diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index a9f63cf701a..e27b1966f18 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -18134,6 +18134,43 @@ Elasticsearch module +*`shard.primary`*:: ++ +-- +type: alias + +alias to: elasticsearch.shard.primary + +-- + +*`shard.state`*:: ++ +-- +type: alias + +alias to: elasticsearch.shard.state + +-- + +*`shard.index`*:: ++ +-- +type: alias + +alias to: elasticsearch.index.name + +-- + +*`shard.node`*:: ++ +-- +type: alias + +alias to: elasticsearch.node.name + +-- + + *`cluster_state.nodes_hash`*:: + -- @@ -22123,6 +22160,21 @@ type: keyword -- + +*`elasticsearch.shard.source_node.name`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.shard.source_node.uuid`*:: ++ +-- +type: keyword + +-- + [[exported-fields-envoyproxy]] == Envoyproxy fields diff --git a/metricbeat/module/elasticsearch/_meta/fields.yml b/metricbeat/module/elasticsearch/_meta/fields.yml index e10da6dc465..d1d69673b66 100644 --- a/metricbeat/module/elasticsearch/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/_meta/fields.yml @@ -6,6 +6,21 @@ settings: ["ssl", "http"] short_config: false fields: + - name: shard + type: group + fields: + - name: primary + type: alias + path: elasticsearch.shard.primary + - name: state + type: alias + path: elasticsearch.shard.state + - name: index + type: alias + path: elasticsearch.index.name + - name: node + type: alias + path: elasticsearch.node.name - name: cluster_state type: group fields: diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index d547f743e59..1c7b8807576 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -162,11 +162,6 @@ func TestXPackEnabled(t *testing.T) { require.NoError(t, err) require.Len(t, events, numShards) - for _, event := range events { - require.Equal(t, "shards", event.RootFields["type"]) - require.Regexp(t, `^.monitoring-es-\d-mb`, event.Index) - } - return } diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index f4ece53c8a7..8595710d912 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded gzipped contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzsXUuP3DiSvtevIHzqAWwBc/VhZ4F57HqB7mlse/ayWGiYUmQmXZIok1S6qn/9QtQjKYlPicpKe7IODXdV5hdfBF/BYDD4AT3D60cEBeaCZBwwy85PCAkiCviI3v1V/f27J4Ry4BkjtSC0+oj+7QkhhCafQSXNmwKeEGJQAObwEZ3wE0IchCDViX9E//uO8+Lde/TuLET97v/av50pE2lGqyM5fURHXPD2+0cCRc4/ShEfUIVL+IiyouECWMoFFiD/gpB4rVshjDZ1/xv1m+q3K5oDT8+Yn8c/DV/HBcFc+W2NxXlmlaQXnrTCufwvJBrIQdoFGCe0iiZqjjfIKbE0Scskmiwd5iBPfiJtGpJHE6eBVKU1PKqkHm6QIEgJXOCyfjIJ6MDf/fv4yXfaTqnwN2HoCfZfG1WmDctANb5/F1/dLK20RGP/9r/rAcdvD3Bf6CHt2qP9V6C9yqL91txcpMrhpQMNNpf8rkMR+ZnEww4DaM1IiRmBZa9VKeloTXqBoAwSTn6HlFTp4VVM8MxtYVdj5Ja48AceOc14ktGmElHFG2BH7eFUQiX2kGyBHqQzODLg50RQgYu0HfatjUpSFCRuI3gLGid7YCfg/ef36xyeYibDiFQnLYd5p0eGjr8ATKX4xWdsuvnqNxBObIJmZGy9YB9Wdonj+nVmVIgCbsrQJXQkN7Ns+Dz4tQH2mmY4O0NSQknZa/R+L0kmQYIGdpJ5jgXel1uAmOsc9rUBLm5huUBRN5nLOmaB89jO8/5grcA5fx9PoCPj6wV8JzN8p9Q9ze56Rvcys8/Y+Qi8emh9qGDJY01/6GbfffpD/yubiBmNfe095eNj6s5j/U6M3bvX/TpgmLKUXg6s5Knr09Fo+Ykb93KUVOKG7DzljaEHze4oIhs9vLIzTC+4aOCG9gmQeY363bR/+YmbrO152g3Y25EME3t1dl8gTw9EpBzE7ciGiVWnlfQCmaDsxrOLt9RZpDgtcX07piFCp77SN0YEsNsxtUqdhAKzjKW4ETQ90qKg31YGBaumPABL6TE9YlK0w6RDI1VOMk1UzysWnbFEYZZ0yMkUeTbfGvkwKKmAdHIYkvabsKj0rIKcbHmTZcD5sSn2sGCPbjah2inWdQR+xixP155zZCxZAFw3mqphV+MbYAYpBeAcWDoPcofJ6ECSKci4Ikjjb5YxwBik9HqcCnrARZqdIXuWftJWncyAM8klfkk5fE0rulWkBmlhy3h6jnZ1azpKj6DrKNaibfulYSqBfLN1rWiDTNoILnCVk+qUMsD59jlznAsV6KSFNkyOKgO5oO1EQWIbOHRyD83x2E7TNTAsWj9g7uuHsVBBkxHUh4Ep9rRBfguZTPE03byu21bYdEo/7et6wIXkIRMinmgj4kK2hIaIok2Ak/h/N9hkoGMR5QiTe42ZdqOsNJw4DMNbLo/wAtku0hV8HRPF/4k821yRbZPN6C1GlT26huZJbhj+XMre3smugJ5i29lAQITubcEcxMp5ZpuiGohpb+5mt8hduFsj7P029hqldFzbEtX33NjS+66rSp7sFCqaw8qtwnFbsocuFLs+vIsvmBT4UBjPi0zWkjk7XXrOsW2sssTsNRnhktCN/hg3oBOzbtfQHrrWIdpQkXYW0X7Mx26Dtn0cQ5m9TOFa5AyWqFtHnKfxWcqJNTLVbvzG59oN4A1klXiWNh4RMnT3Ohl1HUE5rDbEQzQnj+qcb7eTgZph8K1kJJt0zUFG+DFpKMMJcpjZFgki0fqHRypHgK7zzJLVc7ySPnOnqi4TfFYrO8l4uVN1dVk5qxXe57Q/0nhdnKevn+HsJ+Or+Kya33Y7+jcR8VbNejTsZ+Sgg+NQYlfwtX0ebTjyDGU7wY9BOPCILpSvCh+DbnyGEUj5ZgyEUpO4MQh6Z6WEMuyAY1AMTWwIZTrBj0E4MF8glK8KH4vuXjyjEAxLagilqaCvITteR7qUmzZ/pyziwlwUXffYJ65R5NYNsgnbha9RQXeaNP9xtPeXS5mcsuRqk4QWeXLFt+74kUeIwkDb6ZHG4q91UH3JD8RfabOIO0wx7r1VpQbfdbsuNFjVsoo7FG1COUM78+KXDdNuq24JZdJCJSV+2bQgSD4NhzwWoRYrEqMaWAZr9kVLQnXmuy2aHDtMsox8LujaMem2oGZWN9H6YUFxnuILMHyah0rswDZwVcAf52Nm+HG0HeVJVjdJz++UGHFcozbTsd/gCdQNziy9aIutGo5PkFa4oiuj8a3RJIGkp5lIyMQY3fcZiMvuFkfb7MjTrw0VOC1JxqKonGRHnkjMpFmjMprskbD91CLG8n3Nx4QC1/1kR2i+YRWcGqT9XTLDjrqOXzWQBRjSISpvcmjXaTDDjqqBdD9GaOvwW09e4W4cjC7i07DMPmOSVoLRIrX1bW8D9Fs/H0zfQVmQkgibi7KGoAQ1+ioh9LoJPDK9bgoPpTdGoxjNgN+Pw7HamesVkcMq3I0TZ3k0X1O67cL7oSmeo9niawONzutyWELRJWn5JBJnU9CfwRfIhHbSDiUzQK0+VTmBvqTIG1n4BOJuDNxy2Wzf+W2AN7dwd6/oXmzcsdls5fhnslvNfD34vAs793/dbGh53HVPdu4SoO7FzB2bYCvPkrBWJl6muNi23OqqdZmRTGjInRFmB7UBL8CNCQrIa2Nivnk5pHyaahJZdkWB6Wu7M9XlgeyTbqudh+2QNljknY6CItlwWZXCo509U3h25GjK9PmBxqGueozmiuu8eqteRdMUOpShrHQlDp/h9Rtl6gqhqQA7/Ewrwfa4UkpilKoJaUeQSXKzxK4SaFy5ElMrVVuXNSjBeL48m8g6CLc/v9Ac0Ke/aOXMmj+GpGnLq8K6ArNacQdKC8BVmLhPHIkzSGPLf3T48v//pCdQ0Ox56jtspzCAIqjwoYAc0Wqk9adlf8yWN7/nHcMi88+Mcv5h6PAM6oJkMh8eze9aTCsxDz+2Pme8n46svcJ6J+r61YLOJmXfS8EeEJo7+9dvkUrASdHH7BzIhS2ahzC/q+jUZvFlw1XDIKDFhS/rtw39YnEXa5tlrhfCdlmlNdcFnYrrcHSX1byBprfafnw9lYvoP56y2mGh3j6PNzSWt9i9VFzcZHPsyN3qKWVXomm342CwFtVxWkCHGFIHx1vAbAq8Q1Po12hZyCRaP9CFipHD9URuZ6z9+QWXgOixZ2yQdHVINWVPLLYJYvIzfiFlUyLedpkqg/5IuyU3jtLBV+zZSj/GyNZWIsZCWtugw2X376lJB86ORpVGTDpj79Csv4yt2DacFIa+EXEmXUvauVnLFcRneBXX8YIc/TTsGSD/Q+saU8l6NG2nz5HR0r9fynAQJ1UGae/Lr3B8vTT7TEp4j0iFSv4eSYlT9q14dASRnWGhROxhFUT8P6QMdJWB5G2ldvhPTX//U5U3X1OpHQ/S1yCrsWpOAIq5/k0AiL0ciRVKF4pyJ07YggD99n/7ll99UslOx4SEZlGvxVNLU1jTQjANEc3fNQrDcLWz6/vat47CIbSZgu4wivarG2JyamdpOPrpxACq9+gV2mH8HjHI/6CP1snW9O4bHpFBLjkQGUBNvLrWNX1MN9hssXb301BSvf7fOny1PUxOtGXecM66n6nAhTLRSj7t2jbQNPDRBlFjGkMrYGoNK4d15rg6LR34ByjIiRwK8DaMpt6ClVKYVTToU5uEig+wSAvtMMPsXD3WsP3Uwe0ycOMMm15hj/4h/ZXl4LDtt51sfpPustk666/mb3aVZ3bqtLeYyZ0csQetTtarm964cfUrXbKli/0sgVHD200EZYrIhUsHFSPZ8mHJAF/urxIhgisHL5A1rbec1rQg2TyPe4NXtzwiRN6OkMB8ntdqZ2NjZDkeVWENvdNByR2qGPcT8s7fJogMVxkUsr6aBUd/Iqni1JhBJdJWpeXxVxglzdmXCmDOsrffSVEWSSZs9yQdUVklLttUFalOSYW1e2Er2iQdT5aM1Q4TDQuXf3udaLvcgCFQjPq4UCfRMnLl0Wf71eHpqt2ZiTMW/Qw2pPNTxtEZX2Dk1EdZ5BG7bMKm1ivRR8ujnxVmDWP6vP6tS9GfO2Q11nE9w+r1GY1lXi3tVQmjRPHMtPzayRSjDV6s5gDhq1TGAC9zTzsi+TwWMXzpTPJ8ESk1T5FWh29LhJsIggvyu+2I0xpDairMOTlVkKcrfSwNUh/PXQnUrmUX2AiisFkxGjYMpJkWG43KoKAZFqubt2exQY/NCoS24tiCy2BX9EjVPilnn9opyZJzFj/T+w4yf51n18F5uvYclpD3ToMx4UIyU6VYr05vrsqIorSKp9IOviqiTeUAmDMR1uosnjAl4XwtzrTDPdpgLUyMNnA/GesJZHz9PeT7ORSgv2XkdfCnrYCJovSlGM3lrG0WAOZb1S0AMrQQXyDbgJJ5Aci+FRgDIL1rJgZgBhUyDcANLOIZgBxWlS8AOLRQqgP6uqR7v8gdiAgvAljVgkaD9n7Q3BPvLi4weXbWrQ6miuhf0N0CattLrvf4166Hq05Ecpo1cu1DakaZOf60frFdR6+TFEzTzzk5UlZi0d8l2UGbQZ+Wx5DCJ4m3WkihZg32cXGDHVyvMw67c+sFYXdsvSAcTu1j5/nY9Wzf9cRef78r52PXfVIUV9E9pyPfqXu5GJXKSf3htUvI7i1imdDRvnuvPfYcG8baauN26+JYfPtN9kQ7bAl33BpH3g6hm+y6d9scxt3P/utG3x8bwpWgT1qOCYOMXmBS6PINDsKVNzkXSTGOza3hqC48NaTLXhzsMSlkMaH6Wkc8sJsJbZFsJ3f6PLZVtRo+swYQmeUc6mVzsaxbvUHn/x60lbjJsoeiSX4aO4FIYlYA+Swhu+wQYyt3Ys+U68skbhbcIiOc5ww4Rz9ltClydAD06dfxl5TJD7V8DHcSepJxj5BVktODZH3foA3LIGr7/CYh7e3Ti43bPqrgGO3Tk4zbPipJ80E/F7ReN5/KfKmVU3HUmpe2NcFCBpkWzaRvDgfYrJ+b4frRt5bbkRSWTfkbn+gxaHeRWzHkNL8GZuyLy5RUFMVEIzePis9u10xWKIsA1Fl9DdJIheGKF3Qu5k0H4Yaj8046rVI5Ka3DsHVDhzUvwMix3abFnNbkrefU7yliD+MEQUxd/77C3tt6/vEz0h7nUzHPp9aU33FHOb1VWXM05flAazwru2KDC2Jez/XtbdqtgePbRWSsRXsm+n7++1/+vm0NNlfO3U+k+z0X2z4/jJXelzWqGhZO10q9VazSqkGcVwB9nsOx2AoF9B2lpawNFFHwIFDz+gaK0kxK1ZB4RzjOnm5dxXaQZ7w6GL0v3qRXzCXjy+nN5NpaMZLoR87U7vQePunDJ3Xyf/ikD5/04ZM+fNKHTxpD3sMn/V590jH9sEi+0MPTHDcgVluq7mx4oDbmCe8/KvK1AVQW6As9mM94jeUYVwn9L3roIA3PcdAcupTZaF4/A94UItVkjLhUmdHaUgP90GTPIFJcdNf5aZUeMSkaZqmF7ZVn3OVi2ys7WiEKetp4AEJK4AKX9bYjlEQ24y6Ly8H2ipS3L0YvwGLg1JgJYq2GbwEbN4tY4C5JPd6Jjfz78PSs8/k8R3pg3ZhfGfYG6BhtGR+Ec1Kd0s1AtBEpPaaU5cCSa4/fZKILLkie5liA8bqBz/jFXKSyOxi8C+/JaZexB2Utogw/XmPGV78IgrYnZlyPSvsBkjLIKMvXNN5slfx1gOyqasLFHizo5O7TXJgVBLjFf/Q0coFFDJhuHtg4USbqWNOCrU1W/XxWy5J37cJl5X0ERJyBITzMQbKSVNcGiLa/l/8vy511aW0VFegASHbzXJPut/BFvd5zsygw+364L6qvab3BN5RPtfWoevfwy6X0Xu9cjuj//Iw+VUfqVzzWpbVLcw9CAymtAdDC60vOgOuEVJbn/PeOFv4n4FqWIJsECFsd3LFBVYkSv7ytDiV+Wa9CRau3b4pfaPUhQnMMurxli4yq+LfKbG1OdnhT8TiAoxYbcvniR2c07cy8/UWF38YKzggfaCMQ4Ozc52NXCOvfG90WXAgs1H1X523w0hqrXWiHE637PWu7oxOq7+1sajj2CziG2npkuepIUgr1IDfe/N/Fn3c0mVN15Gy62531bTuPmkHJkFOEzeT2w7LJSdXudRtu0RVuVYPiFrrsf7U3RqdWbglvfgRyz+oN1yv5rsXEQ3mfKg7XWVa5BX9j2eqd8xuLlqUKbiyzKzVxY6GTAgo3lq2WQXgD0beWqdRm2FmyEvRpJclYQbS9yU473AHecHEvxmHWbUZxtjoHYxBSA8vmryTMm7SmtIi336Q7HSDa+gqKY3fvijg/4xefGkI14Oe74fwr4Gdf0uk9GVsSL/0svvbS6C7E/9GFzaw7z1fa7LRdu6cmfIyXx3iJM154wy7kQs3vdz+GzNtxfgyZWxM3DRnVxTtlSUaLotsbxXTzBljbQexbl+PfEAuUS/MPpqO2kxzj9YplHQU3JvLr7n8jBSD+ygWUFjE+Nt9cpCTaEB5bgMHGHPjIhPAFE/kI5ZuzGgN4NJ0f5k5pbBmF7jKHMS4FeN6vQH6Z8tdAP86TfaBluDQC9miAeBNNVje79ISC4jzFl1Pyx3mOk0PHCTkTgwjrRd3gLBNJw/EJkq3VuvVWdDN1sZ3IOPLka0MFTrTZ8Z6M0ewyhBPJRt2HvioQClxzyNMaGKG5exh46oPmCfzXe4Z7iVAkGPuOJ/z0GHD/bkQrwWiRutrVlf83RS1IackXm2I6DO2z1qm7iHb87ix4npGV1U2yjCkbhfjEkMW5XX/SmtJ4FQJ2u+DYPbUcpSbgF8iEY5h6LBSnnRL+705RXd24H1TVfZMV7ktX6SD+mKo+zYH63M6kBvngtnxa/8mlsmVz808d4D/lOodJxRFG/R9Q+wcVSQ31rMkv5cBEd4vKdyVwPrXbQqIl5HUBIpQRoS/ZHS7vVx3cOP5kaVutpA1FhhP0N8oQvOCyLlqFGvGhxHU9z7Wb+FqkSrtu7Fs82H3VhZQynVHCLnqoLFu+pUtKgL7zbOpju1VoF2fCEeEyudOjWnuXYRvL+JN7RpKJvVB8zBvbn2U2KxbgI/v6Xnkqs+CruFXRz33JbZlhK7vMN8wHoZCjI6Nl8vT/AQAA///UhdzI" + return "eJzsfUuP3DiS/92fgvBpBmgLmKsP//kD89j1At3T2PHsZbHQsKTITNqSKJNUdtV8+oWoR1ISnyKVlfZWnbrLpV/8IvgKBoPBD+grvHxEUGEuSMEBs+LyDiFBRAUf0fu/qL9//w6hEnjBSCsIbT6i//cOIYQWf4NqWnYVvEOIQQWYw0d0xu8Q4iAEac78I/rv95xX739C7y9CtO//p/+3C2UiL2hzIueP6IQr3n9/IlCV/KMU8QE1uIaPiF8wK+VvEBIvbQ/OaNeOv1G/UL9qGakxe5l/P32LK4K58tsWi8vKFJmUmK0RZj4CC4jAXX4/oZKmhOddqPLLrEfZgDa03Me0//AGOcEVVccFsFzVwL9FekyeXzC/7KI0Cpfm44MRMw3kJO0KjBPaJBO1xpvk1FiaZLeldbJ0mIvel3cdKZOJ00Cq0jqeVNIIN0kQpAYucN2+MwkYwN////kv32s7pcLfhKEnOH42q0w7VoBqfP8uvrtZ5HjT2H8xquMH8Bf6lA/t0f9XoL3qqv9qbS45+wygweZaz3mes5veDsuJn8C216qUdLQWvUBQBhkn/4KcNPnTi1jgmdvCrsbMLXPhTzxKWvCsoF0jkoo3wM7aw7mGRhwh2QI9SWdwYsAvmaACV3k/7Hsb1aSqSNpG8BY0T/bAzsDHvz+uc3iKWQwj0py1HNadHhk6/gYwl+I3f2PTzVe/iXBmE7QiY+sFx7CyS5zXrwujQlRwV4YuoTO5lWXD58FvHbCXvMDFBbIaaspekvd7STILEjSxk8xLLPCx3ALE3Oawbx1wcQ/LBYq6y1w2MAucxw6e9ydrBc75x3gCAxlfL+A7meEHpR5pdtczepSZfcXOR+DNQxuDNlsee/rDMPse0x/GX9lErGgca+8lHx9TDx7rd2Ls0b0e1wHDlKX0cmA1z11/nYyWn7h5L0dJI+7IzlPeHHrQ7I4SstHDKzvD/IqrDu5onwCZt6jfXfuXn7jF2l7mw4C9H8kwsTdn9xnK/ImInIO4H9kwseq0kl+hEJTdeXbxlrqKFOc1bu/HNETo0lf6jREB7H5MrVIXocCiYDnuBM1PtKrobzuDgk1XPwHL6Sk/YVL1w2RAI01JCk1UzysWXbBMYZYNyNkSeTXfGvkwqKmAfHEYko+bsKT0rIKcbHlXFMD5qauOsOCIbjah2in2dQR5WpbvPecoWLYBuG00VcPuxjfATFIqwCWwfP/BXi9jAMmWIPOKII0fLWOCMUgZ9ThX9AlXeXGB4qv0k2J1MgOuJNf4OefwLW9orEgN0saW6fSc7erWdJaeQNdZrEXb/qNpKoEy2rpWtEkm7QQXuClJc84Z4DJ+zpznQgU666ENk6PKQC5oB1GQ2AYOg9yn7nTqp+kWGBa9H7D29cNYqKDZDOrDwBR7ipDfQ2ZLPE03b9u+FaJO6Zd9XQ+4kTzlpKQTbUTcyJbQkFC0CXAR/x8Gmwx0bKIcYXJvMdNhlNWGE4dpeMvlEZ6hOES6gq9jovg/iWebG7Jtspm9xaSyZ9fQPMlNw59L2fGd7AboKbafDQQk6N4WzEmsnGfiFNVALHvzMLsl7sLDGmHvt6nXKKXj2paoseemlj52XVXyYqfQ0BJ2bhVOcckeulDs/vAuvmJS4afKeF5kspbM2RnSc059Y9U1Zi/ZDJeFbvTnuAFdmDVeQ3voWodoQ0XaWUT7Zz52m7Qd4xjK7GUK1yJnsETdOuIyT89STqyJqQ7jNz3XYQBHkFXiWdp4RMjQPepk1HUE5bDaFA/RnDyqc77dTgZqhsG3k5Fs0j0HGeHHpKEMF8hhZtskiCTrHx6pHAG6rjNLds/xSvrMg6q6TfDZrewi4+VB1dVl5exW+JjT/kTjdXOevn+Gs5+M7+Kza3477OjfRMRbNevRsJ+Rgw6OQ4ndwPf2eRRx5BnKdoGfgnDgEV0oXxU+Bd30DBOQ8s0YCKUmcVMQ9M5KCWU4AKegGJrYEMp0gZ+CcGC+QChfFT4V3aN4JiEYltQQSlNB30N2vo50raM2f+ci4cJcVUP3OCauUZXWDbIJ24WvUUF3mrT+cbT3l2udnYvsZpOMVmV2w7fu+JFHiMJA2+mRpuKvdVB9yU/EX2i3iTssMR69VaUG33W7bjTY1bKKO5RsQrlAP/Pi54hpt1e3hjrrobIaP0ctCJJPx6FMRajHSsSoBVbAnn3RllBb+G6LFscOiywjnwu6dkwaF9Qs2i5ZP6woLnN8BYbP61CJHdgGrgr4w3rMTD+OtqM8K9ouG/mdMyOOa9QWOvYRnkDb4cLSi2Js1XF8hrzBDd0Zje+NJglkI81MQmbG6L7PQNx2tzTaFieef+uowHlNCpZE5aw48UxiZt0eldFij4TtpxYplu9bPiZUuB0nO0LLiFVwaZD+d9kKO+k6ftNAFmDIp6i8yaHdp8EKO6kG0v2Yoa3Dbz95hbtxMLqIL8Myx4xJ2ghGq9zWt70NMG79fDB9B2VFaiJsLsoeghLU6KuE0Bsm8MT0hik8lN4cjWK0AP44DsduZ25URA6rcDdOXOTRfEtp3IX3p676mswW3zrodF6XwxKKLlnPJ5M4UUF/Bl+gENpJO5TMBLX7VOUM+pIir2ThM4iHMXDPJdq+69sAr27h4V7Ro9h4YBNt5fRnsrFmvh18PoSdx3+NNrQ87nokOw8JUI9i5oFNsJVXSVg7Ey9zXMUtt7pqXWYkExpyZ4TZQW3AG3BjggLy2piYb15OKZ+mmkSWXVFg+trhTHV5IMek22rnYTukDRZ5p6OgRDbcVqXwaGfPFJ4DOZoyfX6gcairHqO54rqACJ5CpzKUja7E4Vd4+Y0ydYXQ1OKdfpY1eUdcKSUzStWEtBPIJKVZ4lAJNK1ciamVqq3LGpRgvF6eTWQdhPufX2gJ6NOftXJWzZ9C0rLlVWFDgVmtuCdKK8BNmLhPHIkLSGPL/xjw5f//UU+gosXXpe8QT2ECRdDgpwpKRJuZ1h+3/bHY3vxedwyLzD8xyvmHqcMzaCtSyHx4tL5rsayJPf3Y+pzxfjqy9grrnajbpxVdTcq+l4I9IDR39m9fkUbAWdHH7BzIhS2Zh7C+q+jUZvOx4aphENDmwpf1a0O/2NzFirPM7ULYIau05rqgU3Edju6ymjfQ8lbbj6+nchH9x1NWOyzU2+fphsb2FruXipubbI4duVs9pexKMu0OHAzWojpOC+gQQ+rgeAtYTYEPaAr9Gi0LmSTrB7pQMXK4nsjtjPU/v+AaED2NjA2Sbg6ppuyJxTZBTH7Gz6TuasT7LtMUMB5p9+TmUTr5iiNb9c2RLVtbiRgLaW2DTpfdv6cmnTg7GnV47mQw9gHN+svcin3DSWHoNyIuZGhJOzdruYL0DG/iBl5Qot9NewYof9+7xlSynk076HNitPbvlzIcxElTQD768jscXy/NPpMafkKkQTX/CUmJS/a9eHQCUVxgo0TqYRVE/N+kDHSTgeRtpX74L03/+FOVN19TqR0P0rcgq7FqTgCKuf5NAIi9HIkVSheKcidO2IIA4/Y/fsu/flTKTMeEhFZRr81TS0tY00KwDBGt3zUKw3C1s+t77VtH4RDaTEF3GEX7aURMTu0sHUe/OzOA5if0Av0w/gkxKH+vj9bJ1vTuGx6RQS45EBlAzby61i19TDfYbLF299NQUr3xv3X4anuYnGjLvOGcdT9TgStlopV8+rVtomngow2ipjSGVsDSGlYO+8xxc1oG8A9QkTN5qsDbMJp6C1ZKYVbRoC9tEio+wCI9tMMMq3P1VMP20wB3yMBNM2xGhT36h/RXtoPDtt92svm7dJfN1tl/NT/aVV7ZadDeYiZ3csQRtMbXLt305o2rX+mSmC72swRGHe83EZQpIjcuHTSMFNuHJQN8ub9IhASuHDxD0fXect7SihTrPO4Ir257RIi8HSGB+Tqv1c7GxshyPKrCGnqng5I7VDHvJ+SdvyiIAjcFVLK+mgVHfyKp4rSYQSPyXqXt8VcYJc3ZlwpgzrK330lRFkkmbPckHVFZJS7bNQ1pzlmDtXthK9oiHU+WjNUOEw0Ll397m2iH3IApUIzGuNAg0TJy5dFn/+n0dNXhzMQFi3EGm9L5KePogq8wcxqjLPKIXTZh1+qVGKPlyc8Ki44xfV5/7FL0pwFZjXXczrBGfWZjmVdLe1XCJFE8My2/djLFaIMXqzVA+CpVMMDb3NOBSLmORUwfXUhZbiKl5inS6vDFRLiJILgi/7IdcVpjSF2DOSfnBsp8p4+lQRrjuTuB+rXsCpEgCpsdoyFiIK20iDQqg4oWWOxu3pFFhB7RCoS24tyC22BX8kjVMSlnn/opyZJzlj7T+wEyf51n18F5uvYclpD3ToMx4UoKU6VYr05vrsqIkrSKp9IOviqiTeUAmAsR1uosnjA14XwvzrLDvbXBXpgUbeB+MtYTyPj6e8j3JVSgv2XkdfCnrYCJkvSlFM3lrG0WAOZb1S0AMrQQXyDbgJJ5Aci+FRgDIL1rJgZgBhUyDcANLOIZgBxWlS8AOLRQqgP6tqR7v8gdiAjPAljTgyaD9n7Q3BPvIS4weXbWWAdTRfQv6G4Bte0l93v8e9fDXSciJS06ufYhNaPMHH/av9juozdICqbp55ycKKuxGO+SHKDNpE/PY0rhk8R7LaRQswbHuLjBDq7XGYfdufWCsDu2XhAOp/Zt5/m264nf9aRef78r5+PQfVISV9E9pyPfqXu7GNXKSf3Ty5CQPVrEMqGjY/deR+w5IsbabuMO6+JcfPtV9kQHbAkP3Bon3g6hu+y6D9scpt3P/t+Nvr9tCHeCvtNyzBgU9AqLQpevcBCuvMm5SYpxbG4NR3XhqSFD9uJkj0UhiwXVlzbhgd1KaI9kO7nT57HtqtXwmXWAyCrnUC+bi23d6gid/3PSVuJm2x6KFvlp7AwiS1kB5LOEHLJDjK08iL1Qri+TGC24R0a4LBlwjn5X0K4q0ROgT7/Ov6RM/lHPx3AnYSSZ9ghZJbk8SNb3DdqxApK2z98lpL19RrFp20cVnKJ9RpJp20claT7o54K2++ZTmS+1cypOWvPStiZYyCDTopmNzeEAW/VzM9w4+vZyO5HKsil/5RM9Bv0uMhZDTvN7YOa+uE1JRUlMNHPzqPjsds1khbIEQIPV9yDNVBhueEXXYl51EEYcnQ/SaZPLSWkfhq0bOqx5BUZO/TYt5bQmbz3nfk8RexgnCGLp+o8V9l7X80+fkfZ2PpXyfGpP+R13lNNblT1HU54PtKazsis2uCHm9Vzf0aaNDRzfLyJjLdqz0Pfz3/78t7g12Fw59ziR7vdcbPv8MFZ6X9aoalg4XSv1XrFKqwZpXgH0eQ7HYisU0HeUlrI2UELBk0DN6xsoSTMpVUPSHeE4e7p1FTtAnvHqYPK+eJdesZaMr+dXk2trxUSi33KmDqf35pO++aRO/m8+6ZtP+uaTvvmkbz5pCnlvPun36pPO6YdV9oU+vVvjBsRqa9WdDQ/Upjzh/UdDvnWA6gp9oU/mM15jOcZdQv+DPg2Qhuc4aAlDymwyr58B7yqRazJGXKqsaMXUQH/qiq8gclwN1/lpk58wqTpmqYXtlWc85GLbKztaISp6jjwAITVwges27gglk814yOLyZHtFytsXo1dgKXBazASxVsO3gM2bRSzwkKSe7sRG/vv09Kzz+TxHemDbmV8Z9gYYGMWMD8I5ac55NBDtRE5POWUlsOzW46NMdMUVKfMSCzBeN/AZv5iLXHYHg3fhPTkdMvagbkWS4cdbzPjuF0FQfGLG7ah0HCA5g4Kyck/jrVbJXyfIoaomXO3BgkHuMc2FWUWAW/xHTyNXWKSAGeaByIkyU8eaFmxvsurni1qWfGgXLivvIyDiAgzhaQ6SlaSGNkC0/738f1nubEhra6hAT4BkNy816X4bX9TrPTeLAqvvw31RfU3rCN9QPtU2ourdwy/X2nu9czmi//Uz+tScqF/xWJfWLs09CE2ktAZAG68vuwBuM9JYnvM/Olr474BbWYJsESDsdXDHBlUlavz8ujrU+Hm/Cg1tXr8pfqHNhwTNMenymi0yq+LfKqu1OTvgTcXTBI56bCjlix+D0bQzc/yLCn+fKzgj/EQ7gQAXlzEfu0FY/95oXHAhsFD3Q523wXNvrH6hnU60Hves7YFOqL63s6np2C/gGCr2yHLXkaQU6kFuvvl/iD/vaDKn6sjZdPc764s7j1pByZBTgs1k/GHZ4qTq8LoN9+gK96pBcQ9djr/am6JTK7eEox+BPLJ6w+1Kvmsx8VDep4rDbZZVbsHfWbZ65/zOomWpgjvLHEpN3FnoooDCnWWrZRBeQfS9ZSq1GQ6WrAR9ekkyVpBsb3LQDneCN1zcS3GYdZ9RXOzOwZiEtMCK9SsJ6yZtKa3S7TfpQQeItr6C0tjduyLOz/jZp4ZQC/jrw3D+FfBXX9L5IxlbEq/9LL730ughxP8xhM2sO88X2h20XXukJnwbL2/jJc144R27kis1v9/9NmRej/PbkLk3cdOQUV28c5EVtKqGvVFKN2+CtR3EvnY5/ohYoFyafzAdtZ3klK5XbOsouDGRX3f/K6kA8RcuoLaI8bF5dJGSZEN4bgEGkTnwiQnhKybyEcpXZzUH8Gi+Psxd0ogZhe4yhykuBXjer0B+mfK3QD8us2OgZbg0AfZsgHQTTdF2h/SEiuIyx9dz9od1jpNDxwU5E4ME60Xb4aIQWcfxGbLYat16K7qZutguZJx49q2jAmfa7HhPxmh1GcKJZKPuQ18VCBVuOZR5C4zQ0j0MPPVB6wT+2z3Do0QoEox9xxN+eQx4fDeijWC0yl3t6sr/W6JWpLbkiy0xHYb2WevUXUQ/fg8WvM7IKtou28aUjUJ8Ysji0q8/eUtpugoBh11wHJ5aTlIT8AsUwjFMPRaK80EJ/w+nqK5u3A+q6rHJCo+lq3QQf0xV362BxtzOrAX54LZ8Wv+dS2XL5uafOsB/ynUOk4YjjMZ/QP0/qEhqqGdPfikHJoZbVL4rgfOp3R4SbSFvCxChjAh9ye5web/q4ObxJ0vbaiVFFBnO0F8pQ/CM67bqFerEhxq37TrXbuFrkSYfurFv8WD3VRdSy3RGCbvpobJseUyXlABj54nqY4dVaBcXwhHhMrnTo1r7kGGbyviLe0aSib1QfMob259lNisW4CP79l55LrPgm7RV0S9jyW2ZYSu7zG+YT0KhRCdGa1sV8Hx16QlFeYsa5VwKKm749nnz7cf/GwAA//8DfHhl" } diff --git a/metricbeat/module/elasticsearch/shard/_meta/data-xpack.json b/metricbeat/module/elasticsearch/shard/_meta/data-xpack.json deleted file mode 100644 index e7c4eb03c09..00000000000 --- a/metricbeat/module/elasticsearch/shard/_meta/data-xpack.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "_index": ".monitoring-es-6-2018.04.29", - "_type": "doc", - "_id": "zCHJWMeqT1StL1M28ml_Vw:r4XD9O8eTrCHyN_GJswZ5A:heartbeat-6.0.0-rc1-2018.04.14:0:p", - "_score": 2.0136, - "_source": { - "cluster_uuid": "3zVAmPiRRNK6TYXeqCVbqg", - "timestamp": "2018-04-29T00:00:30.108Z", - "interval_ms": 10000, - "type": "shards", - "source_node": { - "uuid": "r4XD9O8eTrCHyN_GJswZ5A", - "host": "172.25.133.112", - "transport_address": "172.25.133.112:19608", - "ip": "172.25.133.112", - "name": "instance-0000000016", - "timestamp": "2018-04-29T00:00:30.073Z" - }, - "state_uuid": "zCHJWMeqT1StL1M28ml_Vw", - "shard": { - "state": "STARTED", - "primary": true, - "node": "r4XD9O8eTrCHyN_GJswZ5A", - "relocating_node": null, - "shard": 0, - "index": "heartbeat-6.0.0-rc1-2018.04.14" - } - } -} diff --git a/metricbeat/module/elasticsearch/shard/_meta/data.json b/metricbeat/module/elasticsearch/shard/_meta/data.json index e76902ab5ca..64a0406003c 100644 --- a/metricbeat/module/elasticsearch/shard/_meta/data.json +++ b/metricbeat/module/elasticsearch/shard/_meta/data.json @@ -1,22 +1,18 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "elasticsearch": { "cluster": { - "id": "91RpCx2xSQ21pVPTZfDK0Q", - "name": "elasticsearch", + "id": "tMjf3CQ_TyCXNfcoR9eTWw", + "name": "docker-cluster", "state": { - "id": "MBE4XrQOSf6ScXRTuCO1Pw" + "id": "n-UoXaqYRoOe9qAC76IG6A" } }, "index": { - "name": "heartbeat-7.0.0-alpha1-2018.08.27" + "name": ".apm-agent-configuration" }, "node": { - "name": "Z4hBonPxQVW9qPKEHpwWCg" + "name": "hx-oJ1-aT_-5pRG22JMI1Q" }, "shard": { "number": 0, @@ -24,17 +20,24 @@ "relocating_node": { "name": null }, + "source_node": { + "name": "1fb2aa83efac", + "uuid": "hx-oJ1-aT_-5pRG22JMI1Q" + }, "state": "STARTED" } }, + "event": { + "dataset": "elasticsearch.shard", + "duration": 115000, + "module": "elasticsearch" + }, "metricset": { - "host": "127.0.0.1:9200", - "module": "elasticsearch", "name": "shard", - "namespace": "elasticsearch.shard", - "rtt": 115 + "period": 10000 }, "service": { - "name": "elasticsearch" + "address": "127.0.0.1:41773", + "type": "elasticsearch" } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/shard/_meta/fields.yml b/metricbeat/module/elasticsearch/shard/_meta/fields.yml index 97902d0d619..8d175a071ca 100644 --- a/metricbeat/module/elasticsearch/shard/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/shard/_meta/fields.yml @@ -20,3 +20,10 @@ type: keyword description: > The node the shard was relocated from. + - name: source_node + type: group + fields: + - name: name + type: keyword + - name: uuid + type: keyword diff --git a/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.710.json b/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.710.json new file mode 100644 index 00000000000..c67c55701ec --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.710.json @@ -0,0 +1,160 @@ +{ + "cluster_name": "docker-cluster", + "cluster_uuid": "tMjf3CQ_TyCXNfcoR9eTWw", + "version": 137, + "state_uuid": "n-UoXaqYRoOe9qAC76IG6A", + "master_node": "hx-oJ1-aT_-5pRG22JMI1Q", + "nodes": { + "hx-oJ1-aT_-5pRG22JMI1Q": { + "name": "1fb2aa83efac", + "ephemeral_id": "7oBOrfgNSQqlcnWZHb3kZw", + "transport_address": "127.0.0.1:9300", + "attributes": { + "ml.machine_memory": "33300463616", + "xpack.installed": "true", + "transform.node": "true", + "ml.max_open_jobs": "20" + } + } + }, + "routing_table": { + "indices": { + ".apm-custom-link": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": ".apm-custom-link", + "allocation_id": { + "id": "UBnwq-49RVie17H7rNxZ5g" + } + } + ] + } + }, + ".kibana-event-log-8.0.0-000001": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": ".kibana-event-log-8.0.0-000001", + "allocation_id": { + "id": "yMoiDQJFTBSsUs8jXU365g" + } + } + ] + } + }, + ".kibana_1": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": ".kibana_1", + "allocation_id": { + "id": "Hjpxq1XrRb-oHdC98LjVKA" + } + } + ] + } + }, + ".kibana_task_manager_1": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": ".kibana_task_manager_1", + "allocation_id": { + "id": "t4fldJzLS_mnOUAyEL-NmQ" + } + } + ] + } + }, + "metricbeat-8.0.0-2020.11.18-000001": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": "metricbeat-8.0.0-2020.11.18-000001", + "allocation_id": { + "id": "n7p13nerRFutSjZvfxd5pQ" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": "metricbeat-8.0.0-2020.11.18-000001", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "CLUSTER_RECOVERED", + "at": "2020-12-09T19:01:54.545Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + "ilm-history-3-000001": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": "ilm-history-3-000001", + "allocation_id": { + "id": "DqiX-m2JS2Kobo6_49MdBA" + } + } + ] + } + }, + ".apm-agent-configuration": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "hx-oJ1-aT_-5pRG22JMI1Q", + "relocating_node": null, + "shard": 0, + "index": ".apm-agent-configuration", + "allocation_id": { + "id": "Ba_BCJuiR2qA5Tdcu9IzrA" + } + } + ] + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/shard/data.go b/metricbeat/module/elasticsearch/shard/data.go index 73486638f1b..6a01b8f2611 100644 --- a/metricbeat/module/elasticsearch/shard/data.go +++ b/metricbeat/module/elasticsearch/shard/data.go @@ -19,6 +19,9 @@ package shard import ( "encoding/json" + "strconv" + + "github.com/elastic/beats/v7/metricbeat/helper/elastic" "github.com/joeshaw/multierror" "github.com/pkg/errors" @@ -67,9 +70,6 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { for _, shard := range shards { event := mb.Event{} - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", elasticsearch.ModuleName) - event.ModuleFields = common.MapStr{} event.ModuleFields.Put("cluster.state.id", stateData.StateID) event.ModuleFields.Put("cluster.id", stateData.ClusterID) @@ -95,13 +95,33 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { continue } - event.ModuleFields.Put("node.name", fields["node"]) - delete(fields, "node") + event.ID, err = generateHashForEvent(stateData.StateID, fields) + if err != nil { + errs = append(errs, errors.Wrap(err, "failure getting event ID")) + continue + } + + event.MetricSetFields = fields + + nodeID, ok := shard["node"] + if !ok { + continue + } + if nodeID != nil { // shard has not been allocated yet + event.ModuleFields.Put("node.name", nodeID) + delete(fields, "node") + + sourceNode, err := getSourceNode(nodeID.(string), stateData) + if err != nil { + errs = append(errs, errors.Wrap(err, "failure getting source node information")) + continue + } + event.MetricSetFields.Put("source_node", sourceNode) + } event.ModuleFields.Put("index.name", fields["index"]) delete(fields, "index") - event.MetricSetFields = fields event.MetricSetFields.Put("number", fields["shard"]) delete(event.MetricSetFields, "shard") @@ -112,5 +132,55 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { } } } + return errs.Err() } + +func getSourceNode(nodeID string, stateData *stateStruct) (common.MapStr, error) { + nodeInfo, ok := stateData.Nodes[nodeID] + if !ok { + return nil, elastic.MakeErrorForMissingField("nodes."+nodeID, elastic.Elasticsearch) + } + + return common.MapStr{ + "uuid": nodeID, + "name": nodeInfo.Name, + }, nil +} + +func generateHashForEvent(stateID string, shard common.MapStr) (string, error) { + var nodeID string + if shard["node"] == nil { + nodeID = "_na" + } else { + var ok bool + nodeID, ok = shard["node"].(string) + if !ok { + return "", elastic.MakeErrorForMissingField("node", elastic.Elasticsearch) + } + } + + indexName, ok := shard["index"].(string) + if !ok { + return "", elastic.MakeErrorForMissingField("index", elastic.Elasticsearch) + } + + shardNumberInt, ok := shard["shard"].(int64) + if !ok { + return "", elastic.MakeErrorForMissingField("shard", elastic.Elasticsearch) + } + shardNumberStr := strconv.FormatInt(shardNumberInt, 10) + + isPrimary, ok := shard["primary"].(bool) + if !ok { + return "", elastic.MakeErrorForMissingField("primary", elastic.Elasticsearch) + } + var shardType string + if isPrimary { + shardType = "p" + } else { + shardType = "r" + } + + return stateID + ":" + nodeID + ":" + indexName + ":" + shardNumberStr + ":" + shardType, nil +} diff --git a/metricbeat/module/elasticsearch/shard/data_test.go b/metricbeat/module/elasticsearch/shard/data_test.go index 377b73617f3..5ba34901a52 100644 --- a/metricbeat/module/elasticsearch/shard/data_test.go +++ b/metricbeat/module/elasticsearch/shard/data_test.go @@ -19,9 +19,13 @@ package shard import ( "io/ioutil" + "net/http" + "net/http/httptest" "path/filepath" "testing" + "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" + "github.com/stretchr/testify/require" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" @@ -42,3 +46,34 @@ func TestStats(t *testing.T) { require.Equal(t, 0, len(reporter.GetErrors())) } } + +func TestData(t *testing.T) { + mux := http.NewServeMux() + + mux.Handle("/_nodes/_local/nodes", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"nodes": { "foobar": {}}}`)) + })) + mux.Handle("/_cluster/state/master_node", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"master_node": "foobar"}`)) + })) + mux.Handle("/_cluster/state/version,nodes,master_node,routing_table", http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + input, _ := ioutil.ReadFile("./_meta/test/routing_table.710.json") + w.Write(input) + })) + + server := httptest.NewServer(mux) + defer server.Close() + + ms := mbtest.NewReportingMetricSetV2Error(t, getConfig(server.URL)) + if err := mbtest.WriteEventsReporterV2Error(ms, t, ""); err != nil { + t.Fatal("write", err) + } +} +func getConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": elasticsearch.ModuleName, + "metricsets": []string{"shard"}, + "hosts": []string{host}, + } +} diff --git a/metricbeat/module/elasticsearch/shard/data_xpack.go b/metricbeat/module/elasticsearch/shard/data_xpack.go deleted file mode 100644 index 30e0e92b4dc..00000000000 --- a/metricbeat/module/elasticsearch/shard/data_xpack.go +++ /dev/null @@ -1,157 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package shard - -import ( - "encoding/json" - "strconv" - "time" - - "github.com/joeshaw/multierror" - "github.com/pkg/errors" - - "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/metricbeat/helper/elastic" - "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" -) - -func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { - stateData := &stateStruct{} - err := json.Unmarshal(content, stateData) - if err != nil { - return errors.Wrap(err, "failure parsing Elasticsearch Cluster State API response") - } - - // TODO: This is currently needed because the cluser_uuid is `na` in stateData in case not the full state is requested. - // Will be fixed in: https://github.com/elastic/elasticsearch/pull/30656 - clusterID, err := elasticsearch.GetClusterID(m.HTTP, m.HostData().SanitizedURI+statePath, stateData.MasterNode) - if err != nil { - return errors.Wrap(err, "failed to get cluster ID from Elasticsearch") - } - - var errs multierror.Errors - for _, index := range stateData.RoutingTable.Indices { - for _, shards := range index.Shards { - for _, shard := range shards { - event := mb.Event{} - fields, err := schema.Apply(shard) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure to apply shard schema")) - continue - } - - // Handle node field: could be string or null - err = elasticsearch.PassThruField("node", shard, fields) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure passing through node field")) - continue - } - - // Handle relocating_node field: could be string or null - err = elasticsearch.PassThruField("relocating_node", shard, fields) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure passing through relocating_node field")) - continue - } - - event.RootFields = common.MapStr{ - "timestamp": time.Now(), - "cluster_uuid": clusterID, - "interval_ms": m.Module().Config().Period.Nanoseconds() / 1000 / 1000, - "type": "shards", - "shard": fields, - "state_uuid": stateData.StateID, - } - - // Build source_node object - nodeID, ok := shard["node"] - if !ok { - continue - } - if nodeID != nil { // shard has not been allocated yet - sourceNode, err := getSourceNode(nodeID.(string), stateData) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure getting source node information")) - continue - } - event.RootFields.Put("source_node", sourceNode) - } - - event.ID, err = getEventID(stateData.StateID, fields) - if err != nil { - errs = append(errs, errors.Wrap(err, "failure getting event ID")) - continue - } - - event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) - r.Event(event) - } - } - } - return errs.Err() -} - -func getSourceNode(nodeID string, stateData *stateStruct) (common.MapStr, error) { - nodeInfo, ok := stateData.Nodes[nodeID] - if !ok { - return nil, elastic.MakeErrorForMissingField("nodes."+nodeID, elastic.Elasticsearch) - } - - return common.MapStr{ - "uuid": nodeID, - "name": nodeInfo.Name, - }, nil -} - -func getEventID(stateID string, shard common.MapStr) (string, error) { - var nodeID string - if shard["node"] == nil { - nodeID = "_na" - } else { - var ok bool - nodeID, ok = shard["node"].(string) - if !ok { - return "", elastic.MakeErrorForMissingField("node", elastic.Elasticsearch) - } - } - - indexName, ok := shard["index"].(string) - if !ok { - return "", elastic.MakeErrorForMissingField("index", elastic.Elasticsearch) - } - - shardNumberInt, ok := shard["shard"].(int64) - if !ok { - return "", elastic.MakeErrorForMissingField("shard", elastic.Elasticsearch) - } - shardNumberStr := strconv.FormatInt(shardNumberInt, 10) - - isPrimary, ok := shard["primary"].(bool) - if !ok { - return "", elastic.MakeErrorForMissingField("primary", elastic.Elasticsearch) - } - var shardType string - if isPrimary { - shardType = "p" - } else { - shardType = "r" - } - - return stateID + ":" + nodeID + ":" + indexName + ":" + shardNumberStr + ":" + shardType, nil -} diff --git a/metricbeat/module/elasticsearch/shard/shard.go b/metricbeat/module/elasticsearch/shard/shard.go index 4367810a8ca..1483b053451 100644 --- a/metricbeat/module/elasticsearch/shard/shard.go +++ b/metricbeat/module/elasticsearch/shard/shard.go @@ -64,18 +64,5 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return err } - if m.XPack { - err = eventsMappingXPack(r, m, content) - if err != nil { - // Since this is an x-pack code path, we log the error but don't - // return it. Otherwise it would get reported into `metricbeat-*` - // indices. - m.Logger().Error(err) - return nil - } - } else { - return eventsMapping(r, content) - } - - return nil + return eventsMapping(r, content) }