UI: Showing Block Size Stats (thanos-io#7233)

* feat(ui): added BlockSizeStats calculation to blocks page A block can have a list of contained files set in `.thanos.files`. If the `files` array is set, all referenced files with `size_bytes` set are counted: - sum of all `chunk/*` file sizes - size of index file - total size (sum of both) Shows statistics about the selected block in the block details view: - Total size of block - Size of index (and percentage of total) - Size of all chunks (and percentage of total) - Daily growth, based on total size and block duration Output is humanized up to Pebibytes and fixed to two decimal places; raw bytes are accessible through mouse over / title text. Signed-off-by: Markus Möslinger <markus.moeslinger@socra.dev> * feat(ui): added aggregated BlockSizeStats to blocks row title Added total size of all blocks from a source to the row title, beneath the source name. The shown total size is humanized up to pebibytes and fixed to two decimal places; raw bytes value is accessible through mouse over / title text. The shown value will refresh with selected compaction levels, but doesn't take block filter into account. I thought about showing daily growth as well, but just summing all milliseconds of all blocks doesn't work with overlapping blocks / multiple resolutions. Signed-off-by: Markus Möslinger <markus.moeslinger@socra.dev> * chore(docs): added UI block size PR to CHANGELOG.md Signed-off-by: Markus Möslinger <markus.moeslinger@socra.dev> * chore(ui): removed comments Automatic code formatting duplicated some comments near import statements. Signed-off-by: Markus Möslinger <markus.moeslinger@socra.dev> --------- Signed-off-by: Markus Möslinger <markus.moeslinger@socra.dev>
jnyi · Apr 4, 2024 · fdcaf92 · fdcaf92
1 parent 62df22e
commit fdcaf92
Show file tree

Hide file tree

Showing 14 changed files with 383 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -26,6 +26,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#7175](https://github.com/thanos-io/thanos/pull/7175): Query: Add `--query.mode=distributed` which enables the new distributed mode of the Thanos query engine.
 - [#7199](https://github.com/thanos-io/thanos/pull/7199): Reloader: Add support for watching and decompressing Prometheus configuration directories
 - [#7200](https://github.com/thanos-io/thanos/pull/7175): Query: Add `--selector.relabel-config` and `--selector.relabel-config-file` flags which allows scoping the Querier to a subset of matched TSDBs.
+- [#7233](https://github.com/thanos-io/thanos/pull/7233): UI: Showing Block Size Stats
 
 ### Changed
 

diff --git a/pkg/ui/react-app/src/thanos/pages/blocks/BlockDetails.test.tsx b/pkg/ui/react-app/src/thanos/pages/blocks/BlockDetails.test.tsx
@@ -2,7 +2,7 @@ import React from 'react';
 import { mount } from 'enzyme';
 import moment from 'moment';
 import { BlockDetails, BlockDetailsProps } from './BlockDetails';
-import { sampleAPIResponse } from './__testdata__/testdata';
+import { sampleAPIResponse, sizeBlock } from './__testdata__/testdata';
 
 const sampleBlock = sampleAPIResponse.data.blocks[0];
 const formatTime = (time: number): string => {
@@ -96,4 +96,60 @@ describe('BlockDetails', () => {
     const labels = list.find('li');
     expect(labels).toHaveLength(Object.keys(sampleBlock.thanos.labels).length);
   });
+
+  it("shouldn't render total size when block doesn't have any", () => {
+    const div = blockDetails.find({ 'data-testid': 'total-size' });
+    expect(div).toHaveLength(0);
+  });
+
+  it("shouldn't render chunk size when block doesn't have any", () => {
+    const div = blockDetails.find({ 'data-testid': 'chunk-size' });
+    expect(div).toHaveLength(0);
+  });
+
+  it("shouldn't render index size when block doesn't have any", () => {
+    const div = blockDetails.find({ 'data-testid': 'index-size' });
+    expect(div).toHaveLength(0);
+  });
+
+  it("shouldn't render daily size when block doesn't have any", () => {
+    const div = blockDetails.find({ 'data-testid': 'daily-bytes' });
+    expect(div).toHaveLength(0);
+  });
+});
+
+describe('BlockDetailsWithSize', () => {
+  const defaultProps: BlockDetailsProps = {
+    block: sizeBlock,
+    selectBlock: (): void => {
+      // do nothing
+    },
+    disableAdminOperations: false,
+  };
+  window.URL.createObjectURL = jest.fn();
+  const blockDetails = mount(<BlockDetails {...defaultProps} />);
+
+  it('renders total size', () => {
+    const div = blockDetails.find({ 'data-testid': 'total-size' });
+    expect(div).toHaveLength(1);
+    expect(div.find('span').text()).toBe('512.38 MiB');
+  });
+
+  it('renders chunk size', () => {
+    const div = blockDetails.find({ 'data-testid': 'chunk-size' });
+    expect(div).toHaveLength(1);
+    expect(div.find('span').text()).toBe('512.14 MiB (99.95%)');
+  });
+
+  it('renders index size', () => {
+    const div = blockDetails.find({ 'data-testid': 'index-size' });
+    expect(div).toHaveLength(1);
+    expect(div.find('span').text()).toBe('251.54 KiB (0.05%)');
+  });
+
+  it('renders daily size', () => {
+    const div = blockDetails.find({ 'data-testid': 'daily-bytes' });
+    expect(div).toHaveLength(1);
+    expect(div.find('span').text()).toBe('144.11 GiB / day');
+  });
 });
diff --git a/pkg/ui/react-app/src/thanos/pages/blocks/BlockDetails.tsx b/pkg/ui/react-app/src/thanos/pages/blocks/BlockDetails.tsx
@@ -3,8 +3,8 @@ import { Block } from './block';
 import styles from './blocks.module.css';
 import moment from 'moment';
 import PathPrefixProps from '../../../types/PathPrefixProps';
-import { Button, Modal, ModalBody, Form, Input, ModalHeader, ModalFooter } from 'reactstrap';
-import { download } from './helpers';
+import { Button, Form, Input, Modal, ModalBody, ModalFooter, ModalHeader } from 'reactstrap';
+import { download, getBlockSizeStats, humanizeBytes } from './helpers';
 
 export interface BlockDetailsProps {
   block: Block | undefined;
@@ -21,6 +21,8 @@ export const BlockDetails: FC<BlockDetailsProps & PathPrefixProps> = ({
   const [modalAction, setModalAction] = useState<string>('');
   const [detailValue, setDetailValue] = useState<string | null>(null);
 
+  const sizeStats = getBlockSizeStats(block);
+
   const submitMarkBlock = async (action: string, ulid: string, detail: string | null) => {
     try {
       const body = detail
@@ -80,6 +82,39 @@ export const BlockDetails: FC<BlockDetailsProps & PathPrefixProps> = ({
             <b>Chunks:</b> <span>{block.stats.numChunks}</span>
           </div>
           <hr />
+          {sizeStats && (
+            <>
+              <div data-testid="total-size">
+                <b>Total size:</b>&nbsp;
+                <span title={sizeStats.totalBytes + ' Bytes'}>{humanizeBytes(sizeStats.totalBytes)}</span>
+              </div>
+              <div data-testid="chunk-size">
+                <b>Chunks:</b>&nbsp;
+                <span title={sizeStats.chunkBytes + ' Bytes'}>
+                  {humanizeBytes(sizeStats.chunkBytes)} ({((sizeStats.chunkBytes / sizeStats.totalBytes) * 100).toFixed(2)}%)
+                </span>
+              </div>
+              <div data-testid="index-size">
+                <b>Index:</b>&nbsp;
+                <span title={sizeStats.indexBytes + ' Bytes'}>
+                  {humanizeBytes(sizeStats.indexBytes)} ({((sizeStats.indexBytes / sizeStats.totalBytes) * 100).toFixed(2)}%)
+                </span>
+              </div>
+              <div data-testid="daily-bytes">
+                <b>Daily:</b>&nbsp;
+                <span
+                  title={
+                    Math.round(sizeStats.totalBytes / moment.duration(block.maxTime - block.minTime, 'ms').as('day')) +
+                    ' Bytes / day'
+                  }
+                >
+                  {humanizeBytes(sizeStats.totalBytes / moment.duration(block.maxTime - block.minTime, 'ms').as('day'))} /
+                  day
+                </span>
+              </div>
+              <hr />
+            </>
+          )}
           <div data-testid="resolution">
             <b>Resolution:</b> <span>{block.thanos.downsample.resolution}</span>
           </div>

diff --git a/pkg/ui/react-app/src/thanos/pages/blocks/SourceView.test.tsx b/pkg/ui/react-app/src/thanos/pages/blocks/SourceView.test.tsx
@@ -1,6 +1,6 @@
 import React from 'react';
 import { mount } from 'enzyme';
-import { SourceView, SourceViewProps, BlocksRow } from './SourceView';
+import { BlocksRow, SourceView, SourceViewProps } from './SourceView';
 import { sampleAPIResponse } from './__testdata__/testdata';
 import { sortBlocks } from './helpers';
 
@@ -22,10 +22,12 @@ describe('Blocks SourceView', () => {
 
   const sourceView = mount(<SourceView {...defaultProps} />);
 
-  it('renders a paragraph with title', () => {
+  it('renders a paragraph with title and size', () => {
     const title = sourceView.find('div > span');
-    expect(title).toHaveLength(1);
-    expect(title.text()).toEqual(source);
+    expect(title).toHaveLength(2);
+
+    expect(title.find('span').at(0).text()).toEqual(source);
+    expect(title.find('span').at(1).text()).toEqual('3.50 GiB');
   });
 
   it('renders a row for each unique resolution and compaction level pair', () => {

diff --git a/pkg/ui/react-app/src/thanos/pages/blocks/SourceView.tsx b/pkg/ui/react-app/src/thanos/pages/blocks/SourceView.tsx
@@ -2,7 +2,7 @@ import React, { FC } from 'react';
 import { Block, BlocksPool } from './block';
 import { BlockSpan } from './BlockSpan';
 import styles from './blocks.module.css';
-import { getBlockByUlid, getBlocksByCompactionLevel } from './helpers';
+import { getBlockByUlid, getBlocksByCompactionLevel, humanizeBytes, sumBlockSizeStats } from './helpers';
 
 export const BlocksRow: FC<{
   blocks: Block[];
@@ -43,11 +43,13 @@ export const SourceView: FC<SourceViewProps> = ({
   blockSearch,
   compactionLevel,
 }) => {
+  const blockSizeStats = sumBlockSizeStats(data, compactionLevel);
   return (
     <>
       <div className={styles.source}>
         <div className={styles.title} title={title}>
           <span>{title}</span>
+          <span title={blockSizeStats.totalBytes + ' Bytes'}>{humanizeBytes(blockSizeStats.totalBytes)}</span>
         </div>
         <div className={styles.rowsContainer}>
           {Object.keys(data).map((k) => (

diff --git a/pkg/ui/react-app/src/thanos/pages/blocks/__testdata__/testdata.ts b/pkg/ui/react-app/src/thanos/pages/blocks/__testdata__/testdata.ts
@@ -85,6 +85,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'sidecar',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEG1J4JXZGQMZA9TQ3DHMTET',
         version: 1,
@@ -147,6 +164,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'compactor',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEEXQJ71TT74ZTQZN50CEETE',
         version: 1,
@@ -195,6 +229,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'sidecar',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEAQ4FPMZXKJE0EXE5P0YCWP',
         version: 1,
@@ -219,6 +270,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'sidecar',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEF8AGCHTPJ1MZ8KH0SEJZ4E',
         version: 1,
@@ -291,6 +359,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'sidecar',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEF75H09C3TEPJHRHFTCFQD4',
         version: 1,
@@ -413,6 +498,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'compactor',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEB0R6V1EX65QW2B4A2HT0HH',
         version: 1,
@@ -511,6 +613,23 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
             monitor: 'prometheus_one',
           },
           source: 'compactor',
+          files: [
+            {
+              rel_path: 'chunks/000001',
+              size_bytes: 536870882,
+            },
+            {
+              rel_path: 'chunks/000002',
+              size_bytes: 143670,
+            },
+            {
+              rel_path: 'index',
+              size_bytes: 257574,
+            },
+            {
+              rel_path: 'meta.json',
+            },
+          ],
         },
         ulid: '01EEFA63X7DYWPW0AAGX47MY09',
         version: 1,
@@ -2058,3 +2177,46 @@ export const sampleAPIResponse: { status: string; data: BlockListProps } = {
   },
   status: 'success',
 };
+
+export const sizeBlock = {
+  ulid: '01FT8X9MJF5G7PFRNGZBYT8SCS',
+  minTime: 1643123700000,
+  maxTime: 1643124000000,
+  stats: {
+    numSamples: 171320,
+    numSeries: 2859,
+    numChunks: 2859,
+  },
+  compaction: {
+    level: 1,
+    sources: ['01FT8X9MJF5G7PFRNGZBYT8SCS'],
+  },
+  version: 1,
+  thanos: {
+    labels: {
+      prometheus: 'prom-2 random:2',
+    },
+    downsample: {
+      resolution: 0,
+    },
+    source: 'sidecar',
+    segment_files: ['000001', '000002'],
+    files: [
+      {
+        rel_path: 'chunks/000001',
+        size_bytes: 536870882,
+      },
+      {
+        rel_path: 'chunks/000002',
+        size_bytes: 143670,
+      },
+      {
+        rel_path: 'index',
+        size_bytes: 257574,
+      },
+      {
+        rel_path: 'meta.json',
+      },
+    ],
+  },
+};