From bb09c846799686d46058c4d1dc1ba5853dda0bd7 Mon Sep 17 00:00:00 2001 From: Yunus M Date: Tue, 5 Dec 2023 11:15:08 +0530 Subject: [PATCH 0001/1178] fix: text formatting issues and upgrade button style updates (#4141) --- frontend/src/container/Header/Header.styles.scss | 13 +++++++++++++ frontend/src/container/Header/index.tsx | 12 ++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/frontend/src/container/Header/Header.styles.scss b/frontend/src/container/Header/Header.styles.scss index 82dd9b81ff..08db1f8b99 100644 --- a/frontend/src/container/Header/Header.styles.scss +++ b/frontend/src/container/Header/Header.styles.scss @@ -8,5 +8,18 @@ .upgrade-link { padding: 0px; padding-right: 4px; + display: inline !important; color: white; + text-decoration: underline; + text-decoration-color: white; + text-decoration-thickness: 2px; + text-underline-offset: 2px; + + &:hover { + color: white; + text-decoration: underline; + text-decoration-color: white; + text-decoration-thickness: 2px; + text-underline-offset: 2px; + } } diff --git a/frontend/src/container/Header/index.tsx b/frontend/src/container/Header/index.tsx index ff4c560c67..b008e65537 100644 --- a/frontend/src/container/Header/index.tsx +++ b/frontend/src/container/Header/index.tsx @@ -1,3 +1,6 @@ +/* eslint-disable jsx-a11y/no-static-element-interactions */ +/* eslint-disable jsx-a11y/anchor-is-valid */ +/* eslint-disable jsx-a11y/click-events-have-key-events */ import './Header.styles.scss'; import { @@ -135,16 +138,17 @@ function HeaderContainer(): JSX.Element { <> {showTrialExpiryBanner && (
- You are in free trial period. Your free trial will end on + You are in free trial period. Your free trial will end on{' '} {getFormattedDate(licenseData?.payload?.trialEnd || Date.now())}. {role === 'ADMIN' ? ( - Please - + to continue using SigNoz features. ) : ( From 4644b1c200007269717d94b7ee020d2654b98d5a Mon Sep 17 00:00:00 2001 From: Yunus M Date: Tue, 5 Dec 2023 16:09:50 +0530 Subject: [PATCH 0002/1178] fix: custom variables options are not populated (#4154) --- .../DashboardVariablesSelection/VariableItem.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/frontend/src/container/NewDashboard/DashboardVariablesSelection/VariableItem.tsx b/frontend/src/container/NewDashboard/DashboardVariablesSelection/VariableItem.tsx index 4e81b0b116..408605a89b 100644 --- a/frontend/src/container/NewDashboard/DashboardVariablesSelection/VariableItem.tsx +++ b/frontend/src/container/NewDashboard/DashboardVariablesSelection/VariableItem.tsx @@ -224,6 +224,14 @@ function VariableItem({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [debouncedVariableValue]); + useEffect(() => { + // Fetch options for CUSTOM Type + if (variableData.type === 'CUSTOM') { + getOptions(null); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + return ( From 112783d618df74371353787adf4ac9df00e2de19 Mon Sep 17 00:00:00 2001 From: Raj Kamal Singh <1133322+raj-k-singh@users.noreply.github.com> Date: Tue, 5 Dec 2023 18:30:46 +0530 Subject: [PATCH 0003/1178] Feat: fe: logs pipelines severity parsing processor (#4149) --- .../AddNewProcessor/FormFields/CSVInput.tsx | 25 ++++++ .../AddNewProcessor/ProcessorForm.tsx | 79 +++++++++++------ .../AddNewProcessor/config.ts | 88 ++++++++++++++++++- .../AddNewProcessor/styles.scss | 27 ++++++ frontend/src/pages/Pipelines/index.tsx | 8 +- 5 files changed, 200 insertions(+), 27 deletions(-) create mode 100644 frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/FormFields/CSVInput.tsx create mode 100644 frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/styles.scss diff --git a/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/FormFields/CSVInput.tsx b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/FormFields/CSVInput.tsx new file mode 100644 index 0000000000..81050c1ba9 --- /dev/null +++ b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/FormFields/CSVInput.tsx @@ -0,0 +1,25 @@ +import { Input, InputProps } from 'antd'; +import { ChangeEventHandler, useState } from 'react'; + +function CSVInput({ value, onChange, ...otherProps }: InputProps): JSX.Element { + const [inputValue, setInputValue] = useState( + ((value as string[]) || []).join(', '), + ); + + const onChangeHandler = (onChange as unknown) as (v: string[]) => void; + + const onInputChange: ChangeEventHandler = (e) => { + const newValue = e.target.value; + setInputValue(newValue); + + if (onChangeHandler) { + const splitValues = newValue.split(',').map((v) => v.trim()); + onChangeHandler(splitValues); + } + }; + + // eslint-disable-next-line react/jsx-props-no-spreading + return ; +} + +export default CSVInput; diff --git a/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/ProcessorForm.tsx b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/ProcessorForm.tsx index 373f2063ac..1d3b12e2c0 100644 --- a/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/ProcessorForm.tsx +++ b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/ProcessorForm.tsx @@ -1,15 +1,13 @@ +import './styles.scss'; + import { Form, Input, Select } from 'antd'; import { ModalFooterTitle } from 'container/PipelinePage/styles'; import { useTranslation } from 'react-i18next'; import { formValidationRules } from '../config'; import { processorFields, ProcessorFormField } from './config'; -import { - Container, - FormWrapper, - PipelineIndexIcon, - StyledSelect, -} from './styles'; +import CSVInput from './FormFields/CSVInput'; +import { FormWrapper, PipelineIndexIcon, StyledSelect } from './styles'; function ProcessorFieldInput({ fieldData, @@ -25,35 +23,63 @@ function ProcessorFieldInput({ return null; } + // Do not render display elements for hidden inputs. + if (fieldData?.hidden) { + return ( + + + + ); + } + + let inputField; + if (fieldData?.options) { + inputField = ( + + {fieldData.options.map(({ value, label }) => ( + + {label} + + ))} + + ); + } else if (Array.isArray(fieldData?.initialValue)) { + inputField = ; + } else { + inputField = ; + } + return ( - - - {Number(fieldData.id) + 1} - +
+ {!fieldData?.compact && ( + + {Number(fieldData.id) + 1} + + )} {fieldData.fieldName}} - key={fieldData.id} name={fieldData.name} initialValue={fieldData.initialValue} rules={fieldData.rules ? fieldData.rules : formValidationRules} dependencies={fieldData.dependencies || []} > - {fieldData?.options ? ( - - {fieldData.options.map(({ value, label }) => ( - - {label} - - ))} - - ) : ( - - )} + {inputField} - +
); } @@ -63,9 +89,12 @@ interface ProcessorFieldInputProps { function ProcessorForm({ processorType }: ProcessorFormProps): JSX.Element { return ( -
+
{processorFields[processorType]?.map((fieldData: ProcessorFormField) => ( - + ))}
); diff --git a/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/config.ts b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/config.ts index 9dc4f1d1f8..2c9a676898 100644 --- a/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/config.ts +++ b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/config.ts @@ -17,6 +17,7 @@ export const processorTypes: Array = [ { key: 'json_parser', value: 'json_parser', label: 'Json Parser' }, { key: 'trace_parser', value: 'trace_parser', label: 'Trace Parser' }, { key: 'time_parser', value: 'time_parser', label: 'Timestamp Parser' }, + { key: 'severity_parser', value: 'severity_parser', label: 'Severity Parser' }, { key: 'add', value: 'add', label: 'Add' }, { key: 'remove', value: 'remove', label: 'Remove' }, // { key: 'retain', value: 'retain', label: 'Retain' }, @Chintan - Commented as per Nitya's suggestion @@ -31,13 +32,15 @@ export type ProcessorFieldOption = { value: string; }; +// TODO(Raj): Refactor Processor Form code after putting e2e UI tests in place. export type ProcessorFormField = { id: number; fieldName: string; placeholder: string; name: string | NamePath; rules?: Array; - initialValue?: string; + hidden?: boolean; + initialValue?: boolean | string | Array; dependencies?: Array; options?: Array; shouldRender?: (form: FormInstance) => boolean; @@ -45,6 +48,10 @@ export type ProcessorFormField = { changedValues: ProcessorData, form: FormInstance, ) => void; + + // Should this field have its own row or should it + // be packed with other compact fields. + compact?: boolean; }; const traceParserFieldValidator: RuleRender = (form) => ({ @@ -317,6 +324,85 @@ export const processorFields: { [key: string]: Array } = { initialValue: '%Y-%m-%dT%H:%M:%S.%f%z', }, ], + severity_parser: [ + { + id: 1, + fieldName: 'Name of Severity Parsing Processor', + placeholder: 'processor_name_placeholder', + name: 'name', + }, + { + id: 2, + fieldName: 'Parse Severity Value From', + placeholder: 'processor_parsefrom_placeholder', + name: 'parse_from', + initialValue: 'attributes.logLevel', + }, + { + id: 3, + fieldName: 'Values for level TRACE', + placeholder: 'Specify comma separated values. Eg: trace, 0', + name: ['mapping', 'trace'], + rules: [], + initialValue: ['trace'], + compact: true, + }, + { + id: 4, + fieldName: 'Values for level DEBUG', + placeholder: 'Specify comma separated values. Eg: debug, 2xx', + name: ['mapping', 'debug'], + rules: [], + initialValue: ['debug'], + compact: true, + }, + { + id: 5, + fieldName: 'Values for level INFO', + placeholder: 'Specify comma separated values. Eg: info, 3xx', + name: ['mapping', 'info'], + rules: [], + initialValue: ['info'], + compact: true, + }, + { + id: 6, + fieldName: 'Values for level WARN', + placeholder: 'Specify comma separated values. Eg: warning, 4xx', + name: ['mapping', 'warn'], + rules: [], + initialValue: ['warn'], + compact: true, + }, + { + id: 7, + fieldName: 'Values for level ERROR', + placeholder: 'Specify comma separated values. Eg: error, 5xx', + name: ['mapping', 'error'], + rules: [], + initialValue: ['error'], + compact: true, + }, + { + id: 8, + fieldName: 'Values for level FATAL', + placeholder: 'Specify comma separated values. Eg: fatal, panic', + name: ['mapping', 'fatal'], + rules: [], + initialValue: ['fatal'], + compact: true, + }, + { + id: 9, + fieldName: 'Override Severity Text', + placeholder: + 'Should the parsed severity set both severity and severityText?', + name: ['overwrite_text'], + rules: [], + initialValue: true, + hidden: true, + }, + ], retain: [ { id: 1, diff --git a/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/styles.scss b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/styles.scss new file mode 100644 index 0000000000..1fabdd233f --- /dev/null +++ b/frontend/src/container/PipelinePage/PipelineListsView/AddNewProcessor/styles.scss @@ -0,0 +1,27 @@ + +.processor-form-container { + position: relative; + width: 100%; + + display: flex; + flex-wrap: wrap +} + +.processor-field-container { + display: flex; + flex-direction: row; + align-items: flex-start; + padding: 0rem; + gap: 1rem; + width: 100%; +} + +.compact-processor-field-container { + display: flex; + flex-direction: row; + align-items: flex-start; + padding: 0rem; + min-width: 40%; + flex-grow: 1; + margin-left: 2.5rem; +} diff --git a/frontend/src/pages/Pipelines/index.tsx b/frontend/src/pages/Pipelines/index.tsx index dd9ea1d185..1a05a4010a 100644 --- a/frontend/src/pages/Pipelines/index.tsx +++ b/frontend/src/pages/Pipelines/index.tsx @@ -5,7 +5,9 @@ import Spinner from 'components/Spinner'; import ChangeHistory from 'container/PipelinePage/Layouts/ChangeHistory'; import PipelinePage from 'container/PipelinePage/Layouts/Pipeline'; import { useNotifications } from 'hooks/useNotifications'; +import ErrorBoundaryFallback from 'pages/ErrorBoundaryFallback/ErrorBoundaryFallback'; import { useEffect, useMemo } from 'react'; +import { ErrorBoundary } from 'react-error-boundary'; import { useTranslation } from 'react-i18next'; import { useQuery } from 'react-query'; import { SuccessResponse } from 'types/api'; @@ -77,7 +79,11 @@ function Pipelines(): JSX.Element { return ; } - return ; + return ( + + ; + + ); } export default Pipelines; From 8072fede85065f1bcf1f7f1e636315ff8c0e5d4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 08:40:47 +0530 Subject: [PATCH 0004/1178] chore(deps): bump tj-actions/branch-names in /.github/workflows (#4164) Bumps [tj-actions/branch-names](https://github.com/tj-actions/branch-names) from 5.1 to 7.0.7. - [Release notes](https://github.com/tj-actions/branch-names/releases) - [Changelog](https://github.com/tj-actions/branch-names/blob/main/HISTORY.md) - [Commits](https://github.com/tj-actions/branch-names/compare/v5.1...v7.0.7) --- updated-dependencies: - dependency-name: tj-actions/branch-names dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/push.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index f8eb005883..e4e5171e33 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -34,7 +34,7 @@ jobs: id: short-sha - name: Get branch name id: branch-name - uses: tj-actions/branch-names@v5.1 + uses: tj-actions/branch-names@v7.0.7 - name: Set docker tag environment run: | if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then @@ -78,7 +78,7 @@ jobs: id: short-sha - name: Get branch name id: branch-name - uses: tj-actions/branch-names@v5.1 + uses: tj-actions/branch-names@v7.0.7 - name: Set docker tag environment run: | if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then @@ -127,7 +127,7 @@ jobs: id: short-sha - name: Get branch name id: branch-name - uses: tj-actions/branch-names@v5.1 + uses: tj-actions/branch-names@v7.0.7 - name: Set docker tag environment run: | if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then @@ -176,7 +176,7 @@ jobs: id: short-sha - name: Get branch name id: branch-name - uses: tj-actions/branch-names@v5.1 + uses: tj-actions/branch-names@v7.0.7 - name: Set docker tag environment run: | if [ '${{ steps.branch-name.outputs.is_tag }}' == 'true' ]; then From 09d579311e6fb73486d68adae61600c197614335 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Wed, 6 Dec 2023 22:10:36 +0545 Subject: [PATCH 0005/1178] =?UTF-8?q?chore(signoz):=20=F0=9F=93=8C=20pin?= =?UTF-8?q?=20versions:=20SigNoz=200.35.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Shahi --- deploy/docker-swarm/clickhouse-setup/docker-compose.yaml | 4 ++-- deploy/docker/clickhouse-setup/docker-compose.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 19d2884644..763e2ce4bf 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -146,7 +146,7 @@ services: condition: on-failure query-service: - image: signoz/query-service:0.34.4 + image: signoz/query-service:0.35.0 command: [ "-config=/root/config/prometheus.yml", @@ -186,7 +186,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:0.34.4 + image: signoz/frontend:0.35.0 deploy: restart_policy: condition: on-failure diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 7429fb648a..ddf80d6e14 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -164,7 +164,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: - image: signoz/query-service:${DOCKER_TAG:-0.34.4} + image: signoz/query-service:${DOCKER_TAG:-0.35.0} container_name: signoz-query-service command: [ @@ -203,7 +203,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:${DOCKER_TAG:-0.34.4} + image: signoz/frontend:${DOCKER_TAG:-0.35.0} container_name: signoz-frontend restart: on-failure depends_on: From 170e5e16864583c77bdd67fcb4df32ba8d961485 Mon Sep 17 00:00:00 2001 From: Avijeet Pandey <40532869+avijeetpandey@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:53:56 +0530 Subject: [PATCH 0006/1178] fix(FE): Fixes the background color of the dashboards full screen view as per the mode selected i.e dark or light mode (#4175) * fix: full screen bg color of graphs as per dark mode * fix: colors from the constants --- frontend/src/container/GridCardLayout/GridCardLayout.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/container/GridCardLayout/GridCardLayout.tsx b/frontend/src/container/GridCardLayout/GridCardLayout.tsx index 93c779f93f..3ee11b4008 100644 --- a/frontend/src/container/GridCardLayout/GridCardLayout.tsx +++ b/frontend/src/container/GridCardLayout/GridCardLayout.tsx @@ -3,6 +3,7 @@ import './GridCardLayout.styles.scss'; import { PlusOutlined, SaveFilled } from '@ant-design/icons'; import { SOMETHING_WENT_WRONG } from 'constants/api'; import { PANEL_TYPES } from 'constants/queryBuilder'; +import { themeColors } from 'constants/theme'; import { useUpdateDashboard } from 'hooks/dashboard/useUpdateDashboard'; import useComponentPermission from 'hooks/useComponentPermission'; import { useIsDarkMode } from 'hooks/useDarkMode'; @@ -155,6 +156,7 @@ function GraphLayout({ onAddPanelHandler }: GraphLayoutProps): JSX.Element { onLayoutChange={setLayouts} draggableHandle=".drag-handle" layout={layouts} + style={{ backgroundColor: isDarkMode ? '' : themeColors.snowWhite }} > {layouts.map((layout) => { const { i: id } = layout; From 6dd34a7f2950c7a87c68a1e33ab760673d57383b Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Fri, 8 Dec 2023 12:37:19 +0530 Subject: [PATCH 0007/1178] Fix/2967 (#4071) --- .../src/container/TriggeredAlerts/Filter.tsx | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/frontend/src/container/TriggeredAlerts/Filter.tsx b/frontend/src/container/TriggeredAlerts/Filter.tsx index 4a54916685..e7ebdb0d76 100644 --- a/frontend/src/container/TriggeredAlerts/Filter.tsx +++ b/frontend/src/container/TriggeredAlerts/Filter.tsx @@ -1,11 +1,35 @@ /* eslint-disable react/no-unstable-nested-components */ import type { SelectProps } from 'antd'; -import { Tag } from 'antd'; -import { Dispatch, SetStateAction, useCallback, useMemo } from 'react'; +import { Tag, Tooltip } from 'antd'; +import { BaseOptionType } from 'antd/es/select'; +import { Dispatch, SetStateAction, useCallback, useMemo, useRef } from 'react'; import { Alerts } from 'types/api/alerts/getTriggered'; import { Container, Select } from './styles'; +function TextOverflowTooltip({ + option, +}: { + option: BaseOptionType; +}): JSX.Element { + const contentRef = useRef(null); + const isOverflow = contentRef.current + ? contentRef.current?.offsetWidth < contentRef.current?.scrollWidth + : false; + return ( + +
+ {option.value} +
+ + ); +} + function Filter({ setSelectedFilter, setSelectedGroup, @@ -51,6 +75,7 @@ function Filter({ const options = uniqueLabels.map((e) => ({ value: e, + title: '', })); const getTags: SelectProps['tagRender'] = (props): JSX.Element => { @@ -88,6 +113,9 @@ function Filter({ placeholder="Group by any tag" tagRender={(props): JSX.Element => getTags(props)} options={options} + optionRender={(option): JSX.Element => ( + + )} /> ); From 3a1e8d523a09de9eef007acbfe0b9a712a115a50 Mon Sep 17 00:00:00 2001 From: Raj Kamal Singh <1133322+raj-k-singh@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:17:06 +0530 Subject: [PATCH 0008/1178] Fix: qs: allow saving pipelines without connected agents (#4189) * chore: add test validating pipelines can be saved without connected agents * chore: allow pipelines to be saved without connected agents --- pkg/query-service/agentConf/db.go | 8 +- pkg/query-service/agentConf/manager.go | 20 --- pkg/query-service/agentConf/version.go | 2 + .../app/logparsingpipeline/controller.go | 6 - .../integration/logparsingpipeline_test.go | 145 ++++++++++++------ 5 files changed, 106 insertions(+), 75 deletions(-) diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 3369dbe23f..ffbc2f53a8 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -50,8 +50,8 @@ func (r *Repo) GetConfigHistory( disabled, deploy_status, deploy_result, - last_hash, - last_config + coalesce(last_hash, '') as last_hash, + coalesce(last_config, '{}') as last_config FROM agent_config_versions AS v WHERE element_type = $1 ORDER BY created_at desc, version desc @@ -89,8 +89,8 @@ func (r *Repo) GetConfigVersion( disabled, deploy_status, deploy_result, - last_hash, - last_config + coalesce(last_hash, '') as last_hash, + coalesce(last_config, '{}') as last_config FROM agent_config_versions v WHERE element_type = $1 AND version = $2`, typ, v) diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index a919185d0d..0e77383f7e 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -172,21 +172,6 @@ func (m *Manager) ReportConfigDeploymentStatus( } } -// Ready indicates if Manager can accept new config update requests -func (mgr *Manager) Ready() bool { - if atomic.LoadUint32(&mgr.lock) != 0 { - return false - } - return opamp.Ready() -} - -// Static methods for working with default manager instance in this module. - -// Ready indicates if Manager can accept new config update requests -func Ready() bool { - return m.Ready() -} - func GetLatestVersion( ctx context.Context, elementType ElementTypeDef, ) (*ConfigVersion, *model.ApiError) { @@ -210,11 +195,6 @@ func StartNewVersion( ctx context.Context, userId string, eleType ElementTypeDef, elementIds []string, ) (*ConfigVersion, *model.ApiError) { - if !m.Ready() { - // agent is already being updated, ask caller to wait and re-try after sometime - return nil, model.UnavailableError(fmt.Errorf("agent updater is busy")) - } - // create a new version cfg := NewConfigversion(eleType) diff --git a/pkg/query-service/agentConf/version.go b/pkg/query-service/agentConf/version.go index 13be5f7cd2..d2bc4547b3 100644 --- a/pkg/query-service/agentConf/version.go +++ b/pkg/query-service/agentConf/version.go @@ -53,6 +53,8 @@ func NewConfigversion(typeDef ElementTypeDef) *ConfigVersion { IsValid: false, Disabled: false, DeployStatus: PendingDeploy, + LastHash: "", + LastConf: "{}", // todo: get user id from context? // CreatedBy } diff --git a/pkg/query-service/app/logparsingpipeline/controller.go b/pkg/query-service/app/logparsingpipeline/controller.go index 7880ac27b7..066123c416 100644 --- a/pkg/query-service/app/logparsingpipeline/controller.go +++ b/pkg/query-service/app/logparsingpipeline/controller.go @@ -73,12 +73,6 @@ func (ic *LogParsingPipelineController) ApplyPipelines( } - if !agentConf.Ready() { - return nil, model.UnavailableError(fmt.Errorf( - "agent updater unavailable at the moment. Please try in sometime", - )) - } - // prepare config elements elements := make([]string, len(pipelines)) for i, p := range pipelines { diff --git a/pkg/query-service/tests/integration/logparsingpipeline_test.go b/pkg/query-service/tests/integration/logparsingpipeline_test.go index 0b4d22973c..4c260596e5 100644 --- a/pkg/query-service/tests/integration/logparsingpipeline_test.go +++ b/pkg/query-service/tests/integration/logparsingpipeline_test.go @@ -367,17 +367,70 @@ func TestLogPipelinesValidation(t *testing.T) { } } +func TestCanSavePipelinesWithoutConnectedAgents(t *testing.T) { + require := require.New(t) + testbed := NewTestbedWithoutOpamp(t) + + getPipelinesResp := testbed.GetPipelinesFromQS() + require.Equal(0, len(getPipelinesResp.Pipelines)) + require.Equal(0, len(getPipelinesResp.History)) + + postablePipelines := logparsingpipeline.PostablePipelines{ + Pipelines: []logparsingpipeline.PostablePipeline{ + { + OrderId: 1, + Name: "pipeline1", + Alias: "pipeline1", + Enabled: true, + Filter: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "method", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + Operator: "=", + Value: "GET", + }, + }, + }, + Config: []logparsingpipeline.PipelineOperator{ + { + OrderId: 1, + ID: "add", + Type: "add", + Field: "attributes.test", + Value: "val", + Enabled: true, + Name: "test add", + }, + }, + }, + }, + } + + testbed.PostPipelinesToQS(postablePipelines) + getPipelinesResp = testbed.GetPipelinesFromQS() + require.Equal(1, len(getPipelinesResp.Pipelines)) + require.Equal(1, len(getPipelinesResp.History)) + +} + // LogPipelinesTestBed coordinates and mocks components involved in // configuring log pipelines and provides test helpers. type LogPipelinesTestBed struct { t *testing.T + testDBFilePath string testUser *model.User apiHandler *app.APIHandler + agentConfMgr *agentConf.Manager opampServer *opamp.Server opampClientConn *opamp.MockOpAmpConnection } -func NewLogPipelinesTestBed(t *testing.T) *LogPipelinesTestBed { +func NewTestbedWithoutOpamp(t *testing.T) *LogPipelinesTestBed { // Create a tmp file based sqlite db for testing. testDBFile, err := os.CreateTemp("", "test-signoz-db-*") if err != nil { @@ -408,22 +461,61 @@ func NewLogPipelinesTestBed(t *testing.T) *LogPipelinesTestBed { t.Fatalf("could not create a new ApiHandler: %v", err) } - opampServer, clientConn := mockOpampAgent(t, testDBFilePath, controller) - user, apiErr := createTestUser() if apiErr != nil { t.Fatalf("could not create a test user: %v", apiErr) } + // Mock an available opamp agent + testDB, err = opampModel.InitDB(testDBFilePath) + require.Nil(t, err, "failed to init opamp model") + + agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{ + DB: testDB, + DBEngine: "sqlite", + AgentFeatures: []agentConf.AgentFeature{ + apiHandler.LogsParsingPipelineController, + }}) + require.Nil(t, err, "failed to init agentConf") + return &LogPipelinesTestBed{ - t: t, - testUser: user, - apiHandler: apiHandler, - opampServer: opampServer, - opampClientConn: clientConn, + t: t, + testDBFilePath: testDBFilePath, + testUser: user, + apiHandler: apiHandler, + agentConfMgr: agentConfMgr, } } +func NewLogPipelinesTestBed(t *testing.T) *LogPipelinesTestBed { + testbed := NewTestbedWithoutOpamp(t) + + opampServer := opamp.InitializeServer(nil, testbed.agentConfMgr) + err := opampServer.Start(opamp.GetAvailableLocalAddress()) + require.Nil(t, err, "failed to start opamp server") + + t.Cleanup(func() { + opampServer.Stop() + }) + + opampClientConnection := &opamp.MockOpAmpConnection{} + opampServer.OnMessage( + opampClientConnection, + &protobufs.AgentToServer{ + InstanceUid: "test", + EffectiveConfig: &protobufs.EffectiveConfig{ + ConfigMap: newInitialAgentConfigMap(), + }, + }, + ) + + testbed.opampServer = opampServer + testbed.opampClientConn = opampClientConnection + + return testbed + +} + func (tb *LogPipelinesTestBed) PostPipelinesToQSExpectingStatusCode( postablePipelines logparsingpipeline.PostablePipelines, expectedStatusCode int, @@ -668,43 +760,6 @@ func assertPipelinesResponseMatchesPostedPipelines( } } -func mockOpampAgent( - t *testing.T, - testDBFilePath string, - pipelinesController *logparsingpipeline.LogParsingPipelineController, -) (*opamp.Server, *opamp.MockOpAmpConnection) { - // Mock an available opamp agent - testDB, err := opampModel.InitDB(testDBFilePath) - require.Nil(t, err, "failed to init opamp model") - - agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{ - DB: testDB, - DBEngine: "sqlite", - AgentFeatures: []agentConf.AgentFeature{pipelinesController}, - }) - require.Nil(t, err, "failed to init agentConf") - - opampServer := opamp.InitializeServer(nil, agentConfMgr) - err = opampServer.Start(opamp.GetAvailableLocalAddress()) - require.Nil(t, err, "failed to start opamp server") - - t.Cleanup(func() { - opampServer.Stop() - }) - - opampClientConnection := &opamp.MockOpAmpConnection{} - opampServer.OnMessage( - opampClientConnection, - &protobufs.AgentToServer{ - InstanceUid: "test", - EffectiveConfig: &protobufs.EffectiveConfig{ - ConfigMap: newInitialAgentConfigMap(), - }, - }, - ) - return opampServer, opampClientConnection -} - func newInitialAgentConfigMap() *protobufs.AgentConfigMap { return &protobufs.AgentConfigMap{ ConfigMap: map[string]*protobufs.AgentConfigFile{ From fd9566d47127b99fc427b5012e3e5b3501f168de Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Mon, 11 Dec 2023 16:09:28 +0530 Subject: [PATCH 0009/1178] fix: incorrect alert description and summary for prom rules (#4190) --- pkg/query-service/rules/promRule.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/rules/promRule.go b/pkg/query-service/rules/promRule.go index 6d0cafa930..94ace4137b 100644 --- a/pkg/query-service/rules/promRule.go +++ b/pkg/query-service/rules/promRule.go @@ -3,7 +3,6 @@ package rules import ( "context" "fmt" - "strconv" "sync" "time" @@ -367,7 +366,10 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time, queriers *Queriers) ( l[lbl.Name] = lbl.Value } - tmplData := AlertTemplateData(l, valueFormatter.Format(smpl.F, r.Unit()), strconv.FormatFloat(r.targetVal(), 'f', 2, 64)+converter.UnitToName(r.ruleCondition.TargetUnit)) + thresholdFormatter := formatter.FromUnit(r.ruleCondition.TargetUnit) + threshold := thresholdFormatter.Format(r.targetVal(), r.ruleCondition.TargetUnit) + + tmplData := AlertTemplateData(l, valueFormatter.Format(smpl.F, r.Unit()), threshold) // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}{{$threshold := .Threshold}}" From 9826ab04b3faa5d991e46be8e9ea8630f48f68ed Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Mon, 11 Dec 2023 17:46:08 +0530 Subject: [PATCH 0010/1178] chore: add new endpoint for variable replacement (#4191) Co-authored-by: Palash Gupta --- pkg/query-service/app/http_handler.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 973e23328c..1d01267860 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -314,6 +314,7 @@ func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMid subRouter.HandleFunc("/autocomplete/attribute_values", am.ViewAccess( withCacheControl(AutoCompleteCacheControlAge, aH.autoCompleteAttributeValues))).Methods(http.MethodGet) subRouter.HandleFunc("/query_range", am.ViewAccess(aH.QueryRangeV3)).Methods(http.MethodPost) + subRouter.HandleFunc("/query_range/format", am.ViewAccess(aH.QueryRangeV3Format)).Methods(http.MethodPost) // live logs subRouter.HandleFunc("/logs/livetail", am.ViewAccess(aH.liveTailLogs)).Methods(http.MethodGet) @@ -3001,6 +3002,18 @@ func (aH *APIHandler) getSpanKeysV3(ctx context.Context, queryRangeParams *v3.Qu return data, nil } +func (aH *APIHandler) QueryRangeV3Format(w http.ResponseWriter, r *http.Request) { + queryRangeParams, apiErrorObj := ParseQueryRangeParams(r) + + if apiErrorObj != nil { + zap.S().Errorf(apiErrorObj.Err.Error()) + RespondError(w, apiErrorObj, nil) + return + } + + aH.Respond(w, queryRangeParams) +} + func (aH *APIHandler) queryRangeV3(ctx context.Context, queryRangeParams *v3.QueryRangeParamsV3, w http.ResponseWriter, r *http.Request) { var result []*v3.Result From 6170b2c5dc4dd517e9e12f2dc9c8668c8b0d4408 Mon Sep 17 00:00:00 2001 From: Rajat Dabade Date: Mon, 11 Dec 2023 18:34:24 +0530 Subject: [PATCH 0011/1178] [Refactor]: added percent 0 - 100 in yaxis for alerts (#4173) --- .../container/FormAlertRules/ChartPreview/index.tsx | 10 ++++++---- .../container/FormAlertRules/ChartPreview/utils.ts | 12 ++++++++++++ frontend/src/container/FormAlertRules/index.tsx | 11 +++++++++-- .../NewWidget/RightContainer/alertFomatCategories.ts | 1 + .../filters/BuilderUnitsFilter/BuilderUnits.tsx | 3 ++- .../QueryBuilder/filters/BuilderUnitsFilter/types.ts | 1 + frontend/src/lib/getConvertedValue.ts | 5 +++++ 7 files changed, 36 insertions(+), 7 deletions(-) diff --git a/frontend/src/container/FormAlertRules/ChartPreview/index.tsx b/frontend/src/container/FormAlertRules/ChartPreview/index.tsx index 400a6f85be..3551c3a87a 100644 --- a/frontend/src/container/FormAlertRules/ChartPreview/index.tsx +++ b/frontend/src/container/FormAlertRules/ChartPreview/index.tsx @@ -32,6 +32,7 @@ export interface ChartPreviewProps { alertDef?: AlertDef; userQueryKey?: string; allowSelectedIntervalForStepGen?: boolean; + yAxisUnit: string; } function ChartPreview({ @@ -44,6 +45,7 @@ function ChartPreview({ userQueryKey, allowSelectedIntervalForStepGen = false, alertDef, + yAxisUnit, }: ChartPreviewProps): JSX.Element | null { const { t } = useTranslation('alerts'); const threshold = alertDef?.condition.target || 0; @@ -112,7 +114,7 @@ function ChartPreview({ () => getUPlotChartOptions({ id: 'alert_legend_widget', - yAxisUnit: query?.unit, + yAxisUnit, apiResponse: queryResponse?.data?.payload, dimensions: containerDimensions, isDarkMode, @@ -129,14 +131,14 @@ function ChartPreview({ optionName, threshold, alertDef?.condition.targetUnit, - query?.unit, + yAxisUnit, )})`, thresholdUnit: alertDef?.condition.targetUnit, }, ], }), [ - query?.unit, + yAxisUnit, queryResponse?.data?.payload, containerDimensions, isDarkMode, @@ -168,7 +170,7 @@ function ChartPreview({ name={name || 'Chart Preview'} panelData={queryResponse.data?.payload.data.newResult.data.result || []} query={query || initialQueriesMap.metrics} - yAxisUnit={query?.unit} + yAxisUnit={yAxisUnit} />
)} diff --git a/frontend/src/container/FormAlertRules/ChartPreview/utils.ts b/frontend/src/container/FormAlertRules/ChartPreview/utils.ts index f17a6e3865..1fc2a2e247 100644 --- a/frontend/src/container/FormAlertRules/ChartPreview/utils.ts +++ b/frontend/src/container/FormAlertRules/ChartPreview/utils.ts @@ -61,8 +61,20 @@ export const getThresholdLabel = ( unit === MiscellaneousFormats.PercentUnit || yAxisUnit === MiscellaneousFormats.PercentUnit ) { + if (unit === MiscellaneousFormats.Percent) { + return `${value}%`; + } return `${value * 100}%`; } + if ( + unit === MiscellaneousFormats.Percent || + yAxisUnit === MiscellaneousFormats.Percent + ) { + if (unit === MiscellaneousFormats.PercentUnit) { + return `${value * 100}%`; + } + return `${value}%`; + } return `${value} ${optionName}`; }; diff --git a/frontend/src/container/FormAlertRules/index.tsx b/frontend/src/container/FormAlertRules/index.tsx index 0076449faf..eafbcd363a 100644 --- a/frontend/src/container/FormAlertRules/index.tsx +++ b/frontend/src/container/FormAlertRules/index.tsx @@ -82,6 +82,7 @@ function FormAlertRules({ // alertDef holds the form values to be posted const [alertDef, setAlertDef] = useState(initialValue); + const [yAxisUnit, setYAxisUnit] = useState(currentQuery.unit || ''); // initQuery contains initial query when component was mounted const initQuery = useMemo(() => initialValue.condition.compositeQuery, [ @@ -400,6 +401,7 @@ function FormAlertRules({ query={stagedQuery} selectedInterval={globalSelectedInterval} alertDef={alertDef} + yAxisUnit={yAxisUnit || ''} /> ); @@ -415,6 +417,7 @@ function FormAlertRules({ query={stagedQuery} alertDef={alertDef} selectedInterval={globalSelectedInterval} + yAxisUnit={yAxisUnit || ''} /> ); @@ -427,7 +430,8 @@ function FormAlertRules({ currentQuery.queryType === EQueryType.QUERY_BUILDER && alertType !== AlertTypes.METRICS_BASED_ALERT; - const onUnitChangeHandler = (): void => { + const onUnitChangeHandler = (value: string): void => { + setYAxisUnit(value); // reset target unit setAlertDef((def) => ({ ...def, @@ -457,7 +461,10 @@ function FormAlertRules({ renderPromAndChQueryChartPreview()} - + ({ label: category, diff --git a/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts b/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts index 693dab7be6..4474c7d20f 100644 --- a/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts +++ b/frontend/src/container/QueryBuilder/filters/BuilderUnitsFilter/types.ts @@ -1,3 +1,4 @@ export interface IBuilderUnitsFilterProps { onChange?: (value: string) => void; + yAxisUnit?: string; } diff --git a/frontend/src/lib/getConvertedValue.ts b/frontend/src/lib/getConvertedValue.ts index 229b2d2677..0e8c267236 100644 --- a/frontend/src/lib/getConvertedValue.ts +++ b/frontend/src/lib/getConvertedValue.ts @@ -232,6 +232,11 @@ const unitsMapping = [ { label: 'Percent (0.0-1.0)', value: 'percentunit', + factor: 100, + }, + { + label: 'Percent (0 - 100)', + value: 'percent', factor: 1, }, ], From fb1dbdc05ea9fabd7f953f17e4c169995c200489 Mon Sep 17 00:00:00 2001 From: guangwu Date: Mon, 11 Dec 2023 21:15:47 +0800 Subject: [PATCH 0012/1178] chore: use bytes.Equal instead (#4201) --- pkg/query-service/app/opamp/model/agent.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go index a6f9dd66ef..758531f345 100644 --- a/pkg/query-service/app/opamp/model/agent.go +++ b/pkg/query-service/app/opamp/model/agent.go @@ -259,7 +259,7 @@ func (agent *Agent) processStatusUpdate( // send the new remote config to the Agent. if configChanged || (agent.Status.RemoteConfigStatus != nil && - bytes.Compare(agent.Status.RemoteConfigStatus.LastRemoteConfigHash, agent.remoteConfig.ConfigHash) != 0) { + !bytes.Equal(agent.Status.RemoteConfigStatus.LastRemoteConfigHash, agent.remoteConfig.ConfigHash)) { // The new status resulted in a change in the config of the Agent or the Agent // does not have this config (hash is different). Send the new config the Agent. response.RemoteConfig = agent.remoteConfig @@ -352,7 +352,7 @@ func isEqualConfigFile(f1, f2 *protobufs.AgentConfigFile) bool { if f1 == nil || f2 == nil { return false } - return bytes.Compare(f1.Body, f2.Body) == 0 && f1.ContentType == f2.ContentType + return bytes.Equal(f1.Body, f2.Body) && f1.ContentType == f2.ContentType } func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { From 9360c61dcafbbc0a07d20301ac8003467e3fd3c8 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 12 Dec 2023 07:24:33 +0530 Subject: [PATCH 0013/1178] chore: update BuilderQuery struct and add PrepareTimeseriesFilterQuery (#4165) --- .../app/metrics/v4/query_builder.go | 86 ++++++++++ .../app/metrics/v4/query_builder_test.go | 150 ++++++++++++++++++ pkg/query-service/model/v3/v3.go | 35 ++++ 3 files changed, 271 insertions(+) create mode 100644 pkg/query-service/app/metrics/v4/query_builder.go create mode 100644 pkg/query-service/app/metrics/v4/query_builder_test.go diff --git a/pkg/query-service/app/metrics/v4/query_builder.go b/pkg/query-service/app/metrics/v4/query_builder.go new file mode 100644 index 0000000000..70d35e8e08 --- /dev/null +++ b/pkg/query-service/app/metrics/v4/query_builder.go @@ -0,0 +1,86 @@ +package v4 + +import ( + "fmt" + "strings" + + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +// PrepareTimeseriesFilterQuery builds the sub-query to be used for filtering timeseries based on the search criteria +func PrepareTimeseriesFilterQuery(mq *v3.BuilderQuery) (string, error) { + var conditions []string + var fs *v3.FilterSet = mq.Filters + var groupTags []v3.AttributeKey = mq.GroupBy + + conditions = append(conditions, fmt.Sprintf("metric_name = %s", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key))) + conditions = append(conditions, fmt.Sprintf("temporality = '%s'", mq.Temporality)) + + if fs != nil && len(fs.Items) != 0 { + for _, item := range fs.Items { + toFormat := item.Value + op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator)))) + if op == v3.FilterOperatorContains || op == v3.FilterOperatorNotContains { + toFormat = fmt.Sprintf("%%%s%%", toFormat) + } + fmtVal := utils.ClickHouseFormattedValue(toFormat) + switch op { + case v3.FilterOperatorEqual: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') = %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotEqual: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') != %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorIn: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') IN %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotIn: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') NOT IN %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorLike: + conditions = append(conditions, fmt.Sprintf("like(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotLike: + conditions = append(conditions, fmt.Sprintf("notLike(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorRegex: + conditions = append(conditions, fmt.Sprintf("match(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotRegex: + conditions = append(conditions, fmt.Sprintf("not match(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorGreaterThan: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') > %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorGreaterThanOrEq: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') >= %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorLessThan: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') < %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorLessThanOrEq: + conditions = append(conditions, fmt.Sprintf("JSONExtractString(labels, '%s') <= %s", item.Key.Key, fmtVal)) + case v3.FilterOperatorContains: + conditions = append(conditions, fmt.Sprintf("like(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorNotContains: + conditions = append(conditions, fmt.Sprintf("notLike(JSONExtractString(labels, '%s'), %s)", item.Key.Key, fmtVal)) + case v3.FilterOperatorExists: + conditions = append(conditions, fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", item.Key.Key)) + case v3.FilterOperatorNotExists: + conditions = append(conditions, fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", item.Key.Key)) + default: + return "", fmt.Errorf("unsupported filter operator") + } + } + } + whereClause := strings.Join(conditions, " AND ") + + var selectLabels string + for _, tag := range groupTags { + selectLabels += fmt.Sprintf("JSONExtractString(labels, '%s') as %s, ", tag.Key, tag.Key) + } + + // The table JOIN key always exists + selectLabels += "fingerprint" + + filterSubQuery := fmt.Sprintf( + "SELECT DISTINCT %s FROM %s.%s WHERE %s", + selectLabels, + constants.SIGNOZ_METRIC_DBNAME, + constants.SIGNOZ_TIMESERIES_LOCAL_TABLENAME, + whereClause, + ) + + return filterSubQuery, nil +} diff --git a/pkg/query-service/app/metrics/v4/query_builder_test.go b/pkg/query-service/app/metrics/v4/query_builder_test.go new file mode 100644 index 0000000000..eb071ecb2f --- /dev/null +++ b/pkg/query-service/app/metrics/v4/query_builder_test.go @@ -0,0 +1,150 @@ +package v4 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func TestPrepareTimeseriesFilterQuery(t *testing.T) { + testCases := []struct { + name string + builderQuery *v3.BuilderQuery + expectedQueryContains string + }{ + { + name: "test prepare time series with no filters and no group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Delta, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Delta'", + }, + { + name: "test prepare time series with no filters and group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative'", + }, + { + name: "test prepare time series with no filters and multiple group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + GroupBy: []v3.AttributeKey{ + { + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + { + Key: "endpoint", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }, + }, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, JSONExtractString(labels, 'endpoint') as endpoint, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative'", + }, + { + name: "test prepare time series with filters and multiple group by", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorNotEqual, + Value: "payment_service", + }, + { + Key: v3.AttributeKey{ + Key: "endpoint", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorIn, + Value: []interface{}{"/paycallback", "/payme", "/paypal"}, + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + // remaining struct fields are not needed here + }, + expectedQueryContains: "SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + query, err := PrepareTimeseriesFilterQuery(testCase.builderQuery) + assert.Nil(t, err) + assert.Contains(t, query, testCase.expectedQueryContains) + }) + } +} diff --git a/pkg/query-service/model/v3/v3.go b/pkg/query-service/model/v3/v3.go index e04be217cf..453c6475a8 100644 --- a/pkg/query-service/model/v3/v3.go +++ b/pkg/query-service/model/v3/v3.go @@ -447,6 +447,38 @@ const ( Cumulative Temporality = "Cumulative" ) +type TimeAggregation string + +const ( + TimeAggregationUnspecified TimeAggregation = "" + TimeAggregationAnyLast TimeAggregation = "latest" + TimeAggregationSum TimeAggregation = "sum" + TimeAggregationAvg TimeAggregation = "avg" + TimeAggregationMin TimeAggregation = "min" + TimeAggregationMax TimeAggregation = "max" + TimeAggregationCount TimeAggregation = "count" + TimeAggregationCountDistinct TimeAggregation = "count_distinct" + TimeAggregationRate TimeAggregation = "rate" + TimeAggregationIncrease TimeAggregation = "increase" +) + +type SpaceAggregation string + +const ( + SpaceAggregationUnspecified SpaceAggregation = "" + SpaceAggregationSum SpaceAggregation = "sum" + SpaceAggregationAvg SpaceAggregation = "avg" + SpaceAggregationMin SpaceAggregation = "min" + SpaceAggregationMax SpaceAggregation = "max" + SpaceAggregationCount SpaceAggregation = "count" +) + +type Function struct { + Category string `json:"category"` + Name string `json:"name"` + Args []interface{} `json:"args,omitempty"` +} + type BuilderQuery struct { QueryName string `json:"queryName"` StepInterval int64 `json:"stepInterval"` @@ -466,6 +498,9 @@ type BuilderQuery struct { OrderBy []OrderBy `json:"orderBy,omitempty"` ReduceTo ReduceToOperator `json:"reduceTo,omitempty"` SelectColumns []AttributeKey `json:"selectColumns,omitempty"` + TimeAggregation TimeAggregation `json:"timeAggregation,omitempty"` + SpaceAggregation SpaceAggregation `json:"spaceAggregation,omitempty"` + Functions []Function `json:"functions,omitempty"` } func (b *BuilderQuery) Validate() error { From bcebe050b1c4a9739d0452456f0cffa1fd013b62 Mon Sep 17 00:00:00 2001 From: Yunus M Date: Tue, 12 Dec 2023 11:19:06 +0530 Subject: [PATCH 0014/1178] fix: variable edit flow - use updated query value on test query --- .../DashboardSettings/Variables/VariableItem/VariableItem.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx index 91faea4fcc..621c3dcd5c 100644 --- a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx +++ b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx @@ -140,7 +140,7 @@ function VariableItem({ enabled: false, queryFn: () => dashboardVariablesQuery({ - query: variableData.queryValue || '', + query: variableQueryValue || '', variables: variablePropsToPayloadVariables(existingVariables), }), refetchOnWindowFocus: false, From 3c284fc9ee053d089cb908cdfab40fac809ab7fc Mon Sep 17 00:00:00 2001 From: Yunus M Date: Tue, 12 Dec 2023 11:38:06 +0530 Subject: [PATCH 0015/1178] Revert "fix: variable edit flow - use updated query value on test query" (#4207) This reverts commit bcebe050b1c4a9739d0452456f0cffa1fd013b62. --- .../DashboardSettings/Variables/VariableItem/VariableItem.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx index 621c3dcd5c..91faea4fcc 100644 --- a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx +++ b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx @@ -140,7 +140,7 @@ function VariableItem({ enabled: false, queryFn: () => dashboardVariablesQuery({ - query: variableQueryValue || '', + query: variableData.queryValue || '', variables: variablePropsToPayloadVariables(existingVariables), }), refetchOnWindowFocus: false, From e557ff273fea2d6474da955bc3f34714942e5a59 Mon Sep 17 00:00:00 2001 From: Palash Gupta Date: Tue, 12 Dec 2023 14:16:06 +0530 Subject: [PATCH 0016/1178] test: metrics application test are added (#4137) * test: metrics application test are added * fix: getTopOperationList is moved under __mocks__ --- .../__mocks__/getTopOperation.ts | 19 +++++ .../MetricsApplication/utils.test.ts | 70 +++++++++++++++++++ .../src/container/MetricsApplication/utils.ts | 8 ++- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts create mode 100644 frontend/src/container/MetricsApplication/utils.test.ts diff --git a/frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts b/frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts new file mode 100644 index 0000000000..27833f5415 --- /dev/null +++ b/frontend/src/container/MetricsApplication/__mocks__/getTopOperation.ts @@ -0,0 +1,19 @@ +import { TopOperationList } from '../TopOperationsTable'; + +interface TopOperation { + numCalls: number; + errorCount: number; +} + +export const getTopOperationList = ({ + errorCount, + numCalls, +}: TopOperation): TopOperationList => + ({ + p50: 0, + errorCount, + name: 'test', + numCalls, + p95: 0, + p99: 0, + } as TopOperationList); diff --git a/frontend/src/container/MetricsApplication/utils.test.ts b/frontend/src/container/MetricsApplication/utils.test.ts new file mode 100644 index 0000000000..7ad338d850 --- /dev/null +++ b/frontend/src/container/MetricsApplication/utils.test.ts @@ -0,0 +1,70 @@ +import { getTopOperationList } from './__mocks__/getTopOperation'; +import { TopOperationList } from './TopOperationsTable'; +import { + convertedTracesToDownloadData, + getErrorRate, + getNearestHighestBucketValue, +} from './utils'; + +describe('Error Rate', () => { + test('should return correct error rate', () => { + const list: TopOperationList = getTopOperationList({ + errorCount: 10, + numCalls: 100, + }); + + expect(getErrorRate(list)).toBe(10); + }); + + test('should handle no errors gracefully', () => { + const list = getTopOperationList({ errorCount: 0, numCalls: 100 }); + expect(getErrorRate(list)).toBe(0); + }); + + test('should handle zero calls', () => { + const list = getTopOperationList({ errorCount: 0, numCalls: 0 }); + expect(getErrorRate(list)).toBe(0); + }); +}); + +describe('getNearestHighestBucketValue', () => { + test('should return nearest higher bucket value', () => { + expect(getNearestHighestBucketValue(50, [10, 20, 30, 40, 60, 70])).toBe('60'); + }); + + test('should return +Inf for value higher than any bucket', () => { + expect(getNearestHighestBucketValue(80, [10, 20, 30, 40, 60, 70])).toBe( + '+Inf', + ); + }); + + test('should return the first bucket for value lower than all buckets', () => { + expect(getNearestHighestBucketValue(5, [10, 20, 30, 40, 60, 70])).toBe('10'); + }); +}); + +describe('convertedTracesToDownloadData', () => { + test('should convert trace data correctly', () => { + const data = [ + { + name: 'op1', + p50: 50000000, + p95: 95000000, + p99: 99000000, + numCalls: 100, + errorCount: 10, + }, + ]; + + expect(convertedTracesToDownloadData(data)).toEqual([ + { + Name: 'op1', + 'P50 (in ms)': '50.00', + 'P95 (in ms)': '95.00', + 'P99 (in ms)': '99.00', + 'Number of calls': '100', + 'Error Rate (%)': '10.00', + }, + ]); + }); +}); diff --git a/frontend/src/container/MetricsApplication/utils.ts b/frontend/src/container/MetricsApplication/utils.ts index a6aec561d7..0fdf307ace 100644 --- a/frontend/src/container/MetricsApplication/utils.ts +++ b/frontend/src/container/MetricsApplication/utils.ts @@ -5,8 +5,12 @@ import history from 'lib/history'; import { TopOperationList } from './TopOperationsTable'; import { NavigateToTraceProps } from './types'; -export const getErrorRate = (list: TopOperationList): number => - (list.errorCount / list.numCalls) * 100; +export const getErrorRate = (list: TopOperationList): number => { + if (list.errorCount === 0 && list.numCalls === 0) { + return 0; + } + return (list.errorCount / list.numCalls) * 100; +}; export const navigateToTrace = ({ servicename, From b557ca5519863a0873fe52c9f6cf7f31885c6f59 Mon Sep 17 00:00:00 2001 From: Yunus M Date: Tue, 12 Dec 2023 16:30:22 +0530 Subject: [PATCH 0017/1178] =?UTF-8?q?fix:=20use=20updated=20query=20value?= =?UTF-8?q?=20on=20test=20query,=20restrict=20direct=20commit=20to?= =?UTF-8?q?=E2=80=A6=20(#4210)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use updated query value on test query, restrict direct commit to develop,main * fix: reset error preview on success --- frontend/.husky/commit-msg | 16 ++++++++++++++++ .../Variables/VariableItem/VariableItem.tsx | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/frontend/.husky/commit-msg b/frontend/.husky/commit-msg index be422b654c..cb50d87a2d 100755 --- a/frontend/.husky/commit-msg +++ b/frontend/.husky/commit-msg @@ -2,3 +2,19 @@ . "$(dirname "$0")/_/husky.sh" cd frontend && yarn run commitlint --edit $1 + +branch="$(git rev-parse --abbrev-ref HEAD)" + +color_red="$(tput setaf 1)" +bold="$(tput bold)" +reset="$(tput sgr0)" + +if [ "$branch" = "main" ]; then + echo "${color_red}${bold}You can't commit directly to the main branch${reset}" + exit 1 +fi + +if [ "$branch" = "develop" ]; then + echo "${color_red}${bold}You can't commit directly to the develop branch${reset}" + exit 1 +fi \ No newline at end of file diff --git a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx index 91faea4fcc..76f0464bd2 100644 --- a/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx +++ b/frontend/src/container/NewDashboard/DashboardSettings/Variables/VariableItem/VariableItem.tsx @@ -140,11 +140,12 @@ function VariableItem({ enabled: false, queryFn: () => dashboardVariablesQuery({ - query: variableData.queryValue || '', + query: variableQueryValue || '', variables: variablePropsToPayloadVariables(existingVariables), }), refetchOnWindowFocus: false, onSuccess: (response) => { + setErrorPreview(null); handleQueryResult(response); }, onError: (error: { From 8b1a781f5894b1511c4faa5996aaddf0bf07228c Mon Sep 17 00:00:00 2001 From: Yunus M Date: Tue, 12 Dec 2023 17:18:57 +0530 Subject: [PATCH 0018/1178] =?UTF-8?q?feat:=20pass=20abort=20signal=20to=20?= =?UTF-8?q?cancel=20api=20request=20on=20query-key=20change=20or=20?= =?UTF-8?q?=E2=80=A6=20(#4193)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: pass abort signal to cancel api request on query-key change or dashboard unmount * fix: transformIgnorePatterns axios * fix: remove axios types * feat: handle error type from dashboardAPI response * feat: remove console.log --- frontend/jest.config.ts | 2 +- frontend/package.json | 4 +-- frontend/src/api/ErrorResponseHandler.ts | 4 +-- frontend/src/api/dashboard/get.ts | 4 +-- frontend/src/api/index.ts | 12 ++++--- frontend/src/api/metrics/getQueryRange.ts | 3 +- .../Preview/hooks/usePipelinePreview.ts | 2 +- .../hooks/queryBuilder/useGetQueryRange.ts | 9 +++-- frontend/src/lib/dashboard/getQueryResults.ts | 3 +- .../src/pages/NewDashboard/DashboardPage.tsx | 4 ++- .../src/providers/Dashboard/Dashboard.tsx | 4 +-- frontend/yarn.lock | 34 +++++++++++++++++-- 12 files changed, 62 insertions(+), 23 deletions(-) diff --git a/frontend/jest.config.ts b/frontend/jest.config.ts index b7cac3cc02..c9a67b3d26 100644 --- a/frontend/jest.config.ts +++ b/frontend/jest.config.ts @@ -22,7 +22,7 @@ const config: Config.InitialOptions = { '^.+\\.(js|jsx)$': 'babel-jest', }, transformIgnorePatterns: [ - 'node_modules/(?!(lodash-es|react-dnd|core-dnd|@react-dnd|dnd-core|react-dnd-html5-backend)/)', + 'node_modules/(?!(lodash-es|react-dnd|core-dnd|@react-dnd|dnd-core|react-dnd-html5-backend|axios)/)', ], setupFilesAfterEnv: ['jest.setup.ts'], testPathIgnorePatterns: ['/node_modules/', '/public/'], diff --git a/frontend/package.json b/frontend/package.json index 25036d24fe..64ac911fc4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -38,7 +38,7 @@ "ansi-to-html": "0.7.2", "antd": "5.11.0", "antd-table-saveas-excel": "2.2.1", - "axios": "^0.21.0", + "axios": "1.6.2", "babel-eslint": "^10.1.0", "babel-jest": "^29.6.4", "babel-loader": "9.1.3", @@ -87,7 +87,7 @@ "react-helmet-async": "1.3.0", "react-i18next": "^11.16.1", "react-markdown": "8.0.7", - "react-query": "^3.34.19", + "react-query": "3.39.3", "react-redux": "^7.2.2", "react-router-dom": "^5.2.0", "react-syntax-highlighter": "15.5.0", diff --git a/frontend/src/api/ErrorResponseHandler.ts b/frontend/src/api/ErrorResponseHandler.ts index 2c42f0951b..3f28ff418d 100644 --- a/frontend/src/api/ErrorResponseHandler.ts +++ b/frontend/src/api/ErrorResponseHandler.ts @@ -1,4 +1,4 @@ -import { AxiosError } from 'axios'; +import { AxiosError, AxiosResponse } from 'axios'; import { ErrorResponse } from 'types/api'; import { ErrorStatusCode } from 'types/common'; @@ -10,7 +10,7 @@ export function ErrorResponseHandler(error: AxiosError): ErrorResponse { const statusCode = response.status as ErrorStatusCode; if (statusCode >= 400 && statusCode < 500) { - const { data } = response; + const { data } = response as AxiosResponse; if (statusCode === 404) { return { diff --git a/frontend/src/api/dashboard/get.ts b/frontend/src/api/dashboard/get.ts index 9b10f6467d..01e04c6c0f 100644 --- a/frontend/src/api/dashboard/get.ts +++ b/frontend/src/api/dashboard/get.ts @@ -3,9 +3,9 @@ import { ApiResponse } from 'types/api'; import { Props } from 'types/api/dashboard/get'; import { Dashboard } from 'types/api/dashboard/getAll'; -const get = (props: Props): Promise => +const getDashboard = (props: Props): Promise => axios .get>(`/dashboards/${props.uuid}`) .then((res) => res.data.data); -export default get; +export default getDashboard; diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index 9739f24e49..bde915f201 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -4,7 +4,7 @@ import getLocalStorageApi from 'api/browser/localstorage/get'; import loginApi from 'api/user/login'; import afterLogin from 'AppRoutes/utils'; -import axios, { AxiosRequestConfig, AxiosResponse } from 'axios'; +import axios, { AxiosResponse, InternalAxiosRequestConfig } from 'axios'; import { ENVIRONMENT } from 'constants/env'; import { LOCALSTORAGE } from 'constants/localStorage'; import store from 'store'; @@ -17,14 +17,16 @@ const interceptorsResponse = ( ): Promise> => Promise.resolve(value); const interceptorsRequestResponse = ( - value: AxiosRequestConfig, -): AxiosRequestConfig => { + value: InternalAxiosRequestConfig, +): InternalAxiosRequestConfig => { const token = store.getState().app.user?.accessJwt || getLocalStorageApi(LOCALSTORAGE.AUTH_TOKEN) || ''; - value.headers.Authorization = token ? `Bearer ${token}` : ''; + if (value && value.headers) { + value.headers.Authorization = token ? `Bearer ${token}` : ''; + } return value; }; @@ -92,8 +94,8 @@ const instance = axios.create({ baseURL: `${ENVIRONMENT.baseURL}${apiV1}`, }); -instance.interceptors.response.use(interceptorsResponse, interceptorRejected); instance.interceptors.request.use(interceptorsRequestResponse); +instance.interceptors.response.use(interceptorsResponse, interceptorRejected); export const AxiosAlertManagerInstance = axios.create({ baseURL: `${ENVIRONMENT.baseURL}${apiAlertManager}`, diff --git a/frontend/src/api/metrics/getQueryRange.ts b/frontend/src/api/metrics/getQueryRange.ts index bee657d904..984d381e10 100644 --- a/frontend/src/api/metrics/getQueryRange.ts +++ b/frontend/src/api/metrics/getQueryRange.ts @@ -9,9 +9,10 @@ import { export const getMetricsQueryRange = async ( props: QueryRangePayload, + signal: AbortSignal, ): Promise | ErrorResponse> => { try { - const response = await axios.post('/query_range', props); + const response = await axios.post('/query_range', props, { signal }); return { statusCode: 200, diff --git a/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts b/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts index 7e739fdf52..cd8748788d 100644 --- a/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts +++ b/frontend/src/container/PipelinePage/PipelineListsView/Preview/hooks/usePipelinePreview.ts @@ -53,7 +53,7 @@ const usePipelinePreview = ({ isLoading: isFetching, outputLogs, isError, - errorMsg: error?.response?.data?.error || '', + errorMsg: error?.message || '', }; }; diff --git a/frontend/src/hooks/queryBuilder/useGetQueryRange.ts b/frontend/src/hooks/queryBuilder/useGetQueryRange.ts index 9aa76405c2..c54a07461d 100644 --- a/frontend/src/hooks/queryBuilder/useGetQueryRange.ts +++ b/frontend/src/hooks/queryBuilder/useGetQueryRange.ts @@ -15,14 +15,19 @@ type UseGetQueryRange = ( export const useGetQueryRange: UseGetQueryRange = (requestData, options) => { const queryKey = useMemo(() => { - if (options?.queryKey) { + if (options?.queryKey && Array.isArray(options.queryKey)) { return [...options.queryKey]; } + + if (options?.queryKey && typeof options.queryKey === 'string') { + return options.queryKey; + } + return [REACT_QUERY_KEY.GET_QUERY_RANGE, requestData]; }, [options?.queryKey, requestData]); return useQuery, Error>({ - queryFn: async () => GetMetricQueryRange(requestData), + queryFn: async ({ signal }) => GetMetricQueryRange(requestData, signal), ...options, queryKey, }); diff --git a/frontend/src/lib/dashboard/getQueryResults.ts b/frontend/src/lib/dashboard/getQueryResults.ts index 47f21bdd25..89ba08f891 100644 --- a/frontend/src/lib/dashboard/getQueryResults.ts +++ b/frontend/src/lib/dashboard/getQueryResults.ts @@ -17,10 +17,11 @@ import { prepareQueryRangePayload } from './prepareQueryRangePayload'; export async function GetMetricQueryRange( props: GetQueryResultsProps, + signal?: AbortSignal, ): Promise> { const { legendMap, queryPayload } = prepareQueryRangePayload(props); - const response = await getMetricsQueryRange(queryPayload); + const response = await getMetricsQueryRange(queryPayload, signal); if (response.statusCode >= 400) { throw new Error( diff --git a/frontend/src/pages/NewDashboard/DashboardPage.tsx b/frontend/src/pages/NewDashboard/DashboardPage.tsx index d880413628..7da194aad0 100644 --- a/frontend/src/pages/NewDashboard/DashboardPage.tsx +++ b/frontend/src/pages/NewDashboard/DashboardPage.tsx @@ -12,7 +12,9 @@ function DashboardPage(): JSX.Element { const { isFetching, isError, isLoading } = dashboardResponse; const errorMessage = isError - ? (dashboardResponse?.error as AxiosError)?.response?.data.errorType + ? // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + (dashboardResponse?.error as AxiosError)?.response?.data?.errorType : 'Something went wrong'; if (isError && !isFetching && errorMessage === ErrorType.NotFound) { diff --git a/frontend/src/providers/Dashboard/Dashboard.tsx b/frontend/src/providers/Dashboard/Dashboard.tsx index 4365b6c35b..3df954e1b7 100644 --- a/frontend/src/providers/Dashboard/Dashboard.tsx +++ b/frontend/src/providers/Dashboard/Dashboard.tsx @@ -1,5 +1,5 @@ import Modal from 'antd/es/modal'; -import get from 'api/dashboard/get'; +import getDashboard from 'api/dashboard/get'; import lockDashboardApi from 'api/dashboard/lockDashboard'; import unlockDashboardApi from 'api/dashboard/unlockDashboard'; import { REACT_QUERY_KEY } from 'constants/reactQueryKeys'; @@ -107,7 +107,7 @@ export function DashboardProvider({ { enabled: (!!isDashboardPage || !!isDashboardWidgetPage) && isLoggedIn, queryFn: () => - get({ + getDashboard({ uuid: dashboardId, }), refetchOnWindowFocus: false, diff --git a/frontend/yarn.lock b/frontend/yarn.lock index bc6da58703..404b83ba28 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -4639,7 +4639,16 @@ axe-core@^4.6.2: resolved "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz" integrity sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ== -axios@^0.21.0, axios@^0.21.1: +axios@1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.2.tgz#de67d42c755b571d3e698df1b6504cde9b0ee9f2" + integrity sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: version "0.21.4" resolved "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz" integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== @@ -7710,6 +7719,11 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0: resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz" integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== +follow-redirects@^1.15.0: + version "1.15.3" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a" + integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + fontfaceobserver@2.3.0: version "2.3.0" resolved "https://registry.npmjs.org/fontfaceobserver/-/fontfaceobserver-2.3.0.tgz" @@ -7759,6 +7773,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + format@^0.2.0: version "0.2.2" resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" @@ -12294,6 +12317,11 @@ proxy-addr@~2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + prr@~1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz" @@ -12962,9 +12990,9 @@ react-markdown@8.0.7, react-markdown@~8.0.0: unist-util-visit "^4.0.0" vfile "^5.0.0" -react-query@^3.34.19: +react-query@3.39.3: version "3.39.3" - resolved "https://registry.npmjs.org/react-query/-/react-query-3.39.3.tgz" + resolved "https://registry.yarnpkg.com/react-query/-/react-query-3.39.3.tgz#4cea7127c6c26bdea2de5fb63e51044330b03f35" integrity sha512-nLfLz7GiohKTJDuT4us4X3h/8unOh+00MLb2yJoGTPjxKs2bc1iDhkNx2bd5MKklXnOD3NrVZ+J2UXujA5In4g== dependencies: "@babel/runtime" "^7.5.5" From 221861230a2de2f50fd2e384c7ec5e73d149b944 Mon Sep 17 00:00:00 2001 From: Yunus M Date: Wed, 13 Dec 2023 01:18:19 +0530 Subject: [PATCH 0019/1178] feat: track channel click event in support page (#4217) --- frontend/src/pages/Support/Support.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/pages/Support/Support.tsx b/frontend/src/pages/Support/Support.tsx index 61d6c11c38..96276507f9 100644 --- a/frontend/src/pages/Support/Support.tsx +++ b/frontend/src/pages/Support/Support.tsx @@ -1,6 +1,7 @@ import './Support.styles.scss'; import { Button, Card, Typography } from 'antd'; +import useAnalytics from 'hooks/analytics/useAnalytics'; import { Book, Cable, @@ -82,6 +83,8 @@ const supportChannels = [ ]; export default function Support(): JSX.Element { + const { trackEvent } = useAnalytics(); + const handleChannelWithRedirects = (url: string): void => { window.open(url, '_blank'); }; @@ -111,6 +114,8 @@ export default function Support(): JSX.Element { }; const handleChannelClick = (channel: Channel): void => { + trackEvent(`Support : ${channel.name}`); + switch (channel.key) { case channelsMap.documentation: case channelsMap.github: From 55664872bd9d9e2b3fbcce747c55ca7dffa4a717 Mon Sep 17 00:00:00 2001 From: Rajat Dabade Date: Wed, 13 Dec 2023 16:26:25 +0530 Subject: [PATCH 0020/1178] [Feat]: only clicked legend graph visible (#4226) * refactor: only clicked legend graph visible * refactor: fix graph manage toggle issue --- .../src/lib/uPlotLib/getUplotChartOptions.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/uPlotLib/getUplotChartOptions.ts b/frontend/src/lib/uPlotLib/getUplotChartOptions.ts index 3a5ec4b2c6..354deb0b5a 100644 --- a/frontend/src/lib/uPlotLib/getUplotChartOptions.ts +++ b/frontend/src/lib/uPlotLib/getUplotChartOptions.ts @@ -55,6 +55,7 @@ export const getUPlotChartOptions = ({ legend: { show: true, live: false, + isolate: true, }, focus: { alpha: 0.3, @@ -158,16 +159,24 @@ export const getUPlotChartOptions = ({ (self): void => { const legend = self.root.querySelector('.u-legend'); if (legend) { - const seriesEls = legend.querySelectorAll('.u-label'); + const seriesEls = legend.querySelectorAll('.u-series'); const seriesArray = Array.from(seriesEls); seriesArray.forEach((seriesEl, index) => { seriesEl.addEventListener('click', () => { if (graphsVisibilityStates) { setGraphsVisibilityStates?.((prev) => { const newGraphVisibilityStates = [...prev]; - newGraphVisibilityStates[index + 1] = !newGraphVisibilityStates[ - index + 1 - ]; + if ( + newGraphVisibilityStates[index + 1] && + newGraphVisibilityStates.every((value, i) => + i === index + 1 ? value : !value, + ) + ) { + newGraphVisibilityStates.fill(true); + } else { + newGraphVisibilityStates.fill(false); + newGraphVisibilityStates[index + 1] = true; + } return newGraphVisibilityStates; }); } From 29b134455747099fa63292ef1fa0b7f05e13e231 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 13 Dec 2023 16:40:17 +0530 Subject: [PATCH 0021/1178] chore: add prepare query for cumulative/unspecified timeseries (#4166) --- .../app/metrics/v4/cumulative/helper.go | 57 +++++ .../app/metrics/v4/cumulative/timeseries.go | 220 +++++++++++++++++ .../metrics/v4/cumulative/timeseries_test.go | 229 ++++++++++++++++++ 3 files changed, 506 insertions(+) create mode 100644 pkg/query-service/app/metrics/v4/cumulative/helper.go create mode 100644 pkg/query-service/app/metrics/v4/cumulative/timeseries.go create mode 100644 pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go diff --git a/pkg/query-service/app/metrics/v4/cumulative/helper.go b/pkg/query-service/app/metrics/v4/cumulative/helper.go new file mode 100644 index 0000000000..5914ee495d --- /dev/null +++ b/pkg/query-service/app/metrics/v4/cumulative/helper.go @@ -0,0 +1,57 @@ +package cumulative + +import ( + "fmt" + "strings" + + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +// groupingSets returns a string of comma separated tags for group by clause +// `ts` is always added to the group by clause +func groupingSets(tags ...string) string { + withTs := append(tags, "ts") + return fmt.Sprintf(`GROUPING SETS ( (%s), (%s) )`, strings.Join(withTs, ", "), strings.Join(tags, ", ")) +} + +// groupingSetsByAttributeKeyTags returns a string of comma separated tags for group by clause +func groupingSetsByAttributeKeyTags(tags ...v3.AttributeKey) string { + groupTags := []string{} + for _, tag := range tags { + groupTags = append(groupTags, tag.Key) + } + return groupingSets(groupTags...) +} + +// groupBy returns a string of comma separated tags for group by clause +func groupByAttributeKeyTags(tags ...v3.AttributeKey) string { + groupTags := []string{} + for _, tag := range tags { + groupTags = append(groupTags, tag.Key) + } + groupTags = append(groupTags, "ts") + return strings.Join(groupTags, ", ") +} + +// orderBy returns a string of comma separated tags for order by clause +// if the order is not specified, it defaults to ASC +func orderByAttributeKeyTags(items []v3.OrderBy, tags []v3.AttributeKey) string { + var orderBy []string + for _, tag := range tags { + found := false + for _, item := range items { + if item.ColumnName == tag.Key { + found = true + orderBy = append(orderBy, fmt.Sprintf("%s %s", item.ColumnName, item.Order)) + break + } + } + if !found { + orderBy = append(orderBy, fmt.Sprintf("%s ASC", tag.Key)) + } + } + + orderBy = append(orderBy, "ts ASC") + + return strings.Join(orderBy, ", ") +} diff --git a/pkg/query-service/app/metrics/v4/cumulative/timeseries.go b/pkg/query-service/app/metrics/v4/cumulative/timeseries.go new file mode 100644 index 0000000000..78d22be4aa --- /dev/null +++ b/pkg/query-service/app/metrics/v4/cumulative/timeseries.go @@ -0,0 +1,220 @@ +package cumulative + +import ( + "fmt" + + v4 "go.signoz.io/signoz/pkg/query-service/app/metrics/v4" + "go.signoz.io/signoz/pkg/query-service/constants" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/utils" +) + +// See https://clickhouse.com/docs/en/sql-reference/window-functions for more details on `lagInFrame` function +// +// Calculating the rate of change of a metric is a common use case. +// Requests and errors are two examples of metrics that are often expressed as a rate of change. +// The rate of change is the difference between the current value and the previous value divided by +// the time difference between the current and previous values (i.e. the time interval). +// +// The value of a cumulative counter always increases. However, the rate of change can be negative +// if the value decreases between two samples. This can happen if the counter is reset when the +// application restarts or if the counter is reset manually. In this case, the rate of change is +// not meaningful and should be ignored. +// +// The condition `(per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0` +// checks if the rate of change is negative. If it is negative, the value is replaced with `nan`. +// +// The condition `ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400` checks +// if the time difference between the current and previous values is greater than or equal to 1 day. +// The first sample of a metric is always `nan` because there is no previous value to compare it to. +// When the first sample is encountered, the previous value for the time is set to default i.e `1970-01-01`. +// Since any difference between the first sample timestamp and the previous value timestamp will be +// greater than or equal to 1 day, the rate of change for the first sample will be `nan`. +// +// If neither of the above conditions are true, the rate of change is calculated as +// `(per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window)` +// where `rate_window` is a window function that partitions the data by fingerprint and orders it by timestamp. +// We want to calculate the rate of change for each time series, so we partition the data by fingerprint. +// +// The `increase` function is similar to the `rate` function, except that it does not divide by the time interval. +const ( + rateWithoutNegative = `If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window)))` + increaseWithoutNegative = `If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window)))` +) + +// prepareTimeAggregationSubQueryTimeSeries prepares the sub-query to be used for temporal aggregation +// of time series data + +// The following example illustrates how the sub-query is used to calculate the sume of values for each +// time series in a 15 seconds interval: + +// ``` +// timestamp 01.00 01.05 01.10 01.15 01.20 01.25 01.30 01.35 01.40 +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | | +// | v1 | v2 | v3 | v4 | v5 | v6 | v7 | v8 | v9 | +// | | | | | | | | | | +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | +// | | | | | | | | | +// | | | +// +------+ +------+ +------+ +// | v1+ | | v4+ | | v7+ | +// | v2+ | | v5+ | | v8+ | +// | v3 | | v6 | | v9 | +// +------+ +------+ +------+ +// 01.00 01.15 01.30 +// ``` + +// Calculating the rate/increase involves an additional step. We first calculate the maximum value for each time series +// in a 15 seconds interval. Then, we calculate the difference between the current maximum value and the previous +// maximum value + +// The following example illustrates how the sub-query is used to calculate the rate of change for each time series +// in a 15 seconds interval: + +// ``` +// timestamp 01.00 01.05 01.10 01.15 01.20 01.25 01.30 01.35 01.40 +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | | +// | v1 | v2 | v3 | v4 | v5 | v6 | v7 | v8 | v9 | +// | | | | | | | | | | +// +------+------+------+------+------+------+------+------+------+ +// | | | | | | | | | +// | | | | | | | | | +// | | | +// +------+ +------+ +------+ +// max(| v1, | max(| v4, | max(| v7, | +// | v2, | | v5, | | v8, | +// | v3 |) | v6 |) | v9 |) +// +------+ +------+ +------+ +// 01.00 01.15 01.30 + +// +-------+ +--------+ +// | V6-V2 | | V9-V6 | +// | | | | +// | | | | +// +------+ +--------+ +// 01.00 01.15 +// ``` + +// The rate of change is calculated as (Vy - Vx) / (Ty - Tx) where Vx and Vy are the values at time Tx and Ty respectively. +// In an ideal scenario, the last value of each interval could be used to calculate the rate of change. Instead, we use +// the maximum value of each interval to calculate the rate of change. This is because any process restart can cause the +// value to be reset to 0. This will produce an inaccurate result. The max is the best approximation we can get. +// We don't expect the process to restart very often, so this should be a good approximation. + +func prepareTimeAggregationSubQueryTimeSeries(start, end, step int64, mq *v3.BuilderQuery) (string, error) { + var subQuery string + + timeSeriesSubQuery, err := v4.PrepareTimeseriesFilterQuery(mq) + if err != nil { + return "", err + } + + samplesTableFilter := fmt.Sprintf("metric_name = %s AND timestamp_ms >= %d AND timestamp_ms <= %d", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key), start, end) + + // Select the aggregate value for interval + queryTmpl := + "SELECT fingerprint, %s" + + " toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL %d SECOND) as ts," + + " %s as per_series_value" + + " FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + constants.SIGNOZ_SAMPLES_TABLENAME + + " INNER JOIN" + + " (%s) as filtered_time_series" + + " USING fingerprint" + + " WHERE " + samplesTableFilter + + " GROUP BY fingerprint, ts" + + " ORDER BY fingerprint, ts" + + var selectLabelsAny string + for _, tag := range mq.GroupBy { + selectLabelsAny += fmt.Sprintf("any(%s) as %s,", tag.Key, tag.Key) + } + + var selectLabels string + for _, tag := range mq.GroupBy { + selectLabels += tag.Key + "," + } + + switch mq.TimeAggregation { + case v3.TimeAggregationAvg: + op := "avg(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationSum: + op := "sum(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationMin: + op := "min(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationMax: + op := "max(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationCount: + op := "count(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationCountDistinct: + op := "count(distinct(value))" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationAnyLast: + op := "anyLast(value)" + subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + case v3.TimeAggregationRate: + op := "max(value)" + innerSubQuery := fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + rateQueryTmpl := + "SELECT %s ts, " + rateWithoutNegative + + " as per_series_value FROM (%s) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)" + subQuery = fmt.Sprintf(rateQueryTmpl, selectLabels, innerSubQuery) + case v3.TimeAggregationIncrease: + op := "max(value)" + innerSubQuery := fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery) + rateQueryTmpl := + "SELECT %s ts, " + increaseWithoutNegative + + " as per_series_value FROM (%s) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)" + subQuery = fmt.Sprintf(rateQueryTmpl, selectLabels, innerSubQuery) + } + return subQuery, nil +} + +// prepareMetricQueryCumulativeTimeSeries prepares the query to be used for fetching metrics +func prepareMetricQueryCumulativeTimeSeries(start, end, step int64, mq *v3.BuilderQuery) (string, error) { + var query string + + temporalAggSubQuery, err := prepareTimeAggregationSubQueryTimeSeries(start, end, step, mq) + if err != nil { + return "", err + } + + groupBy := groupingSetsByAttributeKeyTags(mq.GroupBy...) + orderBy := orderByAttributeKeyTags(mq.OrderBy, mq.GroupBy) + selectLabels := groupByAttributeKeyTags(mq.GroupBy...) + + queryTmpl := + "SELECT %s," + + " %s as value" + + " FROM (%s)" + + " WHERE isNaN(per_series_value) = 0" + + " GROUP BY %s" + + " ORDER BY %s" + + switch mq.SpaceAggregation { + case v3.SpaceAggregationAvg: + op := "avg(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationSum: + op := "sum(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationMin: + op := "min(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationMax: + op := "max(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + case v3.SpaceAggregationCount: + op := "count(per_series_value)" + query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy) + } + + return query, nil +} diff --git a/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go b/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go new file mode 100644 index 0000000000..70f2e1d3ef --- /dev/null +++ b/pkg/query-service/app/metrics/v4/cumulative/timeseries_test.go @@ -0,0 +1,229 @@ +package cumulative + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v3 "go.signoz.io/signoz/pkg/query-service/model/v3" +) + +func TestPrepareTimeAggregationSubQuery(t *testing.T) { + // The time aggregation is performed for each unique series - since the fingerprint represents the + // unique hash of label set, we always group by fingerprint regardless of the GroupBy + // This sub result is then aggregated on dimensions using the provided GroupBy clause keys + testCases := []struct { + name string + builderQuery *v3.BuilderQuery + start int64 + end int64 + expectedQueryContains string + }{ + { + name: "test time aggregation = avg, temporality = cumulative", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorNotEqual, + Value: "payment_service", + }, + { + Key: v3.AttributeKey{ + Key: "endpoint", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorIn, + Value: []interface{}{"/paycallback", "/payme", "/paypal"}, + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationAvg, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND JSONExtractString(labels, 'service_name') != 'payment_service' AND JSONExtractString(labels, 'endpoint') IN ['/paycallback','/payme','/paypal']) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts", + }, + { + name: "test time aggregation = rate, temporality = cumulative", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorContains, + Value: "payment_service", + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationRate, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + query, err := prepareTimeAggregationSubQueryTimeSeries( + testCase.start, + testCase.end, + testCase.builderQuery.StepInterval, + testCase.builderQuery, + ) + assert.Nil(t, err) + assert.Contains(t, query, testCase.expectedQueryContains) + }) + } +} +func TestPrepareTimeseriesQuery(t *testing.T) { + testCases := []struct { + name string + builderQuery *v3.BuilderQuery + start int64 + end int64 + expectedQueryContains string + }{ + { + name: "test time aggregation = avg, space aggregation = sum, temporality = unspecified", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "system_memory_usage", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Unspecified, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "state", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorNotEqual, + Value: "idle", + }, + }, + }, + GroupBy: []v3.AttributeKey{}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationAvg, + SpaceAggregation: v3.SpaceAggregationSum, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT ts, sum(per_series_value) as value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, avg(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'system_memory_usage' AND temporality = 'Unspecified' AND JSONExtractString(labels, 'state') != 'idle') as filtered_time_series USING fingerprint WHERE metric_name = 'system_memory_usage' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WHERE isNaN(per_series_value) = 0 GROUP BY GROUPING SETS ( (ts), () ) ORDER BY ts ASC", + }, + { + name: "test time aggregation = rate, space aggregation = sum, temporality = cumulative", + builderQuery: &v3.BuilderQuery{ + QueryName: "A", + StepInterval: 60, + DataSource: v3.DataSourceMetrics, + AggregateAttribute: v3.AttributeKey{ + Key: "http_requests", + DataType: v3.AttributeKeyDataTypeFloat64, + Type: v3.AttributeKeyTypeUnspecified, + IsColumn: true, + IsJSON: false, + }, + Temporality: v3.Cumulative, + Filters: &v3.FilterSet{ + Operator: "AND", + Items: []v3.FilterItem{ + { + Key: v3.AttributeKey{ + Key: "service_name", + Type: v3.AttributeKeyTypeTag, + DataType: v3.AttributeKeyDataTypeString, + }, + Operator: v3.FilterOperatorContains, + Value: "payment_service", + }, + }, + }, + GroupBy: []v3.AttributeKey{{ + Key: "service_name", + DataType: v3.AttributeKeyDataTypeString, + Type: v3.AttributeKeyTypeTag, + }}, + Expression: "A", + Disabled: false, + TimeAggregation: v3.TimeAggregationRate, + SpaceAggregation: v3.SpaceAggregationSum, + }, + start: 1701794980000, + end: 1701796780000, + expectedQueryContains: "SELECT service_name, ts, sum(per_series_value) as value FROM (SELECT service_name, ts, If((per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) < 0, nan, If((ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window) >= 86400, nan, (per_series_value - lagInFrame(per_series_value, 1, 0) OVER rate_window) / (ts - lagInFrame(ts, 1, toDate('1970-01-01')) OVER rate_window))) as per_series_value FROM (SELECT fingerprint, any(service_name) as service_name, toStartOfInterval(toDateTime(intDiv(timestamp_ms, 1000)), INTERVAL 60 SECOND) as ts, max(value) as per_series_value FROM signoz_metrics.distributed_samples_v2 INNER JOIN (SELECT DISTINCT JSONExtractString(labels, 'service_name') as service_name, fingerprint FROM signoz_metrics.time_series_v2 WHERE metric_name = 'http_requests' AND temporality = 'Cumulative' AND like(JSONExtractString(labels, 'service_name'), '%payment_service%')) as filtered_time_series USING fingerprint WHERE metric_name = 'http_requests' AND timestamp_ms >= 1701794980000 AND timestamp_ms <= 1701796780000 GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window as (PARTITION BY fingerprint ORDER BY fingerprint, ts)) WHERE isNaN(per_series_value) = 0 GROUP BY GROUPING SETS ( (service_name, ts), (service_name) ) ORDER BY service_name ASC, ts ASC", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + query, err := prepareMetricQueryCumulativeTimeSeries( + testCase.start, + testCase.end, + testCase.builderQuery.StepInterval, + testCase.builderQuery, + ) + assert.Nil(t, err) + assert.Contains(t, query, testCase.expectedQueryContains) + }) + } +} From f56b5cb9712f22e092faad1722760f6460e7aab3 Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Wed, 13 Dec 2023 17:05:59 +0530 Subject: [PATCH 0022/1178] fix: createPAT method to return id (#4078) Update token expiry validations --- ee/query-service/app/api/pat.go | 13 ++++++++++++- ee/query-service/dao/interface.go | 2 +- ee/query-service/dao/sqlite/pat.go | 17 ++++++++++++----- ee/query-service/model/pat.go | 2 +- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/ee/query-service/app/api/pat.go b/ee/query-service/app/api/pat.go index 619c875c8f..b0fcf073a4 100644 --- a/ee/query-service/app/api/pat.go +++ b/ee/query-service/app/api/pat.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/mux" "go.signoz.io/signoz/ee/query-service/model" "go.signoz.io/signoz/pkg/query-service/auth" + basemodel "go.signoz.io/signoz/pkg/query-service/model" "go.uber.org/zap" ) @@ -47,8 +48,18 @@ func (ah *APIHandler) createPAT(w http.ResponseWriter, r *http.Request) { req.CreatedAt = time.Now().Unix() req.Token = generatePATToken() + // default expiry is 30 days + if req.ExpiresAt == 0 { + req.ExpiresAt = time.Now().AddDate(0, 0, 30).Unix() + } + // max expiry is 1 year + if req.ExpiresAt > time.Now().AddDate(1, 0, 0).Unix() { + req.ExpiresAt = time.Now().AddDate(1, 0, 0).Unix() + } + zap.S().Debugf("Got PAT request: %+v", req) - if apierr := ah.AppDao().CreatePAT(ctx, &req); apierr != nil { + var apierr basemodel.BaseApiError + if req, apierr = ah.AppDao().CreatePAT(ctx, req); apierr != nil { RespondError(w, apierr, nil) return } diff --git a/ee/query-service/dao/interface.go b/ee/query-service/dao/interface.go index 1a8f3b2460..479ca56edc 100644 --- a/ee/query-service/dao/interface.go +++ b/ee/query-service/dao/interface.go @@ -33,7 +33,7 @@ type ModelDao interface { DeleteDomain(ctx context.Context, id uuid.UUID) basemodel.BaseApiError GetDomainByEmail(ctx context.Context, email string) (*model.OrgDomain, basemodel.BaseApiError) - CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseApiError + CreatePAT(ctx context.Context, p model.PAT) (model.PAT, basemodel.BaseApiError) GetPAT(ctx context.Context, pat string) (*model.PAT, basemodel.BaseApiError) GetPATByID(ctx context.Context, id string) (*model.PAT, basemodel.BaseApiError) GetUserByPAT(ctx context.Context, token string) (*basemodel.UserPayload, basemodel.BaseApiError) diff --git a/ee/query-service/dao/sqlite/pat.go b/ee/query-service/dao/sqlite/pat.go index cc4de546c5..5bd1b78a62 100644 --- a/ee/query-service/dao/sqlite/pat.go +++ b/ee/query-service/dao/sqlite/pat.go @@ -3,14 +3,15 @@ package sqlite import ( "context" "fmt" + "strconv" "go.signoz.io/signoz/ee/query-service/model" basemodel "go.signoz.io/signoz/pkg/query-service/model" "go.uber.org/zap" ) -func (m *modelDao) CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseApiError { - _, err := m.DB().ExecContext(ctx, +func (m *modelDao) CreatePAT(ctx context.Context, p model.PAT) (model.PAT, basemodel.BaseApiError) { + result, err := m.DB().ExecContext(ctx, "INSERT INTO personal_access_tokens (user_id, token, name, created_at, expires_at) VALUES ($1, $2, $3, $4, $5)", p.UserID, p.Token, @@ -19,9 +20,15 @@ func (m *modelDao) CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseAp p.ExpiresAt) if err != nil { zap.S().Errorf("Failed to insert PAT in db, err: %v", zap.Error(err)) - return model.InternalError(fmt.Errorf("PAT insertion failed")) + return model.PAT{}, model.InternalError(fmt.Errorf("PAT insertion failed")) } - return nil + id, err := result.LastInsertId() + if err != nil { + zap.S().Errorf("Failed to get last inserted id, err: %v", zap.Error(err)) + return model.PAT{}, model.InternalError(fmt.Errorf("PAT insertion failed")) + } + p.Id = strconv.Itoa(int(id)) + return p, nil } func (m *modelDao) ListPATs(ctx context.Context, userID string) ([]model.PAT, basemodel.BaseApiError) { @@ -90,7 +97,7 @@ func (m *modelDao) GetUserByPAT(ctx context.Context, token string) (*basemodel.U u.org_id, u.group_id FROM users u, personal_access_tokens p - WHERE u.id = p.user_id and p.token=?;` + WHERE u.id = p.user_id and p.token=? and p.expires_at >= strftime('%s', 'now');` if err := m.DB().Select(&users, query, token); err != nil { return nil, model.InternalError(fmt.Errorf("failed to fetch user from PAT, err: %v", err)) diff --git a/ee/query-service/model/pat.go b/ee/query-service/model/pat.go index c22282060b..f320d0be7c 100644 --- a/ee/query-service/model/pat.go +++ b/ee/query-service/model/pat.go @@ -6,5 +6,5 @@ type PAT struct { Token string `json:"token" db:"token"` Name string `json:"name" db:"name"` CreatedAt int64 `json:"createdAt" db:"created_at"` - ExpiresAt int64 `json:"expiresAt" db:"expires_at"` // unused as of now + ExpiresAt int64 `json:"expiresAt" db:"expires_at"` } From c7b59d44057bfee34316468e4511d1974bb72305 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 13 Dec 2023 17:53:18 +0530 Subject: [PATCH 0023/1178] chore: update .github/CODEOWNERS (#3539) * chore: update .github/CODEOWNERS * chore: remove team --- .github/CODEOWNERS | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index db5cd61a9b..b6d934f88d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,8 +8,4 @@ /frontend/src/container/NewWidget/RightContainer/types.ts @srikanthccv /deploy/ @prashant-shahi /sample-apps/ @prashant-shahi -**/query-service/ @srikanthccv -Makefile @srikanthccv -go.* @srikanthccv -.git* @srikanthccv .github @prashant-shahi From c66c8c28237ef164728d333e75ae4c075a633d7d Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Wed, 13 Dec 2023 18:14:55 +0530 Subject: [PATCH 0024/1178] chore: add new dashboard/alerts info events (#4214) * chore: add new dashboard/alerts info events --- ee/query-service/app/server.go | 2 +- .../app/clickhouseReader/reader.go | 96 +++++++++++++++++++ pkg/query-service/app/server.go | 2 +- pkg/query-service/interfaces/interface.go | 2 + pkg/query-service/model/response.go | 14 +++ pkg/query-service/rules/thresholdRule_test.go | 2 +- pkg/query-service/telemetry/ignored.go | 13 +-- pkg/query-service/telemetry/telemetry.go | 26 ++++- 8 files changed, 142 insertions(+), 15 deletions(-) diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index bed3855f17..699894e691 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -480,7 +480,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { } } - if _, ok := telemetry.IgnoredPaths()[path]; !ok { + if _, ok := telemetry.EnabledPaths()[path]; ok { userEmail, err := auth.GetEmailFromJwt(r.Context()) if err == nil { telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data, userEmail) diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index 8b42c9265a..c8f150cd85 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -43,6 +43,7 @@ import ( promModel "github.com/prometheus/common/model" "go.uber.org/zap" + "go.signoz.io/signoz/pkg/query-service/app/dashboards" "go.signoz.io/signoz/pkg/query-service/app/logs" "go.signoz.io/signoz/pkg/query-service/app/services" "go.signoz.io/signoz/pkg/query-service/auth" @@ -51,6 +52,7 @@ import ( "go.signoz.io/signoz/pkg/query-service/interfaces" "go.signoz.io/signoz/pkg/query-service/model" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" + "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/telemetry" "go.signoz.io/signoz/pkg/query-service/utils" ) @@ -3421,6 +3423,100 @@ func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Contex return &tagsInfo, nil } +// GetDashboardsInfo returns analytics data for dashboards +func (r *ClickHouseReader) GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) { + dashboardsInfo := model.DashboardsInfo{} + // fetch dashboards from dashboard db + query := "SELECT data FROM dashboards" + var dashboardsData []dashboards.Dashboard + err := r.localDB.Select(&dashboardsData, query) + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return &dashboardsInfo, err + } + for _, dashboard := range dashboardsData { + dashboardsInfo = countPanelsInDashboard(dashboard.Data) + } + dashboardsInfo.TotalDashboards = len(dashboardsData) + + return &dashboardsInfo, nil +} + +func countPanelsInDashboard(data map[string]interface{}) model.DashboardsInfo { + var logsPanelCount, tracesPanelCount, metricsPanelCount int + // totalPanels := 0 + if data != nil && data["widgets"] != nil { + widgets, ok := data["widgets"].(interface{}) + if ok { + data, ok := widgets.([]interface{}) + if ok { + for _, widget := range data { + sData, ok := widget.(map[string]interface{}) + if ok && sData["query"] != nil { + // totalPanels++ + query, ok := sData["query"].(interface{}).(map[string]interface{}) + if ok && query["queryType"] == "builder" && query["builder"] != nil { + builderData, ok := query["builder"].(interface{}).(map[string]interface{}) + if ok && builderData["queryData"] != nil { + builderQueryData, ok := builderData["queryData"].([]interface{}) + if ok { + for _, queryData := range builderQueryData { + data, ok := queryData.(map[string]interface{}) + if ok { + if data["dataSource"] == "traces" { + tracesPanelCount++ + } else if data["dataSource"] == "metrics" { + metricsPanelCount++ + } else if data["dataSource"] == "logs" { + logsPanelCount++ + } + } + } + } + } + } + } + } + } + } + } + return model.DashboardsInfo{ + LogsBasedPanels: logsPanelCount, + TracesBasedPanels: tracesPanelCount, + MetricBasedPanels: metricsPanelCount, + } +} + +func (r *ClickHouseReader) GetAlertsInfo(ctx context.Context) (*model.AlertsInfo, error) { + alertsInfo := model.AlertsInfo{} + // fetch alerts from rules db + query := "SELECT data FROM rules" + var alertsData []string + err := r.localDB.Select(&alertsData, query) + if err != nil { + zap.S().Debug("Error in processing sql query: ", err) + return &alertsInfo, err + } + for _, alert := range alertsData { + var rule rules.GettableRule + err = json.Unmarshal([]byte(alert), &rule) + if err != nil { + zap.S().Errorf("msg:", "invalid rule data", "\t err:", err) + continue + } + if rule.AlertType == "LOGS_BASED_ALERT" { + alertsInfo.LogsBasedAlerts = alertsInfo.LogsBasedAlerts + 1 + } else if rule.AlertType == "METRIC_BASED_ALERT" { + alertsInfo.MetricBasedAlerts = alertsInfo.MetricBasedAlerts + 1 + } else if rule.AlertType == "TRACES_BASED_ALERT" { + alertsInfo.TracesBasedAlerts = alertsInfo.TracesBasedAlerts + 1 + } + alertsInfo.TotalAlerts = alertsInfo.TotalAlerts + 1 + } + + return &alertsInfo, nil +} + func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) { // response will contain top level fields from the otel log model response := model.GetFieldsResponse{ diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index e4e43f1648..f7fa328b9f 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -417,7 +417,7 @@ func (s *Server) analyticsMiddleware(next http.Handler) http.Handler { } // if telemetry.GetInstance().IsSampled() { - if _, ok := telemetry.IgnoredPaths()[path]; !ok { + if _, ok := telemetry.EnabledPaths()[path]; ok { userEmail, err := auth.GetEmailFromJwt(r.Context()) if err == nil { telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data, userEmail) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index b25888f607..e2b2b49481 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -71,6 +71,8 @@ type Reader interface { GetListResultV3(ctx context.Context, query string) ([]*v3.Row, error) LiveTailLogsV3(ctx context.Context, query string, timestampStart uint64, idStart string, client *v3.LogsLiveTailClient) + GetDashboardsInfo(ctx context.Context) (*model.DashboardsInfo, error) + GetAlertsInfo(ctx context.Context) (*model.AlertsInfo, error) GetTotalSpans(ctx context.Context) (uint64, error) GetSpansInLastHeartBeatInterval(ctx context.Context) (uint64, error) GetTimeSeriesInfo(ctx context.Context) (map[string]interface{}, error) diff --git a/pkg/query-service/model/response.go b/pkg/query-service/model/response.go index 9bb1d985c0..6d5e65d732 100644 --- a/pkg/query-service/model/response.go +++ b/pkg/query-service/model/response.go @@ -615,6 +615,20 @@ type TagsInfo struct { Env string `json:"env"` } +type AlertsInfo struct { + TotalAlerts int `json:"totalAlerts"` + LogsBasedAlerts int `json:"logsBasedAlerts"` + MetricBasedAlerts int `json:"metricBasedAlerts"` + TracesBasedAlerts int `json:"tracesBasedAlerts"` +} + +type DashboardsInfo struct { + TotalDashboards int `json:"totalDashboards"` + LogsBasedPanels int `json:"logsBasedPanels"` + MetricBasedPanels int `json:"metricBasedPanels"` + TracesBasedPanels int `json:"tracesBasedPanels"` +} + type TagTelemetryData struct { ServiceName string `json:"serviceName" ch:"serviceName"` Env string `json:"env" ch:"env"` diff --git a/pkg/query-service/rules/thresholdRule_test.go b/pkg/query-service/rules/thresholdRule_test.go index 27ad6611f5..031a19b70a 100644 --- a/pkg/query-service/rules/thresholdRule_test.go +++ b/pkg/query-service/rules/thresholdRule_test.go @@ -14,7 +14,7 @@ import ( func TestThresholdRuleCombinations(t *testing.T) { postableRule := PostableRule{ Alert: "Tricky Condition Tests", - AlertType: "METRICS_BASED_ALERT", + AlertType: "METRIC_BASED_ALERT", RuleType: RuleTypeThreshold, EvalWindow: Duration(5 * time.Minute), Frequency: Duration(1 * time.Minute), diff --git a/pkg/query-service/telemetry/ignored.go b/pkg/query-service/telemetry/ignored.go index 29c06fe1ac..c0a739e9ee 100644 --- a/pkg/query-service/telemetry/ignored.go +++ b/pkg/query-service/telemetry/ignored.go @@ -1,16 +1,11 @@ package telemetry -func IgnoredPaths() map[string]struct{} { - ignoredPaths := map[string]struct{}{ - "/api/v1/tags": {}, - "/api/v1/version": {}, - "/api/v1/query_range": {}, - "/api/v2/metrics/query_range": {}, - "/api/v1/health": {}, - "/api/v1/featureFlags": {}, +func EnabledPaths() map[string]struct{} { + enabledPaths := map[string]struct{}{ + "/api/v1/channels": {}, } - return ignoredPaths + return enabledPaths } func ignoreEvents(event string, attributes map[string]interface{}) bool { diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 24a2dbc4e2..206e200f5c 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -38,6 +38,7 @@ const ( TELEMETRY_EVENT_LOGS_FILTERS = "Logs Filters" TELEMETRY_EVENT_DISTRIBUTED = "Distributed" TELEMETRY_EVENT_QUERY_RANGE_V3 = "Query Range V3 Metadata" + TELEMETRY_EVENT_DASHBOARDS_ALERTS = "Dashboards/Alerts Info" TELEMETRY_EVENT_ACTIVE_USER = "Active User" TELEMETRY_EVENT_ACTIVE_USER_PH = "Active User V2" TELEMETRY_EVENT_USER_INVITATION_SENT = "User Invitation Sent" @@ -53,6 +54,7 @@ var SAAS_EVENTS_LIST = map[string]struct{}{ TELEMETRY_EVENT_ENVIRONMENT: {}, TELEMETRY_EVENT_USER_INVITATION_SENT: {}, TELEMETRY_EVENT_USER_INVITATION_ACCEPTED: {}, + TELEMETRY_EVENT_DASHBOARDS_ALERTS: {}, } const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz" @@ -61,9 +63,9 @@ const ph_api_key = "H-htDCae7CR3RV57gUzmol6IAKtm5IMCvbcm_fwnL-w" const IP_NOT_FOUND_PLACEHOLDER = "NA" const DEFAULT_NUMBER_OF_SERVICES = 6 -const HEART_BEAT_DURATION = 6 * time.Hour +const HEART_BEAT_DURATION = 12 * time.Hour -const ACTIVE_USER_DURATION = 30 * time.Minute +const ACTIVE_USER_DURATION = 6 * time.Hour // const HEART_BEAT_DURATION = 30 * time.Second // const ACTIVE_USER_DURATION = 30 * time.Second @@ -241,9 +243,27 @@ func createTelemetry() { } telemetry.SendEvent(TELEMETRY_EVENT_HEART_BEAT, data, "") + alertsInfo, err := telemetry.reader.GetAlertsInfo(context.Background()) + dashboardsInfo, err := telemetry.reader.GetDashboardsInfo(context.Background()) + + if err == nil { + dashboardsAlertsData := map[string]interface{}{ + "totalDashboards": dashboardsInfo.TotalDashboards, + "logsBasedPanels": dashboardsInfo.LogsBasedPanels, + "metricBasedPanels": dashboardsInfo.MetricBasedPanels, + "tracesBasedPanels": dashboardsInfo.TracesBasedPanels, + "totalAlerts": alertsInfo.TotalAlerts, + "logsBasedAlerts": alertsInfo.LogsBasedAlerts, + "metricBasedAlerts": alertsInfo.MetricBasedAlerts, + "tracesBasedAlerts": alertsInfo.TracesBasedAlerts, + } + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, dashboardsAlertsData, "") + } else { + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, map[string]interface{}{"error": err.Error()}, "") + } + getDistributedInfoInLastHeartBeatInterval, _ := telemetry.reader.GetDistributedInfoInLastHeartBeatInterval(context.Background()) telemetry.SendEvent(TELEMETRY_EVENT_DISTRIBUTED, getDistributedInfoInLastHeartBeatInterval, "") - } } }() From a8edc4fd95bfbed871e68e8980211b77b81a30ad Mon Sep 17 00:00:00 2001 From: Vishal Sharma Date: Wed, 13 Dec 2023 19:12:35 +0530 Subject: [PATCH 0025/1178] chore: better error handling in getAlertsInfo (#4230) --- ee/query-service/model/plans.go | 4 +-- pkg/query-service/model/featureSet.go | 4 +-- pkg/query-service/telemetry/telemetry.go | 33 +++++++++++++----------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/ee/query-service/model/plans.go b/ee/query-service/model/plans.go index caa2a4df68..5b6f230550 100644 --- a/ee/query-service/model/plans.go +++ b/ee/query-service/model/plans.go @@ -52,14 +52,14 @@ var BasicPlan = basemodel.FeatureSet{ Name: basemodel.QueryBuilderPanels, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 20, Route: "", }, basemodel.Feature{ Name: basemodel.QueryBuilderAlerts, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 10, Route: "", }, basemodel.Feature{ diff --git a/pkg/query-service/model/featureSet.go b/pkg/query-service/model/featureSet.go index 92aa7b2426..26cd70b908 100644 --- a/pkg/query-service/model/featureSet.go +++ b/pkg/query-service/model/featureSet.go @@ -55,14 +55,14 @@ var BasicPlan = FeatureSet{ Name: QueryBuilderPanels, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 20, Route: "", }, Feature{ Name: QueryBuilderAlerts, Active: true, Usage: 0, - UsageLimit: 5, + UsageLimit: 10, Route: "", }, Feature{ diff --git a/pkg/query-service/telemetry/telemetry.go b/pkg/query-service/telemetry/telemetry.go index 206e200f5c..4797e7d740 100644 --- a/pkg/query-service/telemetry/telemetry.go +++ b/pkg/query-service/telemetry/telemetry.go @@ -244,22 +244,25 @@ func createTelemetry() { telemetry.SendEvent(TELEMETRY_EVENT_HEART_BEAT, data, "") alertsInfo, err := telemetry.reader.GetAlertsInfo(context.Background()) - dashboardsInfo, err := telemetry.reader.GetDashboardsInfo(context.Background()) - - if err == nil { - dashboardsAlertsData := map[string]interface{}{ - "totalDashboards": dashboardsInfo.TotalDashboards, - "logsBasedPanels": dashboardsInfo.LogsBasedPanels, - "metricBasedPanels": dashboardsInfo.MetricBasedPanels, - "tracesBasedPanels": dashboardsInfo.TracesBasedPanels, - "totalAlerts": alertsInfo.TotalAlerts, - "logsBasedAlerts": alertsInfo.LogsBasedAlerts, - "metricBasedAlerts": alertsInfo.MetricBasedAlerts, - "tracesBasedAlerts": alertsInfo.TracesBasedAlerts, - } - telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, dashboardsAlertsData, "") - } else { + if err != nil { telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, map[string]interface{}{"error": err.Error()}, "") + } else { + dashboardsInfo, err := telemetry.reader.GetDashboardsInfo(context.Background()) + if err == nil { + dashboardsAlertsData := map[string]interface{}{ + "totalDashboards": dashboardsInfo.TotalDashboards, + "logsBasedPanels": dashboardsInfo.LogsBasedPanels, + "metricBasedPanels": dashboardsInfo.MetricBasedPanels, + "tracesBasedPanels": dashboardsInfo.TracesBasedPanels, + "totalAlerts": alertsInfo.TotalAlerts, + "logsBasedAlerts": alertsInfo.LogsBasedAlerts, + "metricBasedAlerts": alertsInfo.MetricBasedAlerts, + "tracesBasedAlerts": alertsInfo.TracesBasedAlerts, + } + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, dashboardsAlertsData, "") + } else { + telemetry.SendEvent(TELEMETRY_EVENT_DASHBOARDS_ALERTS, map[string]interface{}{"error": err.Error()}, "") + } } getDistributedInfoInLastHeartBeatInterval, _ := telemetry.reader.GetDistributedInfoInLastHeartBeatInterval(context.Background()) From a805eb75339bfc28003037d0e49170c0c45be678 Mon Sep 17 00:00:00 2001 From: Prashant Shahi Date: Wed, 13 Dec 2023 21:13:09 +0545 Subject: [PATCH 0026/1178] =?UTF-8?q?chore(signoz):=20=F0=9F=93=8C=20pin?= =?UTF-8?q?=20versions:=20SigNoz=200.35.1,=20SigNoz=20OtelCollector=200.88?= =?UTF-8?q?.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Prashant Shahi --- .../docker-swarm/clickhouse-setup/docker-compose.yaml | 10 +++++----- .../docker/clickhouse-setup/docker-compose-core.yaml | 6 +++--- deploy/docker/clickhouse-setup/docker-compose.yaml | 10 +++++----- go.mod | 2 +- go.sum | 4 ++-- .../tests/test-deploy/docker-compose.yaml | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 763e2ce4bf..30e5deecc1 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -146,7 +146,7 @@ services: condition: on-failure query-service: - image: signoz/query-service:0.35.0 + image: signoz/query-service:0.35.1 command: [ "-config=/root/config/prometheus.yml", @@ -186,7 +186,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:0.35.0 + image: signoz/frontend:0.35.1 deploy: restart_policy: condition: on-failure @@ -199,7 +199,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-config.yaml", @@ -237,7 +237,7 @@ services: - query-service otel-collector-migrator: - image: signoz/signoz-schema-migrator:0.88.1 + image: signoz/signoz-schema-migrator:0.88.3 deploy: restart_policy: condition: on-failure @@ -250,7 +250,7 @@ services: # - clickhouse-3 otel-collector-metrics: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-metrics-config.yaml", diff --git a/deploy/docker/clickhouse-setup/docker-compose-core.yaml b/deploy/docker/clickhouse-setup/docker-compose-core.yaml index d4f13e4d26..4e3f5d8bbe 100644 --- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml @@ -66,7 +66,7 @@ services: - --storage.path=/data otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.3} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -81,7 +81,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` otel-collector: container_name: signoz-otel-collector - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-config.yaml", @@ -118,7 +118,7 @@ services: otel-collector-metrics: container_name: signoz-otel-collector-metrics - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 command: [ "--config=/etc/otel-collector-metrics-config.yaml", diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index ddf80d6e14..2e54477bcf 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -164,7 +164,7 @@ services: # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md` query-service: - image: signoz/query-service:${DOCKER_TAG:-0.35.0} + image: signoz/query-service:${DOCKER_TAG:-0.35.1} container_name: signoz-query-service command: [ @@ -203,7 +203,7 @@ services: <<: *db-depend frontend: - image: signoz/frontend:${DOCKER_TAG:-0.35.0} + image: signoz/frontend:${DOCKER_TAG:-0.35.1} container_name: signoz-frontend restart: on-failure depends_on: @@ -215,7 +215,7 @@ services: - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.3} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -229,7 +229,7 @@ services: otel-collector: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.3} container_name: signoz-otel-collector command: [ @@ -269,7 +269,7 @@ services: condition: service_healthy otel-collector-metrics: - image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.3} container_name: signoz-otel-collector-metrics command: [ diff --git a/go.mod b/go.mod index 9009f9da9f..a10d4f904d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/ClickHouse/clickhouse-go/v2 v2.15.0 github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb - github.com/SigNoz/signoz-otel-collector v0.88.1 + github.com/SigNoz/signoz-otel-collector v0.88.3 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974 github.com/antonmedv/expr v1.15.3 diff --git a/go.sum b/go.sum index 9b7192550d..02d869f765 100644 --- a/go.sum +++ b/go.sum @@ -98,8 +98,8 @@ github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb h1:bneLSKPf9YUSFm github.com/SigNoz/govaluate v0.0.0-20220522085550-d19c08c206cb/go.mod h1:JznGDNg9x1cujDKa22RaQOimOvvEfy3nxzDGd8XDgmA= github.com/SigNoz/prometheus v1.9.78 h1:bB3yuDrRzi/Mv00kWayR9DZbyjTuGfendSqISyDcXiY= github.com/SigNoz/prometheus v1.9.78/go.mod h1:MffmFu2qFILQrOHehx3D0XjYtaZMVfI+Ppeiv98x4Ww= -github.com/SigNoz/signoz-otel-collector v0.88.1 h1:Xeu6Kn8VA0g6it60PMIAclayYSIogBq0rnkodlpxllI= -github.com/SigNoz/signoz-otel-collector v0.88.1/go.mod h1:KyEc6JSFS6f8Nw3UdSm4aGDGucEpQYZUdYwjvY8uMVc= +github.com/SigNoz/signoz-otel-collector v0.88.3 h1:30sEJZmCQjfjo8CZGxqXKZkWE7Zij9TeS1uUqNFEZRU= +github.com/SigNoz/signoz-otel-collector v0.88.3/go.mod h1:KyEc6JSFS6f8Nw3UdSm4aGDGucEpQYZUdYwjvY8uMVc= github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc= github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo= github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY= diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index 1f6c718b1f..caed48440b 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -192,7 +192,7 @@ services: <<: *db-depend otel-collector-migrator: - image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.1} + image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.3} container_name: otel-migrator command: - "--dsn=tcp://clickhouse:9000" @@ -205,7 +205,7 @@ services: # condition: service_healthy otel-collector: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 container_name: signoz-otel-collector command: [ @@ -245,7 +245,7 @@ services: condition: service_healthy otel-collector-metrics: - image: signoz/signoz-otel-collector:0.88.1 + image: signoz/signoz-otel-collector:0.88.3 container_name: signoz-otel-collector-metrics command: [ From ec500831ef93f2c7168d6f82490dc3545a4a8c3f Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Thu, 14 Dec 2023 11:22:20 +0530 Subject: [PATCH 0027/1178] feat: upgrade clickhouse to 23.11.1 (#4225) --- deploy/docker-swarm/clickhouse-setup/docker-compose.yaml | 2 +- deploy/docker/clickhouse-setup/docker-compose.yaml | 2 +- pkg/query-service/tests/test-deploy/docker-compose.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml index 30e5deecc1..34be947397 100644 --- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml @@ -1,7 +1,7 @@ version: "3.9" x-clickhouse-defaults: &clickhouse-defaults - image: clickhouse/clickhouse-server:23.7.3-alpine + image: clickhouse/clickhouse-server:23.11.1-alpine tty: true deploy: restart_policy: diff --git a/deploy/docker/clickhouse-setup/docker-compose.yaml b/deploy/docker/clickhouse-setup/docker-compose.yaml index 2e54477bcf..79e4ee2519 100644 --- a/deploy/docker/clickhouse-setup/docker-compose.yaml +++ b/deploy/docker/clickhouse-setup/docker-compose.yaml @@ -3,7 +3,7 @@ version: "2.4" x-clickhouse-defaults: &clickhouse-defaults restart: on-failure # addding non LTS version due to this fix https://github.com/ClickHouse/ClickHouse/commit/32caf8716352f45c1b617274c7508c86b7d1afab - image: clickhouse/clickhouse-server:23.7.3-alpine + image: clickhouse/clickhouse-server:23.11.1-alpine tty: true depends_on: - zookeeper-1 diff --git a/pkg/query-service/tests/test-deploy/docker-compose.yaml b/pkg/query-service/tests/test-deploy/docker-compose.yaml index caed48440b..dfb36a8dbd 100644 --- a/pkg/query-service/tests/test-deploy/docker-compose.yaml +++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml @@ -2,7 +2,7 @@ version: "2.4" x-clickhouse-defaults: &clickhouse-defaults restart: on-failure - image: clickhouse/clickhouse-server:23.7.3-alpine + image: clickhouse/clickhouse-server:23.11.1-alpine tty: true depends_on: - zookeeper-1 From 9c1ea0cde9b8d20899c22381dc2f7bd6f0045b63 Mon Sep 17 00:00:00 2001 From: Rajat Dabade Date: Thu, 14 Dec 2023 11:43:02 +0530 Subject: [PATCH 0028/1178] refactor: pop for unsaved changes (#4188) --- frontend/public/locales/en-GB/dashboard.json | 6 +- frontend/public/locales/en/dashboard.json | 6 +- frontend/src/components/ExplorerCard/utils.ts | 78 ++++++++++--------- frontend/src/container/NewWidget/index.tsx | 75 +++++++++++++++--- frontend/src/container/NewWidget/utils.ts | 15 ++++ 5 files changed, 132 insertions(+), 48 deletions(-) create mode 100644 frontend/src/container/NewWidget/utils.ts diff --git a/frontend/public/locales/en-GB/dashboard.json b/frontend/public/locales/en-GB/dashboard.json index 35e9b47b45..6179004aff 100644 --- a/frontend/public/locales/en-GB/dashboard.json +++ b/frontend/public/locales/en-GB/dashboard.json @@ -21,5 +21,9 @@ "error_while_updating_variable": "Error while updating variable", "dashboard_has_been_updated": "Dashboard has been updated", "do_you_want_to_refresh_the_dashboard": "Do you want to refresh the dashboard?", - "delete_dashboard_success": "{{name}} dashboard deleted successfully" + "delete_dashboard_success": "{{name}} dashboard deleted successfully", + "dashboard_unsave_changes": "There are unsaved changes in the Query builder, please stage and run the query or the changes will be lost. Press OK to discard.", + "dashboard_save_changes": "Your graph built with {{queryTag}} query will be saved. Press OK to confirm.", + "your_graph_build_with": "Your graph built with", + "dashboar_ok_confirm": "query will be saved. Press OK to confirm." } diff --git a/frontend/public/locales/en/dashboard.json b/frontend/public/locales/en/dashboard.json index aa2454c3a3..a74f23d228 100644 --- a/frontend/public/locales/en/dashboard.json +++ b/frontend/public/locales/en/dashboard.json @@ -24,5 +24,9 @@ "do_you_want_to_refresh_the_dashboard": "Do you want to refresh the dashboard?", "locked_dashboard_delete_tooltip_admin_author": "Dashboard is locked. Please unlock the dashboard to enable delete.", "locked_dashboard_delete_tooltip_editor": "Dashboard is locked. Please contact admin to delete the dashboard.", - "delete_dashboard_success": "{{name}} dashboard deleted successfully" + "delete_dashboard_success": "{{name}} dashboard deleted successfully", + "dashboard_unsave_changes": "There are unsaved changes in the Query builder, please stage and run the query or the changes will be lost. Press OK to discard.", + "dashboard_save_changes": "Your graph built with {{queryTag}} query will be saved. Press OK to confirm.", + "your_graph_build_with": "Your graph built with", + "dashboar_ok_confirm": "query will be saved. Press OK to confirm." } diff --git a/frontend/src/components/ExplorerCard/utils.ts b/frontend/src/components/ExplorerCard/utils.ts index 7385fbcc7f..3a2eeac95f 100644 --- a/frontend/src/components/ExplorerCard/utils.ts +++ b/frontend/src/components/ExplorerCard/utils.ts @@ -5,6 +5,7 @@ import { QueryParams } from 'constants/query'; import { initialQueriesMap, PANEL_TYPES } from 'constants/queryBuilder'; import { mapQueryDataFromApi } from 'lib/newQueryBuilder/queryBuilderMappers/mapQueryDataFromApi'; import isEqual from 'lodash-es/isEqual'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; import { DeleteViewHandlerProps, @@ -35,6 +36,45 @@ export const getViewDetailsUsingViewKey: GetViewDetailsUsingViewKey = ( return undefined; }; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const omitIdFromQuery = (query: Query | null): any => ({ + ...query, + builder: { + ...query?.builder, + queryData: query?.builder.queryData.map((queryData) => { + const { id, ...rest } = queryData.aggregateAttribute; + const newAggregateAttribute = rest; + const newGroupByAttributes = queryData.groupBy.map((groupByAttribute) => { + const { id, ...rest } = groupByAttribute; + return rest; + }); + const newItems = queryData.filters.items.map((item) => { + const { id, ...newItem } = item; + if (item.key) { + const { id, ...rest } = item.key; + return { + ...newItem, + key: rest, + }; + } + return newItem; + }); + return { + ...queryData, + aggregateAttribute: newAggregateAttribute, + groupBy: newGroupByAttributes, + filters: { + ...queryData.filters, + items: newItems, + }, + limit: queryData.limit ? queryData.limit : 0, + offset: queryData.offset ? queryData.offset : 0, + pageSize: queryData.pageSize ? queryData.pageSize : 0, + }; + }), + }, +}); + export const isQueryUpdatedInView = ({ viewKey, data, @@ -48,43 +88,7 @@ export const isQueryUpdatedInView = ({ const { query, panelType } = currentViewDetails; // Omitting id from aggregateAttribute and groupBy - const updatedCurrentQuery = { - ...stagedQuery, - builder: { - ...stagedQuery?.builder, - queryData: stagedQuery?.builder.queryData.map((queryData) => { - const { id, ...rest } = queryData.aggregateAttribute; - const newAggregateAttribute = rest; - const newGroupByAttributes = queryData.groupBy.map((groupByAttribute) => { - const { id, ...rest } = groupByAttribute; - return rest; - }); - const newItems = queryData.filters.items.map((item) => { - const { id, ...newItem } = item; - if (item.key) { - const { id, ...rest } = item.key; - return { - ...newItem, - key: rest, - }; - } - return newItem; - }); - return { - ...queryData, - aggregateAttribute: newAggregateAttribute, - groupBy: newGroupByAttributes, - filters: { - ...queryData.filters, - items: newItems, - }, - limit: queryData.limit ? queryData.limit : 0, - offset: queryData.offset ? queryData.offset : 0, - pageSize: queryData.pageSize ? queryData.pageSize : 0, - }; - }), - }, - }; + const updatedCurrentQuery = omitIdFromQuery(stagedQuery); return ( panelType !== currentPanelType || diff --git a/frontend/src/container/NewWidget/index.tsx b/frontend/src/container/NewWidget/index.tsx index 18f05ecec2..e4f40ee2ac 100644 --- a/frontend/src/container/NewWidget/index.tsx +++ b/frontend/src/container/NewWidget/index.tsx @@ -1,5 +1,6 @@ -import { LockFilled } from '@ant-design/icons'; -import { Button, Modal, Tooltip, Typography } from 'antd'; +/* eslint-disable sonarjs/cognitive-complexity */ +import { LockFilled, WarningOutlined } from '@ant-design/icons'; +import { Button, Modal, Space, Tooltip, Typography } from 'antd'; import { SOMETHING_WENT_WRONG } from 'constants/api'; import { FeatureKeys } from 'constants/features'; import { PANEL_TYPES } from 'constants/queryBuilder'; @@ -18,6 +19,7 @@ import { getSelectedWidgetIndex, } from 'providers/Dashboard/util'; import { useCallback, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import { generatePath, useLocation, useParams } from 'react-router-dom'; import { AppState } from 'store/reducers'; @@ -39,6 +41,7 @@ import { RightContainerWrapper, } from './styles'; import { NewWidgetProps } from './types'; +import { getIsQueryModified } from './utils'; function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element { const { @@ -47,7 +50,14 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element { setToScrollWidgetId, } = useDashboard(); - const { currentQuery } = useQueryBuilder(); + const { t } = useTranslation(['dashboard']); + + const { currentQuery, stagedQuery } = useQueryBuilder(); + + const isQueryModified = useMemo( + () => getIsQueryModified(currentQuery, stagedQuery), + [currentQuery, stagedQuery], + ); const { featureResponse } = useSelector( (state) => state.app, @@ -92,6 +102,12 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element { selectedWidget?.fillSpans || false, ); const [saveModal, setSaveModal] = useState(false); + const [discardModal, setDiscardModal] = useState(false); + + const closeModal = (): void => { + setSaveModal(false); + setDiscardModal(false); + }; const [graphType, setGraphType] = useState(selectedGraph); @@ -206,6 +222,14 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element { ]); const onClickDiscardHandler = useCallback(() => { + if (isQueryModified) { + setDiscardModal(true); + return; + } + history.push(generatePath(ROUTES.DASHBOARD, { dashboardId })); + }, [dashboardId, isQueryModified]); + + const discardChanges = useCallback(() => { history.push(generatePath(ROUTES.DASHBOARD, { dashboardId })); }, [dashboardId]); @@ -321,21 +345,54 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element { + + Unsaved Changes + + ) : ( + 'Save Widget' + ) + } focusTriggerAfterClose forceRender destroyOnClose closable - onCancel={(): void => setSaveModal(false)} + onCancel={closeModal} onOk={onClickSaveHandler} centered open={saveModal} width={600} > - - Your graph built with {' '} - query will be saved. Press OK to confirm. - + {!isQueryModified ? ( + + {t('your_graph_build_with')}{' '} + + {t('dashboar_ok_confirm')} + + ) : ( + {t('dashboard_unsave_changes')} + )} + + + + Unsaved Changes + + } + focusTriggerAfterClose + forceRender + destroyOnClose + closable + onCancel={closeModal} + onOk={discardChanges} + centered + open={discardModal} + width={600} + > + {t('dashboard_unsave_changes')} ); diff --git a/frontend/src/container/NewWidget/utils.ts b/frontend/src/container/NewWidget/utils.ts new file mode 100644 index 0000000000..15fcd278d7 --- /dev/null +++ b/frontend/src/container/NewWidget/utils.ts @@ -0,0 +1,15 @@ +import { omitIdFromQuery } from 'components/ExplorerCard/utils'; +import { isEqual } from 'lodash-es'; +import { Query } from 'types/api/queryBuilder/queryBuilderData'; + +export const getIsQueryModified = ( + currentQuery: Query, + stagedQuery: Query | null, +): boolean => { + if (!stagedQuery) { + return false; + } + const omitIdFromStageQuery = omitIdFromQuery(stagedQuery); + const omitIdFromCurrentQuery = omitIdFromQuery(currentQuery); + return !isEqual(omitIdFromStageQuery, omitIdFromCurrentQuery); +}; From a16fca63763d15b8ac78973b1e26592c2c63ca04 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Thu, 14 Dec 2023 16:52:02 +0530 Subject: [PATCH 0029/1178] fix: remove timestamp roundup for logs list api call (#4229) * fix: remove timestamp roundup for logs list api call * fix: test updated --- pkg/query-service/app/logs/v3/query_builder.go | 6 ++++-- pkg/query-service/app/logs/v3/query_builder_test.go | 8 ++++---- pkg/query-service/app/parser.go | 7 ------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pkg/query-service/app/logs/v3/query_builder.go b/pkg/query-service/app/logs/v3/query_builder.go index 32e4bb29f4..c0457358f8 100644 --- a/pkg/query-service/app/logs/v3/query_builder.go +++ b/pkg/query-service/app/logs/v3/query_builder.go @@ -483,8 +483,10 @@ func isOrderByTs(orderBy []v3.OrderBy) bool { func PrepareLogsQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options Options) (string, error) { // adjust the start and end time to the step interval - start = start - (start % (mq.StepInterval * 1000)) - end = end - (end % (mq.StepInterval * 1000)) + if panelType != v3.PanelTypeList { + start = start - (start % (mq.StepInterval * 1000)) + end = end - (end % (mq.StepInterval * 1000)) + } if options.IsLivetailQuery { query, err := buildLogsLiveTailQuery(mq) diff --git a/pkg/query-service/app/logs/v3/query_builder_test.go b/pkg/query-service/app/logs/v3/query_builder_test.go index 1db46e1374..9bbc7da729 100644 --- a/pkg/query-service/app/logs/v3/query_builder_test.go +++ b/pkg/query-service/app/logs/v3/query_builder_test.go @@ -1353,7 +1353,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 5, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) order by timestamp desc LIMIT 1", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by timestamp desc LIMIT 1", }, { Name: "Test limit greater than pageSize - order by ts", @@ -1374,7 +1374,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 10, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by timestamp desc LIMIT 10", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by timestamp desc LIMIT 10", }, { Name: "Test limit less than pageSize - order by custom", @@ -1393,7 +1393,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 5, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 1 OFFSET 0", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 1 OFFSET 0", }, { Name: "Test limit greater than pageSize - order by custom", @@ -1414,7 +1414,7 @@ var testPrepLogsQueryLimitOffsetData = []struct { PageSize: 50, }, TableName: "logs", - ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360000000000 AND timestamp <= 1680066420000000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 50 OFFSET 50", + ExpectedQuery: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, body,CAST((attributes_string_key, attributes_string_value), 'Map(String, String)') as attributes_string,CAST((attributes_int64_key, attributes_int64_value), 'Map(String, Int64)') as attributes_int64,CAST((attributes_float64_key, attributes_float64_value), 'Map(String, Float64)') as attributes_float64,CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as attributes_bool,CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string from signoz_logs.distributed_logs where (timestamp >= 1680066360726000000 AND timestamp <= 1680066458000000000) AND id < '2TNh4vp2TpiWyLt3SzuadLJF2s4' order by attributes_string_value[indexOf(attributes_string_key, 'method')] desc LIMIT 50 OFFSET 50", }, } diff --git a/pkg/query-service/app/parser.go b/pkg/query-service/app/parser.go index 4e6350b70e..96905dc1d5 100644 --- a/pkg/query-service/app/parser.go +++ b/pkg/query-service/app/parser.go @@ -1021,13 +1021,6 @@ func ParseQueryRangeParams(r *http.Request) (*v3.QueryRangeParamsV3, *model.ApiE queryRangeParams.Start = queryRangeParams.End } - // round up the end to neaerest multiple - if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder { - end := (queryRangeParams.End) / 1000 - step := queryRangeParams.Step - queryRangeParams.End = (end / step * step) * 1000 - } - // replace go template variables in clickhouse query if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeClickHouseSQL { for _, chQuery := range queryRangeParams.CompositeQuery.ClickHouseQueries { From 1d1154aa8cc0ed05800932f7325af0a56fdc9602 Mon Sep 17 00:00:00 2001 From: Rajat Dabade Date: Thu, 14 Dec 2023 18:10:52 +0530 Subject: [PATCH 0030/1178] [Refactor]: added tooltip for graph manager (#4236) --- .../GridCardLayout/GridCard/FullView/TableRender/Label.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/container/GridCardLayout/GridCard/FullView/TableRender/Label.tsx b/frontend/src/container/GridCardLayout/GridCard/FullView/TableRender/Label.tsx index 7f5592e592..4b9a33587b 100644 --- a/frontend/src/container/GridCardLayout/GridCard/FullView/TableRender/Label.tsx +++ b/frontend/src/container/GridCardLayout/GridCard/FullView/TableRender/Label.tsx @@ -1,3 +1,4 @@ +import { Tooltip } from 'antd'; import { useIsDarkMode } from 'hooks/useDarkMode'; import { LabelContainer } from '../styles'; @@ -23,7 +24,9 @@ function Label({ disabled={disabled} onClick={onClickHandler} > - {getAbbreviatedLabel(label)} + + {getAbbreviatedLabel(label)} + ); } From 7efe90775762737eac41518019a8f4bec0795b38 Mon Sep 17 00:00:00 2001 From: Vikrant Gupta <54737045+Vikrant2520@users.noreply.github.com> Date: Thu, 14 Dec 2023 22:14:58 +0530 Subject: [PATCH 0031/1178] fix: [GH-3790]: timerange not working for different users (#4192) --- .../TopNav/DateTimeSelection/index.tsx | 48 ++++++++++++------- frontend/src/hooks/logs/useCopyLogLink.ts | 20 ++++++-- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/frontend/src/container/TopNav/DateTimeSelection/index.tsx b/frontend/src/container/TopNav/DateTimeSelection/index.tsx index 742409e455..14f454dbb1 100644 --- a/frontend/src/container/TopNav/DateTimeSelection/index.tsx +++ b/frontend/src/container/TopNav/DateTimeSelection/index.tsx @@ -3,12 +3,16 @@ import { Button, Select as DefaultSelect } from 'antd'; import getLocalStorageKey from 'api/browser/localstorage/get'; import setLocalStorageKey from 'api/browser/localstorage/set'; import { LOCALSTORAGE } from 'constants/localStorage'; +import { QueryParams } from 'constants/query'; +import ROUTES from 'constants/routes'; import dayjs, { Dayjs } from 'dayjs'; import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder'; import { updateStepInterval } from 'hooks/queryBuilder/useStepInterval'; +import useUrlQuery from 'hooks/useUrlQuery'; import GetMinMax from 'lib/getMinMax'; import getTimeString from 'lib/getTimeString'; -import { useCallback, useEffect, useState } from 'react'; +import history from 'lib/history'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { connect, useSelector } from 'react-redux'; import { RouteComponentProps, withRouter } from 'react-router-dom'; import { bindActionCreators, Dispatch } from 'redux'; @@ -34,9 +38,9 @@ function DateTimeSelection({ }: Props): JSX.Element { const [formSelector] = Form.useForm(); - const params = new URLSearchParams(location.search); - const searchStartTime = params.get('startTime'); - const searchEndTime = params.get('endTime'); + const urlQuery = useUrlQuery(); + const searchStartTime = urlQuery.get('startTime'); + const searchEndTime = urlQuery.get('endTime'); const localstorageStartTime = getLocalStorageKey('startTime'); const localstorageEndTime = getLocalStorageKey('endTime'); @@ -169,6 +173,11 @@ function DateTimeSelection({ return `Last refresh - ${secondsDiff} sec ago`; }, [maxTime, minTime, selectedTime]); + const isLogsExplorerPage = useMemo( + () => location.pathname === ROUTES.LOGS_EXPLORER, + [location.pathname], + ); + const onSelectHandler = (value: Time): void => { if (value !== 'custom') { updateTimeInterval(value); @@ -181,12 +190,18 @@ function DateTimeSelection({ setCustomDTPickerVisible(true); } + const { maxTime, minTime } = GetMinMax(value, getTime()); + + if (!isLogsExplorerPage) { + urlQuery.set(QueryParams.startTime, minTime.toString()); + urlQuery.set(QueryParams.endTime, maxTime.toString()); + const generatedUrl = `${location.pathname}?${urlQuery.toString()}`; + history.replace(generatedUrl); + } + if (!stagedQuery) { return; } - - const { maxTime, minTime } = GetMinMax(value, getTime()); - initQueryBuilderData(updateStepInterval(stagedQuery, maxTime, minTime)); }; @@ -207,6 +222,12 @@ function DateTimeSelection({ setLocalStorageKey('startTime', startTimeMoment.toString()); setLocalStorageKey('endTime', endTimeMoment.toString()); updateLocalStorageForRoutes('custom'); + if (!isLogsExplorerPage) { + urlQuery.set(QueryParams.startTime, startTimeMoment.toString()); + urlQuery.set(QueryParams.endTime, endTimeMoment.toString()); + const generatedUrl = `${location.pathname}?${urlQuery.toString()}`; + history.replace(generatedUrl); + } } } }; @@ -234,7 +255,6 @@ function DateTimeSelection({ if (searchEndTime !== null && searchStartTime !== null) { return 'custom'; } - if ( (localstorageEndTime === null || localstorageStartTime === null) && time === 'custom' @@ -252,16 +272,8 @@ function DateTimeSelection({ setRefreshButtonHidden(updatedTime === 'custom'); updateTimeInterval(updatedTime, [preStartTime, preEndTime]); - }, [ - location.pathname, - getTime, - localstorageEndTime, - localstorageStartTime, - searchEndTime, - searchStartTime, - updateTimeInterval, - globalTimeLoading, - ]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [location.pathname, updateTimeInterval, globalTimeLoading]); return ( <> diff --git a/frontend/src/hooks/logs/useCopyLogLink.ts b/frontend/src/hooks/logs/useCopyLogLink.ts index 8401b6433f..35b4293f51 100644 --- a/frontend/src/hooks/logs/useCopyLogLink.ts +++ b/frontend/src/hooks/logs/useCopyLogLink.ts @@ -3,6 +3,7 @@ import ROUTES from 'constants/routes'; import { useNotifications } from 'hooks/useNotifications'; import useUrlQuery from 'hooks/useUrlQuery'; import useUrlQueryData from 'hooks/useUrlQueryData'; +import history from 'lib/history'; import { MouseEventHandler, useCallback, @@ -28,16 +29,27 @@ export const useCopyLogLink = (logId?: string): UseCopyLogLink => { (state) => state.globalTime, ); - const { - queryData: timeRange, - redirectWithQuery: onTimeRangeChange, - } = useUrlQueryData(QueryParams.timeRange, null); + const { queryData: timeRange } = useUrlQueryData( + QueryParams.timeRange, + null, + ); const { queryData: activeLogId } = useUrlQueryData( QueryParams.activeLogId, null, ); + const onTimeRangeChange = useCallback( + (newTimeRange: LogTimeRange | null): void => { + urlQuery.set(QueryParams.timeRange, JSON.stringify(newTimeRange)); + urlQuery.set(QueryParams.startTime, newTimeRange?.start.toString() || ''); + urlQuery.set(QueryParams.endTime, newTimeRange?.end.toString() || ''); + const generatedUrl = `${pathname}?${urlQuery.toString()}`; + history.replace(generatedUrl); + }, + [pathname, urlQuery], + ); + const isActiveLog = useMemo(() => activeLogId === logId, [activeLogId, logId]); const [isHighlighted, setIsHighlighted] = useState(isActiveLog); From 418ab67d5018751fa599379ec5a314c744e15fcb Mon Sep 17 00:00:00 2001 From: Yunus M Date: Thu, 14 Dec 2023 22:56:25 +0530 Subject: [PATCH 0032/1178] Uplot time range (#4144) * feat: show range bound chart based on the selected time range * feat: handle no data * feat: show bigger point if only data point exists * feat: show bigger point if only data point exists * feat: widget ui fixes * feat: no data - full view fix * fix: show closed point on hover * feat: handle widget time preference in case of dashboard, edit view, full view and chart preview --- frontend/.eslintrc.js | 2 +- .../Uplot/{uplot.scss => Uplot.styles.scss} | 8 ++++ frontend/src/components/Uplot/Uplot.tsx | 13 +++++- .../FormAlertRules/ChartPreview/index.tsx | 24 +++++++++-- .../GridCard/FullView/index.tsx | 18 +++++++++ .../GridCard/WidgetGraphComponent.tsx | 6 ++- .../GridCardLayout/GridCard/index.tsx | 24 ++++++++--- .../GridCardLayout/GridCardLayout.styles.scss | 8 ++++ .../WidgetHeader/WidgetHeader.styles.scss | 9 ++++- .../GridCardLayout/WidgetHeader/index.tsx | 6 +-- .../src/container/GridCardLayout/styles.ts | 10 +---- .../WidgetGraph/WidgetGraphs.tsx | 26 +++++++++++- .../LeftContainer/WidgetGraph/styles.ts | 3 +- .../TimeSeriesView/TimeSeriesView.tsx | 23 ++++++++++- .../src/lib/uPlotLib/getUplotChartOptions.ts | 24 ++++++----- .../src/lib/uPlotLib/plugins/tooltipPlugin.ts | 5 ++- frontend/src/lib/uPlotLib/utils/getAxes.ts | 6 +-- .../src/lib/uPlotLib/utils/getGridColor.ts | 4 +- .../src/lib/uPlotLib/utils/getSeriesData.ts | 9 ++++- .../src/lib/uPlotLib/utils/getXAxisScale.ts | 40 +++++++++++++++++++ .../src/lib/uPlotLib/utils/getYAxisScale.ts | 7 +++- frontend/src/utils/getTimeRange.ts | 36 +++++++++++++++++ 22 files changed, 260 insertions(+), 51 deletions(-) rename frontend/src/components/Uplot/{uplot.scss => Uplot.styles.scss} (68%) create mode 100644 frontend/src/lib/uPlotLib/utils/getXAxisScale.ts create mode 100644 frontend/src/utils/getTimeRange.ts diff --git a/frontend/.eslintrc.js b/frontend/.eslintrc.js index 540b1ded70..5034915c00 100644 --- a/frontend/.eslintrc.js +++ b/frontend/.eslintrc.js @@ -86,6 +86,7 @@ module.exports = { }, ], 'import/no-extraneous-dependencies': ['error', { devDependencies: true }], + 'no-plusplus': 'off', 'jsx-a11y/label-has-associated-control': [ 'error', { @@ -109,7 +110,6 @@ module.exports = { // eslint rules need to remove '@typescript-eslint/no-shadow': 'off', 'import/no-cycle': 'off', - 'prettier/prettier': [ 'error', {}, diff --git a/frontend/src/components/Uplot/uplot.scss b/frontend/src/components/Uplot/Uplot.styles.scss similarity index 68% rename from frontend/src/components/Uplot/uplot.scss rename to frontend/src/components/Uplot/Uplot.styles.scss index 55e681cb70..448ef56b18 100644 --- a/frontend/src/components/Uplot/uplot.scss +++ b/frontend/src/components/Uplot/Uplot.styles.scss @@ -13,3 +13,11 @@ height: 100%; width: 100%; } + +.uplot-no-data { + position: relative; + display: flex; + width: 100%; + flex-direction: column; + gap: 8px; +} diff --git a/frontend/src/components/Uplot/Uplot.tsx b/frontend/src/components/Uplot/Uplot.tsx index 84b3d1bb9b..05f050a87c 100644 --- a/frontend/src/components/Uplot/Uplot.tsx +++ b/frontend/src/components/Uplot/Uplot.tsx @@ -1,8 +1,9 @@ /* eslint-disable sonarjs/cognitive-complexity */ -import './uplot.scss'; +import './Uplot.styles.scss'; import { Typography } from 'antd'; import { ToggleGraphProps } from 'components/Graph/types'; +import { LineChart } from 'lucide-react'; import ErrorBoundaryFallback from 'pages/ErrorBoundaryFallback/ErrorBoundaryFallback'; import { forwardRef, @@ -127,6 +128,16 @@ const Uplot = forwardRef( } }, [data, resetScales, create]); + if (data && data[0] && data[0]?.length === 0) { + return ( +
+ + + No Data +
+ ); + } + return (
diff --git a/frontend/src/container/FormAlertRules/ChartPreview/index.tsx b/frontend/src/container/FormAlertRules/ChartPreview/index.tsx index 3551c3a87a..7e22e3a11c 100644 --- a/frontend/src/container/FormAlertRules/ChartPreview/index.tsx +++ b/frontend/src/container/FormAlertRules/ChartPreview/index.tsx @@ -10,7 +10,7 @@ import { useIsDarkMode } from 'hooks/useDarkMode'; import { useResizeObserver } from 'hooks/useDimensions'; import { getUPlotChartOptions } from 'lib/uPlotLib/getUplotChartOptions'; import { getUPlotChartData } from 'lib/uPlotLib/utils/getUplotChartData'; -import { useMemo, useRef } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; @@ -18,6 +18,7 @@ import { AlertDef } from 'types/api/alerts/def'; import { Query } from 'types/api/queryBuilder/queryBuilderData'; import { EQueryType } from 'types/common/dashboard'; import { GlobalReducer } from 'types/reducer/globalTime'; +import { getTimeRange } from 'utils/getTimeRange'; import { ChartContainer, FailedMessageContainer } from './styles'; import { getThresholdLabel } from './utils'; @@ -49,9 +50,13 @@ function ChartPreview({ }: ChartPreviewProps): JSX.Element | null { const { t } = useTranslation('alerts'); const threshold = alertDef?.condition.target || 0; - const { minTime, maxTime } = useSelector( - (state) => state.globalTime, - ); + const [minTimeScale, setMinTimeScale] = useState(); + const [maxTimeScale, setMaxTimeScale] = useState(); + + const { minTime, maxTime, selectedTime: globalSelectedInterval } = useSelector< + AppState, + GlobalReducer + >((state) => state.globalTime); const canQuery = useMemo((): boolean => { if (!query || query == null) { @@ -101,6 +106,13 @@ function ChartPreview({ const graphRef = useRef(null); + useEffect((): void => { + const { startTime, endTime } = getTimeRange(queryResponse); + + setMinTimeScale(startTime); + setMaxTimeScale(endTime); + }, [maxTime, minTime, globalSelectedInterval, queryResponse]); + const chartData = getUPlotChartData(queryResponse?.data?.payload); const containerDimensions = useResizeObserver(graphRef); @@ -117,6 +129,8 @@ function ChartPreview({ yAxisUnit, apiResponse: queryResponse?.data?.payload, dimensions: containerDimensions, + minTimeScale, + maxTimeScale, isDarkMode, thresholds: [ { @@ -141,6 +155,8 @@ function ChartPreview({ yAxisUnit, queryResponse?.data?.payload, containerDimensions, + minTimeScale, + maxTimeScale, isDarkMode, threshold, t, diff --git a/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx b/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx index 3bd60c40ba..db42625f1d 100644 --- a/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx +++ b/frontend/src/container/GridCardLayout/GridCard/FullView/index.tsx @@ -23,6 +23,7 @@ import { useSelector } from 'react-redux'; import { AppState } from 'store/reducers'; import { GlobalReducer } from 'types/reducer/globalTime'; import uPlot from 'uplot'; +import { getTimeRange } from 'utils/getTimeRange'; import { PANEL_TYPES_VS_FULL_VIEW_TABLE } from './contants'; import GraphManager from './GraphManager'; @@ -92,6 +93,21 @@ function FullView({ const isDarkMode = useIsDarkMode(); + const [minTimeScale, setMinTimeScale] = useState(); + const [maxTimeScale, setMaxTimeScale] = useState(); + + const { minTime, maxTime, selectedTime: globalSelectedInterval } = useSelector< + AppState, + GlobalReducer + >((state) => state.globalTime); + + useEffect((): void => { + const { startTime, endTime } = getTimeRange(response); + + setMinTimeScale(startTime); + setMaxTimeScale(endTime); + }, [maxTime, minTime, globalSelectedInterval, response]); + useEffect(() => { if (!response.isFetching && fullViewRef.current) { const width = fullViewRef.current?.clientWidth @@ -114,6 +130,8 @@ function FullView({ graphsVisibilityStates, setGraphsVisibilityStates, thresholds: widget.thresholds, + minTimeScale, + maxTimeScale, }); setChartOptions(newChartOptions); diff --git a/frontend/src/container/GridCardLayout/GridCard/WidgetGraphComponent.tsx b/frontend/src/container/GridCardLayout/GridCard/WidgetGraphComponent.tsx index 2129220427..29abaa9685 100644 --- a/frontend/src/container/GridCardLayout/GridCard/WidgetGraphComponent.tsx +++ b/frontend/src/container/GridCardLayout/GridCard/WidgetGraphComponent.tsx @@ -1,4 +1,5 @@ import { Skeleton, Typography } from 'antd'; +import cx from 'classnames'; import { ToggleGraphProps } from 'components/Graph/types'; import { SOMETHING_WENT_WRONG } from 'constants/api'; import { QueryParams } from 'constants/query'; @@ -298,7 +299,10 @@ function WidgetGraphComponent({
{queryResponse.isLoading && } {queryResponse.isSuccess && ( -
+
(); const { toScrollWidgetId, setToScrollWidgetId } = useDashboard(); + const [minTimeScale, setMinTimeScale] = useState(); + const [maxTimeScale, setMaxTimeScale] = useState(); const onDragSelect = useCallback( (start: number, end: number): void => { @@ -62,16 +65,16 @@ function GridCardGraph({ } }, [toScrollWidgetId, setToScrollWidgetId, widget.id]); - const { minTime, maxTime, selectedTime: globalSelectedInterval } = useSelector< - AppState, - GlobalReducer - >((state) => state.globalTime); - const updatedQuery = useStepInterval(widget?.query); const isEmptyWidget = widget?.id === PANEL_TYPES.EMPTY_WIDGET || isEmpty(widget); + const { minTime, maxTime, selectedTime: globalSelectedInterval } = useSelector< + AppState, + GlobalReducer + >((state) => state.globalTime); + const queryResponse = useGetQueryRange( { selectedTime: widget?.timePreferance, @@ -103,6 +106,13 @@ function GridCardGraph({ const containerDimensions = useResizeObserver(graphRef); + useEffect((): void => { + const { startTime, endTime } = getTimeRange(queryResponse); + + setMinTimeScale(startTime); + setMaxTimeScale(endTime); + }, [maxTime, minTime, globalSelectedInterval, queryResponse]); + const chartData = getUPlotChartData(queryResponse?.data?.payload, fillSpans); const isDarkMode = useIsDarkMode(); @@ -123,6 +133,8 @@ function GridCardGraph({ yAxisUnit: widget?.yAxisUnit, onClickHandler, thresholds: widget.thresholds, + minTimeScale, + maxTimeScale, }), [ widget?.id, @@ -133,6 +145,8 @@ function GridCardGraph({ isDarkMode, onDragSelect, onClickHandler, + minTimeScale, + maxTimeScale, ], ); diff --git a/frontend/src/container/GridCardLayout/GridCardLayout.styles.scss b/frontend/src/container/GridCardLayout/GridCardLayout.styles.scss index 08de391e4c..ab90890541 100644 --- a/frontend/src/container/GridCardLayout/GridCardLayout.styles.scss +++ b/frontend/src/container/GridCardLayout/GridCardLayout.styles.scss @@ -5,3 +5,11 @@ border: none !important; } } + +.widget-graph-container { + height: 100%; + + &.graph { + height: calc(100% - 30px); + } +} diff --git a/frontend/src/container/GridCardLayout/WidgetHeader/WidgetHeader.styles.scss b/frontend/src/container/GridCardLayout/WidgetHeader/WidgetHeader.styles.scss index e03f176570..40d138e7df 100644 --- a/frontend/src/container/GridCardLayout/WidgetHeader/WidgetHeader.styles.scss +++ b/frontend/src/container/GridCardLayout/WidgetHeader/WidgetHeader.styles.scss @@ -2,9 +2,12 @@ display: flex; justify-content: space-between; align-items: center; - height: 30px; + height: 40px; width: 100%; padding: 0.5rem; + box-sizing: border-box; + font-size: 14px; + font-weight: 600; } .widget-header-title { @@ -19,6 +22,10 @@ visibility: hidden; border: none; box-shadow: none; + cursor: pointer; + font: 14px; + font-weight: 600; + padding: 8px; } .widget-header-hover { diff --git a/frontend/src/container/GridCardLayout/WidgetHeader/index.tsx b/frontend/src/container/GridCardLayout/WidgetHeader/index.tsx index 0fb2f90ead..b70e16585c 100644 --- a/frontend/src/container/GridCardLayout/WidgetHeader/index.tsx +++ b/frontend/src/container/GridCardLayout/WidgetHeader/index.tsx @@ -10,7 +10,7 @@ import { MoreOutlined, WarningOutlined, } from '@ant-design/icons'; -import { Button, Dropdown, MenuProps, Tooltip, Typography } from 'antd'; +import { Dropdown, MenuProps, Tooltip, Typography } from 'antd'; import Spinner from 'components/Spinner'; import { QueryParams } from 'constants/query'; import { PANEL_TYPES } from 'constants/queryBuilder'; @@ -199,9 +199,7 @@ function WidgetHeader({ )} - @@ -157,7 +278,9 @@ function VariablesSetting(): JSX.Element { type="text" style={{ padding: 8, color: red[6], cursor: 'pointer' }} onClick={(): void => { - if (_.name) onVariableDeleteHandler(_.name); + if (variable) { + onVariableDeleteHandler(variable); + } }} > @@ -167,6 +290,51 @@ function VariablesSetting(): JSX.Element { }, ]; + const sensors = useSensors( + useSensor(PointerSensor, { + activationConstraint: { + // https://docs.dndkit.com/api-documentation/sensors/pointer#activation-constraints + distance: 1, + }, + }), + ); + + const onDragEnd = ({ active, over }: DragEndEvent): void => { + if (active.id !== over?.id) { + const activeIndex = variablesTableData.findIndex( + (i: { key: UniqueIdentifier }) => i.key === active.id, + ); + const overIndex = variablesTableData.findIndex( + (i: { key: UniqueIdentifier | undefined }) => i.key === over?.id, + ); + + const updatedVariables: IDashboardVariable[] = arrayMove( + variablesTableData, + activeIndex, + overIndex, + ); + + const reArrangedVariables = {}; + + for (let index = 0; index < updatedVariables.length; index += 1) { + const variableName = updatedVariables[index].name; + + if (variableName) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + reArrangedVariables[variableName] = { + ...updatedVariables[index], + order: index, + }; + } + } + + updateVariables(reArrangedVariables); + + setVariablesTableData(updatedVariables); + } + }; + return ( <> {variableViewMode ? ( @@ -176,11 +344,17 @@ function VariablesSetting(): JSX.Element { onSave={onVariableSaveHandler} onCancel={onDoneVariableViewMode} validateName={validateVariableName} - variableViewMode={variableViewMode} + mode={variableViewMode} /> ) : ( <> - +