Remove total field in Log and Trace query (#9094)

docs/en/changes/changes.md

@@ -59,6 +59,7 @@
 * Storage(ElasticSearch): add search options to tolerate inexisting indices.
 * Fix the problem that `MQ` has the wrong `Layer` type.
 * Fix NoneStream model has wrong downsampling(was Second, should be Minute).
+* [Breaking Change] Remove `total` field in Trace and Logs list query.
 
 #### UI
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/storage/EBPFProfilingTargetType.java

@@ -33,6 +33,8 @@ public enum EBPFProfilingTargetType {
     UNKNOWN(0),
 
     ON_CPU(1),
+
+    OFF_CPU(2),
     ;
     private final int value;
     private static final Map<Integer, EBPFProfilingTargetType> DICTIONARY = new HashMap<>();

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/EBPFProfilingAnalyzeAggregateType.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.query.type;
+
+/**
+ * EBPF Profiling analysis data aggregate type
+ */
+public enum EBPFProfilingAnalyzeAggregateType {
+    /**
+     * Aggregate by the total duration of stack
+     * For "OFF_CPU" target type of profiling: Statics the total time spent in off cpu.
+     */
+    DURATION,
+    /**
+     * Aggregate by the trigger count
+     * For "ON_CPU" target type of profiling: Statics the number of dump count.
+     * For "OFF_CPU" target type of profiling: Statics the number of times the process is switched to off cpu by the scheduler.
+     */
+    COUNT
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Logs.java

@@ -27,7 +27,6 @@
 @Getter
 public class Logs {
     private final List<Log> logs;
-    private int total;
 
     public Logs() {
         this.logs = new ArrayList<>();

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java

@@ -30,5 +30,4 @@
 public class Pagination {
     private int pageNum;
     private int pageSize;
-    private boolean needTotal;
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/TraceBrief.java

@@ -21,13 +21,10 @@
 import java.util.ArrayList;
 import java.util.List;
 import lombok.Getter;
-import lombok.Setter;
 
 @Getter
 public class TraceBrief {
     private final List<BasicTrace> traces;
-    @Setter
-    private int total;
 
     public TraceBrief() {
         this.traces = new ArrayList<>();

oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/AlarmQuery.java

@@ -89,7 +89,7 @@ public Alarms getAlarm(final Duration duration, final Scope scope, final String
         long endSecondTB = 0;
         final EventQueryCondition.EventQueryConditionBuilder conditionPrototype =
             EventQueryCondition.builder()
-                               .paging(new Pagination(1, IEventQueryDAO.MAX_SIZE, false));
+                               .paging(new Pagination(1, IEventQueryDAO.MAX_SIZE));
         if (nonNull(duration)) {
             startSecondTB = duration.getStartTimeBucketInSec();
             endSecondTB = duration.getEndTimeBucketInSec();

oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/EBPFProcessProfilingQuery.java

@@ -22,6 +22,7 @@
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingQueryService;
 import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingAnalyzation;
+import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingAnalyzeAggregateType;
 import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingAnalyzeTimeRange;
 import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingSchedule;
 import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTask;
@@ -68,7 +69,9 @@ public List<EBPFProfilingSchedule> queryEBPFProfilingSchedules(String taskId) th
         return getQueryService().queryEBPFProfilingSchedules(taskId);
     }
 
-    public EBPFProfilingAnalyzation analysisEBPFProfilingResult(List<String> scheduleIdList, List<EBPFProfilingAnalyzeTimeRange> timeRanges) throws IOException {
+    public EBPFProfilingAnalyzation analysisEBPFProfilingResult(List<String> scheduleIdList,
+                                                                List<EBPFProfilingAnalyzeTimeRange> timeRanges,
+                                                                EBPFProfilingAnalyzeAggregateType aggregateType) throws IOException {
         return getQueryService().getEBPFProfilingAnalyzation(scheduleIdList, timeRanges);
     }
 }

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBLogQueryDAO.java

@@ -115,7 +115,6 @@ public void apply(StreamQuery query) {
         StreamQueryResponse resp = query(LogRecord.INDEX_NAME, TAGS, tsRange, query);
 
         Logs logs = new Logs();
-        logs.setTotal(resp.size());
 
         for (final RowEntity rowEntity : resp.getElements()) {
             Log log = new Log();

oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBTraceQueryDAO.java

@@ -139,7 +139,6 @@ public void apply(StreamQuery query) {
                 tsRange, q);
 
         TraceBrief traceBrief = new TraceBrief();
-        traceBrief.setTotal(resp.size());
 
         if (resp.size() == 0) {
             return traceBrief;

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java

@@ -141,7 +141,6 @@ public Logs queryLogs(final String serviceId,
         SearchResponse response = getClient().search(index, search.build());
 
         Logs logs = new Logs();
-        logs.setTotal(response.getHits().getTotal());
 
         for (SearchHit searchHit : response.getHits().getHits()) {
             Log log = new Log();

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java

@@ -133,7 +133,6 @@ public TraceBrief queryBasicTraces(long startSecondTB,
                 endSecondTB
             ), search.build());
         final TraceBrief traceBrief = new TraceBrief();
-        traceBrief.setTotal(response.getHits().getTotal());
 
         for (SearchHit searchHit : response.getHits().getHits()) {
             BasicTrace basicTrace = new BasicTrace();

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2LogQueryDAO.java

@@ -162,13 +162,6 @@ public Logs queryLogs(String serviceId,
         Logs logs = new Logs();
         try (Connection connection = h2Client.getConnection()) {
 
-            try (ResultSet resultSet = h2Client.executeQuery(connection, buildCountStatement(sql.toString()), parameters
-                .toArray(new Object[0]))) {
-                while (resultSet.next()) {
-                    logs.setTotal(resultSet.getInt("total"));
-                }
-            }
-
             buildLimit(sql, from, limit);
 
             try (ResultSet resultSet = h2Client.executeQuery(
@@ -199,10 +192,6 @@ public Logs queryLogs(String serviceId,
         return logs;
     }
 
-    protected String buildCountStatement(String sql) {
-        return "select count(1) total from (select 1 " + sql + " )";
-    }
-
     protected void buildLimit(StringBuilder sql, int from, int limit) {
         sql.append(" LIMIT ").append(limit);
         sql.append(" OFFSET ").append(from);

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TraceQueryDAO.java

@@ -162,13 +162,6 @@ public TraceBrief queryBasicTraces(long startSecondTB,
         TraceBrief traceBrief = new TraceBrief();
         try (Connection connection = h2Client.getConnection()) {
 
-            try (ResultSet resultSet = h2Client.executeQuery(connection, buildCountStatement(sql.toString()), parameters
-                .toArray(new Object[0]))) {
-                while (resultSet.next()) {
-                    traceBrief.setTotal(resultSet.getInt("total"));
-                }
-            }
-
             buildLimit(sql, from, limit);
 
             try (ResultSet resultSet = h2Client.executeQuery(
@@ -202,10 +195,6 @@ public TraceBrief queryBasicTraces(long startSecondTB,
         return traceBrief;
     }
 
-    protected String buildCountStatement(String sql) {
-        return "select count(1) total from (select 1 " + sql + " )";
-    }
-
     protected void buildLimit(StringBuilder sql, int from, int limit) {
         sql.append(" LIMIT ").append(limit);
         sql.append(" OFFSET ").append(from);

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLLogQueryDAO.java

@@ -30,11 +30,6 @@ public MySQLLogQueryDAO(final JDBCHikariCPClient h2Client,
         super(h2Client, manager, maxSizeOfArrayColumn, numOfSearchValuesPerTag);
     }
 
-    @Override
-    protected String buildCountStatement(String sql) {
-        return "select count(1) total " + sql;
-    }
-
     @Override
     protected void buildLimit(StringBuilder sql, int from, int limit) {
         sql.append(" LIMIT ").append(from).append(", ").append(limit);

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLTraceQueryDAO.java

@@ -31,11 +31,6 @@ public MySQLTraceQueryDAO(ModuleManager manager,
         super(manager, h2Client, maxSizeOfArrayColumn, numOfSearchableValuesPerTag);
     }
 
-    @Override
-    protected String buildCountStatement(String sql) {
-        return "select count(1) total " + sql;
-    }
-
     @Override
     protected void buildLimit(StringBuilder sql, int from, int limit) {
         sql.append(" LIMIT ").append(from).append(", ").append(limit);

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/postgresql/dao/PostgreSQLLogQueryDAO.java

@@ -29,9 +29,4 @@ public PostgreSQLLogQueryDAO(JDBCHikariCPClient h2Client, ModuleManager manager,
         super(h2Client, manager, maxSizeOfArrayColumn, numOfSearchValuesPerTag);
     }
 
-    @Override
-    protected String buildCountStatement(String sql) {
-        return "select count(*) total from (select 1 " + sql + " ) temp_table ";
-    }
-
 }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/postgresql/dao/PostgreSQLTraceQueryDAO.java

@@ -29,9 +29,4 @@ public PostgreSQLTraceQueryDAO(ModuleManager manager, JDBCHikariCPClient h2Clien
         super(manager, h2Client, maxSizeOfArrayColumn, numOfSearchableValuesPerTag);
     }
 
-    @Override
-    protected String buildCountStatement(String sql) {
-        return "select count(*) total from (select 1 " + sql + " ) temp_table ";
-    }
-
 }

oap-server/server-storage-plugin/storage-zipkin-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/zipkin/elasticsearch/ZipkinTraceQueryEsDAO.java

@@ -142,7 +142,6 @@ public TraceBrief queryBasicTraces(long startSecondTB,
         SearchResponse response = getClient().search(ZipkinSpanRecord.INDEX_NAME, search.build());
 
         TraceBrief traceBrief = new TraceBrief();
-        traceBrief.setTotal(response.getHits().getTotal());
 
         for (SearchHit searchHit : response.getHits().getHits()) {
             BasicTrace basicTrace = new BasicTrace();

test/e2e-v2/cases/browser/expected/error-log.yml

@@ -27,4 +27,4 @@ logs:
   errorurl: http://testui/
   firstreportederror: {{ .firstreportederror }}
 {{- end }}
-total: {{ gt .total 0 }}
+total: {{ gt .total 0 }}

test/e2e-v2/cases/browser/expected/traces.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
   - {{ notEmpty (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/gateway/expected/traces-list.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ index .traceids 0 }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/go/expected/traces-list-consumer.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/go/expected/traces-list-go.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/go/expected/traces-list-provider.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/kafka/log/expected/logs-filebeat.yml

@@ -109,4 +109,3 @@ logs:
     value: {{ notEmpty .value }}
   {{- end }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/kafka/log/expected/logs-fluentd.yml

@@ -79,4 +79,3 @@ logs:
     value: fluentd
   {{- end }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/log/expected/logs.yml

@@ -35,5 +35,4 @@ logs:
   - key: thread
     value: {{ notEmpty .value }}
   {{- end }}
-{{- end }}
-total: {{ gt .total 0 }}
+{{- end }}

test/e2e-v2/cases/lua/expected/traces-list.yml

@@ -23,5 +23,4 @@ traces:
   iserror: false
   traceids:
     - {{ (index .traceids 0) }}
-{{- end }}
-total: {{ gt .total 0 }}
+{{- end }}

test/e2e-v2/cases/nodejs/expected/traces-list.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/php/expected/traces-list-php.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/php/expected/traces-list-provider.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/profiling/trace/expected/traces-list.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/python/expected/traces-list-consumer-java.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/python/expected/traces-list-consumer-py.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/python/expected/traces-list-provider-py-kafka.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/python/expected/traces-list-provider-py.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/satellite/native-protocols/expected/logs-list.yml

@@ -36,4 +36,3 @@ logs:
       value: {{ notEmpty .value }}
     {{- end }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/satellite/native-protocols/expected/traces-list.yml

@@ -26,4 +26,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/simple/expected/traces-list.yml

@@ -24,4 +24,3 @@ traces:
   traceids:
     - {{ index .traceids 0 }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/cases/storage/expected/traces-list.yml

@@ -26,4 +26,3 @@ traces:
   traceids:
     - {{ (index .traceids 0) }}
 {{- end }}
-total: {{ gt .total 0 }}

test/e2e-v2/script/env

@@ -24,4 +24,4 @@ SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
 SW_KUBERNETES_COMMIT_SHA=0f3ec68e5a7e1608cec8688716b848ed15e971e5
 SW_ROVER_COMMIT=90c93c706743aac1f5853b677730edae8cc32a2c
 
-SW_CTL_COMMIT=03dbdcf8cecc3abdef661efaa5734c01ac49adea
+SW_CTL_COMMIT=48238c6d380df52b46bd6f67798ff572fb5723ed