Add query string filtering to Visualizations (#3182)

* Update TimelineSearch
* Add event selection to VisualizationEditor
* Add placeholder image
* Update aggregator
* Adding an API endpoint to get fields per timeline.
* Provide only fields that exist in all selected timelines for the aggregation.
* Minor UI fix for the EventList
* Dynamically sized the placeholder picture

---------

Co-authored-by: Syd Pleno <sydp@google.com>
Co-authored-by: Janosch <99879757+jkppr@users.noreply.github.com>

timesketch/api/v1/resources/timeline.py

@@ -20,6 +20,7 @@
 import six
 
 import opensearchpy
+from flask import jsonify
 from flask import request
 from flask import abort
 from flask import current_app
@@ -40,6 +41,7 @@
 from timesketch.models.sketch import SearchIndex
 from timesketch.models.sketch import Sketch
 from timesketch.models.sketch import Timeline
+from timesketch.lib.aggregators import manager as aggregator_manager
 
 
 logger = logging.getLogger("timesketch.timeline_api")
@@ -531,3 +533,100 @@ def post(self):
         utils.update_sketch_last_activity(sketch)
 
         return self.to_json(searchindex, status_code=HTTP_STATUS_CODE_CREATED)
+
+
+# TODO(Issue 3200): Research more efficient ways to gather unique fields.
+class TimelineFieldsResource(resources.ResourceMixin, Resource):
+    """Resource to retrieve unique fields present in a timeline.
+
+    This resource aggregates data types within a timeline and then queries
+    OpenSearch to retrieve all unique fields present across those data types,
+    excluding default Timesketch fields.
+    """
+
+    @login_required
+    def get(self, sketch_id, timeline_id):
+        """Handles GET request to retrieve unique fields in a timeline.
+
+        Args:
+            sketch_id (int): The ID of the sketch.
+            timeline_id (int): The ID of the timeline.
+
+        Returns:
+            flask.wrappers.Response: A JSON response containing a list of
+                unique fields in the timeline, sorted alphabetically. Returns
+                an empty list if no fields are found or if there's an error.
+                Possible error codes: 400, 403, 404.
+        """
+
+        sketch = Sketch.get_with_acl(sketch_id)
+        if not sketch:
+            abort(HTTP_STATUS_CODE_NOT_FOUND, "No sketch found with this ID.")
+        if not sketch.has_permission(current_user, "read"):
+            abort(
+                HTTP_STATUS_CODE_FORBIDDEN,
+                "User does not have read access controls on sketch.",
+            )
+
+        timeline = Timeline.get_by_id(timeline_id)
+        if not timeline:
+            abort(HTTP_STATUS_CODE_NOT_FOUND, "No timeline found with this ID.")
+
+        # Check that this timeline belongs to the sketch
+        if timeline.sketch.id != sketch.id:
+            abort(
+                HTTP_STATUS_CODE_NOT_FOUND,
+                "The timeline does not belong to the sketch.",
+            )
+
+        index_name = timeline.searchindex.index_name
+        timeline_fields = set()
+
+        # 1. Get distinct data types for the timeline using aggregation
+        aggregator_name = "field_bucket"
+        aggregator_parameters = {
+            "field": "data_type",
+            "limit": "10000",  # Get all data types
+        }
+
+        agg_class = aggregator_manager.AggregatorManager.get_aggregator(aggregator_name)
+        if not agg_class:
+            abort(HTTP_STATUS_CODE_NOT_FOUND, f"Aggregator {aggregator_name} not found")
+
+        aggregator = agg_class(
+            sketch_id=sketch_id, indices=[index_name], timeline_ids=[timeline_id]
+        )
+        result_obj = aggregator.run(**aggregator_parameters)
+
+        if not result_obj:
+            abort(HTTP_STATUS_CODE_BAD_REQUEST, "Error running data type aggregation.")
+
+        data_types = sorted([bucket["data_type"] for bucket in result_obj.values])
+
+        # 2. For each data type, query for a single event to get fields
+        for data_type in data_types:
+            query_filter = {"indices": [timeline_id], "size": 1}
+
+            try:
+                result = self.datastore.search(
+                    sketch_id=sketch_id,
+                    query_string=f'data_type:"{data_type}"',
+                    query_filter=query_filter,
+                    query_dsl=None,
+                    indices=[index_name],
+                    timeline_ids=[timeline_id],
+                )
+            except ValueError as e:
+                abort(HTTP_STATUS_CODE_BAD_REQUEST, str(e))
+
+            if isinstance(result, dict) and result.get("hits", {}).get("hits", []):
+                event = result["hits"]["hits"][0]["_source"]
+                for field in event:
+                    if field not in [
+                        "datetime",
+                        "timestamp",
+                        "__ts_timeline_id",
+                    ]:
+                        timeline_fields.add(field)
+
+        return jsonify({"objects": sorted(list(timeline_fields))})

timesketch/api/v1/routes.py

@@ -56,6 +56,7 @@
 from .resources.explore import QueryResource
 from .resources.timeline import TimelineResource
 from .resources.timeline import TimelineListResource
+from .resources.timeline import TimelineFieldsResource
 from .resources.searchindex import SearchIndexListResource
 from .resources.searchindex import SearchIndexResource
 from .resources.session import SessionResource
@@ -167,6 +168,10 @@
         TimelineResource,
         "/sketches/<int:sketch_id>/timelines/<int:timeline_id>/",
     ),
+    (
+        TimelineFieldsResource,
+        "/sketches/<int:sketch_id>/timelines/<int:timeline_id>/fields/",
+    ),
     (SearchIndexListResource, "/searchindices/"),
     (SearchIndexResource, "/searchindices/<int:searchindex_id>/"),
     (

timesketch/frontend-ng/src/components/Analyzer/TimelineSearch.vue

@@ -92,8 +92,17 @@ export default {
     },
     analyzerTimelineId: {
       handler: function (id) {
-        if (id) this.selectedTimelines.push(id)
-        if (!id) this.selectedTimelines = []
+        if (Array.isArray(id)) {
+          this.selectedTimelines = id
+        } else {
+          if (id) {
+            this.selectedTimelines.push(id)
+          }
+          else {
+            this.selectedTimelines = []
+          }
+        }
+
       },
     },
   },

timesketch/frontend-ng/src/components/Explore/EventList.vue

@@ -30,12 +30,10 @@ limitations under the License.
       <p>
         <v-dialog v-model="saveSearchMenu" v-if="!disableSaveSearch" width="500">
           <template v-slot:activator="{ on, attrs }">
-            <div v-bind="attrs" v-on="on">
-              <v-btn small depressed>
-                <v-icon left small title="Save current search">mdi-content-save-outline</v-icon>
-                Save search
-              </v-btn>
-            </div>
+            <v-btn small depressed v-bind="attrs" v-on="on" title="Save Search">
+              <v-icon left small >mdi-content-save-outline</v-icon>
+              Save search
+            </v-btn>
           </template>
 
           <v-card class="pa-4">

timesketch/frontend-ng/src/components/Visualization/AggregationConfig.vue

@@ -20,6 +20,8 @@ limitations under the License.
         <TsEventFieldSelect
           :field="selectedField"
           @selectedField="selectedField = $event"
+          :timelineFields="timelineFields"
+          :loadingFields="loadingFields"
           :rules="[rules.required]"
           >
           </TsEventFieldSelect>
@@ -114,6 +116,14 @@ export default {
     splitByTimeline: {
       type: Boolean,
     },
+    timelineFields: {
+        type: Array,
+        default: () => [],
+    },
+    loadingFields: {
+        type: Boolean,
+        default: false
+    },
   },
   data() {
     return {