Render flexible plots

extension/src/cli/dvc/constants.ts

@@ -1,4 +1,4 @@
-export const MIN_CLI_VERSION = '2.24.0'
+export const MIN_CLI_VERSION = '2.0.0'
 export const LATEST_TESTED_CLI_VERSION = '2.26.2'
 export const MAX_CLI_VERSION = '3'
 

extension/src/plots/model/collect.ts

@@ -33,6 +33,8 @@ import { TemplateOrder } from '../paths/collect'
 import { extendVegaSpec, isMultiViewPlot } from '../vega/util'
 import { definedAndNonEmpty, splitMatchedOrdered } from '../../util/array'
 import { shortenForLabel } from '../../util/string'
+import { getDvcDataVersionInfo, joinFields } from '../multiSource/collect'
+import { ShapeEncoding, StrokeDashEncoding } from '../multiSource/constants'
 
 type CheckpointPlotAccumulator = {
   iterations: Record<string, number>
@@ -355,7 +357,14 @@ const collectDatapoints = (
   values: Record<string, unknown>[] = []
 ) => {
   for (const value of values) {
-    ;(acc[rev][path] as unknown[]).push({ ...value, rev })
+    const dvc_data_version_info = getDvcDataVersionInfo(value)
+    const data: { rev: string } = {
+      ...value,
+      ...dvc_data_version_info,
+      rev
+    }
+
+    ;(acc[rev][path] as unknown[]).push(data)
   }
 }
 
@@ -487,16 +496,46 @@ export const collectTemplates = (data: PlotsOutput): TemplateAccumulator => {
   return acc
 }
 
-const fillTemplate = (template: string, datapoints: unknown[]) =>
-  JSON.parse(
-    template.replace('"<DVC_METRIC_DATA>"', JSON.stringify(datapoints))
+const fillTemplate = (
+  template: string,
+  datapoints: unknown[],
+  field?: string
+) => {
+  if (!field?.includes('::')) {
+    return JSON.parse(
+      template.replace('"<DVC_METRIC_DATA>"', JSON.stringify(datapoints))
+    ) as TopLevelSpec
+  }
+
+  const mapping = field.split('::')
+  return JSON.parse(
+    template.replace(
+      '"<DVC_METRIC_DATA>"',
+      JSON.stringify(
+        datapoints.map(data => {
+          const obj = data as Record<string, unknown>
+          return {
+            ...obj,
+            [field]: joinFields(mapping.map(field => obj[field] as string))
+          }
+        })
+      )
+    )
   ) as TopLevelSpec
+}
 
 const collectTemplateGroup = (
   paths: string[],
   selectedRevisions: string[],
   templates: TemplateAccumulator,
   revisionData: RevisionData,
+  scales: Record<
+    string,
+    {
+      strokeDash?: StrokeDashEncoding
+      shape?: ShapeEncoding
+    }
+  >,
   size: PlotSize,
   revisionColors: ColorScale | undefined
 ): TemplatePlotEntry[] => {
@@ -509,10 +548,15 @@ const collectTemplateGroup = (
         .flatMap(revision => revisionData?.[revision]?.[path])
         .filter(Boolean)
 
+      const scale = scales[path] || {}
+
       const content = extendVegaSpec(
-        fillTemplate(template, datapoints),
+        fillTemplate(template, datapoints, scale.strokeDash?.field),
         size,
-        revisionColors
+        {
+          ...scale,
+          color: revisionColors
+        }
       )
 
       acc.push({
@@ -532,6 +576,13 @@ export const collectSelectedTemplatePlots = (
   selectedRevisions: string[],
   templates: TemplateAccumulator,
   revisionData: RevisionData,
+  scales: Record<
+    string,
+    {
+      strokeDash?: StrokeDashEncoding
+      shape?: ShapeEncoding
+    }
+  >,
   size: PlotSize,
   revisionColors: ColorScale | undefined
 ): TemplatePlotSection[] | undefined => {
@@ -543,6 +594,7 @@ export const collectSelectedTemplatePlots = (
       selectedRevisions,
       templates,
       revisionData,
+      scales,
       size,
       revisionColors
     )

extension/src/plots/model/index.ts

@@ -33,6 +33,11 @@ import { removeMissingKeysFromObject } from '../../util/object'
 import { TemplateOrder } from '../paths/collect'
 import { PersistenceKey } from '../../persistence/constants'
 import { ModelWithPersistence } from '../../persistence/model'
+import {
+  collectMultiSourceData,
+  collectMultiSourceVariations
+} from '../multiSource/collect'
+import { ShapeEncoding, StrokeDashEncoding } from '../multiSource/constants'
 
 export class PlotsModel extends ModelWithPersistence {
   private readonly experiments: Experiments
@@ -49,6 +54,15 @@ export class PlotsModel extends ModelWithPersistence {
 
   private revisionData: RevisionData = {}
   private templates: TemplateAccumulator = {}
+  private multiSourceData: Record<
+    string,
+    { filename?: string; field?: string }[]
+  > = {}
+
+  private scales: Record<
+    string,
+    { strokeDash?: StrokeDashEncoding; shape?: ShapeEncoding }
+  > = {}
 
   private checkpointPlots?: CheckpointPlot[]
   private selectedMetrics?: string[]
@@ -104,10 +118,12 @@ export class PlotsModel extends ModelWithPersistence {
       ...revs.map(rev => cliIdToLabel[rev])
     ])
 
-    const [{ comparisonData, revisionData }, templates] = await Promise.all([
-      collectData(data, cliIdToLabel),
-      collectTemplates(data)
-    ])
+    const [{ comparisonData, revisionData }, templates, multiSourceData] =
+      await Promise.all([
+        collectData(data, cliIdToLabel),
+        collectTemplates(data),
+        collectMultiSourceVariations(data, this.multiSourceData)
+      ])
 
     const { overwriteComparisonData, overwriteRevisionData } =
       collectWorkspaceRaceConditionData(
@@ -127,6 +143,8 @@ export class PlotsModel extends ModelWithPersistence {
       ...overwriteRevisionData
     }
     this.templates = { ...this.templates, ...templates }
+    this.multiSourceData = multiSourceData
+    this.scales = collectMultiSourceData(this.multiSourceData)
 
     this.setComparisonOrder()
 
@@ -421,6 +439,7 @@ export class PlotsModel extends ModelWithPersistence {
       selectedRevisions,
       this.templates,
       this.revisionData,
+      this.scales,
       this.getPlotSize(Section.TEMPLATE_PLOTS),
       this.getRevisionColors()
     )

extension/src/plots/multiSource/collect.test.ts

@@ -0,0 +1,168 @@
+import { join } from 'path'
+import { collectMultiSourceData } from './collect'
+
+describe('collectMultiSourceData', () => {
+  it('should return an empty object given no multi source data', () => {
+    const multiSourceKeys = collectMultiSourceData({
+      path: [{ field: 'x', filename: 'path' }]
+    })
+    expect(multiSourceKeys).toStrictEqual({})
+  })
+
+  it('should return an object containing a filename strokeDash given data with varying filenames', () => {
+    const otherPath = join('other', 'path')
+    const multiSourceKeys = collectMultiSourceData({
+      combined: [
+        { field: 'x', filename: 'path' },
+        { field: 'x', filename: otherPath }
+      ]
+    })
+    expect(multiSourceKeys).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename',
+          scale: {
+            domain: [otherPath, 'path'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a field strokeDash given data with varying fields', () => {
+    const multiSourceKeys = collectMultiSourceData({
+      path: [
+        { field: 'x', filename: 'path' },
+        { field: 'z', filename: 'path' }
+      ]
+    })
+    expect(multiSourceKeys).toStrictEqual({
+      path: {
+        strokeDash: {
+          field: 'field',
+          scale: {
+            domain: ['x', 'z'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a filename::field strokeDash given data with varying filename and fields', () => {
+    const otherPath = join('other', 'path')
+    const multiSourceKeys = collectMultiSourceData({
+      combined: [
+        { field: 'x', filename: 'path' },
+        { field: 'z', filename: otherPath }
+      ]
+    })
+    expect(multiSourceKeys).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename::field',
+          scale: {
+            domain: [`${otherPath}::z`, 'path::x'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a filename::field strokeDash given data with varying filename and similar field', () => {
+    const multiSourceKeys = collectMultiSourceData({
+      combined: [
+        { field: 'x', filename: join('first', 'path') },
+        { field: 'z', filename: join('second', 'path') },
+        { field: 'z', filename: join('third', 'path') }
+      ]
+    })
+    expect(multiSourceKeys).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename::field',
+          scale: {
+            domain: [
+              `${join('first', 'path')}::x`,
+              `${join('second', 'path')}::z`,
+              `${join('third', 'path')}::z`
+            ],
+            range: [
+              [1, 0],
+              [8, 8],
+              [8, 4]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a filename::field strokeDash given data with different filename and field for each variation', () => {
+    const multiSourceKeys = collectMultiSourceData({
+      combined: [
+        { field: 'x', filename: join('first', 'path') },
+        { field: 'z', filename: join('second', 'path') },
+        { field: 'q', filename: join('third', 'path') }
+      ]
+    })
+    expect(multiSourceKeys).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename::field',
+          scale: {
+            domain: [
+              `${join('first', 'path')}::x`,
+              `${join('second', 'path')}::z`,
+              `${join('third', 'path')}::q`
+            ],
+            range: [
+              [1, 0],
+              [8, 8],
+              [8, 4]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a filename strokeDash and field shape given data with differently varying filename and field', () => {
+    const multiSourceKeys = collectMultiSourceData({
+      combined: [
+        { field: 'x', filename: 'path' },
+        { field: 'z', filename: 'path' },
+        { field: 'z', filename: join('other', 'path') }
+      ]
+    })
+    expect(multiSourceKeys).toStrictEqual({
+      combined: {
+        shape: {
+          field: 'field',
+          scale: { domain: ['x', 'z'], range: ['square', 'circle'] }
+        },
+        strokeDash: {
+          field: 'filename',
+          scale: {
+            domain: [join('other', 'path'), 'path'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+})