Render flexible plots

.vscode/settings.json

@@ -45,6 +45,7 @@
     "unaddable",
     "uncommit",
     "uniqwith",
+    "unmerge",
     "unprotect",
     "unshallow",
     "unstage",

extension/src/plots/model/collect.ts

@@ -33,6 +33,13 @@ import { TemplateOrder } from '../paths/collect'
 import { extendVegaSpec, isMultiViewPlot } from '../vega/util'
 import { definedAndNonEmpty, splitMatchedOrdered } from '../../util/array'
 import { shortenForLabel } from '../../util/string'
+import {
+  getDvcDataVersionInfo,
+  isConcatenatedField,
+  mergeFields,
+  MultiSourceEncoding,
+  unmergeConcatenatedFields
+} from '../multiSource/collect'
 
 type CheckpointPlotAccumulator = {
   iterations: Record<string, number>
@@ -355,7 +362,14 @@ const collectDatapoints = (
   values: Record<string, unknown>[] = []
 ) => {
   for (const value of values) {
-    ;(acc[rev][path] as unknown[]).push({ ...value, rev })
+    const dvc_data_version_info = getDvcDataVersionInfo(value)
+    const data: { rev: string } = {
+      ...value,
+      ...dvc_data_version_info,
+      rev
+    }
+
+    ;(acc[rev][path] as unknown[]).push(data)
   }
 }
 
@@ -487,18 +501,42 @@ export const collectTemplates = (data: PlotsOutput): TemplateAccumulator => {
   return acc
 }
 
-const fillTemplate = (template: string, datapoints: unknown[]) =>
-  JSON.parse(
-    template.replace('"<DVC_METRIC_DATA>"', JSON.stringify(datapoints))
+const fillTemplate = (
+  template: string,
+  datapoints: unknown[],
+  field?: string
+) => {
+  if (!field || !isConcatenatedField(field)) {
+    return JSON.parse(
+      template.replace('"<DVC_METRIC_DATA>"', JSON.stringify(datapoints))
+    ) as TopLevelSpec
+  }
+
+  const fields = unmergeConcatenatedFields(field)
+  return JSON.parse(
+    template.replace(
+      '"<DVC_METRIC_DATA>"',
+      JSON.stringify(
+        datapoints.map(data => {
+          const obj = data as Record<string, unknown>
+          return {
+            ...obj,
+            [field]: mergeFields(fields.map(field => obj[field] as string))
+          }
+        })
+      )
+    )
   ) as TopLevelSpec
+}
 
 const collectTemplateGroup = (
   paths: string[],
   selectedRevisions: string[],
   templates: TemplateAccumulator,
   revisionData: RevisionData,
   size: PlotSize,
-  revisionColors: ColorScale | undefined
+  revisionColors: ColorScale | undefined,
+  multiSourceEncoding: MultiSourceEncoding
 ): TemplatePlotEntry[] => {
   const acc: TemplatePlotEntry[] = []
   for (const path of paths) {
@@ -509,10 +547,19 @@ const collectTemplateGroup = (
         .flatMap(revision => revisionData?.[revision]?.[path])
         .filter(Boolean)
 
+      const multiSourceEncodingUpdate = multiSourceEncoding[path] || {}
+
       const content = extendVegaSpec(
-        fillTemplate(template, datapoints),
+        fillTemplate(
+          template,
+          datapoints,
+          multiSourceEncodingUpdate.strokeDash?.field
+        ),
         size,
-        revisionColors
+        {
+          ...multiSourceEncodingUpdate,
+          color: revisionColors
+        }
       )
 
       acc.push({
@@ -533,7 +580,8 @@ export const collectSelectedTemplatePlots = (
   templates: TemplateAccumulator,
   revisionData: RevisionData,
   size: PlotSize,
-  revisionColors: ColorScale | undefined
+  revisionColors: ColorScale | undefined,
+  multiSourceEncoding: MultiSourceEncoding
 ): TemplatePlotSection[] | undefined => {
   const acc: TemplatePlotSection[] = []
   for (const templateGroup of order) {
@@ -544,7 +592,8 @@ export const collectSelectedTemplatePlots = (
       templates,
       revisionData,
       size,
-      revisionColors
+      revisionColors,
+      multiSourceEncoding
     )
     if (!definedAndNonEmpty(entries)) {
       continue

extension/src/plots/model/index.ts

@@ -33,6 +33,12 @@ import { removeMissingKeysFromObject } from '../../util/object'
 import { TemplateOrder } from '../paths/collect'
 import { PersistenceKey } from '../../persistence/constants'
 import { ModelWithPersistence } from '../../persistence/model'
+import {
+  collectMultiSourceEncoding,
+  collectMultiSourceVariations,
+  MultiSourceEncoding,
+  MultiSourceVariations
+} from '../multiSource/collect'
 
 export class PlotsModel extends ModelWithPersistence {
   private readonly experiments: Experiments
@@ -49,6 +55,8 @@ export class PlotsModel extends ModelWithPersistence {
 
   private revisionData: RevisionData = {}
   private templates: TemplateAccumulator = {}
+  private multiSourceVariations: MultiSourceVariations = {}
+  private multiSourceEncoding: MultiSourceEncoding = {}
 
   private checkpointPlots?: CheckpointPlot[]
   private selectedMetrics?: string[]
@@ -104,10 +112,12 @@ export class PlotsModel extends ModelWithPersistence {
       ...revs.map(rev => cliIdToLabel[rev])
     ])
 
-    const [{ comparisonData, revisionData }, templates] = await Promise.all([
-      collectData(data, cliIdToLabel),
-      collectTemplates(data)
-    ])
+    const [{ comparisonData, revisionData }, templates, multiSourceVariations] =
+      await Promise.all([
+        collectData(data, cliIdToLabel),
+        collectTemplates(data),
+        collectMultiSourceVariations(data, this.multiSourceVariations)
+      ])
 
     const { overwriteComparisonData, overwriteRevisionData } =
       collectWorkspaceRaceConditionData(
@@ -127,6 +137,10 @@ export class PlotsModel extends ModelWithPersistence {
       ...overwriteRevisionData
     }
     this.templates = { ...this.templates, ...templates }
+    this.multiSourceVariations = multiSourceVariations
+    this.multiSourceEncoding = collectMultiSourceEncoding(
+      this.multiSourceVariations
+    )
 
     this.setComparisonOrder()
 
@@ -422,7 +436,8 @@ export class PlotsModel extends ModelWithPersistence {
       this.templates,
       this.revisionData,
       this.getPlotSize(Section.TEMPLATE_PLOTS),
-      this.getRevisionColors()
+      this.getRevisionColors(),
+      this.multiSourceEncoding
     )
   }
 }

extension/src/plots/multiSource/collect.test.ts

@@ -0,0 +1,168 @@
+import { join } from 'path'
+import { collectMultiSourceEncoding } from './collect'
+
+describe('collectMultiSourceEncoding', () => {
+  it('should return an empty object given a single variation collected from the datapoints', () => {
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      path: [{ field: 'x', filename: 'path' }]
+    })
+    expect(multiSourceEncoding).toStrictEqual({})
+  })
+
+  it('should return an object containing a filename strokeDash given variations with differing filenames', () => {
+    const otherPath = join('other', 'path')
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      combined: [
+        { field: 'x', filename: 'path' },
+        { field: 'x', filename: otherPath }
+      ]
+    })
+    expect(multiSourceEncoding).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename',
+          scale: {
+            domain: [otherPath, 'path'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a field strokeDash given variations with differing fields', () => {
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      path: [
+        { field: 'x', filename: 'path' },
+        { field: 'z', filename: 'path' }
+      ]
+    })
+    expect(multiSourceEncoding).toStrictEqual({
+      path: {
+        strokeDash: {
+          field: 'field',
+          scale: {
+            domain: ['x', 'z'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a merged filename::field strokeDash given variations with differing filename and fields', () => {
+    const otherPath = join('other', 'path')
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      combined: [
+        { field: 'x', filename: 'path' },
+        { field: 'z', filename: otherPath }
+      ]
+    })
+    expect(multiSourceEncoding).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename::field',
+          scale: {
+            domain: [`${otherPath}::z`, 'path::x'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a merged filename::field strokeDash given variations with differing filename and similar field', () => {
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      combined: [
+        { field: 'x', filename: join('first', 'path') },
+        { field: 'z', filename: join('second', 'path') },
+        { field: 'z', filename: join('third', 'path') }
+      ]
+    })
+    expect(multiSourceEncoding).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename::field',
+          scale: {
+            domain: [
+              `${join('first', 'path')}::x`,
+              `${join('second', 'path')}::z`,
+              `${join('third', 'path')}::z`
+            ],
+            range: [
+              [1, 0],
+              [8, 8],
+              [8, 4]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a merged filename::field strokeDash given variations with differing filename and field for each variation', () => {
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      combined: [
+        { field: 'x', filename: join('first', 'path') },
+        { field: 'z', filename: join('second', 'path') },
+        { field: 'q', filename: join('third', 'path') }
+      ]
+    })
+    expect(multiSourceEncoding).toStrictEqual({
+      combined: {
+        strokeDash: {
+          field: 'filename::field',
+          scale: {
+            domain: [
+              `${join('first', 'path')}::x`,
+              `${join('second', 'path')}::z`,
+              `${join('third', 'path')}::q`
+            ],
+            range: [
+              [1, 0],
+              [8, 8],
+              [8, 4]
+            ]
+          }
+        }
+      }
+    })
+  })
+
+  it('should return an object containing a filename strokeDash and field shape given variations with unmergable combinations of filename and field', () => {
+    const multiSourceEncoding = collectMultiSourceEncoding({
+      combined: [
+        { field: 'x', filename: 'path' },
+        { field: 'z', filename: 'path' },
+        { field: 'z', filename: join('other', 'path') }
+      ]
+    })
+    expect(multiSourceEncoding).toStrictEqual({
+      combined: {
+        shape: {
+          field: 'field',
+          scale: { domain: ['x', 'z'], range: ['square', 'circle'] }
+        },
+        strokeDash: {
+          field: 'filename',
+          scale: {
+            domain: [join('other', 'path'), 'path'],
+            range: [
+              [1, 0],
+              [8, 8]
+            ]
+          }
+        }
+      }
+    })
+  })
+})