Give option to sort or filter by Created timestamp (#2293)

* add rough implementation for sorting and filtering by created

* start to refactor

* unit test date string comparison

* unit test is free text date

* unit test date filters

* better standardize dates before comparing

* support created being filterable and sortable in the experiments table

* make iso date tz aware

extension/src/experiments/columns/constants.ts

@@ -1,8 +1,11 @@
-import { ColumnType } from '../webview/contract'
+import { Column, ColumnType } from '../webview/contract'
 
-export const timestampColumn = {
+const type = ColumnType.TIMESTAMP
+
+export const timestampColumn: Column = {
   hasChildren: false,
   label: 'Created',
   path: 'Created',
-  type: ColumnType.TIMESTAMP
+  type,
+  types: [type]
 }

extension/src/experiments/columns/quickPick.ts

@@ -1,5 +1,4 @@
 import { ColumnLike } from './like'
-import { timestampColumn } from './constants'
 import { definedAndNonEmpty } from '../../util/array'
 import {
   QuickPickOptionsWithTitle,
@@ -15,13 +14,11 @@ export const pickFromColumnLikes = (
     return Toast.showError('There are no columns to select from.')
   }
   return quickPickValue<ColumnLike>(
-    columnLikes
-      .filter(columnLike => columnLike.path !== timestampColumn.path)
-      .map(columnLike => ({
-        description: columnLike.path,
-        label: columnLike.label,
-        value: columnLike
-      })),
+    columnLikes.map(columnLike => ({
+      description: columnLike.path,
+      label: columnLike.label,
+      value: columnLike
+    })),
     quickPickOptions
   )
 }

extension/src/experiments/model/filterBy/date.test.ts

@@ -0,0 +1,80 @@
+import { Operator } from '.'
+import { compareDateStrings } from './date'
+
+describe('compareDateStrings', () => {
+  it('should compare two date strings and give the expected results', () => {
+    const earlierDate = '2022-01-01'
+    const laterDate = '2023-01-01'
+    expect(
+      compareDateStrings(earlierDate, Operator.GREATER_THAN, laterDate)
+    ).toBe(false)
+    expect(
+      compareDateStrings(laterDate, Operator.GREATER_THAN, earlierDate)
+    ).toBe(true)
+
+    expect(compareDateStrings(earlierDate, Operator.LESS_THAN, laterDate)).toBe(
+      true
+    )
+    expect(compareDateStrings(laterDate, Operator.LESS_THAN, earlierDate)).toBe(
+      false
+    )
+
+    expect(compareDateStrings(earlierDate, Operator.EQUAL, laterDate)).toBe(
+      false
+    )
+  })
+
+  it('should compare a date string and an ISO date strings from the different days and give the expected results', () => {
+    const earlierDate = '2000-01-01'
+    const laterTimestamp = '2000-01-02T00:00:01'
+
+    expect(
+      compareDateStrings(earlierDate, Operator.GREATER_THAN, laterTimestamp)
+    ).toBe(false)
+    expect(
+      compareDateStrings(laterTimestamp, Operator.GREATER_THAN, earlierDate)
+    ).toBe(true)
+
+    expect(
+      compareDateStrings(earlierDate, Operator.LESS_THAN, laterTimestamp)
+    ).toBe(true)
+    expect(
+      compareDateStrings(laterTimestamp, Operator.LESS_THAN, earlierDate)
+    ).toBe(false)
+
+    expect(
+      compareDateStrings(earlierDate, Operator.EQUAL, laterTimestamp)
+    ).toBe(false)
+  })
+
+  it('should compare two ISO date strings from the same day and give the expected results', () => {
+    const earlierTimestamp = '2000-01-01T00:12:00'
+    const laterTimestamp = '2000-01-01T23:12:00'
+
+    expect(
+      compareDateStrings(
+        earlierTimestamp,
+        Operator.GREATER_THAN,
+        laterTimestamp
+      )
+    ).toBe(false)
+    expect(
+      compareDateStrings(
+        laterTimestamp,
+        Operator.GREATER_THAN,
+        earlierTimestamp
+      )
+    ).toBe(false)
+
+    expect(
+      compareDateStrings(earlierTimestamp, Operator.LESS_THAN, laterTimestamp)
+    ).toBe(false)
+    expect(
+      compareDateStrings(laterTimestamp, Operator.LESS_THAN, earlierTimestamp)
+    ).toBe(false)
+
+    expect(
+      compareDateStrings(earlierTimestamp, Operator.EQUAL, laterTimestamp)
+    ).toBe(true)
+  })
+})

extension/src/experiments/model/filterBy/date.ts

@@ -0,0 +1,30 @@
+import { Operator } from '.'
+import { standardizeDate } from '../../../util/date'
+
+export const compareDateStrings = (
+  baseDateString: string | number | boolean,
+  operator: Operator.LESS_THAN | Operator.GREATER_THAN | Operator.EQUAL,
+  comparisonDateString: string | number | boolean
+): boolean => {
+  if (
+    typeof baseDateString !== 'string' ||
+    typeof comparisonDateString !== 'string'
+  ) {
+    return false
+  }
+
+  const epoch = standardizeDate(baseDateString)
+  const otherEpoch = standardizeDate(comparisonDateString)
+
+  switch (operator) {
+    case Operator.LESS_THAN:
+      return epoch < otherEpoch
+    case Operator.GREATER_THAN:
+      return epoch > otherEpoch
+    case Operator.EQUAL:
+      return epoch === otherEpoch
+
+    default:
+      return false
+  }
+}

extension/src/experiments/model/filterBy/index.test.ts

@@ -6,6 +6,7 @@ describe('splitExperimentsByFilters', () => {
   const paramsFile = 'params.yaml'
   const experiments = [
     {
+      Created: '2020-12-29T12:00:01',
       id: 1,
       params: {
         'params.yaml': {
@@ -17,6 +18,7 @@ describe('splitExperimentsByFilters', () => {
       }
     },
     {
+      Created: '2020-12-30T12:00:01',
       id: 2,
       params: {
         'params.yaml': {
@@ -28,6 +30,7 @@ describe('splitExperimentsByFilters', () => {
       }
     },
     {
+      Created: '2021-01-01T00:00:01',
       id: 3,
       params: {
         'params.yaml': {
@@ -260,4 +263,49 @@ describe('splitExperimentsByFilters', () => {
     expect(filtered.map(experiment => experiment.id)).toStrictEqual([1, 3])
     expect(unfiltered.map(experiment => experiment.id)).toStrictEqual([2])
   })
+
+  it('should split the experiments using after Created date', () => {
+    const { filtered, unfiltered } = splitExperimentsByFilters(
+      [
+        {
+          operator: Operator.AFTER_DATE,
+          path: 'Created',
+          value: '2020-12-31T15:40:00'
+        }
+      ],
+      experiments
+    )
+    expect(filtered.map(experiment => experiment.id)).toStrictEqual([1, 2])
+    expect(unfiltered.map(experiment => experiment.id)).toStrictEqual([3])
+  })
+
+  it('should split the experiments using before Created date', () => {
+    const { filtered, unfiltered } = splitExperimentsByFilters(
+      [
+        {
+          operator: Operator.BEFORE_DATE,
+          path: 'Created',
+          value: '2020-12-31T15:40:00'
+        }
+      ],
+      experiments
+    )
+    expect(filtered.map(experiment => experiment.id)).toStrictEqual([3])
+    expect(unfiltered.map(experiment => experiment.id)).toStrictEqual([1, 2])
+  })
+
+  it('should split the experiments using on Created date', () => {
+    const { filtered, unfiltered } = splitExperimentsByFilters(
+      [
+        {
+          operator: Operator.ON_DATE,
+          path: 'Created',
+          value: '2020-12-31T15:40:00'
+        }
+      ],
+      experiments
+    )
+    expect(filtered.map(experiment => experiment.id)).toStrictEqual([1, 2, 3])
+    expect(unfiltered.map(experiment => experiment.id)).toStrictEqual([])
+  })
 })

extension/src/experiments/model/filterBy/index.ts

@@ -1,4 +1,5 @@
 import get from 'lodash.get'
+import { compareDateStrings } from './date'
 import { Experiment } from '../../webview/contract'
 import { definedAndNonEmpty } from '../../../util/array'
 import { splitColumnPath } from '../../columns/paths'
@@ -15,9 +16,18 @@ export enum Operator {
   NOT_CONTAINS = '!∈',
 
   IS_TRUE = '⊤',
-  IS_FALSE = '⊥'
+  IS_FALSE = '⊥',
+
+  BEFORE_DATE = '<d',
+  AFTER_DATE = '>d',
+  ON_DATE = '=d'
 }
 
+export const isDateOperator = (operator: Operator): boolean =>
+  [Operator.AFTER_DATE, Operator.BEFORE_DATE, Operator.ON_DATE].includes(
+    operator
+  )
+
 export interface FilterDefinition {
   path: string
   operator: Operator
@@ -71,6 +81,21 @@ const evaluate = <T extends string | number | boolean>(
       return stringContains(valueToEvaluate, filterValue)
     case Operator.NOT_CONTAINS:
       return !stringContains(valueToEvaluate, filterValue)
+
+    case Operator.AFTER_DATE:
+      return compareDateStrings(
+        valueToEvaluate,
+        Operator.GREATER_THAN,
+        filterValue
+      )
+    case Operator.BEFORE_DATE:
+      return compareDateStrings(
+        valueToEvaluate,
+        Operator.LESS_THAN,
+        filterValue
+      )
+    case Operator.ON_DATE:
+      return compareDateStrings(valueToEvaluate, Operator.EQUAL, filterValue)
     default:
       throw new Error('filter operator not found')
   }

extension/src/experiments/model/filterBy/quickPick.test.ts

@@ -1,5 +1,5 @@
 import { FilterDefinition, getFilterId, Operator } from '.'
-import { operators, pickFiltersToRemove, pickFilterToAdd } from './quickPick'
+import { OPERATORS, pickFiltersToRemove, pickFilterToAdd } from './quickPick'
 import { getInput } from '../../../vscode/inputBox'
 import { appendColumnToPath, buildMetricOrParamPath } from '../../columns/paths'
 import { quickPickManyValues, quickPickValue } from '../../../vscode/quickPick'
@@ -76,9 +76,15 @@ describe('pickFilterToAdd', () => {
     mockedQuickPickValue.mockResolvedValueOnce(mixedParam)
     mockedQuickPickValue.mockResolvedValueOnce(undefined)
     await pickFilterToAdd(params)
-    expect(mockedQuickPickValue).toBeCalledWith(operators, {
-      title: Title.SELECT_OPERATOR
-    })
+    expect(mockedQuickPickValue).toBeCalledWith(
+      OPERATORS.filter(
+        ({ types }) =>
+          !(types.length === 1 && types[0] === ColumnType.TIMESTAMP)
+      ),
+      {
+        title: Title.SELECT_OPERATOR
+      }
+    )
   })
 
   it('should return early if no value is provided', async () => {
@@ -101,7 +107,7 @@ describe('pickFilterToAdd', () => {
       value: undefined
     })
     expect(mockedQuickPickValue).toBeCalledWith(
-      operators.filter(operator => operator.types.includes('boolean')),
+      OPERATORS.filter(operator => operator.types.includes('boolean')),
       {
         title: Title.SELECT_OPERATOR
       }
@@ -121,7 +127,7 @@ describe('pickFilterToAdd', () => {
       value: '5'
     })
     expect(mockedQuickPickValue).toBeCalledWith(
-      operators.filter(operator => operator.types.includes('number')),
+      OPERATORS.filter(operator => operator.types.includes('number')),
       {
         title: Title.SELECT_OPERATOR
       }

extension/src/experiments/model/filterBy/quickPick.ts

@@ -1,13 +1,15 @@
-import { FilterDefinition, getFilterId, Operator } from '.'
+import { FilterDefinition, getFilterId, isDateOperator, Operator } from '.'
 import { definedAndNonEmpty } from '../../../util/array'
-import { getInput } from '../../../vscode/inputBox'
+import { getIsoDate, isFreeTextDate } from '../../../util/date'
+import { getInput, getValidInput } from '../../../vscode/inputBox'
 import { quickPickManyValues, quickPickValue } from '../../../vscode/quickPick'
 import { Title } from '../../../vscode/title'
 import { Toast } from '../../../vscode/toast'
 import { ColumnLike } from '../../columns/like'
 import { pickFromColumnLikes } from '../../columns/quickPick'
+import { ColumnType } from '../../webview/contract'
 
-export const operators = [
+export const OPERATORS = [
   {
     description: 'Equal',
     label: '=',
@@ -67,11 +69,43 @@ export const operators = [
     label: Operator.NOT_CONTAINS,
     types: ['string'],
     value: Operator.NOT_CONTAINS
+  },
+  {
+    description: 'After Date',
+    label: Operator.AFTER_DATE,
+    types: [ColumnType.TIMESTAMP],
+    value: Operator.AFTER_DATE
+  },
+  {
+    description: 'Before Date',
+    label: Operator.BEFORE_DATE,
+    types: [ColumnType.TIMESTAMP],
+    value: Operator.BEFORE_DATE
+  },
+  {
+    description: 'On Day',
+    label: Operator.ON_DATE,
+    types: [ColumnType.TIMESTAMP],
+    value: Operator.ON_DATE
   }
 ]
 
+const getValue = (operator: Operator): Thenable<string | undefined> => {
+  if (isDateOperator(operator)) {
+    return getValidInput(
+      Title.ENTER_FILTER_VALUE,
+      (text?: string): null | string =>
+        isFreeTextDate(text)
+          ? null
+          : 'please enter a valid date of the form yyyy-mm-dd',
+      getIsoDate()
+    )
+  }
+  return getInput(Title.ENTER_FILTER_VALUE)
+}
+
 const addFilterValue = async (path: string, operator: Operator) => {
-  const value = await getInput(Title.ENTER_FILTER_VALUE)
+  const value = await getValue(operator)
   if (!value) {
     return
   }
@@ -93,7 +127,7 @@ export const pickFilterToAdd = async (
     return
   }
 
-  const typedOperators = operators.filter(operator =>
+  const typedOperators = OPERATORS.filter(operator =>
     operator.types.some(type => picked.types?.includes(type))
   )