feat: log storage paging

package.json

@@ -39,7 +39,7 @@
   "license": "MIT",
   "dependencies": {
     "@azure/cosmos": "^3.3.6",
-    "@conversationlearner/models": "0.217.0",
+    "@conversationlearner/models": "0.218.0",
     "@conversationlearner/ui": "0.407.0",
     "@types/supertest": "2.0.4",
     "async-file": "^2.0.2",
@@ -81,7 +81,7 @@
     "jest": "24.5.0",
     "nodemon": "^1.11.0",
     "prettier": "^1.10.2",
-    "tslint": "^5.9.1",
+    "tslint": "^5.20.0",
     "tslint-config-prettier": "^1.12.0",
     "tslint-config-standard": "^7.0.0",
     "tslint-microsoft-contrib": "^5.0.3",

src/AzureFunctions.ts

@@ -5,6 +5,9 @@
 import * as Request from 'request'
 import { CLDebug } from './CLDebug'
 
+/**
+ * Support for calls to Azure Functions (currently disabled in UI)
+ */
 export class AzureFunctions {
     public static Call(azureFunctionsUrl: string, azureFunctionsKey: string, funcName: string, args: string): Promise<string> {
         let apiPath = 'app'

src/CLClient.ts

@@ -26,6 +26,9 @@ export interface ICLClientOptions {
     LUIS_SUBSCRIPTION_KEY?: string
 }
 
+/**
+ * Manages calls to Conversation Learner Service
+ */
 export class CLClient {
     private options: ICLClientOptions
 
@@ -253,9 +256,15 @@ export class CLClient {
         return this.send('GET', this.MakeURL(apiPath))
     }
 
-    public GetLogDialogs(appId: string, packageIds: string[]): Promise<CLM.LogDialogList> {
+    public GetLogDialogs(appId: string, packageIds: string[], continuationToken?: string, maxPageSize?: string): Promise<CLM.LogQueryResult> {
         const packages = packageIds.map(p => `package=${p}`).join("&")
-        const apiPath = `app/${appId}/logdialogs?includeDefinitions=false&${packages}`
+        let apiPath = `app/${appId}/logdialogs?includeDefinitions=false&${packages}`
+        if (continuationToken) {
+            apiPath = apiPath.concat(`&continuationToken=${encodeURIComponent(continuationToken)}`)
+        }
+        if (maxPageSize) {
+            apiPath = apiPath.concat(`&maxPageSize=${maxPageSize}`)
+        }
         return this.send('GET', this.MakeURL(apiPath))
     }
 

src/CLDebug.ts

@@ -25,6 +25,9 @@ interface LogMessage {
     logType: LogType
 }
 
+/***
+ * Handles debug output instead of console.log
+ */
 export class CLDebug {
     private static adapter: BB.BotAdapter
     private static conversationReference: Partial<BB.ConversationReference>

src/CLRunner.ts

@@ -106,6 +106,9 @@ export interface IActionResult {
 
 export type CallbackMap = { [name: string]: InternalCallback<any> }
 
+/**
+ * Runs Conversation Learnern for a given CL Model
+ */
 export class CLRunner {
 
     /* Lookup table for CLRunners.  One CLRunner per CL Model */
@@ -388,6 +391,7 @@ export class CLRunner {
                 scorerSteps: []
             }
             logDialog.rounds.push(newRound)
+            logDialog.lastModifiedDateTime = new Date().toJSON()
             await ConversationLearner.logStorage.Replace(appId, logDialog)
         }
         return extractResponse
@@ -436,6 +440,7 @@ export class CLRunner {
                 throw new Error(`Log Dialogs has no Extractor Step Id:${logDialogId}`)
             }
             lastRound.scorerSteps.push(logScorerStep as any)
+            logDialog.lastModifiedDateTime = new Date().toJSON()
             await ConversationLearner.logStorage.Replace(appId, logDialog)
         }
         return scoreResponse

src/ConversationLearner.ts

@@ -14,6 +14,9 @@ import { CLRecognizerResult } from './CLRecognizeResult'
 import { CLModelOptions } from '.'
 import { ILogStorage } from './Memory/ILogStorage'
 
+/**
+ * Main CL class used by Bot
+ */
 export class ConversationLearner {
     public static options: CLOptions | null = null
     public static clClient: CLClient

src/CosmosLogStorage.ts

@@ -11,14 +11,17 @@ const DATABASE_ID = "LOG_DIALOGS"
 const COLLECTION_ID = "LOG_DIALOGS"
 const MAX_PAGE_SIZE = 100
 const DELETE_BATCH_SIZE = 10
-const PARTITION_KEY = { kind: 'Hash', paths: ['/appId', '/packageId'] }
+const PARTITION_KEY = { kind: 'Hash', paths: ['/appId'] }
 
 interface StoredLogDialog extends CLM.LogDialog {
     // CosmosId
     id: string,
     appId: string
 }
 
+/**
+ * Custom Log Stoage account using Cosmos
+ */
 export class CosmosLogStorage implements ILogStorage {
     private client: Cosmos.CosmosClient
     private database: Cosmos.Database | undefined
@@ -122,6 +125,8 @@ export class CosmosLogStorage implements ILogStorage {
                 querySpec.query = querySpec.query.concat(`${and} NOT ARRAY_CONTAINS(@logIdList, c.logDialogId)`)
                 querySpec.parameters!.push({ name: '@logIdList', value: this.deleteQueue })
             }
+            // Return in reverse order so newest is on top
+            querySpec.query = querySpec.query.concat(' ORDER BY c.lastModifiedDateTime DESC')
 
             const feedOptions: Cosmos.FeedOptions = {
                 maxItemCount: Math.min(pageSize, MAX_PAGE_SIZE),

src/Memory/BotState.ts

@@ -64,6 +64,9 @@ export enum BotStateType {
 type GetKey = (datakey: string) => string
 export type ConvIdMapper = (ref: Partial<BB.ConversationReference> | null) => string | null
 
+/**
+ * Maintains state of Bot
+ */
 export class BotState {
     private readonly storage: CLStorage
     private readonly getKey: GetKey

src/Memory/ClientMemoryManager.ts

@@ -8,6 +8,9 @@ import * as CLM from '@conversationlearner/models'
 
 export type MemoryManagerReturnType<T> = T extends CLM.MemoryValue[] | CLM.MemoryValue ? T extends CLM.MemoryValue[] ? CLM.MemoryValue[] : CLM.MemoryValue : T
 
+/**
+ * Used to read CL Memory in Bot code
+ */
 export class ReadOnlyClientMemoryManager {
     protected allEntities: CLM.EntityBase[] = []
     private sessionInfo: SessionInfo
@@ -165,6 +168,9 @@ export class ReadOnlyClientMemoryManager {
     }
 }
 
+/**
+ * Used to manipulate CL Memory in Bot code
+ */
 export class ClientMemoryManager extends ReadOnlyClientMemoryManager {
 
     public constructor(prevMemories: CLM.FilledEntityMap, curMemories: CLM.FilledEntityMap, allEntities: CLM.EntityBase[], sessionInfo: SessionInfo) {

src/Memory/EntityState.ts

@@ -11,6 +11,9 @@ const NEGATIVE_PREFIX = '~'
 
 export type GetKey = () => string
 
+/**
+ * Memory for a given entity
+ */
 export class EntityState {
     private storage: CLStorage
     private getKey: GetKey

src/Memory/ILogStorage.ts

@@ -9,7 +9,10 @@ export interface LogQueryResult {
     continuationToken: string | undefined
 }
 
-export interface ILogStorage {
+/**
+ * Interface for generating custom LogStorage implimentations
+ */
+export declare class ILogStorage {
 
     /** Add a new log dialog to storage */
     Add(appId: string, logDialog: CLM.LogDialog): Promise<CLM.LogDialog | undefined>

src/Memory/InputQueue.ts

@@ -14,6 +14,9 @@ export interface QueuedInput {
     callback: Function
 }
 
+/**
+ * Used to queue up multiple user inputs when then come in a row
+ */
 export class InputQueue {
 
     private static messageQueue: QueuedInput[] = [];

src/RedisStorage.ts

@@ -18,6 +18,9 @@ export interface RedisStorageSettings {
     port?: number
 }
 
+/**
+ * Bot storage implimentation using a Redis cache
+ */
 export class RedisStorage implements Storage {
     private redisClient: Redis.RedisClient
     private _get: (...args: any[]) => Promise<any>

src/TemplateProvider.ts

@@ -7,6 +7,9 @@ import * as path from 'path'
 import { Template, TemplateVariable, RenderedActionArgument } from '@conversationlearner/models'
 import { CLDebug } from './CLDebug'
 
+/**
+ * Provider for rendering templates
+ */
 export class TemplateProvider {
     private static hasSubmitError = false
 

src/Utils.ts

@@ -12,6 +12,9 @@ import * as CLM from '@conversationlearner/models'
 import * as HttpStatus from 'http-status-codes'
 import { CLClient } from './CLClient'
 
+/**
+ * General utilities
+ */
 export class Utils {
     public static SendTyping(adapter: BB.BotAdapter, address: any) {
         /* TODO

src/http/router.ts

@@ -522,22 +522,21 @@ export const getRouter = (client: CLClient, options: ICLClientOptions): express.
 
     // Get log dialogs
     router.get('/app/:appId/logdialogs', async (req, res, next) => {
-        const { appId, continuationToken, pageSize } = req.params
+        const { appId } = req.params
 
         try {
-            let { package: packageIds } = getQuery(req)
+            let { package: packageIds, continuationToken, maxPageSize } = getQuery(req)
             if (typeof packageIds === "string") {
                 packageIds = [packageIds]
             }
-            let logDialogs
+            let logQueryResult: CLM.LogQueryResult
             if (ConversationLearner.logStorage) {
-                logDialogs = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, pageSize)
+                logQueryResult = await ConversationLearner.logStorage.GetMany(appId, packageIds, continuationToken, maxPageSize)
             }
             else {
-                // TODO: Add paging to server
-                logDialogs = await client.GetLogDialogs(appId, packageIds)
+                logQueryResult = await client.GetLogDialogs(appId, packageIds, continuationToken, maxPageSize)
             }
-            res.send(logDialogs)
+            res.send(logQueryResult)
         } catch (error) {
             HandleError(res, error)
         }