Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

telemetry: add billing categorization to cody events 1 #5493

Merged
merged 4 commits into from
Sep 9, 2024
Merged

telemetry: add billing categorization to cody events 1 #5493

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
50c8eda
telemetry: add billing categorization to cody events 1
dadlerj Sep 6, 2024
24aa773
formatting
dadlerj Sep 7, 2024
8d89ee6
ci
dadlerj Sep 7, 2024
9c9f7de
Comments
dadlerj Sep 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lib/shared/src/telemetry-v2/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
* Events accept billing metadata for ease of categorization in analytics
* pipelines - this type enumerates known categories.
*/
export type BillingCategory = 'exampleBillingCategory'
export type BillingCategory = 'exampleBillingCategory' | 'core' | 'billable'

/**
* Events accept billing metadata for ease of categorization in analytics
* pipelines - this type enumerates known products.
*/
export type BillingProduct = 'exampleBillingProduct'
export type BillingProduct = 'exampleBillingProduct' | 'cody'
dadlerj marked this conversation as resolved.
Show resolved Hide resolved
19 changes: 18 additions & 1 deletion vscode/src/auth/auth.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ export async function showSignInMenu(
const menuID = type || item?.id
telemetryRecorder.recordEvent('cody.auth.signin.menu', 'clicked', {
privateMetadata: { menuID },
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
switch (menuID) {
case 'enterprise': {
Expand Down Expand Up @@ -222,6 +226,10 @@ async function signinMenuForInstanceUrl(instanceUrl: string): Promise<void> {
metadata: {
success: authState.authenticated ? 1 : 0,
},
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
await showAuthResultMessage(instanceUrl, authState)
}
Expand Down Expand Up @@ -292,6 +300,10 @@ export async function tokenCallbackHandler(
metadata: {
success: authState?.authenticated ? 1 : 0,
},
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
if (authState?.authenticated) {
await vscode.window.showInformationMessage(`Signed in to ${endpoint}`)
Expand Down Expand Up @@ -326,7 +338,12 @@ export function formatURL(uri: string): string | null {
}

export async function showSignOutMenu(): Promise<void> {
telemetryRecorder.recordEvent('cody.auth.logout', 'clicked')
telemetryRecorder.recordEvent('cody.auth.logout', 'clicked', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
const { endpoint } = authProvider.instance!.status

if (endpoint) {
Expand Down
30 changes: 28 additions & 2 deletions vscode/src/chat/chat-view/ChatController.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,10 @@ export class ChatController implements vscode.Disposable, vscode.WebviewViewProv
metadata: {
success: authStatus?.authenticated ? 1 : 0,
},
billingMetadata: {
product: 'cody',
category: 'billable',
},
}
)
if (!authStatus?.authenticated) {
Expand Down Expand Up @@ -801,6 +805,10 @@ export class ChatController implements vscode.Disposable, vscode.WebviewViewProv
promptText:
isDotCom(authStatus) && truncatePromptString(inputText, CHAT_INPUT_TOKEN_BUDGET),
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})
}

Expand Down Expand Up @@ -922,7 +930,12 @@ export class ChatController implements vscode.Disposable, vscode.WebviewViewProv
): Promise<void> {
const abortSignal = this.startNewSubmitOrEditOperation()

telemetryRecorder.recordEvent('cody.editChatButton', 'clicked')
telemetryRecorder.recordEvent('cody.editChatButton', 'clicked', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})

try {
const humanMessage = index ?? this.chatModel.getLastSpeakerMessageIndex('human')
Expand All @@ -949,7 +962,12 @@ export class ChatController implements vscode.Disposable, vscode.WebviewViewProv
this.cancelSubmitOrEditOperation()
// Notify the webview there is no message in progress.
this.postViewTranscript()
telemetryRecorder.recordEvent('cody.sidebar.abortButton', 'clicked')
telemetryRecorder.recordEvent('cody.sidebar.abortButton', 'clicked', {
billingMetadata: {
category: 'billable',
product: 'cody',
},
})
}

public async handleGetUserEditorContext(uri?: URI): Promise<void> {
Expand Down Expand Up @@ -1302,6 +1320,10 @@ export class ChatController implements vscode.Disposable, vscode.WebviewViewProv
isDotCom(authStatus) && truncatePromptString(messageText, CHAT_OUTPUT_TOKEN_BUDGET),
chatModel: this.chatModel.modelID,
},
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
}

Expand Down Expand Up @@ -1613,6 +1635,10 @@ export class ChatController implements vscode.Disposable, vscode.WebviewViewProv
isDotCom(authStatus) && truncatePromptString(inputText, CHAT_INPUT_TOKEN_BUDGET),
gitMetadata,
},
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
}
}
Expand Down
14 changes: 12 additions & 2 deletions vscode/src/chat/chat-view/ChatsController.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,12 @@ export class ChatsController implements vscode.Disposable {
}

private async sendEditorContextToChat(uri?: URI): Promise<void> {
telemetryRecorder.recordEvent('cody.addChatContext', 'clicked')
telemetryRecorder.recordEvent('cody.addChatContext', 'clicked', {
billingMetadata: {
category: 'billable',
product: 'cody',
},
})
const provider = await this.getActiveChatController()
if (provider === this.panel) {
await vscode.commands.executeCommand('cody.chat.focus')
Expand Down Expand Up @@ -345,7 +350,12 @@ export class ChatsController implements vscode.Disposable {
* Export chat history to file system
*/
private async exportHistory(): Promise<void> {
telemetryRecorder.recordEvent('cody.exportChatHistoryButton', 'clicked')
telemetryRecorder.recordEvent('cody.exportChatHistoryButton', 'clicked', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
const authStatus = authProvider.instance!.status
if (authStatus.authenticated) {
try {
Expand Down
8 changes: 8 additions & 0 deletions vscode/src/chat/context/chatContext.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,20 @@ export function getMentionMenuData(
source: atMentionSourceTelemetryMetadataMapping[source],
},
privateMetadata: { source },
billingMetadata: {
product: 'cody',
category: 'core',
},
})
},
withProvider: (provider, providerMetadata) => {
telemetryRecorder.recordEvent(`cody.at-mention.${provider}`, 'executed', {
metadata: { source: atMentionSourceTelemetryMetadataMapping[source] },
privateMetadata: { source, providerMetadata },
billingMetadata: {
product: 'cody',
category: 'core',
},
})
},
}
Expand Down
4 changes: 4 additions & 0 deletions vscode/src/commands/execute/explain-history.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ export async function executeExplainHistoryCommand(
source: args?.source,
traceId: span.spanContext().traceId,
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})

const sessionArgs = await explainHistoryCommand(span, commandsProvider, args)
Expand Down
4 changes: 4 additions & 0 deletions vscode/src/commands/execute/explain.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ export async function executeExplainCommand(
source: args?.source,
traceId: span.spanContext().traceId,
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})

const chatArguments = await explainCommand(span, args)
Expand Down
4 changes: 4 additions & 0 deletions vscode/src/commands/execute/smell.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ export async function executeSmellCommand(
source: args?.source,
traceId: span.spanContext().traceId,
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})

const chatArguments = await smellCommand(span, args)
Expand Down
4 changes: 4 additions & 0 deletions vscode/src/commands/execute/terminal.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ export async function executeExplainOutput(
source,
traceId: span.spanContext().traceId,
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})

const promptArgs = PromptString.fromTerminalOutputArguments(args)
Expand Down
4 changes: 4 additions & 0 deletions vscode/src/commands/execute/test-chat.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ export async function executeTestChatCommand(
source: args?.source,
traceId: span.spanContext().traceId,
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})

return {
Expand Down
7 changes: 6 additions & 1 deletion vscode/src/commands/menus/command-builder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,12 @@ export class CustomCommandsBuilderMenu {
const type = prompt && (await this.selectCommandType())

if (key && mode && prompt && type) {
telemetryRecorder.recordEvent('cody.command.custom.build', 'executed')
telemetryRecorder.recordEvent('cody.command.custom.build', 'executed', {
billingMetadata: {
product: 'cody',
category: 'core',
},
})

return { key, prompt: { ...prompt, key, mode }, type }
}
Expand Down
11 changes: 10 additions & 1 deletion vscode/src/commands/menus/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ export async function showCommandMenu(
const source = args?.source
telemetryRecorder.recordEvent(`cody.menu.command.${type}`, 'clicked', {
privateMetadata: { source },
billingMetadata: {
product: 'cody',
category: 'billable',
},
})

// Add items to menus accordingly:
Expand Down Expand Up @@ -219,7 +223,12 @@ function normalize(input: string): string {
export async function showNewCustomCommandMenu(
commands: string[]
): Promise<CustomCommandsBuilder | null> {
telemetryRecorder.recordEvent('cody.menu.custom.build', 'clicked')
telemetryRecorder.recordEvent('cody.menu.custom.build', 'clicked', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})
const builder = new CustomCommandsBuilderMenu()
return builder.start(commands)
}
7 changes: 6 additions & 1 deletion vscode/src/commands/scm/source-control.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,12 @@ export class CodySourceControl implements vscode.Disposable {
* @param scm - The source control instance to use for the commit message generation.
*/
public async generate(scm?: vscode.SourceControl): Promise<void> {
telemetryRecorder.recordEvent('cody.command.generate-commit', 'executed')
telemetryRecorder.recordEvent('cody.command.generate-commit', 'executed', {
billingMetadata: {
product: 'cody',
category: 'core',
},
})

const currentWorkspaceUri = scm?.rootUri ?? vscode.workspace.workspaceFolders?.[0]?.uri
if (!this.gitAPI || !currentWorkspaceUri) {
Expand Down
7 changes: 6 additions & 1 deletion vscode/src/commands/services/code-lenses.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,12 @@ export class CommandCodeLenses implements vscode.CodeLensProvider {
this._disposables.push(vscode.languages.registerCodeLensProvider({ scheme: 'file' }, this))
this._disposables.push(
vscode.commands.registerCommand('cody.editor.codelens.click', async lens => {
telemetryRecorder.recordEvent('cody.command.codelens', 'clicked')
telemetryRecorder.recordEvent('cody.command.codelens', 'clicked', {
billingMetadata: {
product: 'cody',
category: 'core',
},
})
const clickedLens = lens as EditorCodeLens
await this.onCodeLensClick(clickedLens)
})
Expand Down
4 changes: 4 additions & 0 deletions vscode/src/commands/services/runner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ export class CommandRunner implements vscode.Disposable {
source: this.args.source,
traceId: this.span.spanContext().traceId,
},
billingMetadata: {
product: 'cody',
category: 'core',
},
})

// Conditions checks
Expand Down
2 changes: 2 additions & 0 deletions vscode/src/completions/logger.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,15 @@ describe('logger', () => {
interactionID: expect.any(String),
metadata: expect.anything(),
privateMetadata: expect.anything(),
billingMetadata: expect.anything(),
})

expect(recordSpy).toHaveBeenCalledWith('cody.completion', 'accepted', {
version: 0,
interactionID: expect.any(String),
metadata: expect.anything(),
privateMetadata: expect.anything(),
billingMetadata: expect.anything(),
})
})

Expand Down
10 changes: 10 additions & 0 deletions vscode/src/completions/logger.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,16 @@ function writeCompletionEvent<SubFeature extends string, Action extends string,
if (subfeature) {
telemetryRecorder.recordEvent(`cody.completion.${subfeature}`, action, params)
} else {
// Add billing metadata to completion event params.
params = {
...params,
billingMetadata: {
product: 'cody',
// Only acceptance events qualify as "core" usage.
category: action === 'partiallyAccepted' || action === 'accepted' ? 'core' : 'billable',
},
}

telemetryRecorder.recordEvent('cody.completion', action, params)
}
}
Expand Down
21 changes: 18 additions & 3 deletions vscode/src/edit/input/get-input.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,12 @@ export const getInput = async (
if (!acceptedItem) {
return
}
telemetryRecorder.recordEvent('cody.fixup.input.model', 'selected')
telemetryRecorder.recordEvent('cody.fixup.input.model', 'selected', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})

if (acceptedItem.codyProOnly && !isCodyPro) {
const option = await vscode.window.showInformationMessage(
Expand Down Expand Up @@ -238,7 +243,12 @@ export const getInput = async (
if (!acceptedItem) {
return
}
telemetryRecorder.recordEvent('cody.fixup.input.rangeSymbol', 'selected')
telemetryRecorder.recordEvent('cody.fixup.input.rangeSymbol', 'selected', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})

activeRangeItem = acceptedItem
const range =
Expand Down Expand Up @@ -282,7 +292,12 @@ export const getInput = async (
return
}

telemetryRecorder.recordEvent('cody.fixup.input.range', 'selected')
telemetryRecorder.recordEvent('cody.fixup.input.range', 'selected', {
billingMetadata: {
product: 'cody',
category: 'billable',
},
})

activeRangeItem = acceptedItem
const range =
Expand Down
Loading
Loading