-
Notifications
You must be signed in to change notification settings - Fork 491
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
codewhisperer: telemetry metrics apply fix #4007
Merged
ctlai95
merged 5 commits into
aws:feature/codewhisperer/assisted-code-remediation
from
ctlai95:telemetry-metrics-apply-fix
Nov 7, 2023
+178
−139
Merged
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
1b9397e
codewhisperer: add metrics for apply fix
ctlai95 0421806
fix tests
ctlai95 969ccb7
move source values to component
ctlai95 9efb904
use different component values for tests
ctlai95 fd8fc66
revert move source values to component
ctlai95 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,7 +8,7 @@ import assert from 'assert' | |
import * as sinon from 'sinon' | ||
import * as CodeWhispererConstants from '../../../codewhisperer/models/constants' | ||
import { createCodeScanIssue, createMockDocument, resetCodeWhispererGlobalVariables } from '../testUtil' | ||
import { assertTelemetryCurried } from '../../testUtil' | ||
import { assertTelemetry, assertTelemetryCurried } from '../../testUtil' | ||
import { | ||
toggleCodeSuggestions, | ||
showSecurityScan, | ||
|
@@ -138,139 +138,160 @@ describe('CodeWhisperer-basicCommands', function () { | |
await targetCommand.execute() | ||
assert.strictEqual(getTestWindow().shownMessages[0].message, 'Open a valid file to scan.') | ||
}) | ||
}) | ||
|
||
describe('applySecurityFix', function () { | ||
let sandbox: sinon.SinonSandbox | ||
let saveStub: sinon.SinonStub | ||
let openTextDocumentMock: sinon.SinonStub | ||
let writeFileMock: sinon.SinonStub | ||
|
||
beforeEach(function () { | ||
sandbox = sinon.createSandbox() | ||
saveStub = sinon.stub() | ||
openTextDocumentMock = sinon.stub() | ||
writeFileMock = sinon.stub() | ||
}) | ||
|
||
afterEach(function () { | ||
sandbox.restore() | ||
}) | ||
|
||
it('should call applySecurityFix command successfully', async function () { | ||
const fileName = 'sample.py' | ||
const saveStub = sinon.stub() | ||
saveStub.resolves(true) | ||
const textDocumentMock = new MockDocument('first line\n second line\n fourth line', fileName, saveStub) | ||
|
||
const openTextDocumentMock = sinon.stub() | ||
openTextDocumentMock.resolves(textDocumentMock) | ||
const sandbox = sinon.createSandbox() | ||
sandbox.stub(vscode.workspace, 'openTextDocument').value(openTextDocumentMock) | ||
|
||
const writeFileMock = sinon.stub() | ||
writeFileMock.resolves(true) | ||
sinon.stub(FileSystemCommon.prototype, 'writeFile').value(writeFileMock) | ||
|
||
targetCommand = testCommand(applySecurityFix) | ||
await targetCommand.execute({ | ||
filePath: fileName, | ||
...createCodeScanIssue({ | ||
suggestedFixes: [ | ||
{ | ||
description: 'fix', | ||
code: '@@ -1,3 +1,3 @@\n first line\n- second line\n+ third line\n fourth line', | ||
}, | ||
], | ||
}), | ||
const codeScanIssue = createCodeScanIssue({ | ||
suggestedFixes: [ | ||
{ | ||
description: 'fix', | ||
code: '@@ -1,3 +1,3 @@\n first line\n- second line\n+ third line\n fourth line', | ||
}, | ||
], | ||
}) | ||
await targetCommand.execute(codeScanIssue, fileName, 'hover') | ||
assert.ok(saveStub.calledOnce) | ||
assert.ok(writeFileMock.calledOnceWith(fileName, 'first line\n third line\n fourth line')) | ||
|
||
assert.strictEqual( | ||
getTestWindow().shownMessages[0].message, | ||
'Code fix was applied. Run a security scan to validate the fix.' | ||
) | ||
|
||
sandbox.restore() | ||
assertTelemetry('codewhisperer_codeScanIssueApplyFix', { | ||
detectorId: codeScanIssue.detectorId, | ||
findingId: codeScanIssue.findingId, | ||
applyFixSource: 'hover', | ||
result: 'Succeeded', | ||
}) | ||
}) | ||
|
||
it('handles patch failure', async function () { | ||
const textDocumentMock = createMockDocument() | ||
|
||
const openTextDocumentMock = sinon.stub() | ||
openTextDocumentMock.resolves(textDocumentMock) | ||
|
||
const sandbox = sinon.createSandbox() | ||
sandbox.stub(vscode.workspace, 'openTextDocument').value(openTextDocumentMock) | ||
|
||
targetCommand = testCommand(applySecurityFix) | ||
await targetCommand.execute({ | ||
filePath: 'test.py', | ||
...createCodeScanIssue({ | ||
suggestedFixes: [ | ||
{ | ||
code: '@@ -1,1 -1,1 @@\n-mock\n+line5', | ||
description: 'dummy', | ||
}, | ||
], | ||
}), | ||
const codeScanIssue = createCodeScanIssue({ | ||
suggestedFixes: [ | ||
{ | ||
code: '@@ -1,1 -1,1 @@\n-mock\n+line5', | ||
description: 'dummy', | ||
}, | ||
], | ||
}) | ||
await targetCommand.execute(codeScanIssue, 'test.py', 'webview') | ||
|
||
assert.strictEqual(getTestWindow().shownMessages[0].message, 'Failed to apply suggested code fix.') | ||
|
||
sandbox.restore() | ||
assertTelemetry('codewhisperer_codeScanIssueApplyFix', { | ||
detectorId: codeScanIssue.detectorId, | ||
findingId: codeScanIssue.findingId, | ||
applyFixSource: 'webview', | ||
result: 'Failed', | ||
reason: 'Error: Failed to get updated content from applying diff patch', | ||
}) | ||
}) | ||
}) | ||
|
||
it('handles document save failure', async function () { | ||
const fileName = 'sample.py' | ||
const saveStub = sinon.stub() | ||
saveStub.resolves(false) | ||
const textDocumentMock = new MockDocument('first line\n second line\n fourth line', fileName, saveStub) | ||
it('handles document save failure', async function () { | ||
const fileName = 'sample.py' | ||
saveStub.resolves(false) | ||
const textDocumentMock = new MockDocument('first line\n second line\n fourth line', fileName, saveStub) | ||
|
||
const openTextDocumentMock = sinon.stub() | ||
openTextDocumentMock.resolves(textDocumentMock) | ||
openTextDocumentMock.resolves(textDocumentMock) | ||
|
||
const sandbox = sinon.createSandbox() | ||
sandbox.stub(vscode.workspace, 'openTextDocument').value(openTextDocumentMock) | ||
const loggerStub = sinon.stub(getLogger(), 'error') | ||
sandbox.stub(vscode.workspace, 'openTextDocument').value(openTextDocumentMock) | ||
const loggerStub = sinon.stub(getLogger(), 'error') | ||
|
||
targetCommand = testCommand(applySecurityFix) | ||
await targetCommand.execute({ | ||
filePath: fileName, | ||
...createCodeScanIssue({ | ||
targetCommand = testCommand(applySecurityFix) | ||
const codeScanIssue = createCodeScanIssue({ | ||
suggestedFixes: [ | ||
{ | ||
description: 'fix', | ||
code: '@@ -1,3 +1,3 @@\n first line\n- second line\n+ third line\n fourth line', | ||
}, | ||
], | ||
}), | ||
}) | ||
await targetCommand.execute(codeScanIssue, fileName, 'quickfix') | ||
|
||
assert.ok(saveStub.calledOnce) | ||
assert.ok(loggerStub.calledOnce) | ||
const actual = loggerStub.getCall(0).args[0] | ||
assert.strictEqual( | ||
actual, | ||
'Apply fix command failed. Error: Failed to save editor text changes into the file.' | ||
) | ||
assertTelemetry('codewhisperer_codeScanIssueApplyFix', { | ||
detectorId: codeScanIssue.detectorId, | ||
findingId: codeScanIssue.findingId, | ||
applyFixSource: 'quickfix', | ||
result: 'Failed', | ||
reason: 'Error: Failed to save editor text changes into the file.', | ||
}) | ||
}) | ||
|
||
assert.ok(saveStub.calledOnce) | ||
assert.ok(loggerStub.calledOnce) | ||
const actual = loggerStub.getCall(0).args[0] | ||
assert.strictEqual(actual, 'Failed to save editor text changes into the file.') | ||
it('handles document write failure', async function () { | ||
const fileName = 'sample.py' | ||
saveStub.resolves(true) | ||
const textDocumentMock = new MockDocument('first line\n second line\n fourth line', fileName, saveStub) | ||
|
||
sandbox.restore() | ||
}) | ||
it('handles document write failure', async function () { | ||
const fileName = 'sample.py' | ||
const saveStub = sinon.stub() | ||
saveStub.resolves(true) | ||
const textDocumentMock = new MockDocument('first line\n second line\n fourth line', fileName, saveStub) | ||
|
||
const openTextDocumentMock = sinon.stub() | ||
openTextDocumentMock.resolves(textDocumentMock) | ||
const writeFileMock = sinon.stub() | ||
writeFileMock.rejects('writing to file failed.') | ||
|
||
const sandbox = sinon.createSandbox() | ||
sandbox.stub(vscode.workspace, 'openTextDocument').value(openTextDocumentMock) | ||
sinon.stub(FileSystemCommon.prototype, 'writeFile').value(writeFileMock) | ||
const loggerStub = sinon.stub(getLogger(), 'error') | ||
|
||
targetCommand = testCommand(applySecurityFix) | ||
await targetCommand.execute({ | ||
filePath: fileName, | ||
...createCodeScanIssue({ | ||
openTextDocumentMock.resolves(textDocumentMock) | ||
writeFileMock.rejects('Error: Writing to file failed.') | ||
|
||
sandbox.stub(vscode.workspace, 'openTextDocument').value(openTextDocumentMock) | ||
sinon.stub(FileSystemCommon.prototype, 'writeFile').value(writeFileMock) | ||
const loggerStub = sinon.stub(getLogger(), 'error') | ||
|
||
targetCommand = testCommand(applySecurityFix) | ||
const codeScanIssue = createCodeScanIssue({ | ||
suggestedFixes: [ | ||
{ | ||
description: 'fix', | ||
code: '@@ -1,3 +1,3 @@\n first line\n- second line\n+ third line\n fourth line', | ||
}, | ||
], | ||
}), | ||
}) | ||
|
||
assert.ok(saveStub.calledOnce) | ||
assert.ok(loggerStub.calledOnce) | ||
const actual = loggerStub.getCall(0).args[0] | ||
assert.strictEqual(actual, 'Unable to write updated text content into the file: writing to file failed.') | ||
}) | ||
await targetCommand.execute(codeScanIssue, fileName, 'hover') | ||
|
||
sandbox.restore() | ||
assert.ok(saveStub.calledOnce) | ||
assert.ok(loggerStub.calledOnce) | ||
const actual = loggerStub.getCall(0).args[0] | ||
assert.strictEqual(actual, 'Apply fix command failed. Error: Writing to file failed.') | ||
assertTelemetry('codewhisperer_codeScanIssueApplyFix', { | ||
detectorId: codeScanIssue.detectorId, | ||
findingId: codeScanIssue.findingId, | ||
applyFixSource: 'hover', | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: can we use different source values ( |
||
result: 'Failed', | ||
reason: 'Error: Writing to file failed.', | ||
}) | ||
}) | ||
}) | ||
}) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
you could add these values to the existing
component
type: https://github.com/aws/aws-toolkit-common/blob/bbc34c2f5b5f1f0a24befa5c42f4193e3e12032c/telemetry/definitions/commonDefinitions.json#L57then just use
component
incodewhisperer_codeScanIssueApplyFix