Add completion items for operators and variable flags

server/src/BitBakeDocScanner.ts

@@ -6,7 +6,7 @@
 import path from 'path'
 import fs from 'fs'
 import logger from 'winston'
-import { type CompletionItem } from 'vscode-languageserver'
+import { CompletionItemKind, type CompletionItem } from 'vscode-languageserver'
 
 type SuffixType = 'layer' | 'providedItem' | undefined
 
@@ -46,11 +46,12 @@ const variablesRegexForDoc = /^ {3}:term:`(?<name>[A-Z_]*?)`\n(?<definition>.*?)
 
 const yoctoTaskFilePath = path.join(__dirname, '../src/yocto-docs/tasks.rst')
 const yoctoTaskPattern = /(?<=``)((?<name>do_.*)``\n-*\n\n(?<description>(.*\n)*?))\n(?=(\.\. _ref)|((.*)\n(=+)))/g
+
 export class BitBakeDocScanner {
   private _variablesInfos: Record<string, VariableInfos> = {}
   private _variablesRegex = /(?!)/g // Initialize with dummy regex that won't match anything so we don't have to check for undefined
   private _yoctoTasks: CompletionItem[] = []
-
+  private _variableFlags: CompletionItem[] = []
   get variablesInfos (): Record<string, VariableInfos> {
     return this._variablesInfos
   }
@@ -63,6 +64,10 @@ export class BitBakeDocScanner {
     return this._yoctoTasks
   }
 
+  get variablFlags (): CompletionItem[] {
+    return this._variableFlags
+  }
+
   parse (pathToBitbakeFolder: string): void {
     let file = ''
     const docPath = path.join(pathToBitbakeFolder, variablesFolder)
@@ -120,7 +125,10 @@ export class BitBakeDocScanner {
       if (taskName !== undefined) {
         tasks.push({
           label: taskName,
-          documentation: taskDescription
+          documentation: taskDescription,
+          data: {
+            referenceUrl: `https://docs.yoctoproject.org/singleindex.html#${taskName.replace(/_/g, '-')}`
+          }
         })
       }
     }
@@ -136,6 +144,55 @@ export class BitBakeDocScanner {
 
     this._yoctoTasks = tasks
   }
+
+  public parseVariableFlag (pathToBitbakeFolder: string): void {
+    const variableFlagFilePath = 'doc/bitbake-user-manual/bitbake-user-manual-metadata.rst'
+    const variableFlagSectionRegex = /(?<=Variable Flags\n=*\n\n)(?<variable_flag_section>.*\n)*(?<event_section_title>Events\n=*)/g
+    const variableFlagRegex = /(?<=-\s*``\[)(?<name>.*)(?:\]``:)(?<description>(.*\n)*?)(?=\n-\s*``|\nEvents)/g
+
+    const completeVariableFlagFilePath = path.join(pathToBitbakeFolder, variableFlagFilePath)
+    let file = ''
+    try {
+      file = fs.readFileSync(completeVariableFlagFilePath, 'utf8')
+    } catch {
+      logger.error(`Failed to read file at ${completeVariableFlagFilePath}`)
+    }
+
+    const variableFlagSection = file.match(variableFlagSectionRegex)
+    if (variableFlagSection === null) {
+      logger.warn(`No variable flag section found at ${completeVariableFlagFilePath}. Is the regex correct?`)
+      return
+    }
+
+    const variableFlags: CompletionItem[] = []
+    for (const match of variableFlagSection[0].matchAll(variableFlagRegex)) {
+      const name = match.groups?.name
+      const description = match.groups?.description
+        .replace(/:term:|:ref:/g, '')
+        .replace(/::/g, ':')
+        .replace(/``\[/g, '`')
+        .replace(/\]``/g, '`')
+        .replace(/\.\.\s/g, '')
+        .replace(/-\s\s/g, '')
+        .replace(/^\n(\s{2,})/gm, '')
+      if (name === undefined || description === undefined) {
+        return
+      }
+
+      if (name !== undefined) {
+        variableFlags.push({
+          label: name,
+          documentation: description,
+          kind: CompletionItemKind.Keyword,
+          data: {
+            referenceUrl: 'https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-metadata.html#variable-flags'
+          }
+        })
+      }
+    }
+
+    this._variableFlags = variableFlags
+  }
 }
 
 export const bitBakeDocScanner = new BitBakeDocScanner()

server/src/__tests__/completions.test.ts

@@ -117,4 +117,107 @@ describe('On Completion', () => {
 
     expect(result).not.toEqual([])
   })
+
+  it('provides suggestions for overrides when a ":" is typed and it follows an identifier', async () => {
+    await analyzer.analyze({
+      uri: DUMMY_URI,
+      document: FIXTURE_DOCUMENT.COMPLETION
+    })
+
+    const result = onCompletionHandler({
+      textDocument: {
+        uri: DUMMY_URI
+      },
+      position: {
+        line: 2,
+        character: 6
+      }
+    })
+
+    expect(result).toEqual(
+      expect.arrayContaining([
+        {
+          label: 'append',
+          kind: 24
+        }
+        // TODO: add an override item
+      ])
+    )
+  })
+
+  it('provides no suggestions when a ":" is typed but it doesn\'t follow an identifier', async () => {
+    await analyzer.analyze({
+      uri: DUMMY_URI,
+      document: FIXTURE_DOCUMENT.COMPLETION
+    })
+
+    const result = onCompletionHandler({
+      textDocument: {
+        uri: DUMMY_URI
+      },
+      position: {
+        line: 13,
+        character: 1
+      }
+    })
+
+    expect(result).toEqual([])
+  })
+
+  it('provides fallback suggestions for variable flags when a "[" is typed and it follows an identifier', async () => {
+    // TODO: This one only tests the fallback suggestions, the ones from docs require scanning docs in bitbake folder
+    await analyzer.analyze({
+      uri: DUMMY_URI,
+      document: FIXTURE_DOCUMENT.COMPLETION
+    })
+
+    const result = onCompletionHandler({
+      textDocument: {
+        uri: DUMMY_URI
+      },
+      position: {
+        line: 3,
+        character: 6
+      }
+    })
+
+    expect(result).toEqual(
+      expect.arrayContaining([
+        {
+          label: 'cleandirs',
+          kind: 14
+        }
+      ])
+    )
+  })
+
+  it('provides no suggestions when a "[" is typed but it doesn\'t follow a bitbake identifier', async () => {
+    await analyzer.analyze({
+      uri: DUMMY_URI,
+      document: FIXTURE_DOCUMENT.COMPLETION
+    })
+
+    const result1 = onCompletionHandler({
+      textDocument: {
+        uri: DUMMY_URI
+      },
+      position: {
+        line: 12,
+        character: 1
+      }
+    })
+
+    const result2 = onCompletionHandler({
+      textDocument: {
+        uri: DUMMY_URI
+      },
+      position: {
+        line: 6,
+        character: 10
+      }
+    })
+
+    expect(result1).toEqual([])
+    expect(result2).toEqual([])
+  })
 })

server/src/__tests__/fixtures/completion.bb

@@ -1,2 +1,14 @@
 FOO = '123'
-MYVAR = 'F${F}'
+MYVAR = 'F${F}'
+MYVAR:append = '123'
+MYVAR[doc] 'this is docs'
+python (){
+    myvar = [1,2,3]
+    myvar[0] = 4
+}
+
+
+# Wrong code
+
+[]
+:

server/src/completions/bitbake-operator.ts

@@ -0,0 +1,10 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) 2023 Savoir-faire Linux. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+export const BITBAKE_OPERATOR = [
+  'append',
+  'prepend',
+  'remove'
+]

server/src/completions/snippet-utils.ts

@@ -12,9 +12,9 @@ import { InsertTextFormat, type CompletionItem, CompletionItemKind, MarkupKind }
 
 /* eslint-disable no-template-curly-in-string */
 
-export function formatYoctoTaskSnippets (completions: CompletionItem[]): CompletionItem[] {
+export function formatCompletionItems (completions: CompletionItem[]): CompletionItem[] {
   return completions.map((item) => {
-    return {
+    const formatted = {
       ...item,
       insertTextFormat: InsertTextFormat.Snippet,
       documentation: {
@@ -26,13 +26,17 @@ export function formatYoctoTaskSnippets (completions: CompletionItem[]): Complet
           markdownBlock(item.insertText?.replace(/\$\{\d+:(?<code>.*)\}/g, (m, p1) => p1), 'bitbake'),
           '---',
           `${JSON.parse(JSON.stringify(item.documentation))}`,
-          `[Reference](https://docs.yoctoproject.org/singleindex.html#${item.label.replace(/_/g, '-')})`
+          item.data?.referenceUrl !== '' ? `[Reference](${item.data?.referenceUrl})` : ''
         ].join('\n'),
         kind: MarkupKind.Markdown
       },
 
       kind: CompletionItemKind.Snippet
     }
+
+    const { data, ...filtered } = formatted
+
+    return filtered
   })
 }
 

server/src/completions/variable-flags.ts

@@ -0,0 +1,29 @@
+/* --------------------------------------------------------------------------------------------
+ * Copyright (c) 2023 Savoir-faire Linux. All rights reserved.
+ * Licensed under the MIT License. See License.txt in the project root for license information.
+ * ------------------------------------------------------------------------------------------ */
+
+export const VARIABLE_FLAGS = [
+  'cleandirs',
+  'depends',
+  'deptask',
+  'dirs',
+  'file-checksums',
+  'lockfiles',
+  'network',
+  'noexec',
+  'nostamp',
+  'number_threads',
+  'postfuncs',
+  'prefuncs',
+  'rdepends',
+  'rdeptask',
+  'recideptask',
+  'recrdeptask',
+  'stamp-extra-info',
+  'umask',
+  'vardeps',
+  'vardepsexclude',
+  'vardepvalue',
+  'vardepvalueexclude'
+]

server/src/connectionHandlers/onCompletion.ts

@@ -9,8 +9,10 @@ import { symbolKindToCompletionKind } from '../utils/lsp'
 import { BITBAKE_VARIABLES } from '../completions/bitbake-variables'
 import { RESERVED_KEYWORDS } from '../completions/reserved-keywords'
 import { analyzer } from '../tree-sitter/analyzer'
-import { formatYoctoTaskSnippets } from '../completions/snippet-utils'
+import { formatCompletionItems } from '../completions/snippet-utils'
 import { bitBakeDocScanner } from '../BitBakeDocScanner'
+import { BITBAKE_OPERATOR } from '../completions/bitbake-operator'
+import { VARIABLE_FLAGS } from '../completions/variable-flags'
 
 export function onCompletionHandler (textDocumentPositionParams: TextDocumentPositionParams): CompletionItem[] {
   logger.debug('onCompletion')
@@ -34,6 +36,57 @@ export function onCompletionHandler (textDocumentPositionParams: TextDocumentPos
     return []
   }
 
+  // bitbake operators
+  if (word === ':') {
+    const wordBeforeIsIdentifier = analyzer.isIdentifier({
+      ...textDocumentPositionParams,
+      position: {
+        line: textDocumentPositionParams.position.line,
+        // Go two character back as one character back is ':'
+        character: Math.max(textDocumentPositionParams.position.character - 2, 0)
+      }
+    })
+    if (wordBeforeIsIdentifier) {
+      const bitBakeOperatorCompletionItems: CompletionItem[] = BITBAKE_OPERATOR.map(keyword => {
+        return {
+          label: keyword,
+          kind: CompletionItemKind.Operator
+        }
+      })
+
+      return bitBakeOperatorCompletionItems
+    } else {
+      return []
+    }
+  }
+  // TODO: add dynamic overrides, completion type: Property
+
+  // variable flags
+  if (word === '[') {
+    const wordBeforeIsIdentifier = analyzer.isIdentifier({
+      ...textDocumentPositionParams,
+      position: {
+        line: textDocumentPositionParams.position.line,
+        // Go two character back as one character back is ':'
+        character: Math.max(textDocumentPositionParams.position.character - 2, 0)
+      }
+    })
+    if (wordBeforeIsIdentifier) {
+      const variableFlagsFromScanner: CompletionItem[] = formatCompletionItems(bitBakeDocScanner.variablFlags)
+
+      const variableFlagCompletionItems: CompletionItem[] = VARIABLE_FLAGS.map(keyword => {
+        return {
+          label: keyword,
+          kind: CompletionItemKind.Keyword
+        }
+      })
+
+      return variableFlagsFromScanner.length > 0 ? variableFlagsFromScanner : variableFlagCompletionItems
+    } else {
+      return []
+    }
+  }
+
   let symbolCompletions: CompletionItem[] = []
   if (word !== null) {
     const globalDeclarationSymbols = analyzer.getGlobalDeclarationSymbols(textDocumentPositionParams.textDocument.uri)
@@ -61,7 +114,7 @@ export function onCompletionHandler (textDocumentPositionParams: TextDocumentPos
     }
   })
 
-  const yoctoTaskSnippets: CompletionItem[] = formatYoctoTaskSnippets(bitBakeDocScanner.yoctoTasks)
+  const yoctoTaskSnippets: CompletionItem[] = formatCompletionItems(bitBakeDocScanner.yoctoTasks)
 
   const allCompletions = [
     ...reserverdKeywordCompletionItems,

server/src/server.ts

@@ -46,7 +46,8 @@ connection.onInitialize(async (params: InitializeParams): Promise<InitializeResu
     capabilities: {
       textDocumentSync: TextDocumentSyncKind.Incremental,
       completionProvider: {
-        resolveProvider: true
+        resolveProvider: true,
+        triggerCharacters: [':', '[']
       },
       definitionProvider: true,
       executeCommandProvider: {
@@ -84,7 +85,7 @@ connection.onDidChangeConfiguration((change) => {
   bitBakeProjectScanner.pathToBuildFolder = settings.bitbake.pathToBuildFolder
   bitBakeProjectScanner.pathToBitbakeFolder = settings.bitbake.pathToBitbakeFolder
   bitBakeDocScanner.parse(settings.bitbake.pathToBitbakeFolder)
-
+  bitBakeDocScanner.parseVariableFlag(settings.bitbake.pathToBitbakeFolder)
   bitBakeProjectScanner.rescanProject()
 })