Skip to content

Commit

Permalink
semantic tokens
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpogran
jpogran committed Sep 27, 2022
1 parent 2da2a6d commit 2b2b977
Show file tree
Hide file tree
Showing 3 changed files with 310 additions and 35 deletions.
96 changes: 73 additions & 23 deletions decoder/semantic_tokens.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
package decoder

import (
"context"
"sort"

"github.com/zclconf/go-cty/cty"

icontext "github.com/hashicorp/hcl-lang/context"
"github.com/hashicorp/hcl-lang/lang"
"github.com/hashicorp/hcl-lang/reference"
"github.com/hashicorp/hcl-lang/schema"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
)

// SemanticTokensInFile returns a sequence of semantic tokens
// within the config file.
func (d *PathDecoder) SemanticTokensInFile(filename string) ([]lang.SemanticToken, error) {
func (d *PathDecoder) SemanticTokensInFile(ctx context.Context, filename string) ([]lang.SemanticToken, error) {
f, err := d.fileByName(filename)
if err != nil {
return nil, err
Expand All @@ -28,7 +31,7 @@ func (d *PathDecoder) SemanticTokensInFile(filename string) ([]lang.SemanticToke
return []lang.SemanticToken{}, nil
}

tokens := d.tokensForBody(body, d.pathCtx.Schema, []lang.SemanticTokenModifier{})
tokens := d.tokensForBody(ctx, body, d.pathCtx.Schema, []lang.SemanticTokenModifier{})

sort.Slice(tokens, func(i, j int) bool {
return tokens[i].Range.Start.Byte < tokens[j].Range.Start.Byte
Expand All @@ -37,21 +40,43 @@ func (d *PathDecoder) SemanticTokensInFile(filename string) ([]lang.SemanticToke
return tokens, nil
}

func (d *PathDecoder) tokensForBody(body *hclsyntax.Body, bodySchema *schema.BodySchema, parentModifiers []lang.SemanticTokenModifier) []lang.SemanticToken {
func (d *PathDecoder) tokensForBody(ctx context.Context, body *hclsyntax.Body, bodySchema *schema.BodySchema, parentModifiers []lang.SemanticTokenModifier) []lang.SemanticToken {
tokens := make([]lang.SemanticToken, 0)

if bodySchema == nil {
return tokens
}

if bodySchema.Extensions != nil {
ctx = icontext.WithExtensions(ctx, bodySchema.Extensions)
if bodySchema.Extensions.Count {
if _, ok := body.Attributes["count"]; ok {
// append to context we need count provided
ctx = icontext.WithActiveCount(ctx)
}
}
}

for name, attr := range body.Attributes {

attrSchema, ok := bodySchema.Attributes[name]
if !ok {
if bodySchema.AnyAttribute == nil {
// unknown attribute
continue
if bodySchema.Extensions != nil && name == "count" && bodySchema.Extensions.Count {
attrSchema = &schema.AttributeSchema{
Description: lang.MarkupContent{
Kind: lang.MarkdownKind,
Value: "**count** _optional, number_\n\nThe distinct index number (starting with 0) corresponding to the instance",
},
IsOptional: true,
Expr: schema.LiteralTypeOnly(cty.Number),
}
} else {
if bodySchema.AnyAttribute == nil {
// unknown attribute
continue
}
attrSchema = bodySchema.AnyAttribute
}
attrSchema = bodySchema.AnyAttribute
}

attrModifiers := make([]lang.SemanticTokenModifier, 0)
Expand All @@ -65,7 +90,7 @@ func (d *PathDecoder) tokensForBody(body *hclsyntax.Body, bodySchema *schema.Bod
})

ec := ExprConstraints(attrSchema.Expr)
tokens = append(tokens, d.tokensForExpression(attr.Expr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, attr.Expr, ec)...)
}

for _, block := range body.Blocks {
Expand Down Expand Up @@ -106,19 +131,19 @@ func (d *PathDecoder) tokensForBody(body *hclsyntax.Body, bodySchema *schema.Bod
}

if block.Body != nil {
tokens = append(tokens, d.tokensForBody(block.Body, blockSchema.Body, blockModifiers)...)
tokens = append(tokens, d.tokensForBody(ctx, block.Body, blockSchema.Body, blockModifiers)...)
}

depSchema, _, ok := NewBlockSchema(blockSchema).DependentBodySchema(block.AsHCLBlock())
if ok {
tokens = append(tokens, d.tokensForBody(block.Body, depSchema, []lang.SemanticTokenModifier{})...)
tokens = append(tokens, d.tokensForBody(ctx, block.Body, depSchema, []lang.SemanticTokenModifier{})...)
}
}

return tokens
}

func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints ExprConstraints) []lang.SemanticToken {
func (d *PathDecoder) tokensForExpression(ctx context.Context, expr hclsyntax.Expression, constraints ExprConstraints) []lang.SemanticToken {
tokens := make([]lang.SemanticToken, 0)

switch eType := expr.(type) {
Expand All @@ -135,6 +160,31 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
}
}

address, _ := lang.TraversalToAddress(eType.AsTraversal())
countIndexAttr := lang.Address{
lang.RootStep{
Name: "count",
},
lang.AttrStep{
Name: "index",
},
}
countAvailable := icontext.ActiveCountFromContext(ctx)
if address.Equals(countIndexAttr) && countAvailable {
traversal := eType.AsTraversal()
tokens = append(tokens, lang.SemanticToken{
Type: lang.TokenTraversalStep,
Modifiers: []lang.SemanticTokenModifier{},
Range: traversal[0].SourceRange(),
})
tokens = append(tokens, lang.SemanticToken{
Type: lang.TokenTraversalStep,
Modifiers: []lang.SemanticTokenModifier{},
Range: traversal[1].SourceRange(),
})
return tokens
}

tes, ok := constraints.TraversalExprs()
if ok && d.pathCtx.ReferenceTargets != nil {
traversal := eType.AsTraversal()
Expand Down Expand Up @@ -231,7 +281,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
Range: eType.NameRange,
})
for _, arg := range eType.Args {
tokens = append(tokens, d.tokensForExpression(arg, constraints)...)
tokens = append(tokens, d.tokensForExpression(ctx, arg, constraints)...)
}
return tokens
}
Expand All @@ -249,29 +299,29 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
return tokenForTypedExpression(eType, cty.String)
}
case *hclsyntax.TemplateWrapExpr:
return d.tokensForExpression(eType.Wrapped, constraints)
return d.tokensForExpression(ctx, eType.Wrapped, constraints)
case *hclsyntax.TupleConsExpr:
tc, ok := constraints.TupleConsExpr()
if ok {
ec := ExprConstraints(tc.AnyElem)
for _, expr := range eType.Exprs {
tokens = append(tokens, d.tokensForExpression(expr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...)
}
return tokens
}
se, ok := constraints.SetExpr()
if ok {
ec := ExprConstraints(se.Elem)
for _, expr := range eType.Exprs {
tokens = append(tokens, d.tokensForExpression(expr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...)
}
return tokens
}
le, ok := constraints.ListExpr()
if ok {
ec := ExprConstraints(le.Elem)
for _, expr := range eType.Exprs {
tokens = append(tokens, d.tokensForExpression(expr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...)
}
return tokens
}
Expand All @@ -282,7 +332,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
break
}
ec := ExprConstraints(te.Elems[i])
tokens = append(tokens, d.tokensForExpression(expr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, expr, ec)...)
}
return tokens
}
Expand Down Expand Up @@ -316,7 +366,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
})

ec := ExprConstraints(attr.Expr)
tokens = append(tokens, d.tokensForExpression(item.ValueExpr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, item.ValueExpr, ec)...)
}
return tokens
}
Expand All @@ -329,7 +379,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
Range: item.KeyExpr.Range(),
})
ec := ExprConstraints(me.Elem)
tokens = append(tokens, d.tokensForExpression(item.ValueExpr, ec)...)
tokens = append(tokens, d.tokensForExpression(ctx, item.ValueExpr, ec)...)
}
return tokens
}
Expand All @@ -343,7 +393,7 @@ func (d *PathDecoder) tokensForExpression(expr hclsyntax.Expression, constraints
}
_, ok = constraints.TypeDeclarationExpr()
if ok {
return d.tokensForObjectConsTypeDeclarationExpr(eType, constraints)
return d.tokensForObjectConsTypeDeclarationExpr(ctx, eType, constraints)
}
case *hclsyntax.LiteralValueExpr:
valType := eType.Val.Type()
Expand Down Expand Up @@ -389,7 +439,7 @@ func isComplexTypeDeclaration(funcName string) bool {
return false
}

func (d *PathDecoder) tokensForObjectConsTypeDeclarationExpr(expr *hclsyntax.ObjectConsExpr, constraints ExprConstraints) []lang.SemanticToken {
func (d *PathDecoder) tokensForObjectConsTypeDeclarationExpr(ctx context.Context, expr *hclsyntax.ObjectConsExpr, constraints ExprConstraints) []lang.SemanticToken {
tokens := make([]lang.SemanticToken, 0)
for _, item := range expr.Items {
key, _ := item.KeyExpr.Value(nil)
Expand All @@ -405,7 +455,7 @@ func (d *PathDecoder) tokensForObjectConsTypeDeclarationExpr(expr *hclsyntax.Obj
Range: item.KeyExpr.Range(),
})

tokens = append(tokens, d.tokensForExpression(item.ValueExpr, constraints)...)
tokens = append(tokens, d.tokensForExpression(ctx, item.ValueExpr, constraints)...)
}
return tokens
}
Expand Down
16 changes: 12 additions & 4 deletions decoder/semantic_tokens_expr_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
package decoder

import (
"context"
"fmt"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/zclconf/go-cty/cty"

"github.com/hashicorp/hcl-lang/lang"
"github.com/hashicorp/hcl-lang/reference"
"github.com/hashicorp/hcl-lang/schema"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
)

func TestDecoder_SemanticTokensInFile_expressions(t *testing.T) {
Expand Down Expand Up @@ -1433,6 +1435,8 @@ EOT
},
}

ctx := context.Background()

for i, tc := range testCases {
t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) {
bodySchema := &schema.BodySchema{
Expand All @@ -1451,7 +1455,7 @@ EOT
},
})

tokens, err := d.SemanticTokensInFile("test.tf")
tokens, err := d.SemanticTokensInFile(ctx, "test.tf")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -1912,6 +1916,8 @@ func TestDecoder_SemanticTokensInFile_traversalExpression(t *testing.T) {
},
}

ctx := context.Background()

for i, tc := range testCases {
t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) {
bodySchema := &schema.BodySchema{
Expand All @@ -1931,7 +1937,7 @@ func TestDecoder_SemanticTokensInFile_traversalExpression(t *testing.T) {
},
})

tokens, err := d.SemanticTokensInFile("test.tf")
tokens, err := d.SemanticTokensInFile(ctx, "test.tf")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -2227,6 +2233,8 @@ func TestDecoder_SemanticTokensInFile_typeDeclaration(t *testing.T) {
},
}

ctx := context.Background()

for i, tc := range testCases {
t.Run(fmt.Sprintf("%d-%s", i, tc.name), func(t *testing.T) {
bodySchema := &schema.BodySchema{
Expand All @@ -2245,7 +2253,7 @@ func TestDecoder_SemanticTokensInFile_typeDeclaration(t *testing.T) {
},
})

tokens, err := d.SemanticTokensInFile("test.tf")
tokens, err := d.SemanticTokensInFile(ctx, "test.tf")
if err != nil {
t.Fatal(err)
}
Expand Down
Loading

0 comments on commit 2b2b977

Please sign in to comment.