Fix scanner for Windows line endings and add tests

hcl/scanner/scanner.go

@@ -224,7 +224,11 @@ func (s *Scanner) scanComment(ch rune) {
  // single line comments
  if ch == '#' || (ch == '/' && s.peek() != '*') {
  ch = s.next()
- for ch != '\n' && ch >= 0 && ch != eof {
+ for ch != '\n' && ch != '\r' && ch >= 0 && ch != eof {
+ if ch == '\r' && s.peek() == '\n' {
+ ch = s.next()
+ break
+ }
  ch = s.next()
  }
  if ch != eof && ch >= 0 {
@@ -399,6 +403,13 @@ func (s *Scanner) scanHeredoc() {
  return
  }
 
+ // Ignore the '\r' in Windows line endings
+ if ch == '\r' {
+ if s.peek() == '\n' {
+ ch = s.next()
+ }
+ }
+
  // If we didn't reach a newline then that is also not good
  if ch != '\n' {
  s.err("invalid characters in heredoc anchor")

hcl/scanner/scanner_test.go

@@ -6,6 +6,7 @@ import (
  "testing"
 
  "github.com/hashicorp/hcl/hcl/token"
+ "strings"
 )
 
 var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
@@ -307,6 +308,44 @@ func TestFloat(t *testing.T) {
  testTokenList(t, tokenLists["float"])
 }
 
+func TestWindowsLineEndings(t *testing.T) {
+ hcl := `// This should have Windows line endings
+resource "aws_instance" "foo" {
+ user_data=<<HEREDOC
+ test script
+HEREDOC
+}`
+ hclWindowsEndings := strings.Replace(hcl, "\n", "\r\n", -1)
+
+ literals := []struct {
+ tokenType token.Type
+ literal string
+ }{
+ {token.COMMENT, `// This should have Windows line endings`},
+ {token.IDENT, `resource`},
+ {token.STRING, `"aws_instance"`},
+ {token.STRING, `"foo"`},
+ {token.LBRACE, `{`},
+ {token.IDENT, `user_data`},
+ {token.ASSIGN, `=`},
+ {token.HEREDOC, "<<HEREDOC\r\n test script\r\nHEREDOC\r\n"},
+ {token.RBRACE, `}`},
+ }
+
+ s := New([]byte(hclWindowsEndings))
+ for _, l := range literals {
+ tok := s.Scan()
+
+ if l.tokenType != tok.Type {
+ t.Errorf("got: %s want %s for %s\n", tok, l.tokenType, tok.String())
+ }
+
+ if l.literal != tok.Text {
+ t.Errorf("got: %s want %s\n", tok.Text, l.literal)
+ }
+ }
+}
+
 func TestRealExample(t *testing.T) {
  complexHCL := `// This comes from Terraform, as a test
  variable "foo" {