[dev.typeparams] go/token, go/scanner: add the "~" operator

This is an approximate port of CL 307370 to go/token and go/scanner.

Change-Id: I5b789408f825f7e39f569322cb67802117b9d734
Reviewed-on: https://go-review.googlesource.com/c/go/+/324992
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/go/scanner/scanner.go

@@ -969,6 +969,8 @@ scanAgain:
  }
  case '|':
  tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+ case '~':
+ tok = token.TILDE
  default:
  // next reports unexpected BOMs - don't repeat
  if ch != bom {

src/go/scanner/scanner_test.go

@@ -40,7 +40,7 @@ type elt struct {
  class int
 }
 
-var tokens = [...]elt{
+var tokens = []elt{
  // Special tokens
  {token.COMMENT, "/* a comment */", special},
  {token.COMMENT, "// a comment \n", special},
@@ -149,6 +149,7 @@ var tokens = [...]elt{
  {token.RBRACE, "}", operator},
  {token.SEMICOLON, ";", operator},
  {token.COLON, ":", operator},
+ {token.TILDE, "~", operator},
 
  // Keywords
  {token.BREAK, "break", keyword},

src/go/token/token.go

@@ -125,6 +125,11 @@ const (
  TYPE
  VAR
  keyword_end
+
+ additional_beg
+ // additional tokens, handled in an ad-hoc manner
+ TILDE
+ additional_end
 )
 
 var tokens = [...]string{
@@ -225,6 +230,8 @@ var tokens = [...]string{
  SWITCH: "switch",
  TYPE: "type",
  VAR: "var",
+
+ TILDE: "~",
 }
 
 // String returns the string corresponding to the token tok.
@@ -304,7 +311,9 @@ func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_en
 // IsOperator returns true for tokens corresponding to operators and
 // delimiters; it returns false otherwise.
 //
-func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
+func (tok Token) IsOperator() bool {
+ return (operator_beg < tok && tok < operator_end) || tok == TILDE
+}
 
 // IsKeyword returns true for tokens corresponding to keywords;
 // it returns false otherwise.