go/scanner: fall back to next() when encountering 0 bytes in parseIde…

…ntifier

CL 308611 optimized parseIdentifier for ASCII, but inadvertently skipped
error handling for 0 bytes. Don't take the optimized path when
encountering 0.

Fixes #46855

Change-Id: Ic584e077eb74c012611fefa20eb71ca09c81b3c7
Reviewed-on: https://go-review.googlesource.com/c/go/+/329790
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

src/go/scanner/scanner.go

@@ -373,7 +373,7 @@ func (s *Scanner) scanIdentifier() string {
  continue
  }
  s.rdOffset += rdOffset
- if b < utf8.RuneSelf {
+ if 0 < b && b < utf8.RuneSelf {
  // Optimization: we've encountered an ASCII character that's not a letter
  // or number. Avoid the call into s.next() and corresponding set up.
  //

src/go/scanner/scanner_test.go

@@ -812,6 +812,8 @@ var errors = []struct {
  {"//\ufeff", token.COMMENT, 2, "//\ufeff", "illegal byte order mark"}, // only first BOM is ignored
  {"'\ufeff" + `'`, token.CHAR, 1, "'\ufeff" + `'`, "illegal byte order mark"}, // only first BOM is ignored
  {`"` + "abc\ufeffdef" + `"`, token.STRING, 4, `"` + "abc\ufeffdef" + `"`, "illegal byte order mark"}, // only first BOM is ignored
+ {"abc\x00def", token.IDENT, 3, "abc", "illegal character NUL"},
+ {"abc\x00", token.IDENT, 3, "abc", "illegal character NUL"},
 }
 
 func TestScanErrors(t *testing.T) {