sat-arithmetic: correctly tokenize <<|, <<|=

- set state rather than result.tag in tokenizer.zig
- add test to tokenizer.zig for <<, <<|, <<|=

lib/std/zig/tokenizer.zig

@@ -996,7 +996,7 @@ pub const Tokenizer = struct {
                         break;
                     },
                     '|' => {
-                        result.tag = .angle_bracket_angle_bracket_left_pipe;
+                        state = .angle_bracket_angle_bracket_left_pipe;
                     },
                     else => {
                         result.tag = .angle_bracket_angle_bracket_left;
@@ -1998,6 +1998,12 @@ test "tokenizer - invalid builtin identifiers" {
     try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
 }
 
+test "tokenizer - saturating" {
+    try testTokenize("<<", &.{ .angle_bracket_angle_bracket_left });
+    try testTokenize("<<|", &.{ .angle_bracket_angle_bracket_left_pipe });
+    try testTokenize("<<|=", &.{ .angle_bracket_angle_bracket_left_pipe_equal });
+}
+
 fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {

src/stage1/tokenizer.cpp

@@ -995,7 +995,6 @@ void tokenize(const char *source, Tokenization *out) {
                         t.state = TokenizeState_start;
                         break;
                     case '|':
-                        // t.out->ids.last() = TokenIdBitShiftLeftPipe;
                         t.state = TokenizeState_angle_bracket_angle_bracket_left_pipe;
                         break;
                     default: