[mcs] Implements C# 7 digits separators

mcs/errors/cs1013-2.cs

@@ -0,0 +1,7 @@
+// CS1013: Invalid number
+// Line : 6
+
+class X
+{
+    static int i = 0x0_;
+}

mcs/errors/cs1061-18.cs

@@ -0,0 +1,10 @@
+// CS1061: Type `int' does not contain a definition for `__0' and no extension method `__0' of type `int' could be found. Are you missing an assembly reference?
+// Line: 8
+
+static class C
+{
+	static void Main ()
+	{
+		int c = 0.__0;
+	}
+}

mcs/errors/cs1644-61.cs

@@ -0,0 +1,11 @@
+// CS1644: Feature `digit separators' cannot be used because it is not part of the C# 6.0 language specification
+// Line: 9
+// Compiler options: -langversion:6
+
+class X
+{
+	int Test ()
+	{
+		var i = 1_0;
+	}
+}

mcs/mcs/cs-tokenizer.cs

@@ -1575,20 +1575,22 @@ bool decimal_digits (int c)
 					Error_NumericConstantTooLong ();
 				number_builder [number_pos++] = (char) c;
 			}
-			
+
 			//
 			// We use peek_char2, because decimal_digits needs to do a 
 			// 2-character look-ahead (5.ToString for example).
 			//
 			while ((d = peek_char2 ()) != -1){
-				if (d >= '0' && d <= '9'){
+				if (d >= '0' && d <= '9') {
 					if (number_pos == MaxNumberLength)
 						Error_NumericConstantTooLong ();
-					number_builder [number_pos++] = (char) d;
+					number_builder [number_pos++] = (char)d;
 					get_char ();
 					seen_digits = true;
-				} else
-					break;
+					continue;
+				}
+
+				break;
 			}
 
 			return seen_digits;
@@ -1718,9 +1720,8 @@ ILiteralConstant adjust_int (int c, Location loc)
 			} catch (OverflowException) {
 				Error_NumericConstantTooLong ();
 				return new IntLiteral (context.BuiltinTypes, 0, loc);
-			}
-			catch (FormatException) {
-				Report.Error (1013, Location, "Invalid number");
+			} catch (FormatException) {
+				Error_InvalidNumber ();
 				return new IntLiteral (context.BuiltinTypes, 0, loc);
 			}
 		}
@@ -1759,14 +1760,39 @@ ILiteralConstant handle_hex (Location loc)
 		{
 			int d;
 			ulong ul;
+			bool digit_separator = false;
+			int prev = 0;
 
 			get_char ();
 			while ((d = peek_char ()) != -1){
 				if (is_hex (d)){
 					number_builder [number_pos++] = (char) d;
 					get_char ();
-				} else
-					break;
+
+					prev = d;
+					continue;
+				}
+
+				if (d == '_') {
+					get_char ();
+
+					if (!digit_separator) {
+						if (context.Settings.Version < LanguageVersion.V_7)
+							Report.FeatureIsNotAvailable (context, Location, "digit separators");
+
+						digit_separator = true;
+					}
+
+					prev = d;
+					continue;
+				}
+
+				break;
+			}
+
+			if (number_pos == 0 || prev == '_') {
+				Error_InvalidNumber ();
+				return new IntLiteral (context.BuiltinTypes, 0, loc);
 			}
 
 			string s = new String (number_builder, 0, number_pos);
@@ -1781,9 +1807,8 @@ ILiteralConstant handle_hex (Location loc)
 			} catch (OverflowException){
 				Error_NumericConstantTooLong ();
 				return new IntLiteral (context.BuiltinTypes, 0, loc);
-			}
-			catch (FormatException) {
-				Report.Error (1013, Location, "Invalid number");
+			} catch (FormatException) {
+				Error_InvalidNumber ();
 				return new IntLiteral (context.BuiltinTypes, 0, loc);
 			}
 		}
@@ -1792,27 +1817,50 @@ ILiteralConstant handle_binary (Location loc)
 		{
 			int d;
 			ulong ul = 0;
-
-			get_char ();
+			bool digit_separator = false;
+			int prev = -1;
 			int digits = 0;
+
+			get_char ();
 			while ((d = peek_char ()) != -1){
-				if (d == '0' || d == '1'){
+
+				if (d == '0' || d == '1') {
 					ul = (ul << 1);
 					digits++;
 					if (d == '1')
 						ul |= 1;
 					get_char ();
-					if (digits > 64){
+					if (digits > 64) {
 						Error_NumericConstantTooLong ();
 						return new IntLiteral (context.BuiltinTypes, 0, loc);
 					}
-				} else
-					break;
+
+					prev = d;
+					continue;
+				}
+
+				if (d == '_') {
+					get_char ();
+
+					if (!digit_separator) {
+						if (context.Settings.Version < LanguageVersion.V_7)
+							Report.FeatureIsNotAvailable (context, Location, "digit separators");
+
+						digit_separator = true;
+					}
+
+					prev = d;
+					continue;
+				}
+
+				break;
 			}
-			if (digits == 0){
-				Report.Error (1013, Location, "Invalid number");
+
+			if (digits == 0 || prev == '_') {
+				Error_InvalidNumber ();
 				return new IntLiteral (context.BuiltinTypes, 0, loc);
 			}
+
 			return integer_type_suffix (ul, peek_char (), loc);
 		}
 
@@ -1834,6 +1882,7 @@ int is_number (int c, bool dotLead)
 #endif
 			number_pos = 0;
 			var loc = Location;
+			bool digit_separator = false;
 
 			if (!dotLead){
 				if (c == '0'){
@@ -1846,11 +1895,12 @@ int is_number (int c, bool dotLead)
 #endif
 
 						return Token.LITERAL;
-					} else if (peek == 'b' || peek == 'B'){
-						if (context.Settings.Version < LanguageVersion.V_7){
+					}
+
+					if (peek == 'b' || peek == 'B'){
+						if (context.Settings.Version < LanguageVersion.V_7)
 							Report.FeatureIsNotAvailable (context, Location, "binary literals");
-							return Token.ERROR;
-						}
+
 						val = res = handle_binary (loc);
 #if FULL_AST
 						res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
@@ -1859,10 +1909,32 @@ int is_number (int c, bool dotLead)
 						return Token.LITERAL;
 					}
 				}
+
+			digits:
 				decimal_digits (c);
 				c = peek_char ();
+
+				if (c == '_') {
+					if (!digit_separator) {
+						if (context.Settings.Version < LanguageVersion.V_7)
+							Report.FeatureIsNotAvailable (context, Location, "digit separators");
+
+						digit_separator = true;
+					}
+
+					do {
+						get_char ();
+						c = peek_char ();
+					} while (c == '_');
+
+					if (c >= '0' && c <= '9')
+						goto digits;
+				}
 			}
 
+			//TODO: Implement rejection of trailing digit separators
+
+
 			//
 			// We need to handle the case of
 			// "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
@@ -1910,9 +1982,28 @@ int is_number (int c, bool dotLead)
 						Error_NumericConstantTooLong ();
 					number_builder [number_pos++] = '+';
 				}
-
-				decimal_digits (c);
+
+			digits:
+				bool seen_digits = decimal_digits (c);
 				c = peek_char ();
+
+				if (c == '_' && seen_digits) {
+					if (!digit_separator) {
+						if (context.Settings.Version < LanguageVersion.V_7)
+							Report.FeatureIsNotAvailable (context, Location, "digit separators");
+
+						digit_separator = true;
+					}
+
+					do {
+						get_char ();
+						c = peek_char ();
+					} while (c == '_');
+
+					if (c >= '0' && c <= '9')
+						goto digits;
+				}
+
 			}
 
 			var type = real_type_suffix (c);
@@ -2985,6 +3076,11 @@ void Error_NumericConstantTooLong ()
 		{
 			Report.Error (1021, Location, "Integral constant is too large");			
 		}
+
+		void Error_InvalidNumber ()
+		{
+			Report.Error (1013, Location, "Invalid number");
+		}
 
 		void Error_InvalidDirective ()
 		{

mcs/tests/test-950.cs

@@ -0,0 +1,12 @@
+using System;
+
+public class B
+{
+	public static void Main ()
+	{
+		int a = 1_0_3;
+		double b = 0__0e+1_1;
+		int c = 0b__1_0;
+		int d = 0x__F_0;
+	}
+}

mcs/tests/ver-il-net_4_x.xml

@@ -52849,6 +52849,16 @@
       </method>
     </type>
   </test>
+  <test name="test-950.cs">
+    <type name="B">
+      <method name="Void Main()" attrs="150">
+        <size>26</size>
+      </method>
+      <method name="Void .ctor()" attrs="6278">
+        <size>7</size>
+      </method>
+    </type>
+  </test>
   <test name="test-96.cs">
     <type name="N1.A">
       <method name="Int32 Main()" attrs="150">