Skip to content

Commit

Permalink
🔨 rename yytext to token_buffer (fixes #933)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nlohmann
nlohmann committed Jan 29, 2018
1 parent b3bd3b7 commit 8049442
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 36 deletions.
37 changes: 19 additions & 18 deletions develop/detail/input/lexer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,10 @@ class lexer
@brief scan a string literal
This function scans a string according to Sect. 7 of RFC 7159. While
scanning, bytes are escaped and copied into buffer yytext. Then the function
returns successfully, yytext is *not* null-terminated (as it may contain \0
bytes), and yytext.size() is the number of bytes in the string.
scanning, bytes are escaped and copied into buffer token_buffer. Then the
function returns successfully, token_buffer is *not* null-terminated (as it
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.
@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
Expand All @@ -223,7 +224,7 @@ class lexer
*/
token_type scan_string()
{
// reset yytext (ignore opening quote)
// reset token_buffer (ignore opening quote)
reset();

// we entered the function by reading an open quote
Expand Down Expand Up @@ -695,7 +696,7 @@ class lexer
contains cycles, but any cycle can be left when EOF is read. Therefore,
the function is guaranteed to terminate.
During scanning, the read bytes are stored in yytext. This string is
During scanning, the read bytes are stored in token_buffer. This string is
then converted to a signed integer, an unsigned integer, or a
floating-point number.
Expand All @@ -709,7 +710,7 @@ class lexer
*/
token_type scan_number()
{
// reset yytext to store the number's bytes
// reset token_buffer to store the number's bytes
reset();

// the type of the parsed number; initially set to unsigned; will be
Expand Down Expand Up @@ -993,10 +994,10 @@ class lexer
// try to parse integers first and fall back to floats
if (number_type == token_type::value_unsigned)
{
const auto x = std::strtoull(yytext.data(), &endptr, 10);
const auto x = std::strtoull(token_buffer.data(), &endptr, 10);

// we checked the number format before
assert(endptr == yytext.data() + yytext.size());
assert(endptr == token_buffer.data() + token_buffer.size());

if (errno == 0)
{
Expand All @@ -1009,10 +1010,10 @@ class lexer
}
else if (number_type == token_type::value_integer)
{
const auto x = std::strtoll(yytext.data(), &endptr, 10);
const auto x = std::strtoll(token_buffer.data(), &endptr, 10);

// we checked the number format before
assert(endptr == yytext.data() + yytext.size());
assert(endptr == token_buffer.data() + token_buffer.size());

if (errno == 0)
{
Expand All @@ -1026,10 +1027,10 @@ class lexer

// this code is reached if we parse a floating-point number or if an
// integer conversion above failed
strtof(value_float, yytext.data(), &endptr);
strtof(value_float, token_buffer.data(), &endptr);

// we checked the number format before
assert(endptr == yytext.data() + yytext.size());
assert(endptr == token_buffer.data() + token_buffer.size());

return token_type::value_float;
}
Expand Down Expand Up @@ -1058,10 +1059,10 @@ class lexer
// input management
/////////////////////

/// reset yytext; current character is beginning of token
/// reset token_buffer; current character is beginning of token
void reset() noexcept
{
yytext.clear();
token_buffer.clear();
token_string.clear();
token_string.push_back(std::char_traits<char>::to_char_type(current));
}
Expand Down Expand Up @@ -1099,10 +1100,10 @@ class lexer
}
}

/// add a character to yytext
/// add a character to token_buffer
void add(int c)
{
yytext.push_back(std::char_traits<char>::to_char_type(c));
token_buffer.push_back(std::char_traits<char>::to_char_type(c));
}

public:
Expand Down Expand Up @@ -1131,7 +1132,7 @@ class lexer
/// return current string value (implicitly resets the token; useful only once)
std::string move_string()
{
return std::move(yytext);
return std::move(token_buffer);
}

/////////////////////
Expand Down Expand Up @@ -1259,7 +1260,7 @@ class lexer
std::vector<char> token_string {};

/// buffer for variable-length tokens (numbers, strings)
std::string yytext {};
std::string token_buffer {};

/// a description of occurred lexer errors
const char* error_message = "";
Expand Down
37 changes: 19 additions & 18 deletions src/json.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2025,9 +2025,10 @@ class lexer
@brief scan a string literal

This function scans a string according to Sect. 7 of RFC 7159. While
scanning, bytes are escaped and copied into buffer yytext. Then the function
returns successfully, yytext is *not* null-terminated (as it may contain \0
bytes), and yytext.size() is the number of bytes in the string.
scanning, bytes are escaped and copied into buffer token_buffer. Then the
function returns successfully, token_buffer is *not* null-terminated (as it
may contain \0 bytes), and token_buffer.size() is the number of bytes in the
string.

@return token_type::value_string if string could be successfully scanned,
token_type::parse_error otherwise
Expand All @@ -2037,7 +2038,7 @@ class lexer
*/
token_type scan_string()
{
// reset yytext (ignore opening quote)
// reset token_buffer (ignore opening quote)
reset();

// we entered the function by reading an open quote
Expand Down Expand Up @@ -2509,7 +2510,7 @@ class lexer
contains cycles, but any cycle can be left when EOF is read. Therefore,
the function is guaranteed to terminate.

During scanning, the read bytes are stored in yytext. This string is
During scanning, the read bytes are stored in token_buffer. This string is
then converted to a signed integer, an unsigned integer, or a
floating-point number.

Expand All @@ -2523,7 +2524,7 @@ class lexer
*/
token_type scan_number()
{
// reset yytext to store the number's bytes
// reset token_buffer to store the number's bytes
reset();

// the type of the parsed number; initially set to unsigned; will be
Expand Down Expand Up @@ -2807,10 +2808,10 @@ class lexer
// try to parse integers first and fall back to floats
if (number_type == token_type::value_unsigned)
{
const auto x = std::strtoull(yytext.data(), &endptr, 10);
const auto x = std::strtoull(token_buffer.data(), &endptr, 10);

// we checked the number format before
assert(endptr == yytext.data() + yytext.size());
assert(endptr == token_buffer.data() + token_buffer.size());

if (errno == 0)
{
Expand All @@ -2823,10 +2824,10 @@ class lexer
}
else if (number_type == token_type::value_integer)
{
const auto x = std::strtoll(yytext.data(), &endptr, 10);
const auto x = std::strtoll(token_buffer.data(), &endptr, 10);

// we checked the number format before
assert(endptr == yytext.data() + yytext.size());
assert(endptr == token_buffer.data() + token_buffer.size());

if (errno == 0)
{
Expand All @@ -2840,10 +2841,10 @@ class lexer

// this code is reached if we parse a floating-point number or if an
// integer conversion above failed
strtof(value_float, yytext.data(), &endptr);
strtof(value_float, token_buffer.data(), &endptr);

// we checked the number format before
assert(endptr == yytext.data() + yytext.size());
assert(endptr == token_buffer.data() + token_buffer.size());

return token_type::value_float;
}
Expand Down Expand Up @@ -2872,10 +2873,10 @@ class lexer
// input management
/////////////////////

/// reset yytext; current character is beginning of token
/// reset token_buffer; current character is beginning of token
void reset() noexcept
{
yytext.clear();
token_buffer.clear();
token_string.clear();
token_string.push_back(std::char_traits<char>::to_char_type(current));
}
Expand Down Expand Up @@ -2913,10 +2914,10 @@ class lexer
}
}

/// add a character to yytext
/// add a character to token_buffer
void add(int c)
{
yytext.push_back(std::char_traits<char>::to_char_type(c));
token_buffer.push_back(std::char_traits<char>::to_char_type(c));
}

public:
Expand Down Expand Up @@ -2945,7 +2946,7 @@ class lexer
/// return current string value (implicitly resets the token; useful only once)
std::string move_string()
{
return std::move(yytext);
return std::move(token_buffer);
}

/////////////////////
Expand Down Expand Up @@ -3073,7 +3074,7 @@ class lexer
std::vector<char> token_string {};

/// buffer for variable-length tokens (numbers, strings)
std::string yytext {};
std::string token_buffer {};

/// a description of occurred lexer errors
const char* error_message = "";
Expand Down

0 comments on commit 8049442

Please sign in to comment.