yul literal value is struct of u256 + optional formatting hint or 'unlimited'

- unlimited is a string literal for which bytes restrictions don't hold; can occur in builtin function arguments

Author: clonker

libsolidity/codegen/ir/IRGeneratorForStatements.cpp

@@ -37,6 +37,7 @@
 #include <libyul/AsmPrinter.h>
 #include <libyul/AST.h>
 #include <libyul/Dialect.h>
+#include <libyul/Utilities.h>
 #include <libyul/optimiser/ASTCopier.h>
 
 #include <liblangutil/Exceptions.h>
@@ -203,7 +204,7 @@ struct CopyTranslate: public yul::ASTCopier
 			solAssert(false);
 
 		if (isDigit(value.front()))
-			return yul::Literal{_identifier.debugData, yul::LiteralKind::Number, yul::YulString{value}, {}};
+			return yul::Literal{_identifier.debugData, yul::LiteralKind::Number, yul::valueOfNumberLiteral(value), {}};
 		else
 			return yul::Identifier{_identifier.debugData, yul::YulString{value}};
 	}

libsolutil/CommonData.cpp

@@ -161,7 +161,7 @@ std::string solidity::util::getChecksummedAddress(std::string const& _addr)
 	return ret;
 }
 
-bool solidity::util::isValidHex(std::string const& _string)
+bool solidity::util::isValidHex(std::string_view const _string)
 {
 	if (_string.substr(0, 2) != "0x")
 		return false;
@@ -170,7 +170,7 @@ bool solidity::util::isValidHex(std::string const& _string)
 	return true;
 }
 
-bool solidity::util::isValidDecimal(std::string const& _string)
+bool solidity::util::isValidDecimal(std::string_view const _string)
 {
 	if (_string.empty())
 		return false;

libsolutil/CommonData.h

@@ -584,8 +584,8 @@ bool passesAddressChecksum(std::string const& _str, bool _strict);
 /// @param hex strings that look like an address
 std::string getChecksummedAddress(std::string const& _addr);
 
-bool isValidHex(std::string const& _string);
-bool isValidDecimal(std::string const& _string);
+bool isValidHex(std::string_view _string);
+bool isValidDecimal(std::string_view _string);
 
 /// @returns a quoted string if all characters are printable ASCII chars,
 /// or its hex representation otherwise.

libyul/AST.cpp

@@ -0,0 +1,80 @@
+/*
+This file is part of solidity.
+
+solidity is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+solidity is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with solidity.  If not, see <http://www.gnu.org/licenses/>.
+*/
+// SPDX-License-Identifier: GPL-3.0
+
+#include <libyul/AST.h>
+#include <libyul/Exceptions.h>
+
+namespace solidity::yul
+{
+
+LiteralValue::LiteralValue(std::string _builtinStringLiteralValue):
+	m_numericValue(std::nullopt),
+	m_stringValue(std::make_shared<std::string>(std::move(_builtinStringLiteralValue)))
+{ }
+
+LiteralValue::LiteralValue(solidity::yul::LiteralValue::Data const& _data, std::optional<std::string> const& _hint):
+	m_numericValue(_data),
+	m_stringValue(_hint ? std::make_shared<std::string>(*_hint) : nullptr)
+{ }
+
+LiteralValue::Data const& LiteralValue::value() const
+{
+	yulAssert(!unlimited());
+	return *m_numericValue;
+}
+
+LiteralValue::BuiltinStringLiteralData const& LiteralValue::builtinStringLiteralValue() const
+{
+	yulAssert(unlimited());
+	return *m_stringValue;
+}
+
+bool LiteralValue::unlimited() const
+{
+	return !m_numericValue.has_value();
+}
+
+LiteralValue::RepresentationHint const& LiteralValue::hint() const
+{
+	yulAssert(!unlimited());
+	return m_stringValue;
+}
+
+bool LiteralValue::operator==(LiteralValue const& _rhs) const
+{
+	if (unlimited() != _rhs.unlimited())
+		return false;
+
+	if (unlimited())
+		return builtinStringLiteralValue() == _rhs.builtinStringLiteralValue();
+
+	return value() == _rhs.value();
+}
+
+bool LiteralValue::operator<(solidity::yul::LiteralValue const& _rhs) const
+{
+	if (unlimited() != _rhs.unlimited())
+		return unlimited() < _rhs.unlimited();
+
+	if (unlimited())
+		return builtinStringLiteralValue() < _rhs.builtinStringLiteralValue();
+
+	return value() < _rhs.value();
+}
+
+}

libyul/AST.h

@@ -28,6 +28,8 @@
 
 #include <liblangutil/DebugData.h>
 
+#include <libsolutil/Numeric.h>
+
 #include <memory>
 #include <optional>
 
@@ -41,7 +43,32 @@ using TypedNameList = std::vector<TypedName>;
 
 /// Literal number or string (up to 32 bytes)
 enum class LiteralKind { Number, Boolean, String };
-struct Literal { langutil::DebugData::ConstPtr debugData; LiteralKind kind; YulString value; Type type; };
+/// Literal value that holds a u256 word of data, can be of LiteralKind type and - in case of arguments to
+/// builtins - exceed the u256 word (32 bytes), in which case the value is stored as string. The former is constructed
+/// via u256 word and optional hint and leads to unlimited == false, the latter is
+/// constructed via the string constructor and leads to unlimited == true.
+class LiteralValue {
+public:
+	using Data = u256;
+	using BuiltinStringLiteralData = std::string;
+	using RepresentationHint = std::shared_ptr<std::string>;
+
+	LiteralValue() = default;
+	explicit LiteralValue(std::string _builtinStringLiteralValue);
+	explicit LiteralValue(Data const& _data, std::optional<std::string> const& _hint = std::nullopt);
+
+	bool operator==(LiteralValue const& _rhs) const;
+	bool operator<(LiteralValue const& _rhs) const;
+	Data const& value() const;
+	BuiltinStringLiteralData const& builtinStringLiteralValue() const;
+	bool unlimited() const;
+	RepresentationHint const& hint() const;
+
+private:
+	std::optional<Data> m_numericValue;
+	std::shared_ptr<std::string> m_stringValue;
+};
+struct Literal { langutil::DebugData::ConstPtr debugData; LiteralKind kind; LiteralValue value; Type type; };
 /// External / internal identifier or label reference
 struct Identifier { langutil::DebugData::ConstPtr debugData; YulString name; };
 /// Assignment ("x := mload(20:u256)", expects push-1-expression on the right hand

libyul/ASTForward.h

@@ -29,6 +29,7 @@ namespace solidity::yul
 {
 
 enum class LiteralKind;
+class LiteralValue;
 struct Literal;
 struct Label;
 struct Identifier;

libyul/AsmAnalysis.cpp

@@ -41,7 +41,6 @@
 
 #include <fmt/format.h>
 
-#include <memory>
 #include <functional>
 
 using namespace std::string_literals;
@@ -104,24 +103,32 @@ AsmAnalysisInfo AsmAnalyzer::analyzeStrictAssertCorrect(Dialect const& _dialect,
 std::vector<YulString> AsmAnalyzer::operator()(Literal const& _literal)
 {
 	expectValidType(_literal.type, nativeLocationOf(_literal));
-	if (_literal.kind == LiteralKind::String && _literal.value.str().size() > 32)
+	bool erroneousLiteralValue = false;
+	if (_literal.kind == LiteralKind::String && !_literal.value.unlimited() && _literal.value.hint() && _literal.value.hint()->size() > 32)
+	{
+		erroneousLiteralValue = true;
 		m_errorReporter.typeError(
 			3069_error,
 			nativeLocationOf(_literal),
-			"String literal too long (" + std::to_string(_literal.value.str().size()) + " > 32)"
+			"String literal too long (" + std::to_string(formatLiteral(_literal, false /* _validated */ ).size()) + " > 32)"
 		);
-	else if (_literal.kind == LiteralKind::Number && bigint(_literal.value.str()) > u256(-1))
+	}
+	else if (_literal.kind == LiteralKind::Number && _literal.value.hint() && bigint(*_literal.value.hint()) > u256(-1))
+	{
+		erroneousLiteralValue = true;
 		m_errorReporter.typeError(6708_error, nativeLocationOf(_literal), "Number literal too large (> 256 bits)");
-	else if (_literal.kind == LiteralKind::Boolean)
-		yulAssert(_literal.value == "true"_yulstring || _literal.value == "false"_yulstring, "");
+	}
 
 	if (!m_dialect.validTypeForLiteral(_literal.kind, _literal.value, _literal.type))
+	{
 		m_errorReporter.typeError(
 			5170_error,
 			nativeLocationOf(_literal),
-			"Invalid type \"" + _literal.type.str() + "\" for literal \"" + _literal.value.str() + "\"."
+			"Invalid type \"" + _literal.type.str() + "\" for literal \"" + formatLiteral(_literal) + "\"."
 		);
+	}
 
+	yulAssert(erroneousLiteralValue ^ validLiteral(_literal), "Invalid literal after validating it through AsmAnalyzer.");
 	return {_literal.type};
 }
 
@@ -420,23 +427,25 @@ std::vector<YulString> AsmAnalyzer::operator()(FunctionCall const& _funCall)
 				std::string functionName = _funCall.functionName.name.str();
 				if (functionName == "datasize" || functionName == "dataoffset")
 				{
-					if (!m_dataNames.count(std::get<Literal>(arg).value))
+					auto const& argumentAsLiteral = std::get<Literal>(arg);
+					if (!m_dataNames.count(YulString(formatLiteral(argumentAsLiteral))))
 						m_errorReporter.typeError(
 							3517_error,
 							nativeLocationOf(arg),
-							"Unknown data object \"" + std::get<Literal>(arg).value.str() + "\"."
+							"Unknown data object \"" + formatLiteral(argumentAsLiteral) + "\"."
 						);
 				}
 				else if (functionName.substr(0, "verbatim_"s.size()) == "verbatim_")
 				{
-					if (std::get<Literal>(arg).value.empty())
+					auto const& literalValue = std::get<Literal>(arg).value;
+					yulAssert(literalValue.unlimited());  // verbatim literals are always unlimited
+					if (literalValue.builtinStringLiteralValue().empty())
 						m_errorReporter.typeError(
 							1844_error,
 							nativeLocationOf(arg),
 							"The \"verbatim_*\" builtins cannot be used with empty bytecode."
 						);
 				}
-
 				argTypes.emplace_back(expectUnlimitedStringLiteral(std::get<Literal>(arg)));
 				continue;
 			}
@@ -495,12 +504,12 @@ void AsmAnalyzer::operator()(Switch const& _switch)
 			(*this)(*_case.value);
 
 			/// Note: the parser ensures there is only one default case
-			if (watcher.ok() && !cases.insert(valueOfLiteral(*_case.value)).second)
+			if (watcher.ok() && !cases.insert(_case.value->value.value()).second)
 				m_errorReporter.declarationError(
 					6792_error,
 					nativeLocationOf(_case),
 					"Duplicate case \"" +
-					valueOfLiteral(*_case.value).str() +
+					formatLiteral(*_case.value) +
 					"\" defined."
 				);
 		}
@@ -560,8 +569,9 @@ YulString AsmAnalyzer::expectExpression(Expression const& _expr)
 
 YulString AsmAnalyzer::expectUnlimitedStringLiteral(Literal const& _literal)
 {
-	yulAssert(_literal.kind == LiteralKind::String, "");
-	yulAssert(m_dialect.validTypeForLiteral(LiteralKind::String, _literal.value, _literal.type), "");
+	yulAssert(_literal.kind == LiteralKind::String);
+	yulAssert(m_dialect.validTypeForLiteral(LiteralKind::String, _literal.value, _literal.type));
+	yulAssert(_literal.value.unlimited());
 
 	return {_literal.type};
 }

libyul/AsmJsonConverter.cpp

@@ -20,9 +20,10 @@
  * Converts inline assembly AST to JSON format
  */
 
-#include <libyul/AsmJsonConverter.h>
 #include <libyul/AST.h>
+#include <libyul/AsmJsonConverter.h>
 #include <libyul/Exceptions.h>
+#include <libyul/Utilities.h>
 #include <libsolutil/CommonData.h>
 #include <libsolutil/UTF8.h>
 
@@ -47,27 +48,27 @@ Json AsmJsonConverter::operator()(TypedName const& _node) const
 
 Json AsmJsonConverter::operator()(Literal const& _node) const
 {
+	yulAssert(validLiteral(_node));
 	Json ret = createAstNode(originLocationOf(_node), nativeLocationOf(_node), "YulLiteral");
 	switch (_node.kind)
 	{
 	case LiteralKind::Number:
-		yulAssert(
-			util::isValidDecimal(_node.value.str()) || util::isValidHex(_node.value.str()),
-			"Invalid number literal"
-		);
 		ret["kind"] = "number";
 		break;
 	case LiteralKind::Boolean:
 		ret["kind"] = "bool";
 		break;
 	case LiteralKind::String:
 		ret["kind"] = "string";
-		ret["hexValue"] = util::toHex(util::asBytes(_node.value.str()));
+		ret["hexValue"] = util::toHex(util::asBytes(formatLiteral(_node)));
 		break;
 	}
 	ret["type"] = _node.type.str();
-	if (util::validateUTF8(_node.value.str()))
-		ret["value"] = _node.value.str();
+	{
+		auto const formattedLiteral = formatLiteral(_node);
+		if (util::validateUTF8(formattedLiteral))
+			ret["value"] = formattedLiteral;
+	}
 	return ret;
 }
 

libyul/AsmJsonImporter.cpp

@@ -25,6 +25,7 @@
 #include <libyul/AsmJsonImporter.h>
 #include <libyul/AST.h>
 #include <libyul/Exceptions.h>
+#include <libyul/Utilities.h>
 
 #include <liblangutil/Exceptions.h>
 #include <liblangutil/Scanner.h>
@@ -165,26 +166,25 @@ Literal AsmJsonImporter::createLiteral(Json const& _node)
 	std::string kind = member(_node, "kind").get<std::string>();
 
 	solAssert(member(_node, "hexValue").is_string() || member(_node, "value").is_string(), "");
+	std::string value;
 	if (_node.contains("hexValue"))
-		lit.value = YulString{util::asString(util::fromHex(member(_node, "hexValue").get<std::string>()))};
+		value = util::asString(util::fromHex(member(_node, "hexValue").get<std::string>()));
 	else
-		lit.value = YulString{member(_node, "value").get<std::string>()};
-
-	lit.type= YulString{member(_node, "type").get<std::string>()};
-
+		value = member(_node, "value").get<std::string>();
+	lit.type = YulString{member(_node, "type").get<std::string>()};
 	if (kind == "number")
 	{
-		langutil::CharStream charStream(lit.value.str(), "");
+		langutil::CharStream charStream(value, "");
 		langutil::Scanner scanner{charStream};
 		lit.kind = LiteralKind::Number;
 		yulAssert(
 			scanner.currentToken() == Token::Number,
-			"Expected number but got " + langutil::TokenTraits::friendlyName(scanner.currentToken()) + std::string(" while scanning ") + lit.value.str()
+			"Expected number but got " + langutil::TokenTraits::friendlyName(scanner.currentToken()) + std::string(" while scanning ") + value
 		);
 	}
 	else if (kind == "bool")
 	{
-		langutil::CharStream charStream(lit.value.str(), "");
+		langutil::CharStream charStream(value, "");
 		langutil::Scanner scanner{charStream};
 		lit.kind = LiteralKind::Boolean;
 		yulAssert(
@@ -197,13 +197,17 @@ Literal AsmJsonImporter::createLiteral(Json const& _node)
 	{
 		lit.kind = LiteralKind::String;
 		yulAssert(
-			lit.value.str().size() <= 32,
-			"String literal too long (" + std::to_string(lit.value.str().size()) + " > 32)"
+			value.size() <= 32,
+			"String literal too long (" + std::to_string(value.size()) + " > 32)"
 		);
 	}
 	else
 		yulAssert(false, "unknown type of literal");
 
+	// import only for inline assembly, no unlimited string literals there
+	lit.value = valueOfLiteral(value, lit.kind, false /* _unlimitedLiteralArgument */);
+
+	yulAssert(validLiteral(lit));
 	return lit;
 }