Skip to content

Commit

Permalink
NODE-2524 Add base16 and FOLD to keywords (#3882)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ivan-mashonskiy
ivan-mashonskiy authored Sep 28, 2023
1 parent 3c40276 commit cf5593b
Show file tree
Hide file tree
Showing 17 changed files with 216 additions and 124 deletions.
8 changes: 4 additions & 4 deletions lang/shared/src/main/scala/com/wavesplatform/lang/v1/BaseGlobal.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ trait BaseGlobal {
estimator: ScriptEstimator
): Either[String, (Array[Byte], Long, Expressions.SCRIPT, Iterable[CompilationError])] = {
(for {
compRes <- ExpressionCompiler.compileWithParseResult(input, offset, context)
compRes <- ExpressionCompiler.compileWithParseResult(input, offset, context, stdLibVersion)
(compExpr, exprScript, compErrorList) = compRes
illegalBlockVersionUsage = letBlockOnly && com.wavesplatform.lang.v1.compiler.containsBlockV2(compExpr)
_ <- Either.cond(!illegalBlockVersionUsage, (), "UserFunctions are only enabled in STDLIB_VERSION >= 3").leftMap((_, 0, 0))
Expand Down Expand Up @@ -165,7 +165,7 @@ trait BaseGlobal {
val compileFreeCall
: (String, LibrariesOffset, CompilerContext, StdLibVersion, ScriptType, ScriptEstimator) => Either[String, (Array[Byte], EXPR, Long)] =
(input, offset, ctx, version, scriptType, estimator) =>
compile(input, offset, ctx, version, scriptType, estimator, ContractCompiler.compileFreeCall(_, _, _, version))
compile(input, offset, ctx, version, scriptType, estimator, ContractCompiler.compileFreeCall)

val compileDecls
: (String, LibrariesOffset, CompilerContext, StdLibVersion, ScriptType, ScriptEstimator) => Either[String, (Array[Byte], EXPR, Long)] =
Expand All @@ -178,11 +178,11 @@ trait BaseGlobal {
version: StdLibVersion,
scriptType: ScriptType,
estimator: ScriptEstimator,
compiler: (String, LibrariesOffset, CompilerContext) => Either[String, EXPR]
compiler: (String, LibrariesOffset, CompilerContext, StdLibVersion) => Either[String, EXPR]
): Either[String, (Array[Byte], EXPR, Long)] = {
val isFreeCall = scriptType == Call
for {
expr <- if (isFreeCall) ContractCompiler.compileFreeCall(input, offset, context, version) else compiler(input, offset, context)
expr <- if (isFreeCall) ContractCompiler.compileFreeCall(input, offset, context, version) else compiler(input, offset, context, version)
bytes = serializeExpression(expr, version)
_ <- ExprScript.validateBytes(bytes, isFreeCall)
complexity <- ExprScript.estimate(expr, version, isFreeCall, estimator, scriptType == Account)
Expand Down
6 changes: 3 additions & 3 deletions lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/ContractCompiler.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,7 +372,7 @@ object ContractCompiler {
removeUnusedCode: Boolean = false,
allowIllFormedStrings: Boolean = false
): Either[String, DApp] = {
val parser = new Parser()(offset)
val parser = new Parser(version)(offset)
parser.parseContract(input) match {
case fastparse.Parsed.Success(xs, _) =>
ContractCompiler(ctx, xs, version, source, needCompaction, removeUnusedCode, allowIllFormedStrings) match {
Expand All @@ -393,7 +393,7 @@ object ContractCompiler {
removeUnusedCode: Boolean = false,
saveExprContext: Boolean = true
): Either[(String, Int, Int), (Option[DApp], Expressions.DAPP, Iterable[CompilationError])] =
new Parser()(offset)
new Parser(version)(offset)
.parseDAPPWithErrorRecovery(input)
.flatMap { case (parseResult, removedCharPosOpt) =>
compileContract(parseResult, version, needCompaction, removeUnusedCode, ScriptResultSource.CallableFunction, saveExprContext)
Expand Down Expand Up @@ -425,7 +425,7 @@ object ContractCompiler {
ctx: CompilerContext,
version: StdLibVersion
): Either[String, EXPR] = {
val parser = new Parser()(offset)
val parser = new Parser(version)(offset)
parser.parseExpr(input) match {
case fastparse.Parsed.Success(expr, _) =>
val p = AnyPos
Expand Down
26 changes: 17 additions & 9 deletions lang/shared/src/main/scala/com/wavesplatform/lang/v1/compiler/ExpressionCompiler.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import cats.implicits.*
import cats.{Id, Show}
import com.wavesplatform.common.state.ByteStr
import com.wavesplatform.lang.CommonError
import com.wavesplatform.lang.directives.values.StdLibVersion
import com.wavesplatform.lang.v1.compiler.CompilationError.*
import com.wavesplatform.lang.v1.compiler.CompilerContext.*
import com.wavesplatform.lang.v1.compiler.Terms.*
Expand Down Expand Up @@ -52,33 +53,40 @@ object ExpressionCompiler {
errors: Iterable[CompilationError] = Iterable.empty
)

def compile(input: String, offset: LibrariesOffset, ctx: CompilerContext, allowIllFormedStrings: Boolean = false): Either[String, (EXPR, FINAL)] = {
val parser = new Parser()(offset)
def compile(
input: String,
offset: LibrariesOffset,
ctx: CompilerContext,
version: StdLibVersion,
allowIllFormedStrings: Boolean = false
): Either[String, (EXPR, FINAL)] = {
val parser = new Parser(version)(offset)
parser.parseExpr(input) match {
case fastparse.Parsed.Success(xs, _) => ExpressionCompiler(ctx, xs, allowIllFormedStrings)
case f: fastparse.Parsed.Failure => Left(parser.toString(input, f))
}
}

def compileBoolean(input: String, offset: LibrariesOffset, ctx: CompilerContext): Either[String, EXPR] = {
compile(input, offset, ctx).flatMap {
def compileBoolean(input: String, offset: LibrariesOffset, ctx: CompilerContext, version: StdLibVersion): Either[String, EXPR] = {
compile(input, offset, ctx, version).flatMap {
case (expr, BOOLEAN) => Right(expr)
case _ => Left("Script should return boolean")
}
}

def compileUntyped(input: String, offset: LibrariesOffset, ctx: CompilerContext): Either[String, EXPR] = {
compile(input, offset, ctx)
def compileUntyped(input: String, offset: LibrariesOffset, ctx: CompilerContext, version: StdLibVersion): Either[String, EXPR] = {
compile(input, offset, ctx, version)
.map { case (expr, _) => expr }
}

def compileWithParseResult(
input: String,
offset: LibrariesOffset,
ctx: CompilerContext,
version: StdLibVersion,
saveExprContext: Boolean = true
): Either[(String, Int, Int), (EXPR, Expressions.SCRIPT, Iterable[CompilationError])] =
new Parser()(offset)
new Parser(version)(offset)
.parseExpressionWithErrorRecovery(input)
.flatMap { case (parseResult, removedCharPosOpt) =>
compileExprWithCtx(parseResult.expr, saveExprContext, allowIllFormedStrings = false)
Expand All @@ -104,9 +112,9 @@ object ExpressionCompiler {
.leftMap(e => (s"Compilation failed: ${Show[CompilationError].show(e)}", e.start, e.end))
}

def compileDecls(input: String, offset: LibrariesOffset, ctx: CompilerContext): Either[String, EXPR] = {
def compileDecls(input: String, offset: LibrariesOffset, ctx: CompilerContext, version: StdLibVersion): Either[String, EXPR] = {
val adjustedDecls = s"$input\n${GlobalValNames.Unit}"
compileUntyped(adjustedDecls, offset, ctx)
compileUntyped(adjustedDecls, offset, ctx, version)
}

private def compileExpr(expr: Expressions.EXPR): CompileM[(Terms.EXPR, FINAL, Expressions.EXPR, Iterable[CompilationError])] =
Expand Down
45 changes: 27 additions & 18 deletions lang/shared/src/main/scala/com/wavesplatform/lang/v1/parser/Parser.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import cats.instances.list.*
import cats.syntax.either.*
import cats.syntax.traverse.*
import com.wavesplatform.common.state.ByteStr
import com.wavesplatform.lang.directives.values.{StdLibVersion, V8}
import com.wavesplatform.lang.v1.evaluator.ctx.impl.GlobalValNames
import com.wavesplatform.lang.v1.evaluator.ctx.impl.PureContext.MaxListLengthV4
import com.wavesplatform.lang.v1.parser.BinaryOperation.*
Expand All @@ -21,14 +22,15 @@ import fastparse.Parsed.Failure

import scala.annotation.tailrec

class Parser(implicit offset: LibrariesOffset) {
class Parser(stdLibVersion: StdLibVersion)(implicit offset: LibrariesOffset) {

private val Global = com.wavesplatform.lang.hacks.Global // Hack for IDEA
implicit def hack(p: fastparse.P[Any]): fastparse.P[Unit] = p.map(_ => ())

val keywords = Set("let", "strict", "base58", "base64", "true", "false", "if", "then", "else", "match", "case", "func")
val excludeInError = Set('(', ')', ':', ']', '[', '=', ',', ';')

val keywords: Set[String] = if (stdLibVersion >= V8) keywordsBeforeV8 ++ additionalV8Keywords else keywordsBeforeV8

def lowerChar[A: P] = CharIn("a-z")
def upperChar[A: P] = CharIn("A-Z")
def nonLatinChar[A: P] = (CharPred(_.isLetter) ~~/ Fail).opaque("only latin charset for definitions")
Expand Down Expand Up @@ -102,11 +104,6 @@ class Parser(implicit offset: LibrariesOffset) {
}
.map(posAndVal => CONST_STRING(posAndVal._1, posAndVal._2))

def correctVarName[A: P]: P[PART[String]] =
(Index ~~ (char ~~ (digit | char).repX()).! ~~ Index)
.filter { case (_, x, _) => !keywords.contains(x) }
.map { case (start, x, end) => PART.VALID(Pos(start, end), x) }

def declNameP[A: P](check: Boolean = false): P[Unit] = {
def symbolsForError = CharPred(c => !c.isWhitespace && !excludeInError.contains(c))
def checkedUnderscore = ("_" ~~/ !"_".repX(1)).opaque("not more than 1 underscore in a row")
Expand Down Expand Up @@ -361,7 +358,12 @@ class Parser(implicit offset: LibrariesOffset) {
}

def matchP[A: P]: P[EXPR] =
P(Index ~~ "match" ~~ &(border) ~/ baseExpr ~ "{" ~ comment ~ matchCaseP.rep(0, comment) ~ comment ~ "}" ~~ Index)
P(
Index ~~ "match" ~~ &(border) ~/ baseExpr.opaque("expression to match") ~ "{" ~ comment ~ matchCaseP.rep(
0,
comment
) ~ comment ~ "}" ~~ Index
)
.map {
case (start, _, Nil, end) => INVALID(Pos(start, end), "pattern matching requires case branches")
case (start, e, cases, end) => MATCH(Pos(start, end), e, cases.toList)
Expand Down Expand Up @@ -719,15 +721,21 @@ class Parser(implicit offset: LibrariesOffset) {
}

object Parser {
private val defaultParser = new Parser()(NoLibraries)
def parseExpr(str: String): Parsed[EXPR] = defaultParser.parseExpr(str)
def parseReplExpr(str: String): Parsed[EXPR] = defaultParser.parseReplExpr(str)
def parseContract(str: String): Parsed[DAPP] = defaultParser.parseContract(str)
def toString(input: String, f: Failure): String = defaultParser.toString(input, f)
def parseExpressionWithErrorRecovery(str: String): Either[(String, Int, Int), (SCRIPT, Option[Pos])] =
defaultParser.parseExpressionWithErrorRecovery(str)
def parseDAPPWithErrorRecovery(str: String): Either[(String, Int, Int), (DAPP, Option[Pos])] =
defaultParser.parseDAPPWithErrorRecovery(str)
private def parser(version: StdLibVersion) = new Parser(version)(NoLibraries)
def parseExpr(str: String, version: StdLibVersion = StdLibVersion.VersionDic.all.last): Parsed[EXPR] = parser(version).parseExpr(str)
def parseReplExpr(str: String, version: StdLibVersion = StdLibVersion.VersionDic.all.last): Parsed[EXPR] = parser(version).parseReplExpr(str)
def parseContract(str: String, version: StdLibVersion = StdLibVersion.VersionDic.all.last): Parsed[DAPP] = parser(version).parseContract(str)
def toString(input: String, f: Failure, version: StdLibVersion = StdLibVersion.VersionDic.all.last): String = parser(version).toString(input, f)
def parseExpressionWithErrorRecovery(
str: String,
version: StdLibVersion = StdLibVersion.VersionDic.all.last
): Either[(String, Int, Int), (SCRIPT, Option[Pos])] =
parser(version).parseExpressionWithErrorRecovery(str)
def parseDAPPWithErrorRecovery(
str: String,
version: StdLibVersion = StdLibVersion.VersionDic.all.last
): Either[(String, Int, Int), (DAPP, Option[Pos])] =
parser(version).parseDAPPWithErrorRecovery(str)

sealed trait Accessor
case class Method(name: PART[String], args: Seq[EXPR]) extends Accessor
Expand All @@ -741,7 +749,8 @@ object Parser {
val KnownMethods: Set[String] = Set(ExactAs, As)
}

val keywords = Set("let", "strict", "base58", "base64", "true", "false", "if", "then", "else", "match", "case", "func")
val keywordsBeforeV8 = Set("let", "strict", "base58", "base64", "true", "false", "if", "then", "else", "match", "case", "func")
val additionalV8Keywords = Set("base16", "FOLD")

sealed trait LibrariesOffset {
def shiftStart(idx: Int): Int = idx
Expand Down
6 changes: 3 additions & 3 deletions lang/testkit/src/main/scala/com/wavesplatform/lang/v1/compiler/TestCompiler.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ class TestCompiler(version: StdLibVersion) {
): ExprScript =
ExprScript(
version,
ExpressionCompiler.compile(script, offset, expressionCompilerContext, allowIllFormedStrings).explicitGet()._1,
ExpressionCompiler.compile(script, offset, expressionCompilerContext, version, allowIllFormedStrings).explicitGet()._1,
checkSize = checkSize
).explicitGet()

Expand All @@ -82,11 +82,11 @@ class TestCompiler(version: StdLibVersion) {
checkSize: Boolean = true
): Either[String, ExprScript] =
ExpressionCompiler
.compile(script, offset, expressionCompilerContext, allowIllFormedStrings)
.compile(script, offset, expressionCompilerContext, version, allowIllFormedStrings)
.map(s => ExprScript(version, s._1, checkSize = checkSize).explicitGet())

def compileAsset(script: String, offset: LibrariesOffset = NoLibraries): Script =
ExprScript(version, ExpressionCompiler.compile(script, offset, assetCompilerContext).explicitGet()._1).explicitGet()
ExprScript(version, ExpressionCompiler.compile(script, offset, assetCompilerContext, version).explicitGet()._1).explicitGet()

def compileFreeCall(script: String, offset: LibrariesOffset = NoLibraries): ExprScript = {
val expr = ContractCompiler.compileFreeCall(script, offset, compilerContext, version).explicitGet()
Expand Down
32 changes: 21 additions & 11 deletions lang/testkit/src/main/scala/com/wavesplatform/lang/v1/testing/ScriptGen.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,18 @@ package com.wavesplatform.lang.v1.testing
import com.wavesplatform.common.state.ByteStr
import com.wavesplatform.common.utils.Base58
import com.wavesplatform.lang.v1.parser.BinaryOperation
import com.wavesplatform.lang.v1.parser.BinaryOperation._
import com.wavesplatform.lang.v1.parser.BinaryOperation.*
import com.wavesplatform.lang.v1.parser.Expressions.Pos.AnyPos
import com.wavesplatform.lang.v1.parser.Expressions._
import com.wavesplatform.lang.v1.parser.Parser.keywords
import org.scalacheck._
import com.wavesplatform.lang.v1.parser.Expressions.*
import com.wavesplatform.lang.v1.parser.Parser.{additionalV8Keywords, keywordsBeforeV8}
import org.scalacheck.*

import scala.reflect.ClassTag

trait ScriptGen {

val allKeywords: Set[String] = keywordsBeforeV8 ++ additionalV8Keywords

def CONST_LONGgen: Gen[(EXPR, Long)] = Gen.choose(Long.MinValue, Long.MaxValue).map(v => (CONST_LONG(AnyPos, v), v))

def BOOLgen(gas: Int): Gen[(EXPR, Boolean)] =
Expand Down Expand Up @@ -87,26 +89,34 @@ trait ScriptGen {
(cnd, vcnd) <- BOOLgen((gas - 3) / 3)
(t, vt) <- BOOLgen((gas - 3) / 3)
(f, vf) <- BOOLgen((gas - 3) / 3)
} yield (IF(AnyPos, cnd, t, f), if (vcnd) { vt } else { vf })
} yield (
IF(AnyPos, cnd, t, f),
if (vcnd) { vt }
else { vf }
)

def IF_INTgen(gas: Int): Gen[(EXPR, Long)] =
for {
(cnd, vcnd) <- BOOLgen((gas - 3) / 3)
(t, vt) <- INTGen((gas - 3) / 3)
(f, vf) <- INTGen((gas - 3) / 3)
} yield (IF(AnyPos, cnd, t, f), if (vcnd) { vt } else { vf })
} yield (
IF(AnyPos, cnd, t, f),
if (vcnd) { vt }
else { vf }
)

def STRgen: Gen[EXPR] =
Gen.identifier.map(PART.VALID[String](AnyPos, _)).map(CONST_STRING(AnyPos, _))

def LETgen(gas: Int): Gen[LET] =
for {
name <- Gen.identifier.filter(!keywords(_))
name <- Gen.identifier.filter(!allKeywords(_))
(value, _) <- BOOLgen((gas - 3) / 3)
} yield LET(AnyPos, PART.VALID(AnyPos, name), value)

def REFgen: Gen[EXPR] =
Gen.identifier.filter(!keywords(_)).map(PART.VALID[String](AnyPos, _)).map(REF(AnyPos, _))
Gen.identifier.filter(!allKeywords(_)).map(PART.VALID[String](AnyPos, _)).map(REF(AnyPos, _))

def BLOCKgen(gas: Int): Gen[EXPR] =
for {
Expand Down Expand Up @@ -136,11 +146,11 @@ trait ScriptGen {

def toString(expr: EXPR): Gen[String] = expr match {
case CONST_LONG(_, x, _) => withWhitespaces(s"$x")
case REF(_, x, _, _) => withWhitespaces(toString(x))
case REF(_, x, _, _) => withWhitespaces(toString(x))
case CONST_STRING(_, x, _) => withWhitespaces(s"""\"${toString(x)}\"""")
case CONST_BYTESTR(_, x, _) => withWhitespaces(s"""base58'${toString(x)}'""")
case _: TRUE => withWhitespaces("true")
case _: FALSE => withWhitespaces("false")
case _: TRUE => withWhitespaces("true")
case _: FALSE => withWhitespaces("false")
case BINARY_OP(_, x, op: BinaryOperation, y, _, _) =>
for {
arg1 <- toString(x)
Expand Down
4 changes: 2 additions & 2 deletions lang/tests/src/test/scala/com/wavesplatform/lang/IntegrationTest.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class IntegrationTest extends PropSpec with Inside {
)
)

val compiled = ExpressionCompiler.compile(code, NoLibraries, ctx.compilerContext)
val compiled = ExpressionCompiler.compile(code, NoLibraries, ctx.compilerContext, StdLibVersion.VersionDic.all.last)
val evalCtx = ctx.evaluationContext(env).asInstanceOf[EvaluationContext[Environment, Id]]
compiled.flatMap(v =>
EvaluatorV2
Expand Down Expand Up @@ -287,7 +287,7 @@ class IntegrationTest extends PropSpec with Inside {
}

def compile(script: String): Either[String, Terms.EXPR] =
ExpressionCompiler.compileBoolean(script, NoLibraries, CTX.empty.compilerContext)
ExpressionCompiler.compileBoolean(script, NoLibraries, CTX.empty.compilerContext, StdLibVersion.VersionDic.all.last)

property("wrong script return type") {
compile("1") should produce("should return boolean")
Expand Down
6 changes: 3 additions & 3 deletions lang/tests/src/test/scala/com/wavesplatform/lang/compiler/ExpressionCompilerV1Test.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ class ExpressionCompilerV1Test extends PropSpec {
| func f(a: Any) = a._1 == a._2
| true
""".stripMargin
ExpressionCompiler.compile(script, NoLibraries, compilerContext) should produce(
ExpressionCompiler.compile(script, NoLibraries, compilerContext, StdLibVersion.VersionDic.all.last) should produce(
"Compilation failed: [" +
"Undefined field `_1` of variable of type `Any` in 19-23; " +
"Undefined field `_2` of variable of type `Any` in 27-31" +
Expand All @@ -585,7 +585,7 @@ class ExpressionCompilerV1Test extends PropSpec {
|t._2[ind] == 3
|""".stripMargin

ExpressionCompiler.compile(script, NoLibraries, compilerContextV4) shouldBe Right(
ExpressionCompiler.compile(script, NoLibraries, compilerContextV4, StdLibVersion.VersionDic.all.last) shouldBe Right(
(
LET_BLOCK(
LET(
Expand Down Expand Up @@ -623,7 +623,7 @@ class ExpressionCompilerV1Test extends PropSpec {
|t.some[ind] == 3
|""".stripMargin

ExpressionCompiler.compile(script, NoLibraries, compilerContextV4) should produce(
ExpressionCompiler.compile(script, NoLibraries, compilerContextV4, StdLibVersion.VersionDic.all.last) should produce(
"Compilation failed: [Non-matching types: expected: List[T], actual: Nothing in 39-50; Undefined field `some` of variable of type `(Int, List[Int], Int)` in 39-45]"
)
}
Expand Down
5 changes: 4 additions & 1 deletion ...s/src/test/scala/com/wavesplatform/lang/compiler/ExpressionCompilerWithParserV2Test.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package com.wavesplatform.lang.compiler

import cats.implicits.toBifunctorOps
import com.wavesplatform.common.utils.EitherExt2
import com.wavesplatform.lang.directives.values.StdLibVersion
import com.wavesplatform.lang.directives.{Directive, DirectiveParser}
import com.wavesplatform.lang.utils
import com.wavesplatform.lang.v1.compiler.ExpressionCompiler
Expand All @@ -20,7 +21,9 @@ class ExpressionCompilerWithParserV2Test extends PropSpec {
directives <- DirectiveParser(script)
ds <- Directive.extractDirectives(directives)
ctx = utils.compilerContext(ds)
compResult <- ExpressionCompiler.compileWithParseResult(script, NoLibraries, ctx, saveExprContext).leftMap(_._1)
compResult <- ExpressionCompiler
.compileWithParseResult(script, NoLibraries, ctx, StdLibVersion.VersionDic.all.last, saveExprContext)
.leftMap(_._1)
} yield compResult

result.map(_._2.expr)
Expand Down
Loading

0 comments on commit cf5593b

Please sign in to comment.