executeFromVar() error on costing

build.sbt

@@ -114,9 +114,9 @@ libraryDependencies ++= Seq(
 ) ++ testingDependencies
 
 val circeVersion = "0.10.0"
-val circeCore = "io.circe" %% "circe-core" % circeVersion 
-val circeGeneric = "io.circe" %% "circe-generic" % circeVersion 
-val circeParser = "io.circe" %% "circe-parser" % circeVersion 
+val circeCore = "io.circe" %% "circe-core" % circeVersion
+val circeGeneric = "io.circe" %% "circe-generic" % circeVersion
+val circeParser = "io.circe" %% "circe-parser" % circeVersion
 
 libraryDependencies ++= Seq( circeCore, circeGeneric, circeParser )
 
@@ -235,7 +235,7 @@ lazy val sigmastate = (project in file("sigmastate"))
   .dependsOn(sigmaimpl % allConfigDependency, sigmalibrary % allConfigDependency)
   .settings(libraryDefSettings)
   .settings(libraryDependencies ++= Seq(
-    scorexUtil, kiama, fastparse, circeCore, circeGeneric, circeParser)) 
+    scorexUtil, kiama, fastparse, circeCore, circeGeneric, circeParser))
 
 lazy val sigma = (project in file("."))
     .settings(commonSettings, rootSettings)
@@ -250,7 +250,7 @@ lazy val rootSettings = Seq(
   libraryDependencies := libraryDependencies.all(aggregateCompile).value.flatten,
   mappings in (Compile, packageSrc) ++= (mappings in(Compile, packageSrc)).all(aggregateCompile).value.flatten,
   mappings in (Test, packageBin) ++= (mappings in(Test, packageBin)).all(aggregateCompile).value.flatten,
-  mappings in(Test, packageSrc) ++= (mappings in(Test, packageSrc)).all(aggregateCompile).value.flatten,
+  mappings in (Test, packageSrc) ++= (mappings in(Test, packageSrc)).all(aggregateCompile).value.flatten,
 )
 
 def runErgoTask(task: String, sigmastateVersion: String, log: Logger): Unit = {
@@ -308,7 +308,7 @@ commands += Command.command("ergoItTest") { state =>
 }
 
 def runSpamTestTask(task: String, sigmastateVersion: String, log: Logger): Unit = {
-  val spamBranch = "master"
+  val spamBranch = "v4"
   val envVars = Seq("SIGMASTATE_VERSION" -> sigmastateVersion,
     "SPECIAL_VERSION" -> specialVersion,
     // SSH_SPAM_REPO_KEY should be set (see Jenkins Credentials Binding Plugin)

core/src/main/scala/scalan/staged/Transforming.scala

@@ -184,9 +184,9 @@ trait Transforming { self: Scalan =>
 
       // new effects may appear during body mirroring
       // thus we need to forget original Reify node and create a new one
-//      val oldStack = lambdaStack
+      val oldStack = lambdaStack
       try {
-//        lambdaStack = newLambdaCandidate :: lambdaStack
+        lambdaStack = newLambdaCandidate :: lambdaStack
         val newRoot = { // reifyEffects block
           val schedule = lam.scheduleIds
           val t2 = mirrorSymbols(t1, rewriter, lam, schedule)
@@ -196,7 +196,7 @@ trait Transforming { self: Scalan =>
         ySym.assignDefFrom(newRoot)
       }
       finally {
-//        lambdaStack = oldStack
+        lambdaStack = oldStack
       }
 
       // we don't use toExp here to avoid rewriting pass for new Lambda

docs/hard-fork-changes.md

@@ -0,0 +1,31 @@
+## A list of hard-fork changes
+
+Please describe here all changes which may lead to hard fork (HF for short).
+
+**Note for reviewers: Pull requests targeted to the next HF branch (e.g. v4.x) should be rejected, 
+if they contain HF change, which is not described here**.
+
+### Hard-fork changes in v4.0 (since v3.0.0)
+
+1. This version v4.0.x is based on special/v0.6.x and inherit its hard-fork changes 
+  [see corresponding file](https://github.com/scalan/special/blob/v0.6.0/docs/hard-fork-changes.md).
+
+2. CheckAppendInFoldLoop validation rule added.
+
+3. Costing rule for Append is changed. The same script will have different cost in this version.
+   Hence some transactions may become invalid.
+
+4. ModQ operations serializers removed from serializers table, because these operations
+are not fully implemented.
+This means old v3.0 nodes can successfully serialize but throw exception later.
+New v4.0 nodes will throw during deserialization. For this reason this opCodes 
+cannot be used, even after new operations are fully implemented via soft-fork.
+Thus this is better to be fixed as part of upcoming hard-fork.
+
+5. Complexity of Fold.opCode changed 4034 -> 8034 (i.e. increased for better approximations of the costs)
+
+6. Append operation cost also increased (see `("Append", "(Coll[IV],Coll[IV]) => Coll[IV]", appendCost)`)
+
+7. Removed RW Rule: replicate(n,x).zip(ys).map(f)  ==>  ys.map(y => f(x, y))
+
+8. Changes marked as // HF change:

sigmastate/src/main/scala/org/ergoplatform/ErgoBoxCandidate.scala

@@ -2,6 +2,7 @@ package org.ergoplatform
 
 import java.util
 
+import scalan.util.Extensions.LongOps
 import org.ergoplatform.ErgoBox._
 import org.ergoplatform.settings.ErgoAlgos
 import scorex.crypto.hash.Digest32
@@ -130,14 +131,14 @@ object ErgoBoxCandidate {
       r.positionLimit = r.position + ErgoBox.MaxBoxSize
       val value = r.getULong()                  // READ
       val tree = DefaultSerializer.deserializeErgoTree(r, SigmaSerializer.MaxPropositionSize)  // READ
-      val creationHeight = r.getUInt().toInt    // READ
+      val creationHeight = r.getUInt().toIntExact // READ   // HF change: was r.getUInt().toInt
       val nTokens = r.getUByte()                // READ
       val tokenIds = new Array[Digest32](nTokens)
       val tokenAmounts = new Array[Long](nTokens)
       val tokenIdSize = TokenId.size
       cfor(0)(_ < nTokens, _ + 1) { i =>
         val tokenId = if (digestsInTx.isDefined) {
-          val digestIndex = r.getUInt().toInt   // READ
+          val digestIndex = r.getUInt().toIntExact // READ  // HF change: was r.getUInt().toInt
           val digests = digestsInTx.get
           if (!digests.isDefinedAt(digestIndex)) sys.error(s"failed to find token id with index $digestIndex")
           digests(digestIndex)

sigmastate/src/main/scala/org/ergoplatform/ErgoLikeTransaction.scala

@@ -1,5 +1,6 @@
 package org.ergoplatform
 
+import scalan.util.Extensions.LongOps
 import org.ergoplatform.ErgoBox.TokenId
 import scorex.crypto.authds.ADKey
 import scorex.crypto.hash.{Blake2b256, Digest32}
@@ -154,7 +155,7 @@ object ErgoLikeTransactionSerializer extends SigmaSerializer[ErgoLikeTransaction
       dataInputsBuilder += DataInput(ADKey @@ r.getBytes(ErgoBox.BoxId.size))
     }
     // parse distinct ids of tokens in transaction outputs
-    val tokensCount = r.getUInt().toInt
+    val tokensCount = r.getUInt().toIntExact             // HF change: was r.getUInt().toInt
     val tokensBuilder = mutable.ArrayBuilder.make[TokenId]()
     for (_ <- 0 until tokensCount) {
       tokensBuilder += Digest32 @@ r.getBytes(TokenId.size)

sigmastate/src/main/scala/org/ergoplatform/validation/ValidationRules.scala

@@ -265,7 +265,7 @@ object ValidationRules {
    * because at the time of parsing we may assume that `ChangedRule.newValue`
    * has correct length, so we just parse it until end of bytes (of cause
    * checking consistency). */
-  object CheckCostFuncOperation extends ValidationRule(1013,
+  object CheckCostFuncOpCode extends ValidationRule(1013,
     "Check the opcode is allowed in cost function") with SoftForkWhenCodeAdded {
     final def apply[Ctx <: IRContext, T](ctx: Ctx)(opCode: OpCodeExtra): Unit = {
       checkRule()
@@ -319,6 +319,17 @@ object ValidationRules {
     }
   }
 
+  object CheckAppendInFoldLoop extends ValidationRule(1016,
+    "Check that Append operation is not appear in loop body") with SoftForkWhenReplaced {
+    def apply(level: Int): Unit = {
+      checkRule()
+      if (level > 0)
+        throwValidationException(
+          new CosterException(s"Append is not allowed in Fold loop", None),
+          Array(level))
+    }
+  }
+
   val ruleSpecs: Seq[ValidationRule] = Seq(
     CheckDeserializedScriptType,
     CheckDeserializedScriptIsSigmaProp,
@@ -333,9 +344,10 @@ object ValidationRules {
     CheckTypeWithMethods,
     CheckAndGetMethod,
     CheckHeaderSizeBit,
-    CheckCostFuncOperation,
+    CheckCostFuncOpCode,
     CheckPositionLimit,
-    CheckLoopLevelInCostFunction
+    CheckLoopLevelInCostFunction,
+    CheckAppendInFoldLoop
   )
 
   /** Validation settings that correspond to the current version of the ErgoScript implementation.

sigmastate/src/main/scala/sigmastate/AvlTreeData.scala

@@ -2,7 +2,7 @@ package sigmastate
 
 import java.util
 import java.util.{Arrays, Objects}
-
+import scalan.util.Extensions.LongOps
 import scorex.crypto.authds.ADDigest
 import sigmastate.interpreter.CryptoConstants
 import sigmastate.serialization.SigmaSerializer
@@ -93,8 +93,8 @@ object AvlTreeData {
     override def parse(r: SigmaByteReader): AvlTreeData = {
       val digest = r.getBytes(DigestSize)
       val tf = AvlTreeFlags(r.getByte())
-      val keyLength = r.getUInt().toInt
-      val valueLengthOpt = r.getOption(r.getUInt().toInt)
+      val keyLength = r.getUInt().toIntExact  // HF change: was r.getUInt().toInt
+      val valueLengthOpt = r.getOption(r.getUInt().toIntExact)  // HF change: was r.getUInt().toInt
       AvlTreeData(ADDigest @@ digest, tf, keyLength, valueLengthOpt)
     }
   }

sigmastate/src/main/scala/sigmastate/eval/CostingRules.scala

@@ -583,7 +583,17 @@ trait CostingRules extends SigmaLibrary { IR: IRContext =>
 
   object OptionCoster extends CostingHandler[WOption[Any]]((obj, m, costedArgs, args) => new OptionCoster[Any](obj, m, costedArgs, args))
 
-  /** Costing rules for SCollection methods (see object SCollection) */
+  object IsValidFlatMapLambda {
+    def unapply(d: Def[_]): Boolean = d match {
+      // pattern: x => x.property
+      case Lambda(l,_,_,Def(MethodCall(x, _, Nil, _))) if x == l.x => true
+      // pattern: x => x
+      case IdentityLambda() => true
+      case _ => false
+    }
+  }
+
+  /** Costing rules for SCollection methods (see object SCollection). */
   class CollCoster[T](obj: RCosted[Coll[T]], method: SMethod, costedArgs: Seq[RCosted[_]], args: Seq[Sym]) extends Coster[Coll[T]](obj, method, costedArgs, args) {
     import Coll._
     implicit val eT = obj.elem.eVal.eItem
@@ -642,8 +652,8 @@ trait CostingRules extends SigmaLibrary { IR: IRContext =>
     def flatMap[B](fC: RCosted[T => Coll[B]]): RCostedColl[B] = {
       val fV = fC.value
       fV match {
-        // Pattern: xs.flatMap(x => x.property)
-        case Def(Lambda(l,_,_,Def(mc @ MethodCall(x, m, Nil, _)))) if x == l.x =>
+        // Pattern: xs.flatMap(x => x.property) || xss.flatMap(xs => xs)
+        case Def(IsValidFlatMapLambda()) =>
           val cfC = asCostedFunc[T, Coll[B]](fC)
           val calcF = cfC.sliceCalc
           val sizeF = cfC.sliceSize

sigmastate/src/main/scala/sigmastate/eval/Evaluation.scala

@@ -5,11 +5,9 @@ import java.math.BigInteger
 
 import org.bouncycastle.math.ec.ECPoint
 import org.ergoplatform._
-import org.ergoplatform.validation.ValidationRules.{CheckLoopLevelInCostFunction, CheckCostFuncOperation}
+import org.ergoplatform.validation.ValidationRules.{CheckLoopLevelInCostFunction, CheckCostFuncOpCode, CheckAppendInFoldLoop}
 import sigmastate._
 import sigmastate.Values.{Value, GroupElementConstant, SigmaBoolean, Constant}
-import sigmastate.lang.Terms.OperationId
-import sigmastate.utxo.CostTableStat
 
 import scala.reflect.ClassTag
 import scala.util.Try
@@ -320,19 +318,29 @@ trait Evaluation extends RuntimeCosting { IR: IRContext =>
     }
   }
 
-  /** Recursively traverse the hierarchy of loop operations. */
-  private def traverseScope(scope: AstGraph, level: Int): Unit = {
+  /** Recursively traverse the hierarchy of loop operations.
+    * Due to graph IR, some lambdas may serve as a body of many loop operations, which may
+    * be on different nesting levels. Such lambdas may be traversed many times in order to
+    * check level rules.
+    * @param loopStack stack of parent loop operations
+    * @param scope    graph of the scope to traverse
+    * @param level    current nesting level of the loop body, where 0 is a top level calcF scope
+    */
+  private def traverseScope(loopStack: List[Def[_]], scope: AstGraph, level: Int): Unit = {
+    def isInsideFold: Boolean = loopStack.exists { case CollM.foldLeft(_,_,_) => true  case _ => false }
     scope.schedule.foreach { sym =>
       sym.node match {
         case op @ LoopOperation(bodyLam) =>
-          CheckCostFuncOperation(this)(getOpCodeEx(op))
+          CheckCostFuncOpCode(this)(getOpCodeEx(op))
           val nextLevel = level + 1
           CheckLoopLevelInCostFunction(nextLevel)
-          traverseScope(bodyLam, nextLevel)
-        case CollM.flatMap(_, Def(lam: Lambda[_,_])) =>
-          traverseScope(lam, level) // special case because the body is limited (so don't increase level)
+          traverseScope(op :: loopStack, bodyLam, nextLevel)
+        case op @ CollM.flatMap(_, Def(lam: Lambda[_,_])) =>
+          traverseScope(op :: loopStack, lam, level) // special case because the body is limited (so don't increase level)
+        case CollM.append(_,_) if isInsideFold =>
+          CheckAppendInFoldLoop(level)
         case op =>
-          CheckCostFuncOperation(this)(getOpCodeEx(op))
+          CheckCostFuncOpCode(this)(getOpCodeEx(op))
       }
     }
   }
@@ -342,7 +350,7 @@ trait Evaluation extends RuntimeCosting { IR: IRContext =>
   def verifyCostFunc(costF: Ref[Any => Int]): Try[Unit] = {
     val Def(Lambda(lam,_,_,_)) = costF
     Try {
-      traverseScope(lam, level = 0)
+      traverseScope(Nil, lam, level = 0)
       if (debugModeSanityChecks) {
         val backDeps = mutable.HashMap.empty[Sym, ArrayBuffer[Sym]]
         lam.flatSchedule.foreach { sym =>

sigmastate/src/main/scala/sigmastate/eval/RuntimeCosting.scala

@@ -556,10 +556,11 @@ trait RuntimeCosting extends CostingRules { IR: IRContext =>
         case _ => super.rewriteDef(d)
       }
 
-      case CM.map(CM.zip(CBM.replicate(_, n, x: Ref[a]), ys: RColl[b]@unchecked), _f) =>
-        val f = asRep[((a,b)) => Any](_f)
-        implicit val eb = ys.elem.eItem
-        ys.map(fun { y: Ref[b] => f(Pair(x, y))})
+//      // Rule: replicate(n,x).zip(ys).map(f)  ==>  ys.map(y => f(x, y))
+//      case CM.map(CM.zip(CBM.replicate(_, n, x: Ref[a]), ys: RColl[b]@unchecked), _f) =>
+//        val f = asRep[((a,b)) => Any](_f)
+//        implicit val eb = ys.elem.eItem
+//        ys.map(fun { y: Ref[b] => f(Pair(x, y))})
 
       // Rule: l.isValid op Thunk {... root} => (l op TrivialSigma(root)).isValid
       case ApplyBinOpLazy(op, SigmaM.isValid(l), Def(ThunkDef(root, sch))) if root.elem == BooleanElement =>
@@ -1418,9 +1419,10 @@ trait RuntimeCosting extends CostingRules { IR: IRContext =>
         val col1 = asRep[CostedColl[Any]](_col1)
         val col2 = asRep[CostedColl[Any]](_col2)
         val values = col1.values.append(col2.values)
-        val costs = col1.costs.append(col2.costs)
+        val len = col1.costs.length + col2.costs.length
+        val costs = colBuilder.replicate(len, IntZero)
         val sizes = col1.sizes.append(col2.sizes)
-        RCCostedColl(values, costs, sizes, opCost(values, Array(col1.cost, col2.cost), costOf(node)))
+        RCCostedColl(values, costs, sizes, opCost(values, Array(col1.cost, col2.cost), costOf(node) + perItemCostOf(node, len)))
 
       case Filter(input, p) =>
         val inputC = evalNode(ctx, env, input)

sigmastate/src/main/scala/sigmastate/eval/TreeBuilding.scala

@@ -405,8 +405,8 @@ trait TreeBuilding extends RuntimeCosting { IR: IRContext =>
     var curEnv = env
     for (s <- subG.schedule) {
       val d = s.node
-//      val nonRootLoop = LoopOperation.unapply(d).isDefined && !subG.roots.contains(s)
-      if ((mainG.hasManyUsagesGlobal(s)/* || nonRootLoop*/)
+      val nonRootLoop = LoopOperation.unapply(d).isDefined && !subG.roots.contains(s)
+      if ((mainG.hasManyUsagesGlobal(s) || nonRootLoop)
         && IsContextProperty.unapply(d).isEmpty
         && IsInternalDef.unapply(d).isEmpty
           // to increase effect of constant segregation we need to treat the constants specially

sigmastate/src/main/scala/sigmastate/serialization/ConstantPlaceholderSerializer.scala

@@ -1,5 +1,6 @@
 package sigmastate.serialization
 
+import scalan.util.Extensions.LongOps
 import sigmastate.Values._
 import sigmastate._
 import sigmastate.utils.{SigmaByteReader, SigmaByteWriter}
@@ -13,8 +14,8 @@ case class ConstantPlaceholderSerializer(cons: (Int, SType) => Value[SType])
   }
 
   override def parse(r: SigmaByteReader): Value[SType] = {
-    val id = r.getUInt().toInt
-    val constant = r.constantStore.get(id)  // TODO HF move this under if branch
+    val id = r.getUInt().toIntExact         // HF change: was r.getUInt().toInt
+    val constant = r.constantStore.get(id)
     if (r.resolvePlaceholdersToConstants)
       constant
     else

sigmastate/src/main/scala/sigmastate/serialization/DataSerializer.scala

@@ -3,6 +3,7 @@ package sigmastate.serialization
 import java.math.BigInteger
 import java.nio.charset.StandardCharsets
 
+import scalan.util.Extensions.LongOps
 import org.ergoplatform.ErgoBox
 import org.ergoplatform.validation.ValidationRules.CheckSerializableTypeCode
 import scalan.RType
@@ -92,7 +93,7 @@ object DataSerializer {
       case SInt => r.getInt()
       case SLong => r.getLong()
       case SString =>
-        val size = r.getUInt().toInt
+        val size = r.getUInt().toIntExact  // HF change: was r.getUInt().toInt
         val bytes = r.getBytes(size)
         new String(bytes, StandardCharsets.UTF_8)
       case SBigInt =>

sigmastate/src/main/scala/sigmastate/serialization/ErgoTreeSerializer.scala

@@ -207,7 +207,7 @@ class ErgoTreeSerializer {
     */
   private def deserializeConstants(header: Byte, r: SigmaByteReader): Array[Constant[SType]] = {
     val constants = if (ErgoTree.isConstantSegregation(header)) {
-      val nConsts = r.getUInt().toInt
+      val nConsts = r.getUInt().toIntExact
       val res = new Array[Constant[SType]](nConsts)
       cfor(0)(_ < nConsts, _ + 1) { i =>
         res(i) = constantSerializer.deserialize(r)

sigmastate/src/main/scala/sigmastate/serialization/FuncValueSerializer.scala

@@ -25,7 +25,7 @@ case class FuncValueSerializer(cons: (IndexedSeq[(Int, SType)], Value[SType]) =>
     val argsSize = r.getUInt().toIntExact
     val argsBuilder = mutable.ArrayBuilder.make[(Int, SType)]()
     for (_ <- 0 until argsSize) {
-      val id = r.getUInt().toInt
+      val id = r.getUInt().toIntExact   // HF change: was r.getUInt().toInt
       val tpe = r.getType()
       r.valDefTypeStore(id) = tpe
       argsBuilder += ((id, tpe))

sigmastate/src/main/scala/sigmastate/serialization/ValDefSerializer.scala

@@ -24,7 +24,7 @@ case class ValDefSerializer(override val opDesc: ValueCompanion) extends ValueSe
   }
 
   override def parse(r: SigmaByteReader): Value[SType] = {
-    val id = r.getUInt().toInt
+    val id = r.getUInt().toIntExact   // HF change: was r.getUInt().toInt
     val tpeArgs: Seq[STypeVar] = opCode match {
       case FunDefCode =>
         val nTpeArgs = r.getByte()