feat: added serialization header to CS and debug info to all constrai…

…nts with -tags=debug (#347)

* feat: added GnarkVersion semver tag in cs serialized objects

* feat: store debug info stack for all constraints if debug tag is set

* doc: added TODO

* fix: round trip serialization test of cs

* fix: stack parsing

* style: better comments

* feat: ignore some fields when serializing a constraint system

* fix: fix previous commit, unexport fields in cs

* test: added version test to ensure we update hardcoded v

* test: added debug.ParseStack test

* fix: fix version test to work on CI using git ls-remote

debug/debug.go

@@ -1,12 +1,35 @@
 package debug
 
 import (
+	"errors"
 	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
 )
 
+type StackLine struct {
+	Line uint32
+	File string
+}
+
+// ParseStack parses a stack as stored in a log entry and return readable data
+func ParseStack(stack []uint64, stackPaths map[uint32]string) ([]StackLine, error) {
+	r := make([]StackLine, len(stack))
+
+	for i, s := range stack {
+		pID := uint32(s >> 32)
+		line := uint32(s)
+		path, ok := stackPaths[pID]
+		if !ok {
+			return nil, errors.New("missing stack path in stackPaths map")
+		}
+		r[i] = StackLine{Line: line, File: path}
+	}
+
+	return r, nil
+}
+
 func Stack() string {
 	var sbb strings.Builder
 	WriteStack(&sbb)

debug/debug_test.go

@@ -0,0 +1,46 @@
+package debug
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseStack(t *testing.T) {
+	assert := require.New(t)
+
+	const (
+		f1 = "/usr/local/file1.go"
+		f2 = "/usr/local/file2.go"
+		f3 = "/usr/local/lib/file3.go"
+	)
+
+	stackPaths := make(map[uint32]string)
+	stackPaths[0] = f1
+	stackPaths[1] = f2
+	stackPaths[2] = f3
+
+	stack := []uint64{
+		uint64(0<<32) | 27,
+		uint64(1<<32) | 42,
+		uint64(2<<32) | 2,
+	}
+
+	parsed, err := ParseStack(stack, stackPaths)
+	assert.NoError(err)
+
+	assert.True(len(parsed) == 3)
+	assert.Equal(f1, parsed[0].File)
+	assert.Equal(uint32(27), parsed[0].Line)
+
+	assert.Equal(f2, parsed[1].File)
+	assert.Equal(uint32(42), parsed[1].Line)
+
+	assert.Equal(f3, parsed[2].File)
+	assert.Equal(uint32(2), parsed[2].Line)
+
+	stack = append(stack, uint64(8000<<32)|2)
+	_, err = ParseStack(stack, stackPaths)
+	assert.Error(err)
+
+}

doc.go

@@ -17,7 +17,12 @@
 // https://docs.gnark.consensys.net
 package gnark
 
-import "github.com/consensys/gnark-crypto/ecc"
+import (
+	"github.com/blang/semver/v4"
+	"github.com/consensys/gnark-crypto/ecc"
+)
+
+var Version = semver.MustParse("0.8.0-alpha")
 
 // Curves return the curves supported by gnark
 func Curves() []ecc.ID {

frontend/ccs.go

@@ -47,4 +47,10 @@ type CompiledConstraintSystem interface {
 
 	// GetConstraints return a human readable representation of the constraints
 	GetConstraints() [][]string
+
+	// GetDebugInfo return the list of debug info per constraints and the map to restore file path
+	// from the debug info. When compiled with -tags=debug, each constraint has an associated
+	// stack encoded as a []uint64. The uint64 pack uint32(fileID) | uint32(lineNumber).
+	// The file string can be retrieved from associated map.
+	GetDebugInfo() ([][]uint64, map[uint32]string)
 }

frontend/compiled/cs.go

@@ -3,19 +3,26 @@ package compiled
 import (
 	"fmt"
 	"math/big"
+	"runtime"
 	"strings"
 
+	"github.com/blang/semver/v4"
+	"github.com/consensys/gnark"
 	"github.com/consensys/gnark-crypto/ecc"
 	"github.com/consensys/gnark/backend"
 	"github.com/consensys/gnark/backend/hint"
 	"github.com/consensys/gnark/debug"
 	"github.com/consensys/gnark/frontend/schema"
 	"github.com/consensys/gnark/internal/tinyfield"
 	"github.com/consensys/gnark/internal/utils"
+	"github.com/consensys/gnark/logger"
 )
 
 // ConstraintSystem contains common element between R1CS and ConstraintSystem
 type ConstraintSystem struct {
+	// serialization header
+	GnarkVersion string
+	ScalarField  string
 
 	// schema of the circuit
 	Schema *schema.Schema
@@ -34,12 +41,17 @@ type ConstraintSystem struct {
 	// debug info contains stack trace (including line number) of a call to a cs.API that
 	// results in an unsolved constraint
 	DebugInfo []LogEntry
+	// maps unique id to a path (optimized for reading debug info stacks)
+	DebugStackPaths map[uint32]string
+	// maps a path to an id (optimized for storing debug info stacks)
+	debugPathsIds map[string]uint32 `cbor:"-"`
+	debugPathId   uint32            `cbor:"-"`
 
 	// maps constraint id to debugInfo id
 	// several constraints may point to the same debug info
 	MDebug map[int]int
 
-	Counters []Counter // TODO @gbotrel no point in serializing these
+	Counters []Counter `cbor:"-"`
 
 	MHints             map[int]*Hint      // maps wireID to hint
 	MHintsDependencies map[hint.ID]string // maps hintID to hint string identifier
@@ -50,25 +62,49 @@ type ConstraintSystem struct {
 	Levels [][]int
 
 	// scalar field
-	q      *big.Int
-	bitLen int
+	q      *big.Int `cbor:"-"`
+	bitLen int      `cbor:"-"`
 }
 
 // NewConstraintSystem initialize the common structure among constraint system
 func NewConstraintSystem(scalarField *big.Int) ConstraintSystem {
 	return ConstraintSystem{
+		GnarkVersion:       gnark.Version.String(),
+		ScalarField:        scalarField.Text(16),
 		MDebug:             make(map[int]int),
 		MHints:             make(map[int]*Hint),
 		MHintsDependencies: make(map[hint.ID]string),
+		DebugStackPaths:    make(map[uint32]string),
+		debugPathsIds:      make(map[string]uint32),
 		q:                  new(big.Int).Set(scalarField),
 		bitLen:             scalarField.BitLen(),
 	}
 }
 
-// SetScalarField sets the scalar field on the constraint system object
+// CheckSerializationHeader parses the scalar field and gnark version headers
 //
-// This is meant to be use at the deserialization step
-func (cs *ConstraintSystem) SetScalarField(scalarField *big.Int) error {
+// This is meant to be use at the deserialization step, and will error for illegal values
+func (cs *ConstraintSystem) CheckSerializationHeader() error {
+	// check gnark version
+	binaryVersion := gnark.Version
+	objectVersion, err := semver.Parse(cs.GnarkVersion)
+	if err != nil {
+		return fmt.Errorf("when parsing gnark version: %w", err)
+	}
+
+	if binaryVersion.Compare(objectVersion) != 0 {
+		log := logger.Logger()
+		log.Warn().Str("binary", binaryVersion.String()).Str("object", objectVersion.String()).Msg("gnark version (binary) mismatch with constraint system. there are no guarantees on compatibilty")
+	}
+
+	// TODO @gbotrel maintain version changes and compare versions properly
+	// (ie if major didn't change,we shouldn't have a compat issue)
+
+	scalarField := new(big.Int)
+	_, ok := scalarField.SetString(cs.ScalarField, 16)
+	if !ok {
+		return fmt.Errorf("when parsing serialized modulus: %s", cs.ScalarField)
+	}
 	curveID := utils.FieldToCurve(scalarField)
 	if curveID == ecc.UNKNOWN && scalarField.Cmp(tinyfield.Modulus()) != 0 {
 		return fmt.Errorf("unsupported scalard field %s", scalarField.Text(16))
@@ -108,6 +144,15 @@ func (cs *ConstraintSystem) AddDebugInfo(errName string, i ...interface{}) int {
 
 	var l LogEntry
 
+	// add the stack info
+	// TODO @gbotrel duplicate with Debug.stack below
+	l.Stack = cs.stack()
+
+	if errName == "" {
+		cs.DebugInfo = append(cs.DebugInfo, l)
+		return len(cs.DebugInfo) - 1
+	}
+
 	const minLogSize = 500
 	var sbb strings.Builder
 	sbb.Grow(minLogSize)
@@ -147,3 +192,61 @@ func (cs *ConstraintSystem) AddDebugInfo(errName string, i ...interface{}) int {
 func (cs *ConstraintSystem) FieldBitLen() int {
 	return cs.bitLen
 }
+
+func (cs *ConstraintSystem) GetDebugInfo() ([][]uint64, map[uint32]string) {
+	r := make([][]uint64, len(cs.DebugInfo))
+	for _, l := range cs.DebugInfo {
+		r = append(r, l.Stack)
+	}
+	return r, cs.DebugStackPaths
+}
+
+func (cs *ConstraintSystem) stack() (r []uint64) {
+	if !debug.Debug {
+		return
+	}
+	// derived from: https://golang.org/pkg/runtime/#example_Frames
+	// we stop when func name == Define as it is where the gnark circuit code should start
+
+	// Ask runtime.Callers for up to 10 pcs
+	pc := make([]uintptr, 10)
+	n := runtime.Callers(3, pc)
+	if n == 0 {
+		// No pcs available. Stop now.
+		// This can happen if the first argument to runtime.Callers is large.
+		return
+	}
+	pc = pc[:n] // pass only valid pcs to runtime.CallersFrames
+	frames := runtime.CallersFrames(pc)
+	// Loop to get frames.
+	// A fixed number of pcs can expand to an indefinite number of Frames.
+	for {
+		frame, more := frames.Next()
+		fe := strings.Split(frame.Function, "/")
+		function := fe[len(fe)-1]
+		file := frame.File
+
+		if strings.Contains(frame.File, "gnark/frontend") {
+			continue
+		}
+
+		// TODO @gbotrel this stores an absolute path, so will work only locally
+		id, ok := cs.debugPathsIds[file]
+		if !ok {
+			id = cs.debugPathId
+			cs.debugPathId++
+			cs.debugPathsIds[file] = id
+			cs.DebugStackPaths[id] = file
+		}
+		r = append(r, ((uint64(id) << 32) | uint64(frame.Line)))
+		if !more {
+			break
+		}
+		if strings.HasSuffix(function, "Define") {
+			break
+		}
+	}
+
+	return
+
+}

frontend/compiled/log.go

@@ -27,6 +27,11 @@ type LogEntry struct {
 	Caller    string
 	Format    string
 	ToResolve []Term
+
+	// When compiled with -tags=debug, each constraint has an associated
+	// stack encoded as a []uint64. The uint64 pack uint32(fileID) | uint32(lineNumber).
+	// The actual string describing the file is stored in a map in the compiled.ConstraintSystem.
+	Stack []uint64
 }
 
 func (l *LogEntry) WriteVariable(le LinearExpression, sbb *strings.Builder) {

frontend/cs/r1cs/api.go

@@ -99,7 +99,7 @@ func (system *r1cs) Mul(i1, i2 frontend.Variable, in ...frontend.Variable) front
 		// v1 and v2 are both unknown, this is the only case we add a constraint
 		if !v1Constant && !v2Constant {
 			res := system.newInternalVariable()
-			system.Constraints = append(system.Constraints, newR1C(v1, v2, res))
+			system.addConstraint(newR1C(v1, v2, res))
 			return res
 		}
 
@@ -287,7 +287,7 @@ func (system *r1cs) Xor(_a, _b frontend.Variable) frontend.Variable {
 	c := system.Neg(res).(compiled.LinearExpression)
 	c = append(c, a[0], b[0])
 	aa := system.Mul(a, 2)
-	system.Constraints = append(system.Constraints, newR1C(aa, b, c))
+	system.addConstraint(newR1C(aa, b, c))
 
 	return res
 }
@@ -307,7 +307,7 @@ func (system *r1cs) Or(_a, _b frontend.Variable) frontend.Variable {
 	system.MarkBoolean(res)
 	c := system.Neg(res).(compiled.LinearExpression)
 	c = append(c, a[0], b[0])
-	system.Constraints = append(system.Constraints, newR1C(a, b, c))
+	system.addConstraint(newR1C(a, b, c))
 
 	return res
 }

frontend/cs/r1cs/builder.go

@@ -30,6 +30,7 @@ import (
 	"github.com/consensys/gnark-crypto/ecc"
 	"github.com/consensys/gnark/backend"
 	"github.com/consensys/gnark/backend/hint"
+	"github.com/consensys/gnark/debug"
 	"github.com/consensys/gnark/frontend"
 	"github.com/consensys/gnark/frontend/compiled"
 	"github.com/consensys/gnark/frontend/cs"
@@ -176,6 +177,8 @@ func (system *r1cs) addConstraint(r1c compiled.R1C, debugID ...int) {
 	system.Constraints = append(system.Constraints, r1c)
 	if len(debugID) > 0 {
 		system.MDebug[len(system.Constraints)-1] = debugID[0]
+	} else if debug.Debug {
+		system.MDebug[len(system.Constraints)-1] = system.AddDebugInfo("")
 	}
 }
 

frontend/cs/scs/builder.go

@@ -30,6 +30,7 @@ import (
 	"github.com/consensys/gnark-crypto/ecc"
 	"github.com/consensys/gnark/backend"
 	"github.com/consensys/gnark/backend/hint"
+	"github.com/consensys/gnark/debug"
 	"github.com/consensys/gnark/frontend"
 	"github.com/consensys/gnark/frontend/compiled"
 	"github.com/consensys/gnark/frontend/cs"
@@ -89,6 +90,8 @@ func (system *scs) addPlonkConstraint(l, r, o compiled.Term, cidl, cidr, cidm1,
 
 	if len(debugID) > 0 {
 		system.MDebug[len(system.Constraints)] = debugID[0]
+	} else if debug.Debug {
+		system.MDebug[len(system.Constraints)] = system.AddDebugInfo("")
 	}
 
 	l.SetCoeffID(cidl)

go.mod

@@ -1,11 +1,13 @@
 module github.com/consensys/gnark
 
-go 1.17
+go 1.18
 
 require (
+	github.com/blang/semver/v4 v4.0.0
 	github.com/consensys/bavard v0.1.11-0.20220418151131-08e04eec2153
 	github.com/consensys/gnark-crypto v0.7.1-0.20220603201101-938eff486457
 	github.com/fxamacker/cbor/v2 v2.2.0
+	github.com/google/go-cmp v0.5.8
 	github.com/leanovate/gopter v0.2.9
 	github.com/rs/zerolog v1.26.1
 	github.com/stretchr/testify v1.7.1