Add --experimental_src_map to ethier gen command (#35)

* Solidity source mapping from OpCodes at runtime!

* Fix test failure introduced by upgrade of go-ethereum + bring new test code in line with implementation variable names.

* Run prettier on Solidity test contract

* Properly handle source-code resolution from EVM trace when executing libraries.

* GitHub test Action is failing despite tests passing so attempting to change the node_modules cache key to clear it.

* Revert GitHub Action cache key and change testing to non-verbose as it hides failures; also output `solc` + `abigen` versions.

* Deliberate panic in ethier/gen.go to diagnose difference from local run to GitHub Actions

* A further deliberate panic in ethier/gen.go to see full output; abigen differs and regexp is a bad idea here because it's too fragile

* Update ethier/gen.go to match output of old and new versions of abigen after ethereum/go-ethereum#24835.

See the TODO in ethier/gen.go re direct modification of the bind.Bind() template.

* Simplify calculation of PUSH<N> instruction offset

.github/workflows/test.yml

@@ -40,4 +40,4 @@ jobs:
  sudo apt-get install -y solc abigen
 
  - name: Run tests
- run: npm run test:verbose
+ run: npm run test

eth/signer_test.go

@@ -14,6 +14,7 @@ import (
  "github.com/google/tink/go/keyset"
  "github.com/google/tink/go/prf"
  "github.com/google/tink/go/tink"
+ "github.com/h-fam/errdiff"
 
  // These tests require ethtest.SimulatedBackend but that would result in a
  // cyclical dependency. As this is limited to these tests and not the
@@ -161,19 +162,19 @@ func TestTransactorWithChainID(t *testing.T) {
  const gasLimit = 21000
  txFee := new(big.Int).Mul(gasPrice, big.NewInt(gasLimit))
 
- sendEth := func(t *testing.T, opts *bind.TransactOpts, to common.Address, value *big.Int) {
+ sendEth := func(t *testing.T, opts *bind.TransactOpts, to common.Address, value *big.Int, errDiffAgainst interface{}) {
  t.Helper()
  unsigned := types.NewTransaction(0, to, value, gasLimit, gasPrice, nil)
  tx, err := opts.Signer(opts.From, unsigned)
  if err != nil {
  t.Fatalf("%T.Signer(%+v) error %v", opts, unsigned, err)
  }
- if err := sim.SendTransaction(ctx, tx); err != nil {
- t.Fatalf("%T.SendTransaction() error %v", sim, err)
+ if diff := errdiff.Check(sim.SendTransaction(ctx, tx), errDiffAgainst); diff != "" {
+ t.Fatalf("%T.SendTransaction() %s", sim, diff)
  }
  }
 
- sendEth(t, sim.Acc(0), signer.Address(), Ether(42))
+ sendEth(t, sim.Acc(0), signer.Address(), Ether(42), nil)
  wantBalance(ctx, t, "faucet after sending 42", sim.Addr(0), new(big.Int).Sub(Ether(100-42), txFee))
  wantBalance(ctx, t, "signer after receiving 42", signer.Address(), Ether(42))
 
@@ -183,32 +184,17 @@ func TestTransactorWithChainID(t *testing.T) {
  if err != nil {
  t.Fatalf("%T.TransactorWithChainID(%d) error %v", signer, chainID, err)
  }
- sendEth(t, opts, sim.Addr(0), Ether(21))
+ sendEth(t, opts, sim.Addr(0), Ether(21), nil)
  wantBalance(ctx, t, "faucet after sending 42 and receiving 21", sim.Addr(0), new(big.Int).Sub(Ether(100-42+21), txFee))
  wantBalance(ctx, t, "signer after receiving 42 and sending 21", signer.Address(), new(big.Int).Sub(Ether(42-21), txFee))
  })
 
  t.Run("incorrect chain ID", func(t *testing.T) {
- // The SimulatedBackend panics instead of returning an error when the
- // chain ID is incorrect. #java
- defer func() {
- const wantContains = "invalid chain id"
- r := recover()
-
- if err, ok := r.(error); ok && strings.Contains(err.Error(), wantContains) {
- return
- }
- t.Errorf("%T.SendTransaction(%T.TransactorWithChainID(<incorrect ID>)) recovered %T(%v); want panic with error containing %q", sim, signer, r, r, wantContains)
- }()
-
  chainID := new(big.Int).Add(sim.Blockchain().Config().ChainID, big.NewInt(1))
  opts, err := signer.TransactorWithChainID(chainID)
  if err != nil {
  t.Fatalf("%T.TransactorWithChainID(%d) error %v", signer, chainID, err)
  }
- sendEth(t, opts, sim.Addr(0), Ether(1))
- // We should never reach here because sendEth results in a panic inside
- // go-ethereum's SimulatedBackend.
- t.Errorf("%T.SendTransaction(%T.TransactorWithChainID(<incorrect ID>)) did not panic", sim, signer)
+ sendEth(t, opts, sim.Addr(0), Ether(1), "invalid chain id")
  })
 }

ethier/gen.go

@@ -1,20 +1,33 @@
 package main
 
 import (
+ "bytes"
+ "encoding/json"
  "errors"
  "fmt"
+ "go/format"
+ "go/parser"
+ "go/token"
  "io"
  "log"
  "os"
  "os/exec"
  "path/filepath"
+ "regexp"
  "strings"
+ "text/template"
 
+ "github.com/ethereum/go-ethereum/common/compiler"
  "github.com/spf13/cobra"
+ "golang.org/x/tools/go/ast/astutil"
+
+ _ "embed"
 )
 
+const srcMapFlag = "experimental_src_map"
+
 func init() {
- rootCmd.AddCommand(&cobra.Command{
+ cmd := &cobra.Command{
  Use: "gen",
  Short: "Compiles Solidity contracts to generate Go ABI bindings with go:generate",
  RunE: gen,
@@ -29,12 +42,16 @@ func init() {
  }
  return nil
  },
- })
+ }
+
+ cmd.Flags().Bool(srcMapFlag, false, "Generate source maps to determine Solidity code location from EVM traces")
+
+ rootCmd.AddCommand(cmd)
 }
 
 // gen runs `solc | abigen` on the Solidity source files passed as the args.
 // TODO: support wildcard / glob matching of files.
-func gen(_ *cobra.Command, args []string) (retErr error) {
+func gen(cmd *cobra.Command, args []string) (retErr error) {
  pwd, err := os.Getwd()
  if err != nil {
  return fmt.Errorf("os.Getwd(): %v", err)
@@ -62,7 +79,7 @@ func gen(_ *cobra.Command, args []string) (retErr error) {
  args,
  "--base-path", basePath,
  "--include-path", filepath.Join(basePath, "node_modules"),
- "--combined-json", "abi,bin",
+ "--combined-json", "abi,bin,bin-runtime,hashes,srcmap-runtime",
  )
  solc := exec.Command("solc", args...)
  solc.Stderr = os.Stderr
@@ -75,13 +92,16 @@ func gen(_ *cobra.Command, args []string) (retErr error) {
  "abigen",
  "--combined-json", "/dev/stdin",
  "--pkg", pkg,
- "--out", "generated.go",
  )
  abigen.Stderr = os.Stderr
 
  r, w := io.Pipe()
  solc.Stdout = w
- abigen.Stdin = r
+ combinedJSON := bytes.NewBuffer(nil)
+ abigen.Stdin = io.TeeReader(r, combinedJSON)
+
+ generated := bytes.NewBuffer(nil)
+ abigen.Stdout = generated
 
  if err := solc.Start(); err != nil {
  return fmt.Errorf("start `solc`: %v", err)
@@ -99,5 +119,146 @@ func gen(_ *cobra.Command, args []string) (retErr error) {
  if err := abigen.Wait(); err != nil {
  return fmt.Errorf("`abigen` returned: %v", err)
  }
- return r.Close()
+ if err := r.Close(); err != nil {
+ return fmt.Errorf("close read-half of pipe from solc to abigen: %v", err)
+ }
+
+ extend, err := cmd.Flags().GetBool(srcMapFlag)
+ if err != nil {
+ return fmt.Errorf("%T.Flags().GetBool(%q): %v", cmd, srcMapFlag, err)
+ }
+ if !extend {
+ return os.WriteFile("generated.go", generated.Bytes(), 0644)
+ }
+
+ out, err := extendGeneratedCode(generated, combinedJSON)
+ if err != nil {
+ return err
+ }
+ return os.WriteFile("generated.go", out, 0644)
+}
+
+var (
+ //go:embed gen_extra.go.tmpl
+ extraCode string
+
+ // extraTemplate is the template for use by extendGeneratedCode().
+ extraTemplate = template.Must(
+ template.New("extra").
+ Funcs(template.FuncMap{
+ "quote": func(s interface{}) string {
+ return fmt.Sprintf("%q", s)
+ },
+ "stringSlice": func(strs []string) string {
+ q := make([]string, len(strs))
+ for i, s := range strs {
+ q[i] = fmt.Sprintf("%q", s)
+ }
+ return fmt.Sprintf("[]string{%s}", strings.Join(q, ", "))
+ },
+ "contract": func(s string) (string, error) {
+ parts := strings.Split(s, ".sol:")
+ if len(parts) != 2 {
+ return "", fmt.Errorf("invalid contract name %q must have format path/to/file.sol:ContractName", s)
+ }
+ return parts[1], nil
+ },
+ }).
+ Parse(extraCode),
+ )
+
+ // Regular expressions for modifying abigen-generated code to work with the
+ // extraTemplate code above.
+ deployedRegexp = regexp.MustCompile(`^\s*return address, tx, &(.+?)\{.*Transactor.*\}, nil\s*$`)
+ // Note the option for matching strings.Replace or strings.ReplaceAll due to
+ // a recent change in abigen.
+ libReplacementRegexp = regexp.MustCompile(`^\s*(.+?)Bin = strings.Replace(?:All)?\(.+?, "__\$([0-9a-f]{34})\$__", (.+?)(?:, -1)?\)\s*$`)
+ // TODO(aschlosberg) replace regular expressions with a more explicit
+ // approach for modifying the output code. This likely requires a PR to the
+ // go-ethereum repo to allow bind.Bind (+/- abigen) to accept an alternate
+ // template.
+)
+
+// extendGeneratedCode adds ethier-specific functionality to code generated by
+// abigen, allowing for interoperability with the ethier/solidity package for
+// source-map interpretation at runtime.
+func extendGeneratedCode(generated, combinedJSON *bytes.Buffer) ([]byte, error) {
+ meta := struct {
+ SourceList []string `json:"sourceList"`
+ Version string `json:"version"`
+
+ Contracts map[string]*compiler.Contract
+ CombinedJSON string
+ }{CombinedJSON: combinedJSON.String()}
+
+ if err := json.Unmarshal(combinedJSON.Bytes(), &meta); err != nil {
+ return nil, fmt.Errorf("json.Unmarshal([solc output], %T): %v", &meta, err)
+ }
+
+ cs, err := compiler.ParseCombinedJSON(combinedJSON.Bytes(), "", "", meta.Version, "")
+ if err != nil {
+ return nil, fmt.Errorf("compiler.ParseCombinedJSON(): %v", err)
+ }
+ meta.Contracts = cs
+ for k, c := range meta.Contracts {
+ if c.RuntimeCode == "0x" {
+ delete(meta.Contracts, k)
+ }
+ }
+
+ if err := extraTemplate.Execute(generated, meta); err != nil {
+ return nil, fmt.Errorf("%T.Execute(): %v", extraTemplate, err)
+ }
+
+ // When using vm.Config.Trace, the only contract-identifying information is
+ // the address to which the transaction was sent. We must therefore modify
+ // every DeployFoo() function to save the address(es) at which the contract
+ // is deployed.
+ lines := strings.Split(generated.String(), "\n")
+ for i, l := range lines {
+ matches := deployedRegexp.FindStringSubmatch(l)
+ if len(matches) == 0 {
+ continue
+ }
+ lines[i] = fmt.Sprintf(
+ `deployedContracts[address] = %q // Added by ethier gen
+ %s`,
+ matches[1], l,
+ )
+ }
+
+ // Libraries have their addresses string-replaced directly into contract
+ // code, which we need to mirror for the runtime code too.
+ for i, l := range lines {
+ matches := libReplacementRegexp.FindStringSubmatch(l)
+ if len(matches) == 0 {
+ continue
+ }
+ lines[i] = fmt.Sprintf(
+ `%s
+ RuntimeSourceMaps[%q].RuntimeCode = strings.Replace(RuntimeSourceMaps[%[2]q].RuntimeCode, "__$%s$__", %s, -1)`,
+ l, matches[1], matches[2], matches[3],
+ )
+ }
+
+ // Effectively the same as running goimports on the (ugly) generated code.
+ fset := token.NewFileSet()
+ f, err := parser.ParseFile(fset, "generated.go", strings.Join(lines, "\n"), parser.ParseComments|parser.AllErrors)
+ if err != nil {
+ return nil, fmt.Errorf("parser.ParseFile(%T, …): %v", fset, err)
+ }
+ for _, pkg := range []string{
+ "github.com/ethereum/go-ethereum/common/compiler",
+ "github.com/divergencetech/ethier/solidity",
+ } {
+ if !astutil.AddImport(fset, f, pkg) {
+ return nil, fmt.Errorf("add import %q to generated Go: %v", pkg, err)
+ }
+ }
+
+ buf := bytes.NewBuffer(nil)
+ if err := format.Node(buf, fset, f); err != nil {
+ return nil, fmt.Errorf("format.Node(%T, %T, %T): %v", buf, fset, f, err)
+ }
+ return buf.Bytes(), nil
 }

ethier/gen_extra.go.tmpl

@@ -0,0 +1,52 @@
+/**
+ *
+ * Additional code added by ethier, beyond standard abigen output.
+ *
+ */
+
+const (
+ // SolCVersion is the version of the Solidity compiler used to create this
+ // file.
+ SolCVersion = {{quote .Version}}
+
+ // CombinedJSON is the raw combined-JSON output of solc, passed to abigen to
+ // create this file.
+ CombinedJSON = {{quote .CombinedJSON}}
+)
+
+var (
+ // SourceList is the list of source files used by solc when compiling these
+ // contracts. Their indices correspond to the file numbers in the source
+ // maps.
+ SourceList = {{stringSlice .SourceList}}
+
+ // RuntimeSourceMaps contains, for each compiled contract, the runtime
+ // binary and its associated source map. With a program counter pointing to
+ // an instruction in the runtime binary, this is sufficient to determine the
+ // respective location in the Solidity code from which the binary was
+ // compiled.
+ RuntimeSourceMaps = map[string]*compiler.Contract{
+ {{- range $src, $c := .Contracts }}
+ {{quote (contract $src)}}: {
+ RuntimeCode: {{quote $c.RuntimeCode}},
+ Info: compiler.ContractInfo{
+ SrcMapRuntime: {{quote $c.Info.SrcMapRuntime}},
+ },
+ },
+ {{- end }}
+ }
+)
+
+// deployedContracts tracks which contract is deployed at each address. The
+// standard abigen Deploy<ContractName>() functions have been modified to set
+// the value of this map to <ContractName> before returning the deployment
+// address. This allows SourceMap() to function correctly.
+var deployedContracts = make(map[common.Address]string)
+
+// SourceMap returns a new SourceMap, able to convert program counters to
+// Solidity source offsets. SourceMap() must be called after contracts are
+// deployed otherwise they won't be registered by contract address (only by
+// contract name).
+func SourceMap() (*solidity.SourceMap, error) {
+ return solidity.NewSourceMap(SourceList, RuntimeSourceMaps, deployedContracts)
+}