V1.10.23 statediff v4 wip

.github/workflows/checks.yml

@@ -0,0 +1,15 @@
+name: checks
+
+on: [pull_request]
+
+jobs:
+ linter-check:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/setup-go@v3
+ with:
+ go-version: "1.18"
+ check-latest: true
+ - uses: actions/checkout@v2
+ - name: Run linter
+ run: go run build/ci.go lint

.github/workflows/on-pr.yml

@@ -0,0 +1,7 @@
+name: Build and test
+
+on: [pull_request]
+
+jobs:
+ run-tests:
+ uses: ./.github/workflows/tests.yml

.github/workflows/publish.yaml

@@ -0,0 +1,61 @@
+name: Publish geth to release
+on:
+ release:
+ types: [published]
+jobs:
+ run-tests:
+ uses: ./.github/workflows/tests.yml
+ build:
+ name: Run docker build and publish
+ needs: run-tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Run docker build
+ run: docker build -t vulcanize/go-ethereum -f Dockerfile .
+ - name: Get the version
+ id: vars
+ run: echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+ - name: Tag docker image
+ run: docker tag vulcanize/go-ethereum docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+ - name: Docker Login
+ run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+ - name: Docker Push
+ run: docker push docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+ push_to_registries:
+ name: Publish assets to Release
+ runs-on: ubuntu-latest
+ needs: build
+ steps:
+ - name: Get the version
+ id: vars
+ run: |
+ echo ::set-output name=sha::$(echo ${GITHUB_SHA:0:7})
+ echo ::set-output name=tag::$(echo ${GITHUB_REF#refs/tags/})
+ - name: Docker Login to Github Registry
+ run: echo ${{ secrets.GITHUB_TOKEN }} | docker login https://docker.pkg.github.com -u vulcanize --password-stdin
+ - name: Docker Pull
+ run: docker pull docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}}
+ - name: Copy ethereum binary file
+ run: docker run --rm --entrypoint cat docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} /usr/local/bin/geth > geth-linux-amd64
+ - name: Docker Login to Docker Registry
+ run: echo ${{ secrets.VULCANIZEJENKINS_PAT }} | docker login -u vulcanizejenkins --password-stdin
+ - name: Tag docker image
+ run: docker tag docker.pkg.github.com/vulcanize/go-ethereum/go-ethereum:${{steps.vars.outputs.sha}} vulcanize/vdb-geth:${{steps.vars.outputs.tag}}
+ - name: Docker Push to Docker Hub
+ run: docker push vulcanize/vdb-geth:${{steps.vars.outputs.tag}}
+ - name: Get release
+ id: get_release
+ uses: bruceadams/get-release@v1.2.0
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ - name: Upload Release Asset
+ id: upload-release-asset
+ uses: actions/upload-release-asset@v1
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ with:
+ upload_url: ${{ steps.get_release.outputs.upload_url }}
+ asset_path: geth-linux-amd64
+ asset_name: geth-linux-amd64
+ asset_content_type: application/octet-stream

.github/workflows/tests.yml

@@ -0,0 +1,142 @@
+name: Tests for Geth that are used in multiple jobs.
+
+on:
+ workflow_call:
+
+env:
+ stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'f2fd766f5400fcb9eb47b50675d2e3b1f2753702'}}
+ ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'be345e0733d2c025e4082c5154e441317ae94cf7' }}
+ GOPATH: /tmp/go
+
+jobs:
+ build:
+ name: Run docker build
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Run docker build
+ run: docker build -t vulcanize/go-ethereum .
+
+ geth-unit-test:
+ name: Run geth unit test
+ runs-on: ubuntu-latest
+ env:
+ GO111MODULE: on
+ steps:
+ - name: Create GOPATH
+ run: mkdir -p /tmp/go
+
+ - uses: actions/setup-go@v3
+ with:
+ go-version: "1.18"
+ check-latest: true
+
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Run unit tests
+ run: |
+ make test
+
+ statediff-unit-test:
+ name: Run state diff unit test
+ runs-on: ubuntu-latest
+ steps:
+ - name: Create GOPATH
+ run: mkdir -p /tmp/go
+
+ - uses: actions/setup-go@v3
+ with:
+ go-version: "1.18"
+ check-latest: true
+
+ - name: Checkout code
+ uses: actions/checkout@v2
+
+ - name: Run docker compose
+ run: |
+ docker-compose up -d
+
+ - name: Give the migration a few seconds
+ run: sleep 30;
+
+ - name: Run unit tests
+ run: make statedifftest
+
+ private-network-test:
+ name: Start Geth in a private network.
+ runs-on: ubuntu-latest
+ steps:
+ - name: Create GOPATH
+ run: mkdir -p /tmp/go
+
+ - uses: actions/setup-go@v3
+ with:
+ go-version: "1.18"
+ check-latest: true
+
+ - name: Checkout code
+ uses: actions/checkout@v3
+ with:
+ path: "./go-ethereum"
+
+ - uses: actions/checkout@v3
+ with:
+ ref: ${{ env.stack-orchestrator-ref }}
+ path: "./stack-orchestrator/"
+ repository: vulcanize/stack-orchestrator
+ fetch-depth: 0
+
+ - uses: actions/checkout@v3
+ with:
+ ref: ${{ env.ipld-eth-db-ref }}
+ repository: vulcanize/ipld-eth-db
+ path: "./ipld-eth-db/"
+ fetch-depth: 0
+
+ - name: Create config file
+ run: |
+ echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > $GITHUB_WORKSPACE/config.sh
+ echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> $GITHUB_WORKSPACE/config.sh
+ echo db_write=true >> $GITHUB_WORKSPACE/config.sh
+ echo genesis_file_path=start-up-files/go-ethereum/genesis.json >> $GITHUB_WORKSPACE/config.sh
+ cat $GITHUB_WORKSPACE/config.sh
+
+ - name: Compile Geth
+ run: |
+ cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts
+ ./compile-geth.sh -e docker -p $GITHUB_WORKSPACE/config.sh
+ cd -
+
+ - name: Run docker compose
+ run: |
+ docker-compose \
+ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \
+ -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" \
+ --env-file $GITHUB_WORKSPACE/config.sh \
+ up -d --build
+
+ - name: Make sure the /root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS exists within a certain time frame.
+ shell: bash
+ run: |
+ COUNT=0
+ ATTEMPTS=15
+ docker ps
+ until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" cp go-ethereum:/root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS ./STATEFUL_TEST_DEPLOYED_ADDRESS) || [[ $COUNT -eq $ATTEMPTS ]]; do echo -e "$(( COUNT++ ))... \c"; sleep 10; done
+ [[ $COUNT -eq $ATTEMPTS ]] && echo "Could not find the successful contract deployment" && (exit 1)
+ cat ./STATEFUL_TEST_DEPLOYED_ADDRESS
+ sleep 15;
+
+ - name: Create a new transaction.
+ shell: bash
+ run: |
+ docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum /bin/bash /root/transaction_info/NEW_TRANSACTION
+ echo $?
+
+ - name: Make sure we see entries in the header table
+ shell: bash
+ run: |
+ rows=$(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT COUNT(*) FROM eth.header_cids")
+ [[ "$rows" -lt "1" ]] && echo "We could not find any rows in postgres table." && (exit 1)
+ echo $rows
+ docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db-sharding.yml" -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec ipld-eth-db psql -U vdbm -d vulcanize_testing -AXqtc "SELECT * FROM eth.header_cids"

.gitignore

@@ -47,3 +47,15 @@ profile.cov
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log
+foundry/deployments/local-private-network/geth-linux-amd64
+foundry/projects/local-private-network/geth-linux-amd64
+
+# Helpful repos
+related-repositories/foundry-test/**
+related-repositories/hive/**
+related-repositories/ipld-eth-db/**
+statediff/indexer/database/sql/statediffing_test_file.sql
+statediff/statediffing_test_file.sql
+statediff/known_gaps.sql
+related-repositories/foundry-test/
+related-repositories/ipld-eth-db/

.gitmodules

@@ -5,4 +5,4 @@
 [submodule "evm-benchmarks"]
  path = tests/evm-benchmarks
  url = https://github.com/ipsilon/evm-benchmarks
- shallow = true
+ shallow = true

Dockerfile.amd64

@@ -0,0 +1,7 @@
+# Build Geth in a stock Go builder container
+FROM golang:1.15.5 as builder
+
+#RUN apk add --no-cache make gcc musl-dev linux-headers git
+
+ADD . /go-ethereum
+RUN cd /go-ethereum && make geth

Makefile

@@ -4,10 +4,31 @@
 
 .PHONY: geth android ios evm all test clean
 
+BIN = $(GOPATH)/bin
+
+## Migration tool
+GOOSE = $(BIN)/goose
+$(BIN)/goose:
+ go get -u github.com/pressly/goose/cmd/goose
+
 GOBIN = ./build/bin
 GO ?= latest
 GORUN = env GO111MODULE=on go run
 
+#Database
+HOST_NAME = localhost
+PORT = 5432
+USER = vdbm
+PASSWORD = password
+
+# Set env variable
+# `PGPASSWORD` is used by `createdb` and `dropdb`
+export PGPASSWORD=$(PASSWORD)
+
+#Test
+TEST_DB = vulcanize_public
+TEST_CONNECT_STRING = postgresql://$(USER):$(PASSWORD)@$(HOST_NAME):$(PORT)/$(TEST_DB)?sslmode=disable
+
 geth:
  $(GORUN) build/ci.go install ./cmd/geth
  @echo "Done building."
@@ -48,3 +69,13 @@ devtools:
  env GOBIN= go install ./cmd/abigen
  @type "solc" 2> /dev/null || echo 'Please install solc'
  @type "protoc" 2> /dev/null || echo 'Please install protoc'
+
+.PHONY: statedifftest
+statedifftest: | $(GOOSE)
+ GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0
+ GO111MODULE=on MODE=statediff go test -p 1 ./statediff/... -v
+
+.PHONY: statediff_filewriting_test
+statediff_filetest: | $(GOOSE)
+ GO111MODULE=on go get github.com/stretchr/testify/assert@v1.7.0
+ GO111MODULE=on MODE=statediff STATEDIFF_DB=file go test -p 1 ./statediff/... -v