Merge pull request #438 from district0x/master-checkpoint-debugging

Monkey-patch server.web3-events to avoir repeating last-processed block

.github/workflows/docker-image.yml

@@ -41,11 +41,11 @@ jobs:
           repository: district0x/ethlance-config
           path: ethlance-config
           token: ${{ secrets.ETHLANCE_CONFIG_PAT }}
-          ref: master
+          ref: docker_build
 
       - name: Run ganache service
         run: |
-          docker run --rm -d --name ganache -p 8550:8550 trufflesuite/ganache:v7.9.1 -v -m "easy leave proof verb wait patient fringe laptop intact opera slab shine" -p 8550 -b 0.5 --chain.allowUnlimitedContractSize=true
+          docker run --rm -d --name ganache -p 8550:8550 trufflesuite/ganache:v7.9.1 -v -m '${{vars.ETHLANCE_DEPLOY_SEED}}' -p 8550 -b 0.5 --chain.allowUnlimitedContractSize=true
 
       - name: Restore Top level cache directory for Ethlance
         uses: actions/cache@v3
@@ -76,6 +76,8 @@ jobs:
           cd ..
 
       - name: Deploy contracts to ganache-test
+        env:
+          ETHLANCE_DEPLOY_SEED: ${{ vars.ETHLANCE_DEPLOY_SEED }}
         run: |
           npx truffle migrate --network ganache-test --reset
 

.gitignore

@@ -4,6 +4,7 @@ node_modules
 .cpcache
 ui/out/**
 server/out/**
+server/out-syncer/**
 yarn-error.log
 ui/resources/public/js
 ui/resources/public/css

bb.edn

@@ -43,6 +43,12 @@
                                    command-args (format "-A:dev:shadow-cljs%s watch dev-server" (clojure.string.join "" aliases))]
                                 (println "Starting watch dev-server args:" command-args)
                                 (clojure {:dir "server"} command-args))}
+         watch-syncer {:doc "Watch and compile syncer code changes CLJS -> JS in server/src/ethlance/server/syncer*"
+                       :requires ([babashka.cli :as cli])
+                       :task (let [aliases (or *command-line-args* [])
+                                   command-args (format "-A:dev:shadow-cljs%s watch new-syncer" (clojure.string.join "" aliases))]
+                                (println "Starting watch dev-server args:" command-args)
+                                (clojure {:dir "server"} command-args))}
          run-server {:doc "Start Node.js API server process"
                      :task (do
                              (println "Starting Ethlance API server")
@@ -60,6 +66,7 @@
                            [build-id folder] (case build-arg
                                                    "ui" [:dev-ui "ui"]
                                                    "server" [:dev-server "server"]
+                                                   "syncer" [:new-syncer "server"]
                                                    "test" [:test-repl "server"])
                            _ (println ">>> build-id folder" build-id folder)
                            parse-port (fn [folder] (get-in (clojure.edn/read-string (slurp (str folder "/shadow-cljs.edn"))) [:nrepl :port]))

docker-builds/base/Dockerfile

@@ -1,5 +1,5 @@
 ARG BUILD_ENV=qa
-FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:node-18.17.0
+FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:node-18.20.1
 USER root
 
 RUN npm i -g truffle@v5.11
@@ -12,7 +12,7 @@ WORKDIR /build/ethlance
 COPY . .
 
 WORKDIR /build/ethlance-config
-COPY config .
+COPY ethlance-config .
 
 WORKDIR /build
 RUN git clone https://github.com/district0x/d0x-libs

docker-builds/server/Dockerfile

@@ -1,12 +1,14 @@
-FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:node-18.17.0 as init
+FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:node-18.20.1 as init
 ARG BUILD_ENV=qa
+ARG ETHLANCE_DEPLOY_SEED="sign bachelor state zoo expire boat morning situate scene unveil oven crew"
 USER root
 
 ENV ETHLANCE_COMPONENT="server"
 ENV ETHLANCE_SOURCE_ROOT=/build/ethlance
 ENV ETHLANCE_SERVER_ROOT=/build/ethlance/server
 ENV ETHLANCE_ENV=$BUILD_ENV
 ENV DEPLOY_TARGET=/deploy/
+ENV ETHLANCE_DEPLOY_SEED=${ETHLANCE_DEPLOY_SEED}
 
 ENV SMART_CONTRACTS_PATH="/build/ethlance/shared/src/ethlance/shared/smart_contracts_${BUILD_ENV}.cljs"
 ENV SMART_CONTRACTS_BUILD_PATH="/build/ethlance/resources/public/contracts/build"
@@ -18,18 +20,19 @@ COPY . .
 
 RUN cat "${SMART_CONTRACTS_PATH}"
 
-WORKDIR /build/ethlance-config
-COPY config .
-
+# WORKDIR /build/ethlance-config/
+# COPY ethlance-config .
 WORKDIR /build
-# RUN git clone https://github.com/district0x/d0x-libs
+COPY ethlance-config/config  /build/ethlance/config
+# RUN cp /build/ethlance/ethlance-config/config  /build/ethlance/
+RUN git clone https://github.com/district0x/d0x-libs
 
 FROM init as build_server
-WORKDIR $ETHLANCE_SOURCE_ROOT
+WORKDIR ${ETHLANCE_SOURCE_ROOT}
 RUN yarn install && ETHLANCE_ENV="${BUILD_ENV}" npx truffle compile
 
 # Release deployment
-WORKDIR $ETHLANCE_SOURCE_ROOT/ui 
+WORKDIR "${ETHLANCE_SOURCE_ROOT}/ui"
 RUN yarn && \
     ETHLANCE_ENV="${BUILD_ENV}" npx shadow-cljs release dev-ui && \
     ./node_modules/less/bin/lessc resources/public/less/main.less resources/public/css/main.css --verbose
@@ -49,8 +52,8 @@ ENV UI_CONFIG_PATH="/deploy/resources/config/ui-config-${BUILD_ENV}.edn"
 ENV SERVER_CONFIG_PATH="/deploy/resources/config/server-config-${BUILD_ENV}.edn"
 WORKDIR "${ETHLANCE_SERVER_ROOT}"
 
-COPY --from=build_server /build/ethlance-config/ui-config-${ETHLANCE_ENV}.edn /deploy/resources/config/ui-config-${ETHLANCE_ENV}.edn
-COPY --from=build_server /build/ethlance-config/server-config-${ETHLANCE_ENV}.edn /deploy/resources/config/server-config-${ETHLANCE_ENV}.edn
+COPY --from=build_server /build/ethlance/config/ui-config-${ETHLANCE_ENV}.edn /deploy/resources/config/ui-config-${ETHLANCE_ENV}.edn
+COPY --from=build_server /build/ethlance/config/server-config-${ETHLANCE_ENV}.edn /deploy/resources/config/server-config-${ETHLANCE_ENV}.edn
 COPY --from=build_server /build/ethlance/resources /deploy/resources
 COPY --from=build_server /build/ethlance/server/node_modules /deploy/server/node_modules/
 COPY --from=build_server /build/ethlance/server/package.json /build/ethlance/server/out/ethlance_server* /deploy/server/

docker-builds/ui/Dockerfile

@@ -1,29 +1,33 @@
-FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:node-18.17.0 as init
+FROM --platform=amd64 487920318758.dkr.ecr.us-west-2.amazonaws.com/cljs-web3-ci:node-18.20.1 as init
 ARG BUILD_ENV=qa
+ARG ETHLANCE_DEPLOY_SEED="sign bachelor state zoo expire boat morning situate scene unveil oven crew"
 USER root
 
 ENV ETHLANCE_COMPONENT="ui"
 ENV ETHLANCE_SOURCE_ROOT=/build/ethlance
 ENV ETHLANCE_SERVER_ROOT=/build/ethlance/server
 ENV ETHLANCE_ENV=$BUILD_ENV
 ENV DEPLOY_TARGET=/deploy/
-
+ENV ETHLANCE_DEPLOY_SEED="${ETHLANCE_DEPLOY_SEED}"
 ENV SMART_CONTRACTS_PATH="/build/ethlance/shared/src/ethlance/shared/smart_contracts_${BUILD_ENV}.cljs"
 ENV SMART_CONTRACTS_BUILD_PATH="/build/ethlance/resources/public/contracts/build"
 ENV ETHLANCE_CONFIG_PATH="/build/ethlance/config/${ETHLANCE_COMPONENT}-config-${BUILD_ENV}.edn"
 
 WORKDIR /build/ethlance
 COPY . .
 
-RUN cat shared/src/ethlance/shared/smart_contracts_${ETHLANCE_ENV}.cljs
+RUN cat "${SMART_CONTRACTS_PATH}"
 
-WORKDIR /build/ethlance-config
-COPY config .
+# WORKDIR /build/ethlance-config
+# COPY config .
 
-# WORKDIR /build
-# RUN git clone https://github.com/district0x/d0x-libs
+WORKDIR /build
+COPY ethlance-config/config  /build/ethlance/config
+RUN git clone https://github.com/district0x/d0x-libs
 
 
+ARG BUILD_ENV=qa
+ARG ETHLANCE_ENV=qa
 FROM init as build_stage
 ENV ETHLANCE_ENV=qa
 WORKDIR $ETHLANCE_SOURCE_ROOT

migrations/2_unchangeable_and_proxy_contracts.js

@@ -14,8 +14,16 @@ let Job = artifacts.require("Job");
 let EthlanceStructs = artifacts.require("EthlanceStructs");
 let EthlanceProxy = artifacts.require("EthlanceProxy");
 let MutableForwarder = artifacts.require("MutableForwarder");
-// copy("EthlanceProxy", "EthlanceProxy", contracts_build_directory);
-// let EthlanceProxy = artifacts.require("EthlanceProxy");
+
+let deployedContracts = {};
+
+function getDeployedAddress(contractName) {
+  const address = deployedContracts[contractName];
+  if (address == null) {
+    throw new Error("Contract with" + contractName + " not founc in deployedContracts");
+  }
+  return address;
+}
 
 // Deployment Functions
 
@@ -54,7 +62,8 @@ async function deploy_EthlanceStructs(deployer, opts){
 
 async function deploy_JobHelpers(deployer, opts){
   let jobHelpers = await deployer.deploy(JobHelpers, {...opts, gas: 6e6});
-  deployer.link(JobHelpers, Job);
+  // Linking to Job to be done in deploy_Job()
+  // deployer.link(JobHelpers, Job);
   assignContract(jobHelpers, "JobHelpers", "job-helpers");
 }
 
@@ -67,6 +76,10 @@ async function deploy_proxies(deployer, opts) {
 }
 
 async function deploy_Job(deployer, opts){
+  jobHelpersInstance = await JobHelpers.at(getDeployedAddress("JobHelpers"));
+  ethlanceStructsInstance = await EthlanceStructs.at(getDeployedAddress("EthlanceStructs"));
+  deployer.link(jobHelpersInstance, Job);
+  deployer.link(ethlanceStructsInstance, Job);
   let job = await deployer.deploy(Job, {...opts, gas: 18e6});
   assignContract(job, "Job", "job");
 }
@@ -77,6 +90,7 @@ async function deploy_all(deployer, opts) {
   await deploy_TestMultiToken(deployer, opts);
   await deploy_EthlanceStructs(deployer, opts);
   await deploy_JobHelpers(deployer, opts);
+
   await deploy_Job(deployer, opts);
   await deploy_proxies(deployer, opts);
 }
@@ -93,6 +107,7 @@ let smart_contract_listing = [];
  */
 function assignContract(contract_instance, contract_name, contract_key, opts) {
   console.log("- Assigning '" + contract_name + "' to smart contract listing...");
+  deployedContracts[contract_name] = contract_instance.address;
   opts = opts || {};
   smart_contract_listing = smart_contract_listing.concat(
     encodeContractEDN(contract_instance, contract_name, contract_key, opts));
@@ -114,6 +129,7 @@ function writeSmartContracts() {
 //
 module.exports = async function(deployer, network, accounts) {
   const gas = 4e6;
+  console.log("Deploying FROM:", deployer.options.from);
   const from = deployer.options.from || accounts[0];
   const opts = {gas: gas, from: from};
 

migrations/3_ethlance_initial_implementation.js

@@ -1,5 +1,6 @@
 const edn = require ("jsedn");
 const {env, smart_contracts_path, contracts_build_directory} = require ('../truffle-config.js');
+console.log(">>> Migra 3: env", env);
 const {copy, encodeContractEDN, readSmartContractsFile, getSmartContractAddress, setSmartContractAddress, writeSmartContracts, setSmartContractForwardsTo, linkBytecode} = require("./utils");
 
 const Ethlance = artifacts.require("Ethlance");

server/deps.edn

@@ -1,7 +1,7 @@
-{:paths ["src" "test"]
+{:paths ["src"]
  :deps
- {org.clojure/clojure {:mvn/version "1.10.3"},
-  org.clojure/clojurescript {:mvn/version "1.11.60"}
+ {org.clojure/clojure {:mvn/version "1.12.0"},
+  org.clojure/clojurescript {:mvn/version "1.11.132"}
 
   is.d0x/ethlance-shared {:local/root "../shared"}
   mvxcvi/alphabase {:mvn/version "2.1.1"}
@@ -60,7 +60,7 @@
  :aliases
  {:shadow-cljs
   {:extra-deps
-   {thheller/shadow-cljs {:mvn/version "2.26.2"}}
+   {thheller/shadow-cljs {:mvn/version "2.28.16"}}
    :main-opts ["-m" "shadow.cljs.devtools.cli"]}
   :local-deps
   {:override-deps
@@ -76,4 +76,6 @@
    {; Vim Iced deps
     refactor-nrepl/refactor-nrepl {:mvn/version "3.3.2"}
     cider/cider-nrepl {:mvn/version "0.28.2"}
-    com.github.liquidz/iced-nrepl {:mvn/version "1.2.8"}}}}}
+    com.github.liquidz/iced-nrepl {:mvn/version "1.2.8"}}}
+  :test
+  {:extra-paths ["test"]}}}

server/package.json

@@ -23,11 +23,12 @@
     "graphql": "14.2.1",
     "graphql-middleware": "4.0.1",
     "graphql-tools": "4.0.5",
+    "@helia/verified-fetch": "1.5.0",
     "is-ipfs": "2.0.0",
     "jsedn": "0.4.1",
     "jsonwebtoken": "8.5.1",
     "pg": "8.8.0",
-    "shadow-cljs": "2.26.2",
+    "shadow-cljs": "2.28.16",
     "web3": "1.7.3",
     "ws": "4.0.0",
     "xhr2": "0.2.1",

server/shadow-cljs.edn

@@ -18,6 +18,19 @@
            {:compiler-options {:optimizations :simple
                                :source-map true}}}
 
+          :new-syncer
+          {:target :node-script
+           :main ethlance.server.new-syncer/-main
+           :output-dir "out-syncer/"
+           :output-to "out-syncer/new-syncer.js"
+           :optimizations :none
+           :source-map true
+           :source-map-timestamp true
+           :closure-defines {goog.DEBUG true}
+
+           :release
+           {:compiler-options {:optimizations :simple
+                               :source-map true}}}
           :server-tests
           {:target :node-test
            :js-options {:js-package-dirs ["./server/node_modules"]}

server/src/ethlance/server/config.cljs

@@ -0,0 +1,82 @@
+(ns ethlance.server.config
+  (:require
+    [ethlance.server.db :as server-db]
+    [ethlance.server.new-syncer.handlers :as new-syncer.handlers]
+    [ethlance.server.new-syncer :as new-syncer]
+    [ethlance.shared.config :as shared-config]
+    [ethlance.shared.smart-contracts-dev :as smart-contracts-dev]
+    [ethlance.shared.smart-contracts-prod :as smart-contracts-prod]
+    [ethlance.shared.smart-contracts-qa :as smart-contracts-qa]
+    [ethlance.shared.smart-contracts-qa-base :as smart-contracts-qa-base]
+    [ethlance.shared.utils :include-macros true :as shared-utils]
+    ))
+
+
+(def environment (shared-utils/get-environment))
+
+(println "Ethlance server starting in environment:" environment)
+
+
+(def contracts-var
+  (condp = environment
+    "prod" #'smart-contracts-prod/smart-contracts
+    "qa" #'smart-contracts-qa/smart-contracts
+    "qa-base" #'smart-contracts-qa-base/smart-contracts
+    "dev" #'smart-contracts-dev/smart-contracts
+    ; "dev" #'smart-contracts-qa-base/smart-contracts
+    ))
+
+
+(def default-config
+  {:web3 {:url  "ws://127.0.0.1:8549"}
+   :new-syncer {:auto-start-listening-new-events? false
+                :handlers new-syncer.handlers/handlers
+                :save-checkpoint server-db/save-processed-events-checkpoint}
+   :web3-events {:events
+                 {:ethlance/job-created [:ethlance :JobCreated]
+                  :ethlance/invoice-created [:ethlance :InvoiceCreated]
+                  :ethlance/invoice-paid [:ethlance :InvoicePaid]
+                  :ethlance/dispute-raised [:ethlance :DisputeRaised]
+                  :ethlance/dispute-resolved [:ethlance :DisputeResolved]
+                  :ethlance/candidate-added [:ethlance :CandidateAdded]
+                  :ethlance/quote-for-arbitration-set [:ethlance :QuoteForArbitrationSet]
+                  :ethlance/quote-for-arbitration-accepted [:ethlance :QuoteForArbitrationAccepted]
+                  :ethlance/job-ended [:ethlance :JobEnded]
+                  :ethlance/arbiters-invited [:ethlance :ArbitersInvited]
+                  :ethlance/funds-in [:ethlance :FundsIn]
+                  :ethlance/funds-out [:ethlance :FundsOut]
+                  :ethlance/test-event [:ethlance :TestEvent]}
+                 :from-block 1000
+                 :block-step 5 ; 1000
+                 :dispatch-logging? true
+                 :crash-on-event-fail? true
+                 :skip-past-events-replay? false
+                 :load-checkpoint server-db/load-processed-events-checkpoint
+                 :save-checkpoint server-db/save-processed-events-checkpoint
+                 :callback-after-past-events new-syncer/start-listening-new-events
+                 }
+   :smart-contracts {:contracts-var contracts-var
+                     :contracts-build-path "../resources/public/contracts/build"
+                     :print-gas-usage? false
+                     :auto-mining? false}
+   :graphql {:port 6300
+             :sign-in-secret "SECRET"
+             :graphiql (= environment "dev")}
+   :district/db {:user "ethlanceuser"
+                 :host "localhost"
+                 :database "ethlance"
+                 :password "pass"
+                 :port 5432}
+   :ethlance/db {:resync? false}
+   :ipfs {:host "http://host-machine:5001"
+          :endpoint "/api/v0"
+          :gateway "http://host-machine:8080/ipfs"}
+   :logging {:level "debug"
+             :console? true}})
+
+
+(defn env-config
+  [_env]
+  (shared-utils/deep-merge
+    default-config
+    shared-config/config))