From 2de9310137550ba0b1a5830fac1ea00fef8d9ed1 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:10:26 -0500 Subject: [PATCH 01/18] Cleans up the scripts folder --- .gitignore | 3 +-- scripts/README.md | 4 +++- scripts/up.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 385b33a..5020944 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .DS_Store .env -test.sh .save_env docker-compose.local.yml -trueblocks-dappnode + diff --git a/scripts/README.md b/scripts/README.md index 90dc16b..b8f27c9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,3 +1,5 @@ # Scripts -The scripts in this folder are for use of the developers only. You may use them, but they are unsupported and undocumented. +The scripts in this folder are for use of the TrueBlocks developers. + +You may use them, but they are unsupported and undocumented. diff --git a/scripts/up.sh b/scripts/up.sh index 9bc8cd9..f9f9ae3 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Start the containers defined in docker-compose.yml and docker-compose.local.yml +# Starts the containers defined in docker-compose.yml and docker-compose.local.yml docker compose -f docker-compose.yml -f docker-compose.local.yml up \ No newline at end of file From 03b54493c0b934d54f1f84b96240e19bb255030c Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:10:34 -0500 Subject: [PATCH 02/18] Moves test.sh to scripts folder --- test.sh => scripts/test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test.sh => scripts/test.sh (100%) diff --git a/test.sh b/scripts/test.sh similarity index 100% rename from test.sh rename to scripts/test.sh From f02263c8723fc99bc14a1053dbd8b0fd0d475ebf Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:20:55 -0500 Subject: [PATCH 03/18] Simply removes all comments to make the scripts / Dockerfiles easier to understand --- build/core/Dockerfile | 24 ++++-------- build/core/core.entrypoint.sh | 9 ----- build/core/test/core-test.entrypoint.sh | 5 --- docker-compose.local.example | 27 ------------- docker-compose.yml | 52 ------------------------- env.example | 43 -------------------- scripts/publish.sh | 5 --- 7 files changed, 7 insertions(+), 158 deletions(-) diff --git a/build/core/Dockerfile b/build/core/Dockerfile index a87d1ef..5dff99c 100644 --- a/build/core/Dockerfile +++ b/build/core/Dockerfile @@ -1,47 +1,37 @@ -# Latest golang as builder FROM golang:1.18-alpine as builder -# Install build depenedencies RUN apk --no-cache add g++ gcc make cmake git nano libcurl python3 python3-dev \ curl bash curl-dev linux-headers sqlite-dev -# Set workdir WORKDIR /root -# Try to get upstream version (default v0.41.0-beta) ARG UPSTREAM_VER=v0.41.0-beta -# Clone and make TrueBlocks Core -# make -j 5 is a fairly safe number RUN git clone -b "${UPSTREAM_VER}" --single-branch --progress --depth 1 \ - https://github.com/TrueBlocks/trueblocks-core.git && \ - cd trueblocks-core && mkdir build && cd build && cmake ../src && make -j 5 + https://github.com/TrueBlocks/trueblocks-core.git && \ + cd trueblocks-core && \ + mkdir -p build && \ + cd build && \ + cmake ../src && \ + make -j 5 -# Switch to alpine container FROM alpine:latest -# Install binary dependencies and nice to haves RUN apk --no-cache add gzip libstdc++ libgcc libcurl python3 python3-dev procps bash curl nano findutils -# Copy files from builder COPY --from=builder /root/trueblocks-core/bin /usr/local/bin COPY --from=builder /root/.local/bin/chifra /root/.local/bin/chifra COPY --from=builder /root/.local/share/trueblocks /root/.local/share/trueblocks -# Copy entrypoint file COPY core.entrypoint.sh /root - -# Testing COPY test/core-test.entrypoint.sh /root + RUN chmod +x /root/core-test.entrypoint.sh -# Expose ports ARG SERVE_PORT=8080 EXPOSE ${SERVE_PORT} -# Set correct paths ENV TB_SETTINGS_CACHEPATH=/cache ENV TB_SETTINGS_INDEXPATH=/index -# Run entrypoint script ENTRYPOINT bash /root/core.entrypoint.sh diff --git a/build/core/core.entrypoint.sh b/build/core/core.entrypoint.sh index a4771d1..5f10165 100644 --- a/build/core/core.entrypoint.sh +++ b/build/core/core.entrypoint.sh @@ -1,7 +1,5 @@ #!/bin/bash -# index_chain() starts scraper of the given chain, reading arguments and file -# content from env variables (optional). index_chain() { CHAIN_NAME=`echo $1 | tr '[:upper:]' '[:lower:]'` CHAIN_VAR=$1 @@ -52,12 +50,8 @@ read_configuration_from_tool() { fi } -# Read configuration from our configuration tool (only if REQUIRE_CONFIGURATION_TOOL -# is set to true). read_configuration_from_tool -# Run `chifra init` in the background if we want to bootstrap -# There seems to be a bug that prevents chifra init -all from running before chifra init if [ "${BOOTSTRAP_BLOOM_FILTERS:-true}" = true ] ; then if [ "${BOOTSTRAP_FULL_INDEX:-false}" = true ] ; then echo "Downloading bloom filters AND full index in the background" @@ -68,9 +62,7 @@ if [ "${BOOTSTRAP_BLOOM_FILTERS:-true}" = true ] ; then fi fi -# Run scraper if enabled if [ "${RUN_SCRAPER:-true}" = true ] ; then - # turn all TB_CHAINS_[CHAIN]_CHAINID into a list of space-separated [chain] chain names CHAINS=`env | grep TB_CHAINS_.*_CHAINID | sed -E 's/TB_CHAINS_(.*)_CHAINID.*/\1/g'` for CHAIN in ${CHAINS} do @@ -78,5 +70,4 @@ if [ "${RUN_SCRAPER:-true}" = true ] ; then done fi -# Run chifra serve chifra serve --port 0.0.0.0:${SERVE_PORT:-8080} diff --git a/build/core/test/core-test.entrypoint.sh b/build/core/test/core-test.entrypoint.sh index 82f6f61..a573da3 100644 --- a/build/core/test/core-test.entrypoint.sh +++ b/build/core/test/core-test.entrypoint.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Locations of config files CONFIG_FILE=/root/.local/share/trueblocks/trueBlocks.toml echo "Environment:" @@ -8,10 +7,6 @@ env echo "Testing..." -# Because C++ code used by chifra status always returns success, -# we will use anonymous pipe to grep stderr and check if there are -# any strings containing "error". Grep with -c option will return -# the count of matched strings, so we can use it as our exit code ERRORS=`chifra status --terse &> >(grep -ci 'error\|Quitting')` if [ "$ERRORS" -gt 0 ] then diff --git a/docker-compose.local.example b/docker-compose.local.example index 89261e7..9a3c543 100644 --- a/docker-compose.local.example +++ b/docker-compose.local.example @@ -1,30 +1,3 @@ -# ------------------------------------------------------------------------------------------- -# qblocks - fast, easily-accessible, fully-decentralized data from blockchains -# copyright (c) 2016, 2021 TrueBlocks, LLC (http://trueblocks.io) -# ------------------------------------------------------------------------------------------- -# -# This file gives an example of how one might customize this docker contain to expose its -# internal folders to the host machine. The majority of the configuration for this container -# is found in the file `docker-compose.yml`. Do not need to edit that file. Instead, customize -# this file after copying it to `docker-compose.local.yml` -# -# It is important to use the file name `docker-compose.local.yml` as it is ignored in `.gitignore`. -# -# Once your customization is complete, you may start the container with: -# -# docker compose -f docker-compose.yml -f docker-compose.local.yml up -# -# The settings in this file expose the internal docker folders externally on the host machine -# (i.e., the machine on which this docker image is running). One might want to do this in order to -# get direct access to the Unchained Index, the cache, or -- most importantly -- the exported data. -# -# Note that we customize the same services as those found in the `docker-compose.yml`. We expose the -# container's internal folders to the host machine with the `bind` directive. The following assumes the -# existance on your host machine of a folder called ~/Data/. You may adjust this location. -# -# Bind the same folders for each service (/cache and /unchained). The `source` folder is the name of the -# folder on the host machine. The `target` is the name of the folder internal to the docker image. -# services: core: volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 384345e..73a1ad2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,81 +1,29 @@ -# ------------------------------------------------------------------------------------------- -# qblocks - fast, easily-accessible, fully-decentralized data from blockchains -# copyright (c) 2016, 2021 TrueBlocks, LLC (http://trueblocks.io) -# ------------------------------------------------------------------------------------------- - -# -# This is the system-wide docker compose file. Generally speaking, you will not edit this file. -# Instead, you may customize how this image works by copying the `docker-compose.local.example` -# file to `docker-compose.local.yml`, customizing things there, and standing up the container with: -# -# docker compose -f docker-compose.yml -f docker-compose.local.yml up -# -# In this file, we create two services: core and monitors. -# - name: TrueBlocksCore services: core: - # The tag (on docker-hub) of the core image we're using. image: trueblocks/core:v0.41.0-beta - - # The location on the local disc of the core image's Dockerfile. build: ./build/core - - # An instruction to expose the container's internal :8080 port to the host machine. Note, if you - # get an error message such as "Ports are not available...bind: address already in use", - # or similar, restate this setting in the `docker-compose.local.yml` file and change the first - # of these two ports (i.e. the port on the host machine). It's best not to change the setting here. ports: - "8080:8080" - - # An instruction telling docker where to find customizations to how chifra is to be run. See the - # README.md file more information. env_file: .env - - # Define the volumes where the images cache and the Unchained Index are stored. Note that this - # should be consistent with the definitions below. You may expose these folders to the host machine - # by customizing the `docker-compose.local.yml` file. volumes: - # The name:/folder (internal to docker) where chifra stores the Unchained Index. - unchained:/unchained - # The name:/folder (internal to docker) where chifra stores its cached data. - cache:/cache monitors: - # Tells docker that the monitors service cannot run without the core image depends_on: - core - - # Instructs docker to restart the service if it stops other than if it's explicity halted restart: unless-stopped - - # The tag (on docker-hub) of the monitors docker image image: trueblocks/monitors:v0.41.0-beta - - # The location of the monitors image's Dockerfile build: ./build/monitors - - # The same environment file as above providing the same configuration for both services env_file: .env - - # A list of volumes (corresponding to the above) with two additional volumes used - # exclusively by the monitors service. volumes: - # The name:/folder (internal to docker) where chifra stores the Unchained Index (same as above). - unchained:/unchained - # The name:/folder (internal to docker) where chifra stores its cached data (same as above). - cache:/cache - # The name:/folder (internal to docker) where `chifra monitors --watch` exports its results. See the comments - # in docker-compose.local.example for information on exposing this folder externally to the host machine. - exports:/exports - # The name:/folder (internal to docker) where `chifra monitors --watch` looks for its list of addresses - # to monitor. See the comments in docker-compose.local.example for information on exposing this - # folder externally to the host machine. - monitors:/monitors -# Instructions telling docker to persist these volumes when the container is brought down. volumes: unchained: cache: diff --git a/env.example b/env.example index 6b8db37..4f7acad 100644 --- a/env.example +++ b/env.example @@ -1,56 +1,13 @@ -# -# TrueBlocks docker -- example .env file -# -# In order for this package to work, it needs a `.env` file in the local folder. This present -# example file can serve as the basis for your `.env` file. Simply copy this file to `.env` -# and things should just work. If you experience a problem, it's most likely related to the -# RPC provider. Pay careful attention to that value. - -# This first value points to your RPC provider endpoint. If you're running your node -# locally -- you should be! -- use docker's naming conventions for local hosts. If your RPC -# endpoint is remote or on a different local machine than this host, use your endpoint directly. TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:8545 - -# This variable specifies which chain `chifra` operates on in the absence of the `--chain ` option. TB_SETTINGS_DEFAULTCHAIN=mainnet - -# This variable, while optional, allows chifra to "articulate" transactions (that is, convert byte data -# into human readable text). If you try to use a command that requires this value, you will get notified. -# You may obtain this key at https://etherscan.io. It's likely you'll want to specify this value. #TB_KEYS_ETHERSCAN_APIKEY= - -# This variable is only required if you are interested in pinning the Unchained Index (and thereby -# sharing it with others). Please see the documentation for more information. #TB_SETTINGS_DEFAULTGATEWAY=https://ipfs.unchainedindex.io/ipfs/ - -# These three variables are only required if you are interested in pinning the Unchained Index (and thereby -# sharing it with others). Please see the documentation for more information. #TB_KEYS_PINATA_APIKEY= #TB_KEYS_PINATA_JWT= #TB_KEYS_PINATA_SECRET= - -# The following variables dictate how chifra operates on startup. - -# When the container starts, it starts `chifra scrape` against the given chain. Use these values (which -# must be specified per chain) to send command-line options to the scraper. See `chifra scrape --help`. -# No additional options is the default. #SCRAPER_MAINNET_ARGS= #SCRAPER_MAINNET_FILE= - -# When the container starts, it starts `chifra monitors --watch` against the given chain. Use these values (which -# must be specified per chain) to send command-line options to the scraper. See `chifra chifra monitors --help`. -# No additional options is the default. #MONITORS_WATCH_FILE= #MONITORS_WATCH_ARGS= - -# The following variables specify settings for a given chain. You may specify more than one chain. -# -# The MAINNET, SEPOLIA, and GNOSIS chains are preconfigured and will only require you to set the -# `_RPCPROVIDER` value. See the README and the documentation for more information on supporting -# multiple or custom chains. -# -# To support other chains, replace the word MAINNET in the variables (both above and below) with the -# name of your chain. (Update the `_DEFAULTCHAIN` value above if you wish.) In order to access other -# configured chains, use the `--chain ` command-line option. #TB_CHAINS_MAINNET_CHAINID=1 #TB_CHAINS_MAINNET_SYMBOL=ETH diff --git a/scripts/publish.sh b/scripts/publish.sh index 8780cd4..810aa44 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -1,12 +1,7 @@ #!/usr/bin/env bash -# This script builds the images and pushes them to docker hub - -# tagged version VERSION=v0.44.0-beta -# Add --no-cache to the build commands to create a fresh build - docker build build/core --tag trueblocks/core:$VERSION docker push trueblocks/core:$VERSION From 1fdab73aa55bd9dc13533b0af17613e2dadc06d9 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:31:44 -0500 Subject: [PATCH 04/18] Re-write README for minimal version. We will be removing a lot of stuff from here. --- README.md | 232 +++++++++--------------------------------------------- 1 file changed, 36 insertions(+), 196 deletions(-) diff --git a/README.md b/README.md index c3d4d52..ac8bc35 100644 --- a/README.md +++ b/README.md @@ -1,212 +1,52 @@ -# TrueBlocks Docker - -## Table of Contents - - [Introduction](#introduction) - - [Prerequisite](#prerequisite) - - [Quick start](#quick-start) - - [Testing](#testing) - - [Configuration](#configuration) - - [Building](#building) - - [Using the container](#using-the-container) - - [Monitoring addresses](#monitoring-addresses) - - [Contributing](#contributing) - - [List of Contributors](#contributors) - - [Contact](#contact) + +

TrueBlocks / Docker Version

-## Introduction - -The TrueBlocks docker version makes it easy to run the `trueblocks-core` in a docker container. - -TrueBlocks builds an index of 'every appearance of every address anywhere on the chain.' The index turns your node software ([Erigon](https://github.com/ledgerwatch/erigon), for example) into a true Ethereum data server. With such a server, you can build truly distributed applications (dApps) running locally on your end users' machines. dApps that are truly trustless and perfectly private. dApps that share among themselves effortlessly and whose data is uncensorable, naturally sharded, and private. - -## Prerequisite - -In order for this docker version to work, it needs an RPC endpoint. Erigon, is an excellent choice for this and provides both the archive node functionality and the `trace_` namespace TrueBlocks requires. - -There are commercially available RPC endpoints, however, we find them inadequate due to excessive cost and significantly slower speed of access. Erigon is easy to install, syncs to the tip of the chain very quickly, and is blazingly fast at serving data -- if one has an index. A great "devops" solution to running both Erigon and TrueBlocks is the [dAppNode project](https://github.com/dappnode). - -A [docker build environment](https://docs.docker.com/get-docker/) is also required to build this package. +![GitHub repo size](https://img.shields.io/github/repo-size/TrueBlocks/trueblocks-docker) +[![GitHub contributors](https://img.shields.io/github/contributors/TrueBlocks/trueblocks-docker)](https://github.com/TrueBlocks/trueblocks-docker/contributors) +[![GitHub stars](https://img.shields.io/github/stars/TrueBlocks/trueblocks-docker?style%3Dsocial)](https://github.com/TrueBlocks/trueblocks-docker/stargazers) +[![GitHub forks](https://img.shields.io/github/forks/TrueBlocks/trueblocks-docker?style=social)](https://github.com/TrueBlocks/trueblocks-docker/network/members) +[![Twitter Follow](https://img.shields.io/twitter/follow/trueblocks?style=social)](https://twitter.com/trueblocks) -## Quick start - -When properly installed, `trueblocks-docker`: - -- builds `trublocks-core`, -- runs `chifra init` to initialize the Unchained Index (this takes a while to complete), -- starts `chifra scrape` (in the background) to actively build the index against the head of the chain, -- starts `chifra serve` (in the background) to serve an API of all `chifra` commands, -- optionally, starts `chifra monitors --watch` (in the background) to monitor a collection of addresses. - -### Getting started - -Copy the `env.example` file to `.env` and modify values (see the comments in the `env.example` for more details), or just do: - -```bash -echo "TB_CHAINS_MAINNET_RPCPROVIDER=http://yourRpcEndpoint:port" >.env -docker compose up -``` +**Table of Contents** -Make sure to provide a valid RPC endpoint that exposes both an archive node and the `trace_` namespace. If you experience problems, you -may find useful answers in the comments of the `docker-compose.yml` and `docker-compose.local.example` files. - -### Running against other chains - -If you want to run against other EVM-compatible chains, edit a file in the local folder called `.env` (or copy the `env.example` file first) and add these items: - -```bash -TB_SETTINGS_DEFAULTCHAIN="" -TB_CHAINS_``_CHAINID="" -TB_CHAINS_``_RPCPROVIDER="" -TB_CHAINS_``_SYMBOL="" -``` - -Of course, replace `` with (what else?) your chain's name. - -TODO: This is confusing - -Note: `MAINNET`, `SEPOLIA` and `GNOSIS` chains are pre-configured and you may use these values and the first two settings alone without further adue. - -After configuring for your custom chain, run `docker compose up`. - -## Interacting with the container - -Once the container is running, you may interact with `chifra` directly from the command line by calling: - -```bash -scripts/chifra.sh -``` +- [Introduction](#introduction) +- [Configuration](#configuration) +- [Running the container](#running-the-container) +- [Other](#other) -(This should produce `chifra`'s help screen.) - -Alternatively, you may use the API server, thus: - -```bash -curl "localhost:8080/when?blocks=london" -``` +## Introduction -## Testing +TrueBlocks docker provides a docker version of trueblocks-core. This container is intentionally very minimal. -To test the image, run `test.sh` script. This script builds a container and tries to call `chifra status --terse` checking for any errors and returning the right error code (`0` when no errors, error count otherwise). +Please see [the core repo for information](https://github.com/TrueBlocks/trueblocks-core) about TrueBlocks. -For testing purposes a different entrypoint is used: `build/core/test/core-test.entrypoint.sh`. +This software is pre-alpha. Use at your own risk. ## Configuration -You may configure both the way the docker image is built and the way `chifra` operates. Doing so is explained in its own page: - -- [Configuring the build and/or chifra](CONFIGURE.md) - -## Building - ---- -1. Copy the `env.example` file to `.env` and modify values (see the comments in the `env.example` for more details), or just do: - -```bash -echo "TB_CHAINS_MAINNET_RPCPROVIDER=http://yourRpcEndpoint:port" > .env -# Make sure to provide a valid RPC endpoint that exposes both an archive node and the trace_ namespace -``` - -2. Build the docker image (example tagged with `latest`) - -## Using the container - -### Calling the `chifra` command line +TODO: Edit `.env` -You may use `scripts/chifra.sh` to call `chifra` commands inside running core container: +TODO: `docker-compose.local.yml`. -```bash -# Getting the list of available chifra commands -scripts/chifra.sh +## Running the container -# Show the latest block -scripts/chifra.sh when latest +Assuming you've completed the above instructions, start the container with this command from this folder: -# Export JSON data for every 100th block between blocks 0 and 10,000 -scripts/chifra.sh blocks 0-10000:100 - -# Show all the transactions for a given address (note: you must have initialized the Unchained Index for this work) -scripts/chifra.sh export trueblocks.eth -``` - -The `scripts/chifra.sh` script calls `docker compose exec` internally, so the above commands are equivalent to: - -```bash -docker compose exec core bash -c "chifra" -docker compose exec core bash -c "chifra when latest" -docker compose exec core bash -c "chifra blocks 0-10000:100" -docker compose exec core bash -c "chifra export trueblocks.eth" +```[bash] +docker compose -f docker.compose.yml -f docker.compose.local.yml up ``` -### Connecting into the chifra API server - -By default, the `core` container exposes an API server on port `8080` serving exactly the same routes and options as the `chifra` command line does sub-commands and options. Access the API server with: - -```bash -curl -s "http://localhost:8080/when?blocks=latest" -curl -s "http://localhost:8080/blocks?blocks=0-10000:100" -curl -s "http://localhost:8080/export?addrs=trueblocks.eth" -``` +## Other -## Monitoring addresses - -TrueBlocks (via `chira`) allows you to "monitor" a collection or set of addresses. This section describes how to do that: - -1. Create a new folder on your host machine's file system. For example, - - ```bash - mkdir ~/Data/monitors/ - ``` - -3. Run the container - - ```bash - # By default, both scraper and chifra serve (API server) are started - docker run \ - --name trueblocks-core \ - --env-file ./.env \ - --publish 8080:8080 \ - -v ~/REPLACE/WITH/PATH/TO/CACHE:/cache \ - -v ~/REPLACE/WITH/PATH/TO/INDEX:/index \ - --rm \ - trueblocks-core:latest - - # Try to connect to the container - curl localhost:8080/status - ``` - -3. Edit `docker-compose.local.yml` (create it by copying from `docker-compose.local.example` if need be). Specify the path you created above to instruct docker where to pick up the list of monitored addresses and where to drop the results. - - -```yaml - monitors: - volumes: - # unchanged - - type: bind - source: ~/Data/cache - target: /cache - # unchanged - - type: bind - source: ~/Data/unchained - target: /unchained - # HERE - - type: bind - source: ~/Data/monitors - target: /monitors - # HERE - - type: bind - source: ~/Data/monitors/export - target: /export -``` +**Documentation** -4. Restart (or run for the first time) the container with `docker compose restart` or `scripts/up.sh`. You should see message in the logs, thus: +See the TrueBlocks website for the [most recent documentation](https://trueblocks.io/docs/). - ``` - trueblockscore-monitors-1 | Addresses file found, linking it - ``` +**License** -and also the results of the monitoring in the same folder. The monitor service is now watching your addresses. +This software is licensed under [GNU Version 3](https://github.com/TrueBlocks/trueblocks-docker/blob/master/LICENSE). -## Contributing +**Contributing** We love contributors. Please see information about our [work flow](https://github.com/TrueBlocks/trueblocks-core/blob/develop/docs/BRANCHING.md) before proceeding. @@ -216,15 +56,15 @@ We love contributors. Please see information about our [work flow](https://githu 4. Push back to the original branch: `git push origin TrueBlocks/trueblocks-core` 5. Create the pull request. -## List of Contributors +**Contact** -Thanks to the following people who have contributed to this project: +If you have questions, comments, or complaints, please join the discussion on our discord server which is [linked from our website](https://trueblocks.io). -* [@tjayrush](https://github.com/tjayrush) -* [@dszlachta](https://github.com/dszlachta) -* [@MysticRyuujin](https://github.com/MysticRyuujin) -* [@wildmolasses](https://github.com/wildmolasses) +**List of Contributors** -## Contact +Thanks to the following people who have contributed to this project: -If you have questions, comments, or complaints, please join the discussion on our discord server which is [linked from our website](https://trueblocks.io). +- [@tjayrush](https://github.com/tjayrush) +- [@dszlachta](https://github.com/dszlachta) +- [@wildmolasses](https://github.com/wildmolasses) +- [@MysticRyuujin](https://github.com/MysticRyuujin) From ad1148ac0e450023cde7035cfb6e26dc02f8a494 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:33:29 -0500 Subject: [PATCH 05/18] Removes testing. We will put it back if we need it. --- build/core/test/core-test.entrypoint.sh | 18 ------------------ scripts/test.sh | 17 ----------------- 2 files changed, 35 deletions(-) delete mode 100644 build/core/test/core-test.entrypoint.sh delete mode 100755 scripts/test.sh diff --git a/build/core/test/core-test.entrypoint.sh b/build/core/test/core-test.entrypoint.sh deleted file mode 100644 index a573da3..0000000 --- a/build/core/test/core-test.entrypoint.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -CONFIG_FILE=/root/.local/share/trueblocks/trueBlocks.toml - -echo "Environment:" -env - -echo "Testing..." - -ERRORS=`chifra status --terse &> >(grep -ci 'error\|Quitting')` -if [ "$ERRORS" -gt 0 ] -then - echo "Errors detected" -else - echo "Done" -fi - -exit $ERRORS diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index 700d335..0000000 --- a/scripts/test.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -ENV_FILE=$1 - -if [ "$ENV_FILE" = "" ] -then - echo "This script requires env file" - exit 1 -fi - -IMAGE=`docker build -q ./build/core` -docker run -it --rm --env-file $ENV_FILE --entrypoint /root/core-test.entrypoint.sh $IMAGE -RESULT=$? - -docker image rm $IMAGE - -exit $RESULT \ No newline at end of file From 29c5d8062e0076291c0f90df669f2897b4710b57 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:37:45 -0500 Subject: [PATCH 06/18] Removes comments --- build/monitors/Dockerfile | 2 -- build/monitors/test/monitors-test.entrypoint.sh | 4 ---- scripts/chifra.sh | 2 -- scripts/enter.sh | 2 -- scripts/publish.sh | 3 --- scripts/up.sh | 4 +--- 6 files changed, 1 insertion(+), 16 deletions(-) diff --git a/build/monitors/Dockerfile b/build/monitors/Dockerfile index 08e9db1..48e7b53 100644 --- a/build/monitors/Dockerfile +++ b/build/monitors/Dockerfile @@ -2,11 +2,9 @@ FROM trueblocks/core:v0.41.0-beta RUN apk --no-cache add jq -# Testing COPY test/monitors-test.entrypoint.sh /root RUN chmod +x /root/monitors-test.entrypoint.sh COPY monitors.entrypoint.sh /root -# Run entrypoint script ENTRYPOINT bash /root/monitors.entrypoint.sh diff --git a/build/monitors/test/monitors-test.entrypoint.sh b/build/monitors/test/monitors-test.entrypoint.sh index 12eb8b7..913f028 100644 --- a/build/monitors/test/monitors-test.entrypoint.sh +++ b/build/monitors/test/monitors-test.entrypoint.sh @@ -5,10 +5,6 @@ env echo "Testing..." -# Because C++ code used by chifra status always returns success, -# we will use anonymous pipe to grep stderr and check if there are -# any strings containing "error". Grep with -c option will return -# the count of matched strings, so we can use it as our exit code ERRORS=`chifra status --terse &> >(grep -ci 'error\|Quitting')` if [ "$ERRORS" -gt 0 ] then diff --git a/scripts/chifra.sh b/scripts/chifra.sh index 12b4f27..03d713a 100755 --- a/scripts/chifra.sh +++ b/scripts/chifra.sh @@ -1,6 +1,4 @@ #!/bin/bash -# Assuming the docker image is running, call into chifra running in the docker image. Results returned to screen. - ARGS="$@" docker compose exec core bash -c "chifra $ARGS" \ No newline at end of file diff --git a/scripts/enter.sh b/scripts/enter.sh index fc012d0..a979f9a 100755 --- a/scripts/enter.sh +++ b/scripts/enter.sh @@ -1,5 +1,3 @@ #!/bin/bash -# Enter the shell of the docker image - docker compose exec core bin/sh diff --git a/scripts/publish.sh b/scripts/publish.sh index 810aa44..d1cd147 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -4,6 +4,3 @@ VERSION=v0.44.0-beta docker build build/core --tag trueblocks/core:$VERSION docker push trueblocks/core:$VERSION - -# docker build build/monitors --tag trueblocks/monitor:$VERSION -# docker push trueblocks/monitor:$VERSION diff --git a/scripts/up.sh b/scripts/up.sh index f9f9ae3..44ce43f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,5 +1,3 @@ #!/bin/bash -# Starts the containers defined in docker-compose.yml and docker-compose.local.yml - -docker compose -f docker-compose.yml -f docker-compose.local.yml up \ No newline at end of file +docker compose -f docker-compose.yml -f docker-compose.local.yml up From 80ddf1e911226c40eba2db440bbd4baa4f1ad81d Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:40:05 -0500 Subject: [PATCH 07/18] Removes monitors from the repo. --- build/monitors/Dockerfile | 10 --- build/monitors/monitors.entrypoint.sh | 62 ------------------- .../monitors/test/monitors-test.entrypoint.sh | 16 ----- docker-compose.yml | 16 ----- 4 files changed, 104 deletions(-) delete mode 100644 build/monitors/Dockerfile delete mode 100644 build/monitors/monitors.entrypoint.sh delete mode 100644 build/monitors/test/monitors-test.entrypoint.sh diff --git a/build/monitors/Dockerfile b/build/monitors/Dockerfile deleted file mode 100644 index 48e7b53..0000000 --- a/build/monitors/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM trueblocks/core:v0.41.0-beta - -RUN apk --no-cache add jq - -COPY test/monitors-test.entrypoint.sh /root -RUN chmod +x /root/monitors-test.entrypoint.sh - -COPY monitors.entrypoint.sh /root - -ENTRYPOINT bash /root/monitors.entrypoint.sh diff --git a/build/monitors/monitors.entrypoint.sh b/build/monitors/monitors.entrypoint.sh deleted file mode 100644 index ed9e795..0000000 --- a/build/monitors/monitors.entrypoint.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -read_configuration_from_tool() { - config_file=/configuration/configuration.sh - - if [ "${REQUIRE_CONFIGURATION_TOOL}" != true ] - then - return - fi - - while [ ! -f $config_file ] - do - echo "No configuration found. Please use TrueBlocks Configuration Tool first." - echo "If you are using DAppNode, click Info tab above, then Settings link." - echo "Will try to re-read the configuration in a few seconds" - sleep 5 - done - - # Source env file: https://zwbetz.com/set-environment-variables-in-your-bash-shell-from-a-env-file/ - . $config_file - if [ $? -lt 1 ] - then - echo "Configuration has been read successfully" - else - echo "Error while reading configuration" - fi -} - -# Read configuration from our configuration tool (only if REQUIRE_CONFIGURATION_TOOL -# is set to true). -read_configuration_from_tool - -echo "${MONITORS_WATCH_FILE}" > /tmp/monitors - -ADDRESSES_TARGET=/addresses/addresses.tsv -ADDRESSES_LINK=/addresses.tsv - -# Create link to addresses file -if [ -f "$ADDRESSES_TARGET" ] -then - echo "Addresses file found, linking it" - ln -s "$ADDRESSES_TARGET" "$ADDRESSES_LINK" -fi - -CORE_HOST="${CORE_URL:-http://core}:${CORE_PORT:-8080}" - -echo "Will try to reach ${CORE_HOST}" - -while : -do - STATUS=`curl --silent --show-error ${CORE_HOST}/status | jq ".data[].isScraping"` - if [ "$STATUS" == "true" ] - then - echo "Scraper is running, continuing..." - break - fi - - echo "Scraper is not running, waiting..." - sleep 5 # second -done - -chifra monitors --watch $MONITORS_WATCH_ARGS --file /tmp/monitors diff --git a/build/monitors/test/monitors-test.entrypoint.sh b/build/monitors/test/monitors-test.entrypoint.sh deleted file mode 100644 index 913f028..0000000 --- a/build/monitors/test/monitors-test.entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -echo "Environment:" -env - -echo "Testing..." - -ERRORS=`chifra status --terse &> >(grep -ci 'error\|Quitting')` -if [ "$ERRORS" -gt 0 ] -then - echo "Errors detected" -else - echo "Done" -fi - -exit $ERRORS diff --git a/docker-compose.yml b/docker-compose.yml index 73a1ad2..b5f18c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,22 +10,6 @@ services: volumes: - unchained:/unchained - cache:/cache - - monitors: - depends_on: - - core - restart: unless-stopped - image: trueblocks/monitors:v0.41.0-beta - build: ./build/monitors - env_file: .env - volumes: - - unchained:/unchained - - cache:/cache - - exports:/exports - - monitors:/monitors - volumes: unchained: cache: - exports: - monitors: From 44e5ca032bbb74683d5f3491d7a3b8d6ed1b1257 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:41:05 -0500 Subject: [PATCH 08/18] Removes unneeded docuements on building and configuring --- BUILDING.md | 68 ---------------------------------------------------- CONFIGURE.md | 65 ------------------------------------------------- 2 files changed, 133 deletions(-) delete mode 100644 BUILDING.md delete mode 100644 CONFIGURE.md diff --git a/BUILDING.md b/BUILDING.md deleted file mode 100644 index c850e9e..0000000 --- a/BUILDING.md +++ /dev/null @@ -1,68 +0,0 @@ -# Building the images - -There are two ways to build the docker version of the TrueBlocks package. The first (building with docker compose) is preferred. - -## Building with docker compose - -The preferred way of building and running TrueBlocks is by running - -``` -docker compose -``` - -Details can be found in [Configuration section](CONFIGURE.md). However, you may wish to expose the Unchained Index on the host machine. If you wish to do that, you must customize the docker-compose build slightly. - -1. Create a file called `.env` in project's root directory (see [Configuration](#CONFIGURE.md)) by copying and editing `env.example`. -2. Create a local configuration copying the example: `cp docker-compose.local.example docker-compose.local.yml`. -3. Edit `docker-compose.local.yml` so that each volume's source points to an external location on your hard drive. The docker container will store the results of its processing there. -4. Run the project by calling `scripts/up.sh` (this simply calls: `docker compose -f docker-compose.yml -f docker-compose.local.yml up`) - -If you don't need the Unchained Index or the results of monitoring address on your host machine, you may skip this step and simply run `docker compose up` as described above. - -Both the `.env` and `docker-compose.local.yml` are ignored by Git so as to protect your privacy. - -## Building with docker directly - -1. Build the core docker image directly (for example, against `latest`) with: - - ```bash - docker build ./build/core --tag=trueblocks-core:latest - ``` - -2. You may run the newly built image with: - - ```bash - # By default, both scraper and chifra serve (API server) are started - docker run \ - --name trueblocks-core \ - --env-file ./.env \ - --publish 8080:8080 \ - --mount type=bind,source=~/Data/cache,target=/cache \ - --mount type=bind,source=~/Data/unchained,target=/unchained \ - trueblocks-core:latest -``` - -Note that the above assumes the existence of a folder called `~/Data/`. Create it if it doesn't exist. - -Now, you can try to connect to the container: - -``` - curl localhost:8080/status - ``` - -3. Build and run the monitors (this is an optional step): - - ```bash - docker build ./build/monitors --tag=trueblocks-monitor:latest - - # Note: monitor has to use the same cache and unchained volumes as core as above - docker run \ - --name trueblocks-monitor \ - --env-file ./.env \ - --publish 8080:8080 \ - --mount type=bind,source=~/Data/cache,target=/cache \ - --mount type=bind,source=~/Data/unchained,target=/unchained \ - --mount type=bind,source=~/Data/monitors/exports,target=/exports \ - --mount type=bind,source=~/Data/monitors,target=/monitors \ - trueblocks-monitor:latest - ``` diff --git a/CONFIGURE.md b/CONFIGURE.md deleted file mode 100644 index 547992c..0000000 --- a/CONFIGURE.md +++ /dev/null @@ -1,65 +0,0 @@ -# Configuring this docker image - -It's possible to config both the build process to create this docker image as well as how the TrueBlocks executable (called `chifra`) works internally. Both of these possibilities are described below. - -## Configuring the build - -There are two build arguments you may configure before building the image: - -1. `UPSTREAM_VER`: source code branch to build `chifra` from (default: `master`) -2. `SERVE_PORT`: port that `chifra serve` should bind to (default: `8080`) - -You may invoke the build process with `docker build [...] --build-arg X=Y` to modify these settings. (See [Docker documentation](https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg)). - -**A quick note on SERVE_PORT:** Distinguish the internal port used by chifra inside the running container from the port exposed to the host machine. See the comments in the `docker-compose.yml` file information on exposing an external port in the event that the port is already in use on the host machine. The setting here is for the port internal to the docker image. - -## Configuring `chifra` - -In addition to customizing the way the image is built, you may customize the way `chifra` works internally to the image. - -`chifra` supports multiple configuration items. Store these customizations in a local file to this folder called `.env`. Copy the example file (`env.example`) to `.env` and adjust values there. Using the name `.env` will ensure any private information in this file won't be pushed to GitHub, as the file is ignored by the `.gitignore` file. - -### Primary settings - -Each of these values defaults to settings for mainnet, if you have a locally running RPC provider at :8545, this should just work. It's likely, though, -that you will have to specify at least the RPC Provider setting. The other values should work without modification. - -| Item | Default | Description | -| -------------------------------- | -------------------------------------- | ------------------------------------------------ | -| TB_SETTINGS_DEFAULTCHAIN | `mainnet` | Chain to use if `--chain` option is not supplied | -| TB_CHAINS_MAINNET_RPCPROVIDER | `host.docker.internal:8545` | RPC provider URL | -| | | | -| TB_CHAINS_MAINNET_CHAINID | `1` | Chain ID (for a chain called `mainnet`) | -| TB_CHAINS_MAINNET_SYMBOL | `ETH` | Token symbol for a chain called `mainnet` | -| TB_CHAINS_MAINNET_PINGATEWAY | `https://ipfs.unchainedindex.io/ipfs/` | Unchained Index pin gateway | -| TB_CHAINS_MAINNET_REMOTEEXPLORER | `https://etherscan.io` | Remote explorer URL | -| TB_CHAINS_MAINNET_LOCALEXPLORER | `http://localhost:1234` | URL of the local explorer (TrueBlocks Explorer) | - -The words `MAINNET` and `mainnet` above may be replaced for different chains, however you will have to customize additional settings. SEPOLIA -and GNOSIS are actively indexed by TrueBlocks, LLC, and you may use them directly (assuming you have associated RPC endpoints). - -You may adjust these values for your chain. SEPOLIA and GNOSIS are being actively indexed by us, and you can simply use them (assuming you have an associated RPC endpoint). If you wish to index you're own chain, you must provide the above values. - -We're working on a configuration tool that will make this setup easier, but in the meantime, you will find information on running against different -chains here: [NOT READY-Not ready yet](./). - -### Optional settings - -Additional things you may wish to customize, along with their default values are summarize here: - -| Item | Default | Description | Use | -| ------------------------ | ------- | ------------------------------------------------------------------------- | ------------------------ | -| RUN_SCRAPER | `true` | Whether or not to run the scraper | starts the index scraper | -| BOOTSTRAP_BLOOM_FILTERS | `true` | If `true`, the container will run `chifra init` downloading bloom filters | | -| BOOTSTRAP_FULL_INDEX | `true` | If `true`, `chifra init` will download full index | | -| | | | | -| SCRAPER_MAINNET_ARGS | *empty* | Command line arguments passed to scraper | custom options | -| SCRAPER_MAINNET_FILE | *empty* | Contents of a file with scraper arguments | | -| MONITORS_WATCH_ARGS | *empty* | Command line arguments passed to `monitors --watch` | | -| MONITORS_WATCH_FILE | *empty* | Contents of a file with `monitors --watch` arguments | | -| | | | | -| TB_KEYS_ETHERSCAN_APIKEY | *empty* | Your Etherscan API key | | - -You may configure different scrapers for different chains in a similar way. By default, the core container starts one scraper per chain -as specified with the `TB_CHAINS_[chain name]_CHAINID` values found in this file. If variables `SCRAPER_[chain name]_ARGS` and/or -`SCRAPER_[chain name]_FILE` are present, they will be used as arguments when starting `chifra scrape`. From e0d93de4fd5e07815e5f7bdf7e725d4865a9ec1d Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:43:10 -0500 Subject: [PATCH 09/18] Updated env.example, but it will be greatly simplified shortly --- env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/env.example b/env.example index 4f7acad..787d836 100644 --- a/env.example +++ b/env.example @@ -11,3 +11,6 @@ TB_SETTINGS_DEFAULTCHAIN=mainnet #MONITORS_WATCH_ARGS= #TB_CHAINS_MAINNET_CHAINID=1 #TB_CHAINS_MAINNET_SYMBOL=ETH +RUN_SCRAPER=true +BOOTSTRAP_BLOOM_FILTERS=true +BOOTSTRAP_FULL_INDEX=true From 7352e677e7f18dffa15e8fcb71b045ba5f4f0284 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 20:43:59 -0500 Subject: [PATCH 10/18] Updates for v0.44.0 --- build/core/Dockerfile | 5 +++-- docker-compose.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/core/Dockerfile b/build/core/Dockerfile index 5dff99c..040f19b 100644 --- a/build/core/Dockerfile +++ b/build/core/Dockerfile @@ -5,7 +5,7 @@ RUN apk --no-cache add g++ gcc make cmake git nano libcurl python3 python3-dev \ WORKDIR /root -ARG UPSTREAM_VER=v0.41.0-beta +ARG UPSTREAM_VER=v0.44.0-beta RUN git clone -b "${UPSTREAM_VER}" --single-branch --progress --depth 1 \ https://github.com/TrueBlocks/trueblocks-core.git && \ @@ -26,12 +26,13 @@ COPY --from=builder /root/.local/share/trueblocks /root/.local/share/trueblocks COPY core.entrypoint.sh /root COPY test/core-test.entrypoint.sh /root +RUN chmod +x /root/core.entrypoint.sh RUN chmod +x /root/core-test.entrypoint.sh ARG SERVE_PORT=8080 EXPOSE ${SERVE_PORT} ENV TB_SETTINGS_CACHEPATH=/cache -ENV TB_SETTINGS_INDEXPATH=/index +ENV TB_SETTINGS_INDEXPATH=/unchained ENTRYPOINT bash /root/core.entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index b5f18c8..4ac1942 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ name: TrueBlocksCore services: core: - image: trueblocks/core:v0.41.0-beta + image: trueblocks/core:v0.44.0-beta build: ./build/core ports: - "8080:8080" From 7e89010e4b8821eac1fe705635cac744a64ec9f7 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sat, 10 Dec 2022 23:52:26 -0500 Subject: [PATCH 11/18] First version using chifra daemon --- build/core/Dockerfile | 12 ++---- build/core/core.entrypoint.sh | 72 +---------------------------------- docker-compose.local.example | 18 +-------- env.example | 19 +++------ scripts/chifra.sh | 3 +- scripts/up.sh | 3 ++ 6 files changed, 16 insertions(+), 111 deletions(-) diff --git a/build/core/Dockerfile b/build/core/Dockerfile index 040f19b..8e533f9 100644 --- a/build/core/Dockerfile +++ b/build/core/Dockerfile @@ -1,12 +1,14 @@ FROM golang:1.18-alpine as builder RUN apk --no-cache add g++ gcc make cmake git nano libcurl python3 python3-dev \ - curl bash curl-dev linux-headers sqlite-dev + curl bash curl-dev linux-headers sqlite-dev WORKDIR /root -ARG UPSTREAM_VER=v0.44.0-beta +#ARG UPSTREAM_VER=v0.44.0-beta +ARG UPSTREAM_VER=feature/docker-version +ADD https://api.github.com/repos/TrueBlocks/trueblocks-core/git/refs/heads/$UPSTREAM_VER version.json RUN git clone -b "${UPSTREAM_VER}" --single-branch --progress --depth 1 \ https://github.com/TrueBlocks/trueblocks-core.git && \ cd trueblocks-core && \ @@ -24,15 +26,9 @@ COPY --from=builder /root/.local/bin/chifra /root/.local/bin/chifra COPY --from=builder /root/.local/share/trueblocks /root/.local/share/trueblocks COPY core.entrypoint.sh /root -COPY test/core-test.entrypoint.sh /root - RUN chmod +x /root/core.entrypoint.sh -RUN chmod +x /root/core-test.entrypoint.sh ARG SERVE_PORT=8080 EXPOSE ${SERVE_PORT} -ENV TB_SETTINGS_CACHEPATH=/cache -ENV TB_SETTINGS_INDEXPATH=/unchained - ENTRYPOINT bash /root/core.entrypoint.sh diff --git a/build/core/core.entrypoint.sh b/build/core/core.entrypoint.sh index 5f10165..966ce50 100644 --- a/build/core/core.entrypoint.sh +++ b/build/core/core.entrypoint.sh @@ -1,73 +1,3 @@ #!/bin/bash -index_chain() { - CHAIN_NAME=`echo $1 | tr '[:upper:]' '[:lower:]'` - CHAIN_VAR=$1 - - args_varname="SCRAPER_${CHAIN_VAR}_ARGS" - file_varname="SCRAPER_${CHAIN_VAR}_FILE" - - ARGS="${!args_varname}" - FILE="${!file_varname}" - LOCALFILE="/tmp/scraper-$RANDOM" - - echo "Starting index scraper for chain $CHAIN_NAME" - echo " With arguments: '$ARGS'" - if [ -z "$FILE" ] - then - echo " And no command file" - else - echo " And command file saved as $LOCALFILE" - fi - - echo "${FILE}" > $LOCALFILE - chifra scrape --chain $CHAIN_NAME $ARGS --file $LOCALFILE & -} - -read_configuration_from_tool() { - config_file=/configuration/configuration.sh - - if [ "${REQUIRE_CONFIGURATION_TOOL}" != true ] - then - return - fi - - while [ ! -f $config_file ] - do - echo "No configuration found. Please use TrueBlocks Configuration Tool first." - echo "If you are using DAppNode, click Info tab above, then Settings link." - echo "Will try to re-read the configuration in a few seconds" - sleep 5 - done - - # Source env file: https://zwbetz.com/set-environment-variables-in-your-bash-shell-from-a-env-file/ - . $config_file - if [ $? -lt 1 ] - then - echo "Configuration has been read successfully" - else - echo "Error while reading configuration" - fi -} - -read_configuration_from_tool - -if [ "${BOOTSTRAP_BLOOM_FILTERS:-true}" = true ] ; then - if [ "${BOOTSTRAP_FULL_INDEX:-false}" = true ] ; then - echo "Downloading bloom filters AND full index in the background" - chifra init --all - else - echo "Downloading bloom filters in the background" - chifra init - fi -fi - -if [ "${RUN_SCRAPER:-true}" = true ] ; then - CHAINS=`env | grep TB_CHAINS_.*_CHAINID | sed -E 's/TB_CHAINS_(.*)_CHAINID.*/\1/g'` - for CHAIN in ${CHAINS} - do - index_chain $CHAIN - done -fi - -chifra serve --port 0.0.0.0:${SERVE_PORT:-8080} +chifra node --api on --scrape blooms --monitor diff --git a/docker-compose.local.example b/docker-compose.local.example index 9a3c543..1182d43 100644 --- a/docker-compose.local.example +++ b/docker-compose.local.example @@ -2,22 +2,8 @@ services: core: volumes: - type: bind - source: ~/Data/cache + source: /Data/cache target: /cache - type: bind - source: ~/Data/unchained + source: /Data/unchained target: /unchained - monitors: - volumes: - - type: bind - source: ~/Data/cache - target: /cache - - type: bind - source: ~/Data/unchained - target: /unchained - - type: bind - source: ~/Data/monitors - target: /monitors - - type: bind - source: ~/Data/monitors/export # notice difference in path - target: /export diff --git a/env.example b/env.example index 787d836..43a9c50 100644 --- a/env.example +++ b/env.example @@ -1,16 +1,7 @@ TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:8545 +TB_CHAINS_MAINNET_CHAINID=1 +TB_CHAINS_MAINNET_SYMBOL=ETH + TB_SETTINGS_DEFAULTCHAIN=mainnet -#TB_KEYS_ETHERSCAN_APIKEY= -#TB_SETTINGS_DEFAULTGATEWAY=https://ipfs.unchainedindex.io/ipfs/ -#TB_KEYS_PINATA_APIKEY= -#TB_KEYS_PINATA_JWT= -#TB_KEYS_PINATA_SECRET= -#SCRAPER_MAINNET_ARGS= -#SCRAPER_MAINNET_FILE= -#MONITORS_WATCH_FILE= -#MONITORS_WATCH_ARGS= -#TB_CHAINS_MAINNET_CHAINID=1 -#TB_CHAINS_MAINNET_SYMBOL=ETH -RUN_SCRAPER=true -BOOTSTRAP_BLOOM_FILTERS=true -BOOTSTRAP_FULL_INDEX=true +TB_SETTINGS_CACHEPATH=/cache +TB_SETTINGS_INDEXPATH=/unchained diff --git a/scripts/chifra.sh b/scripts/chifra.sh index 03d713a..d15ce0e 100755 --- a/scripts/chifra.sh +++ b/scripts/chifra.sh @@ -1,4 +1,3 @@ #!/bin/bash -ARGS="$@" -docker compose exec core bash -c "chifra $ARGS" \ No newline at end of file +docker compose exec core bash -c "chifra $@" \ No newline at end of file diff --git a/scripts/up.sh b/scripts/up.sh index 44ce43f..01cb367 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,3 +1,6 @@ #!/bin/bash +VERSION=v0.44.0-beta + +docker build build/core --tag trueblocks/core:$VERSION docker compose -f docker-compose.yml -f docker-compose.local.yml up From c35bedb3439e01457b3d69602212e097679001b2 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sun, 11 Dec 2022 10:49:09 -0500 Subject: [PATCH 12/18] Catching up --- .gitignore | 2 +- build/core/core.entrypoint.sh | 2 +- scripts/build.sh | 4 ++++ scripts/chifra.sh | 3 ++- scripts/config.sh | 3 +++ scripts/up.sh | 3 --- 6 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 scripts/build.sh create mode 100755 scripts/config.sh diff --git a/.gitignore b/.gitignore index 5020944..dda6b81 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ .env .save_env docker-compose.local.yml - +publish.sh \ No newline at end of file diff --git a/build/core/core.entrypoint.sh b/build/core/core.entrypoint.sh index 966ce50..5898e3a 100644 --- a/build/core/core.entrypoint.sh +++ b/build/core/core.entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/bash -chifra node --api on --scrape blooms --monitor +chifra daemon --api on --scrape blooms --monitor diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..870307e --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +VERSION=v0.44.0-beta +docker build build/core --tag trueblocks/core:$VERSION diff --git a/scripts/chifra.sh b/scripts/chifra.sh index d15ce0e..4285785 100755 --- a/scripts/chifra.sh +++ b/scripts/chifra.sh @@ -1,3 +1,4 @@ #!/bin/bash -docker compose exec core bash -c "chifra $@" \ No newline at end of file +ARGS="$@" +docker compose exec core bash -c "chifra $ARGS" diff --git a/scripts/config.sh b/scripts/config.sh new file mode 100755 index 0000000..d1618a3 --- /dev/null +++ b/scripts/config.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +open https://google.com diff --git a/scripts/up.sh b/scripts/up.sh index 01cb367..44ce43f 100755 --- a/scripts/up.sh +++ b/scripts/up.sh @@ -1,6 +1,3 @@ #!/bin/bash -VERSION=v0.44.0-beta - -docker build build/core --tag trueblocks/core:$VERSION docker compose -f docker-compose.yml -f docker-compose.local.yml up From dab7b8520f14f3be206f625c8e630e34b65147e8 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Sun, 11 Dec 2022 23:10:29 -0500 Subject: [PATCH 13/18] Removes unwanted build parts --- build/core/Dockerfile | 7 ++++++- docker-compose.yml | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/build/core/Dockerfile b/build/core/Dockerfile index 8e533f9..74cb937 100644 --- a/build/core/Dockerfile +++ b/build/core/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.18-alpine as builder RUN apk --no-cache add g++ gcc make cmake git nano libcurl python3 python3-dev \ - curl bash curl-dev linux-headers sqlite-dev + curl bash curl-dev linux-headers sqlite-dev sed WORKDIR /root @@ -12,6 +12,11 @@ ADD https://api.github.com/repos/TrueBlocks/trueblocks-core/git/refs/heads/$UPST RUN git clone -b "${UPSTREAM_VER}" --single-branch --progress --depth 1 \ https://github.com/TrueBlocks/trueblocks-core.git && \ cd trueblocks-core && \ + cat src/libs/CMakeLists.txt | grep -v "test-libs" >x && \ + cat x >src/libs/CMakeLists.txt && \ + cat src/CMakeLists.txt | grep -v "examples" | grep -v dev_tools >x && \ + cat x >src/CMakeLists.txt && \ + rm -f x && \ mkdir -p build && \ cd build && \ cmake ../src && \ diff --git a/docker-compose.yml b/docker-compose.yml index 4ac1942..fe392f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,9 @@ services: core: image: trueblocks/core:v0.44.0-beta build: ./build/core - ports: - - "8080:8080" env_file: .env + ports: + - "8080:${SERVE_PORT-8080}" volumes: - unchained:/unchained - cache:/cache From 019cef1f2c0de4c69e7662c89b2daf812c8e42ab Mon Sep 17 00:00:00 2001 From: tjayrush Date: Thu, 15 Dec 2022 22:06:47 -0500 Subject: [PATCH 14/18] Updates README --- README.md | 93 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ac8bc35..f459d7f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ - +

TrueBlocks / Docker Version

![GitHub repo size](https://img.shields.io/github/repo-size/TrueBlocks/trueblocks-docker) @@ -11,31 +11,104 @@ - [Introduction](#introduction) - [Configuration](#configuration) -- [Running the container](#running-the-container) +- [Running the tool](#running-the-tool) +- [The Unchained Index](#the-unchained-index) - [Other](#other) ## Introduction -TrueBlocks docker provides a docker version of trueblocks-core. This container is intentionally very minimal. +This docker version of TrueBlocks is intentionally minimal. Please see [the core repo for information](https://github.com/TrueBlocks/trueblocks-core) about TrueBlocks, the Unchained Index, our command-line tool (`chifra`) and our API. The software is pre-alpha. Use at your own risk. -Please see [the core repo for information](https://github.com/TrueBlocks/trueblocks-core) about TrueBlocks. +## Configuration -This software is pre-alpha. Use at your own risk. +---- +TODO: COMPLETE THIS SECTION -## Configuration -TODO: Edit `.env` +Add these items to `.env`. Adjust the RPC provider to point to the proper port (if you're node is running locally) or the URL if the node is remote. The EtherScan key is optional, but useful. + +```[shell] +TB_SETTINGS_DEFAULTCHAIN=mainnet +TB_SETTINGS_CACHEPATH=/cache +TB_SETTINGS_INDEXPATH=/unchained +TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:23456 +TB_KEYS_ETHERSCAN_APIKEY= +``` + +---- +TODO: COMPLETE THIS SECTION + + +By default, the docker stores the Unchained Index internally to the container. If you wish to store the Unchained Index on your host machine as well, do this: + +Create two folders on your host machine. We will attach these folders to the container's `CACHEPATH` and `INDEXPATH` in a moment. -TODO: `docker-compose.local.yml`. +On our machines, we use: -## Running the container +```[shell] +mkdir -p /Users/user/Data/cache +mkdir -p /Users/user/Data/unchained +``` + +**Note:** Adjust these values appropriately for your machine. + +Next, edit a file in the current folder called `docker-compose.local.yml`. Add the following code. (Adjust the paths.) + +```[shell] +services: + core: + volumes: + - type: bind + source: /Users/user/Data/docker/cache + target: /cache + - type: bind + source: /Users/user/Data/docker/unchained + target: /unchained +``` + +Notice that the above process attaches the internal-to-docker `target` folders to the external-on-the-host `source` folders. This allows the files created by the Unchained Index to be visible outside of the docker container. +## Running the tool -Assuming you've completed the above instructions, start the container with this command from this folder: +Assuming you've completed the above instructions, open a new terminal window and start the container with this command: ```[bash] docker compose -f docker.compose.yml -f docker.compose.local.yml up ``` +The above will start the [TrueBlocks API server](https://trueblocks.io/api/). You may now use `curl` to access the server through [http://localhost:8008](http://localhost:8008). For example, + +```[shell] +curl "http://localhost:8080/blocks?blocks=1-1000:10" +``` + +will extract every 10th block between blocks 1 and 1,000. You may also use the `chifra` command line tool directly. From the current folder, type: + +```[shell] +./scripts/chifra.sh blocks 1-1000:10 +``` + +This will produce the same results as the command running against the API server. + +See the [most recent documentation](https://trueblocks.io/docs/) here. + +## The Unchained Index + +In the future, this docker will initialize and maintain [the Unchained Index](https://trueblocks.io/papers/2022/file-format-spec-v0.40.0-beta.pdf). Until then, you must initialize it and maintain it manually. Do this by running this command: + +```[shell] +./scripts/chifra.sh init --all +``` + +Depending on your connection, this may take several hours. When it finishes, you may optionally choose to run the scraper which maintains the index at the front of the chain. (Note: if you're exploring older data, this step is optional.) + +To start the scraper, do this once the `chifra init` command finishes: + +```[shell] +chifra scrape +``` + +Allow this process to continue running in its own terminal window. + ## Other **Documentation** From abf103676d5beceaaebbd1a4fc9b5a5a62a0a530 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Fri, 16 Dec 2022 08:58:06 -0500 Subject: [PATCH 15/18] Final version for this release --- build/core/Dockerfile => Dockerfile | 5 +--- README.md | 46 ++++++++++++++++------------- build/core/core.entrypoint.sh | 3 -- docker-compose.local.example | 10 +++++-- docker-compose.yml | 2 +- env.example | 32 ++++++++++++++++++-- 6 files changed, 65 insertions(+), 33 deletions(-) rename build/core/Dockerfile => Dockerfile (91%) delete mode 100644 build/core/core.entrypoint.sh diff --git a/build/core/Dockerfile b/Dockerfile similarity index 91% rename from build/core/Dockerfile rename to Dockerfile index 74cb937..d2e1b88 100644 --- a/build/core/Dockerfile +++ b/Dockerfile @@ -30,10 +30,7 @@ COPY --from=builder /root/trueblocks-core/bin /usr/local/bin COPY --from=builder /root/.local/bin/chifra /root/.local/bin/chifra COPY --from=builder /root/.local/share/trueblocks /root/.local/share/trueblocks -COPY core.entrypoint.sh /root -RUN chmod +x /root/core.entrypoint.sh - ARG SERVE_PORT=8080 EXPOSE ${SERVE_PORT} -ENTRYPOINT bash /root/core.entrypoint.sh +CMD ["chifra", "daemon", "--api", "on", "--scrape", "blooms", "--monitor"] diff --git a/README.md b/README.md index f459d7f..af7862a 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ - [Configuration](#configuration) - [Running the tool](#running-the-tool) - [The Unchained Index](#the-unchained-index) +- [Data science](#data-science) - [Other](#other) ## Introduction @@ -22,28 +23,20 @@ This docker version of TrueBlocks is intentionally minimal. Please see [the core ## Configuration ---- -TODO: COMPLETE THIS SECTION - - -Add these items to `.env`. Adjust the RPC provider to point to the proper port (if you're node is running locally) or the URL if the node is remote. The EtherScan key is optional, but useful. +You must create a file called `.env` in the local folder. An `env.example` file is provided with more information. Adjust the RPC provider to point to your RPC provider (preferably a local one). The EtherScan key is optional, but useful. ```[shell] TB_SETTINGS_DEFAULTCHAIN=mainnet +TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:8545 +TB_KEYS_ETHERSCAN_APIKEY= TB_SETTINGS_CACHEPATH=/cache TB_SETTINGS_INDEXPATH=/unchained -TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:23456 -TB_KEYS_ETHERSCAN_APIKEY= ``` ---- -TODO: COMPLETE THIS SECTION - +By default, the docker stores the Unchained Index and binary cache internally. If you wish to be able to access this data on your host machine as well (you may if you want the fastest access), do this: -By default, the docker stores the Unchained Index internally to the container. If you wish to store the Unchained Index on your host machine as well, do this: - -Create two folders on your host machine. We will attach these folders to the container's `CACHEPATH` and `INDEXPATH` in a moment. - -On our machines, we use: +Create two folders on your host machine. We will use these folders in a moment. ```[shell] mkdir -p /Users/user/Data/cache @@ -52,7 +45,7 @@ mkdir -p /Users/user/Data/unchained **Note:** Adjust these values appropriately for your machine. -Next, edit a file in the current folder called `docker-compose.local.yml`. Add the following code. (Adjust the paths.) +Next, edit a file in the current folder called `docker-compose.local.yml`. See the `docker-compose.local.example` for more information. ```[shell] services: @@ -66,7 +59,10 @@ services: target: /unchained ``` -Notice that the above process attaches the internal-to-docker `target` folders to the external-on-the-host `source` folders. This allows the files created by the Unchained Index to be visible outside of the docker container. +**Note:** Adjust the `source` paths appropriately for your machine. + +Notice that the above process attaches the *internal-to-docker* `target` folders to the *external-on-the-host* `source` folders. This allows the files created by the Unchained Index to be visible on your host machine. + ## Running the tool Assuming you've completed the above instructions, open a new terminal window and start the container with this command: @@ -93,21 +89,31 @@ See the [most recent documentation](https://trueblocks.io/docs/) here. ## The Unchained Index -In the future, this docker will initialize and maintain [the Unchained Index](https://trueblocks.io/papers/2022/file-format-spec-v0.40.0-beta.pdf). Until then, you must initialize it and maintain it manually. Do this by running this command: +In the future, this docker will initialize and maintain [the Unchained Index](https://trueblocks.io/papers/2022/file-format-spec-v0.40.0-beta.pdf). Until then, you must initialize it and maintain it manually. + +Please [read and understand this discussion](https://trueblocks.io/docs/install/get-the-index/) before proceeding. It will have an important impact on how `chifra` works for you. + +Once you've undertstood the above, run one of the following two commands: ```[shell] -./scripts/chifra.sh init --all +./scripts/chifra.sh init --all # if you want to initialize the full index (recommended if you have space), or + +./scripts/chifra.sh init # if you want a minimal index and don't mind slower initial access. ``` -Depending on your connection, this may take several hours. When it finishes, you may optionally choose to run the scraper which maintains the index at the front of the chain. (Note: if you're exploring older data, this step is optional.) +Depending on your connection, this above will take either several minutes or several hours. When it finishes, you must decide if you wish to run the `scraper` which maintains the index at the front of the chain. (Note: if you're exploring older data, this step is optional.) -To start the scraper, do this once the `chifra init` command finishes: +To start the scraper, do this only after the `chifra init` command finishes: ```[shell] chifra scrape ``` -Allow this process to continue running in its own terminal window. +Allow this process to continue running in its own terminal window or `tmux` session. + +## Data science + +`chifra` is a great data science tool. See a few of our articles ([here](https://trueblocks.io/tags/community/), [here](https://trueblocks.io/tags/trueblocks/), and [here](https://trueblocks.io/tags/recipes/)) for ideas on how to take advantage of this very useful tool. ## Other diff --git a/build/core/core.entrypoint.sh b/build/core/core.entrypoint.sh deleted file mode 100644 index 5898e3a..0000000 --- a/build/core/core.entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -chifra daemon --api on --scrape blooms --monitor diff --git a/docker-compose.local.example b/docker-compose.local.example index 1182d43..2ebf3a1 100644 --- a/docker-compose.local.example +++ b/docker-compose.local.example @@ -1,9 +1,15 @@ +# There are many things you can do with a docker-compose file. Here we simple attach the internal-to-docker +# folders to the external-to-the-host folders we've described in the .env file. This is a good way to +# keep your data safe and persistent. +# +# Note that the name of the service (`core`) must match that found in `docker-compose.yml`. Also, we've +# found that the easiest way to get this to work is to make the `source` and `target` paths absolute. services: core: volumes: - type: bind - source: /Data/cache + source: /Users/user/Data/cache target: /cache - type: bind - source: /Data/unchained + source: /Users/user/unchained target: /unchained diff --git a/docker-compose.yml b/docker-compose.yml index fe392f5..8d71fe2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ name: TrueBlocksCore services: core: image: trueblocks/core:v0.44.0-beta - build: ./build/core + build: ./ env_file: .env ports: - "8080:${SERVE_PORT-8080}" diff --git a/env.example b/env.example index 43a9c50..02b49c9 100644 --- a/env.example +++ b/env.example @@ -1,7 +1,33 @@ -TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:8545 -TB_CHAINS_MAINNET_CHAINID=1 -TB_CHAINS_MAINNET_SYMBOL=ETH +# This is an example `.env` file. Copy this file and rename it if you wish. +#--------- +# To various degrees, TrueBlocks runs against any evm-based blockchain (although we only officially +# support `mainnet`, `sepolia`, and `gnosis`). Specify the default chain you wish to use. The default +# chain is the one `chifra` processes if you don't specify a `--chain` parameter. +#--------- TB_SETTINGS_DEFAULTCHAIN=mainnet + +#--------- +# For each chain you wish to support, you must instruct TrueBlocks how to connect to it. Below, we +# configure for "mainnet" and "sepolia". If you wish to support other chains, add their RPC providers +# below. Variables are named as follows: TB_CHAINS__RPCPROVIDER. The value provided should be +# the URL of the RPC provider for the chain. It's highly suggested that the URL is local. +#--------- +# Note: The following values most likely will not work on your machine. Replace these vaules with +# values that work on your machine. The values below work for us on a Mac running Docker Desktop. +TB_CHAINS_MAINNET_RPCPROVIDER=http://host.docker.internal:8545 +TB_CHAINS_SEPOLIA_RPCPROVIDER=http://host.docker.internal:8546 + +#--------- +# This value, which is optional, enables the articulation feature. Eventually, we will provide a +# way to use the articulation feature without an API key. In the meantime, you can get an API +# key from https://etherscan.io/apis. +#--------- +TB_KEYS_ETHERSCAN_APIKEY= + +#--------- +# These two paths, while optional, allow us to more easily use the data produced by TrueBlocks +# outside the docker image. We recommend you leave these settings as they are. +#--------- TB_SETTINGS_CACHEPATH=/cache TB_SETTINGS_INDEXPATH=/unchained From 9cda22d279b645883e6d9284455d30c06e8184ee Mon Sep 17 00:00:00 2001 From: tjayrush Date: Fri, 16 Dec 2022 11:29:39 -0500 Subject: [PATCH 16/18] Updates README --- README.md | 60 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index af7862a..7e7a236 100644 --- a/README.md +++ b/README.md @@ -12,18 +12,17 @@ - [Introduction](#introduction) - [Configuration](#configuration) - [Running the tool](#running-the-tool) -- [The Unchained Index](#the-unchained-index) +- [The unchained index](#the-unchained-index) - [Data science](#data-science) - [Other](#other) ## Introduction -This docker version of TrueBlocks is intentionally minimal. Please see [the core repo for information](https://github.com/TrueBlocks/trueblocks-core) about TrueBlocks, the Unchained Index, our command-line tool (`chifra`) and our API. The software is pre-alpha. Use at your own risk. +TrueBlocks docker is intentionally minimal. See [the core repo for information](https://github.com/TrueBlocks/trueblocks-core) about TrueBlocks, the Unchained Index, chifra (our command-line tool) and the core API. This repo is pre-alpha. Use at your own risk. ## Configuration ----- -You must create a file called `.env` in the local folder. An `env.example` file is provided with more information. Adjust the RPC provider to point to your RPC provider (preferably a local one). The EtherScan key is optional, but useful. +Begin by creating a file called `.env` in this folder. An [env.example](env.example) file is provided to help you. Adjust the RPC provider to point to a (preferably local) RPC endpoint. The `ETHERSCAN_APIKEY` key is optional, but useful. ```[shell] TB_SETTINGS_DEFAULTCHAIN=mainnet @@ -33,19 +32,18 @@ TB_SETTINGS_CACHEPATH=/cache TB_SETTINGS_INDEXPATH=/unchained ``` ----- -By default, the docker stores the Unchained Index and binary cache internally. If you wish to be able to access this data on your host machine as well (you may if you want the fastest access), do this: +By default, we store the Unchained Index and binary caches internally to docker. If you wish to to access this data from your host machine (you do, because you want faster access), complete these steps. (Otherwise skip to [Running the tool](https://github.com/TrueBlocks/trueblocks-docker/tree/feature/use-config-tool#running-the-tool)) -Create two folders on your host machine. We will use these folders in a moment. +Create two folders on your host machine: ```[shell] mkdir -p /Users/user/Data/cache mkdir -p /Users/user/Data/unchained ``` -**Note:** Adjust these values appropriately for your machine. +**Note:** Adjust these paths appropriately for your machine. -Next, edit a file in the current folder called `docker-compose.local.yml`. See the `docker-compose.local.example` for more information. +Next, create a file called `docker-compose.local.yml` in the current folder. See the [docker-compose.local.example](docker-compose.local.example) for more information. ```[shell] services: @@ -61,47 +59,59 @@ services: **Note:** Adjust the `source` paths appropriately for your machine. -Notice that the above process attaches the *internal-to-docker* `target` folders to the *external-on-the-host* `source` folders. This allows the files created by the Unchained Index to be visible on your host machine. +The above process attaches (binds) the *internal-to-docker* `target` folders to the *external-on-the-host* `source` folders. This allows the files created internally by the docker to be visible on your host machine. ## Running the tool -Assuming you've completed the above instructions, open a new terminal window and start the container with this command: +Assuming you've completed the above instructions, start the container by running this command: ```[bash] docker compose -f docker.compose.yml -f docker.compose.local.yml up ``` -The above will start the [TrueBlocks API server](https://trueblocks.io/api/). You may now use `curl` to access the server through [http://localhost:8008](http://localhost:8008). For example, +This will start the [TrueBlocks API server](https://trueblocks.io/api/). Leave this process running and open a new terminal window. + +Use `curl` to access the API through [http://localhost:8080](http://localhost:8080). For example, the command ```[shell] curl "http://localhost:8080/blocks?blocks=1-1000:10" ``` -will extract every 10th block between blocks 1 and 1,000. You may also use the `chifra` command line tool directly. From the current folder, type: +will extract every 10th block between blocks 1 and 1,000. + +You may also use the `chifra` command line tool directly. From the current folder, type: ```[shell] ./scripts/chifra.sh blocks 1-1000:10 ``` -This will produce the same results as the command running against the API server. +This will produce the same results as the `curl` command. -See the [most recent documentation](https://trueblocks.io/docs/) here. +```[shell] +./scripts/chifra.sh --help +``` -## The Unchained Index +will show all available chifra tools. See the [full documentation](https://trueblocks.io/docs/) for detailed information. -In the future, this docker will initialize and maintain [the Unchained Index](https://trueblocks.io/papers/2022/file-format-spec-v0.40.0-beta.pdf). Until then, you must initialize it and maintain it manually. +## The unchained index -Please [read and understand this discussion](https://trueblocks.io/docs/install/get-the-index/) before proceeding. It will have an important impact on how `chifra` works for you. +In the future, this docker will initialize and maintain [the Unchained Index](https://trueblocks.io/papers/2022/file-format-spec-v0.40.0-beta.pdf). Until then, you must initialize it and maintain it yourself. -Once you've undertstood the above, run one of the following two commands: +Before doing that, please [read and understand this discussion](https://trueblocks.io/docs/install/get-the-index/). It will have an important impact on how `chifra` works for you. + +Once you've read the above, run one of the following two commands: ```[shell] -./scripts/chifra.sh init --all # if you want to initialize the full index (recommended if you have space), or +# If you want to initialize the full index (recommended if you have space), or +./scripts/chifra.sh init --all -./scripts/chifra.sh init # if you want a minimal index and don't mind slower initial access. +# If you want a minimal index and don't mind slower initial queries +./scripts/chifra.sh init ``` -Depending on your connection, this above will take either several minutes or several hours. When it finishes, you must decide if you wish to run the `scraper` which maintains the index at the front of the chain. (Note: if you're exploring older data, this step is optional.) +Depending on your connection, this above will take several minutes or as much as several hours. + +When the initialization finishes, decide if you want to run the `scraper`. The scrape maintains the index to the front of the chain. (Note: if you're exploring older data, this step is optional.) To start the scraper, do this only after the `chifra init` command finishes: @@ -109,11 +119,13 @@ To start the scraper, do this only after the `chifra init` command finishes: chifra scrape ``` -Allow this process to continue running in its own terminal window or `tmux` session. +Allow this process to continue running in its own terminal window or `tmux` session. If you stop it, the next time you run `chifra`, you will have to re-run the scraper to catch up to the chain. ## Data science -`chifra` is a great data science tool. See a few of our articles ([here](https://trueblocks.io/tags/community/), [here](https://trueblocks.io/tags/trueblocks/), and [here](https://trueblocks.io/tags/recipes/)) for ideas on how to take advantage of this very useful tool. +`chifra` is an excellent data science tool. See a few of our articles ([here](https://trueblocks.io/tags/community/), [here](https://trueblocks.io/tags/trueblocks/), and [here](https://trueblocks.io/tags/recipes/)) for ideas on how to take advantage of this very useful tool. + +TODO: Add tutorials. ## Other From 43e1c396c0c618cf8ab098d5b59a288461262fe7 Mon Sep 17 00:00:00 2001 From: tjayrush Date: Fri, 16 Dec 2022 11:33:00 -0500 Subject: [PATCH 17/18] Updates README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7e7a236..d552184 100644 --- a/README.md +++ b/README.md @@ -107,16 +107,18 @@ Once you've read the above, run one of the following two commands: # If you want a minimal index and don't mind slower initial queries ./scripts/chifra.sh init + +# Do not run both commands, chose one or the other ``` -Depending on your connection, this above will take several minutes or as much as several hours. +Depending on your connection, the above will take several minutes or as much as several hours. When the initialization finishes, decide if you want to run the `scraper`. The scrape maintains the index to the front of the chain. (Note: if you're exploring older data, this step is optional.) To start the scraper, do this only after the `chifra init` command finishes: ```[shell] -chifra scrape +./scripts/chifra.sh scrape ``` Allow this process to continue running in its own terminal window or `tmux` session. If you stop it, the next time you run `chifra`, you will have to re-run the scraper to catch up to the chain. From 6f94ddcd8b8cc2ddae8c29c9b9bec26576644afe Mon Sep 17 00:00:00 2001 From: tjayrush Date: Fri, 16 Dec 2022 16:41:25 -0500 Subject: [PATCH 18/18] Cleans up --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d552184..5153f5c 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,15 @@ ## Introduction -TrueBlocks docker is intentionally minimal. See [the core repo for information](https://github.com/TrueBlocks/trueblocks-core) about TrueBlocks, the Unchained Index, chifra (our command-line tool) and the core API. This repo is pre-alpha. Use at your own risk. +TrueBlocks is a local-first indexing / data access solution that you may use for data science or as a locally-running backend for your Web 3.0 projects. + +This docker repo is intentionally minimal. See [the core repo](https://github.com/TrueBlocks/trueblocks-core) for more information about the Unchained Index, chifra (our command-line tool), and the TrueBlocks data API. + +This repo is pre-alpha and comes with no warrenty implied or otherwise. Use at your own discretion. ## Configuration -Begin by creating a file called `.env` in this folder. An [env.example](env.example) file is provided to help you. Adjust the RPC provider to point to a (preferably local) RPC endpoint. The `ETHERSCAN_APIKEY` key is optional, but useful. +To get started, create a file called `.env` in this folder. An [env.example](env.example) file explaining each setting is provided to help you. Adjust the RPC provider to point to a (preferably local) RPC endpoint. ```[shell] TB_SETTINGS_DEFAULTCHAIN=mainnet @@ -32,7 +36,9 @@ TB_SETTINGS_CACHEPATH=/cache TB_SETTINGS_INDEXPATH=/unchained ``` -By default, we store the Unchained Index and binary caches internally to docker. If you wish to to access this data from your host machine (you do, because you want faster access), complete these steps. (Otherwise skip to [Running the tool](https://github.com/TrueBlocks/trueblocks-docker/tree/feature/use-config-tool#running-the-tool)) +The `ETHERSCAN_APIKEY` key is optional, but useful to enable the articulation feature. + +By default, the system stores the Unchained Index and binary caches internally to docker. If you wish to to access this data from your host machine (you do, because you want faster access), complete these steps. (Otherwise skip to [Running the tool](https://github.com/TrueBlocks/trueblocks-docker/tree/feature/use-config-tool#running-the-tool)) Create two folders on your host machine: @@ -113,7 +119,7 @@ Once you've read the above, run one of the following two commands: Depending on your connection, the above will take several minutes or as much as several hours. -When the initialization finishes, decide if you want to run the `scraper`. The scrape maintains the index to the front of the chain. (Note: if you're exploring older data, this step is optional.) +When the initialization finishes, decide if you want to run the `scraper`. The scrape maintains the index to the front of the chain. (Note: if you're exploring older data, this step may be optional.) To start the scraper, do this only after the `chifra init` command finishes: