eth_log_ingester.org

Base Request Construct

!pip install pandas

import requests
import json
from types import SimpleNamespace
from ast import literal_eval
from datetime import datetime
import pandas as p
import os

# url = 'http://10.50.0.4:22000'
url = 'http://localhost:8545'
headers = {"Content-Type": "application/json; charset=utf-8"}

log_file.close()
log_file = open("ethereum.log", "a")

Sample - Get Peer Count

input = '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":74}'

response = requests.post(url, headers=headers, data=input)

if response.status_code == 200:
    data = response.json()
    print(data)

Sample - Get Latest Block Number

input = '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":83}'

response = requests.post(url, headers=headers, data=input)

if response.status_code == 200:
    data = response.json()
    print(data)

Sample - Get Block Data

input = '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0xADAD02", true],"id":1}'

response = requests.post(url, headers=headers, data=input)

if response.status_code == 200:
    data = SimpleNamespace(**(response.json()))
    print(json.dumps(data.result, indent=4))

Get Latest Block number

def get_latest_block_number():
    input = '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":83}'

    response = requests.post(url, headers=headers, data=input)

    if response.status_code == 200:
        data = response.json()

        return literal_eval(data['result'])

def get_block_by_number(n):

    input = json.loads('{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x4", true],"id":1}')

    input['params'] = [hex(n), True]

    response = requests.post(url, headers=headers, data=json.dumps(input))

    if response.status_code == 200:
        response_data = response.json()
        block = SimpleNamespace(** SimpleNamespace(** response_data).result)

        return block

Tranform Block data

def transform_block_data(b):
    delattr(b, 'extraData')
    delattr(b, 'logsBloom')
    delattr(b, 'mixHash')
    delattr(b, 'nonce')
    delattr(b, 'receiptsRoot')
    delattr(b, 'sha3Uncles')
    delattr(b, 'stateRoot')
    delattr(b, 'transactions')
    delattr(b, 'transactionsRoot')
    delattr(b, 'uncles')

    setattr(b, 'difficulty', literal_eval(b.difficulty))
    setattr(b, 'gasLimit', literal_eval(b.gasLimit))
    setattr(b, 'gasUsed', literal_eval(b.gasUsed))
    setattr(b, 'hash', literal_eval(b.hash))
    setattr(b, 'miner', b.miner)
    setattr(b, 'blockNumber', literal_eval(b.number))
    setattr(b, 'parentHash', b.parentHash)
    setattr(b, 'size', literal_eval(b.size))
    setattr(b, 'timestamp', pd.to_datetime(literal_eval(b.timestamp), unit='s').strftime('%Y-%m-%d %H:%M:%S'))
    setattr(b, 'totalDifficulty', literal_eval(b.totalDifficulty))

    delattr(b, 'number')

    return b

Test Transform block Data

b = get_block_by_number(1)

# print(b)
b = transform_block_data(b)

print(json.dumps(vars(b), indent=4))

Tranform Transaction data

def transform_log_data(b, t):
    delattr(t, 'input')
    delattr(t, 'v')
    delattr(t, 'r')
    delattr(t, 's')
    delattr(t, 'nonce')

    setattr(t, 'difficulty', literal_eval(b.difficulty))
    setattr(t, 'gasLimit', literal_eval(b.gasLimit))
    setattr(t, 'gasUsed', literal_eval(b.gasUsed))
    setattr(t, 'hash', literal_eval(b.hash))
    setattr(t, 'miner', b.miner)
    setattr(t, 'parentHash', b.parentHash)
    setattr(t, 'size', literal_eval(b.size))
    setattr(t, 'timestamp', pd.to_datetime(literal_eval(b.timestamp), unit='s').strftime('%Y-%m-%d %H:%M:%S'))
    setattr(t, 'totalDifficulty', literal_eval(b.totalDifficulty))

    t.blockNumber = literal_eval(t.blockNumber)
    t.gas = literal_eval(t.gas)
    t.gasPrice = literal_eval(t.gasPrice)
    t.transactionIndex = literal_eval(t.transactionIndex)
    t.value = literal_eval(t.value)

    return t

Sample - Transform Block Data by transactions

input = '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x4", true],"id":1}'

response = requests.post(url, headers=headers, data=input)

if response.status_code == 200:
    response_data = response.json()
    block = SimpleNamespace(** (SimpleNamespace(** response_data).result))

    if len(block.transactions) > 0 :
        for t in block.transactions:
            t = transform_log_data(block, t)
            print(json.dumps(vars(t), indent=4))

Print Transactions in block

def print_txn_in_block(n):
    block = get_block_by_number(n)

    if len(block.transactions) > 0 :
        for t in block.transactions:
            t = transform_log_data(block, SimpleNamespace(** t))
            # print(json.dumps(vars(t), indent=4))
            log_file.write(json.dumps(vars(t)) + "\n")
            log_file.flush()
    else:
        # print(json.dumps(vars(transform_block_data(block))))
        log_file.write(json.dumps(vars(transform_block_data(block))) + "\n")
        log_file.flush()

Test print transactions in block

print_txn_in_block(11)

Log transactions

block_height = get_latest_block_number()

current_block = 1

while True:
    print_txn_in_block(current_block)

    if current_block == block_height:
        block_height = get_latest_block_number()

        if current_block == block_height:
            break

    current_block += 1

Test to limited blocks

block_height = 12

current_block = 1

while True:
    print_txn_in_block(current_block)

    b = get_block_by_number(current_block)
    if current_block == block_height:
        break

    current_block += 1

Get Last written block number

def last_written_block_number():
    try:
        with open('ethereum.log', 'rb') as f:
            try:  # catch OSError in case of a one line file
                f.seek(-2, os.SEEK_END)
                while f.read(1) != b'\n':
                    f.seek(-2, os.SEEK_CUR)
            except OSError:
                f.seek(0)
            last_line = f.readline().decode()
            block = json.loads(last_line)

            return block['blockNumber']

    except OSError:
        return 0

Test Last Written block number

print(last_written_block_number())