bsq-index

#!/usr/bin/python3

import subprocess
import inspect
import simplejson
import time
import git
import os
import logging
import sys
from pprint import pprint

def run_command(command, input_str=None, ignore_stderr=False):
    if ignore_stderr:
        if input_str!=None:
            p = subprocess.Popen(command, shell=True,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE)
            return p.communicate(input_str)
        else:
            p = subprocess.Popen(command, shell=True,
                stdout=subprocess.PIPE)
            return p.communicate()
    else:
        if input_str!=None:
            p = subprocess.Popen(command, shell=True,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT)
            return p.communicate(input_str)
        else:
            p = subprocess.Popen(command, shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT)
            return p.communicate()

def error(msg):
    last_block_msg=''
    func_name='unknown'
    try:
        func_name=inspect.stack()[1][3]
    except IndexError:
        pass
    # on parse: update last block
    if func_name.startswith('parse'):
        # store last parsed block
        try:
            last_block_msg='('+str(last_block)+')'
        except IOError:
            pass
    print('[E] '+func_name+': '+str(msg)+last_block_msg)
    exit(1)

def info(msg):
    func_name='unknown'
    try:
        func_name=inspect.stack()[1][3]
    except IndexError:
        pass
    print('[I] '+func_name+': '+str(msg))

def debug(msg):
    if d == True:
        func_name='unknown'
        try:
            func_name=inspect.stack()[1][3]
        except IndexError:
            pass
        print('[D] '+func_name+': '+str(msg))

def formatted_decimal(float_number):
    s=str("{0:.8f}".format(float_number))
    if s.strip('0.') == '':      # only zero and/or decimal point
        return '0.0'
    else:
        trimmed=s.rstrip('0')     # remove zeros on the right
        if trimmed.endswith('.'): # make sure there is at least one zero on the right
            return trimmed+'0'
        else:
            if trimmed.find('.')==-1:
                return trimmed+'.0'
            else:
                return trimmed

def format_time_from_struct(st, short=False):
    if short:
        return time.strftime('%Y%m%d',st)
    else:
        return time.strftime('%d %b %Y %H:%M:%S GMT',st)

def format_time_from_epoch(epoch, short=False):
    return format_time_from_struct(time.localtime(int(epoch)), short)

def get_now():
    return format_time_from_struct(time.gmtime())

def get_today():
    return format_time_from_struct(time.gmtime(), True)

def get_string_xor(s1,s2):
    result = int(s1, 16) ^ int(s2, 16)
    return '{:x}'.format(result)

def load_dict_from_file(filename, all_list=False, skip_error=False):
    tmp_dict={}
    try:
        f=open(filename,'r')
        if all_list == False:
            tmp_dict=simplejson.load(f)[0]
        else:
            tmp_dict=simplejson.load(f)
        f.close()
    except IOError: # no such file?
        if skip_error:
            info('dict load failed. missing '+filename)
        else:
            error('dict load failed. missing '+filename)
    return tmp_dict

# mkdir -p function
def mkdirp(directory):
    if not os.path.isdir(directory):
        os.makedirs(directory)

# dump json to a file, and replace it atomically
def atomic_json_dump(tmp_dict, filename, add_brackets=True):
    # check if filename already exists
    # if exists, write to a tmp file first
    # then move atomically

    # make sure path exists
    path, only_filename = os.path.split(filename)
    mkdirp(path)

    f=open(filename,'w')
    if add_brackets:
        f.write('[')
    f.write(simplejson.dumps(tmp_dict, sort_keys=True, use_decimal=True))
    if add_brackets:
        f.write(']')
    f.write('\n')
    f.close()

def load_json_file(filename):
    f=open(filename,'r')
    json_data=f.read()
    f.close()
    data=simplejson.loads(str(json_data))
    return data

def get_git_details():
    return ("","")
    # repo=git.Repo(config.bisqHome);
    # assert repo.bare == False
    # head_commit=repo.head.commit
    # timestamp=format_time_from_epoch(int(head_commit.authored_date), True)
    # return(head_commit.hexsha,timestamp)

def main():
    lines_per_page=100
    lines=[]
    last_block=0
    chainstate_dict=load_json_file(os.getenv('BSQ_EXPLORER_ROOT') + '/data/json/all/blocks.json')
    for block in chainstate_dict['blocks']:
        last_block=block['height']
        for tx in block['txs']:
            txid=tx['id']
            time=tx['time']
            txType=tx['txType']
            burntFee=tx['burntFee']
            outputsNum=0
            txBsqAmount=0
            # take address from first output as tx details
            if "u'address'" in list(tx['outputs'][0].keys()):
                address=tx['outputs'][0]['address']
            else:
                address="n/a"

            if txType == 'UNVERIFIED':
                txTypeDisplayString='Unverified'
            elif txType == 'INVALID':
                txTypeDisplayString='Invalid'
            elif txType == 'GENESIS':
                txTypeDisplayString='Genesis'
            elif txType == 'TRANSFER_BSQ':
                txTypeDisplayString='Transfer BSQ'
            elif txType == 'PAY_TRADE_FEE':
                txTypeDisplayString='Pay trade fee'
            elif txType == 'PROPOSAL':
                txTypeDisplayString='Proposal'
            elif txType == 'COMPENSATION_REQUEST':
                txTypeDisplayString='Compensation request'
            elif txType == 'REIMBURSEMENT_REQUEST':
                txTypeDisplayString='Reimbursement request'
            elif txType == 'BLIND_VOTE':
                txTypeDisplayString='Blind vote'
            elif txType == 'VOTE_REVEAL':
                txTypeDisplayString='Vote reveal'
            elif txType == 'LOCKUP':
                txTypeDisplayString='Lockup'
            elif txType == 'UNLOCK':
                txTypeDisplayString='Unlock'
            elif txType == 'ASSET_LISTING_FEE':
                txTypeDisplayString='Asset listing fee'
            elif txType == 'PROOF_OF_BURN':
                txTypeDisplayString='Proof of burn'
            else:
                txTypeDisplayString='Undefined'

            for o in tx['outputs']:
                index=o['index']
                if ('opReturn' not in o):
                    bsqAmount = o['bsqAmount']
                    txBsqAmount += bsqAmount
                    addr=o['address']
                    unspent=o['isUnspent']
                    outputsNum+=1
                    txo_entry={'bsqAmount':bsqAmount, 'time':time, 'txType':txType, 'txTypeDisplayString':txTypeDisplayString, 'txId':txid, 'index':str(index)}
                    if addr in addr_dict:
                        if unspent==True:
                            stats_dict['Unspent TXOs']+=1
                            if 'utxos' in addr_dict[addr]:
                                addr_dict[addr]['utxos'].append(txo_entry)
                            else:
                                addr_dict[addr]['utxos']=[txo_entry]
                        else:
                            stats_dict['Spent TXOs']+=1
                            if 'stxos' in addr_dict[addr]:
                                addr_dict[addr]['stxos'].append(txo_entry)
                            else:
                                addr_dict[addr]['stxos']=[txo_entry]

                    else:
                        if unspent==True:
                            stats_dict['Unspent TXOs']+=1
                            addr_dict[addr]={'utxos':[txo_entry]}
                        else:
                            stats_dict['Spent TXOs']+=1
                            addr_dict[addr]={'stxos':[txo_entry]}

                    if (o['txOutputType'] == 'GENESIS_OUTPUT' or
                            o['txOutputType'] == 'ISSUANCE_CANDIDATE_OUTPUT') and \
                            o['isVerified'] == True:
                        # collect minted coins for stats
                        stats_dict['Minted amount']+=o['bsqAmount']

            # collect the fee for stats
            stats_dict['Burnt amount']+=float(tx['burntFee'])

            line_dict={'bsqAmount':txBsqAmount, 'txType':txType, 'txTypeDisplayString':txTypeDisplayString, 'txId':txid, 'time':time, 'burntFee':burntFee, 'outputsNum':outputsNum, 'height':last_block}
            lines.append(line_dict)

    # divide by 100 Satoshi/BSQ (1 BSQ = 100 Sat)
    stats_dict['Minted amount']/=100
    stats_dict['Burnt amount']/=100
    stats_dict['Existing amount']/=100

    # calculate more stats
    # TODO missing issued amount
    stats_dict['Existing amount']=stats_dict['Minted amount']-stats_dict['Burnt amount']

    stats_dict['Addresses']=len(list(addr_dict.keys()))

    # TODO not provided yet in jsons (though in app available)
    # stats_dict['Price']=0.001234
    # stats_dict['Marketcap']=stats_dict['Price']*stats_dict['Existing amount']


    stats_json=[]
    for k in ["Existing amount", "Minted amount", "Burnt amount", "Addresses", "Unspent TXOs", "Spent TXOs", "Price", "Marketcap"]:
       stats_json.append({"name":k, "value":stats_dict[k]})

    atomic_json_dump(stats_json, os.getenv('BSQ_EXPLORER_ROOT') + '/general/stats.json', add_brackets=False)


    # split recent tx to pages
    lines.reverse()
    pages=int((len(lines)-1)/lines_per_page)+1
    for i in range(pages):
        strnum=str(i+1).zfill(4)
        atomic_json_dump(lines[i*lines_per_page:(i+1)*lines_per_page], os.getenv('BSQ_EXPLORER_ROOT') + '/general/BSQ_'+strnum+'.json', add_brackets=False)

    atomic_json_dump({"currency": "BSQ", "name": "BSQ token", "pages": pages}, os.getenv('BSQ_EXPLORER_ROOT') + '/values.json')


    # update field in addr
    for addr in list(addr_dict.keys()):
        balance=0
        totalReserved=0
        burntNum=0
        genesisTxNum=0
        receivedOutputsNum=0
        spentOutputsNum=0
        totalGenesis=0
        totalReceived=0
        totalSpent=0
        if 'utxos' in addr_dict[addr]:
            for u in addr_dict[addr]['utxos']:
                balance+=u['bsqAmount']
                if u['txType']=='GENESIS':
                    genesisTxNum+=1
                    totalGenesis+=u['bsqAmount']
                else:
                    receivedOutputsNum+=1
                    totalReceived+=u['bsqAmount']
        if 'stxos' in addr_dict[addr]:
            for s in addr_dict[addr]['stxos']:
                if s['txType']=='GENESIS':
                    genesisTxNum+=1
                    totalGenesis+=s['bsqAmount']
                    spentOutputsNum+=1
                    totalSpent+=s['bsqAmount']
                else:
                    receivedOutputsNum+=1
                    totalReceived+=s['bsqAmount']
                    spentOutputsNum+=1
                    totalSpent+=s['bsqAmount']

        totalBurnt=totalGenesis+totalReceived-totalSpent-balance

        addr_dict[addr]['address']=addr
        addr_dict[addr]['balance']=balance
        addr_dict[addr]['genesisTxNum']=genesisTxNum
        addr_dict[addr]['totalGenesis']=totalGenesis
        addr_dict[addr]['receivedOutputsNum']=receivedOutputsNum
        addr_dict[addr]['totalReceived']=totalReceived
        addr_dict[addr]['spentOutputsNum']=spentOutputsNum
        addr_dict[addr]['totalSpent']=totalSpent
        addr_dict[addr]['burntNum']=burntNum
        addr_dict[addr]['totalBurnt']=totalBurnt
        addr_dict[addr]['totalReserved']=totalReserved


    for addr in list(addr_dict.keys()):
        atomic_json_dump(addr_dict[addr], os.getenv('BSQ_EXPLORER_ROOT') + '/addr/'+addr+'.json', add_brackets=False)

    (commit_hexsha,commit_time)=get_git_details()
    now=get_now()
    revision_dict={"commit_hexsha":commit_hexsha, "commit_time":commit_time, "last_block":last_block, "last_parsed":now, "url":"https://github.com/bisq-network/bisq"}

    atomic_json_dump(revision_dict, os.getenv('BSQ_EXPLORER_ROOT') + '/revision.json', add_brackets=False)

# global variables
d=False # debug mode
last_block=0
bsqutxo_dict={}
bsqo_dict={}
tx_dict={}
addr_dict={}
chainstate_dict={}
stats_dict={"Existing amount":0,
            "Minted amount":0,
            "Burnt amount":0,
            "Addresses":0,
            "Unspent TXOs":0,
            "Spent TXOs":0,
            "Price":0,
            "Marketcap":0}

if __name__== "__main__" :
    main()