nbtext.py

import json
import os
import random
from random import sample
import numpy as np
import numpy.random
import re
from collections import Counter
import inspect

import pandas as pd
import matplotlib.pyplot as plt
import requests
from IPython.display import HTML
import seaborn as sns
import networkx as nx
from pylab import rcParams
try:
    from wordcloud import WordCloud
except ImportError:
    print("wordcloud er ikke installert, kan ikke lage ordskyer")


#************** For defining wordbag search

def dict2pd(dictionary):
    res = pd.DataFrame.from_dict(dictionary).fillna(0)
    s = (res.mean(axis=0))
    s = s.rename('snitt')
    res = res.append(s)
    return res.sort_values(by='snitt', axis=1, ascending=False).transpose()

def def2dict(ddef):
    res = dict()
    defs = ddef.split(';')
    for d in defs:
        lex = d.split(':')
        if len(lex) == 2:
            #print('#'.join(lex))
            hyper = lex[0].strip()
            occurrences = [x.strip() for x in lex[1].split(',')]
            res[hyper] = occurrences
    for x in res:
        for y in res[x]:
            if y.capitalize() not in res[x]:
                res[x].append(y.capitalize())
    return res

def wordbag_eval(wordbag, urns):
    if type(urns) is list:
        if isinstance(urns[0], list):
            urns = [u[0] for u in urns]
        else:
            urns = urns
    else:
        urns = [urns]
    param = dict()
    param['wordbags'] = wordbag
    param['urns'] = urns
    r = requests.post("https://api.nb.no/ngram/wordbags", json = param)
    return dict2pd(r.json())

def wordbag_eval_para(wordbag, urns):
    if type(urns) is list:
        if isinstance(urns[0], list):
            urns = [u[0] for u in urns]
        else:
            urns = urns
    else:
        urns = [urns]
    param = dict()
    param['wordbags'] = wordbag
    param['urns'] = urns
    r = requests.post("https://api.nb.no/ngram/wordbags_para", json = param)
    return r.json()

def get_paragraphs(urn, paras):
    """Return paragraphs for urn"""
    param = dict()
    param['paragraphs'] = paras
    param['urn'] = urn
    r = requests.get("https://api.nb.no/ngram/paragraphs", json=param)
    return dict2pd(r.json())

### ******************* wordbag search end

def ner(text = None, dist=False):
    """Analyze text for named entities - set dist = True will return the four values that go into decision"""
    r = []
    if text != None:
        r = requests.post("https://api.nb.no/ngram/ner", json={'text':text,'dist':dist})
    return r.json()
    
#**** names ****

def check_navn(navn, limit=2, remove='Ja Nei Nå Dem De Deres Unnskyld Ikke Ah Hmm Javel Akkurat Jaja Jaha'.split()):
    """Removes all items in navn with frequency below limit and words in all case as well as all words in list 'remove'"""
    r = {x:navn[x] for x in navn if navn[x] > limit and x.upper() != x and not x in remove}
    return r


def sentences(urns, num=300):
    if isinstance(urns[0], list):
        urns = [str(x[0]) for x in urns]
    params = {'urns':urns,
             'num':num}
    res = requests.get("https://api.nb.no/ngram/sentences", params=params)
    return res.json()


def names(urn, ratio = 0.3, cutoff = 2):
    """ Return namens in book with urn. Returns uni- , bi-, tri- and quadgrams """
    if type(urn) is list:
        urn = urn[0]
    r = requests.get('https://api.nb.no/ngram/names', json={'urn':urn, 'ratio':ratio, 'cutoff':cutoff})
    x = r.json()
    result = (
        Counter(x[0][0]),
        Counter({tuple(x[1][i][0]):x[1][i][1] for i in range(len(x[1]))}),
        Counter({tuple(x[2][i][0]):x[2][i][1] for i in range(len(x[2]))}),
        Counter({tuple(x[3][i][0]):x[3][i][1] for i in range(len(x[3]))})
    )
    return result


def name_graph(name_struct):
    m = []
    for n in name_struct[0]:
        m.append(frozenset([n]))
    for n in name_struct[1:]:
        m += [frozenset(x) for x in n]
        
    G = []
    for x in m:
        for y in m:
            if x < y:
                G.append((' '.join(x), ' '.join(y)))
    N = []
    for x in m:
        N.append(' '.join(x))
    Gg = nx.Graph()
    Gg.add_nodes_from(N)
    Gg.add_edges_from(G)
    return Gg

def aggregate_urns(urnlist):
    """Sum up word frequencies across urns"""
    
    if isinstance(urnlist[0], list):
        urnlist = [u[0] for u in urnlist]
    r = requests.post("https://api.nb.no/ngram/book_aggregates", json={'urns':urnlist})
    return r.json()

# Norweigan word bank
def word_variant(word, form):
    """ Find alternative form for a given word form, e.g. word_variant('spiste', 'pres-part') """
    r = requests.get("https://api.nb.no/ngram/variant_form", params={'word':word, 'form':form})
    return r.json()

def word_paradigm(word):
    """ Find alternative form for a given word form, e.g. word_variant('spiste', 'pres-part') """
    r = requests.get("https://api.nb.no/ngram/paradigm", params = {'word': word})
    return r.json()


def word_form(word):
    """ Find alternative form for a given word form, e.g. word_variant('spiste', 'pres-part') """
    r = requests.get("https://api.nb.no/ngram/word_form", params = {'word': word})
    return r.json()

def word_lemma(word):
    """ Find lemma form for a given word form """
    r = requests.get("https://api.nb.no/ngram/word_lemma", params = {'word': word})
    return r.json()


def word_freq(urn, words):
    """ Find frequency of words within urn """
    params = {'urn':urn, 'words':words}
    r = requests.post("https://api.nb.no/ngram/freq", json=params)
    return dict(r.json())

def tot_freq(words):
    """ Find total frequency of words """
    params = {'words':words}
    r = requests.post("https://api.nb.no/ngram/word_frequencies", json=params)
    return dict(r.json())

def book_count(urns):
    params = {'urns':urns}
    r = requests.post("https://api.nb.no/ngram/book_count", json=params)
    return dict(r.json())

def sttr(urn, chunk=5000):
    r = requests.get("https://api.nb.no/ngram/sttr", json = {'urn':urn, 'chunk':chunk})
    return r.json()
                     
def totals(top=200):
    r = requests.get("https://api.nb.no/ngram/totals", json={'top':top})
    return dict(r.json())

def navn(urn):
    if type(urn) is list:
        urn = urn[0]
    r = requests.get('https://api.nb.no/ngram/tingnavn', json={'urn':urn})
    return dict(r.json())
    
def digibokurn_from_text(T):
    """Return URNs as 13 digits (any sequence of 13 digits is counted as an URN)"""
    return re.findall("(?<=digibok_)[0-9]{13}", T)

def urn_from_text(T):
    """Return URNs as 13 digits (any sequence of 13 digits is counted as an URN)"""
    return re.findall("[0-9]{13}", T)

def metadata(urn=None):
    urns = pure_urn(urn)
    #print(urns)
    r = requests.post("https://api.nb.no/ngram/meta", json={'urn':urns})
    return r.json()


def pure_urn(data):
    """Convert URN-lists with extra data into list of serial numbers.
    Args:
        data: May be a list of URNs, a list of lists with URNs as their
            initial element, or a string of raw texts containing URNs
            Any pandas dataframe or series. Urns must be in the first column of dataframe.
    Returns:
        List[str]: A list of URNs. Empty list if input is on the wrong
            format or contains no URNs
    """
    
    korpus_def = []
    if isinstance(data, list):
        if not data:  # Empty list
            korpus_def = []
        if isinstance(data[0], list):  # List of lists
            try:
                korpus_def = [str(x[0]) for x in data]
            except IndexError:
                korpus_def = []
        else:  # Assume data is already a list of URNs
            korpus_def = [str(int(x)) for x in data]
    elif isinstance(data, str):
        korpus_def = [str(x) for x in urn_from_text(data)]
    elif isinstance(data, (int, np.integer)):
        korpus_def = [str(data)]    
    elif isinstance(data, pd.DataFrame):
        col = data.columns[0]
        urns = pd.to_numeric(data[col])
        korpus_def = [str(int(x)) for x in urns.dropna()]
    elif isinstance(data, pd.Series):
        korpus_def = [str(int(x)) for x in data.dropna()]
    return korpus_def

####  N-Grams from fulltext updated

def unigram(word, period=(1950, 2020), media = 'bok', ddk=None, topic=None, gender=None, publisher=None, lang=None, trans=None):
    r = requests.get("https://api.nb.no/ngram/unigrams", params={
        'word':word,
        'ddk':ddk,
        'topic':topic,
        'gender':gender,
        'publisher':publisher,
        'lang':lang,
        'trans':trans,
        'period0':period[0],
        'period1':period[1],
        'media':media
    })
    return frame(dict(r.json()))

def bigram(first,second, period=(1950, 2020), media = 'bok', ddk=None, topic=None, gender=None, publisher=None, lang=None, trans=None):
    r = requests.get("https://api.nb.no/ngram/bigrams", params={
        'first':first,
        'second':second,
        'ddk':ddk,
        'topic':topic,
        'gender':gender,
        'publisher':publisher,
        'lang':lang,
        'trans':trans,
        'period0':period[0],
        'period1':period[1],
        'media':media
    })
    return frame(dict(r.json()))

def book_counts(period=(1800, 2050)):
    r = requests.get("https://api.nb.no/ngram/book_counts", params={
    
        'period0':period[0],
        'period1':period[1],
    })
    return frame(dict(r.json()))

####    

def difference(first, second, rf, rs, years=(1980, 2000),smooth=1, corpus='bok'):
    """Compute difference of difference (first/second)/(rf/rs)"""
    try:
        a_first = nb_ngram(first, years=years, smooth=smooth, corpus=corpus)
        a_second = nb_ngram(second, years=years, smooth=smooth, corpus=corpus)
        a = a_first.join(a_second)  
        b_first = nb_ngram(rf, years=years, smooth=smooth, corpus=corpus)
        b_second = nb_ngram(rs, years=years, smooth=smooth, corpus=corpus)
        if rf == rs:
            b_second.columns = [rs + '2']
        b = b_first.join(b_second)
        s_a = a.mean()
        s_b = b.mean()
        f1 = s_a[a.columns[0]]/s_a[a.columns[1]]
        f2 = s_b[b.columns[0]]/s_b[b.columns[1]]
        res = f1/f2
    except:
        res = 'Mangler noen data - har bare for: ' + ', '.join([x for x in a.columns.append(b.columns)])
    return res

def df_combine(array_df):
    """Combine one columns dataframes"""
    import pandas as pd
    cols = []
    for i in range(len(a)):
        #print(i)
        if array_df[i].columns[0] in cols:
            array_df[i].columns = [array_df[i].columns[0] + '_' + str(i)]
        cols.append(array_df[i].columns[0])
    return pd.concat(a, axis=1, sort=True)

def col_agg(df, col='sum'):
    c = df.sum(axis=0)
    c = pd.DataFrame(c)
    c.columns = [col]
    return c

def row_agg(df, col='sum'):
    c = df.sum(axis=1)
    c = pd.DataFrame(c)
    c.columns = [col]
    return c


def get_freq(urn, top=50, cutoff=3):
    """Get frequency list for urn"""
    if isinstance(urn, list):
        urn = urn[0]
    r = requests.get("https://api.nb.no/ngram/urnfreq", json={'urn':urn, 'top':top, 'cutoff':cutoff})
    return Counter(dict(r.json()))

####=============== GET URNS ==================##########
def book_corpus(words = None, author = None, 
             title = None, subtitle = None, ddk  = None, subject = None, 
             period=(1100, 2020), 
             gender=None, 
             lang = None, 
             trans= None, 
             limit=20 ):
    return frame(book_urn(words, author, title, subtitle, ddk, subject, period, gender, lang, trans, limit),
                "urn author title year".split())
                
def book_urn(words = None, author = None, 
             title = None, subtitle = None, ddk  = None, subject = None, 
             period=(1100, 2020), 
             gender=None, 
             lang = None, 
             trans= None, 
             limit=20 ):
    """Get URNs for books with metadata"""
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)
    query = {i:values[i] for i in args if values[i] != None and i != 'period'}
    query['year'] = period[0]
    query['next'] = period[1] - period[0]
    return get_urn(query)

def unique_urns(korpus, newest=True):
    author_title = {(c[1],c[2]) for c in korpus}
    corpus = {(c[0], c[1]):[d for d in korpus if c[0] == d[1] and c[1]==d[2]] for c in author_title  }
    for c in corpus:
        corpus[c].sort(key=lambda c: c[3])

    if newest == True:
        res = [corpus[c][-1] for c in corpus]
    else:
        res = [corpus[c][0] for c in corpus]
    return res

def refine_book_urn(urns = None, words = None, author = None, 
             title = None, ddk  = None, subject = None, period=(1100, 2020), gender=None, lang = None, trans= None, limit=20 ):

    """Refine URNs for books with metadata"""
    
    # if empty urns nothing to refine
    
    if urns is None or urns == []:
        return []
    
    # check if urns is a metadata list, and pick out first elements if that is the case
    if isinstance(urns[0], list):
        urns = [x[0] for x in urns]
        
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)
    query = {i:values[i] for i in args if values[i] != None and i != 'period' and i != 'urns'}
    query['year'] = period[0]
    query['next'] = period[1] - period[0]
    #print(query)
    return refine_urn(urns, query)

def best_book_urn(word = None, author = None, 
             title = None, ddk  = None, subject = None, period=(1100, 2020), gender=None, lang = None, trans= None, limit=20 ):
    """Get URNs for books with metadata"""
    
    if word is None:
        return []
    
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)
    query = {i:values[i] for i in args if values[i] != None and i != 'period' and i != 'word'}
    query['year'] = period[0]
    query['next'] = period[1] - period[0]
    return get_best_urn(word, query)

def get_urn(metadata=None):
    """Get urns from metadata"""
    if metadata is None:
        metadata = {}
    if not ('next' in metadata or 'neste' in metadata):
        metadata['next'] = 100
    if not 'year' in metadata:
        metadata['year'] = 1900
    r = requests.get('https://api.nb.no/ngram/urn', json=metadata)
    return r.json()

def refine_urn(urns, metadata=None):
    """Refine a list urns using extra information"""
    if metadata is None:
        metadata = {}
    metadata['urns'] = urns
    if not ('words' in metadata):
        metadata['words'] = []
    if not ('next' in metadata or 'neste' in metadata):
        metadata['next'] = 520
    if not 'year' in metadata:
        metadata['year'] = 1500
    r = requests.post('https://api.nb.no/ngram/refineurn', json=metadata)
    return r.json()

def get_best_urn(word, metadata=None):
    """Get the best urns from metadata containing a specific word"""
    metadata['word'] = word
    if not ('next' in metadata or 'neste' in metadata):
        metadata['next'] = 600
    if not 'year' in metadata:
        metadata['year'] = 1500
    r = requests.get('https://api.nb.no/ngram/best_urn', json=metadata)
    return r.json()

def get_papers(top=5, cutoff=5, navn='%', yearfrom=1800, yearto=2020, samplesize=100):
    """Get newspapers"""
    div = lambda x, y: (int(x/y), x % y)
    chunks = 20
    
    
    # split samplesize into chunks, go through the chunks and then the remainder
    
    (first, second) = div(samplesize, chunks)
    r = []
    
    # collect chunkwise 
    for i in range(first):
        r += requests.get("https://api.nb.no/ngram/avisfreq", json={'navn':navn, 'top':top, 'cutoff':cutoff,
                                                              'yearfrom':yearfrom, 'yearto':yearto,'samplesize':chunks}
                         ).json()
    
    
    # collect the remainder
    r += requests.get("https://api.nb.no/ngram/avisfreq", json={'navn':navn, 'top':top, 'cutoff':cutoff,
                                                              'yearfrom':yearfrom, 'yearto':yearto,'samplesize':second}
                         ).json()

    return [dict(x) for x in r]


def urn_coll(word, urns=[], after=5, before=5, limit=1000):
    """Find collocations for word in a set of book URNs. Only books at the moment"""
    
    if isinstance(urns[0], list):  # urns assumed to be list of list with urn-serial as first element
        urns = [u[0] for u in urns]
        
    r = requests.post("https://api.nb.no/ngram/urncoll", json={'word':word, 'urns':urns, 
                                                                   'after':after, 'before':before, 'limit':limit})
    res = pd.DataFrame.from_dict(r.json(), orient='index')
    if not res.empty:
        res = res.sort_values(by=res.columns[0], ascending = False)
    return res

def urn_coll_words(words, urns=None, after=5, before=5, limit=1000):
    """Find collocations for a group of words within a set of books given by a list of URNs. Only books at the moment"""
    coll = pd.DataFrame()
    if urns != None:
        if isinstance(urns[0], list):  # urns assumed to be list of list with urn-serial as first element
            urns = [u[0] for u in urns]
        colls = Counter()
        if isinstance(words, str):
            words = words.split()
        res = Counter()
        for word in words: 
            try:
                res += Counter(
                    requests.post(
                        "https://api.nb.no/ngram/urncoll", 
                        json={
                            'word':word, 
                            'urns':urns, 
                            'after':after, 
                            'before':before, 
                            'limit':limit}
                    ).json()
                )
            except:
                True
        coll = pd.DataFrame.from_dict(res, orient='index')
        if not coll.empty:
            coll = coll.sort_values(by=coll.columns[0], ascending = False) 
    return coll


def get_aggregated_corpus(urns, top=0, cutoff=0):
    res = Counter()
    if isinstance(urns[0], list):  # urns assumed to be list of list with urn-serial as first element
        urns = [u[0] for u in urns]
    for u in urns:
        #print(u)
        res += get_freq(u, top = top, cutoff = cutoff)
    return pd.DataFrame.from_dict(res, orient='index').sort_values(by=0, ascending = False)


def compare_word_bags(bag_of_words, another_bag_of_words, first_freq = 0, another_freq = 1, top=100, first_col = 0, another_col= 0):
    """Compare two columns taken from two or one frame. Parameters x_freq are frequency limits used to cut down candidate words
    from the bag of words. Compare along the columns where first_col and another_col are column numbers. Typical situation is that
    bag_of_words is a one column frame and another_bag_of_words is another one column frame. When the columns are all from one frame, 
    just change column numbers to match the columns"""
    diff = bag_of_words[bag_of_words > first_freq][bag_of_words.columns[first_col]]/another_bag_of_words[another_bag_of_words > another_freq][another_bag_of_words.columns[another_col]] 
       
    return frame(diff, 'diff').sort_values(by='diff', ascending=False)[:top]


def collocation(
    word, 
    yearfrom=2010, 
    yearto=2018, 
    before=3, 
    after=3, 
    limit=1000, 
    corpus='avis',
    lang='nob',
    title='%',
    ddk='%', 
    subtitle='%'):
    """Defined collects frequencies for a given word"""
    
    data =  requests.get(
        "https://api.nb.no/ngram/collocation", 
        params={
            'word':word,
            'corpus':corpus, 
            'yearfrom':yearfrom, 
            'before':before,
            'after':after,
            'limit':limit,
            'yearto':yearto,
        'title':title,
        'ddk':ddk,
        'subtitle':subtitle}).json()
    return pd.DataFrame.from_dict(data['freq'], orient='index')


def collocation_data(words, yearfrom = 2000, yearto = 2005, limit = 1000, before = 5, after = 5, title = '%', corpus='bok'):
    """Collocation for a set of words sum up all the collocations words is a list of words or a blank separated string of words"""
    import sys
    a = dict()
    
    if isinstance(words, str):
        words = words.split()
    
    for word in words:
        
        print(word)
        try:
            
            a[word] = collocation(
                word, 
                yearfrom = yearfrom, yearto = yearto, limit = limit, 
                corpus = corpus, before = before, 
                after = after, title = title
            )
            
            a[word].columns = [word]
        
        except:    
            print(word, ' feilsituasjon', sys.exc_info())
    result = pd.DataFrame()
    for w in a:
        result = result.join(a[w], how='outer')
    return pd.DataFrame(result.sum(axis=1)).sort_values(by=0, ascending=False)

class CollocationCorpus:
    from random import sample
    
    def __init__(self, corpus = None, name='', maximum_texts = 500):
        urns = pure_urn(corpus)
        
        if len(urns) > maximum_texts:      
            selection = random(urns, maximum_texts)
        else:
            selection = urns
            
        self.corpus_def = selection
        self.corpus = get_aggregated_corpus(self.corpus_def, top=0, cutoff=0)


    def summary(self, head=10):
        info = {
            'corpus_definition':self.corpus[:head],
            'number_of_words':len(self.corpus)
            
        }
        return info

def collocation_old(word, yearfrom=2010, yearto=2018, before=3, after=3, limit=1000, corpus='avis'):
    data =  requests.get(
        "https://api.nb.no/ngram/collocation", 
        params={
            'word':word,
            'corpus':corpus, 
            'yearfrom':yearfrom, 
            'before':before,
            'after':after,
            'limit':limit,
            'yearto':yearto}).json()
    return pd.DataFrame.from_dict(data['freq'], orient='index')

def heatmap(df, color='green'):
    return df.fillna(0).style.background_gradient(cmap=sns.light_palette(color, as_cmap=True))

def get_corpus_text(urns, top = 0, cutoff=0):
    k = dict()
    if isinstance(urns, list):
        # a list of urns, or a korpus with urns as first element
        if isinstance(urns[0], list):
            urns = [u[0] for u in urns]
    else:
        # assume it is a single urn, text or number
        urns = [urns]
    for u in urns:
        #print(u)
        k[u] = get_freq(u, top = top, cutoff = cutoff)
    df = pd.DataFrame(k)
    res = df.sort_values(by=df.columns[0], ascending=False)
    return res


def normalize_corpus_dataframe(df):
    colsums = df.sum()
    for x in colsums.index:
        #print(x)
        df[x] = df[x].fillna(0)/colsums[x]
    return True

def show_korpus(korpus, start=0, size=4, vstart=0, vsize=20, sortby = ''):
    """Show corpus as a panda dataframe
    start = 0 indicates which dokument to show first, dataframe is sorted according to this
    size = 4 how many documents (or columns) are shown
    top = 20 how many words (or rows) are shown"""
    if sortby != '':
        val = sortby
    else:
        val = korpus.columns[start]
    return korpus[korpus.columns[start:start+size]].sort_values(by=val, ascending=False)[vstart:vstart + vsize]

def aggregate(korpus):
    """Make an aggregated sum of all documents across the corpus, here we use average"""
    return pd.DataFrame(korpus.fillna(0).mean(axis=1))

def convert_list_of_freqs_to_dataframe(referanse):
    """The function get_papers() returns a list of frequencies - convert it"""
    res = []
    for x in referanse:
        res.append( dict(x))
    result = pd.DataFrame(res).transpose()
    normalize_corpus_dataframe(result)
    return result

def get_corpus(top=0, cutoff=0, navn='%', corpus='avis', yearfrom=1800, yearto=2020, samplesize=10):
    if corpus == 'avis':
        result = get_papers(top=top, cutoff=cutoff, navn=navn, yearfrom=yearfrom, yearto=yearto, samplesize=samplesize)
        res = convert_list_of_freqs_to_dataframe(result)
    else:
        urns = get_urn({'author':navn, 'year':yearfrom, 'neste':yearto-yearfrom, 'limit':samplesize})
        res = get_corpus_text([x[0] for x in urns], top=top, cutoff=cutoff)
    return res


class Cluster:
    def __init__(self, word = '', filename = '', period = (1950,1960) , before = 5, after = 5, corpus='avis', reference = 200, 
                 word_samples=1000):
        if word != '':
            self.collocates = collocation(word, yearfrom=period[0], yearto = period[1], before=before, after=after,
                                corpus=corpus, limit=word_samples)
            self.collocates.columns = [word]
            if type(reference) is pd.core.frame.DataFrame:
                reference = reference
            elif type(reference) is int:
                reference = get_corpus(yearfrom=period[0], yearto=period[1], corpus=corpus, samplesize=reference)
            else:
                reference = get_corpus(yearfrom=period[0], yearto=period[1], corpus=corpus, samplesize=int(reference))

            self.reference = aggregate(reference)
            self.reference.columns = ['reference_corpus']        
            self.word = word
            self.period = period
            self.corpus = corpus
        else:
            if filename != '':
                self.load(filename)
            
                
    def cluster_set(self, exponent=1.1, top = 200, aslist=True):
        combo_corp = self.reference.join(self.collocates, how='outer')
        normalize_corpus_dataframe(combo_corp)
        korpus = compute_assoc(combo_corp, self.word, exponent)
        korpus.columns = [self.word]
        if top <= 0:
            res = korpus.sort_values(by=self.word, ascending=False)
        else:
            res = korpus.sort_values(by=self.word, ascending=False).iloc[:top]
        if aslist == True:
            res = HTML(', '.join(list(res.index)))
        return res
    
    def add_reference(self, number=20):
        ref = get_corpus(yearfrom=self.period[0], yearto=self.period[1], samplesize=number)
        ref = aggregate(ref)
        ref.columns = ['add_ref']
        normalize_corpus_dataframe(ref)
        self.reference = aggregate(self.reference.join(ref, how='outer'))
        return True
    
    def save(self, filename=''):
        if filename == '':
            filename = "{w}_{p}-{q}.json".format(w=self.word,p=self.period[0], q = self.period[1])
        model = {
            'word':self.word,
            'period':self.period,
            'reference':self.reference.to_dict(),
            'collocates':self.collocates.to_dict(),
            'corpus':self.corpus
        }
        with open(filename, 'w', encoding = 'utf-8') as outfile:
            print('lagrer til:', filename)
            outfile.write(json.dumps(model))
        return True

    def load(self, filename):
        with open(filename, 'r') as infile:
            try:
                model = json.loads(infile.read())
                #print(model['word'])
                self.word = model['word']
                self.period = model['period']
                self.corpus = model['corpus']
                self.reference = pd.DataFrame(model['reference'])
                self.collocates = pd.DataFrame(model['collocates'])
            except:
                print('noe gikk galt')
        return True
    
       
    def search_words(self, words, exponent=1.1):
        if type(words) is str:
            words = [w.strip() for w in words.split()]
        df = self.cluster_set(exponent=exponent, top=0, aslist=False)
        sub= [w for w in words if w in df.index]
        res = df.transpose()[sub].transpose().sort_values(by=df.columns[0], ascending=False)
        return res


def wildcardsearch(params=None):
    if params is None:
        params = {'word': '', 'freq_lim': 50, 'limit': 50, 'factor': 2}
    res = requests.get('https://api.nb.no/ngram/wildcards', params=params)
    if res.status_code == 200:
        result = res.json()
    else:
        result = {'status':'feil'}
    resultat = pd.DataFrame.from_dict(result, orient='index')
    if not(resultat.empty):
        resultat.columns = [params['word']]
    return resultat

def sorted_wildcardsearch(params):
    res = wildcardsearch(params)
    if not res.empty:
        res = res.sort_values(by=params['word'], ascending=False)
    return res
            
def make_newspaper_network(key, wordbag, titel='%', yearfrom='1980', yearto='1990', limit=500):
    if type(wordbag) is str:
        wordbag = wordbag.split()
    r = requests.post("https://api.nb.no/ngram/avisgraph", json={
        'key':key, 
        'words':wordbag,
        'yearto':yearto,
    'yearfrom':yearfrom,
    'limit':limit})
    G = nx.Graph()
    if r.status_code == 200:
        G.add_weighted_edges_from([(x,y,z) for (x,y,z) in r.json() if z > 0 and x != y])
    else:
        print(r.text)
    return G

def make_network(urn, wordbag, cutoff=0):
    if type(urn) is list:
        urn = urn[0]
    if type(wordbag) is str:
        wordbag = wordbag.split()
    G = make_network_graph(urn, wordbag, cutoff)
    return G

def make_network_graph(urn, wordbag, cutoff=0):
    r = requests.post("https://api.nb.no/ngram/graph", json={'urn':urn, 'words':wordbag})
    G = nx.Graph()
    G.add_weighted_edges_from([(x,y,z) for (x,y,z) in r.json() if z > cutoff and x != y])
    return G

def make_network_name_graph(urn, tokens, tokenmap=None, cutoff=2):
    if isinstance(urn, list):
        urn = urn[0]
        
    # tokens should be a list of list of tokens. If it is list of dicts pull out the keys (= tokens)   
    if isinstance(tokens[0], dict):
        tokens = [list(x.keys()) for x in tokens]
        
    r = requests.post("https://api.nb.no/ngram/word_graph", json={'urn':urn, 'tokens':tokens, 'tokenmap':tokenmap})
    #print(r.text)
    G = nx.Graph()
    G.add_weighted_edges_from([(x,y,z) for (x,y,z) in r.json() if z > cutoff and x != y])
    return G



def token_convert_back(tokens, sep='_'):
    """ convert a list of tokens to string representation"""
    res = [tokens[0]]
    for y in tokens:
        res.append([tuple(x.split(sep)) for x in y])
    l = len(res)
    for x in range(1, 4-l):
        res.append([])
    return res

def token_convert(tokens, sep='_'):
    """ convert back to tuples """
    tokens = [list(x.keys()) for x in tokens]
    tokens = [[(x,) for x in tokens[0]], tokens[1], tokens[2], tokens[3]]
    conversion = []
    for x in tokens:
        conversion.append([sep.join(t) for t in x])
    return conversion

def token_map_to_tuples(tokens_as_strings, sep='_', arrow='==>'):
    tuples = []
    for x in tokens_as_strings:
        token = x.split(arrow)[0].strip()
        mapsto = x.split(arrow)[1].strip()
        tuples.append((tuple(token.split(sep)), tuple(mapsto.split(sep))))
    #tuples = [(tuple(x.split(arrow).strip()[0].split(sep)), tuple(x.split(arrow)[1].strip().split(sep))) for x in tokens_as_strings]
    return tuples
        
def token_map(tokens, strings=False, sep='_', arrow= '==>'):
    """ tokens as from nb.names()"""
    if isinstance(tokens[0], dict):
        # get the keys(), otherwise it is already just a list of tokens up to length 4
        tokens = [list(x.keys()) for x in tokens]
        
    # convert tokens to tuples and put them all in one list
    tokens = [(x,) for x in tokens[0]] + tokens[1] + tokens[2] + tokens[3]
    tm = []
    #print(tokens)
    for token in tokens:
        if isinstance(token, str):
            trep = (token,)
        elif isinstance(token, list):
            trep = tuple(token)
            token = tuple(token)
        else:
            trep = token
        n = len(trep)
        #print(trep)

        if trep[-1].endswith('s'):
            cp = list(trep[:n-1])
            cp.append(trep[-1][:-1])
            cp = tuple(cp)

            #print('copy', cp, trep)
            if cp in tokens:
                #print(trep, cp)
                trep = cp

        larger = [ts for ts in tokens if set(ts) >= set(trep)]
        #print(trep, ' => ', larger)
        larger.sort(key=lambda x: len(x), reverse=True)
        tm.append((token,larger[0]))
        res = tm
        if strings == True:
            res = [sep.join(x[0]) + ' ' + arrow + ' ' + sep.join(x[1]) for x in tm]
            
    return res



def draw_graph_centrality(G, h=15, v=10, fontsize=20, k=0.2, arrows=False, font_color='black', threshold=0.01): 
    node_dict = nx.degree_centrality(G)
    subnodes = dict({x:node_dict[x] for x in node_dict if node_dict[x] >= threshold})
    x, y = rcParams['figure.figsize']
    rcParams['figure.figsize'] = h, v
    pos =nx.spring_layout(G, k=k)
    ax = plt.subplot()
    ax.set_xticks([])
    ax.set_yticks([])
    G = G.subgraph(subnodes)
    nx.draw_networkx_labels(G, pos, font_size=fontsize, font_color=font_color)
    nx.draw_networkx_nodes(G, pos, alpha=0.5, nodelist=subnodes.keys(), node_size=[v * 1000 for v in subnodes.values()])
    nx.draw_networkx_edges(G, pos, alpha=0.7, arrows=arrows, edge_color='lightblue', width=1)

    rcParams['figure.figsize'] = x, y
    return True

def combine(clusters):
    """Make new collocation analyses from data in clusters"""
    colls = []
    collocates = clusters[0].collocates
    for c in clusters[1:]:
        collocates = collocates.join(c.collocates, rsuffix='-' + str(c.period[0]))
    return collocates

def cluster_join(cluster):
    clusters = [cluster[i] for i in cluster]
    clst = clusters[0].cluster_set(aslist=False)
    for c in clusters[1:]:
        clst = clst.join(c.cluster_set(aslist=False), rsuffix = '_'+str(c.period[0]))
    return clst

def serie_cluster(word, startår, sluttår, inkrement, before=5, after=5, reference=150, word_samples=500):
    tidscluster = dict()
    for i in range(startår, sluttår, inkrement):
        tidscluster[i] = Cluster(
        word, 
        corpus='avis', 
        period=(i, i + inkrement - 1), 
        before=after, 
        after=after, 
        reference=reference, 
        word_samples=word_samples)
        print(i, i+inkrement - 1)
    return tidscluster

def save_serie_cluster(tidscluster):
    for i in tidscluster:
        tidscluster[i].save()
    return 'OK'

def les_serie_cluster(word, startår, sluttår, inkrement):
    tcluster = dict()
    for i in range(startår, sluttår, inkrement):
        print(i, i+inkrement - 1)
        tcluster[i] = Cluster(filename='{w}_{f}-{t}.json'.format(w=word, f=i,t=i+inkrement - 1))
    return tcluster




def make_cloud(json_text, top=100, background='white', stretch=lambda x: 2**(10*x), width=500, height=500, font_path=None):
    pairs0 = Counter(json_text).most_common(top)
    pairs = {x[0]:stretch(x[1]) for x in pairs0}
    wc = WordCloud(
    font_path=font_path,  
    background_color=background,
    width=width, 
    #color_func=my_colorfunc,
    ranks_only=True,
    height=height).generate_from_frequencies(pairs)
    return wc

def draw_cloud(sky, width=20, height=20, fil=''):
    plt.figure(figsize=(width,height))
    plt.imshow(sky, interpolation='bilinear')
    figplot = plt.gcf()
    if fil != '':
        figplot.savefig(fil, format='png')
    return 

def cloud(pd, column='', top=200, width=1000, height=1000, background='black', file='', stretch=10, font_path=None):
    if column == '':
        column = pd.columns[0]
    data = json.loads(pd[column].to_json())
    a_cloud = make_cloud(data, top=top, 
                         background=background, font_path=font_path, 
                         stretch=lambda x: 2**(stretch*x), width=width, height=height)
    draw_cloud(a_cloud, fil=file)
    return


def make_a_collocation(word, period=(1990, 2000), before=5, after=5, corpus='avis', samplesize=100, limit=2000):
    collocates = collocation(word, yearfrom=period[0], yearto=period[1], before=before, after=after,
                             corpus=corpus, limit=limit)
    collocates.columns = [word]
    reference = get_corpus(yearfrom=period[0], yearto=period[1], samplesize=samplesize)
    ref_agg = aggregate(reference)
    ref_agg.columns = ['reference_corpus']
    return  ref_agg



def compute_assoc(coll_frame, column, exponent=1.1, refcolumn = 'reference_corpus'):
    return pd.DataFrame(coll_frame[column]**exponent/coll_frame.mean(axis=1))
    


class Corpus:
    def __init__(self, filename = '', target_urns = None, reference_urns = None,  period = (1950,1960), author='%', 
                 title='%', ddk='%', gender='%', subject='%', reference = 100, max_books=100):
        params = {
            'year':period[0], 
            'next': period[1]-period[0], 
            'subject':subject,
            'ddk':ddk, 
            'author':author, 
            #'gender':gender, ser ikke ut til å virke for get_urn - sjekk opp APIet
            'title':title, 
            'limit':max_books,
            'reference':reference
        }
        self.params = params
        self.coll = dict()
        self.coll_graph = dict()
        if filename == '':
            if target_urns != None:
                målkorpus_def = target_urns
            else:
                målkorpus_def = get_urn(params)

            #print("Antall bøker i målkorpus ", len(målkorpus_def))
            if isinstance(målkorpus_def[0], list):
                målkorpus_urn = [str(x[0]) for x in målkorpus_def]
                #print(målkorpus_urn)
            else:
                målkorpus_urn = målkorpus_def
            if len(målkorpus_urn) > max_books and max_books > 0:
                target_urn = list(numpy.random.choice(målkorpus_urn, max_books))
            else:
                target_urn = målkorpus_urn

            if reference_urns != None:
                referansekorpus_def = reference_urns
            else:
                # select from period, usually used only of target is by metadata
                referansekorpus_def = get_urn({'year':period[0], 'next':period[1]-period[0], 'limit':reference})


            #print("Antall bøker i referanse: ", len(referansekorpus_def))
            # referansen skal være distinkt fra målkorpuset
            referanse_urn = [str(x[0]) for x in referansekorpus_def]
            self.reference_urn = referanse_urn
            self.target_urn = target_urn
            # make sure there is no overlap between target and reference
            # 
            referanse_urn = list(set(referanse_urn) - set(target_urn))


            målkorpus_txt = get_corpus_text(target_urn)
            normalize_corpus_dataframe(målkorpus_txt)
            if referanse_urn != []:
                referanse_txt = get_corpus_text(referanse_urn)
                normalize_corpus_dataframe(referanse_txt)
                combo = målkorpus_txt.join(referanse_txt)

            else:
                referanse_txt = målkorpus_txt
                combo = målkorpus_txt
                
            self.combo = combo 
            self.reference = referanse_txt
            self.target = målkorpus_txt

            self.reference = aggregate(self.reference)
            self.reference.columns = ['reference_corpus']

            ## dokumentfrekvenser

            mål_docf = pd.DataFrame(pd.DataFrame(målkorpus_txt/målkorpus_txt).sum(axis=1))
            combo_docf = pd.DataFrame(pd.DataFrame(combo/combo).sum(axis=1))
            ref_docf = pd.DataFrame(pd.DataFrame(referanse_txt/referanse_txt).sum(axis=1))

            ### Normaliser dokumentfrekvensene
            normalize_corpus_dataframe(mål_docf)
            normalize_corpus_dataframe(combo_docf)
            normalize_corpus_dataframe(ref_docf)

            self.målkorpus_tot = aggregate(målkorpus_txt)
            self.combo_tot = aggregate(combo)
            self.mål_docf = mål_docf
            self.combo_docf = combo_docf
            self.lowest = self.combo_tot.sort_values(by=0)[0][0]
        else:
            self.load(filename)
        return 
            
    
    def difference(self, freq_exp=1.1, doc_exp=1.1, top = 200, aslist=True):
        res = pd.DataFrame(
            (self.målkorpus_tot**freq_exp/self.combo_tot)*(self.mål_docf**doc_exp/self.combo_docf)
        )
        res.columns = ['diff']
        if top > 0:
            res = res.sort_values(by=res.columns[0], ascending=False).iloc[:top]
        else:
            res = res.sort_values(by=res.columns[0], ascending=False)
        if aslist == True:
            res = HTML(', '.join(list(res.index)))
        return res
    

    def save(self, filename):

        model = {
            'params':self.params,   
            'target': self.målkorpus_tot.to_json(),
            'combo': self.combo_tot.to_json(),
            'target_df': self.mål_docf.to_json(),
            'combo_df': self.combo_docf.to_json()
        }

        with open(filename, 'w', encoding = 'utf-8') as outfile:
            outfile.write(json.dumps(model))
        return True

    def load(self, filename):
        with open(filename, 'r') as infile:
            try:
                model = json.loads(infile.read())
                #print(model['word'])
                self.params = model['params']
                #print(self.params)
                self.målkorpus_tot = pd.read_json(model['target'])
                #print(self.målkorpus_tot[:10])
                self.combo_tot = pd.read_json(model['combo'])
                self.mål_docf = pd.read_json(model['target_df'])
                self.combo_docf = pd.read_json(model['combo_df'])
            except:
                print('noe gikk galt')
        return True
    
    def collocations(self, word, after=5, before=5, limit=1000):
        """Find collocations for word in a set of book URNs. Only books at the moment"""
        
        r = requests.post(
            "https://api.nb.no/ngram/urncoll", 
            json={
                'word': word, 
                'urns': self.target_urn,
                'after': after, 
                'before': before, 
                'limit': limit
            }
        )

        temp = pd.DataFrame.from_dict(r.json(), orient='index')
        normalize_corpus_dataframe(temp)
        self.coll[word] = temp.sort_values(by = temp.columns[0], ascending = False)
        return True
    
    def conc(self, word, before=8, after=8, size=10, combo=0):
        
        
        if combo == 0:
            urns = self.target_urn + self.reference_urn
        elif combo == 1:
            urns = self.target_urn
        else:
            urns = self.reference_urn
        if len(urns) > 300:
            urns = list(numpy.random.choice(urns, 300, replace=False))
        return get_urnkonk(word, {'urns':urns, 'before':before, 'after':after, 'limit':size})
    
    def sort_collocations(self, word, comparison = None, exp = 1.0, above = None):
        
        if comparison == None:
            comparison = self.combo_tot[0]
        try:
            res = pd.DataFrame(self.coll[word][0]**exp/comparison)

        except KeyError:
            print('Constructing a collocation for {w} with default parameters.'.format(w=word))
            self.collocations(word)
            res = pd.DataFrame(self.coll[word][0]**exp/comparison)
        if above == None:
        	above = self.lowest
        res = res[self.combo_tot > above]
        return res.sort_values(by = 0, ascending = False)
    
    def search_collocations(self, word, words, comparison = None, exp = 1.0):
        
        if comparison == None:
            comparison = self.combo_tot[0]
        try:
            res = pd.DataFrame(self.coll[word][0]**exp/comparison)
        except KeyError:
            print('Constructing a collocation for {w} with default parameters.'.format(w=word))
            self.collocations(word)
            res = pd.DataFrame(self.coll[word][0]**exp/comparison)
        search_items = list(set(res.index) & set(words))
        return res.transpose()[search_items].transpose().sort_values(by = 0, ascending = False)
    
    def summary(self, head=10):
        info = {
            'parameters':self.params,
            'target_urn':self.target_urn[:head],
            'reference urn':self.reference_urn[:head],
            
        }
        return info

    def search_words(self, words, freq_exp=1.1, doc_exp=1.1):
        if type(words) is str:
            words = [w.strip() for w in words.split()]
        df = self.difference(freq_exp = freq_exp, doc_exp=doc_exp,top=0, aslist=False)
        sub = [w for w in words if w in df.index]
        res = df.transpose()[sub].transpose().sort_values(by=df.columns[0], ascending=False)
        return res
    
    def make_collocation_graph(self, target_word, top = 15, before = 4, after = 4, limit = 1000, exp=1):
        """Make a cascaded network of collocations"""

        self.collocations(target_word, before=before, after=after, limit=limit)
        coll = self.sort_collocations(target_word, exp = exp)
        target_graf = dict()
        edges = []
        for word in coll[:top].index:
            edges.append((target_word, word))
            if word.isalpha():
                self.collocations(word, before=before, after=after,  limit=limit)
                for w in self.sort_collocations(word, exp = exp)[:top].index:
                    if w.isalpha():
                        edges.append((word, w)) 

        target_graph = nx.Graph()
        target_graph.add_edges_from(edges)
        self.coll_graph[target_word] = target_graph
        return target_graph
        
def vekstdiagram(urn, params=None):
    if params is None:
        params = {}

    # if urn is the value of get_urn() it is a list
    # otherwise it just passes
    if type(urn) is list:
        urn = urn[0]
    
    para = params
    para['urn']= urn
    r = requests.post('https://api.nb.no/ngram/vekstdiagram', json = para)
    return pd.DataFrame(r.json())

def plot_book_wordbags(urn, wordbags, window=5000, pr = 100):
    """Generate a diagram of wordbags in book """
    return plot_sammen_vekst(urn, wordbags, window=window, pr=pr)
    

def plot_sammen_vekst(urn, ordlister, window=5000, pr = 100):
    """Plott alle seriene sammen"""
    rammer = []
    c = dict()
    if isinstance(ordlister, list):
        if isinstance(ordlister[0], list):
            for l in ordlister:
                if l != []:
                    c[l[0]] = l
        else:
            c[ordlister[0]] = ordlister
    else:
        c = ordlister
    for key in c:
        vekst = vekstdiagram(urn, params = {'words': c[key], 'window':window, 'pr': pr} )
        vekst.columns = [key]
        rammer.append(vekst)
    return pd.concat(rammer)

def spurious_names(n=300):
    topwords = totals(n)
    Removals = [x.capitalize() for x in topwords if x.isalpha()]
    return Removals

def relaterte_ord(word, number = 20, score=False):
    G = make_graph(word)
    res = Counter(nx.eigenvector_centrality(G)).most_common(number) 
    if score == False:
        res = [x[0] for x in res]
    return res


def check_words(urn, ordbag):
    if type(urn) is list:
        urn = urn[0]
    ordliste = get_freq(urn, top=50000, cutoff=0)
    res = Counter()
    for w in ordbag:
        res[w] = ordliste[w]
    for p in res.most_common():
        if p[1] != 0:
            print(p[0], p[1])
        else:
            break
    return True


def nb_ngram(terms, corpus='bok', smooth=3, years=(1810, 2010), mode='relative'):
    df = ngram_conv(get_ngram(terms, corpus=corpus), smooth=smooth, years=years, mode=mode)
    df.index = df.index.astype(int)
    return df

def get_ngram(terms, corpus='avis'):
    req = requests.get(
        "http://www.nb.no/sp_tjenester/beta/ngram_1/ngram/query?terms={terms}&corpus={corpus}".format(
            terms=terms,
            corpus=corpus
        ))
    if req.status_code == 200:
        res = req.text
    else:
        res = "[]"
    return json.loads(res)

def ngram_conv(ngrams, smooth=1, years=(1810,2013), mode='relative'):
    ngc = {}
    # check if relative frequency or absolute frequency is in question
    if mode.startswith('rel') or mode=='y':
        arg = 'y'
    else:
        arg = 'f'
    for x in ngrams:
        if x != []:
            ngc[x['key']] = {z['x']:z[arg] for z in x['values'] if int(z['x']) <= years[1] and int(z['x']) >= years[0]}
    return pd.DataFrame(ngc).rolling(window=smooth, win_type='triang').mean()


def make_graph(words, lang='nob', cutoff=20, leaves=0):
    """Get galaxy from ngram-database. English and German provided by Google N-gram. 
    Set leaves=1 to get the leaves. Parameter cutoff only works for lang='nob'. 
    Specify English by setting lang='eng' and German by lang='ger'"""
    
    params = dict()
    params['terms'] = words
    params['corpus'] = lang
    params['limit'] = cutoff
    params['leaves'] = leaves
    result = requests.get("https://www.nb.no/sp_tjenester/beta/ngram_1/galaxies/query", params=params)
    G = nx.DiGraph()
    edgelist = []
    if result.status_code == 200:
        graph = json.loads(result.text)
        #print(graph)
        nodes = graph['nodes']
        edges = graph['links']
        for edge in edges:
            edgelist += [(nodes[edge['source']]['name'], nodes[edge['target']]['name'], abs(edge['value']))]
    #print(edgelist)
    G.add_weighted_edges_from(edgelist)

    return G

def urn_concordance(urns = None, word = None, size = 5, before = None, after = None ):
    """ Find a concordance within a corpus as list of URNs. This is a wrapper for get_urnkonk """
    
    # exit if list of urns is empty
    if urns is None or word is None:
        return []
    
    # The URNs may be presented in different ways. 
    urns = pure_urn(urns)
    
    # find values and feed everything to get_urnkonk
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)
    query = {i:values[i] for i in args if values[i] != None and i != 'word'}
    return get_urnkonk(word, query)


def konk(word, urns=None, before=5, after=5):
    if urns == None:
        print('URNer mangler')
        return
    urner = refine_book_urn(words=[word], urns=urns)
    return urn_concordance(word=word, urns = sample(urner, min(20, len(urner))),before = before, after = after)


def concordance(word = None, corpus='bok', author=None, title=None, subtitle=None, lang=None, ddk=None, subject=None,
               yearfrom = None, yearto=None, before=None, after=None, size=5, gender=None, offset=None, kind='html'):
    if word == None:
        return []
    frame = inspect.currentframe()
    args, _, _, values = inspect.getargvalues(frame)
    query = {i:values[i] for i in args if values[i] != None and i != 'word'  and i != 'kind'}
    return get_konk(word, query, kind=kind)

def get_konk(word, params=None, kind='html'):
    if params is None:
        params = {}

    para = params
    para['word']= word

    corpus = 'bok'
    if 'corpus' in para:
        corpus = para['corpus']
    else:
        para['corpus'] = corpus
        
    r = requests.get('https://api.nb.no/ngram/konk', params=para)
    if kind=='html':
        rows = ""
        row_template = ("<tr>"
                        "<td><a href='{urn}?searchText={kw}' target='_'>{urnredux}</a></td>"
                        "<td>{b}</td>"
                        "<td>{w}</td>"
                        "<td style='text-align:left'>{a}</td>"
                        "</tr>\n")
        if corpus == 'bok':
            for x in r.json():
                rows += row_template.format(
                    kw = word,
                    urn=x['urn'],
                    urnredux=','.join([x['author'], x['title'], str(x['year'])]),
                    b=x['before'],
                    w=x['word'],
                    a=x['after'])
        else:
            #print(r.json())
            for x in r.json():
                rows += row_template.format(
                    kw = word,
                    urn=x['urn'],
                    urnredux='-'.join(x['urn'].split('_')[2:6:3]),
                    b=x['before'],
                    w=x['word'],
                    a=x['after'])
        res = "<table>{rows}</table>".format(rows=rows)
        res = HTML(res)
    elif kind == 'json':
        res = r.json()
    else:
        try:
            if corpus == 'bok':
                res = pd.DataFrame(r.json())
                res = res[['urn','author','title','year','before','word','after']]
            else:
                res = pd.DataFrame(r.json())
                res = res[['urn','before','word','after']]
            
        except:
            res= pd.DataFrame()
        #r = r.style.set_properties(subset=['after'],**{'text-align':'left'})
    return res



def konk_to_html(jsonkonk):
    rows = ""
    row_template = ("<tr>"
                    "<td><a href='{urn}' target='_'>{urnredux}</a></td>"
                    "<td>{b}</td>"
                    "<td>{w}</td>"
                    "<td style='text-align:left'>{a}</td>"
                    "</tr>\n")
    for x in jsonkonk:
        rows += row_template.format(
            urn=x['urn'], urnredux=x['urn'], b=x['before'], w=x['word'], a=x['after'])
    res = "<table>{rows}</table>".format(rows=rows)
    return res

def central_characters(graph, n=10):
    res = Counter(nx.degree_centrality(graph)).most_common(n)
    return res

def central_betweenness_characters(graph, n=10):
    res = Counter(nx.betweenness_centrality(graph)).most_common(n)
    return res
    

def get_urnkonk(word, params=None, html=True):
    if params is None:
        params = {}

    para = params
    para['word']= word
    try:
        para['urns'] = pure_urn(para['urns'])
    except:
        print('Parameter urns missing')
    r = requests.post('https://api.nb.no/ngram/urnkonk', json = para)
    if html:
        rows = ""
        for x in r.json():
            rows += """<tr>
                <td>
                    <a href='{urn}?searchText={kw}' target='_blank' style='text-decoration:none'>{urnredux}</a>
                </td>
                <td>{b}</td>
                <td>{w}</td>
                <td style='text-align:left'>{a}</td>
            </tr>\n""".format(kw=word,
                              urn=x['urn'],
                              urnredux="{t}, {f}, {y}".format(t=x['title'], f=x['author'], y=x['year']),
                              b=x['before'],
                              w=x['word'],
                              a=x['after']
                             )
        res = """<table>{rows}</table>""".format(rows=rows)
        res = HTML(res)
    else:
        res = pd.DataFrame(r.json())
        res = res[['urn','before','word','after']]
        #r = r.style.set_properties(subset=['after'],**{'text-align':'left'})
    return res

def frame(something, name = None):
    """Try to make a frame out of something and name columns according to name, which should be a string or a list of strings,
    one for each column. Mismatch in numbers is taken care of."""
    
    if isinstance(something, dict):
        res = pd.DataFrame.from_dict(something, orient='index')
    else:
        res =  pd.DataFrame(something)
    number_of_columns = len(res.columns)
    if name != None:
        if isinstance(name, list):
            if len(name) >= number_of_columns:
                res.columns = name[:number_of_columns]
            else:
                res.columns = name + list(range(len(name), number_of_columns))
        else:
            res.columns = [name] + list(range(1, number_of_columns))
    return res

def frame_sort(frame, by = 0, ascending = False):
    """sort a dataframe, if value of key is a column it will sort by that, otherwise value is
    interpreted as an index into the columns"""
    if by in frame.columns:
        res = frame.sort_values(by = by, ascending = ascending)
    elif isinstance(by, int):
        col = max(by, len(frame.columns) - 1)
        res = frame.sort_values(by = frame.columns[col], ascending = ascending)
    else:
        res = frame.sort_values(by = frame.columns[0], ascending = ascending)
    return res

def get_urns_from_docx(document):
    """Find all URNs specified in a Word document - typically .docx"""
    import sys
    import zipfile
    import re
    from bs4 import BeautifulSoup

    with zipfile.ZipFile(document, 'r') as zfp:
        with zfp.open('word/document.xml') as fp:
            soup = BeautifulSoup(fp.read(), 'xml')

    return re.findall("[0-9]{13}", str(soup))

def get_urns_from_text(document):
    """Find all URNs in a text-file"""
    
    import re

    with open(document) as fp:
        text = fp.read()
    #print(text)
    return re.findall("[0-9]{13}", text)


def get_urns_from_files(mappe, file_type='txt'):
    """Find URNs in files in a folder - specify folder"""
    import os
    froot, _, files = next(os.walk(mappe))
    urns = dict()
    for f in files:
        fn = (os.path.join(froot, f))
        #print(fn)
        if f.endswith('.docx'):
            urns[f] = get_urns_from_docx(fn)
        elif f.endswith('.txt'):
            urns[f] = get_urns_from_text(fn)
    return urns


#======================== Utilities

def xmlpretty(xmls):
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(xmls, features='lxml')
    soup.prettify()
    # '<html>\n <head>\n </head>\n <body>\n  <a href="http://example.com/">\n...'

    print(soup.prettify())

def dewey(dewey):
    r = requests.get("https://api.nb.no:443/dewey/v1/list", params={'class':dewey, 'language':'nob'})
    try:
        ddk = r.json()

        ddc = dict()

        if 'deweyPathList' in ddk:
            for item in ddk['deweyPathList']:
                ddc[str(item['level'])] = [item['classValue'], item['heading']]
    except:
        ddc = []
    return ddc

def metadata_xml(URN, kind='marcxml'):
    if isinstance(URN, int):
        URN = "URN:NBN:no-nb_digibok_{urn}".format(urn=str(URN))
    elif isinstance(URN, str):
        if URN.startswith('URN'):
            URN = URN
        else:
            URN = "URN:NBN:no-nb_digibok_{urn}".format(urn=URN)
    
    r = requests.get("https://api.nb.no:443/catalog/v1/metadata/{urn}/{kind}".format(urn=URN, kind=kind))
    try:
        res = r.text
    except:
        res = ""
    return res


def save_frame_to_excel(df, filename, index = False):
    import os
    if os.path.exists(filename):
        print('Det eksisterer allerede en fil {filename} - velg nytt navn og prøv igjen'.format(filename=filename))
    else:
        df.to_excel(filename, index=index)
    return

def restore_metadata_from_excel(data):
    import os
    df = pd.DataFrame()
    try:
        df = pd.read_excel(data)
        # From excel some stray rows with null values for urn may occur. Drop those.
        indexNames = df[df[df.columns[0]].isnull()].index
        df.drop(indexNames , inplace=True)
        try:
            urn = df.columns[0]
            year = df.columns[2]
            df = df.astype({urn: 'int64', year:'int'})
            df = df.astype({urn: 'str'})
        except:
            True
    except:
        if not os.path.exists(data):
            print('filen {data} ble ikke funnet'.format(data=data))
        else:
            print('noe gikk galt ved import av dataene')
    return df