generate.py

#!/usr/bin/python
# -*- coding:utf-8 -*-

"""Documentation"""

import shutil
import os
import codecs
import shelve
from datetime import datetime

from jinja2 import ChoiceLoader
from jinja2 import Environment
from jinja2 import FileSystemLoader
from jinja2 import PackageLoader
from markdown import Markdown
import pypinyin

# 静态文件路径，默认为`/static/`
# 表示使用flask发布网站时的`http://ip:port/static/`目录
# 也可指定为固定地址的静态文件url，例如："http://192.168.62.47:5000/static/"
# 注意，使用其他域名的静态文件时有可能引起跨域问题
STATIC_ROOT = "/static/"

# Markdown文件读取目录
INPUT_CONTENT = os.path.abspath('.')+"/in/"

# 索引文件
INDEX_DAT = os.path.abspath('.')+"/static/out/index.dat"

# html生成输出目录
OUTPUT_CONTENT = os.path.abspath('.')+"/static/out/"

#     loader=ChoiceLoader([
#         FileSystemLoader('templates'),
#         PackageLoader('flask_bootstrap')
#     ])


# 标签倒排索引
TAG_INVERTED_INDEX = {}
# 作者倒排索引
AUTHOR_INVERTED_INDEX = {}
# 文章索引
ARTICLE_INDEX = {}
# 类型索引
CATEGORIES_INDEX = {}

_MD_FILES = []

_current_file_index = None
_pinyin_names = set()

TAG_HTML_TEMPLATE = u" <a href='/tags/{tag}/' class='tag-index'>{tag}&nbsp</a> "
AUTHOR_HTML_TEMPLATE = u"<a href='/about/' class='tag-index'> {author} </a>"
TITLE_HTML_TEMPLATE = u"<div class='sidebar-module-inset'><h3 class='sidebar-title fa fa-angle-down' style='color:#dddddd;font-size: 16px'>    标题</h3><p>{title_str}</p></div>"


# 原来的方式是使用 flask_bootstrap模板，所以需要PackageLoader，
# 现在已经把bootstrap移动到自己的templates目录中
# loader=ChoiceLoader([
#         PackageLoader('flask_bootstrap', 'templates'),
#         FileSystemLoader("templates")])

env = Environment(
    loader=FileSystemLoader("templates")
)

# env.globals['bootstrap_find_resource'] = app.jinja_env.globals['bootstrap_find_resource']
# 因为不想引入app  所以 使用到的模板不中不能使用url_for，
# 如果一定要使用url_for，可以重新写一份模板引用
# 也可以引入app
# env.globals['url_for'] = app.jinja_env.globals['url_for']


def _reload_global():
    global TAG_INVERTED_INDEX, AUTHOR_INVERTED_INDEX, ARTICLE_INDEX, CATEGORIES_INDEX, \
        _MD_FILES, _current_file_index, _pinyin_names

    TAG_INVERTED_INDEX = {}
    AUTHOR_INVERTED_INDEX = {}
    ARTICLE_INDEX = {}
    CATEGORIES_INDEX = {}
    _MD_FILES = []
    _current_file_index = None
    _pinyin_names = set()


def clean():
    """清理输出文件夹
    """
    if os.path.exists(OUTPUT_CONTENT):
        shutil.rmtree(OUTPUT_CONTENT)


def parse_time(timestamp, pattern="%Y-%m-%d %H:%M:%S"):
    """解析时间
    """
    return datetime.fromtimestamp(timestamp).strftime(pattern)


def str2pinyin(hans, style=pypinyin.FIRST_LETTER):
    """字符串转拼音，默认只获取首字母
    """
    pinyin_str = pypinyin.slug(hans, style=style, separator="")
    num = 2
    while pinyin_str in _pinyin_names:
        pinyin_str += str(num)
        num += 1
    return pinyin_str


def dump_index():
    """持久化索引信息
    """
    dat = shelve.open(INDEX_DAT)
    # dat = shelve.open(INDEX_DAT, writeback=True, protocol=4, flag='c')
    dat["tag_inverted_index"] = TAG_INVERTED_INDEX
    dat["article_index"] = ARTICLE_INDEX
    dat["author_inverted_index"] = AUTHOR_INVERTED_INDEX
    dat["category_index"] = CATEGORIES_INDEX
    # dat.close()
    dat.sync()


def index_tags(tags, fid):
    """为标签倒排索引添加标签
    """
    for tag in tags:
        if tag in TAG_INVERTED_INDEX:
            TAG_INVERTED_INDEX[tag].append(fid)
        else:
            TAG_INVERTED_INDEX[tag] = [fid]


def index_authors(authors, fid):
    """为作者倒排索引添加作者
    """
    for author in authors:
        if author in AUTHOR_INVERTED_INDEX:
            AUTHOR_INVERTED_INDEX[author].append(fid)
        else:
            AUTHOR_INVERTED_INDEX[author] = [fid]


def index_categories(categories, fid):
    """为作者倒排索引添加作者
    """
    for category in categories:
        if category in CATEGORIES_INDEX:
            CATEGORIES_INDEX[category].append(fid)
        else:
            CATEGORIES_INDEX[category] = [fid]


def create_index(filename, meta):
    """创建索引信息
    :param filename: 文件从INPUT_CONTENT开始的全路径
    :param meta:
    :type meta: dict
    :return:
    """

    filename = codecs.decode(filename.encode('utf-8'), "gb2312")
    index_tags(meta.get("tags", []), _current_file_index)
    index_authors(meta.get("authors", []), _current_file_index)
    index_categories(meta.get("categories", []), _current_file_index)

    title = meta.get("title", [""])[0]
    if title == "":
        title = os.path.splitext(os.path.basename(filename))[0]

    publish_dates = meta.get("publish_date", [])
    if len(publish_dates) == 0:
        publish_date = parse_time(os.path.getctime(filename), "%Y-%m-%d")
    else:
        publish_date = publish_dates[0]

    ARTICLE_INDEX[_current_file_index] = {
        "filename": filename,
        "modify_time": parse_time(os.path.getmtime(filename)),
        "title": title,
        "summary": meta.get("summary", [u""])[0],
        "authors": meta.get("authors", [u"匿名"]),
        "publish_date": publish_date,
        "tags": meta.get("tags", []),
        "categories": meta.get("categories", [u"未分类"])
    }


def get_out_dir(md_file):
    """获取md文件的输出路径
    :param md_file:
    :return:
    """

    return os.path.join(OUTPUT_CONTENT, _current_file_index + ".html")


def save_html(out_path, html):
    """保存html至文件
    :param out_path:
    :param html:
    :return:
    """
    base_folder = os.path.dirname(out_path)
    if not os.path.exists(base_folder):
        os.makedirs(base_folder)
    with codecs.open(out_path, "w+", "utf-8") as f:
        f.write(html)


def render_tags_html(tags):
    """渲染tags的html
    """
    tags_html = ""
    for tag in tags:
        tags_html += TAG_HTML_TEMPLATE.format(tag=tag)
    return tags_html


def render_authors_html(authors):
    """渲染作者html
    """
    authors_html = ""
    for author in authors:
        authors_html += AUTHOR_HTML_TEMPLATE.format(author=author)
    return authors_html


def render_title_html(title):
    """渲染标题html
    """
    title_html = ""
    if title.strip() != "":
        title_html = TITLE_HTML_TEMPLATE.format(title_str=title)
    return title_html


"""
传入文件名字
"""


def render(md_file, site_id):
    """渲染html页面
    :param md_file:
    :return:
    """
    with codecs.open(md_file, "r", "utf-8") as f:
        text = f.read()
        md = Markdown(
            extensions=[
                "fenced_code",
                "attr_list",
                "codehilite(css_class=highlight,linenums=None)",
                "meta",
                "admonition",
                "tables",
                "toc",
                "wikilinks",
            ],
        )
        html = md.convert(text)
        meta = md.Meta if hasattr(md, "Meta") else {}
        toc = md.toc if hasattr(md, "toc") else ""
        create_index(md_file, meta)
        article_url = '/posts/' + site_id
        template = env.get_template("posts/article_base.html")
        text = template.render(
            blog_content=html,
            static_root=STATIC_ROOT,
            site_id=site_id,
            article_url=article_url,
            title=ARTICLE_INDEX[_current_file_index].get("title"),
            title_html=render_title_html(ARTICLE_INDEX[_current_file_index].get("title")),
            summary=ARTICLE_INDEX[_current_file_index].get("summary", ""),
            authors=render_authors_html(ARTICLE_INDEX[_current_file_index].get("authors")),
            tags=render_tags_html(ARTICLE_INDEX[_current_file_index].get("tags")),
            toc=toc,
        )

    return text


def gen(md_file_path, site_id):
    """将markdown生成html文件
    :param site_id:
    :param md_file_path:
    """
    out_path = get_out_dir(md_file_path)
    html = render(md_file_path, site_id)
    save_html(out_path, html)


def scan_md():
    """扫描md文件
    """
    global _current_file_index
    for f in _MD_FILES:
        file_base_name = os.path.splitext(os.path.basename(f))[0]
        _current_file_index = str2pinyin(
            codecs.decode(file_base_name.encode('utf-8'), "gb2312")
        )
        _pinyin_names.add(_current_file_index)
        gen(f, _current_file_index)


def load_md_files(folder):
    """从指定文件夹载入Markdown文件
    """
    global _MD_FILES
    for root, dirs, files in os.walk(folder):
        for f in files:
            if os.path.splitext(f)[1].lower() == ".md":
                _MD_FILES.append(os.path.join(root, f))


def generate():
    _reload_global()
    clean()
    load_md_files(INPUT_CONTENT)
    scan_md()
    dump_index()
    from utils.helper import IndexData
    IndexData.reload_index_data()
    pass

if __name__ == "__main__":
    generate()
    pass