diff --git a/.gitignore b/.gitignore
index d8c0f77..d1027f6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@ tmp/
download/
log/
-production.yml
\ No newline at end of file
+production.yml
+sitemap.xml
\ No newline at end of file
diff --git a/HISTORY.md b/HISTORY.md
index 052e150..96f0d18 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,3 +1,64 @@
+# Release v1.6.0
+
+
+- author: Yves Vindevogel (vindevoy)
+- date: 2020-05-17
+
+## New Features
+
+- URL rerouting has been implemented on the level of the posts, pages and tags. This means that you can change the URL of the publication and through the rerouting, it will map the correct document(s). This can be used to make URLs more readable.
+- Support for draft pages has been included. If you are writing a new post or page, but you don't want it to be published yet, you can put it as draft in the meta data. You can decide through the environment.yml if you want to show drafts or not (development vs. production).
+
+## Enhancements
+
+- A sitemap.xml file can be generated from the Makefile. This allows Google to crawl the website better.
+- The application controller class now uses htmlmin to remove all the comment in the HTML, remove double spaces and so on. This results in a smaller HTML going over the wire, but it also results in a better memory footprint for caching.
+
+## Other
+
+- A Facebook page has been created for marketing reasons. Why else ?
+
+### Github
+
+For more information on this release, see the issues for this milestone:
+
+- [https://github.com/vindevoy/cherryblog/milestone/15](https://github.com/vindevoy/cherryblog/milestone/15)
+
+
+# Release v1.5.2
+
+
+- author: Yves Vindevogel (vindevoy)
+- date: 2020-05-08
+
+## Hotfix
+
+- There was a runtime-error on the posts and pages if they had to tags. This resulted in a TypeError, and in code the KeyError was captured.
+
+### Github
+
+For more information on this release, see the issues for this milestone:
+
+- [https://github.com/vindevoy/cherryblog/milestone/15](https://github.com/vindevoy/cherryblog/milestone/15)
+
+
+# Release v1.5.1
+
+
+- author: Yves Vindevogel (vindevoy)
+- date: 2020-05-07
+
+## Bug Fixes
+
+- There was a problem with the search results. The font weight (bold) of found text in the search results is now case-insensitive.
+
+### Github
+
+For more information on this release, see the issues for this milestone:
+
+- [https://github.com/vindevoy/cherryblog/milestone/14](https://github.com/vindevoy/cherryblog/milestone/14)
+
+
# Release v1.5.0
diff --git a/Makefile b/Makefile
index ca60ab3..3c75365 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,12 @@
###
#
-# Version: 1.2.1
-# Date: 2020-04-25
+# Version: 1.3.0
+# Date: 2020-05-07
# Author: Yves Vindevogel (vindevoy)
#
-# Fixes:
-# - all logging is now in application.log instead of ENVIRONMENT.log (settings determine it for the run)
-# - better history generation
+# Features:
+# - dependency for django-htmlmin
+# - create sitemap.xml
#
###
@@ -59,6 +59,8 @@ dependencies:
@pip3 install jinja2
@pip3 install pyyaml
@pip3 install markdown
+ @pip3 install django-htmlmin
+
@echo '[OK] Dependencies in Python installed'
@@ -99,11 +101,21 @@ history:
@echo '[OK] history copied to pages'
-develop:
+sitemap:
+ @mkdir -p ./src/data/sitemap
+ @python3 ./src/application/sitemap.py
+
+
+ @echo '[OK] sitemap.xml created'
+
+google: sitemap
+ @curl http://www.google.com/ping?sitemap=https://cherryblog.org/sitemap.xml > /dev/null
+
+develop: sitemap
@mkdir -p ./log
@python3 ./src/application/main.py 2>&1 | tee ./log/application.log
-production:
+production: sitemap
@mkdir -p /var/log/cherryblog
@python3 ./src/application/main.py --env production 2>&1 | tee /var/log/cherryblog/application.log &
@@ -113,6 +125,14 @@ stop:
###
#
+# Version: 1.2.1
+# Date: 2020-04-25
+# Author: Yves Vindevogel (vindevoy)
+#
+# Fixes:
+# - all logging is now in application.log instead of ENVIRONMENT.log (settings determine it for the run)
+# - better history generation
+#
# Version: 1.2.0
# Date: 2020-04-11
# Author: Yves Vindevogel (vindevoy)
diff --git a/README.md b/README.md
index c45dbe4..9b3279f 100644
--- a/README.md
+++ b/README.md
@@ -9,6 +9,9 @@ CherryBlog implements a very simple blogging website.
- Pages: the extra static pages like the 'about'
- Tags: each post has one or multiple tags
- Search: word based search in posts and pages
+- Caching: in memory caching of data and content
+- Drafts: drafts are pages that are not published yet, only visible in development
+- URL rewriting: the final URL is not necessarily the path to the file
At this moment, it has the basic news of the project where each article is a post about the new version that was released.
diff --git a/src/application/common/options.py b/src/application/common/options.py
index 306d856..90dc6d0 100644
--- a/src/application/common/options.py
+++ b/src/application/common/options.py
@@ -2,11 +2,12 @@
#
# Full history: see below
#
-# Version: 2.2.0
-# Date: 2020-04-26
+# Version: 2.3.0
+# Date: 2020-05-07
# Author: Yves Vindevogel (vindevoy)
#
-# Caching enabled or not
+# Features:
+# - Support for drafts
#
###
@@ -29,6 +30,7 @@ class Options(metaclass=Singleton):
daemon = False
caching = True
meta_content_separator = ''
+ include_drafts = False
# SSL settings (for instance for LetsEncrypt)
use_ssl = False
@@ -46,6 +48,12 @@ class Options(metaclass=Singleton):
###
#
+# Version: 2.2.0
+# Date: 2020-04-26
+# Author: Yves Vindevogel (vindevoy)
+#
+# Caching enabled or not
+#
# Version: 2.1.0
# Date: 2020-04-23
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/controller/application.py b/src/application/controller/application.py
index 686c6b8..080a5de 100644
--- a/src/application/controller/application.py
+++ b/src/application/controller/application.py
@@ -2,11 +2,12 @@
#
# Full history: see below
#
-# Version: 2.3.0
-# Date: 2020-04-26
+# Version: 2.5.0
+# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
-# Caching enabled or not
+# Features:
+# - Remapping of URLs to documents
#
###
@@ -14,11 +15,13 @@
import logging
from datetime import datetime
+from htmlmin.minify import html_minify
from common.options import Options
from common.singleton import Singleton
from controller.data_loader import DataLoader
from controller.page_cacher import PageCacher
+from controller.remapper import Remapper
from view.templateloader import TemplateLoader
@@ -31,11 +34,12 @@ def __init__(self):
@cherrypy.expose
def index(self, page_index=1, **_):
- request = '/index/{0}'.format(page_index)
start = datetime.now()
+ request = '/index/{0}'.format(page_index)
+
if Options().caching and PageCacher().cached_already(request):
- rendered = PageCacher().get_cached(request)
+ minified = PageCacher().get_cached(request)
else:
data = DataLoader().index_data(page_index)
@@ -43,100 +47,136 @@ def index(self, page_index=1, **_):
template = TemplateLoader().get_template('screen_index.html')
rendered = template.render(data=data)
+ minified = html_minify(rendered)
if Options().caching:
- PageCacher().cache(request, rendered)
+ PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
- return rendered
+ return minified
@cherrypy.expose
def pages(self, page, **_):
- request = '/pages/{0}'.format(page)
start = datetime.now()
+ request = '/pages/{0}'.format(page)
+ self.__logger.debug('pages - request: {0}'.format(request))
+
if Options().caching and PageCacher().cached_already(request):
- rendered = PageCacher().get_cached(request)
+ minified = PageCacher().get_cached(request)
else:
+ remapped = Remapper().remap_url(request)
+ self.__logger.debug('pages - remapped: {0}'.format(remapped))
+
+ if request != remapped:
+ page = remapped.split('/')[2]
+ self.__logger.debug('pages - page: {0}'.format(page))
+
# page on the URL: http://www.yoursite.ext/pages/page
data = DataLoader().pages_data(page)
- data['url'] = request
+ data['url'] = remapped
template = TemplateLoader().get_template('screen_page.html')
rendered = template.render(data=data)
+ minified = html_minify(rendered)
if Options().caching:
- PageCacher().cache(request, rendered)
+ PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
- return rendered
+ return minified
@cherrypy.expose
def posts(self, post, **_):
- request = '/posts/{0}'.format(post)
start = datetime.now()
+ request = '/posts/{0}'.format(post)
+ self.__logger.debug('posts - request: {0}'.format(request))
+
if Options().caching and PageCacher().cached_already(request):
- rendered = PageCacher().get_cached(request)
+ minified = PageCacher().get_cached(request)
else:
+ remapped = Remapper().remap_url(request)
+ self.__logger.debug('posts - remapped: {0}'.format(remapped))
+
+ if request != remapped:
+ post = remapped.split('/')[2]
+ self.__logger.debug('posts - post: {0}'.format(post))
+
data = DataLoader().posts_data(post)
- data['url'] = request
+ data['url'] = remapped
template = TemplateLoader().get_template('screen_post.html')
rendered = template.render(data=data)
+ minified = html_minify(rendered)
if Options().caching:
- PageCacher().cache(request, rendered)
+ PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
- return rendered
+ return minified
@cherrypy.expose
def tags(self, tag, page_index=1, **_):
- request = '/tags/{0}/{1}'.format(tag, page_index)
start = datetime.now()
+ request = '/tags/{0}/{1}'.format(tag, page_index)
+ self.__logger.debug('tags - tag: {0}'.format(tag))
+
if Options().caching and PageCacher().cached_already(request):
- rendered = PageCacher().get_cached(request)
+ minified = PageCacher().get_cached(request)
else:
+ short_request = 'tags/{0}'.format(tag)
+ self.__logger.debug('tags - short_request: {0}'.format(short_request))
+
+ remapped = Remapper().remap_url(short_request)
+ self.__logger.debug('tags - remapped: {0}'.format(remapped))
+
+ if short_request != remapped:
+ tag = remapped.split('/')[2]
+ self.__logger.debug('tags - tag: {0}'.format(tag))
+
data = DataLoader().tags_data(tag, page_index)
- data['url'] = request
+ data['url'] = remapped
template = TemplateLoader().get_template('screen_tag.html')
rendered = template.render(data=data)
+ minified = html_minify(rendered)
if Options().caching:
- PageCacher().cache(request, rendered)
+ PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
- return rendered
+ return minified
@cherrypy.expose
- def search(self, page_index=1, query='', **_):
- request = '/search/{0}/{1}'.format(page_index, query)
+ def search(self, query='', page_index=1, **_):
start = datetime.now()
+ request = '/search/{0}/{1}'.format(query, page_index)
+
data = DataLoader().search_data(query, page_index)
data['url'] = request
template = TemplateLoader().get_template('screen_search.html')
rendered = template.render(data=data)
+ minified = html_minify(rendered)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
- return rendered
+ return minified
# @cherrypy.expose
# def print_page(self, page, **_):
@@ -160,6 +200,18 @@ def search(self, page_index=1, query='', **_):
###
#
+# Version: 2.4.0
+# Date: 2020-05-07
+# Author: Yves Vindevogel (vindevoy)
+#
+# Using HTML minify
+#
+# Version: 2.3.0
+# Date: 2020-04-26
+# Author: Yves Vindevogel (vindevoy)
+#
+# Caching enabled or not
+#
# Version: 2.2.0
# Date: 2020-04-22
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/controller/data_cacher.py b/src/application/controller/data_cacher.py
index 5dc7ced..014eb54 100644
--- a/src/application/controller/data_cacher.py
+++ b/src/application/controller/data_cacher.py
@@ -1,3 +1,13 @@
+###
+#
+# Version: 1.0.0
+# Date: 2020-04-17
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# Simple data caching
+#
+###
import logging
diff --git a/src/application/controller/data_loader.py b/src/application/controller/data_loader.py
index 137ebed..f3780e4 100644
--- a/src/application/controller/data_loader.py
+++ b/src/application/controller/data_loader.py
@@ -2,12 +2,13 @@
#
# Full history: see below
#
-# Version: 1.3.0
-# Date: 2020-04-26
+# Version: 1.4.0
+# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
# Features:
-# - Caching enabled or not
+# - Support for drafts
+# - Remapping of URLs to documents
#
###
@@ -20,6 +21,7 @@
from model.i8n import I8N
from model.important_news import ImportantNews
from model.index import Index
+from model.mapping import Mapping
from model.pages import Pages
from model.posts import Posts
from model.search import Search
@@ -75,6 +77,7 @@ def common_data(self):
'settings': self.global_settings,
'tags_list': self.tags_list,
'tags_list_count': self.tags_list_count,
+ 'tags_skip_list': self.tags_skip_list,
'main_menu': self.index_main_menu,
'footer_menu': self.index_footer_menu,
'important_news': self.important_news_data,
@@ -112,7 +115,12 @@ def index_data(self, page_index):
return DataCacher().get_cached(key)
common = self.common_data
- data, _ = Index().data(page_index, self.posts_directory, self.posts_count)
+ if Options().include_drafts:
+ posts = self.posts_files
+ else:
+ posts = self.posts_published
+
+ data, _ = Index().data(page_index, posts)
# We don't care yet about the introduction content
combined = self.__combine(common, data)
@@ -134,6 +142,13 @@ def index_spotlight_posts(self):
def index_highlight_posts(self):
return self.__get_data('index_highlight_posts', Index(), 'highlight_posts')
+ # mapping
+ def mapping_incoming(self):
+ return self.__get_data('mapping_incoming', Mapping(), 'incoming')
+
+ def mapping_outgoing(self):
+ return self.__get_data('mapping_outgoing', Mapping(), 'outgoing')
+
# pages
@property
def pages_directory(self):
@@ -143,10 +158,18 @@ def pages_directory(self):
def pages_files(self):
return self.__get_data('pages_files', Pages(), 'files')
+ @property
+ def pages_published(self):
+ return self.__get_data('pages_files', Pages(), 'files_published')
+
@property
def pages_count(self):
return self.__get_data('pages_count', Pages(), 'count')
+ @property
+ def pages_count_published(self):
+ return self.__get_data('pages_count', Pages(), 'count_published')
+
def pages_data(self, page):
key = '/pages/{0}'.format(page)
@@ -173,10 +196,18 @@ def posts_directory(self):
def posts_files(self):
return self.__get_data('posts_files', Posts(), 'files')
+ @property
+ def posts_published(self):
+ return self.__get_data('posts_files_published', Posts(), 'files_published')
+
@property
def posts_count(self):
return self.__get_data('posts_count', Posts(), 'count')
+ @property
+ def posts_count_published(self):
+ return self.__get_data('posts_count', Posts(), 'count_published')
+
def posts_data(self, post):
key = '/posts/{0}'.format(post)
@@ -199,10 +230,17 @@ def posts_data(self, post):
def search_data(self, query, page_index):
search_base = []
- for page in self.pages_files:
+ if Options().include_drafts:
+ pages = self.pages_files
+ posts = self.posts_files
+ else:
+ pages = self.pages_published
+ posts = self.posts_published
+
+ for page in pages:
search_base.append(page)
- for post in self.posts_files:
+ for post in posts:
search_base.append(post)
common = self.common_data
@@ -225,7 +263,12 @@ def tags_posts_count(self, tag):
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
- data = Tags().count_posts(self.posts_directory, tag)
+ if Options().include_drafts:
+ posts = self.posts_files
+ else:
+ posts = self.posts_published
+
+ data = Tags().count_posts(posts, tag)
if Options().caching:
DataCacher().cache(key, data)
@@ -239,7 +282,12 @@ def tags_list(self):
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
- tags_list = Tags().list(self.posts_directory)
+ if Options().include_drafts:
+ posts = self.posts_files
+ else:
+ posts = self.posts_published
+
+ tags_list = Tags().list(posts)
if Options().caching:
DataCacher().cache(key, tags_list)
@@ -270,8 +318,13 @@ def tags_data(self, tag, page_index):
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
+ if Options().include_drafts:
+ posts = self.posts_files
+ else:
+ posts = self.posts_published
+
common = self.common_data
- data = Tags().data(self.posts_directory, tag, page_index, self.index_max_posts, self.tags_posts_count(tag))
+ data = Tags().data(posts, tag, page_index, self.index_max_posts, self.tags_posts_count(tag))
combined = self.__combine(common, data)
if Options().caching:
@@ -281,6 +334,13 @@ def tags_data(self, tag, page_index):
###
#
+# Version: 1.3.0
+# Date: 2020-04-26
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# - Caching enabled or not
+#
# Version: 1.2.0
# Date: 2020-04-17
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/controller/page_cacher.py b/src/application/controller/page_cacher.py
index 900f8af..7781ab4 100644
--- a/src/application/controller/page_cacher.py
+++ b/src/application/controller/page_cacher.py
@@ -33,4 +33,3 @@ def get_cached(self, key):
def cache(self, key, data):
self.__logger.info('Caching {0}'.format(key))
self.__cached_pages[key] = data
-
diff --git a/src/application/controller/remapper.py b/src/application/controller/remapper.py
new file mode 100644
index 0000000..415f698
--- /dev/null
+++ b/src/application/controller/remapper.py
@@ -0,0 +1,48 @@
+###
+#
+# Version: 1.0.0
+# Date: 2020-05-08
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# Remapping of incoming requests to content documents
+# Remapping of content to outgoing URLs
+#
+###
+
+import logging
+
+from common.options import Options
+from common.singleton import Singleton
+
+
+class Remapper(metaclass=Singleton):
+ outgoing_content = None
+ incoming_content = None
+
+ __logger = None
+
+ def __init__(self):
+ self.__logger = logging.getLogger('REMAPPER')
+ self.__logger.setLevel(Options().default_logging_level)
+
+ def remap_document(self, doc):
+ self.__logger.debug('remap_document - doc: {0}'.format(doc))
+
+ return self.__remap(doc, self.outgoing_content, 'OUT')
+
+ def remap_url(self, url):
+ self.__logger.debug('remap_url - url: {0}'.format(url))
+
+ return self.__remap(url, self.incoming_content, 'IN')
+
+ def __remap(self, mapping, mappings, way):
+ self.__logger.debug('__remap - mapping: {0}'.format(mapping))
+ self.__logger.debug('__remap - mappings: {0}'.format(mappings))
+
+ if mapping in mappings:
+ mapped = mappings[mapping]
+ self.__logger.debug('{0}: {1} -> {2}'.format(way, mapping, mapped))
+ return mapped
+ else:
+ return mapping
diff --git a/src/application/controller/settings_loader.py b/src/application/controller/settings_loader.py
index 48e6c0a..8b68a76 100644
--- a/src/application/controller/settings_loader.py
+++ b/src/application/controller/settings_loader.py
@@ -2,12 +2,12 @@
#
# Full history: see below
#
-# Version: 1.5.0
-# Date: 2020-04-28
+# Version: 1.6.0
+# Date: 2020-05-07
# Author: Yves Vindevogel (vindevoy)
#
# Features:
-# Handling missing keys and using defaults
+# - Support for drafts
#
###
@@ -142,6 +142,11 @@ def __option_settings(settings_yaml):
except KeyError:
Options().meta_content_separator = '__________'
+ try:
+ Options().include_drafts = settings_yaml['content']['include_drafts']
+ except KeyError:
+ Options().include_drafts = False
+
try:
Options().daemon = settings_yaml['engine']['daemon']
except KeyError:
@@ -190,6 +195,13 @@ def __option_settings(settings_yaml):
###
#
+# Version: 1.5.0
+# Date: 2020-04-28
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# Handling missing keys and using defaults
+#
# Version: 1.4.0
# Date: 2020-04-23
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/main.py b/src/application/main.py
index 0241757..421da0c 100644
--- a/src/application/main.py
+++ b/src/application/main.py
@@ -25,6 +25,7 @@
from controller.application import Application
from controller.data_loader import DataLoader
from controller.logging_loader import LoggingLoader
+from controller.remapper import Remapper
from controller.settings_loader import SettingsLoader
__application = 'CherryBlog'
@@ -78,6 +79,10 @@
logger.info('Default date input format set to \'{0}\'.'.format(DateTimeSupport().input_format))
logger.info('Default date output format set to \'{0}\'.'.format(DateTimeSupport().output_format))
+ # Loading the remapping
+ Remapper().outgoing_content = DataLoader().mapping_outgoing()
+ Remapper().incoming_content = DataLoader().mapping_incoming()
+
if Options().use_ssl:
# vindevoy - 2020-04-25
# You must use the builtin SSL option. If you use pyOpenSLL you get the error below (see history)
@@ -200,4 +205,4 @@
# raise WantWriteError()
# OpenSSL.SSL.WantWriteError
#
-###
\ No newline at end of file
+###
diff --git a/src/application/model/index.py b/src/application/model/index.py
index b1f9f97..717bce3 100644
--- a/src/application/model/index.py
+++ b/src/application/model/index.py
@@ -2,18 +2,18 @@
#
# Full history: see below
#
-# Version: 1.3.0
-# Date: 2020-05-01
+# Version: 1.4.0
+# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
# Features:
-# - Rewrite date format
+# - Support for drafts
+# - Remapping of URLs to documents
#
###
import logging
import math
-import os
from pathlib import Path
@@ -21,6 +21,7 @@
from common.datetime_support import DateTimeSupport
from common.options import Options
from common.singleton import Singleton
+from controller.remapper import Remapper
class Index(metaclass=Singleton):
@@ -73,7 +74,7 @@ def footer_menu(self):
return content
- def data(self, page_index, posts_dir, total_posts):
+ def data(self, page_index, published_posts):
self.__logger.debug('data - page_index: {0}'.format(page_index))
data = {}
@@ -108,46 +109,48 @@ def data(self, page_index, posts_dir, total_posts):
skip_entries = (int(page_index) - 1) * max_entries
self.__logger.debug('data - skip_entries: {0}'.format(skip_entries))
- try:
- for file in sorted(os.listdir(posts_dir), reverse=True):
- count_entries += 1
+ for entry in published_posts:
+ directory = entry['directory']
+ file = entry['file']
+ count_entries += 1
- # We count the entries, but for pages 2 and more, you don't show them
- if skip_entries >= count_entries:
- continue
+ # We count the entries, but for pages 2 and more, you don't show them
+ if skip_entries >= count_entries:
+ continue
- post, _, post['content'] = Content().read_content(posts_dir, file)
+ post, _, post['content'] = Content().read_content(directory, file)
- stem = Path(file).stem
- post['url'] = stem
- post['date'] = DateTimeSupport().rewrite_date(post['date'])
+ stem = Path(file).stem
+ url = '/posts/{0}'.format(stem)
+ url = Remapper().remap_document(url)
- self.__logger.debug('data - post: {0}'.format(post))
+ post['url'] = url
+ post['date'] = DateTimeSupport().rewrite_date(post['date'])
- if page_index == 1:
- if count_entries <= spotlight_entries:
- self.__logger.debug('data - post added to spotlight_posts.')
- data['spotlight_posts'].append(post)
+ self.__logger.debug('data - post: {0}'.format(post))
- if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries):
- self.__logger.debug('data - post added to highlight_posts.')
- data['highlight_posts'].append(post)
+ if page_index == 1:
+ if count_entries <= spotlight_entries:
+ self.__logger.debug('data - post added to spotlight_posts.')
+ data['spotlight_posts'].append(post)
- if count_entries > (spotlight_entries + highlight_entries):
- self.__logger.debug('data - post added to (standard) posts.')
- data['posts'].append(post)
+ if spotlight_entries < count_entries <= (spotlight_entries + highlight_entries):
+ self.__logger.debug('data - post added to highlight_posts.')
+ data['highlight_posts'].append(post)
- else:
+ if count_entries > (spotlight_entries + highlight_entries):
self.__logger.debug('data - post added to (standard) posts.')
data['posts'].append(post)
- if count_entries == (max_entries + skip_entries):
- self.__logger.debug('data - enough posts for this index page.')
- break
- except FileNotFoundError:
- self.__logger.warning('COULD NOT FIND THE POSTS DIRECTORY {0}'.format(posts_dir))
- pass
+ else:
+ self.__logger.debug('data - post added to (standard) posts.')
+ data['posts'].append(post)
+ if count_entries == (max_entries + skip_entries):
+ self.__logger.debug('data - enough posts for this index page.')
+ break
+
+ total_posts = len(published_posts)
total_index_pages = math.ceil(total_posts / max_entries)
self.__logger.debug('data - total_posts: {0}'.format(total_posts))
self.__logger.debug('data - total_index_pages: {0}'.format(total_index_pages))
@@ -165,6 +168,13 @@ def data(self, page_index, posts_dir, total_posts):
###
#
+# Version: 1.3.0
+# Date: 2020-05-01
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# - Rewrite date format
+#
# Version: 1.2.0
# Date: 2020-04-17
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/model/mapping.py b/src/application/model/mapping.py
new file mode 100644
index 0000000..b78650f
--- /dev/null
+++ b/src/application/model/mapping.py
@@ -0,0 +1,63 @@
+###
+#
+# Full history: see below
+#
+# Version: 1.0.0
+# Date: 2020-05-08
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# - Mapping of URLs to physical files
+#
+###
+
+import logging
+
+from common.options import Options
+from common.content import Content
+from common.singleton import Singleton
+
+
+class Mapping(metaclass=Singleton):
+ __base_dir = 'mapping'
+ __logger = None
+
+ incoming = None
+ outgoing = None
+
+ def __init__(self):
+ self.__logger = logging.getLogger('MODEL.MAPPING')
+ self.__logger.setLevel(Options().default_logging_level)
+
+ self.__data()
+
+ def __data(self):
+ content = Content().load_data_settings_yaml(self.__base_dir)
+ self.__logger.debug('__data - content: {0}'.format(content))
+
+ incoming = {}
+ outgoing = {}
+
+ for mapping in content['content']:
+ self.__logger.debug('__data - mapping: {0}'.format(mapping))
+
+ target = mapping['target']
+ source = mapping['source']
+
+ if target[0:1] != '/':
+ target = '/{0}'.format(target)
+
+ if source[0:1] != '/':
+ source = '/{0}'.format(source)
+
+ self.__logger.debug('__data - target: {0}'.format(target))
+ self.__logger.debug('__data - source: {0}'.format(source))
+
+ incoming[target] = source
+ self.__logger.info('mapping incoming URL \'{0}\' to source \'{1}\''.format(target, source))
+
+ outgoing[source] = target
+ self.__logger.info('mapping outgoing source \'{0}\' to URL \'{1}\''.format(source, target))
+
+ self.incoming = incoming
+ self.outgoing = outgoing
diff --git a/src/application/model/pages.py b/src/application/model/pages.py
index 39db26f..530af3d 100644
--- a/src/application/model/pages.py
+++ b/src/application/model/pages.py
@@ -2,12 +2,12 @@
#
# Full history: see below
#
-# Version: 1.3.1
+# Version: 1.4.0
# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
-# Hotfix:
-# - Page with no tags returns a TypeError and not a KeyError
+# Features:
+# - Support for drafts
#
###
@@ -50,6 +50,29 @@ def files(self):
return files
+ @property
+ def files_published(self):
+ directory = self.directory
+ files = []
+
+ for entry in self.files:
+ file = entry['file']
+ meta, _, _ = Content().read_content(directory, file) # only need the meta data
+
+ try:
+ value = meta['draft']
+ draft = bool(value)
+
+ except KeyError:
+ draft = False
+
+ self.__logger.debug('published_files - {0}/{1} is a draft: {2}'.format(directory, file, draft))
+
+ if not draft:
+ files.append(entry)
+
+ return files
+
@property
def count(self):
try:
@@ -62,6 +85,13 @@ def count(self):
return count
+ @property
+ def count_published(self):
+ count = len(self.files_published)
+ self.__logger.debug('count_published - count: {0}'.format(count))
+
+ return count
+
def data(self, page, skip_tags):
self.__logger.debug('data - page: {0}'.format(page))
@@ -101,6 +131,13 @@ def data(self, page, skip_tags):
###
#
+# Version: 1.3.1
+# Date: 2020-05-08
+# Author: Yves Vindevogel (vindevoy)
+#
+# Hotfix:
+# - Page with no tags returns a TypeError and not a KeyError
+#
# Version: 1.3.0
# Date: 2020-05-01
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/model/posts.py b/src/application/model/posts.py
index 2d3bbb6..af80845 100644
--- a/src/application/model/posts.py
+++ b/src/application/model/posts.py
@@ -2,12 +2,12 @@
#
# Full history: see below
#
-# Version: 1.3.1
+# Version: 1.4.0
# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
-# Hotfix:
-# - Page with no tags returns a TypeError and not a KeyError
+# Feature:
+# - Support for drafts
#
###
@@ -51,17 +51,42 @@ def files(self):
return files
@property
- def count(self):
- try:
- count = len(os.listdir(self.directory))
- except FileNotFoundError:
- self.__logger.warning('COULD NOT FIND THE POSTS DIRECTORY {0}'.format(self.directory))
- count = 0
+ def files_published(self):
+ directory = self.directory
+ files = []
+
+ for entry in self.files:
+ file = entry['file']
+ meta, _, _ = Content().read_content(directory, file) # only need the meta data
+
+ try:
+ value = meta['draft']
+ draft = bool(value)
+
+ except KeyError:
+ draft = False
+
+ self.__logger.debug('published_files - {0}/{1} is a draft: {2}'.format(directory, file, draft))
+
+ if not draft:
+ files.append(entry)
+
+ return files
+ @property
+ def count(self):
+ count = len(self.files)
self.__logger.debug('count - count: {0}'.format(count))
return count
+ @property
+ def count_published(self):
+ count = len(self.files_published)
+ self.__logger.debug('count_published - count: {0}'.format(count))
+
+ return count
+
def data(self, post, skip_tags):
self.__logger.debug('data - post: {0}'.format(post))
@@ -101,6 +126,13 @@ def data(self, post, skip_tags):
###
#
+# Version: 1.3.1
+# Date: 2020-05-08
+# Author: Yves Vindevogel (vindevoy)
+#
+# Hotfix:
+# - Page with no tags returns a TypeError and not a KeyError
+#
# Version: 1.3.0
# Date: 2020-05-01
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/model/search.py b/src/application/model/search.py
index 0cc1173..b287625 100644
--- a/src/application/model/search.py
+++ b/src/application/model/search.py
@@ -2,12 +2,13 @@
#
# Full history: see below
#
-# Version: 1.1.1
-# Date: 2020-05-06
+# Version: 1.2.0
+# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
# Fixes:
-# - The replace with the bold tags works, but it does not capture capital letters. Introducing another method.
+# - Support for drafts
+# - Remapping of URLs to documents
#
###
@@ -21,6 +22,7 @@
from common.singleton import Singleton
from controller.data_cacher import DataCacher
+from controller.remapper import Remapper
class Search(metaclass=Singleton):
@@ -56,24 +58,28 @@ def data(self, query, page_index, search_base, max_entries):
file = item['file']
stem = Path(file).stem
+ unmapped = '/{0}/{1}'.format(directory, stem)
+ remapped = Remapper().remap_document(unmapped)
+
+ if unmapped != remapped:
+ stem = remapped.split('/')[2]
+
# adding the stem to the object for the final url
item['stem'] = stem
- url = '/{0}/{1}'.format(directory, stem)
-
- if url in excluded:
+ if remapped in excluded:
self.__logger.debug('data - excluded item: {0}'.format(item))
continue
- cached_already = DataCacher().cached_already(url)
+ cached_already = DataCacher().cached_already(unmapped)
if cached_already:
- self.__logger.debug('data - cached: {0}'.format(url))
+ self.__logger.debug('data - cached: {0}'.format(unmapped))
- meta = DataCacher().get_cached('{0}/meta'.format(url))
- content = DataCacher().get_cached('{0}/content'.format(url))
+ meta = DataCacher().get_cached('{0}/meta'.format(unmapped))
+ content = DataCacher().get_cached('{0}/content'.format(unmapped))
else:
- self.__logger.debug('data - not cached: {0}'.format(url))
+ self.__logger.debug('data - not cached: {0}'.format(unmapped))
meta, content, _ = Content().read_content(directory, file)
@@ -178,6 +184,13 @@ def data(self, query, page_index, search_base, max_entries):
###
#
+# Version: 1.1.1
+# Date: 2020-05-06
+# Author: Yves Vindevogel (vindevoy)
+#
+# Fixes:
+# - The replace with the bold tags works, but it does not capture capital letters. Introducing another method.
+#
# Version: 1.1.0
# Date: 2020-05-01
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/model/tags.py b/src/application/model/tags.py
index 76effa3..f2cddda 100644
--- a/src/application/model/tags.py
+++ b/src/application/model/tags.py
@@ -2,19 +2,18 @@
#
# Full history: see below
#
-# Version: 1.3.0
-# Date: 2020-05-01
+# Version: 1.4.0
+# Date: 2020-05-08
# Author: Yves Vindevogel (vindevoy)
#
# Changes:
-# - Moved the tag_label and tag_text methods to a support class in common
-# - Rewrite date format
+# - Support for drafts
+# - Remapping of URLs to documents
#
###
import logging
import math
-import os
import string
from operator import itemgetter
@@ -25,6 +24,7 @@
from common.options import Options
from common.singleton import Singleton
from common.tags_support import TagsSupport
+from controller.remapper import Remapper
class Tags(metaclass=Singleton):
@@ -47,8 +47,8 @@ def skip_tags(self):
return tags
- def list(self, posts_dir):
- self.__logger.debug('list - posts_dir: {0}'.format(posts_dir))
+ def list(self, posts):
+ self.__logger.debug('list - posts: {0}'.format(posts))
settings = Content().load_data_settings_yaml(self.__base_dir)
self.__logger.debug('list - settings: {0}'.format(settings))
@@ -56,31 +56,30 @@ def list(self, posts_dir):
# Starting with a dictionary as this is the easiest to find existing tags
tags = {}
- try:
- for file in os.listdir(posts_dir):
- meta, _, _ = Content().read_content(posts_dir, file) # No need to catch the content
+ for entry in posts:
+ directory = entry['directory']
+ file = entry['file']
+
+ meta, _, _ = Content().read_content(directory, file) # No need to catch the content
+
+ if meta['tags'] is None:
+ continue
- if meta['tags'] is None:
+ for tag in meta['tags']:
+ label = TagsSupport().tag_label(tag)
+
+ if label in settings['skip_tags']:
+ self.__logger.debug('list - tag {0} found in skip_tags'.format(tag))
continue
- for tag in meta['tags']:
- label = TagsSupport().tag_label(tag)
-
- if label in settings['skip_tags']:
- self.__logger.debug('list - tag {0} found in skip_tags'.format(tag))
- continue
-
- if label in tags.keys():
- self.__logger.debug('list - tag {0} already exists, +1'.format(tag))
- current_count = tags[label]['count']
- tags[label]['count'] = current_count + 1
- else:
- self.__logger.debug('list - tag {0} does not already exist'.format(tag))
- data = {'label': label, 'count': 1, 'text': string.capwords(tag)}
- tags[label] = data
- except FileNotFoundError:
- self.__logger.warning('COULD NOT FIND THE POSTS DIRECTORY {0}'.format(posts_dir))
- pass
+ if label in tags.keys():
+ self.__logger.debug('list - tag {0} already exists, +1'.format(tag))
+ current_count = tags[label]['count']
+ tags[label]['count'] = current_count + 1
+ else:
+ self.__logger.debug('list - tag {0} does not already exist'.format(tag))
+ data = {'label': label, 'count': 1, 'text': string.capwords(tag)}
+ tags[label] = data
self.__logger.debug('list - tags: '.format(tags))
@@ -90,13 +89,23 @@ def list(self, posts_dir):
for _, value in tags.items(): # Only need the value
tags_array.append(value)
+ for tag in tags_array:
+ label = tag['label']
+ unmapped = '/tags/{0}'.format(label)
+ remapped = Remapper().remap_document(unmapped)
+
+ if unmapped != remapped:
+ label = remapped.split('/')[2]
+ tag['label'] = label
+ tag['text'] = TagsSupport().tag_text(label)
+
tags_list = sorted(tags_array, key=itemgetter('count'), reverse=True)
self.__logger.debug('list - sorted tags: '.format(tags_list))
return tags_list
- def data(self, posts_dir, tag, page_index, index_max_posts, count_tag_posts):
- self.__logger.debug('data - posts_dir: {0}'.format(posts_dir))
+ def data(self, posts, tag, page_index, index_max_posts, count_tag_posts):
+ self.__logger.debug('data - posts: {0}'.format(posts))
self.__logger.debug('data - tag: {0}'.format(tag))
self.__logger.debug('data - page_index tags: {0}'.format(page_index))
self.__logger.debug('data - index_max_posts tags: {0}'.format(index_max_posts))
@@ -110,46 +119,48 @@ def data(self, posts_dir, tag, page_index, index_max_posts, count_tag_posts):
self.__logger.debug('data - max_entries: {0}'.format(max_entries))
self.__logger.debug('data - skip_entries: {0}'.format(skip_entries))
- try:
- for file in sorted(os.listdir(posts_dir), reverse=True):
- post, _, post['content'] = Content().read_content(posts_dir, file)
+ for entry in posts:
+ directory = entry['directory']
+ file = entry['file']
- if post['tags'] is None:
- continue
+ post, _, post['content'] = Content().read_content(directory, file)
- self.__logger.debug('data - post: {0}'.format(post))
+ if post['tags'] is None:
+ continue
- must_include = False
+ self.__logger.debug('data - post: {0}'.format(post))
- for tag_raw in post['tags']:
- if TagsSupport().tag_label(tag_raw) == tag:
- must_include = True
- break
+ must_include = False
- self.__logger.debug('data - must_include: {0}'.format(must_include))
+ for tag_raw in post['tags']:
+ if TagsSupport().tag_label(tag_raw) == tag:
+ must_include = True
+ break
- if must_include:
- count_entries += 1
+ self.__logger.debug('data - must_include: {0}'.format(must_include))
+
+ if must_include:
+ count_entries += 1
- # We count the entries, but for pages 2 and more, you don't show them
- if skip_entries >= count_entries:
- self.__logger.debug('data - post skipped}')
- continue
- else:
- self.__logger.debug('data - post added')
+ # We count the entries, but for pages 2 and more, you don't show them
+ if skip_entries >= count_entries:
+ self.__logger.debug('data - post skipped}')
+ continue
+ else:
+ self.__logger.debug('data - post added')
- stem = Path(file).stem
- post['url'] = stem
+ stem = Path(file).stem
+ url = '/posts/{0}'.format(stem)
+ url = Remapper().remap_document(url)
- post['date'] = DateTimeSupport().rewrite_date(post['date'])
+ post['url'] = url
+ post['date'] = DateTimeSupport().rewrite_date(post['date'])
- data['posts'].append(post)
+ data['posts'].append(post)
- if count_entries == (max_entries + skip_entries):
- self.__logger.debug('data - enough posts')
- break
- except FileNotFoundError:
- pass
+ if count_entries == (max_entries + skip_entries):
+ self.__logger.debug('data - enough posts')
+ break
data['tag'] = {'name': TagsSupport().tag_text(tag), 'path': tag}
@@ -162,26 +173,26 @@ def data(self, posts_dir, tag, page_index, index_max_posts, count_tag_posts):
return data
- def count_posts(self, posts_dir, tag):
+ def count_posts(self, posts, tag):
self.__logger.debug('count_posts - tag: {0}'.format(tag))
- self.__logger.debug('count_posts - posts_dir: {0}'.format(posts_dir))
+ self.__logger.debug('count_posts - posts: {0}'.format(posts))
count_entries = 0
- try:
- for file in os.listdir(posts_dir):
- post, _, _ = Content().read_content(posts_dir, file)
+ for entry in posts:
+ directory = entry['directory']
+ file = entry['file']
- if post['tags'] is None:
- continue
+ post, _, _ = Content().read_content(directory, file)
+
+ if post['tags'] is None:
+ continue
- for tag_raw in post['tags']:
- if TagsSupport().tag_label(tag_raw) == tag:
- count_entries += 1
- self.__logger.debug('count_posts - file {0} includes tag '.format(file))
- break
- except FileNotFoundError:
- pass
+ for tag_raw in post['tags']:
+ if TagsSupport().tag_label(tag_raw) == tag:
+ count_entries += 1
+ self.__logger.debug('count_posts - file {0} includes tag '.format(file))
+ break
self.__logger.debug('count_posts - tag {0} has {1} posts'.format(tag, count_entries))
@@ -189,6 +200,14 @@ def count_posts(self, posts_dir, tag):
###
#
+# Version: 1.3.0
+# Date: 2020-05-01
+# Author: Yves Vindevogel (vindevoy)
+#
+# Changes:
+# - Moved the tag_label and tag_text methods to a support class in common
+# - Rewrite date format
+#
# Version: 1.2.1
# Date: 2020-04-23
# Author: Yves Vindevogel (vindevoy)
diff --git a/src/application/sitemap.py b/src/application/sitemap.py
new file mode 100644
index 0000000..385d7c3
--- /dev/null
+++ b/src/application/sitemap.py
@@ -0,0 +1,128 @@
+###
+#
+# Full history: see below
+#
+# Version: 1.1.0
+# Date: 2020-05-08
+# Author: Yves Vindevogel (vindevoy)
+#
+# Features:
+# - Added remapper
+#
+###
+
+import logging
+import os
+
+from pathlib import Path
+
+from common.options import Options
+from common.content import Content
+from controller.data_loader import DataLoader
+from controller.logging_loader import LoggingLoader
+from controller.remapper import Remapper
+from controller.settings_loader import SettingsLoader
+
+environment = 'localhost'
+data_dir = os.path.join(os.getcwd(), 'src', 'data')
+
+Options().environment = environment
+Options().data_dir = data_dir
+
+settings = SettingsLoader(environment).parse()
+
+LoggingLoader().configure()
+
+logger = logging.getLogger('SITEMAP')
+
+logger.debug('data_dir: {0}'.format(data_dir))
+
+Remapper().outgoing_content = DataLoader().mapping_outgoing()
+Remapper().incoming_content = DataLoader().mapping_incoming()
+
+# Override the drafts because Google will never find them in production most likely
+Options().include_drafts = False
+
+xml = '\n'
+xml += '
{{ post.summary}}
- {{ data.i8n.navigation.read_more_label }} + {{ data.i8n.navigation.read_more_label }}{{ post.summary}}
- {{ data.i8n.navigation.read_more_label }} + {{ data.i8n.navigation.read_more_label }}{{ post.summary}}
- {{ data.i8n.navigation.read_more_label }} + {{ data.i8n.navigation.read_more_label }}