1.0.0 beta4 merge request (#5)

* Improvements to readibility

README,
benchmarking code,
and removed debugging code from tinymp main module

* Included some benchmarking information

Also, tidied up tinymp.py

* Included some benchmarking information

Also, tidied up tinymp.py

* Reformatted table in README

* Fleshed out README.rst file

* Quick amendment to README to sort typo

* Update README.rst

* Changes for Beta4 release wrapped

Author: alshapton

CHANGELOG.md

@@ -1,5 +1,8 @@
+* Version 1.0.0-Beta4 - 11/26/2017
+    * Added descriptions of benchmarking and tidied up repo
+    * Further added information to README about usage
 
-* Version 1.0.0-Beta3 - 24/11/2017
+* Version 1.0.0-Beta3 - 11/24/2017
     * Added support for u-msgpack-python alternative msgpack library
 
 * Version 1.0.0-Beta2 - 11/21/2017

README.rst

@@ -10,6 +10,23 @@
 
 TinyMP is a storage backend for TinyDB https://github.com/msiemens/tinydb which is based around the MessagePack compressed JSON format (https://msgpack.org/index.html)   
 
+Syntax :
+========
+TinyMP extends the syntax of the ``tinydb`` class using one of the optional ``kwargs`` as follows:
+
+
+.. code:: python
+    class tinydb.database.TinyDB(*args, **kwargs)
+
+.. csv-table:: Values for ``**kwargs``
+   :header: "Value","Effect"
+   :widths: 10,90
+
+   "``storage=MsgPackStorage``","Default option, will use the ``MsgPack`` library"
+   "``storage=MsgPackStorage,Lib='msgpack'``","Will use the ``MsgPack`` library"
+   "``storage=MsgPackStorage,Lib='umsgpack'``","Will use the ``U-MsgPack`` Library"
+
+
 Example Usage:
 ==============
 
@@ -26,9 +43,11 @@ Example Usage:
     dbins()
 
 As you can see, it's a simple drop-in replacement for any storage engine
-and it can be nested and cached.
+and it can be nested and cached. Don't forget, you will need to install as a minimum,
+the ``msgpack-python`` library using ``pip install msgpack-python`` and the ``U-MsgPack``
+library from https://github.com/vsergeev/u-msgpack-python in order to use that option.
 
-Example Usage using alternative MessgaePack Library:
+Example Usage using alternative MessagePack Library:
 ====================================================
 
 .. code:: python
@@ -45,10 +64,33 @@ Example Usage using alternative MessgaePack Library:
 
 As you can see, it's a simple drop-in replacement for any storage engine
 and it can be nested and cached.
+
+Why would I use this?
+=====================
+Looking at the statistics below, it's apparent that compared to the "standard"
+JSON Storage mechanism, MessagePack isn't as quick, however, the filesizes on
+disc are smaller - consider the table below, with 1,000 JSON documents of 
+minute size - clearly, the MessagePack compressed format is smaller than
+the JSON format. Whether you choose the default MsgPack library, which is 
+marginally slower than the U-MsgPack library (at the cost of a small increase
+in storage footprint with U-MsgPack) is dependent on your use case.
+
+.. csv-table:: Timings (seconds)
+   :header: "Format","Run 1", "Run 2", "Run 3", "FileSize"
+   :widths: 10,30, 30, 30,10 
+
+   "JSON Write:", 2.147,2.011,2.040,"37.0 Kb"
+   "MsgPack Write:", 9.562,9.732,9.716,"21.1 Kb"
+   "U-MsgPack Write:", 9.354,9.066,8.949,"24.1 Kb"
+
 Changes
 =======
 
-* Version 1.0.0-Beta3 - 24/11/2017
+* Version 1.0.0-Beta4 - 11/26/2017
+    * Added descriptions of benchmarking and tidied up repo
+    * Further added information to README about usage
+
+* Version 1.0.0-Beta3 - 11/24/2017
     * Added support for u-msgpack-python alternative msgpack library
 
 * Version 1.0.0-Beta2 - 11/21/2017
@@ -70,4 +112,5 @@ References:
 
 * TinyDB      https://github.com/msiemens/tinydb 
 * MessagePack https://msgpack.org/index.html
+* U-MsgPack   https://github.com/vsergeev/u-msgpack-python
 

benchmarks/benchmark.py

@@ -4,22 +4,53 @@
 
 import msgpack
 from tinymp import *
+	
+import os
 
-db1 = TinyDB('data.msg',storage=MsgPackStorage)
-#db1 = TinyDB('data.json')
 
-def dbins1():
-   db1.insert({'type': 'apple', 'count': 7})
+def convert_bytes(num):
+    """
+    this function will convert bytes to MB.... GB... etc
+    """
+    for x in ['bytes', 'KB', 'MB', 'GB', 'TB']:
+        if num < 1024.0:
+            return "%3.1f %s" % (num, x)
+        num /= 1024.0
 
-def dbins2():
-   db2.insert({'type': 'apple', 'count': 7})
+
+def file_size(file_path):
+    """
+    this function will return the file size
+    """
+    if os.path.isfile(file_path):
+        file_info = os.stat(file_path)
+        return convert_bytes(file_info.st_size)
+
+def dbins(db):
+   db.insert({'type': 'apple', 'count': 7})
+
+def insertstuff(db,n,method,file_name):
+    t0 = time.time()
+    for i in range(n): dbins(db)
+    t1 = time.time()
+
+    print method
+    print "Write Time: = " + repr(t1-t0) + "s"
+    print file_size(file_name)
+
+
+
+db1 = TinyDB('data.json')
+db2 = TinyDB('data.msgpack.msg',storage=MsgPackStorage)
+db3 = TinyDB('data.umsgpack.msg',storage=MsgPackStorage,Lib='umsgpack')
+
+
+insertstuff(db1,1000,'JSON','data.json')
+insertstuff(db2,1000,'MsgPack','data.msgpack.msg')
+insertstuff(db3,1000,'U-MsgpPack','data.umsgpack.msg')
 
 db1.purge_tables()
+db2.purge_tables()
+db3.purge_tables()
 
-n = 1000
-t0 = time.time()
-for i in range(n): dbins1()
-t1 = time.time()
 
-print (t1-t0)
-print "Write Benchmark: JSON = " + repr(t1-t0)

changenotes

@@ -0,0 +1,2 @@
+removed debugging statements from tinymp.py
+better inbenchmarking

tests/modifytests.sh

@@ -1,4 +1,15 @@
 #!/bin/sh
+
+# Make a new directory for working
+#mkdir originaltests
+
+# Copy ALL tests and duplicate them for both messagepack tests
+#copy test*py originaltests
+
+# Create new copies of the Python scripts for each StorageClass subtype
+#ls -1  test*.py| awk 'BEGIN     { FS = "." } { print "cp " $0 " " $1".msgpack.py; cp "$0 " " $1 ".umsgpack.py"  }' 
+
+
 echo "import umsgpack as msgpack" >> conftest.py
 echo "import tinymp" >> conftest.py
 

tinymp.py

@@ -14,7 +14,6 @@ def touch(fname, create_dirs):
 
 class MsgPackStorage(Storage):
     def __init__(self, path, create_dirs=False, **kwargs):
-        #self.filename = filename
         super(MsgPackStorage, self).__init__()
         touch(path, create_dirs=create_dirs)  # Create file if not exists
         self.kwargs = kwargs
@@ -25,15 +24,12 @@ def __init__(self, path, create_dirs=False, **kwargs):
            self.library=self.kwargs['Lib']
 
            if self.library == 'umsgpack':
-              print 'ultra-messagepack'
               import umsgpack as msgpack
               self.msgpack=msgpack
            else:
-              print 'messagepack'
               import msgpack
               self.msgpack=msgpack
         else:
-           print 'default'
            import msgpack
            self.msgpack=msgpack
 
@@ -42,7 +38,6 @@ def __init__(self, path, create_dirs=False, **kwargs):
     def write(self, data):
         self._handle.seek(0)
         serialized = self.msgpack.dump(data, self._handle)
-        #self._handle.write(serialized)
         self._handle.flush()
         self._handle.truncate()