apply data compression to strings, new datatype char

NeuroJSON · Jun 2, 2020 · 8829d6b · 8829d6b
1 parent 270cbf6
commit 8829d6b
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 18 deletions.
diff --git a/examples/jsonlab_basictest.matlab b/examples/jsonlab_basictest.matlab
@@ -49,9 +49,9 @@ ans =
 
 
 >> 
-json2data = 
+json2data =
 
-   Empty cell array: 0-by-1
+     []
 
 >> >> 
 %=================================================
@@ -148,9 +148,10 @@ ans =
 
 
 >> 
-json2data = 
+json2data =
 
-    'AC'    'EG'
+AE
+CG
 
 >> >> 
 %=================================================
@@ -737,7 +738,7 @@ ans =
 >> 
 json2data = 
 
-    empty_0by0_real: {0x1 cell}
+    empty_0by0_real: []
 
 >> >> 
 %=================================================
@@ -1085,7 +1086,7 @@ ans =
 >> 
 json2data = 
 
-    data2json: {{1x3 cell}  {1x3 cell}}
+    data2json: {2x3 cell}
 
 >> >> 
 %=================================================
@@ -1142,7 +1143,7 @@ ans =
 >> 
 json2data = 
 
-    data2json: {{1x2 cell}  {1x2 cell}  {1x2 cell}}
+    data2json: {3x2 cell}
 
 >> >> >> 
 %=================================================

diff --git a/jdatadecode.m b/jdatadecode.m
@@ -122,10 +122,19 @@
             end
             if(~isempty(strmatch(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc'})))
                 decompfun=str2func([zipmethod 'decode']);
+                arraytype=data(j).(N_('_ArrayType_'));
+                chartype=0;
+                if(strcmp(arraytype,'char'))
+                    chartype=1;
+                    arraytype='uint8';
+                end
                 if(needbase64)
-                    ndata=reshape(typecast(decompfun(base64decode(data(j).(N_('_ArrayZipData_')))),data(j).(N_('_ArrayType_'))),dims);
+                    ndata=reshape(typecast(decompfun(base64decode(data(j).(N_('_ArrayZipData_')))),arraytype),dims);
                 else
-                    ndata=reshape(typecast(decompfun(data(j).(N_('_ArrayZipData_'))),data(j).(N_('_ArrayType_'))),dims);
+                    ndata=reshape(typecast(decompfun(data(j).(N_('_ArrayZipData_'))),arraytype),dims);
+                end
+                if(chartype)
+                    ndata=char(ndata);
                 end
             else
                 error('compression method is not supported');

diff --git a/savejson.m b/savejson.m
@@ -88,6 +88,8 @@
 %                             compressed binary array data. 
 %           CompressArraySize [100|int]: only to compress an array if the total 
 %                         element count is larger than this number.
+%           CompressStringSize [400|int]: only to compress a string if the total 
+%                         element count is larger than this number.
 %           FormatVersion [2|float]: set the JSONLab output version; since
 %                         v2.0, JSONLab uses JData specification Draft 1
 %                         for output format, it is incompatible with all
@@ -144,6 +146,7 @@
 opt.singletarray=jsonopt('SingletArray',0,opt);
 opt.formatversion=jsonopt('FormatVersion',2,opt);
 opt.compressarraysize=jsonopt('CompressArraySize',100,opt);
+opt.compressstringsize=jsonopt('CompressStringSize',opt.compressarraysize*4,opt);
 opt.intformat=jsonopt('IntFormat','%d',opt);
 opt.floatformat=jsonopt('FloatFormat','%.10g',opt);
 opt.unpackhex=jsonopt('UnpackHex',1,opt);
@@ -254,7 +257,11 @@
 elseif(isnumeric(item) || islogical(item))
     txt=mat2json(name,item,level,varargin{:});
 elseif(ischar(item))
-    txt=str2json(name,item,level,varargin{:});
+    if(numel(item)>=varargin{1}.compressstringsize)
+        txt=mat2json(name,item,level,varargin{:});
+    else
+        txt=str2json(name,item,level,varargin{:});
+    end
 elseif(isa(item,'function_handle'))
     txt=struct2json(name,functions(item),level,varargin{:});
 elseif(isa(item,'containers.Map'))
@@ -533,7 +540,7 @@
 
 %%-------------------------------------------------------------------------
 function txt=mat2json(name,item,level,varargin)
-if(~isnumeric(item) && ~islogical(item))
+if(~isnumeric(item) && ~islogical(item) && ~ischar(item))
         error('input is not an array');
 end
 ws=varargin{1}.whitespaces_;
@@ -598,7 +605,7 @@
         end
         txt=sprintf(dataformat,txt,padding0,'"_ArrayZipSize_": ',regexprep(mat2str(size(fulldata)),'\s+',','), sep);
         txt=sprintf(dataformat,txt,padding0,'"_ArrayZipType_": "',dozip, ['"' sep]);
-	    compfun=str2func([dozip 'encode']);
+	compfun=str2func([dozip 'encode']);
         txt=sprintf(dataformat,txt,padding0,'"_ArrayZipData_": "',base64encode(compfun(typecast(fulldata(:),'uint8'))),['"' nl]);
     else
         if(size(item,1)==1)
@@ -621,7 +628,7 @@
     if(~isempty(dozip) && numel(item)>zipsize)
         if(isreal(item))
             fulldata=item(:)';
-            if(islogical(fulldata))
+            if(islogical(fulldata) || ischar(fulldata))
                 fulldata=uint8(fulldata);
             end
         else

diff --git a/saveubjson.m b/saveubjson.m
@@ -87,6 +87,8 @@
 %                            'base64' encoding
 %          CompressArraySize [100|int]: only to compress an array if the total 
 %                         element count is larger than this number.
+%          CompressStringSize [400|int]: only to compress a string if the total 
+%                         element count is larger than this number.
 %          MessagePack [0|1]: output MessagePack (https://msgpack.org/)
 %                         binary stream instead of BJD/UBJSON
 %          UBJSON [0|1]: 0: (default)-encode data based on BJData Draft 1
@@ -146,6 +148,7 @@
 opt.nestarray=jsonopt('NestArray',0,opt);
 opt.formatversion=jsonopt('FormatVersion',2,opt);
 opt.compressarraysize=jsonopt('CompressArraySize',100,opt);
+opt.compressstringsize=jsonopt('CompressStringSize',opt.compressarraysize*4,opt);
 opt.singletcell=jsonopt('SingletCell',1,opt);
 opt.singletarray=jsonopt('SingletArray',0,opt);
 opt.arraytostruct=jsonopt('ArrayToStruct',0,opt);
@@ -271,7 +274,11 @@
 elseif(isnumeric(item) || islogical(item))
     txt=mat2ubjson(name,item,level,varargin{:});
 elseif(ischar(item))
-    txt=str2ubjson(name,item,level,varargin{:});
+    if(numel(item)>=varargin{1}.compressstringsize)
+        txt=mat2ubjson(name,item,level,varargin{:});
+    else
+        txt=str2ubjson(name,item,level,varargin{:});
+    end
 elseif(isa(item,'function_handle'))
     txt=struct2ubjson(name,functions(item),level,varargin{:});
 elseif(isa(item,'containers.Map'))
@@ -502,7 +509,7 @@
 
 %%-------------------------------------------------------------------------
 function txt=mat2ubjson(name,item,level,varargin)
-if(~isnumeric(item) && ~islogical(item))
+if(~isnumeric(item) && ~islogical(item) && ~ischar(item))
         error('input is not an array');
 end
 
@@ -610,7 +617,7 @@
     if(~isempty(dozip) && numel(item)>zipsize)
         if(isreal(item))
             fulldata=item(:)';
-            if(islogical(fulldata))
+            if(islogical(fulldata) || ischar(fulldata))
                 fulldata=uint8(fulldata);
             end
         else
@@ -621,8 +628,8 @@
         cid=I_(uint32(max(size(fulldata))),varargin{:});
         txt=[txt, N_('_ArrayZipSize_',opt),I_a(size(fulldata),cid(1),varargin{:})];
         txt=[txt, N_('_ArrayZipType_',opt),S_(dozip,opt)];
-	    compfun=str2func([dozip 'encode']);
-	    txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),varargin{:})];
+	compfun=str2func([dozip 'encode']);
+	txt=[txt,N_('_ArrayZipData_',opt), I_a(compfun(typecast(fulldata(:),'uint8')),Imarker(1),varargin{:})];
         childcount=childcount+3;
     else
         if(ismsgpack)