Skip to content

Commit 0a76e14

Browse files
guzman-raphaelguzman-raphael
authored
Merge pull request #196 from guzman-raphael/external
Add External Storage (File)
2 parents cb1c2ac + 06d401c commit 0a76e14

32 files changed

+1378
-619
lines changed

‎+dj/+internal/Declare.m‎

Lines changed: 72 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,23 @@
33
% table definitions, and to declare the corresponding mysql tables.
44

55
properties(Constant)
6+
UUID_DATA_TYPE = 'binary(16)'
67
CONSTANT_LITERALS = {'CURRENT_TIMESTAMP'}
8+
EXTERNAL_TABLE_ROOT = '~external'
79
TYPE_PATTERN = struct( ...
810
'NUMERIC', '^((tiny|small|medium|big)?int|decimal|double|float)', ...
911
'STRING', '^((var)?char|enum|date|(var)?year|time|timestamp)', ...
10-
'INTERNAL_BLOB', '^(tiny|medium|long)?blob', ...
12+
'INTERNAL_BLOB', '^(tiny|medium|long)?blob$', ...
13+
'EXTERNAL_BLOB', 'blob@(?<store>[a-z]\w*)$', ...
1114
'UUID', 'uuid$' ...
1215
)
13-
UUID_DATA_TYPE = 'binary(16)'
14-
SPECIAL_TYPES = {'UUID'}
16+
SPECIAL_TYPES = {'UUID', 'EXTERNAL_BLOB'}
17+
EXTERNAL_TYPES = {'EXTERNAL_BLOB'} % data referenced by a UUID in external tables
18+
SERIALIZED_TYPES = {'EXTERNAL_BLOB'} % requires packing data
1519
end
1620

1721
methods(Static)
18-
function sql = declare(table_instance, def)
22+
function [sql, external_stores] = declare(table_instance, def)
1923
% sql = DECLARE(query, definition)
2024
% Parse table declaration and declares the table.
2125
% sql: <string> Generated SQL to create a table.
@@ -36,12 +40,13 @@
3640
switch true
3741

3842
case {isa(table_instance, 'dj.internal.UserRelation'), isa(table_instance, ...
39-
'dj.Part'), isa(table_instance, 'dj.Jobs')}
43+
'dj.Part'), isa(table_instance, 'dj.Jobs'), ...
44+
isa(table_instance, 'dj.internal.ExternalTable')}
4045
% New-style declaration using special classes for each tier
4146
tableInfo = struct;
4247
if isa(table_instance, 'dj.Part')
4348
tableInfo.tier = 'part';
44-
else
49+
elseif ~isa(table_instance, 'dj.internal.ExternalTable')
4550
specialClass = find(cellfun(@(c) isa(table_instance, c), ...
4651
dj.Schema.tierClasses));
4752
assert(length(specialClass)==1, ...
@@ -70,11 +75,14 @@
7075
dj.internal.fromCamelCase(table_instance.className(length( ...
7176
table_instance.master.className)+1:end))));
7277
%#ok<MCNPN>
73-
else
78+
elseif ~isa(table_instance, 'dj.internal.ExternalTable')
7479
tableName = sprintf('%s%s%s', ...
7580
table_instance.schema.prefix, dj.Schema.tierPrefixes{ ...
7681
strcmp(tableInfo.tier, dj.Schema.allowedTiers)}, ...
7782
dj.internal.fromCamelCase(tableInfo.className));
83+
else
84+
tableName = [dj.internal.Declare.EXTERNAL_TABLE_ROOT '_' ...
85+
table_instance.store];
7886
end
7987

8088
otherwise
@@ -105,12 +113,13 @@
105113
stableInfo.className));
106114
end
107115

108-
sql = sprintf('CREATE TABLE `%s`.`%s` (\n', table_instance.schema.dbname, ...
109-
tableName);
110-
111116
% fields and foreign keys
112117
inKey = true;
113118
primaryFields = {};
119+
foreignKeySql = {};
120+
indexSql = {};
121+
attributeSql = {};
122+
external_stores = {};
114123
fields = {};
115124
for iLine = 1:length(def)
116125
line = def{iLine};
@@ -119,19 +128,20 @@ case strncmp(line,'---',3)
119128
inKey = false;
120129
% foreign key
121130
case regexp(line, '^(\s*\([^)]+\)\s*)?->.+$')
122-
[sql, newFields] = dj.internal.Declare.makeFK( ...
123-
sql, line, fields, inKey, ...
131+
[fk_attr_sql, fk_sql, newFields] = dj.internal.Declare.makeFK( ...
132+
line, fields, inKey, ...
124133
dj.internal.shorthash(sprintf('`%s`.`%s`', ...
125134
table_instance.schema.dbname, tableName)));
126-
sql = sprintf('%s,\n', sql);
135+
attributeSql = [attributeSql, fk_attr_sql]; %#ok<AGROW>
136+
foreignKeySql = [foreignKeySql, fk_sql]; %#ok<AGROW>
127137
fields = [fields, newFields]; %#ok<AGROW>
128138
if inKey
129139
primaryFields = [primaryFields, newFields]; %#ok<AGROW>
130140
end
131141

132142
% index
133143
case regexpi(line, '^(unique\s+)?index[^:]*$')
134-
sql = sprintf('%s%s,\n', sql, line); % add checks
144+
indexSql = [indexSql, line]; %#ok<AGROW>
135145

136146
% attribute
137147
case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
@@ -144,26 +154,37 @@ case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
144154
primaryFields{end+1} = fieldInfo.name; %#ok<AGROW>
145155
end
146156
fields{end+1} = fieldInfo.name; %#ok<AGROW>
147-
sql = sprintf('%s%s', sql, ...
148-
dj.internal.Declare.compileAttribute(fieldInfo));
149-
157+
[attr_sql, store, foreignKeySql] = ...
158+
dj.internal.Declare.compileAttribute(fieldInfo, foreignKeySql);
159+
attributeSql = [attributeSql, attr_sql]; %#ok<AGROW>
160+
if ~isempty(store)
161+
external_stores{end+1} = store; %#ok<AGROW>
162+
end
150163
otherwise
151164
error('Invalid table declaration line "%s"', line)
152165
end
153166
end
154167

155-
% add primary key declaration
168+
% create declaration
169+
create_sql = sprintf('CREATE TABLE `%s`.`%s` (\n', table_instance.schema.dbname,...
170+
tableName);
171+
% add attribute, primary key, foreign key, and index declaration
156172
assert(~isempty(primaryFields), 'table must have a primary key')
157-
sql = sprintf('%sPRIMARY KEY (%s),\n' ,sql, backquotedList(primaryFields));
158-
173+
table_sql = {attributeSql', {['PRIMARY KEY (`' strjoin(primaryFields, '`,`') ...
174+
'`)']}, foreignKeySql', indexSql'};
175+
table_sql = sprintf([strjoin(cat(1, table_sql{:}), ',\n') '\n']);
159176
% finish the declaration
160-
sql = sprintf('%s\n) ENGINE = InnoDB, COMMENT "%s"', sql(1:end-2), ...
161-
tableInfo.comment);
177+
engine_sql = sprintf(') ENGINE = InnoDB, COMMENT "%s"', tableInfo.comment);
178+
179+
sql = sprintf('%s%s%s', create_sql, table_sql, engine_sql);
180+
162181

163182
% execute declaration
164-
fprintf \n<SQL>\n
165-
fprintf(sql)
166-
fprintf \n</SQL>\n\n
183+
if strcmpi(dj.config('loglevel'), 'DEBUG')
184+
fprintf \n<SQL>\n
185+
fprintf(sql)
186+
fprintf \n</SQL>\n\n
187+
end
167188
end
168189

169190
function fieldInfo = parseAttrDef(line)
@@ -178,7 +199,7 @@ case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
178199
'^(?<name>[a-z][a-z\d_]*)\s*' % field name
179200
['=\s*(?<default>".*"|''.*''|\w+|[-+]?[0-9]*\.?[0-9]+([eE][-+]?' ...
180201
'[0-9]+)?)\s*'] % default value
181-
[':\s*(?<type>\w[\w\s]+(\(.*\))?(\s*[aA][uU][tT][oO]_[iI][nN]' ...
202+
[':\s*(?<type>\w[@\w\s]+(\(.*\))?(\s*[aA][uU][tT][oO]_[iI][nN]' ...
182203
'[cC][rR][eE][mM][eE][nN][tT])?)\s*'] % datatype
183204
'#(?<comment>.*)' % comment
184205
'$' % end of line
@@ -208,7 +229,7 @@ case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
208229
fieldInfo.isnullable = strcmpi(fieldInfo.default,'null');
209230
end
210231

211-
function [sql, newattrs] = makeFK(sql, line, existingFields, inKey, hash)
232+
function [all_attr_sql, fk_sql, newattrs] = makeFK(line, existingFields, inKey, hash)
212233
% [sql, newattrs] = MAKEFK(sql, line, existingFields, inKey, hash)
213234
% Add foreign key to SQL table definition.
214235
% sql: <string> Modified in-place SQL to include foreign keys.
@@ -217,6 +238,8 @@ case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
217238
% existingFields: <struct> Existing field attributes.
218239
% inKey: <logical> Set as primary key.
219240
% hash: <string> Current hash as base.
241+
fk_sql = '';
242+
all_attr_sql = '';
220243
pat = ['^(?<newattrs>\([\s\w,]*\))?' ...
221244
'\s*->\s*' ...
222245
'(?<cname>\w+\.[A-Z][A-Za-z0-9]*)' ...
@@ -269,29 +292,38 @@ case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
269292
rel.tableHeader.names));
270293
fieldInfo.name = newattrs{i};
271294
fieldInfo.nullabe = ~inKey; % nonprimary references are nullable
272-
sql = sprintf('%s%s', sql, dj.internal.Declare.compileAttribute(fieldInfo));
295+
[attr_sql, ~, ~] = dj.internal.Declare.compileAttribute(fieldInfo, []);
296+
all_attr_sql = sprintf('%s%s,\n', all_attr_sql, attr_sql);
273297
end
298+
all_attr_sql = all_attr_sql(1:end-2);
274299

275300
fkattrs = rel.primaryKey;
276301
fkattrs(ismember(fkattrs, attrs))=newattrs;
277302
hash = dj.internal.shorthash([{hash rel.fullTableName} newattrs]);
278-
sql = sprintf(...
303+
fk_sql = sprintf(...
279304
['%sCONSTRAINT `%s` FOREIGN KEY (%s) REFERENCES %s (%s) ' ...
280-
'ON UPDATE CASCADE ON DELETE RESTRICT'], sql, hash, backquotedList(fkattrs), ...
281-
rel.fullTableName, backquotedList(rel.primaryKey));
305+
'ON UPDATE CASCADE ON DELETE RESTRICT'], fk_sql, hash, ...
306+
backquotedList(fkattrs), rel.fullTableName, backquotedList(rel.primaryKey));
282307
end
283308

284-
function field = substituteSpecialType(field, category)
309+
function [field, foreignKeySql] = substituteSpecialType(field, category, foreignKeySql)
285310
% field = SUBSTITUTESPECIALTYPE(field, category)
286311
% Substitute DataJoint type with sql type.
287312
% field: <struct> Modified in-place field attributes.
288313
% category: <string> DataJoint type match based on TYPE_PATTERN.
289314
if strcmpi(category, 'UUID')
290315
field.type = dj.internal.Declare.UUID_DATA_TYPE;
316+
elseif any(strcmpi(category, dj.internal.Declare.EXTERNAL_TYPES))
317+
field.store = strtrim(field.type((strfind(field.type,'@')+1):end));
318+
field.type = dj.internal.Declare.UUID_DATA_TYPE;
319+
foreignKeySql = [foreignKeySql, sprintf( ...
320+
['FOREIGN KEY (`%s`) REFERENCES `{database}`.`%s_%s` (`hash`) ON ' ...
321+
'UPDATE RESTRICT ON DELETE RESTRICT'], field.name, ...
322+
dj.internal.Declare.EXTERNAL_TABLE_ROOT, field.store)]; %#ok<AGROW>
291323
end
292324
end
293325

294-
function sql = compileAttribute(field)
326+
function [sql, store, foreignKeySql] = compileAttribute(field, foreignKeySql)
295327
% sql = COMPILEATTRIBUTE(field)
296328
% Convert the structure field with header {'name' 'type' 'default' 'comment'}
297329
% to the SQL column declaration.
@@ -317,11 +349,16 @@ case regexp(line, ['^[a-z][a-z\d_]*\s*' ... % name
317349
'illegal characters in attribute comment "%s"', field.comment)
318350

319351
category = dj.internal.Declare.matchType(field.type);
352+
store = [];
320353
if any(strcmpi(category, dj.internal.Declare.SPECIAL_TYPES))
321354
field.comment = [':' strip(field.type) ':' field.comment];
322-
field = dj.internal.Declare.substituteSpecialType(field, category);
355+
[field, foreignKeySql] = dj.internal.Declare.substituteSpecialType(field, ...
356+
category, foreignKeySql);
357+
if isfield(field, 'store')
358+
store = field.store;
359+
end
323360
end
324-
sql = sprintf('`%s` %s %s COMMENT "%s",\n', ...
361+
sql = sprintf('`%s` %s %s COMMENT "%s"', ...
325362
field.name, strtrim(field.type), default, field.comment);
326363
end
327364

‎+dj/+internal/ExternalMapping.m‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
% dj.internal.ExternalMapping - The external manager contains all the tables for all external
2+
% stores for a given schema.
3+
% :Example:
4+
% e = dj.internal.ExternalMapping(schema)
5+
% external_table = e.table(store)
6+
classdef ExternalMapping < handle
7+
properties
8+
schema
9+
tables
10+
end
11+
methods
12+
function self = ExternalMapping(schema)
13+
self.schema = schema;
14+
self.tables = struct();
15+
end
16+
function store_table = table(self, store)
17+
keys = fieldnames(self.tables);
18+
if all(~strcmp(store, keys))
19+
self.tables.(store) = dj.internal.ExternalTable(...
20+
self.schema.conn, store, self.schema);
21+
end
22+
store_table = self.tables.(store);
23+
end
24+
end
25+
end

0 commit comments

Comments
 (0)