diff --git a/+dj/+config/load.m b/+dj/+config/load.m new file mode 100644 index 00000000..54edf67f --- /dev/null +++ b/+dj/+config/load.m @@ -0,0 +1,10 @@ +function load(fname) + switch nargin + case 0 + dj.internal.Settings.load(); + case 1 + dj.internal.Settings.load(fname); + otherwise + error('Exceeded 1 input limit.'); + end +end \ No newline at end of file diff --git a/+dj/+config/restore.m b/+dj/+config/restore.m new file mode 100644 index 00000000..e23c8272 --- /dev/null +++ b/+dj/+config/restore.m @@ -0,0 +1,3 @@ +function out = restore() + out = dj.internal.Settings.restore(); +end \ No newline at end of file diff --git a/+dj/+config/save.m b/+dj/+config/save.m new file mode 100644 index 00000000..40938192 --- /dev/null +++ b/+dj/+config/save.m @@ -0,0 +1,10 @@ +function save(fname) + switch nargin + case 0 + dj.internal.Settings.save(); + case 1 + dj.internal.Settings.save(fname); + otherwise + error('Exceeded 1 input limit.'); + end +end \ No newline at end of file diff --git a/+dj/+config/saveGlobal.m b/+dj/+config/saveGlobal.m new file mode 100644 index 00000000..6ace6d77 --- /dev/null +++ b/+dj/+config/saveGlobal.m @@ -0,0 +1,3 @@ +function saveGlobal() + dj.internal.Settings.saveGlobal(); +end \ No newline at end of file diff --git a/+dj/+config/saveLocal.m b/+dj/+config/saveLocal.m new file mode 100644 index 00000000..9e7d4cf1 --- /dev/null +++ b/+dj/+config/saveLocal.m @@ -0,0 +1,3 @@ +function saveLocal() + dj.internal.Settings.saveLocal(); +end \ No newline at end of file diff --git a/+dj/+internal/AutoPopulate.m b/+dj/+internal/AutoPopulate.m index 9d078353..2034385d 100755 --- a/+dj/+internal/AutoPopulate.m +++ b/+dj/+internal/AutoPopulate.m @@ -111,12 +111,12 @@ % % See also dj.internal.AutoPopulate/parpopulate - if ~dj.set('populateAncestors') + if ~dj.config('queryPopulate_ancestors') rels = {self}; else % get all ancestors to be populated before self assert(nargout==0, ... - 'parpopulate cannot return output when populateAncestors is true') + 'parpopulate cannot return output when queryPopulate_ancestors is true') rels = cellfun(@feval, self.ancestors, 'uni', false); rels = rels(cellfun(@(x) isa(x,'dj.internal.AutoPopulate'), rels)); end @@ -171,7 +171,7 @@ function parpopulate(self, varargin) % % See also dj.internal.AutoPopulate/populate - if ~dj.set('populateAncestors') + if ~dj.config('queryPopulate_ancestors') rels = {self}; else % get all ancestors to be populated before self @@ -375,7 +375,7 @@ function cleanup(self, key) success = false; end end - if ~success && dj.set('verbose') + if ~success && strcmpi(dj.config('loglevel'), 'DEBUG') fprintf('** %s: skipping already reserved\n', self.className) disp(key) end @@ -433,8 +433,8 @@ function createJobTable(self) function populateSanityChecks(self) % Performs sanity checks that are common to populate, % parpopulate and batch_populate. - % To disable the sanity check: dj.set('populateCheck',false) - if dj.set('populateCheck') + % To disable the sanity check: dj.config('queryPopulate_check',false) + if dj.config('queryPopulate_check') source = self.getKeySource; abovePopRel = setdiff(self.primaryKey(1:min(end,length(source.primaryKey))), source.primaryKey); if ~all(ismember(source.primaryKey, self.primaryKey)) diff --git a/+dj/+internal/GeneralRelvar.m b/+dj/+internal/GeneralRelvar.m index be241311..2947f80a 100644 --- a/+dj/+internal/GeneralRelvar.m +++ b/+dj/+internal/GeneralRelvar.m @@ -83,7 +83,7 @@ function disp(self) attrList{i} = hdr.names{i}; end end - maxRows = dj.set('maxPreviewRows'); + maxRows = dj.config('displayLimit'); preview = self.fetch(attrList{:}, sprintf('LIMIT %d', maxRows+1)); if ~isempty(preview) hasMore = length(preview) > maxRows; diff --git a/+dj/+internal/Settings.m b/+dj/+internal/Settings.m new file mode 100644 index 00000000..4e98b442 --- /dev/null +++ b/+dj/+internal/Settings.m @@ -0,0 +1,153 @@ +classdef Settings < matlab.mixin.Copyable + properties (Constant) + LOCALFILE = './dj_local_conf.json' + GLOBALFILE = '~/.datajoint_config.json' + DEFAULTS = struct( ... + 'databaseHost', 'localhost', ... + 'databasePassword', [], ... + 'databaseUser', [], ... + 'databasePort', 3306, ... + 'databaseUse_tls', [], ... + 'databaseReconnect_transaction', false, ... + 'connectionInit_function', [], ... + 'loglevel', 'INFO', ... + 'safemode', true, ... + 'displayLimit', 12, ... how many rows to display when previewing a relation + 'displayDiagram_hierarchy_radius', [2 1], ... levels up and down the hierachy to display in `erd schema.Table` + 'displayDiagram_font_size', 12, ... font size to use in ERD labels + 'queryPopulate_check', true, ... + 'queryPopulate_ancestors', false, ... + 'queryBigint_to_double', false, ... + 'queryIgnore_extra_insert_fields', false ... when false, throws an error in `insert(self, tuple)` when tuple has extra fields. + ) + end + properties + result + end + methods(Static) + function out = Settings(name, value) + current_state = stateAccess; + out.result = current_state; + if nargin == 1 || nargin == 2 + assert(ischar(name), 'DataJoint:Config:InvalidType', ... + 'Setting name must be a string'); + token = regexp(['.', name], '(\W)(\w+)', 'tokens'); + token = vertcat(token{:}).'; + token(1,:) = strrep(strrep(token(1,:), '{', '{}'), '(', '()'); + value_vector = str2double(token(2,:)); + index = ~isnan(value_vector); + token(2, index) = num2cell(num2cell(value_vector(index))); + subscript = substruct(token{:}); + if nargout + try + out.result = subsref(current_state, subscript); + catch ME + switch ME.identifier + case 'MATLAB:nonExistentField' + error('DataJoint:Config:InvalidKey', ... + 'Setting `%s` does not exist', name); + otherwise + rethrow(ME); + end + end + else + out.result = []; + end + end + if nargin == 2 + new_state = subsasgn(current_state, subscript, value); + stateAccess('set', new_state); + end + end + function out = restore() + out = stateAccess('restore'); + end + function save(fname) + c = dj.internal.Settings; + dj.lib.saveJSONfile(c.result, fname); + end + function load(fname) + if ~nargin + fname = dj.internal.Settings.LOCALFILE; + end + raw = fileread(fname); + new_state = fixProps(jsondecode(raw), raw); + stateAccess('load', new_state); + end + function saveLocal() + dj.internal.Settings.save(dj.internal.Settings.LOCALFILE); + end + function saveGlobal() + dj.internal.Settings.save(dj.internal.Settings.GLOBALFILE); + end + end +end +function data = fixProps(data, raw) + newFields = fieldnames(data); + for i=1:length(newFields) + for j=1:length(data.(newFields{i})) + if isstruct(data.(newFields{i})(j)) + if exist('res','var') + res(end + 1) = fixProps(data.(newFields{i})(j), raw); + else + res = fixProps(data.(newFields{i})(j), raw); + end + if j == length(data.(newFields{i})) + data.(newFields{i}) = res; + clear res; + end + end + end + newFields{i} = regexprep(regexp(raw, ... + regexprep(newFields{i},'_','.'), 'match', 'once'), ... + '\.[a-zA-Z0-9]','${upper($0(2))}'); + end + data = cell2struct(struct2cell(data), newFields); +end +function out = stateAccess(operation, new) + function envVarUpdate() + % optional environment variables specifying the connection. + if getenv('DJ_HOST') + STATE.databaseHost = getenv('DJ_HOST'); + end + if getenv('DJ_USER') + STATE.databaseUser = getenv('DJ_USER'); + end + if getenv('DJ_PASS') + STATE.databasePassword = getenv('DJ_PASS'); + end + if getenv('DJ_INIT') + STATE.connectionInit_function = getenv('DJ_INIT'); + end + end + switch nargin + case 0 + operation = ''; + case 1 + case 2 + otherwise + error('Exceeded 2 input limit.') + end + persistent STATE + if (isempty(STATE) && ~strcmpi(operation, 'load')) || strcmpi(operation, 'restore') + % default settings + STATE = orderfields(dj.internal.Settings.DEFAULTS); + if exist(dj.internal.Settings.LOCALFILE, 'file') == 2 + dj.internal.Settings.load(dj.internal.Settings.LOCALFILE); + elseif exist(dj.internal.Settings.GLOBALFILE, 'file') == 2 + dj.internal.Settings.load(dj.internal.Settings.GLOBALFILE); + end + envVarUpdate(); + end + % return STATE prior to change + out = STATE; + if any(strcmpi(operation, {'set', 'load'})) + % merge with existing STATE + STATE = rmfield(STATE, intersect(fieldnames(STATE), fieldnames(new))); + names = [fieldnames(STATE); fieldnames(new)]; + STATE = orderfields(cell2struct([struct2cell(STATE); struct2cell(new)], names, 1)); + if strcmpi(operation, 'load') + envVarUpdate(); + end + end +end \ No newline at end of file diff --git a/+dj/+internal/Table.m b/+dj/+internal/Table.m index 2fb32377..5d19b421 100755 --- a/+dj/+internal/Table.m +++ b/+dj/+internal/Table.m @@ -191,11 +191,11 @@ function erd(self, up, down) self.create if nargin<=2 - down = dj.set('tableErdRadius'); + down = dj.config('displayDiagram_hierarchy_radius'); down = down(2); end if nargin<=1 - up = dj.set('tableErdRadius'); + up = dj.config('displayDiagram_hierarchy_radius'); up = up(1); end @@ -465,7 +465,7 @@ function syncDef(self) if isempty(path) fprintf('File %s.m is not found\n', self.className); else - if ~dj.set('suppressPrompt') ... + if dj.config('safemode') ... && ~strcmpi('yes', dj.internal.ask(sprintf('Update the table definition and class definition in %s?',path))) disp 'No? Table definition left untouched.' else @@ -549,7 +549,7 @@ function drop(self) end % if any table has data, give option to cancel - doPrompt = doPrompt && ~dj.set('suppressPrompt'); % suppress prompt + doPrompt = doPrompt && dj.config('safemode'); % suppress prompt if doPrompt && ~strcmpi('yes', dj.internal.ask('Proceed to drop?')) disp 'User cancelled table drop' else diff --git a/+dj/+lib/saveJSONfile.m b/+dj/+lib/saveJSONfile.m new file mode 100644 index 00000000..25d31bf9 --- /dev/null +++ b/+dj/+lib/saveJSONfile.m @@ -0,0 +1,83 @@ +function saveJSONfile(data, jsonFileName) +% Modified from FileExchange entry: +% https://www.mathworks.com/matlabcentral/fileexchange/... +% 50965-structure-to-json?focused=3876199&tab=function +% saves the values in the structure 'data' to a file in JSON indented format. +% +% Example: +% data.name = 'chair'; +% data.color = 'pink'; +% data.metrics.height = 0.3; +% data.metrics.width = 1.3; +% saveJSONfile(data, 'out.json'); +% +% Output 'out.json': +% { +% "name" : "chair", +% "color" : "pink", +% "metrics" : { +% "height" : 0.3, +% "width" : 1.3 +% } +% } +% + fid = fopen(jsonFileName,'w'); + writeElement(fid, data,''); + fprintf(fid,'\n'); + fclose(fid); +end +function writeElement(fid, data,tabs) + namesOfFields = fieldnames(data); + tabs = sprintf('%s\t',tabs); + fprintf(fid,'{\n%s',tabs); + key = true; + for i = 1:length(namesOfFields) - 1 + currentField = namesOfFields{i}; + currentElementValue = data.(currentField); + writeSingleElement(fid, currentField,currentElementValue,tabs, key); + fprintf(fid,',\n%s',tabs); + end + currentField = namesOfFields{end}; + currentElementValue = data.(currentField); + writeSingleElement(fid, currentField,currentElementValue,tabs, key); + fprintf(fid,'\n%s}',tabs(1:end-1)); +end +function writeSingleElement(fid, currentField,currentElementValue,tabs, key) + % if this is an array and not a string then iterate on every + % element, if this is a single element write it + currentField = regexprep(currentField,'[a-z0-9][A-Z]','${$0(1)}.${lower($0(2))}'); + if key + fprintf(fid,'"%s" : ' , currentField); + end + if length(currentElementValue) > 1 && ~ischar(currentElementValue) + fprintf(fid,'[\n%s\t',tabs); + for m = 1:length(currentElementValue)-1 + if isstruct(currentElementValue(m)) + writeElement(fid, currentElementValue(m),tabs); + else + writeSingleElement(fid, '',currentElementValue(m),tabs, false) + end + fprintf(fid,',\n%s\t',tabs); + end + if isstruct(currentElementValue(end)) + writeElement(fid, currentElementValue(end),tabs); + else + writeSingleElement(fid, '',currentElementValue(end),tabs, false) + end + fprintf(fid,'\n%s]',tabs); + elseif isstruct(currentElementValue) + writeElement(fid, currentElementValue,tabs); + elseif isempty(currentElementValue) + fprintf(fid,'null'); + elseif isnumeric(currentElementValue) + fprintf(fid,'%g' ,currentElementValue); + elseif islogical(currentElementValue) + if currentElementValue + fprintf(fid,'true'); + else + fprintf(fid,'false'); + end + else %ischar or something else ... + fprintf(fid,'"%s"',currentElementValue); + end +end \ No newline at end of file diff --git a/+dj/Connection.m b/+dj/Connection.m index 650814ed..1d393fa8 100644 --- a/+dj/Connection.m +++ b/+dj/Connection.m @@ -150,7 +150,7 @@ function reload(self) ret = ~isempty(self.connId) && 0==mym(self.connId, 'status'); if ~ret && self.inTransaction - if dj.set('reconnectTimedoutTransaction') + if dj.config('databaseReconnect_transaction') warning 'Reconnected after server disconnected during a transaction' else error 'Server disconnected during a transaction' @@ -170,7 +170,7 @@ function reload(self) end end v = varargin; - if dj.set('bigint_to_double') + if dj.config('queryBigint_to_double') v{end+1} = 'bigint_to_double'; end if nargout>0 diff --git a/+dj/ERD.m b/+dj/ERD.m index 669894a0..157ba363 100644 --- a/+dj/ERD.m +++ b/+dj/ERD.m @@ -144,7 +144,7 @@ function draw(self) for i=1:self.graph.numnodes if tiers(i)<7 % ignore jobs, logs, etc. isPart = tiers(i)==6; - fs = dj.set('erdFontSize')*(1 - 0.3*isPart); + fs = dj.config('displayDiagram_font_size')*(1 - 0.3*isPart); fc = isPart*0.3*[1 1 1]; name = self.conn.tableToClass(self.graph.Nodes.Name{i}); text(h.XData(i)+0.1, h.YData(i), name, ... diff --git a/+dj/Relvar.m b/+dj/Relvar.m index 802845ba..5b6c95ce 100755 --- a/+dj/Relvar.m +++ b/+dj/Relvar.m @@ -95,7 +95,7 @@ function cleanup(self) rels = rels(counts>0); % confirm and delete - if ~dj.set('suppressPrompt') && ~strcmpi('yes',dj.internal.ask('Proceed to delete?')) + if dj.config('safemode') && ~strcmpi('yes',dj.internal.ask('Proceed to delete?')) disp 'delete canceled' else self.schema.conn.startTransaction @@ -183,7 +183,7 @@ function insert(self, tuples, command) % The input argument tuples must a structure array with field % names exactly matching those in the table. % - % The ignoreExtraFields setting in dj.set allows ignoring fields + % The ignoreExtraFields setting in dj.config allows ignoring fields % in the tuples structure that are not found in the table. % % The optional argument 'command' can be of the following: @@ -219,7 +219,7 @@ function insert(self, tuples, command) fnames = fieldnames(tuples); found = ismember(fnames,header.names); if any(~found) - if dj.set('ignore_extra_insert_fields') + if dj.config('queryIgnore_extra_insert_fields') tuples = rmfield(tuples, fnames(~found)); fnames = fnames(found); else diff --git a/+dj/Schema.m b/+dj/Schema.m index 3b9042b7..759dd81f 100755 --- a/+dj/Schema.m +++ b/+dj/Schema.m @@ -177,7 +177,7 @@ function reload(self, force) self.tableNames.remove(self.tableNames.keys); % reload schema information into memory: table names and field named. - if dj.set('verbose') + if strcmpi(dj.config('loglevel'), 'DEBUG') fprintf('loading table definitions from %s... ', self.dbname), tic end tableInfo = self.conn.query(sprintf(... @@ -189,7 +189,7 @@ function reload(self, force) re = cellfun(@(x) sprintf('^%s%s[a-z][a-z0-9_]*$',self.prefix,x), ... dj.Schema.tierPrefixes, 'UniformOutput', false); % regular expressions to determine table tier - if dj.set('verbose') + if strcmpi(dj.config('loglevel'), 'DEBUG') fprintf('%.3g s\nloading field information... ', toc), tic end for info = dj.struct.fromFields(tableInfo)' @@ -200,11 +200,11 @@ function reload(self, force) self.headers(info.name) = dj.internal.Header.initFromDatabase(self,info); end - if dj.set('verbose') + if strcmpi(dj.config('loglevel'), 'DEBUG') fprintf('%.3g s\nloading dependencies... ', toc), tic end self.conn.loadDependencies(self) - if dj.set('verbose') + if strcmpi(dj.config('loglevel'), 'DEBUG') fprintf('%.3g s\n',toc) end end diff --git a/+dj/config.m b/+dj/config.m new file mode 100644 index 00000000..c49ead24 --- /dev/null +++ b/+dj/config.m @@ -0,0 +1,22 @@ +function res = config(name, value) + switch nargin + case 0 + out = dj.internal.Settings; + res = out.result; + case 1 + out = dj.internal.Settings(name); + res = out.result; + case 2 + switch nargout + case 0 + dj.internal.Settings(name, value); + case 1 + out = dj.internal.Settings(name, value); + res = out.result; + otherwise + error('Exceeded 1 output limit.') + end + otherwise + error('Exceeded 2 input limit.') + end +end \ No newline at end of file diff --git a/+dj/conn.m b/+dj/conn.m index b6c4cf10..7559ae8c 100644 --- a/+dj/conn.m +++ b/+dj/conn.m @@ -30,6 +30,12 @@ nogui = false; end +if nargin<1 || isempty(host) + host = dj.config('databaseHost'); +end +if ~contains(host, ':') + host = [host ':' num2str(dj.config('databasePort'))]; +end if isa(CONN, 'dj.Connection') && ~reset if nargin>0 @@ -46,24 +52,15 @@ end else % invoke setupDJ - % optional environment variables specifying the connection. - env = struct(... - 'host', 'DJ_HOST', ... - 'user', 'DJ_USER', ... - 'pass', 'DJ_PASS', ... - 'init', 'DJ_INIT'); % get host address - if nargin<1 || isempty(host) - host = getenv(env.host); - end if isempty(host) host = input('Enter datajoint host address> ','s'); end % get username if nargin<2 || isempty(user) - user = getenv(env.user); + user = dj.config('databaseUser'); end if isempty(user) user = input('Enter datajoint username> ', 's'); @@ -71,7 +68,7 @@ % get password if nargin<3 || isempty(pass) - pass = getenv(env.pass); + pass = dj.config('databasePassword'); end if isempty(pass) if nogui @@ -83,12 +80,12 @@ % get initial query (if any) to execute when a connection is (re)established if nargin<4 || isempty(initQuery) - initQuery = getenv(env.init); + initQuery = dj.config('connectionInit_function'); end % get tls option if nargin<6 || isempty(use_tls) - use_tls = dj.set('use_tls'); + use_tls = dj.config('databaseUse_tls'); end if islogical(use_tls) && ~use_tls diff --git a/+dj/set.m b/+dj/set.m deleted file mode 100644 index 9fcddb61..00000000 --- a/+dj/set.m +++ /dev/null @@ -1,49 +0,0 @@ -function out = set(name, value) -% dj.set - get or set a DataJoint setting -% -% USAGE: -% dj.set - view current settings -% v = dj.set('settingName') - get the value of a setting -% oldValue = dj.set('settingName', value) - set the value of a setting -% dj.set('restore') - restore defaults - -persistent STATE -if isempty(STATE) || (nargin==1 && strcmpi(name,'restore')) - % default settings - STATE = struct( ... - 'suppressPrompt', false, ... - 'reconnectTimedoutTransaction', true, ... - 'populateCheck', true, ... - 'tableErdRadius', [2 1], ... levels up and down the hierachy to display in `erd schema.Table` - 'erdFontSize', 12, ... font size to use in ERD labels - 'verbose', false, ... - 'populateAncestors', false, ... - 'bigint_to_double', false, ... - 'maxPreviewRows', 12, ... how many rows to display when previewing a relation - 'ignore_extra_insert_fields', false, ... when false, throws an error in `insert(self, tuple)` when tuple has extra fields. - 'use_tls', nan ... - ); -end - -if ~nargin && ~nargout - disp(STATE) -elseif nargout - out = STATE; -end - -if nargin~=1 || ~strcmpi(name, 'restore') - if nargin - assert(ischar(name), 'Setting name must be a string') - assert(isfield(STATE,name), 'Setting `%s` does not exist', name) - end - switch nargin - case 1 - out = STATE.(name); - case 2 - if nargout - out = STATE.(name); - end - STATE.(name) = value; - end -end -end diff --git a/+dj/version.m b/+dj/version.m index 2b79a3cb..273db493 100644 --- a/+dj/version.m +++ b/+dj/version.m @@ -1,7 +1,7 @@ function varargout = version % report DataJoint version -v = struct('major',3,'minor',3,'bugfix',1); +v = struct('major',3,'minor',4,'bugfix',0); if nargout varargout{1}=v; diff --git a/+dj/+lib/compareVersions.m b/+tests/+lib/compareVersions.m similarity index 100% rename from +dj/+lib/compareVersions.m rename to +tests/+lib/compareVersions.m diff --git a/+tests/Main.m b/+tests/Main.m index e7986e8e..702007b7 100644 --- a/+tests/Main.m +++ b/+tests/Main.m @@ -1,102 +1,5 @@ classdef Main < ... + tests.TestConfig & ... tests.TestConnection & ... tests.TestTls - - properties (Constant) - CONN_INFO_ROOT = struct(... - 'host', getenv('DJ_HOST'), ... - 'user', getenv('DJ_USER'), ... - 'password', getenv('DJ_PASS')); - CONN_INFO = struct(... - 'host', getenv('DJ_TEST_HOST'), ... - 'user', getenv('DJ_TEST_USER'), ... - 'password', getenv('DJ_TEST_PASSWORD')); - end - - methods (TestClassSetup) - function init(testCase) - disp('---------------INIT---------------'); - clear functions; - testCase.addTeardown(@testCase.dispose); - - curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... - testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password,'',true); - - ver = curr_conn.query('select @@version as version').version; - if dj.lib.compareVersions(ver,'5.8') - cmd = {... - 'CREATE USER IF NOT EXISTS ''datajoint''@''%%'' ' - 'IDENTIFIED BY ''datajoint'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'CREATE USER IF NOT EXISTS ''djview''@''%%'' ' - 'IDENTIFIED BY ''djview'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'CREATE USER IF NOT EXISTS ''djssl''@''%%'' ' - 'IDENTIFIED BY ''djssl'' ' - 'REQUIRE SSL;' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - else - cmd = {... - 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'' ' - 'IDENTIFIED BY ''datajoint'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'' ' - 'IDENTIFIED BY ''djview'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'' ' - 'IDENTIFIED BY ''djssl'' ' - 'REQUIRE SSL;' - }; - curr_conn.query(sprintf('%s',cmd{:})); - end - end - end - - methods (Static) - function dispose() - disp('---------------DISP---------------'); - warning('off','MATLAB:RMDIR:RemovedFromPath'); - - curr_conn = dj.conn(tests.Main.CONN_INFO_ROOT.host, ... - tests.Main.CONN_INFO_ROOT.user, tests.Main.CONN_INFO_ROOT.password, '',true); - - cmd = {... - 'DROP USER ''datajoint''@''%%'';' - 'DROP USER ''djview''@''%%'';' - 'DROP USER ''djssl''@''%%'';' - }; - res = curr_conn.query(sprintf('%s',cmd{:})); - curr_conn.delete; - - warning('on','MATLAB:RMDIR:RemovedFromPath'); - end - end -end +end \ No newline at end of file diff --git a/+tests/Prep.m b/+tests/Prep.m new file mode 100644 index 00000000..c2c3b8f5 --- /dev/null +++ b/+tests/Prep.m @@ -0,0 +1,128 @@ +classdef Prep < matlab.unittest.TestCase + % Setup and teardown for tests. + properties (Constant) + CONN_INFO_ROOT = struct(... + 'host', getenv('DJ_HOST'), ... + 'user', getenv('DJ_USER'), ... + 'password', getenv('DJ_PASS')); + CONN_INFO = struct(... + 'host', getenv('DJ_TEST_HOST'), ... + 'user', getenv('DJ_TEST_USER'), ... + 'password', getenv('DJ_TEST_PASSWORD')); + PREFIX = 'djtest'; + end + properties + test_root; + end + methods + function obj = Prep() + % Initialize test_root + test_pkg_details = what('tests'); + [test_root, ~, ~] = fileparts(test_pkg_details.path); + obj.test_root = [test_root '/+tests']; + end + end + methods (TestClassSetup) + function init(testCase) + disp('---------------INIT---------------'); + clear functions; + addpath([testCase.test_root '/test_schemas']); + + curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... + testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password,'',true); + + ver = curr_conn.query('select @@version as version').version; + if tests.lib.compareVersions(ver,'5.8') + cmd = {... + 'CREATE USER IF NOT EXISTS ''datajoint''@''%%'' ' + 'IDENTIFIED BY ''datajoint'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'CREATE USER IF NOT EXISTS ''djview''@''%%'' ' + 'IDENTIFIED BY ''djview'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'CREATE USER IF NOT EXISTS ''djssl''@''%%'' ' + 'IDENTIFIED BY ''djssl'' ' + 'REQUIRE SSL;' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + else + cmd = {... + 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'' ' + 'IDENTIFIED BY ''datajoint'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'' ' + 'IDENTIFIED BY ''djview'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'' ' + 'IDENTIFIED BY ''djssl'' ' + 'REQUIRE SSL;' + }; + curr_conn.query(sprintf('%s',cmd{:})); + end + end + end + methods (TestClassTeardown) + function dispose(testCase) + disp('---------------DISP---------------'); + warning('off','MATLAB:RMDIR:RemovedFromPath'); + + curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... + testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password, '',true); + + curr_conn.query('SET FOREIGN_KEY_CHECKS=0;'); + res = curr_conn.query(['SHOW DATABASES LIKE "' testCase.PREFIX '_%";']); + for i = 1:length(res.(['Database (' testCase.PREFIX '_%)'])) + curr_conn.query(['DROP DATABASE ' ... + res.(['Database (' testCase.PREFIX '_%)']){i} ';']); + end + curr_conn.query('SET FOREIGN_KEY_CHECKS=1;'); + + cmd = {... + 'DROP USER ''datajoint''@''%%'';' + 'DROP USER ''djview''@''%%'';' + 'DROP USER ''djssl''@''%%'';' + }; + res = curr_conn.query(sprintf('%s',cmd{:})); + curr_conn.delete; + + % Remove getSchemas to ensure they are created by tests. + files = dir([testCase.test_root '/test_schemas']); + dirFlags = [files.isdir] & ~strcmp({files.name},'.') & ~strcmp({files.name},'..'); + subFolders = files(dirFlags); + for k = 1 : length(subFolders) + delete([testCase.test_root '/test_schemas/' subFolders(k).name ... + '/getSchema.m']); + % delete(['test_schemas/+University/getSchema.m']) + end + rmpath([testCase.test_root '/test_schemas']); + warning('on','MATLAB:RMDIR:RemovedFromPath'); + end + end +end \ No newline at end of file diff --git a/+tests/TestConfig.m b/+tests/TestConfig.m new file mode 100644 index 00000000..1c4cc29e --- /dev/null +++ b/+tests/TestConfig.m @@ -0,0 +1,227 @@ +classdef TestConfig < tests.Prep + % TestConfig tests scenarios related to initializing DJ config. + methods (Static) + function obj = configRemoveEnvVars(obj, type) + switch type + case 'file' + if isfield(obj, 'database_host') + obj = rmfield(obj, 'database_host'); + end + if isfield(obj, 'database_user') + obj = rmfield(obj, 'database_user'); + end + if isfield(obj, 'database_password') + obj = rmfield(obj, 'database_password'); + end + if isfield(obj, 'connection_init_function') + obj = rmfield(obj, 'connection_init_function'); + end + case 'config' + if isfield(obj, 'databaseHost') + obj = rmfield(obj, 'databaseHost'); + end + if isfield(obj, 'databaseUser') + obj = rmfield(obj, 'databaseUser'); + end + if isfield(obj, 'databasePassword') + obj = rmfield(obj, 'databasePassword'); + end + if isfield(obj, 'connectionInit_function') + obj = rmfield(obj, 'connectionInit_function'); + end + end + end + function configSingleFileTest(test_instance, type, fname, base) + switch type + case 'save-local' + dj.config.saveLocal(); + fname = dj.internal.Settings.LOCALFILE; + case 'save-global' + dj.config.saveGlobal(); + fname = dj.internal.Settings.GLOBALFILE; + case 'save-custom' + dj.config.save(fname); + case 'load-custom' + dj.config.load(fname); + end + % load raw + read_data = fileread(fname); + obj1 = tests.TestConfig.configRemoveEnvVars(jsondecode(read_data), 'file'); + % optional merge from base + if strcmpi(type, 'load-custom') + tmp = rmfield(base, intersect(fieldnames(base), fieldnames(obj1))); + names = [fieldnames(tmp); fieldnames(obj1)]; + obj1 = orderfields(cell2struct([struct2cell(tmp); ... + struct2cell(obj1)], names, 1)); + end + % stringify + file = jsonencode(obj1); + % load config + obj2 = tests.TestConfig.configRemoveEnvVars(dj.config(), 'config'); + curr = jsonencode(obj2); + curr = regexprep(curr,'[a-z0-9][A-Z]','${$0(1)}_${lower($0(2))}'); + % checks + verifyEqual(test_instance, curr, file); + assert(~contains(read_data, '[]')); + % optional remove file + if ~strcmpi(type, 'load-custom') + delete(fname); + end + end + end + methods (Test) + function testGetSet(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + function verifyConfig(new, previous_value, subref, subref_value, subref_prev) + keys = fieldnames(new); + try + c_prev = dj.config(keys{1}, new.(keys{1})); + catch ME + switch ME.identifier + case 'DataJoint:Config:InvalidKey' + dj.config(keys{1}, new.(keys{1})); + otherwise + rethrow(ME); + end + end + c_curr = dj.config(keys{1}); + if nargin > 1 + verifyEqual(testCase, c_prev, previous_value); + elseif exist('c_prev','var') + verifyNotEqual(testCase, c_curr, c_prev); + end + verifyEqual(testCase, c_curr, new.(keys{1})); + if exist('subref','var') + % eval less efficient but keeps test simple for 1 test + eval(['c_prev = dj.config(''' subref ''', ''' subref_value ''');']); + eval(['c_curr = dj.config(''' subref ''');']); + verifyEqual(testCase, c_prev, subref_prev); + verifyEqual(testCase, c_curr, subref_value); + eval(['new.' subref ' = subref_value;']); + verifyEqual(testCase, dj.config(keys{1}), new.(keys{1})); + end + end + dj.config.restore; + % check update a default config + verifyConfig(struct('displayLimit', 15)); + % check create new config + prev = 'Neuro'; + verifyConfig(struct('project', prev)); + % check update newly created config + verifyConfig(struct('project', 'Lab'), prev); + % check create new struct array config + prev = [ ... + struct(... + 'protocol', 'file', ... + 'location', '/tmp', ... + 'subfolding', [1,1] ... + ), struct(... + 'protocol', 's3', ... + 'location', '/home', ... + 'subfolding', [2,2] ... + )]; + verifyConfig(struct('stores', prev)); + % check update to cell array config, and check update nested config + verifyConfig(struct('stores', {{ ... + struct(... + 'protocol', 'file', ... + 'location', '/tmp', ... + 'subfolding', [1,1] ... + ), struct(... + 'protocol', 's3', ... + 'location', '/home', ... + 'subfolding', [2,2] ... + )}}), prev, 'stores{2}.protocol', 'http', 's3'); + end + function testConfigChecks(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + testCase.verifyError(@() dj.config(9), ... + 'DataJoint:Config:InvalidType'); + d = testCase.verifyError(@() dj.config('none'), ... + 'DataJoint:Config:InvalidKey'); + end + function testRestore(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + dj.config.restore; + obj1 = tests.TestConfig.configRemoveEnvVars(dj.config(), 'config'); + obj2 = tests.TestConfig.configRemoveEnvVars( ... + orderfields(dj.internal.Settings.DEFAULTS), 'config'); + testCase.verifyEqual(jsonencode(obj1), jsonencode(obj2)); + end + function testSave(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + dj.config.restore; + + % local + dj.config('font', 10); + tests.TestConfig.configSingleFileTest(testCase, 'save-local'); + % global + dj.config('font', 12); + tests.TestConfig.configSingleFileTest(testCase, 'save-global'); + % custom + dj.config('font', 16); + tests.TestConfig.configSingleFileTest(testCase, 'save-custom', './config.json'); + + dj.config.restore; + end + function testLoad(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + pkg = what('tests'); + % generate default base + default_file = [pkg.path '/test_schemas/default.json']; + dj.config.restore; + dj.config.save(default_file); + defaults = tests.TestConfig.configRemoveEnvVars( ... + jsondecode(fileread(default_file)), 'file'); + delete(default_file); + % load test config + tests.TestConfig.configSingleFileTest(testCase, 'load-custom', ... + [pkg.path '/test_schemas/config.json'], defaults); + % load new config on top of existing + base = tests.TestConfig.configRemoveEnvVars(dj.config, 'config'); + base = jsonencode(base); + base = regexprep(base,'[a-z0-9][A-Z]','${$0(1)}_${lower($0(2))}'); + tests.TestConfig.configSingleFileTest(testCase, 'load-custom', ... + [pkg.path '/test_schemas/config_lite.json'], jsondecode(base)); + % cleanup + dj.config.restore; + end + function testEnv(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + function validateEnvVarConfig(type, values) + switch type + case 'set' + dj.config('databaseHost', values{1}); + dj.config('databaseUser', values{2}); + dj.config('databasePassword', values{3}); + dj.config('connectionInit_function', values{4}); + end + testCase.verifyEqual(dj.config('databaseHost'), values{1}); + testCase.verifyEqual(dj.config('databaseUser'), values{2}); + testCase.verifyEqual(dj.config('databasePassword'), values{3}); + testCase.verifyEqual(dj.config('connectionInit_function'), values{4}); + end + pkg = what('tests'); + setenv('DJ_INIT', 'select @@version;'); + dj.config.restore; + % check pulling from env vars + env = {getenv('DJ_HOST'), getenv('DJ_USER'), getenv('DJ_PASS'), getenv('DJ_INIT')}; + validateEnvVarConfig('env', env); + % check after load if env vars take precedence + dj.config.load([pkg.path '/test_schemas/config.json']); + validateEnvVarConfig('env', env); + % check if overriding env vars is persisted + validateEnvVarConfig('set', ... + {'localhost', 'john', 'secure', 'SET SESSION sql_mode="TRADITIONAL";'}); + % cleanup + setenv('DJ_INIT', ''); + dj.config.restore; + end + end +end \ No newline at end of file diff --git a/+tests/TestConnection.m b/+tests/TestConnection.m index d57d901b..61914dc7 100644 --- a/+tests/TestConnection.m +++ b/+tests/TestConnection.m @@ -1,4 +1,4 @@ -classdef TestConnection < matlab.unittest.TestCase +classdef TestConnection < tests.Prep % TestConnection tests typical connection scenarios. methods (Test) function testConnection(testCase) @@ -15,8 +15,8 @@ function testConnectionExists(testCase) % Fix https://github.com/datajoint/datajoint-matlab/issues/160 st = dbstack; disp(['---------------' st(1).name '---------------']); - dj.conn(testCase.CONN_INFO.host, '', '', '', '', true) - dj.conn(testCase.CONN_INFO.host, '', '', '', '', true) + dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); + dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); end function testConnectionDiffHost(testCase) % testConnectionDiffHost tests that will fail if connection open @@ -24,11 +24,20 @@ function testConnectionDiffHost(testCase) % Fix https://github.com/datajoint/datajoint-matlab/issues/160 st = dbstack; disp(['---------------' st(1).name '---------------']); - dj.conn(testCase.CONN_INFO.host, '', '', '', '', true) + dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); testCase.verifyError(@() dj.conn(... 'anything', '', '', '', '', true), ... 'DataJoint:Connection:AlreadyInstantiated'); end + function testPort(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + testCase.verifyError(@() dj.conn(... + [testCase.CONN_INFO.host ':3307'], ... + testCase.CONN_INFO.user,... + testCase.CONN_INFO.password,'',true), ... + 'MySQL:Error'); + end end end \ No newline at end of file diff --git a/+tests/TestTls.m b/+tests/TestTls.m index 9da70f8a..cdda43d0 100644 --- a/+tests/TestTls.m +++ b/+tests/TestTls.m @@ -1,4 +1,4 @@ -classdef TestTls < matlab.unittest.TestCase +classdef TestTls < tests.Prep % TestTls tests TLS connection scenarios. methods (Test) function testSecureConn(testCase) diff --git a/+tests/test_schemas/config.json b/+tests/test_schemas/config.json new file mode 100644 index 00000000..e035eef8 --- /dev/null +++ b/+tests/test_schemas/config.json @@ -0,0 +1,31 @@ +{ + "database.host": "", + "database.password": "", + "database.user": "", + "database.port": 3306, + "database.reconnect": true, + "connection.init_function": null, + "connection.charset": "", + "loglevel": "INFO", + "safemode": true, + "fetch_format": "array", + "display.limit": [12, 5], + "display.width": 14, + "display.show_tuple_count": true, + "database.use_tls": null, + "enable_python_native_blobs": false, + "custom.accessories" : { + "item.type": "keyboard", + "location" : "office" + }, + "stores.options" : [ + { + "protocol.type": "file", + "location" : "/tmp" + }, + { + "protocol.type": "s3", + "location" : "/home" + } + ] +} \ No newline at end of file diff --git a/+tests/test_schemas/config_lite.json b/+tests/test_schemas/config_lite.json new file mode 100644 index 00000000..d9c94a15 --- /dev/null +++ b/+tests/test_schemas/config_lite.json @@ -0,0 +1,3 @@ +{ + "database.use_tls" : true +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3a9626a7..5dbb4ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ mym/ *.mltbx *.env +notebook +docker-compose.yml \ No newline at end of file