From 236a4d0348e0a0562ef25888302254e0a02f3d32 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 15 Apr 2020 07:47:30 -0500 Subject: [PATCH 1/6] Fix update issue and add appropriate schema + tests. --- +dj/Relvar.m | 15 ++- +tests/+lib/compareVersions.m | 100 +++++++++++++++++++ +tests/Main.m | 101 +------------------ +tests/Prep.m | 135 ++++++++++++++++++++++++++ +tests/TestConnection.m | 23 +++-- +tests/TestRelationalOperand.m | 36 +++++++ +tests/TestTls.m | 14 +-- +tests/test_schemas/+University/All.m | 12 +++ .gitignore | 7 ++ 9 files changed, 328 insertions(+), 115 deletions(-) create mode 100644 +tests/+lib/compareVersions.m create mode 100644 +tests/Prep.m create mode 100644 +tests/TestRelationalOperand.m create mode 100644 +tests/test_schemas/+University/All.m diff --git a/+dj/Relvar.m b/+dj/Relvar.m index 802845ba..44ecda84 100755 --- a/+dj/Relvar.m +++ b/+dj/Relvar.m @@ -352,14 +352,25 @@ function update(self, attrname, value) switch true case isNull + assert(header.attributes(ix).isnullable, ... + 'attribute `%s` is not nullable.', attrname) valueStr = 'NULL'; value = {}; case header.attributes(ix).isString assert(dj.lib.isString(value), 'Value must be a string') - valueStr = '"{S}"'; - value = {char(value)}; + if isempty(value) + assert(header.attributes(ix).isnullable, ... + 'attribute `%s` is not nullable.', attrname) + valueStr = 'NULL'; + value = {}; + else + valueStr = '"{S}"'; + value = {char(value)}; + end case header.attributes(ix).isBlob if isempty(value) && header.attributes(ix).isnullable + assert(header.attributes(ix).isnullable, ... + 'attribute `%s` is not nullable.', attrname) valueStr = 'NULL'; value = {}; else diff --git a/+tests/+lib/compareVersions.m b/+tests/+lib/compareVersions.m new file mode 100644 index 00000000..46d7b27d --- /dev/null +++ b/+tests/+lib/compareVersions.m @@ -0,0 +1,100 @@ +function res = compareVersions(verArray, verComp) + % compareVersions - Semantic version comparison (greater than or equal) + % + % This function evaluates if an array of semantic versions is greater than + % or equal to a reference version. + % + % DISTRIBUTION: + % GitHub: https://github.com/guzman-raphael/compareVersions + % FileExchange: https://www.mathworks.com/matlabcentral/fileexchange/71849-compareversions + % + % res = compareVersions(verArray, verComp) + % INPUT: + % verArray: Cell array with the following conditions: + % - be of length >= 1, + % - contain only string elements, and + % - each element must be of length >= 1. + % verComp: String or Char array that verArray will compare against for + % greater than evaluation. Must be: + % - be of length >= 1, and + % - a string. + % OUTPUT: + % res: Logical array that identifies if each cell element in verArray + % is greater than or equal to verComp. + % TESTS: + % Tests included for reference. From root package directory, + % use command: runtests + % + % EXAMPLES: + % output = compareVersions({'3.2.4beta','9.5.2.1','8.0'}, '8.0.0'); %logical([0 1 1]) + % + % NOTES: + % Tests included for reference. From root package directory, + % use command: runtests + % + % Tested: Matlab 9.5.0.944444 (R2018b) Linux + % Author: Raphael Guzman, DataJoint + % + % $License: MIT (use/copy/change/redistribute on own risk) $ + % $File: compareVersions.m $ + % History: + % 001: 2019-06-12 11:00, First version. + % + % OPEN BUGS: + % - None + res_n = length(verArray); + if ~res_n || max(cellfun(@(c) ~ischar(c) && ... + ~isstring(c),verArray)) > 0 || min(cellfun('length',verArray)) == 0 + msg = { + 'compareVersions:Error:CellArray' + 'Cell array to verify must:' + '- be of length >= 1,' + '- contain only string elements, and' + '- each element must be of length >= 1.' + }; + error('compareVersions:Error:CellArray', sprintf('%s\n',msg{:})); + end + if ~ischar(verComp) && ~isstring(verComp) || length(verComp) == 0 + msg = { + 'compareVersions:Error:VersionRef' + 'Version reference must:' + '- be of length >= 1, and' + '- a string.' + }; + error('compareVersions:Error:VersionRef', sprintf('%s\n',msg{:})); + end + res = false(1, res_n); + for i = 1:res_n + shortVer = strsplit(verArray{i}, '.'); + shortVer = cellfun(@(x) str2double(regexp(x,'\d*','Match')), shortVer(1,:)); + longVer = strsplit(verComp, '.'); + longVer = cellfun(@(x) str2double(regexp(x,'\d*','Match')), longVer(1,:)); + shortVer_p = true; + longVer_p = false; + shortVer_s = length(shortVer); + longVer_s = length(longVer); + + if shortVer_s > longVer_s + [longVer shortVer] = deal(shortVer,longVer); + [longVer_s shortVer_s] = deal(shortVer_s,longVer_s); + [longVer_p shortVer_p] = deal(shortVer_p,longVer_p); + end + + shortVer = [shortVer zeros(1,longVer_s - shortVer_s)]; + diff = shortVer - longVer; + match = diff ~= 0; + + if ~match + res(i) = true; + else + pos = 1:longVer_s; + pos = pos(match); + val = diff(pos(1)); + if val > 0 + res(i) = shortVer_p; + elseif val < 0 + res(i) = longVer_p; + end + end + end +end \ No newline at end of file diff --git a/+tests/Main.m b/+tests/Main.m index e7986e8e..4f14f0a7 100644 --- a/+tests/Main.m +++ b/+tests/Main.m @@ -1,102 +1,5 @@ classdef Main < ... tests.TestConnection & ... + tests.TestRelationalOperand & ... tests.TestTls - - properties (Constant) - CONN_INFO_ROOT = struct(... - 'host', getenv('DJ_HOST'), ... - 'user', getenv('DJ_USER'), ... - 'password', getenv('DJ_PASS')); - CONN_INFO = struct(... - 'host', getenv('DJ_TEST_HOST'), ... - 'user', getenv('DJ_TEST_USER'), ... - 'password', getenv('DJ_TEST_PASSWORD')); - end - - methods (TestClassSetup) - function init(testCase) - disp('---------------INIT---------------'); - clear functions; - testCase.addTeardown(@testCase.dispose); - - curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... - testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password,'',true); - - ver = curr_conn.query('select @@version as version').version; - if dj.lib.compareVersions(ver,'5.8') - cmd = {... - 'CREATE USER IF NOT EXISTS ''datajoint''@''%%'' ' - 'IDENTIFIED BY ''datajoint'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'CREATE USER IF NOT EXISTS ''djview''@''%%'' ' - 'IDENTIFIED BY ''djview'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'CREATE USER IF NOT EXISTS ''djssl''@''%%'' ' - 'IDENTIFIED BY ''djssl'' ' - 'REQUIRE SSL;' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - else - cmd = {... - 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'' ' - 'IDENTIFIED BY ''datajoint'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'' ' - 'IDENTIFIED BY ''djview'';' - }; - curr_conn.query(sprintf('%s',cmd{:})); - - cmd = {... - 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'' ' - 'IDENTIFIED BY ''djssl'' ' - 'REQUIRE SSL;' - }; - curr_conn.query(sprintf('%s',cmd{:})); - end - end - end - - methods (Static) - function dispose() - disp('---------------DISP---------------'); - warning('off','MATLAB:RMDIR:RemovedFromPath'); - - curr_conn = dj.conn(tests.Main.CONN_INFO_ROOT.host, ... - tests.Main.CONN_INFO_ROOT.user, tests.Main.CONN_INFO_ROOT.password, '',true); - - cmd = {... - 'DROP USER ''datajoint''@''%%'';' - 'DROP USER ''djview''@''%%'';' - 'DROP USER ''djssl''@''%%'';' - }; - res = curr_conn.query(sprintf('%s',cmd{:})); - curr_conn.delete; - - warning('on','MATLAB:RMDIR:RemovedFromPath'); - end - end -end +end \ No newline at end of file diff --git a/+tests/Prep.m b/+tests/Prep.m new file mode 100644 index 00000000..47395249 --- /dev/null +++ b/+tests/Prep.m @@ -0,0 +1,135 @@ +classdef Prep < matlab.unittest.TestCase + % Setup and teardown for tests. + properties (Constant) + CONN_INFO_ROOT = struct(... + 'host', getenv('DJ_HOST'), ... + 'user', getenv('DJ_USER'), ... + 'password', getenv('DJ_PASS')); + CONN_INFO = struct(... + 'host', getenv('DJ_TEST_HOST'), ... + 'user', getenv('DJ_TEST_USER'), ... + 'password', getenv('DJ_TEST_PASSWORD')); + S3_CONN_INFO = struct(... + 'endpoint', getenv('S3_ENDPOINT'), ... + 'access_key', getenv('S3_ACCESS_KEY'), ... + 'secret_key', getenv('S3_SECRET_KEY'), ... + 'bucket', getenv('S3_BUCKET')); + PREFIX = 'djtest'; + end + properties + test_root; + end + methods + function obj = Prep() + % Initialize test_root + test_pkg_details = what('tests'); + [test_root, ~, ~] = fileparts(test_pkg_details.path); + obj.test_root = [test_root '/+tests']; + end + end + methods (TestClassSetup) + function init(testCase) + disp('---------------INIT---------------'); + clear functions; + addpath([testCase.test_root '/test_schemas']); + + curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... + testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password,'',true); + % create test users + ver = curr_conn.query('select @@version as version').version; + if tests.lib.compareVersions(ver,'5.8') + cmd = {... + 'CREATE USER IF NOT EXISTS ''datajoint''@''%%'' ' + 'IDENTIFIED BY ''datajoint'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'CREATE USER IF NOT EXISTS ''djview''@''%%'' ' + 'IDENTIFIED BY ''djview'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'CREATE USER IF NOT EXISTS ''djssl''@''%%'' ' + 'IDENTIFIED BY ''djssl'' ' + 'REQUIRE SSL;' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + else + cmd = {... + 'GRANT ALL PRIVILEGES ON `djtest%%`.* TO ''datajoint''@''%%'' ' + 'IDENTIFIED BY ''datajoint'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djview''@''%%'' ' + 'IDENTIFIED BY ''djview'';' + }; + curr_conn.query(sprintf('%s',cmd{:})); + + cmd = {... + 'GRANT SELECT ON `djtest%%`.* TO ''djssl''@''%%'' ' + 'IDENTIFIED BY ''djssl'' ' + 'REQUIRE SSL;' + }; + curr_conn.query(sprintf('%s',cmd{:})); + end + end + end + methods (TestClassTeardown) + function dispose(testCase) + disp('---------------DISP---------------'); + warning('off','MATLAB:RMDIR:RemovedFromPath'); + + curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ... + testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password, '',true); + + % remove databases + curr_conn.query('SET FOREIGN_KEY_CHECKS=0;'); + res = curr_conn.query(['SHOW DATABASES LIKE "' testCase.PREFIX '_%";']); + for i = 1:length(res.(['Database (' testCase.PREFIX '_%)'])) + curr_conn.query(['DROP DATABASE ' ... + res.(['Database (' testCase.PREFIX '_%)']){i} ';']); + end + curr_conn.query('SET FOREIGN_KEY_CHECKS=1;'); + + % remove users + cmd = {... + 'DROP USER ''datajoint''@''%%'';' + 'DROP USER ''djview''@''%%'';' + 'DROP USER ''djssl''@''%%'';' + }; + res = curr_conn.query(sprintf('%s',cmd{:})); + curr_conn.delete; + + % Remove getSchemas to ensure they are created by tests. + files = dir([testCase.test_root '/test_schemas']); + dirFlags = [files.isdir] & ~strcmp({files.name},'.') & ~strcmp({files.name},'..'); + subFolders = files(dirFlags); + for k = 1 : length(subFolders) + delete([testCase.test_root '/test_schemas/' subFolders(k).name ... + '/getSchema.m']); + % delete(['test_schemas/+University/getSchema.m']) + end + rmpath([testCase.test_root '/test_schemas']); + warning('on','MATLAB:RMDIR:RemovedFromPath'); + end + end +end \ No newline at end of file diff --git a/+tests/TestConnection.m b/+tests/TestConnection.m index d57d901b..92b113a3 100644 --- a/+tests/TestConnection.m +++ b/+tests/TestConnection.m @@ -1,7 +1,7 @@ -classdef TestConnection < matlab.unittest.TestCase +classdef TestConnection < tests.Prep % TestConnection tests typical connection scenarios. methods (Test) - function testConnection(testCase) + function TestConnection_testConnection(testCase) st = dbstack; disp(['---------------' st(1).name '---------------']); testCase.verifyTrue(dj.conn(... @@ -9,26 +9,35 @@ function testConnection(testCase) testCase.CONN_INFO.user,... testCase.CONN_INFO.password,'',true).isConnected); end - function testConnectionExists(testCase) + function TestConnection_testConnectionExists(testCase) % testConnectionExists tests that will not fail if connection open % to the same host. % Fix https://github.com/datajoint/datajoint-matlab/issues/160 st = dbstack; disp(['---------------' st(1).name '---------------']); - dj.conn(testCase.CONN_INFO.host, '', '', '', '', true) - dj.conn(testCase.CONN_INFO.host, '', '', '', '', true) + dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); + dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); end - function testConnectionDiffHost(testCase) + function TestConnection_testConnectionDiffHost(testCase) % testConnectionDiffHost tests that will fail if connection open % to a different host. % Fix https://github.com/datajoint/datajoint-matlab/issues/160 st = dbstack; disp(['---------------' st(1).name '---------------']); - dj.conn(testCase.CONN_INFO.host, '', '', '', '', true) + dj.conn(testCase.CONN_INFO.host, '', '', '', '', true); testCase.verifyError(@() dj.conn(... 'anything', '', '', '', '', true), ... 'DataJoint:Connection:AlreadyInstantiated'); end + function TestConnection_testPort(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + testCase.verifyError(@() dj.conn(... + [testCase.CONN_INFO.host ':3307'], ... + testCase.CONN_INFO.user,... + testCase.CONN_INFO.password,'',true), ... + 'MySQL:Error'); + end end end \ No newline at end of file diff --git a/+tests/TestRelationalOperand.m b/+tests/TestRelationalOperand.m new file mode 100644 index 00000000..caf31207 --- /dev/null +++ b/+tests/TestRelationalOperand.m @@ -0,0 +1,36 @@ +classdef TestRelationalOperand < tests.Prep + % TestRelationalOperand tests relational operations. + methods (Test) + function TestRelationalOperand_testUpdateDate(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + % https://github.com/datajoint/datajoint-matlab/issues/211 + package = 'University'; + + c1 = dj.conn(... + testCase.CONN_INFO.host,... + testCase.CONN_INFO.user,... + testCase.CONN_INFO.password,'',true); + + dj.createSchema(package,[testCase.test_root '/test_schemas'], ... + [testCase.PREFIX '_university']); + + insert(University.All, struct( ... + 'id', 2, ... + 'date', '2019-12-20' ... + )); + q = University.All & 'id=2'; + + new_value = ''; + q.update('date', new_value); + testCase.verifyEqual(q.fetch1('date'), new_value); + + new_value = '2020-04-14'; + q.update('date', new_value); + testCase.verifyEqual(q.fetch1('date'), new_value); + + q.update('date'); + testCase.verifyEqual(q.fetch1('date'), ''); + end + end +end \ No newline at end of file diff --git a/+tests/TestTls.m b/+tests/TestTls.m index 9da70f8a..a340afb6 100644 --- a/+tests/TestTls.m +++ b/+tests/TestTls.m @@ -1,7 +1,7 @@ -classdef TestTls < matlab.unittest.TestCase +classdef TestTls < tests.Prep % TestTls tests TLS connection scenarios. methods (Test) - function testSecureConn(testCase) + function TestTls_testSecureConn(testCase) % secure connection test st = dbstack; disp(['---------------' st(1).name '---------------']); @@ -12,7 +12,7 @@ function testSecureConn(testCase) '',true,true).query(... 'SHOW STATUS LIKE ''Ssl_cipher''').Value{1}) > 0); end - function testInsecureConn(testCase) + function TestTls_testInsecureConn(testCase) % insecure connection test st = dbstack; disp(['---------------' st(1).name '---------------']); @@ -24,7 +24,7 @@ function testInsecureConn(testCase) 'SHOW STATUS LIKE ''Ssl_cipher''').Value{1}, ... ''); end - function testPreferredConn(testCase) + function TestTls_testPreferredConn(testCase) % preferred connection test st = dbstack; disp(['---------------' st(1).name '---------------']); @@ -35,7 +35,7 @@ function testPreferredConn(testCase) '',true).query(... 'SHOW STATUS LIKE ''Ssl_cipher''').Value{1}) > 0); end - function testRejectException(testCase) + function TestTls_testRejectException(testCase) % test exception on require TLS st = dbstack; disp(['---------------' st(1).name '---------------']); @@ -54,7 +54,7 @@ function testRejectException(testCase) ["requires secure connection","Access denied"])); %MySQL8,MySQL5 end end - function testStructException(testCase) + function TestTls_testStructException(testCase) % test exception on TLS struct st = dbstack; disp(['---------------' st(1).name '---------------']); @@ -63,7 +63,7 @@ function testStructException(testCase) testCase.CONN_INFO.user, ... testCase.CONN_INFO.password, ... '',true,struct('ca','fake/path/some/where')), ... - 'DataJoint:TLS:InvalidStruct'); + 'mYm:TLS:InvalidStruct'); end end end \ No newline at end of file diff --git a/+tests/test_schemas/+University/All.m b/+tests/test_schemas/+University/All.m new file mode 100644 index 00000000..55d4be71 --- /dev/null +++ b/+tests/test_schemas/+University/All.m @@ -0,0 +1,12 @@ +%{ +# All +id : int +--- +string=null : varchar(30) +datetime=null : datetime +date=null : date +number=null : float +blob=null : longblob +%} +classdef All < dj.Manual +end \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3a9626a7..c9a8c43f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,10 @@ mym/ *.mltbx *.env +notebook +*getSchema.m +docker-compose.yml +.vscode +matlab.prf +win.* +macos.* \ No newline at end of file From 9bc424fb96c2d6d2fd4f585b127024a37939d86c Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 15 Apr 2020 08:59:31 -0500 Subject: [PATCH 2/6] Fix travis. --- .travis.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6ee789ba..8bb0c8ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ branches: - only: - - master + except: + - /^stage.*$/ sudo: required services: - docker @@ -8,9 +8,9 @@ env: global: - DOCKER_CLIENT_TIMEOUT: 120 - COMPOSE_HTTP_TIMEOUT: 120 - - MATLAB_USER: muser - MATLAB_UID: 2000 - MATLAB_GID: 2000 + - MINIO_VER: RELEASE.2019-09-26T19-42-35Z slim: &slim stage: Slim9.9 os: linux @@ -18,18 +18,22 @@ slim: &slim script: - license=MATLAB_LICENSE_${MATLAB_VERSION} - export MATLAB_LICENSE=$(eval echo "\$$license") - - docker-compose -f LNX-docker-compose.yml up --exit-code-from dj + - docker-compose -f LNX-docker-compose.yml up --exit-code-from app jobs: include: - <<: *slim env: - - MATLAB_VERSION: R2018b + - MATLAB_VERSION: R2019a - MYSQL_TAG: 8.0 - <<: *slim env: - - MATLAB_VERSION: R2018b + - MATLAB_VERSION: R2019a - MYSQL_TAG: 5.7 + - <<: *slim + env: + - MATLAB_VERSION: R2019a + - MYSQL_TAG: 5.6 - <<: *slim env: - MATLAB_VERSION: R2018b - - MYSQL_TAG: 5.6 \ No newline at end of file + - MYSQL_TAG: 5.7 \ No newline at end of file From fd21e7c955dedca498ef13cff523a48cf0ce9a78 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 15 Apr 2020 09:06:29 -0500 Subject: [PATCH 3/6] Update test structure and add docs on how to run tests locally. --- LNX-docker-compose.yml | 30 ++++++++++------- README.md | 46 ++++++++++++++++++++++++++ local-docker-compose.yml | 70 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 11 deletions(-) create mode 100644 local-docker-compose.yml diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 42897e1a..062b80af 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -1,24 +1,38 @@ -# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --build --exit-code-from dj +# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --build --exit-code-from app version: '2.2' x-net: &net networks: - main services: - dj: + db: + <<: *net + image: datajoint/mysql:${MYSQL_TAG} + environment: + - MYSQL_ROOT_PASSWORD=simple + fakeservices.datajoint.io: + <<: *net + image: raphaelguzman/nginx:v0.0.4 + environment: + - ADD_db_TYPE=DATABASE + - ADD_db_ENDPOINT=db:3306 + depends_on: + db: + condition: service_healthy + app: <<: *net environment: - DISPLAY - MATLAB_LICENSE - MATLAB_USER - - DJ_HOST=mysql + - DJ_HOST=fakeservices.datajoint.io - DJ_USER=root - DJ_PASS=simple - - DJ_TEST_HOST=mysql + - DJ_TEST_HOST=fakeservices.datajoint.io - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint image: raphaelguzman/matlab:${MATLAB_VERSION}-MIN depends_on: - mysql: + fakeservices.datajoint.io: condition: service_healthy user: ${MATLAB_UID}:${MATLAB_GID} working_dir: /src @@ -33,11 +47,5 @@ services: mac_address: $MATLAB_HOSTID volumes: - .:/src - - /tmp/.X11-unix:/tmp/.X11-unix:rw - mysql: - <<: *net - image: datajoint/mysql:${MYSQL_TAG} - environment: - - MYSQL_ROOT_PASSWORD=simple networks: main: \ No newline at end of file diff --git a/README.md b/README.md index 6fdd4c3c..b8da6eb4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,49 @@ DataJoint for MATLAB is a high-level programming interface for relational databases designed to support data processing chains in science labs. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, and querying data. DataJoint was initially developed in 2009 by Dimitri Yatsenko in [Andreas Tolias' Lab](http://toliaslab.org) for the distributed processing and management of large volumes of data streaming from regular experiments. Starting in 2011, DataJoint has been available as an open-source project adopted by other labs and improved through contributions from several developers. + + +Running Tests Locally +===================== + + +* Create an `.env` with desired development environment values e.g. +``` sh +MATLAB_USER=raphael +MATLAB_LICENSE="#\ BEGIN----...---------END" # For image usage instructions see https://github.com/guzman-raphael/matlab, https://hub.docker.com/r/raphaelguzman/matlab +MATLAB_VERSION=R2018b +MATLAB_HOSTID=XX:XX:XX:XX:XX:XX +MATLAB_UID=1000 +MATLAB_GID=1000 +MYSQL_TAG=5.7 +``` +* `cp local-docker-compose.yml docker-compose.yml` +* `docker-compose up` (Note configured `JUPYTER_PASSWORD`) +* Select a means of running MATLAB e.g. Jupyter Notebook, GUI, or Terminal (see bottom) +* Run desired tests. Some examples are as follows: + +| Use Case | MATLAB Code | +| ---------------------------- | ------------------------------------------------------------------------------ | +| Run all tests | `run(tests.Main)` | +| Run one class of tests | `run(tests.TestTls)` | +| Run one specific test | `runtests('tests.TestTls/testInsecureConn')` | +| Run tests based on test name | `import matlab.unittest.TestSuite;`
`import matlab.unittest.selectors.HasName;`
`import matlab.unittest.constraints.ContainsSubstring;`
`suite = TestSuite.fromClass(?tests.Main, ... `
    `HasName(ContainsSubstring('Conn')));`
`run(suite)`| + + +Launch Jupyter Notebook +----------------------- +* Navigate to `localhost:8888` +* Input Jupyter password +* Launch a notebook i.e. `New > MATLAB` + + +Launch MATLAB GUI (supports remote interactive debugger) +-------------------------------------------------------- +* Shell into `datajoint-matlab_app_1` i.e. `docker exec -it datajoint-matlab_app_1 bash` +* Launch Matlab by runnning command `matlab` + + +Launch MATLAB Terminal +---------------------- +* Shell into `datajoint-matlab_app_1` i.e. `docker exec -it datajoint-matlab_app_1 bash` +* Launch Matlab with no GUI by runnning command `matlab -nodisplay` \ No newline at end of file diff --git a/local-docker-compose.yml b/local-docker-compose.yml new file mode 100644 index 00000000..288d65c1 --- /dev/null +++ b/local-docker-compose.yml @@ -0,0 +1,70 @@ +# docker-compose -f local-docker-compose.yml --env-file LNX.env up --build +version: '2.4' +x-net: &net + networks: + - main +services: + db: + <<: *net + image: datajoint/mysql:${MYSQL_TAG} + environment: + - MYSQL_ROOT_PASSWORD=simple + # ports: + # - "3306:3306" + ## To persist MySQL data + # volumes: + # - ./mysql/data:/var/lib/mysql + fakeservices.datajoint.io: + <<: *net + image: raphaelguzman/nginx:v0.0.4 + environment: + - ADD_db_TYPE=DATABASE + - ADD_db_ENDPOINT=db:3306 + ports: + - "443:443" + - "3306:3306" + depends_on: + db: + condition: service_healthy + app: + <<: *net + environment: + - DJ_HOST=fakeservices.datajoint.io + - DJ_USER=root + - DJ_PASS=simple + - DJ_TEST_HOST=fakeservices.datajoint.io + - DJ_TEST_USER=datajoint + - DJ_TEST_PASSWORD=datajoint + - MATLAB_USER + - MATLAB_LICENSE + - JUPYTER_PASSWORD=datajoint + - DISPLAY + image: raphaelguzman/matlab:${MATLAB_VERSION}-GUI + depends_on: + fakeservices.datajoint.io: + condition: service_healthy + ports: + - "8888:8888" + user: ${MATLAB_UID}:${MATLAB_GID} + working_dir: /home/muser/notebooks + command: > + /bin/bash -c " + ## Remove mym + rm -R /src/mym; + ## Set dj path + matlab -nodisplay -r \"\ + addpath('/src');\ + savepath;\ + \"; + ## Interactive Jupyter Notebook environment + jupyter notebook; + " + mac_address: $MATLAB_HOSTID + volumes: + ## Dev mounts + - .:/src + - /tmp/.X11-unix:/tmp/.X11-unix:rw + ## Additional mounts may go here + # - ./notebook:/home/muser/notebooks +networks: + main: \ No newline at end of file From 4354bc6c852fd3493f593ce627ff312fc31e9737 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 15 Apr 2020 09:20:28 -0500 Subject: [PATCH 4/6] Fix issue in reading schema MySQL8. --- +dj/createSchema.m | 159 +++++++++++++++++++++++---------------------- 1 file changed, 83 insertions(+), 76 deletions(-) diff --git a/+dj/createSchema.m b/+dj/createSchema.m index 972cbf98..1523886d 100644 --- a/+dj/createSchema.m +++ b/+dj/createSchema.m @@ -1,89 +1,96 @@ function createSchema(package,parentdir,db) -% DJ.CREATESCHEMA - interactively create a new DataJoint schema -% -% INPUT: -% (optional) package - name of the package to be associated with the schema -% (optional) parentdir - name of the dirctory where to create new package -% (optional) db - database name to associate with the schema - -if nargin < 3 - dbname = input('Enter database name >> ','s'); -else - dbname = db; -end - -if ~dbname - disp 'No database name entered. Quitting.' -elseif isempty(regexp(dbname,'^[a-z][a-z0-9_]*$','once')) - error 'Invalid database name. Begin with a letter, only lowercase alphanumerical and underscores.' -else - % create database - s = query(dj.conn, ... - sprintf('SELECT schema_name FROM information_schema.schemata WHERE schema_name = "%s"', dbname)); - - if ~isempty(s.schema_name) - disp 'database already exists' + % DJ.CREATESCHEMA - interactively create a new DataJoint schema + % + % INPUT: + % (optional) package - name of the package to be associated with the schema + % (optional) parentdir - name of the dirctory where to create new package + % (optional) db - database name to associate with the schema + + if nargin < 3 + dbname = input('Enter database name >> ','s'); else - query(dj.conn, sprintf('create schema %s',dbname)) - disp 'database created' + dbname = db; end - - if nargin < 1 - if usejava('desktop') - disp 'Please select a package folder. Opening UI...' - folder = uigetdir('./','Select a package folder'); + + if ~dbname + disp 'No database name entered. Quitting.' + elseif isempty(regexp(dbname,'^[a-z][a-z0-9_]*$','once')) + error(['Invalid database name. Begin with a letter, only lowercase alphanumerical and ' ... + 'underscores.']) + else + % create database + s = query(dj.conn, ... + sprintf(['SELECT schema_name as `schema_name` ' ... + 'FROM information_schema.schemata WHERE schema_name = "%s"'], dbname)); + + if ~isempty(s.schema_name) + disp 'database already exists' else - folder = input('Enter package folder path >> ','s'); + query(dj.conn, sprintf('create schema %s',dbname)) + disp 'database created' end - else - if nargin < 3 + + if nargin < 1 if usejava('desktop') - fprintf('Please select folder to create package %s in. Opening UI...\n', ['+', package]) - folder = uigetdir('./', sprintf('Select folder to create package %s in', ['+', package])); + disp 'Please select a package folder. Opening UI...' + folder = uigetdir('./','Select a package folder'); else - folder = input('Enter parent folder path >> ','s'); + folder = input('Enter package folder path >> ','s'); end else - folder = parentdir; - end - - if folder - folder = fullfile(folder, ['+', package]); - mkdir(folder) - end - end - - if ~folder - disp 'No package selected. Cancelled.' - else - [filepath,package] = fileparts(folder); - if package(1)~='+' - error 'Package folders must start with a +' + if nargin < 3 + if usejava('desktop') + fprintf('Please select folder to create package %s in. Opening UI...\n', ... + ['+', package]) + folder = uigetdir('./', sprintf('Select folder to create package %s in', ... + ['+', package])); + else + folder = input('Enter parent folder path >> ','s'); + end + else + folder = parentdir; + end + + if folder + folder = fullfile(folder, ['+', package]); + mkdir(folder) + end end - package = package(2:end); % discard + - - % create the getSchema function - schemaFile = fullfile(folder,'getSchema.m'); - if exist(schemaFile,'file') - fprintf('%s.getSchema.m already exists\n', package) + + if ~folder + disp 'No package selected. Cancelled.' else - f = fopen(schemaFile,'wt'); - assert(-1 ~= f, 'Could not open %s', f) - - fprintf(f,'function obj = getSchema\n'); - fprintf(f,'persistent schemaObject\n'); - fprintf(f,'if isempty(schemaObject)\n'); - fprintf(f,' schemaObject = dj.Schema(dj.conn, ''%s'', ''%s'');\n', package, dbname); - fprintf(f,'end\n'); - fprintf(f,'obj = schemaObject;\n'); - fprintf(f,'end\n'); - fclose(f); - end - - % test that getSchema is on the path - whichpath = which(sprintf('%s.getSchema',package)); - if isempty(whichpath) - warning('Could not open %s.getSchema. Ensure that %s is on the path', package, filepath) + [filepath,package] = fileparts(folder); + if package(1)~='+' + error 'Package folders must start with a +' + end + package = package(2:end); % discard + + + % create the getSchema function + schemaFile = fullfile(folder,'getSchema.m'); + if exist(schemaFile,'file') + fprintf('%s.getSchema.m already exists\n', package) + else + f = fopen(schemaFile,'wt'); + assert(-1 ~= f, 'Could not open %s', f) + + fprintf(f,'function obj = getSchema\n'); + fprintf(f,'persistent schemaObject\n'); + fprintf(f,'if isempty(schemaObject)\n'); + fprintf(f,' schemaObject = dj.Schema(dj.conn, ''%s'', ''%s'');\n', ... + package, dbname); + fprintf(f,'end\n'); + fprintf(f,'obj = schemaObject;\n'); + fprintf(f,'end\n'); + fclose(f); + end + + % test that getSchema is on the path + whichpath = which(sprintf('%s.getSchema',package)); + if isempty(whichpath) + warning('Could not open %s.getSchema. Ensure that %s is on the path', ... + package, filepath) + end end end -end +end \ No newline at end of file From c1e3af30bbab92e92437a7a079ca2ff44fe944b1 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 22 Apr 2020 15:43:15 -0500 Subject: [PATCH 5/6] Fix linting. --- +dj/Relvar.m | 39 ++++++--------------- +tests/TestRelationalOperand.m | 63 +++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/+dj/Relvar.m b/+dj/Relvar.m index 44ecda84..f56f55eb 100755 --- a/+dj/Relvar.m +++ b/+dj/Relvar.m @@ -343,12 +343,14 @@ function update(self, attrname, value) % update(v2p.Mice & key, 'mouse_dob', '2011-01-01') % update(v2p.Scan & key, 'lens') % set the value to NULL - assert(count(self)==1, 'Update is only allowed on one tuple at a time') - isNull = nargin<3; + assert(count(self)==1, 'Update is only allowed on one tuple at a time'); header = self.header; ix = find(strcmp(attrname,header.names)); - assert(numel(ix)==1, 'invalid attribute name') - assert(~header.attributes(ix).iskey, 'cannot update a key value. Use insert(..,''REPLACE'') instead') + assert(numel(ix)==1, 'invalid attribute name'); + assert(~header.attributes(ix).iskey, ... + 'cannot update a key value. Use insert(..,''REPLACE'') instead'); + isNull = nargin<3 || (header.attributes(ix).isNumeric && isnan(value)) || ... + (~header.attributes(ix).isNumeric && ~ischar(value) && isempty(value)); switch true case isNull @@ -358,36 +360,15 @@ function update(self, attrname, value) value = {}; case header.attributes(ix).isString assert(dj.lib.isString(value), 'Value must be a string') - if isempty(value) - assert(header.attributes(ix).isnullable, ... - 'attribute `%s` is not nullable.', attrname) - valueStr = 'NULL'; - value = {}; - else valueStr = '"{S}"'; value = {char(value)}; - end case header.attributes(ix).isBlob - if isempty(value) && header.attributes(ix).isnullable - assert(header.attributes(ix).isnullable, ... - 'attribute `%s` is not nullable.', attrname) - valueStr = 'NULL'; - value = {}; - else - valueStr = '"{M}"'; - value = {value}; - end + valueStr = '"{M}"'; + value = {value}; case header.attributes(ix).isNumeric assert(isscalar(value) && isnumeric(value), 'Numeric value must be scalar') - if isnan(value) - assert(header.attributes(ix).isnullable, ... - 'attribute `%s` is not nullable. NaNs not allowed', attrname) - valueStr = 'NULL'; - value = {}; - else - valueStr = sprintf('%1.16g',value); - value = {}; - end + valueStr = sprintf('%1.16g',value); + value = {}; otherwise error 'invalid update command' end diff --git a/+tests/TestRelationalOperand.m b/+tests/TestRelationalOperand.m index caf31207..b88147f0 100644 --- a/+tests/TestRelationalOperand.m +++ b/+tests/TestRelationalOperand.m @@ -21,16 +21,71 @@ function TestRelationalOperand_testUpdateDate(testCase) )); q = University.All & 'id=2'; - new_value = ''; + new_value = []; q.update('date', new_value); - testCase.verifyEqual(q.fetch1('date'), new_value); + res = mym(['select date from `' testCase.PREFIX ... + '_university`.`all` where id=2 and date is null;']); + testCase.verifyEqual(length(res.date), 1); new_value = '2020-04-14'; q.update('date', new_value); - testCase.verifyEqual(q.fetch1('date'), new_value); + res = mym(['select date from `' testCase.PREFIX ... + '_university`.`all` where id=2 and date like ''' new_value ''';']); + testCase.verifyEqual(length(res.date), 1); q.update('date'); - testCase.verifyEqual(q.fetch1('date'), ''); + res = mym(['select date from `' testCase.PREFIX ... + '_university`.`all` where id=2 and date is null;']); + testCase.verifyEqual(length(res.date), 1); + end + function TestRelationalOperand_testUpdateString(testCase) + st = dbstack; + disp(['---------------' st(1).name '---------------']); + % related https://github.com/datajoint/datajoint-matlab/issues/211 + package = 'University'; + + c1 = dj.conn(... + testCase.CONN_INFO.host,... + testCase.CONN_INFO.user,... + testCase.CONN_INFO.password,'',true); + + dj.createSchema(package,[testCase.test_root '/test_schemas'], ... + [testCase.PREFIX '_university']); + + insert(University.All, struct( ... + 'id', 3, ... + 'string', 'normal' ... + )); + q = University.All & 'id=3'; + + new_value = ''; + q.update('string', new_value); + res = mym(['select string from `' testCase.PREFIX ... + '_university`.`all` where id=3 and string like ''' new_value ''';']); + testCase.verifyEqual(length(res.string), 1); + + new_value = ' '; + q.update('string', new_value); + res = mym(['select string from `' testCase.PREFIX ... + '_university`.`all` where id=3 and string like ''' new_value ''';']); + testCase.verifyEqual(length(res.string), 1); + + new_value = []; + q.update('string', new_value); + res = mym(['select string from `' testCase.PREFIX ... + '_university`.`all` where id=3 and string is null;']); + testCase.verifyEqual(length(res.string), 1); + + new_value = 'diff'; + q.update('string', new_value); + res = mym(['select string from `' testCase.PREFIX ... + '_university`.`all` where id=3 and string like ''' new_value ''';']); + testCase.verifyEqual(length(res.string), 1); + + q.update('string'); + res = mym(['select string from `' testCase.PREFIX ... + '_university`.`all` where id=3 and string is null;']); + testCase.verifyEqual(length(res.string), 1); end end end \ No newline at end of file From 1916bf096224aa0b31e71d1ec70ee67fa2bc0f6e Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 22 Apr 2020 15:50:24 -0500 Subject: [PATCH 6/6] Fix linting2. --- +dj/Relvar.m | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/+dj/Relvar.m b/+dj/Relvar.m index f56f55eb..1ca2eceb 100755 --- a/+dj/Relvar.m +++ b/+dj/Relvar.m @@ -355,13 +355,13 @@ function update(self, attrname, value) switch true case isNull assert(header.attributes(ix).isnullable, ... - 'attribute `%s` is not nullable.', attrname) + 'attribute `%s` is not nullable.', attrname); valueStr = 'NULL'; value = {}; case header.attributes(ix).isString - assert(dj.lib.isString(value), 'Value must be a string') - valueStr = '"{S}"'; - value = {char(value)}; + assert(dj.lib.isString(value), 'Value must be a string'); + valueStr = '"{S}"'; + value = {char(value)}; case header.attributes(ix).isBlob valueStr = '"{M}"'; value = {value};