Merge pull request #235 from datajoint/stage-external-storage

Add R2016b tests
datajoint · May 19, 2020 · 6c8094b · 6c8094b
2 parents 2559c56 + 43c1f1f
commit 6c8094b
Show file tree

Hide file tree

Showing 30 changed files with 144 additions and 98 deletions.
diff --git a/+dj/+store_plugins/S3.m b/+dj/+store_plugins/S3.m
@@ -193,8 +193,8 @@ function remove_object(self, external_filepath)
                 credential_scope ', '  'SignedHeaders=' signed_headers ', ' ...
                 'Signature=' signature];
             url = [protocol host canonical_uri '?' canonical_querystring];
-            
-            if ~strcmpi(method, 'head')
+
+            try
                 headers = {...
                     'content_type', content_type; ...
                     'host', host; ...
@@ -207,6 +207,7 @@ function remove_object(self, external_filepath)
                     'HeaderFields', headers, ...
                     'RequestMethod', lower(method), ...
                     'Timeout', 60, ...
+                    'CertificateFilename', 'default', ...
                     'ContentType', 'binary', ...
                     'MediaType', content_type, ...
                     'CharacterEncoding', 'ISO-8859-1'...
@@ -217,30 +218,34 @@ function remove_object(self, external_filepath)
                 else
                     data = webwrite(url, char(payload_bin'), options);
                 end
-            else
-                headers = [...
-                    matlab.net.http.HeaderField('content_type', content_type), ...
-                    matlab.net.http.HeaderField('host', host), ...
-                    matlab.net.http.HeaderField('X-Amz-Date', amzdate), ...
-                    matlab.net.http.HeaderField('X-Amz-Content-Sha256', payload_hash), ...
-                    matlab.net.http.HeaderField('Authorization', authorization_header), ...
-                    matlab.net.http.field.AcceptField(...
-                        [matlab.net.http.MediaType(content_type)]) ...
-                ];
-
-                request = matlab.net.http.RequestMessage(lower(method), headers, ...
-                    matlab.net.http.MessageBody(payload_bin));
-                response = request.send(url, matlab.net.http.HTTPOptions(...
-                    'ConnectTimeout', 60));
-                if response.StatusCode ~= matlab.net.http.StatusCode.OK && ...
-                        response.StatusCode ~= matlab.net.http.StatusCode.NoContent
-                    error(['DataJoint:RequestError:' char(response.StatusCode)], ...
-                        'Request failed on route `%s` with status `%s`.', canonical_uri, ...
-                        response.StatusLine);
+            catch ME
+                if strcmp(ME.identifier,'MATLAB:weboptions:unrecognizedStringChoice')
+                    headers = [...
+                        matlab.net.http.HeaderField('content_type', content_type), ...
+                        matlab.net.http.HeaderField('host', host), ...
+                        matlab.net.http.HeaderField('X-Amz-Date', amzdate), ...
+                        matlab.net.http.HeaderField('X-Amz-Content-Sha256', payload_hash), ...
+                        matlab.net.http.HeaderField('Authorization', authorization_header), ...
+                        matlab.net.http.field.AcceptField(...
+                            [matlab.net.http.MediaType(content_type)]) ...
+                    ];
+
+                    request = matlab.net.http.RequestMessage(lower(method), headers, ...
+                        matlab.net.http.MessageBody(payload_bin));
+                    response = request.send(url, matlab.net.http.HTTPOptions(...
+                        'ConnectTimeout', 60, 'CertificateFilename', 'default'));
+                    if response.StatusCode ~= matlab.net.http.StatusCode.OK && ...
+                            response.StatusCode ~= matlab.net.http.StatusCode.NoContent
+                        error(['DataJoint:RequestError:' char(response.StatusCode)], ...
+                            'Request failed on route `%s` with status `%s`.', canonical_uri,...
+                            response.StatusLine);
+                    else
+                        data = response.Body.Data;
+                    end
                 else
-                    data = response.Body.Data;
+                    rethrow(ME);
                 end
-            end   
+            end
         end
     end
 end

diff --git a/+tests/Main.m b/+tests/Main.m
diff --git a/.travis.yml b/.travis.yml
@@ -23,25 +23,21 @@ jobs:
   include:
     - <<: *slim
       env:
-      - MATLAB_VERSION: R2018b
+      - MATLAB_VERSION: R2019a
       - MYSQL_TAG: 8.0
     - <<: *slim
       env:
-      - MATLAB_VERSION: R2018b
+      - MATLAB_VERSION: R2019a
       - MYSQL_TAG: 5.7
-    - <<: *slim
-      env:
-      - MATLAB_VERSION: R2018b
-      - MYSQL_TAG: 5.6
     - <<: *slim
       env:
       - MATLAB_VERSION: R2019a
-      - MYSQL_TAG: 8.0
+      - MYSQL_TAG: 5.6
     - <<: *slim
       env:
-      - MATLAB_VERSION: R2019a
+      - MATLAB_VERSION: R2018b
       - MYSQL_TAG: 5.7
     - <<: *slim
       env:
-      - MATLAB_VERSION: R2019a
-      - MYSQL_TAG: 5.6
+      - MATLAB_VERSION: R2016b
+      - MYSQL_TAG: 5.7
diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml
@@ -66,7 +66,8 @@ services:
     command: > 
       /bin/bash -c "
         matlab -nodisplay -r \"\
-            res=run(tests.Main);\
+            addpath('/src/tests');\
+            res=run(Main);\
             disp(res);\
             if all([res.Passed])  exit, else exit(1), end;\
           \";

diff --git a/README.md b/README.md
@@ -20,14 +20,15 @@ MYSQL_TAG=5.7
 * `cp local-docker-compose.yml docker-compose.yml`
 * `docker-compose up` (Note configured `JUPYTER_PASSWORD`)
 * Select a means of running MATLAB e.g. Jupyter Notebook, GUI, or Terminal (see bottom)
+* Add `tests` directory to path e.g. in MATLAB, `addpath('tests')`
 * Run desired tests. Some examples are as follows:
 
 | Use Case                     | MATLAB Code                                                                    |
 | ---------------------------- | ------------------------------------------------------------------------------ |
-| Run all tests                | `run(tests.Main)`                                                              |
-| Run one class of tests       | `run(tests.TestTls)`                                                           |
-| Run one specific test        | `runtests('tests.TestTls/testInsecureConn')`                                   |
-| Run tests based on test name | `import matlab.unittest.TestSuite;`<br>`import matlab.unittest.selectors.HasName;`<br>`import matlab.unittest.constraints.ContainsSubstring;`<br>`suite = TestSuite.fromClass(?tests.Main, ... `<br><code>&nbsp;&nbsp;&nbsp;&nbsp;</code>`HasName(ContainsSubstring('Conn')));`<br>`run(suite)`|
+| Run all tests                | `run(Main)`                                                              |
+| Run one class of tests       | `run(TestTls)`                                                           |
+| Run one specific test        | `runtests('TestTls/testInsecureConn')`                                   |
+| Run tests based on test name | `import matlab.unittest.TestSuite;`<br>`import matlab.unittest.selectors.HasName;`<br>`import matlab.unittest.constraints.ContainsSubstring;`<br>`suite = TestSuite.fromClass(?Main, ... `<br><code>&nbsp;&nbsp;&nbsp;&nbsp;</code>`HasName(ContainsSubstring('Conn')));`<br>`run(suite)`|
 
 
 Launch Jupyter Notebook

diff --git a/local-docker-compose.yml b/local-docker-compose.yml
@@ -82,6 +82,7 @@ services:
         ## Set dj path
         matlab -nodisplay -r \"\
             addpath('/src');\
+            addpath('/src/tests');\
             savepath;\
           \";
         ## Interactive Jupyter Notebook environment

diff --git a/+tests/+lib/compareVersions.m → tests/+lib/compareVersions.m b/+tests/+lib/compareVersions.m → tests/+lib/compareVersions.m
diff --git a/tests/Main.m b/tests/Main.m
@@ -0,0 +1,11 @@
+classdef Main < ...
+    TestConfig & ...
+    TestConnection & ...
+    TestERD & ...
+    TestExternalFile & ...
+    TestExternalS3 & ...
+    TestFetch & ...
+    TestProjection & ...
+    TestTls & ...
+    TestUuid
+end
diff --git a/+tests/Prep.m → tests/Prep.m b/+tests/Prep.m → tests/Prep.m
@@ -23,9 +23,9 @@
     methods
         function obj = Prep()
             % Initialize test_root
-            test_pkg_details = what('tests');
+            test_pkg_details = what('dj');
             [test_root, ~, ~] = fileparts(test_pkg_details.path);
-            obj.test_root = [test_root '/+tests'];
+            obj.test_root = [test_root '/tests'];
             if ispc
                 obj.external_file_store_root = [getenv('TEMP') '\root'];
             else
@@ -37,13 +37,14 @@
         function init(testCase)
             disp('---------------INIT---------------');
             clear functions;
+            addpath(testCase.test_root);
             addpath([testCase.test_root '/test_schemas']);
 
             curr_conn = dj.conn(testCase.CONN_INFO_ROOT.host, ...
                 testCase.CONN_INFO_ROOT.user, testCase.CONN_INFO_ROOT.password,'',true);
             % create test users
             ver = curr_conn.query('select @@version as version').version;
-            if tests.lib.compareVersions(ver,'5.8')
+            if lib.compareVersions(ver,'5.8')
                 cmd = {...
                 'CREATE USER IF NOT EXISTS ''datajoint''@''%%'' '
                 'IDENTIFIED BY ''datajoint'';'
@@ -154,6 +155,7 @@ function dispose(testCase)
                 % delete(['test_schemas/+University/getSchema.m'])
             end
             rmpath([testCase.test_root '/test_schemas']);
+            % rmpath(testCase.test_root);
             warning('on','MATLAB:RMDIR:RemovedFromPath');
         end
     end

diff --git a/+tests/TestConfig.m → tests/TestConfig.m b/+tests/TestConfig.m → tests/TestConfig.m
@@ -1,4 +1,4 @@
-classdef TestConfig < tests.Prep
+classdef TestConfig < Prep
     % TestConfig tests scenarios related to initializing DJ config.
     methods (Static)
         function obj = TestConfig_configRemoveEnvVars(obj, type)
@@ -49,7 +49,7 @@ function TestConfig_configSingleFileTest(test_instance, type, fname, base)
             end
             % load raw
             read_data = fileread(fname);           
-            obj1 = tests.TestConfig.TestConfig_configRemoveEnvVars(jsondecode(read_data), 'file');
+            obj1 = TestConfig.TestConfig_configRemoveEnvVars(jsondecode(read_data), 'file');
             % optional merge from base
             if strcmpi(type, 'load-custom')
                 tmp = rmfield(base, intersect(fieldnames(base), fieldnames(obj1)));
@@ -60,7 +60,7 @@ function TestConfig_configSingleFileTest(test_instance, type, fname, base)
             % stringify
             file = jsonencode(obj1);
             % load config
-            obj2 = tests.TestConfig.TestConfig_configRemoveEnvVars(dj.config(), 'config');
+            obj2 = TestConfig.TestConfig_configRemoveEnvVars(dj.config(), 'config');
             curr = jsonencode(obj2);
             curr = regexprep(curr,'[a-z0-9][A-Z]','${$0(1)}_${lower($0(2))}');
             % checks
@@ -142,15 +142,20 @@ function TestConfig_testConfigChecks(testCase)
             disp(['---------------' st(1).name '---------------']);
             testCase.verifyError(@() dj.config(9), ...
                 'DataJoint:Config:InvalidType');
-            d = testCase.verifyError(@() dj.config('none'), ...
-                'DataJoint:Config:InvalidKey');
+            try
+                d = dj.config('none');
+            catch ME
+                if ~strcmp(ME.identifier,'DataJoint:Config:InvalidKey')
+                    rethrow(ME);
+                end
+            end
         end
         function TestConfig_testRestore(testCase)
             st = dbstack;
             disp(['---------------' st(1).name '---------------']);
             dj.config.restore;
-            obj1 = tests.TestConfig.TestConfig_configRemoveEnvVars(dj.config(), 'config');
-            obj2 = tests.TestConfig.TestConfig_configRemoveEnvVars( ...
+            obj1 = TestConfig.TestConfig_configRemoveEnvVars(dj.config(), 'config');
+            obj2 = TestConfig.TestConfig_configRemoveEnvVars( ...
                 orderfields(dj.internal.Settings.DEFAULTS), 'config');
             testCase.verifyEqual(jsonencode(obj1), jsonencode(obj2));
         end
@@ -161,36 +166,37 @@ function TestConfig_testSave(testCase)
 
             % local
             dj.config('font', 10);
-            tests.TestConfig.TestConfig_configSingleFileTest(testCase, 'save-local');
+            TestConfig.TestConfig_configSingleFileTest(testCase, 'save-local');
             % global
             dj.config('font', 12);
-            tests.TestConfig.TestConfig_configSingleFileTest(testCase, 'save-global');
+            TestConfig.TestConfig_configSingleFileTest(testCase, 'save-global');
             % custom
             dj.config('font', 16);
-            tests.TestConfig.TestConfig_configSingleFileTest(testCase, 'save-custom', './config.json');
+            TestConfig.TestConfig_configSingleFileTest(...
+                testCase, 'save-custom', './config.json');
 
             dj.config.restore;
         end
         function TestConfig_testLoad(testCase)
             st = dbstack;
             disp(['---------------' st(1).name '---------------']);
-            pkg = what('tests');
+            pkg_path = testCase.test_root;
             % generate default base
-            default_file = [pkg.path '/test_schemas/default.json'];
+            default_file = [pkg_path '/test_schemas/default.json'];
             dj.config.restore;
             dj.config.save(default_file);
-            defaults = tests.TestConfig.TestConfig_configRemoveEnvVars( ...
+            defaults = TestConfig.TestConfig_configRemoveEnvVars( ...
                 jsondecode(fileread(default_file)), 'file');
             delete(default_file);
             % load test config
-            tests.TestConfig.TestConfig_configSingleFileTest(testCase, 'load-custom', ...
-                [pkg.path '/test_schemas/config.json'], defaults);
+            TestConfig.TestConfig_configSingleFileTest(testCase, 'load-custom', ...
+                [pkg_path '/test_schemas/config.json'], defaults);
             % load new config on top of existing
-            base = tests.TestConfig.TestConfig_configRemoveEnvVars(dj.config, 'config');
+            base = TestConfig.TestConfig_configRemoveEnvVars(dj.config, 'config');
             base = jsonencode(base);
             base = regexprep(base,'[a-z0-9][A-Z]','${$0(1)}_${lower($0(2))}');
-            tests.TestConfig.TestConfig_configSingleFileTest(testCase, 'load-custom', ...
-                [pkg.path '/test_schemas/config_lite.json'], jsondecode(base));
+            TestConfig.TestConfig_configSingleFileTest(testCase, 'load-custom', ...
+                [pkg_path '/test_schemas/config_lite.json'], jsondecode(base));
             % cleanup
             dj.config.restore;
         end
@@ -210,14 +216,14 @@ function validateEnvVarConfig(type, values)
                 testCase.verifyEqual(dj.config('databasePassword'), values{3});
                 testCase.verifyEqual(dj.config('connectionInit_function'), values{4});
             end
-            pkg = what('tests');
+            pkg_path = testCase.test_root;
             setenv('DJ_INIT', 'select @@version;');
             dj.config.restore;
             % check pulling from env vars
             env = {getenv('DJ_HOST'), getenv('DJ_USER'), getenv('DJ_PASS'), getenv('DJ_INIT')};
             validateEnvVarConfig('env', env);
             % check after load if env vars take precedence
-            dj.config.load([pkg.path '/test_schemas/config.json']);
+            dj.config.load([pkg_path '/test_schemas/config.json']);
             validateEnvVarConfig('env', env);
             % check if overriding env vars is persisted
             validateEnvVarConfig('set', ...

diff --git a/+tests/TestConnection.m → tests/TestConnection.m b/+tests/TestConnection.m → tests/TestConnection.m
@@ -1,4 +1,4 @@
-classdef TestConnection < tests.Prep
+classdef TestConnection < Prep
     % TestConnection tests typical connection scenarios.
     methods (Test)
         function TestConnection_testConnection(testCase)
@@ -39,5 +39,39 @@ function TestConnection_testPort(testCase)
                 testCase.CONN_INFO.password,'',true), ...
                 'MySQL:Error');
         end
+        function TestConnection_testTransactionRollback(testCase)
+            st = dbstack;
+            disp(['---------------' st(1).name '---------------']);
+            package = 'University';
+
+            c1 = dj.conn(...
+                testCase.CONN_INFO.host,... 
+                testCase.CONN_INFO.user,...
+                testCase.CONN_INFO.password,'',true);
+            dj.createSchema(package,[testCase.test_root '/test_schemas'], ...
+                [testCase.PREFIX '_university']);
+            schema = University.getSchema;
+            tmp = {
+                20   'Henry'   'Jupyter' '2020-11-25 12:34:56'
+                21   'Lacy'   'Mars' '2017-11-25 12:34:56'
+            };
+
+            insert(University.Student, tmp(1, :));
+
+            schema.conn.startTransaction
+            try
+                insert(University.Student, tmp(2, :));
+                assert(false, 'Customer:Error', 'Message')
+            catch ME
+                schema.conn.cancelTransaction
+                if ~strcmp(ME.identifier,'Customer:Error')
+                    rethrow(ME);
+                end
+            end
+
+            q = University.Student & 'student_id in (20,21)';
+            testCase.verifyEqual(q.count, 1);
+            testCase.verifyEqual(q.fetch1('student_id'), 20);
+        end
     end
 end
diff --git a/+tests/TestERD.m → tests/TestERD.m b/+tests/TestERD.m → tests/TestERD.m
@@ -1,4 +1,4 @@
-classdef TestERD < tests.Prep
+classdef TestERD < Prep
     % TestERD tests unusual ERD scenarios.
     methods (Test)
         function TestERD_testDraw(testCase)