From 6eb605c31d7a2dc08e2dc95298352cac3428d366 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 17 Apr 2014 12:16:53 +0200 Subject: [PATCH 001/113] Improved self-test for MarkLogic 7 --- deploy/test/data/ml7-config.xml | 3 +++ deploy/test/test_server_config.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index bcbeb1f0..90b9b0c4 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -299,6 +299,7 @@ + test @@ -341,6 +342,7 @@ + test2 @@ -380,6 +382,7 @@ + diff --git a/deploy/test/test_server_config.rb b/deploy/test/test_server_config.rb index 35bc1e2c..f06d624e 100644 --- a/deploy/test/test_server_config.rb +++ b/deploy/test/test_server_config.rb @@ -62,7 +62,7 @@ def bootstrap_version(version) end def test_bootstrap - version = ENV['ROXY_TEST_SERVER_VERSION'] || 4 + version = ENV['ROXY_TEST_SERVER_VERSION'] || 7 bootstrap_version version end end \ No newline at end of file From bc550aa0726bbcc16d633b5d8e63cc90de812b8a Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 22 Apr 2014 11:54:11 +0200 Subject: [PATCH 002/113] Fix for #210: new usage when running ml from roxy root --- ml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ml b/ml index 56062795..41c17e3a 100755 --- a/ml +++ b/ml @@ -125,6 +125,9 @@ then else printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" fi +elif [ "${PWD##*/}" == 'roxy' ] +then + usage else if [ -e deploy/lib/ml.rb ] then From 90046d98039c679cb7c5386f93361732905c169b Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 22 Apr 2014 15:47:43 +0200 Subject: [PATCH 003/113] Fix for #176: improved Roxy documentation --- deploy/lib/Help.rb | 69 +++++++++++++++++++++++++--------------------- deploy/lib/ml.rb | 26 ++++++++++++----- ml | 13 +++++++-- 3 files changed, 67 insertions(+), 41 deletions(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index 63c5c656..d0019e5f 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -2,37 +2,41 @@ class Help def self.usage <<-DOC.strip_heredoc - Usage: ml ENVIRONMENT COMMAND [ARGS] - - Deployment Commands: - init Creates configuration files for you to customize - initcpf Creates cpf configuration files for you to customize - info Return settings for a given environment - credentials Configures user and password for a given environment - bootstrap Configures your application on the MarkLogic server - wipe Remove your configuration from the MarkLogic server - restart Restart your MarkLogic server - deploy Loads modules, data, cpf configuration into the server - load Loads a file or folder into the server - clean Removes all files from the cpf, modules, or content databases - info Prints the environment-specific configuration information - test Runs xquery unit tests - recordloader Runs RecordLoader - xqsync Runs XQSync - corb Runs Corb - - Roxy Scaffolding commands: - create Creates a controller or view or model - index Adds an index to the configuration - extend Create a REST API service extension - transform Create a REST API transformation - - Other commands: - upgrade Upgrades the Roxy files - capture Capture the source code of an existing App Builder application - - All commands can be run with -h for more information. - + Usage: + ml [ENVIRONMENT] COMMAND [ARGS] + + Generic commands (no environment): + init Creates configuration files for you to customize + initcpf Creates cpf configuration files for you to customize + jar Creates a Roxy jar + new Creates a new project directory structure + upgrade Upgrades the Roxy files + usage Shows this help + + Scaffolding commands (no environment): + create Creates a controller, model, test or layout + index Adds an index to the configuration + extend Creates a REST API service extension + transform Creates a REST API transformation + + Bootstrapping commands (with environment): + bootstrap Configures your application on the given environment + capture Captures the source code of an existing App Builder application + clean Removes all files from the cpf, modules, or content databases on the given environment + credentials Configures user and password for the given environment + info Returns settings for the given environment + restart Restarts the given environment + wipe Removes your application from the given environment + + Deployment commands (with environment): + corb Runs Corb against the given environment + deploy Loads modules, data, cpf configuration into the given environment + load Loads a file or folder into the given environment + recordloader Runs RecordLoader against the given environment + test Runs xquery unit tests against the given environment + xqsync Runs XQSync against the given environment + + All commands can be run with -h or --help for more information. Leave out environment.. DOC end @@ -453,7 +457,8 @@ def self.jar end def self.doHelp(logger, command, error_message = nil) - logger.error "#{error_message}\n" if error_message + logger.info "" if error_message + logger.error "#{error_message}" if error_message logger.info Help.send(command) end end diff --git a/deploy/lib/ml.rb b/deploy/lib/ml.rb index 2f675d2b..f2238837 100755 --- a/deploy/lib/ml.rb +++ b/deploy/lib/ml.rb @@ -62,7 +62,9 @@ def need_help? # # Roxy framework is a convenience utility for create MVC code # - if command == "create" + if command == "usage" + Help.doHelp(@logger, :usage) + elsif command == "create" if need_help? Help.doHelp(@logger, command) else @@ -97,7 +99,7 @@ def need_help? # # put things in ServerConfig class methods that don't depend on environment or server info # - elsif ServerConfig.respond_to?(command.to_sym) || ServerConfig.respond_to?(command) + elsif ServerConfig.respond_to?(command.to_sym) || ServerConfig.respond_to?(command) || ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) if need_help? Help.doHelp(@logger, command) else @@ -105,19 +107,28 @@ def need_help? ServerConfig.send command end break + elsif ARGV.length == 0 + Help.doHelp(@logger, :usage, "Unknown generic command #{command}!") # # ServerConfig methods require environment to be set in order to talk to a ML server # else + # [GJo] check second arg before checking properties, makes help available within Roxy folder too.. + command2 = ARGV[0] + if need_help? && (ServerConfig.instance_methods.include?(command2.to_sym) || ServerConfig.instance_methods.include?(command2)) + Help.doHelp(@logger, command2) + break + elsif ! (ServerConfig.instance_methods.include?(command2.to_sym) || ServerConfig.instance_methods.include?(command2)) + Help.doHelp(@logger, :usage, "Unknown environment command #{command2}!") + break + end + # unshift to get the environment in ServerConfig.properties ARGV.unshift command @properties = ServerConfig.properties command = ARGV.shift - if need_help? && Help.respond_to?(command) - Help.doHelp(@logger, command) - break - elsif ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) + if ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) raise HelpException.new(command, "Missing environment for #{command}") if @properties["environment"].nil? raise ExitException.new("Missing ml-config.xml file. Check config.file property") if @properties["ml.config.file"].nil? @@ -127,7 +138,8 @@ def need_help? :logger => @logger ).send(command) else - Help.doHelp(@logger, :usage) + # [GJo] no longer reached.. + Help.doHelp(@logger, :usage, :error_message => "Unknown command #{command}!") break end end diff --git a/ml b/ml index 41c17e3a..1e73e5a5 100755 --- a/ml +++ b/ml @@ -35,7 +35,10 @@ PARAMS=("${@}") if [ "$1" == 'new' ] then shift - if [[ "$1" == '-h' ]] + if [ "$1" == '-h' ] + then + usage + elif [ "$1" == '--help' ] then usage elif [ $1 ] @@ -125,7 +128,13 @@ then else printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" fi -elif [ "${PWD##*/}" == 'roxy' ] +elif [ "${PWD##*/}" == 'roxy' ] && [ "$#" -eq 0 ] +then + usage +elif [ "${PWD##*/}" == 'roxy' ] && [ "$1" == '-h' ] +then + usage +elif [ "${PWD##*/}" == 'roxy' ] && [ "$1" == '--help' ] then usage else From 4d645b740ac9e271a6c35f5d350d66113931deb2 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 22 Apr 2014 16:56:19 +0200 Subject: [PATCH 004/113] Removed usage as explicit command --- deploy/lib/Help.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index d0019e5f..e2776894 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -11,7 +11,6 @@ def self.usage jar Creates a Roxy jar new Creates a new project directory structure upgrade Upgrades the Roxy files - usage Shows this help Scaffolding commands (no environment): create Creates a controller, model, test or layout From 4c3abb2e6c07d6f2d02e0ad9c6920f72464c2a1f Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 22 Apr 2014 16:56:26 +0200 Subject: [PATCH 005/113] Removed usage as explicit command --- deploy/lib/ml.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deploy/lib/ml.rb b/deploy/lib/ml.rb index f2238837..73f10be6 100755 --- a/deploy/lib/ml.rb +++ b/deploy/lib/ml.rb @@ -62,9 +62,7 @@ def need_help? # # Roxy framework is a convenience utility for create MVC code # - if command == "usage" - Help.doHelp(@logger, :usage) - elsif command == "create" + if command == "create" if need_help? Help.doHelp(@logger, command) else From 8ca73adefdc2fe0a61ede8f5dd765dacc102658e Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 22 Apr 2014 21:51:01 +0200 Subject: [PATCH 006/113] Fix for #207: used eval instead of xdmp:user to get scheduled user --- deploy/lib/xquery/setup.xqy | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index a1ef75ad..8da9331c 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3400,7 +3400,7 @@ declare function setup:create-scheduled-task( $task/gr:task-start-time, admin:database-get-id($admin-config, $task/gr:task-database/@name), admin:database-get-id($admin-config, $task/gr:task-modules/@name), - xdmp:user($task/gr:task-user/@name), + setup:get-user-id($task/gr:task-user/@name), $task/gr:task-host/xdmp:host(.), $task/gr:task-priority) else if ($task/gr:task-type eq "hourly") then @@ -3411,7 +3411,7 @@ declare function setup:create-scheduled-task( $task/gr:task-minute, admin:database-get-id($admin-config, $task/gr:task-database/@name), admin:database-get-id($admin-config, $task/gr:task-modules/@name), - xdmp:user($task/gr:task-user/@name), + setup:get-user-id($task/gr:task-user/@name), $task/gr:task-host/xdmp:host(.), $task/gr:task-priority) else if ($task/gr:task-type eq "minutely") then @@ -3421,7 +3421,7 @@ declare function setup:create-scheduled-task( $task/gr:task-period, admin:database-get-id($admin-config, $task/gr:task-database/@name), admin:database-get-id($admin-config, $task/gr:task-modules/@name), - xdmp:user($task/gr:task-user/@name), + setup:get-user-id($task/gr:task-user/@name), $task/gr:task-host/xdmp:host(.), $task/gr:task-priority) else if ($task/gr:task-type eq "monthly") then @@ -3433,7 +3433,7 @@ declare function setup:create-scheduled-task( $task/gr:task-start-time, admin:database-get-id($admin-config, $task/gr:task-database/@name), admin:database-get-id($admin-config, $task/gr:task-modules/@name), - xdmp:user($task/gr:task-user/@name), + setup:get-user-id($task/gr:task-user/@name), $task/gr:task-host/xdmp:host(.), $task/gr:task-priority) else if ($task/gr:task-type eq "once") then @@ -3443,7 +3443,7 @@ declare function setup:create-scheduled-task( $task/gr:task-start, admin:database-get-id($admin-config, $task/gr:task-database/@name), admin:database-get-id($admin-config, $task/gr:task-modules/@name), - xdmp:user($task/gr:task-user/@name), + setup:get-user-id($task/gr:task-user/@name), $task/gr:task-host/xdmp:host(.), $task/gr:task-priority) else if ($task/gr:task-type eq "weekly") then @@ -3455,7 +3455,7 @@ declare function setup:create-scheduled-task( $task/gr:task-start-time, admin:database-get-id($admin-config, $task/gr:task-database/@name), admin:database-get-id($admin-config, $task/gr:task-modules/@name), - xdmp:user($task/gr:task-user/@name), + setup:get-user-id($task/gr:task-user/@name), $task/gr:task-host/xdmp:host(.), $task/gr:task-priority) else () @@ -4182,6 +4182,17 @@ declare function setup:get-users($ids as xs:unsignedLong*) as element(sec:users) }; +declare function setup:get-user-id($user-name as xs:string) as xs:unsignedLong? { + xdmp:eval( + 'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy"; + declare variable $user-name as xs:string external; + /sec:user[sec:user-name = $user-name]/sec:user-id', + (xs:QName("user-name"), $user-name), + + {$default-security} + ) +}; + declare function setup:get-roles($ids as xs:unsignedLong*) as element(sec:roles)? { let $roles := xdmp:eval( From f059a2525b1b5e5c8daadb84f2669d57e882011a Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 22 Apr 2014 22:09:32 +0200 Subject: [PATCH 007/113] Initial attempt on #8: capturing the full environment ml-config with capture --full-ml-config --- deploy/lib/server_config.rb | 83 +++++++++++++++++++++++++------------ deploy/lib/xquery/setup.xqy | 12 +++--- 2 files changed, 62 insertions(+), 33 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a13ebab4..a96ea149 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -708,40 +708,47 @@ def credentials end def capture - - if @properties['ml.app-type'] != 'rest' - raise ExitException.new("This is a #{@properties['ml.app-type']} application; capture only works for app-type=rest") - end - + full_config = find_arg(['--full-ml-config']) target_db = find_arg(['--modules-db']) + + # check params + if full_config == nil && target_db == nil + raise HelpException.new("capture", "either full-ml-config or modules-db is required") + elsif target_db != nil && @properties['ml.app-type'] != 'rest' + raise ExitException.new("This is a #{@properties['ml.app-type']} application; capture modules only works for app-type=rest") + end - if target_db == nil - raise HelpException.new("capture", "modules-db is required") + # retrieve full setup config from environment + if full_config != nil + capture_environment_config end - tmp_dir = Dir.mktmpdir - logger.debug "using temp dir " + tmp_dir - logger.info "Retrieving source and REST config from #{target_db}..." + # retrieve modules from selected database from environment + if target_db != nil + tmp_dir = Dir.mktmpdir + logger.debug "using temp dir " + tmp_dir + logger.info "Retrieving source and REST config from #{target_db}..." + + save_files_to_fs(target_db, "#{tmp_dir}/src") + + # set up the options + FileUtils.cp_r( + "#{tmp_dir}/src/#{@properties['ml.group']}/" + target_db.sub("-modules", "") + "/rest-api/.", + @properties['ml.rest-options.dir'] + ) + FileUtils.rm_rf("#{tmp_dir}/src/#{@properties['ml.group']}/") + + # If we have an application/custom directory, we've probably done a capture + # before. Don't overwrite that directory. Kill the downloaded custom directory + # to avoid overwriting. + if Dir.exists? "#{@properties["ml.xquery.dir"]}/application/custom" + FileUtils.rm_rf("#{tmp_dir}/src/application/custom") + end - save_files_to_fs(target_db, "#{tmp_dir}/src") + FileUtils.cp_r("#{tmp_dir}/src/.", @properties["ml.xquery.dir"]) - # set up the options - FileUtils.cp_r( - "#{tmp_dir}/src/#{@properties['ml.group']}/" + target_db.sub("-modules", "") + "/rest-api/.", - @properties['ml.rest-options.dir'] - ) - FileUtils.rm_rf("#{tmp_dir}/src/#{@properties['ml.group']}/") - - # If we have an application/custom directory, we've probably done a capture - # before. Don't overwrite that directory. Kill the downloaded custom directory - # to avoid overwriting. - if Dir.exists? "#{@properties["ml.xquery.dir"]}/application/custom" - FileUtils.rm_rf("#{tmp_dir}/src/application/custom") + FileUtils.rm_rf(tmp_dir) end - - FileUtils.cp_r("#{tmp_dir}/src/.", @properties["ml.xquery.dir"]) - - FileUtils.rm_rf(tmp_dir) end private @@ -800,6 +807,28 @@ def save_files_to_fs(target_db, target_dir) end + def capture_environment_config + raise ExitException.new("Capture requires the target environment's hostname to be defined") unless @hostname.present? + + logger.info "Capturing configuration of MarkLogic on #{@hostname}..." + setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) + r = execute_query %Q{#{setup} setup:get-configuration((), (), (), (), (), ())} + + if r.body.match("error log") + logger.error r.body + logger.error "... Capture FAILED" + return false + else + JSON.parse(r.body).each do |item| + contents = item['result'] + name = "#{@properties["ml.config.file"].sub( %r{.xml}, '' )}-#{@properties["ml.environment"]}.xml" + File.open(name, 'w') { |file| file.write(contents) } + logger.info("... Captured full configuration into #{name}") + end + return true + end + end + # Build an array of role/capability objects. def permissions(role, capabilities) if capabilities.is_a?(Array) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 8da9331c..370111ce 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3914,7 +3914,7 @@ declare function setup:get-app-servers($names as xs:string*) as element()* { let $groups := setup:read-config-file("groups.xml")/gr:groups/gr:group return ( - let $http-servers := $groups/gr:http-servers/gr:http-server[gr:http-server-name = $names] + let $http-servers := $groups/gr:http-servers/gr:http-server[empty($names) or gr:http-server-name = $names] where $http-servers return , - let $xdbc-servers := $groups/gr:xdbc-servers/gr:xdbc-server[gr:xdbc-server-name = $names] + let $xdbc-servers := $groups/gr:xdbc-servers/gr:xdbc-server[empty($names) or gr:xdbc-server-name = $names] where $xdbc-servers return , - let $odbc-servers := $groups/gr:odbc-servers/gr:odbc-server[gr:odbc-server-name = $names] + let $odbc-servers := $groups/gr:odbc-servers/gr:odbc-server[empty($names) or gr:odbc-server-name = $names] where $odbc-servers return Date: Wed, 23 Apr 2014 16:15:38 +0200 Subject: [PATCH 008/113] Added JSON Parsing on response bodies for ML 7 --- deploy/lib/server_config.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a96ea149..ce03c60a 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -439,7 +439,12 @@ def validate_install logger.info "code: #{r.code.to_i}" logger.info r.body - if r.body.match(" e - logger.error e.response.body + if e.response.body.match('"result":" Date: Wed, 23 Apr 2014 16:17:33 +0200 Subject: [PATCH 009/113] Extended ml7-config.xml self-test data with non-default values, schemas, module-locations, and request-blackouts --- deploy/test/data/ml7-config.xml | 171 ++++++++++++++++++++++---------- 1 file changed, 121 insertions(+), 50 deletions(-) diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index bc101498..ebf2af2d 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -79,57 +79,72 @@ - + + @ml.app-name + false + /@ml.app-name/ @ml.app-port - + + - 0 - false -
0.0.0.0
- 256 - 32 - 30 - 5 - 3600 - 3600 - 600 - 3600 - 1000 - 10000 - http://marklogic.com/collation/ - digest + + Last-Login + true +
1.2.3.4
+ 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + http://marklogic.com/collation/codepoint + basic + + @ml.default-user admin-ui - 0 - false - true - true - true - 1.0-ml - contemporaneous - none - UTF-8 - default - default - - - - - default - default - default - default - - none - default - omit - default - - default + 123 + + false + false + false + 1.0 + nonblocking + strict + + normal + ASCII + xml + yes + a + b + c + d + yes + yes + yes + yes + text/xml + NFC + yes + yes + yes + 1.0 + yes + /roxy/error.xqy - + + + http://www.ns.com/ns0 + /test.xsd + + ns1 @@ -139,16 +154,72 @@ ns2 http://www.ns.com/ns2 + + http://www.ns.com/ns3 + - + + + http://www.ns.com/ns4 + /test-lib.xqy + + + + + + + recurring + + monday + + + 12:34:00+02:00 + 23:45:00+02:00 + + + + + + recurring + + monday + + + 12:34:00+02:00 + PT1H23M + + + + + + recurring + + monday + + + + + + + once + + + 1234-11-22 + 12:34:00+02:00 + 1234-12-23 + 23:34:00+02:00 + + + /roxy/rewrite.xqy true - - true - true + + false + false sixer - ALL:!LOW:@STRENGTH - true + ALL:!LOW:@STRENGTH:ROXY + false + @ml.test-appserver From 7fad34a8e1230c0ded2882c7ef297a9f74c46998 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Wed, 23 Apr 2014 16:43:24 +0200 Subject: [PATCH 010/113] Fixes for #209, #211, #212: enabled appserver settings validation, and added functionality to set schemas, module-locations, and request-blockouts on appservers --- deploy/lib/xquery/setup.xqy | 188 +++++++++++++++++++++++++++++------- 1 file changed, 154 insertions(+), 34 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 370111ce..1138f595 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -121,6 +121,7 @@ declare variable $http-server-settings := enabled root port + modules-database database last-login @@ -138,9 +139,12 @@ declare variable $http-server-settings := pre-commit-trigger-limit collation authentication + + default-user - privilege2 + privilege concurrent-request-limit + log-errors debug-allow profile-allow @@ -200,7 +204,7 @@ declare variable $webdav-server-settings := collation authentication default-user - privilege2 + privilege concurrent-request-limit compute-content-length log-errors @@ -237,7 +241,7 @@ declare variable $webdav-server-settings := ; -declare variable $xcc-server-settings := +declare variable $xdbc-server-settings := enabled root @@ -258,7 +262,7 @@ declare variable $xcc-server-settings := pre-commit-trigger-limit collation authentication - privilege2 + privilege concurrent-request-limit log-errors debug-allow @@ -317,7 +321,7 @@ declare variable $odbc-server-settings := pre-commit-trigger-limit collation authentication - privilege2 + privilege concurrent-request-limit log-errors debug-allow @@ -3155,14 +3159,15 @@ declare function setup:validate-http-server( { setup:validate-server( $server-config, - xdmp:server($server-config/gr:http-server-name[fn:string-length(fn:string(.)) > 0])) + xdmp:server($server-config/gr:http-server-name[fn:string-length(fn:string(.)) > 0]), + $http-server-settings) }; declare function setup:configure-xdbc-server( $server-config as element(gr:xdbc-server)) as item()* { let $server-name as xs:string? := $server-config/gr:xdbc-server-name[fn:string-length(fn:string(.)) > 0] - let $admin-config := setup:configure-server($server-config, xdmp:server($server-name), $xcc-server-settings) + let $admin-config := setup:configure-server($server-config, xdmp:server($server-name), $xdbc-server-settings) return ( if (admin:save-configuration-without-restart($admin-config)) then @@ -3177,7 +3182,8 @@ declare function setup:validate-xdbc-server( { setup:validate-server( $server-config, - xdmp:server($server-config/gr:xdbc-server-name[fn:string-length(fn:string(.)) > 0])) + xdmp:server($server-config/gr:xdbc-server-name[fn:string-length(fn:string(.)) > 0]), + $xdbc-server-settings) }; declare function setup:configure-odbc-server( @@ -3199,7 +3205,8 @@ declare function setup:validate-odbc-server( { setup:validate-server( $server-config, - xdmp:server($server-config/gr:odbc-server-name[fn:string-length(fn:string(.)) > 0])) + xdmp:server($server-config/gr:odbc-server-name[fn:string-length(fn:string(.)) > 0]), + $odbc-server-settings) }; declare function setup:configure-task-server( @@ -3296,7 +3303,7 @@ declare function setup:configure-server( if ($namespaces) then let $old-ns := admin:appserver-get-namespaces($admin-config, $server-id) let $config := - (: First delete any namespace that matches the prefix and uri :) + (: First delete any namespace that matches the prefix :) admin:appserver-delete-namespace( $admin-config, $server-id, @@ -3319,38 +3326,126 @@ declare function setup:configure-server( admin:group-namespace($ns/gr:prefix, $ns/gr:namespace-uri)) else $admin-config - (: TODO: schemas, request-blackouts :) + + let $schemas := $server-config/gr:schemas/gr:schema + let $admin-config := + if ($schemas) then + let $old-schemas := admin:appserver-get-schemas($admin-config, $server-id) + let $config := + (: First delete any schema that matches the namespace :) + admin:appserver-delete-schema( + $admin-config, + $server-id, + for $schema in $schemas + let $same-ns := + $old-schemas[gr:namespace-uri = $schema/gr:namespace-uri][gr:schema-location ne $schema/gr:schema-location] + return + if ($same-ns) then $schema + else ()) + return + (: Then add in any schema whose namespace isn't already defined :) + admin:appserver-add-schema( + $config, + $server-id, + for $schema in $schemas + return + if ($old-schemas[gr:namespace-uri = $schema/gr:namespace-uri][gr:schema-location = $schema/gr:schema-location]) then () + else + $schema) + else + $admin-config + + let $module-locations := $server-config/gr:module-locations/gr:module-location + let $admin-config := + if ($module-locations) then + let $old-module-locations := admin:appserver-get-module-locations($admin-config, $server-id) + let $config := + (: First delete any module-location that matches the namespace :) + admin:appserver-delete-module-location( + $admin-config, + $server-id, + for $module-location in $module-locations + let $same-ns := + $old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location ne $module-location/gr:location] + return + if ($same-ns) then $module-location + else ()) + return + (: Then add in any module-location whose namespace isn't already defined :) + admin:appserver-add-module-location( + $config, + $server-id, + for $module-location in $module-locations + return + if ($old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location = $module-location/gr:location]) then () + else + $module-location) + else + $admin-config + + let $request-blackouts := $server-config/gr:request-blackouts/gr:request-blackout + let $admin-config := + if ($request-blackouts) then + let $old-request-blackouts := admin:appserver-get-request-blackouts($admin-config, $server-id) + let $config := + (: First delete any request-blackout that matches type and period :) + admin:appserver-delete-request-blackout( + $admin-config, + $server-id, + for $request-blackout in $request-blackouts + let $same-blackout := + $old-request-blackouts[setup:get-request-blackout-hash(.) = setup:get-request-blackout-hash($request-blackout)][gr:users ne $request-blackout/gr:users or gr:roles ne $request-blackout/gr:roles] + return + if ($same-blackout) then $request-blackout + else ()) + return + (: Then add in any request-blackout whose type and period aren't already defined :) + admin:appserver-add-request-blackout( + $config, + $server-id, + for $request-blackout in $request-blackouts + return + if ($old-request-blackouts[setup:get-request-blackout-hash(.) = setup:get-request-blackout-hash($request-blackout)][gr:users eq $request-blackout/gr:users and gr:roles eq $request-blackout/gr:roles]) then () + else + $request-blackout) + else + $admin-config + return $admin-config }; +declare function setup:get-request-blackout-hash($blackout) { + string-join(($blackout/gr:blackout-type, $blackout//gr:day, $blackout//start-date, $blackout//start-time, $blackout//end-date, $blackout//end-time), '-') +}; + declare function setup:validate-server( $server-config as element(), - $server-id as xs:unsignedLong) as element(configuration) + $server-id as xs:unsignedLong, + $settings as element(settings)) as item()* { let $admin-config := admin:get-configuration() - let $_ := - let $actual := admin:appserver-get-last-login($admin-config, $server-id) - let $expected := - if (fn:data($server-config/gr:last-login/(@name|text()))) then - xdmp:database(fn:data($server-config/gr:last-login/(@name|text()))) - else 0 - return - if ($actual = $expected) then () - else - setup:validation-fail(fn:concat("Appserver last-login mismatch: ", $expected, " != ", $actual)) - for $setting in $http-server-settings/*:setting - let $min-version as xs:string? := $setting/@min-version - let $expected := - if (fn:empty($min-version) or setup:at-least-version($min-version)) then - fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, "[fn:string-length(fn:string(.)) > 0]"))) - else () + + for $setting in $settings/*:setting + let $setting-test := + if ($setting/@accept-blank = "true") then + "" + else + "[fn:string-length(fn:string(.)) > 0]" + let $expected := + if ($setting/@value) then + xdmp:value($setting/@value) + else + fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, $setting-test))) let $actual := xdmp:value(fn:concat("admin:appserver-get-", $setting, "($admin-config, $server-id)")) - where $expected + let $min-version as xs:string? := $setting/@min-version + where (fn:exists($expected)) return - if ($expected = $actual) then () - else - setup:validation-fail(fn:concat("Appserver ", $setting, " mismatch: ", $expected, " != ", $actual)), + if (fn:empty($min-version) or setup:at-least-version($min-version)) then + if ($expected = $actual) then () + else + setup:validation-fail(fn:concat("Appserver ", $setting, " mismatch: ", $expected, " != ", $actual)) + else (), let $admin-config := admin:get-configuration() let $existing := admin:appserver-get-namespaces($admin-config, $server-id) @@ -3358,7 +3453,31 @@ declare function setup:validate-server( return if ($existing[fn:deep-equal(., $expected)]) then () else - setup:validation-fail(fn:concat("Appserver missing namespace: ", $expected/gr:namespace-uri)) + setup:validation-fail(fn:concat("Appserver missing namespace: ", $expected/gr:namespace-uri)), + + let $admin-config := admin:get-configuration() + let $existing := admin:appserver-get-schemas($admin-config, $server-id) + for $expected in $server-config/gr:schemas/gr:schema + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing schema: ", $expected/gr:schema)), + + let $admin-config := admin:get-configuration() + let $existing := admin:appserver-get-module-locations($admin-config, $server-id) + for $expected in $server-config/gr:module-locations/gr:module-location + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing module location: ", $expected/gr:module-location)), + + let $admin-config := admin:get-configuration() + let $existing := admin:appserver-get-request-blackouts($admin-config, $server-id) + for $expected in $server-config/gr:request-blackouts/gr:request-blackout + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing request blackout: ", $expected/gr:request-blackout)) }; declare function setup:create-scheduled-tasks( @@ -4012,7 +4131,7 @@ declare function setup:get-appserver-modules-database($server-config as element( { let $modules as xs:string? := $server-config/gr:modules/(@name|text()) return - if ($modules eq "filesystem") then 0 + if ($modules = ("filesystem", "0")) then 0 else if ($modules) then xdmp:database($modules) else 0 }; @@ -4637,6 +4756,7 @@ declare function setup:validate-install($import-config as element(configuration) setup:validate-database-settings($import-config), setup:validate-databases-indexes($import-config), setup:validate-appservers($import-config), + setup:validate-appservers-settings($import-config), setup:validate-scheduled-tasks($import-config) } catch($ex) From 2bad1c9c3c7059e75a7955f02cca0cdb824ebc12 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Wed, 23 Apr 2014 14:51:48 -0400 Subject: [PATCH 011/113] Fix #177 - url encode password for corb --- deploy/lib/server_config.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a13ebab4..805d66ce 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -632,7 +632,8 @@ def xqsync end def corb - connection_string = %Q{xcc://#{@properties['ml.user']}:#{@properties['ml.password']}@#{@properties['ml.server']}:#{@properties['ml.xcc-port']}/#{@properties['ml.content-db']}} + encoded_password = url_encode(@properties['ml.password']) + connection_string = %Q{xcc://#{@properties['ml.user']}:#{encoded_password}@#{@properties['ml.server']}:#{@properties['ml.xcc-port']}/#{@properties['ml.content-db']}} collection_name = find_arg(['--collection']) || '""' xquery_module = find_arg(['--modules']) uris_module = find_arg(['--uris']) || '""' From fa6bf5daa41e93117328756693933954468a6685 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 24 Apr 2014 11:10:35 +0200 Subject: [PATCH 012/113] Fix for #214: allowing --server-version param for self-test --- ml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ml b/ml index 1e73e5a5..bfd7bec3 100755 --- a/ml +++ b/ml @@ -124,6 +124,13 @@ elif [ "$1" == 'self-test' ] then if [ -e deploy/test/test_main.rb ] then + shift + version="$1" + if [ "${version:0:17}" == '--server-version=' ] + then + # This exports the new version only to sub-processes, e.g. the ruby call below.. + export ROXY_TEST_SERVER_VERSION="${version:17}" + fi ruby -I deploy -I deploy/lib -I deploy/test deploy/test/test_main.rb else printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" From 6bf917fa192531b835a3f0db8c8b80784eb195cf Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 24 Apr 2014 11:11:34 +0200 Subject: [PATCH 013/113] Extra log line showing test version --- deploy/test/test_server_config.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/test/test_server_config.rb b/deploy/test/test_server_config.rb index f06d624e..38c267da 100644 --- a/deploy/test/test_server_config.rb +++ b/deploy/test/test_server_config.rb @@ -63,6 +63,7 @@ def bootstrap_version(version) def test_bootstrap version = ENV['ROXY_TEST_SERVER_VERSION'] || 7 + Logger.new(STDOUT).info "Testing against MarkLogic version #{version}.." bootstrap_version version end end \ No newline at end of file From ca268f26bf3d148ee2be8b0148eadd074079e66a Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 24 Apr 2014 11:12:40 +0200 Subject: [PATCH 014/113] Added mimetype sample to ml-config sample --- deploy/sample/ml-config.sample.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/sample/ml-config.sample.xml b/deploy/sample/ml-config.sample.xml index cbaa99bc..4f609a4d 100644 --- a/deploy/sample/ml-config.sample.xml +++ b/deploy/sample/ml-config.sample.xml @@ -490,4 +490,14 @@ --> + + + From 2b2a3e769dd44334e6ee86c9dd41b3ddf5bb6a85 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 24 Apr 2014 11:17:09 +0200 Subject: [PATCH 015/113] Added version check to module-locations (#209), removed server setting definition redundancy --- deploy/lib/xquery/setup.xqy | 408 +++++++++++++----------------------- 1 file changed, 151 insertions(+), 257 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 1138f595..b2346988 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -121,7 +121,7 @@ declare variable $http-server-settings := enabled root port - + modules-database database last-login @@ -139,37 +139,37 @@ declare variable $http-server-settings := pre-commit-trigger-limit collation authentication - - + + default-user - privilege + privilege2 concurrent-request-limit - + log-errors debug-allow profile-allow default-xquery-version distribute-timestamps - multi-version-concurrency-control + multi-version-concurrency-control output-sgml-character-entities output-encoding - output-method - output-byte-order-mark - output-cdata-section-namespace-uri - output-cdata-section-localname - output-doctype-public - output-doctype-system - output-escape-uri-attributes - output-include-content-type - output-indent - output-indent-untyped - output-media-type - output-normalization-form - output-omit-xml-declaration - output-standalone - output-undeclare-prefixes - output-version - output-include-default-attributes + output-method + output-byte-order-mark + output-cdata-section-namespace-uri + output-cdata-section-localname + output-doctype-public + output-doctype-system + output-escape-uri-attributes + output-include-content-type + output-indent + output-indent-untyped + output-media-type + output-normalization-form + output-omit-xml-declaration + output-standalone + output-undeclare-prefixes + output-version + output-include-default-attributes error-handler url-rewriter rewrite-resolves-globally @@ -183,178 +183,20 @@ declare variable $http-server-settings := ; declare variable $webdav-server-settings := - - enabled - root - port - database - last-login - display-last-login - address - backlog - threads - request-timeout - keep-alive-timeout - session-timeout - max-time-limit - default-time-limit - static-expires - pre-commit-trigger-depth - pre-commit-trigger-limit - collation - authentication - default-user - privilege - concurrent-request-limit + { + $http-server-settings/*[fn:not(fn:data(.) = 'modules-database')], compute-content-length - log-errors - debug-allow - profile-allow - default-xquery-version - multi-version-concurrency-control - output-sgml-character-entities - output-encoding - output-method - output-byte-order-mark - output-cdata-section-namespace-uri - output-cdata-section-localname - output-doctype-public - output-doctype-system - output-escape-uri-attributes - output-include-content-type - output-indent - output-indent-untyped - output-media-type - output-normalization-form - output-omit-xml-declaration - output-standalone - output-undeclare-prefixes - output-version - output-include-default-attributes - rewrite-resolves-globally - ssl-certificate-template - ssl-allow-sslv3 - ssl-allow-tls - ssl-hostname - ssl-ciphers - ssl-require-client-certificate - + } ; declare variable $xdbc-server-settings := - - enabled - root - port - modules-database - database - last-login - display-last-login - address - backlog - threads - request-timeout - keep-alive-timeout - session-timeout - max-time-limit - default-time-limit - pre-commit-trigger-depth - pre-commit-trigger-limit - collation - authentication - privilege - concurrent-request-limit - log-errors - debug-allow - profile-allow - default-xquery-version - distribute-timestamps - multi-version-concurrency-control - output-sgml-character-entities - output-encoding - output-method - output-byte-order-mark - output-cdata-section-namespace-uri - output-cdata-section-localname - output-doctype-public - output-doctype-system - output-escape-uri-attributes - output-include-content-type - output-indent - output-indent-untyped - output-media-type - output-normalization-form - output-omit-xml-declaration - output-standalone - output-undeclare-prefixes - output-version - output-include-default-attributes - ssl-certificate-template - ssl-allow-sslv3 - ssl-allow-tls - ssl-hostname - ssl-ciphers - ssl-require-client-certificate - ssl-client-certificate-authorities - + { + $http-server-settings/*[fn:not(fn:data(.) = ('static-expires', 'default-user', 'error-handler', 'url-rewriter', 'rewrite-resolves-globally'))] + } ; declare variable $odbc-server-settings := - - enabled - root - port - modules-database - database - last-login - display-last-login - address - backlog - threads - request-timeout - keep-alive-timeout - session-timeout - max-time-limit - default-time-limit - static-expires - pre-commit-trigger-depth - pre-commit-trigger-limit - collation - authentication - privilege - concurrent-request-limit - log-errors - debug-allow - profile-allow - default-xquery-version - distribute-timestamps - multi-version-concurrency-control - output-sgml-character-entities - output-encoding - output-method - output-byte-order-mark - output-cdata-section-namespace-uri - output-cdata-section-localname - output-doctype-public - output-doctype-system - output-escape-uri-attributes - output-include-content-type - output-indent - output-indent-untyped - output-media-type - output-normalization-form - output-omit-xml-declaration - output-standalone - output-undeclare-prefixes - output-version - output-include-default-attributes - ssl-certificate-template - ssl-allow-sslv3 - ssl-allow-tls - ssl-hostname - ssl-ciphers - ssl-require-client-certificate - + $xdbc-server-settings ; declare variable $task-server-settings := @@ -3261,6 +3103,14 @@ declare function setup:get-appserver-privilege( else 0 }; +declare function admin:appserver-get-privilege2( + $config as element(configuration), + $appserver-id as xs:unsignedLong) +as xs:unsignedLong +{ + admin:appserver-get-privilege($config, $appserver-id) +}; + declare function admin:appserver-set-privilege2( $config as element(configuration), $appserver-id as xs:unsignedLong, @@ -3296,7 +3146,10 @@ declare function setup:configure-server( if (fn:empty($min-version) or setup:at-least-version($min-version)) then xdmp:set($admin-config, xdmp:value(fn:concat("admin:appserver-set-", $setting, "($admin-config, $server-id, $value)"))) - else () + else + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support ", $setting, ". Use ", $min-version, " or higher.")) let $namespaces := $server-config/gr:namespaces/gr:namespace let $admin-config := @@ -3358,28 +3211,48 @@ declare function setup:configure-server( let $module-locations := $server-config/gr:module-locations/gr:module-location let $admin-config := if ($module-locations) then - let $old-module-locations := admin:appserver-get-module-locations($admin-config, $server-id) - let $config := - (: First delete any module-location that matches the namespace :) - admin:appserver-delete-module-location( - $admin-config, - $server-id, - for $module-location in $module-locations - let $same-ns := - $old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location ne $module-location/gr:location] - return - if ($same-ns) then $module-location - else ()) - return - (: Then add in any module-location whose namespace isn't already defined :) - admin:appserver-add-module-location( - $config, - $server-id, - for $module-location in $module-locations + if (setup:at-least-version("7.0-0")) then + xdmp:eval(' + xquery version "1.0-ml"; + + import module namespace admin = "http://marklogic.com/xdmp/admin" at "/MarkLogic/admin.xqy"; + + declare namespace gr="http://marklogic.com/xdmp/group"; + + declare variable $admin-config external; + declare variable $server-id external; + declare variable $module-locations external; + + let $old-module-locations := admin:appserver-get-module-locations($admin-config, $server-id) + let $config := + (: First delete any module-location that matches the namespace :) + admin:appserver-delete-module-location( + $admin-config, + $server-id, + for $module-location in $module-locations + let $same-ns := + $old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location ne $module-location/gr:location] + return + if ($same-ns) then $module-location + else ()) return - if ($old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location = $module-location/gr:location]) then () - else - $module-location) + (: Then add in any module-location whose namespace isnt already defined :) + admin:appserver-add-module-location( + $config, + $server-id, + for $module-location in $module-locations + return + if ($old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location = $module-location/gr:location]) then () + else + $module-location) + ', + (xs:QName("admin-config"),$admin-config, + xs:QName("server-id"),$server-id, + xs:QName("module-locations"),$module-locations)) + else + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support module-locations. Use 7.0-0 or higher.")) else $admin-config @@ -3425,59 +3298,80 @@ declare function setup:validate-server( $settings as element(settings)) as item()* { let $admin-config := admin:get-configuration() + return ( - for $setting in $settings/*:setting - let $setting-test := - if ($setting/@accept-blank = "true") then - "" - else - "[fn:string-length(fn:string(.)) > 0]" - let $expected := - if ($setting/@value) then - xdmp:value($setting/@value) - else - fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, $setting-test))) - let $actual := xdmp:value(fn:concat("admin:appserver-get-", $setting, "($admin-config, $server-id)")) - let $min-version as xs:string? := $setting/@min-version - where (fn:exists($expected)) - return - if (fn:empty($min-version) or setup:at-least-version($min-version)) then - if ($expected = $actual) then () + for $setting in $settings/*:setting + let $setting-test := + if ($setting/@accept-blank = "true") then + "" else - setup:validation-fail(fn:concat("Appserver ", $setting, " mismatch: ", $expected, " != ", $actual)) - else (), + "[fn:string-length(fn:string(.)) > 0]" + let $expected := + if ($setting/@value) then + xdmp:value($setting/@value) + else + fn:data(xdmp:value(fn:concat("$server-config/gr:", $setting, $setting-test))) + let $min-version as xs:string? := $setting/@min-version + where (fn:exists($expected)) + return + if (fn:empty($min-version) or setup:at-least-version($min-version)) then + let $actual := xdmp:value(fn:concat("admin:appserver-get-", $setting, "($admin-config, $server-id)")) + return + if ($expected = $actual) then () + else + setup:validation-fail(fn:concat("Appserver ", $setting, " mismatch: ", $expected, " != ", $actual)) + else + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support ", $setting, ". Use ", $min-version, " or higher.")), - let $admin-config := admin:get-configuration() - let $existing := admin:appserver-get-namespaces($admin-config, $server-id) - for $expected in $server-config/gr:namespaces/gr:namespace - return - if ($existing[fn:deep-equal(., $expected)]) then () - else - setup:validation-fail(fn:concat("Appserver missing namespace: ", $expected/gr:namespace-uri)), + let $existing := admin:appserver-get-namespaces($admin-config, $server-id) + for $expected in $server-config/gr:namespaces/gr:namespace + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing namespace: ", $expected/gr:namespace-uri)), - let $admin-config := admin:get-configuration() - let $existing := admin:appserver-get-schemas($admin-config, $server-id) - for $expected in $server-config/gr:schemas/gr:schema - return - if ($existing[fn:deep-equal(., $expected)]) then () - else - setup:validation-fail(fn:concat("Appserver missing schema: ", $expected/gr:schema)), + let $existing := admin:appserver-get-schemas($admin-config, $server-id) + for $expected in $server-config/gr:schemas/gr:schema + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing schema: ", $expected/gr:schema)), - let $admin-config := admin:get-configuration() - let $existing := admin:appserver-get-module-locations($admin-config, $server-id) - for $expected in $server-config/gr:module-locations/gr:module-location - return - if ($existing[fn:deep-equal(., $expected)]) then () - else - setup:validation-fail(fn:concat("Appserver missing module location: ", $expected/gr:module-location)), + if ($server-config/gr:module-locations/gr:module-location) then + if (setup:at-least-version("7.0-0")) then + let $existing := + xdmp:eval(' + xquery version "1.0-ml"; - let $admin-config := admin:get-configuration() - let $existing := admin:appserver-get-request-blackouts($admin-config, $server-id) - for $expected in $server-config/gr:request-blackouts/gr:request-blackout - return - if ($existing[fn:deep-equal(., $expected)]) then () - else - setup:validation-fail(fn:concat("Appserver missing request blackout: ", $expected/gr:request-blackout)) + import module namespace admin = "http://marklogic.com/xdmp/admin" at "/MarkLogic/admin.xqy"; + + declare variable $admin-config external; + declare variable $server-id external; + + admin:appserver-get-module-locations($admin-config, $server-id) + ', + (xs:QName("admin-config"),$admin-config, + xs:QName("server-id"),$server-id)) + for $expected in $server-config/gr:module-locations/gr:module-location + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing module location: ", $expected/gr:module-location)) + else + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support module-locations. Use 7.0-0 or higher.")) + else (), + + let $existing := admin:appserver-get-request-blackouts($admin-config, $server-id) + for $expected in $server-config/gr:request-blackouts/gr:request-blackout + return + if ($existing[fn:deep-equal(., $expected)]) then () + else + setup:validation-fail(fn:concat("Appserver missing request blackout: ", $expected/gr:request-blackout)) + ) }; declare function setup:create-scheduled-tasks( From 4de7a2f76836d8120d0cb7f2d59647e45e90b39a Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 24 Apr 2014 13:50:59 +0200 Subject: [PATCH 016/113] Fix for #216 and #217: exit early at ml new failure, and --fork option for ml new --- ml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/ml b/ml index bfd7bec3..e625a97c 100755 --- a/ml +++ b/ml @@ -52,6 +52,7 @@ then INIT_GIT=0 FORCE_INSTALL=0 APPTYPE="mvc" + FORK="marklogic" for (( i = 0; i < ${#PARAMS[@]}; i++ )); do if [[ ${PARAMS[${i}]} == --branch=* ]] then @@ -63,10 +64,14 @@ then elif [[ ${PARAMS[${i}]} == --force* ]] then FORCE_INSTALL=1 - elif [[ ${PARAMS[${i}]} == --app-type* ]] + elif [[ ${PARAMS[${i}]} == --app-type=* ]] then splits=(${PARAMS[${i}]//=/ }) APPTYPE=${splits[1]} + elif [[ ${PARAMS[${i}]} == --fork=* ]] + then + splits=(${PARAMS[${i}]//=/ }) + FORK=${splits[1]} fi done @@ -82,28 +87,28 @@ then exit 1 fi - printf "\nCreating new Application: ${app_name}..." + printf "\nCreating new Application: ${app_name}...\n" if [ -e $app_name ] then - git clone git://github.com/marklogic/roxy.git -b ${BRANCH} ${app_name}.tmp_1 - mv ${app_name}.tmp_1/* ${app_name}/ - rm -rf ${app_name}.tmp_1 + git clone git://github.com/${FORK}/roxy.git -b ${BRANCH} ${app_name}.tmp_1 || exit 1 + mv ${app_name}.tmp_1/* ${app_name}/ || exit 1 + rm -rf ${app_name}.tmp_1 || exit 1 else - git clone git://github.com/marklogic/roxy.git -b ${BRANCH} ${app_name} + git clone git://github.com/${FORK}/roxy.git -b ${BRANCH} ${app_name} || exit 1 fi - pushd ${app_name} > /dev/null - rm -rf .git* + pushd ${app_name} > /dev/null || exit 1 + rm -rf .git* || exit 1 if [ "$APPTYPE" = "rest" ] then # For a REST application, we won't be using the MVC code. Remove it. # mvc and hybrid apps will use it. - rm -rf src/* + rm -rf src/* || exit 1 printf "\nNo initial source code is provided for REST apps. You can copy code from Application Builder under the source directory.\n" fi - ./ml init ${app_name} ${@} - popd > /dev/null + ./ml init ${app_name} ${@} || exit 1 + popd > /dev/null || exit 1 printf " done\n" if [ -e $app_name ] then @@ -111,9 +116,9 @@ then then printf "Creating a git repository:\n" cd ${app_name} - git init - git add . - git commit -q -m "Initial Commit" + git init || exit 1 + git add . || exit 1 + git commit -q -m "Initial Commit" || exit 1 printf "...done\n" fi fi @@ -131,7 +136,7 @@ then # This exports the new version only to sub-processes, e.g. the ruby call below.. export ROXY_TEST_SERVER_VERSION="${version:17}" fi - ruby -I deploy -I deploy/lib -I deploy/test deploy/test/test_main.rb + ruby -I deploy -I deploy/lib -I deploy/test deploy/test/test_main.rb || exit 1 else printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" fi @@ -147,7 +152,7 @@ then else if [ -e deploy/lib/ml.rb ] then - ruby -I deploy -I deploy/lib deploy/lib/ml.rb $* + ruby -I deploy -I deploy/lib deploy/lib/ml.rb $* || exit 1 else printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" fi From 6f34ea673175ec9b00302fe335d50907bba6c88d Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 24 Apr 2014 14:22:27 +0200 Subject: [PATCH 017/113] Added --fork to upgrade as well (#217) --- deploy/lib/upgrader.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deploy/lib/upgrader.rb b/deploy/lib/upgrader.rb index c70744cb..c17b33d4 100644 --- a/deploy/lib/upgrader.rb +++ b/deploy/lib/upgrader.rb @@ -50,6 +50,7 @@ def upgrade_base(tmp_dir) end def upgrade(args) + fork = find_arg(['--fork']) || 'marklogic' branch = find_arg(['--branch']) raise HelpException.new("upgrade", "Missing branch name") unless branch @@ -62,11 +63,11 @@ def upgrade(args) confirm = gets.chomp if confirm.match(/^y(es)?$/i) - @logger.info "Upgrading to the #{branch} branch" + @logger.info "Upgrading to the #{branch} branch from #{fork}/roxy" tmp_dir = Dir.mktmpdir @logger.info "Cloning Roxy in a temp directory..." - system("git clone git://github.com/marklogic/roxy.git -b #{branch} #{tmp_dir}") + system("git clone git://github.com/#{fork}/roxy.git -b #{branch} #{tmp_dir}") @logger.info "Upgrading base project files\n" upgrade_base(tmp_dir) From fe775a8fcbeabeaa8c672872a83ced9a2b063d9f Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Thu, 24 Apr 2014 08:55:55 -0400 Subject: [PATCH 018/113] Getting rid of "privilege2" setting. The extra setup:get-appserver-privilege() function is needed, but the admin:appserver-set-privilege2() function is not. --- deploy/lib/xquery/setup.xqy | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index a1ef75ad..920a58a1 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -139,7 +139,7 @@ declare variable $http-server-settings := collation authentication default-user - privilege2 + privilege concurrent-request-limit log-errors debug-allow @@ -200,7 +200,7 @@ declare variable $webdav-server-settings := collation authentication default-user - privilege2 + privilege concurrent-request-limit compute-content-length log-errors @@ -258,7 +258,7 @@ declare variable $xcc-server-settings := pre-commit-trigger-limit collation authentication - privilege2 + privilege concurrent-request-limit log-errors debug-allow @@ -317,7 +317,7 @@ declare variable $odbc-server-settings := pre-commit-trigger-limit collation authentication - privilege2 + privilege concurrent-request-limit log-errors debug-allow @@ -3254,17 +3254,6 @@ declare function setup:get-appserver-privilege( else 0 }; -declare function admin:appserver-set-privilege2( - $config as element(configuration), - $appserver-id as xs:unsignedLong, - $value as xs:unsignedLong) -as element(configuration) -{ - if ($value = 0) then $config - else - admin:appserver-set-privilege($config, $appserver-id, $value) -}; - declare function setup:configure-server( $server-config as element(), $server-id as xs:unsignedLong, From ddc1b1fd30b61f1188c4f7972b45b13083f603ba Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Thu, 24 Apr 2014 09:02:13 -0400 Subject: [PATCH 019/113] Renaming $xcc-server-settings to $xdbc-server-settings --- deploy/lib/xquery/setup.xqy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 920a58a1..956e91ec 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -237,7 +237,7 @@ declare variable $webdav-server-settings := ; -declare variable $xcc-server-settings := +declare variable $xdbc-server-settings := enabled root @@ -3162,7 +3162,7 @@ declare function setup:configure-xdbc-server( $server-config as element(gr:xdbc-server)) as item()* { let $server-name as xs:string? := $server-config/gr:xdbc-server-name[fn:string-length(fn:string(.)) > 0] - let $admin-config := setup:configure-server($server-config, xdmp:server($server-name), $xcc-server-settings) + let $admin-config := setup:configure-server($server-config, xdmp:server($server-name), $xdbc-server-settings) return ( if (admin:save-configuration-without-restart($admin-config)) then From 124ed964a6aae7144bbaba74f28e6e537c77ca51 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 11:52:56 +0200 Subject: [PATCH 020/113] Removed roxy dir check from ml, improved ml help a bit --- ml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/ml b/ml index e625a97c..f3039d1b 100755 --- a/ml +++ b/ml @@ -35,10 +35,7 @@ PARAMS=("${@}") if [ "$1" == 'new' ] then shift - if [ "$1" == '-h' ] - then - usage - elif [ "$1" == '--help' ] + if [ "$1" == '-h' ] || [ "$1" == '--help' ] then usage elif [ $1 ] @@ -123,6 +120,7 @@ then fi fi else + printf "\nNOTE: Please provide an app name..\n\n" usage fi elif [ "$1" == 'self-test' ] @@ -138,22 +136,15 @@ then fi ruby -I deploy -I deploy/lib -I deploy/test deploy/test/test_main.rb || exit 1 else - printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" + printf "\nERROR: You must run this command inside a valid Roxy Project. Use 'ml new' to create a project.\n\n" + usage fi -elif [ "${PWD##*/}" == 'roxy' ] && [ "$#" -eq 0 ] -then - usage -elif [ "${PWD##*/}" == 'roxy' ] && [ "$1" == '-h' ] -then - usage -elif [ "${PWD##*/}" == 'roxy' ] && [ "$1" == '--help' ] -then - usage else if [ -e deploy/lib/ml.rb ] then ruby -I deploy -I deploy/lib deploy/lib/ml.rb $* || exit 1 else - printf "\nERROR: You must run this command inside a valid Roxy Project\n\n" + printf "\nERROR: You must run this command inside a valid Roxy Project. Use 'ml new' to create a project.\n\n" + usage fi fi \ No newline at end of file From 9edac78d76befd19929c1f531b293efe8110c802 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 11:55:09 +0200 Subject: [PATCH 021/113] Improved ml self-test param handling (#214) --- ml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ml b/ml index f3039d1b..30db9a61 100755 --- a/ml +++ b/ml @@ -127,13 +127,15 @@ elif [ "$1" == 'self-test' ] then if [ -e deploy/test/test_main.rb ] then - shift - version="$1" - if [ "${version:0:17}" == '--server-version=' ] - then - # This exports the new version only to sub-processes, e.g. the ruby call below.. - export ROXY_TEST_SERVER_VERSION="${version:17}" - fi + # Look for --server-version param, and export that as env variable. Unit testing doesn't allow cmd params.. + for (( i = 0; i < ${#PARAMS[@]}; i++ )); do + if [[ ${PARAMS[${i}]} == --server-version=* ]] + then + splits=(${PARAMS[1]//=/ }) + # This exports the version only to sub-processes, e.g. the ruby call below.. + export ROXY_TEST_SERVER_VERSION=${splits[1]} + fi + done ruby -I deploy -I deploy/lib -I deploy/test deploy/test/test_main.rb || exit 1 else printf "\nERROR: You must run this command inside a valid Roxy Project. Use 'ml new' to create a project.\n\n" From 6752925a12b5a402404fce9f75794f58c7cdcb5e Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 12:47:43 +0200 Subject: [PATCH 022/113] Fix for #83: ask confirmation for running ml new from a Roxy project --- ml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ml b/ml index 30db9a61..657e6c0f 100755 --- a/ml +++ b/ml @@ -40,6 +40,17 @@ then usage elif [ $1 ] then + # check if we are already in a valid Roxy project + if [ -e deploy/lib/ml.rb ] + then + read -r -n 1 -p "Running ml new from within a Roxy project is not recommended. Continue? [y/N] " response + printf "\n" + if ! [[ $response =~ ^(Y|y) ]] + then + exit 1 + fi + fi + app_name="$1" shift From ceaf45131addefa67f643427e7aaa2491ce719f6 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 14:21:22 +0200 Subject: [PATCH 023/113] Normalized indentation in ml.bat --- ml.bat | 90 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/ml.bat b/ml.bat index 62ffe84a..8128a24b 100644 --- a/ml.bat +++ b/ml.bat @@ -38,16 +38,16 @@ set FORCE_INSTALL=0 :loop if not "%1"=="" ( if "%1"=="--branch" ( - set BRANCH=%2 - shift + set BRANCH=%2 + shift ) if "%1"=="--app-type" ( set APPTYPE=%2 - shift + shift ) if "%1"=="--force" ( - set FORCE_INSTALL=1 - shift + set FORCE_INSTALL=1 + shift ) shift goto :loop @@ -68,12 +68,12 @@ echo. echo Creating new Application: %app_name%... if EXIST %app_name% ( - cmd /c git clone git://github.com/marklogic/roxy.git -b %BRANCH% %app_name%.tmp_1 - xcopy %app_name%.tmp_1\* %app_name%\ /E - rmdir /s /q %app_name%.tmp_1 + cmd /c git clone git://github.com/marklogic/roxy.git -b %BRANCH% %app_name%.tmp_1 + xcopy %app_name%.tmp_1\* %app_name%\ /E + rmdir /s /q %app_name%.tmp_1 ) if NOT EXIST %app_name% ( - cmd /c git clone git://github.com/marklogic/roxy.git -b %BRANCH% %app_name% + cmd /c git clone git://github.com/marklogic/roxy.git -b %BRANCH% %app_name% ) pushd %app_name% @@ -90,9 +90,9 @@ if "%APPTYPE%"=="rest" ( ) for /f "tokens=1-2*" %%a in ("%*") do ( - set arg-command=%%a - set arg-appname=%%b - set arg-options=%%c + set arg-command=%%a + set arg-appname=%%b + set arg-options=%%c ) cmd /c ml init %app_name% %arg-options% @@ -121,50 +121,50 @@ popd goto end :selftest - if NOT EXIST deploy\test\test_main.rb GOTO missingdeploy - ruby -Ideploy -Ideploy\lib -Ideploy\test deploy\test\test_main.rb - goto end + if NOT EXIST deploy\test\test_main.rb GOTO missingdeploy + ruby -Ideploy -Ideploy\lib -Ideploy\test deploy\test\test_main.rb + goto end :rubydeployer - if NOT EXIST deploy\lib\ml.rb GOTO missingdeploy - ruby -Ideploy -Ideploy\lib deploy\lib\ml.rb %* - goto end + if NOT EXIST deploy\lib\ml.rb GOTO missingdeploy + ruby -Ideploy -Ideploy\lib deploy\lib\ml.rb %* + goto end :missingdeploy - echo. - echo You must run this command inside a valid Roxy Project - echo. - goto end + echo. + echo You must run this command inside a valid Roxy Project + echo. + goto end :needruby - echo. - echo Ruby is required to run the ml scripts. - echo. - goto end + echo. + echo Ruby is required to run the ml scripts. + echo. + goto end :needgit - echo. - echo Git is required to use the new command. - echo. - goto end + echo. + echo Git is required to use the new command. + echo. + goto end :alreadyexists - echo. - echo %app_name% already exists. Aborting - echo. - goto end + echo. + echo %app_name% already exists. Aborting + echo. + goto end :usage - echo Usage: ml new app-name --server-version=[version] [--branch=branch] [--git] [--force] - echo. - echo. - echo use --server-version to specify the major version of MarkLogic you will - echo target in your project (4, 5, 6, 7) - echo use --branch to specify the GitHub branch of the Roxy project your project - echo will be based on (master, dev) - echo use --git to automatically configure a git repo - echo use --force to overwrite an existing directory - echo. - goto end + echo Usage: ml new app-name --server-version=[version] [--branch=branch] [--git] [--force] + echo. + echo. + echo use --server-version to specify the major version of MarkLogic you will + echo target in your project (4, 5, 6, 7) + echo use --branch to specify the GitHub branch of the Roxy project your project + echo will be based on (master, dev) + echo use --git to automatically configure a git repo + echo use --force to overwrite an existing directory + echo. + goto end :end From e9c917bda9d8c33730ef0a79cc5db1be94b50ce5 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 14:26:46 +0200 Subject: [PATCH 024/113] Synching ml changes with ml.bat --- ml.bat | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ml.bat b/ml.bat index 8128a24b..9d89116c 100644 --- a/ml.bat +++ b/ml.bat @@ -14,8 +14,9 @@ if "%1"=="self-test" goto selftest IF not "%1"=="new" goto rubydeployer SHIFT -IF "%1"=="" goto usage +IF "%1"=="" goto providename IF "%1"=="-h" goto usage +IF "%1"=="--help" goto usage set app_name=%1 SHIFT @@ -47,7 +48,6 @@ if not "%1"=="" ( ) if "%1"=="--force" ( set FORCE_INSTALL=1 - shift ) shift goto :loop @@ -132,9 +132,9 @@ goto end :missingdeploy echo. - echo You must run this command inside a valid Roxy Project + echo You must run this command inside a valid Roxy Project. Use 'ml new' to create a project. echo. - goto end + goto usage :needruby echo. @@ -150,10 +150,15 @@ goto end :alreadyexists echo. - echo %app_name% already exists. Aborting + echo %app_name% already exists. Aborting. echo. goto end +:providename + echo. + echo "NOTE: Please provide an app name.." + echo. + :usage echo Usage: ml new app-name --server-version=[version] [--branch=branch] [--git] [--force] echo. From 5696a589aa6fed3e166515eed2fd3ce7bd9fff09 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 14:29:21 +0200 Subject: [PATCH 025/113] Synched changes for #217 with ml.bat --- ml.bat | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ml.bat b/ml.bat index 9d89116c..3a946ee2 100644 --- a/ml.bat +++ b/ml.bat @@ -35,6 +35,7 @@ set BRANCH=master set INIT_GIT=0 set APPTYPE=mvc set FORCE_INSTALL=0 +set FORK=marklogic :loop if not "%1"=="" ( @@ -49,6 +50,10 @@ if not "%1"=="" ( if "%1"=="--force" ( set FORCE_INSTALL=1 ) + if "%1"=="--fork" ( + set FORK=%2 + shift + ) shift goto :loop ) @@ -68,12 +73,12 @@ echo. echo Creating new Application: %app_name%... if EXIST %app_name% ( - cmd /c git clone git://github.com/marklogic/roxy.git -b %BRANCH% %app_name%.tmp_1 + cmd /c git clone git://github.com/%FORK%/roxy.git -b %BRANCH% %app_name%.tmp_1 xcopy %app_name%.tmp_1\* %app_name%\ /E rmdir /s /q %app_name%.tmp_1 ) if NOT EXIST %app_name% ( - cmd /c git clone git://github.com/marklogic/roxy.git -b %BRANCH% %app_name% + cmd /c git clone git://github.com/%FORK%/roxy.git -b %BRANCH% %app_name% ) pushd %app_name% From c1e4994915d678959bd9fba7c7c38b405b9e5262 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 14:31:13 +0200 Subject: [PATCH 026/113] Synched changes for #214 with ml.bat --- ml.bat | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ml.bat b/ml.bat index 3a946ee2..b1785fe4 100644 --- a/ml.bat +++ b/ml.bat @@ -127,7 +127,26 @@ goto end :selftest if NOT EXIST deploy\test\test_main.rb GOTO missingdeploy + + REM Save original env variable value + set ROXY_TEST_SERVER_VERSION_ORG=%ROXY_TEST_SERVER_VERSION% + +:loop2 + if not "%1"=="" ( + REM Look for --server-version param, and export that as env variable. Unit testing doesn't allow cmd params.. + if "%1"=="--server-version" ( + set ROXY_TEST_SERVER_VERSION=%2 + shift + ) + shift + goto :loop2 + ) + ruby -Ideploy -Ideploy\lib -Ideploy\test deploy\test\test_main.rb + + REM Restore original env variable value + set ROXY_TEST_SERVER_VERSION=%ROXY_TEST_SERVER_VERSION_ORG% + set ROXY_TEST_SERVER_VERSION_ORG= goto end :rubydeployer From 3879a8c28492602a210887fddb037676411990a3 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 14:32:39 +0200 Subject: [PATCH 027/113] Synched changes for #83 with ml.bat --- ml.bat | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ml.bat b/ml.bat index b1785fe4..3a64c8c7 100644 --- a/ml.bat +++ b/ml.bat @@ -18,6 +18,13 @@ IF "%1"=="" goto providename IF "%1"=="-h" goto usage IF "%1"=="--help" goto usage +REM check if we are already in a valid Roxy project +if NOT EXIST deploy\lib\ml.rb GOTO skip_roxy_exists +set /p response= "Running ml new from within a Roxy project is not recommended. Continue? [y/N] " +if /i "%response:~,1%" NEQ "Y" exit /b + +:skip_roxy_exists + set app_name=%1 SHIFT From 447eb327aede2df8794ad607a20502b12ade6a36 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 14:35:23 +0200 Subject: [PATCH 028/113] Added a TODO in ml.bat for issue #216 --- ml.bat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ml.bat b/ml.bat index 3a64c8c7..95e1d008 100644 --- a/ml.bat +++ b/ml.bat @@ -79,6 +79,8 @@ if not "%APPTYPE%"=="mvc" if not "%APPTYPE%"=="rest" if not "%APPTYPE%"=="hybrid echo. echo Creating new Application: %app_name%... +REM TODO: check errorlevel and bail out if any of the below commands fail.. + if EXIST %app_name% ( cmd /c git clone git://github.com/%FORK%/roxy.git -b %BRANCH% %app_name%.tmp_1 xcopy %app_name%.tmp_1\* %app_name%\ /E From f2778f564499028e6b8d71a2fb0423cf7f9c7ef7 Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Fri, 25 Apr 2014 09:54:17 -0400 Subject: [PATCH 029/113] Issue #220 - fixing broken excluded and included field elements --- deploy/lib/xquery/setup.xqy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 956e91ec..e37993d9 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -1429,10 +1429,10 @@ declare function setup:add-field-includes-R( admin:database-included-element( $e/db:namespace-uri, $e/db:localname, - $e/db:weight, + ($e/db:weight, 1.0)[1], $e/db:attribute-namespace-uri, - $e/db:attribute-localname, - $e/db:attribute-value)), + ($e/db:attribute-localname, "")[1], + ($e/db:attribute-value, "")[1])), $database, fn:subsequence($field-configs, 2)) else @@ -1476,9 +1476,9 @@ declare function setup:add-field-excludes-R( admin:database-excluded-element( $e/db:namespace-uri, $e/db:localname, - $e/db:attribute-namespace-uri, - $e/db:attribute-localname, - $e/db:attribute-value)', + ($e/db:attribute-namespace-uri, "")[1], + ($e/db:attribute-localname, "")[1], + ($e/db:attribute-value, "")[1])', (xs:QName("e"), $e), same-statement From 7034b8bc2bd9403d2a22be924ba630f9535666ce Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 17:10:51 +0200 Subject: [PATCH 030/113] Reverted unwanted and non-relevant changes (#176) --- deploy/lib/ml.rb | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/deploy/lib/ml.rb b/deploy/lib/ml.rb index 73f10be6..1da7c6df 100755 --- a/deploy/lib/ml.rb +++ b/deploy/lib/ml.rb @@ -27,6 +27,11 @@ def need_help? ARGV << '--help' if ARGV.empty? +if ARGV.length == 1 && need_help? + Help.doHelp(@logger, :usage) + exit +end + @profile = find_arg(['-p', '--profile']) if @profile then begin @@ -97,7 +102,7 @@ def need_help? # # put things in ServerConfig class methods that don't depend on environment or server info # - elsif ServerConfig.respond_to?(command.to_sym) || ServerConfig.respond_to?(command) || ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) + elsif ServerConfig.respond_to?(command.to_sym) || ServerConfig.respond_to?(command) if need_help? Help.doHelp(@logger, command) else @@ -105,22 +110,10 @@ def need_help? ServerConfig.send command end break - elsif ARGV.length == 0 - Help.doHelp(@logger, :usage, "Unknown generic command #{command}!") # # ServerConfig methods require environment to be set in order to talk to a ML server # else - # [GJo] check second arg before checking properties, makes help available within Roxy folder too.. - command2 = ARGV[0] - if need_help? && (ServerConfig.instance_methods.include?(command2.to_sym) || ServerConfig.instance_methods.include?(command2)) - Help.doHelp(@logger, command2) - break - elsif ! (ServerConfig.instance_methods.include?(command2.to_sym) || ServerConfig.instance_methods.include?(command2)) - Help.doHelp(@logger, :usage, "Unknown environment command #{command2}!") - break - end - # unshift to get the environment in ServerConfig.properties ARGV.unshift command @properties = ServerConfig.properties @@ -136,8 +129,7 @@ def need_help? :logger => @logger ).send(command) else - # [GJo] no longer reached.. - Help.doHelp(@logger, :usage, :error_message => "Unknown command #{command}!") + Help.doHelp(@logger, :usage, "Unknown command #{command}!") break end end From 504ebb05c18037f976ebb9a94290ed1c5ff66b13 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 25 Apr 2014 17:12:46 +0200 Subject: [PATCH 031/113] Reverted unwanted and non-relevant changes (#176) --- deploy/lib/ml.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/lib/ml.rb b/deploy/lib/ml.rb index 1da7c6df..1ef67749 100755 --- a/deploy/lib/ml.rb +++ b/deploy/lib/ml.rb @@ -119,7 +119,10 @@ def need_help? @properties = ServerConfig.properties command = ARGV.shift - if ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) + if need_help? && Help.respond_to?(command) + Help.doHelp(@logger, command) + break + elsif ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) raise HelpException.new(command, "Missing environment for #{command}") if @properties["environment"].nil? raise ExitException.new("Missing ml-config.xml file. Check config.file property") if @properties["ml.config.file"].nil? From 8cf06dcfdf10fce55a47f1e888456c12a93bcd99 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Fri, 25 Apr 2014 11:53:48 -0400 Subject: [PATCH 032/113] #164 rlouapre's fix, applied to dev branch instead of master --- deploy/lib/MLClient.rb | 2 +- deploy/lib/util.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/deploy/lib/MLClient.rb b/deploy/lib/MLClient.rb index 0136afd0..daabb6f3 100644 --- a/deploy/lib/MLClient.rb +++ b/deploy/lib/MLClient.rb @@ -23,7 +23,7 @@ class MLClient def initialize(options) @ml_username = options[:user_name] - @ml_password = options[:password] + @ml_password = options[:password].xquery_unsafe @logger = options[:logger] || logger @request = {} end diff --git a/deploy/lib/util.rb b/deploy/lib/util.rb index d0c9d20a..2ea3f722 100644 --- a/deploy/lib/util.rb +++ b/deploy/lib/util.rb @@ -63,6 +63,11 @@ def strip_heredoc def xquery_safe REXML::Text::normalize(self).gsub(/\{/, '{{').gsub(/\}/, '}}') end + + def xquery_unsafe + REXML::Text::unnormalize(self).gsub(/\{{/, '{').gsub(/\}}/, '}') + end + end class Object From dc6a08585a176200ead8287a9dd03cd2faf78017 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 12:56:50 -0400 Subject: [PATCH 033/113] tweaking Geert's help changes --- deploy/lib/Help.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index e2776894..e66422ec 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -5,7 +5,7 @@ def self.usage Usage: ml [ENVIRONMENT] COMMAND [ARGS] - Generic commands (no environment): + General commands (no environment): init Creates configuration files for you to customize initcpf Creates cpf configuration files for you to customize jar Creates a Roxy jar @@ -27,7 +27,7 @@ def self.usage restart Restarts the given environment wipe Removes your application from the given environment - Deployment commands (with environment): + Deployment/Data commands (with environment): corb Runs Corb against the given environment deploy Loads modules, data, cpf configuration into the given environment load Loads a file or folder into the given environment @@ -35,7 +35,7 @@ def self.usage test Runs xquery unit tests against the given environment xqsync Runs XQSync against the given environment - All commands can be run with -h or --help for more information. Leave out environment.. + All commands can be run with -h or --help for more information. DOC end From 24c030b3bed5b62092fa1cbad4d66335a3d2459b Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 12:57:17 -0400 Subject: [PATCH 034/113] Added a comment about capture_enrivonment_config --- deploy/lib/server_config.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 6f186d24..1bfd48f4 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -723,7 +723,7 @@ def credentials def capture full_config = find_arg(['--full-ml-config']) target_db = find_arg(['--modules-db']) - + # check params if full_config == nil && target_db == nil raise HelpException.new("capture", "either full-ml-config or modules-db is required") @@ -820,6 +820,13 @@ def save_files_to_fs(target_db, target_dir) end + # Note: this is the beginning of a feature; not really useful yet. What we want is to specify one or more app servers, + # get all configuration related to them, and write that into the ml-config.xml format. This format is very similar to + # MarkLogic's databases.xml and other config files, but there are some differences. + # The related configuration is to include any databases connected to the app server(s) -- modules, content, triggers, + # schemas; CPF configuration; along with users and roles. For users and roles, we probably need an interactive system -- + # we don't want or need to capture built-in users and roles. If the application uses app-level security, then we + # could start with "Capture user #{default user}?" and then check on each role that user has. def capture_environment_config raise ExitException.new("Capture requires the target environment's hostname to be defined") unless @hostname.present? From cb32f7566762bf448fde1f2fb396fcc967444069 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 14:25:44 -0400 Subject: [PATCH 035/113] Added webdav to self-test for ML7 --- deploy/lib/xquery/setup.xqy | 19 ++- deploy/test/data/ml7-config.xml | 130 +++++++++++++++++- .../test/data/ml7-properties/build.properties | 3 +- 3 files changed, 139 insertions(+), 13 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 56c58d7e..a506f8dc 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -121,7 +121,6 @@ declare variable $http-server-settings := enabled root port - modules-database database last-login @@ -139,12 +138,9 @@ declare variable $http-server-settings := pre-commit-trigger-limit collation authentication - - default-user privilege concurrent-request-limit - log-errors debug-allow profile-allow @@ -184,7 +180,7 @@ declare variable $http-server-settings := declare variable $webdav-server-settings := { - $http-server-settings/*[fn:not(fn:data(.) = 'modules-database')], + $http-server-settings/*[fn:not(fn:data(.) = ('modules-database', 'error-handler', 'url-rewriter', 'rewrite-resolves-globally'))], compute-content-length } ; @@ -3002,7 +2998,10 @@ declare function setup:validate-http-server( setup:validate-server( $server-config, xdmp:server($server-config/gr:http-server-name[fn:string-length(fn:string(.)) > 0]), - $http-server-settings) + if ($server-config/gr:webDAV = fn:true()) then + $webdav-server-settings + else + $http-server-settings) }; declare function setup:configure-xdbc-server( @@ -3199,11 +3198,11 @@ declare function setup:configure-server( import module namespace admin = "http://marklogic.com/xdmp/admin" at "/MarkLogic/admin.xqy"; declare namespace gr="http://marklogic.com/xdmp/group"; - + declare variable $admin-config external; declare variable $server-id external; declare variable $module-locations external; - + let $old-module-locations := admin:appserver-get-module-locations($admin-config, $server-id) let $config := (: First delete any module-location that matches the namespace :) @@ -3330,7 +3329,7 @@ declare function setup:validate-server( declare variable $admin-config external; declare variable $server-id external; - + admin:appserver-get-module-locations($admin-config, $server-id) ', (xs:QName("admin-config"),$admin-config, @@ -3351,7 +3350,7 @@ declare function setup:validate-server( return if ($existing[fn:deep-equal(., $expected)]) then () else - setup:validation-fail(fn:concat("Appserver missing request blackout: ", $expected/gr:request-blackout)) + setup:validation-fail(fn:concat("Appserver missing request blackout: ", $expected/gr:request-blackout)) ) }; diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index ebf2af2d..78931ce2 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -117,7 +117,7 @@ 1.0 nonblocking strict - + normal ASCII xml @@ -222,6 +222,132 @@ @ml.test-appserver + + @ml.app-name-dav + false + /@ml.app-name/ + @ml.dav-port + true + true + + Last-Login + true +
1.2.3.4
+ 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + http://marklogic.com/collation/codepoint + basic + @ml.default-user + admin-ui + 123 + false + false + false + 1.0 + nonblocking + strict + + normal + ASCII + xml + yes + a + b + c + d + yes + yes + yes + yes + text/xml + NFC + yes + yes + yes + 1.0 + yes + + + + http://www.ns.com/ns0 + /test.xsd + + + + + ns1 + http://www.ns.com/ns1 + + + ns2 + http://www.ns.com/ns2 + + + http://www.ns.com/ns3 + + + + + + + recurring + + monday + + + 12:34:00+02:00 + 23:45:00+02:00 + + + + + + recurring + + monday + + + 12:34:00+02:00 + PT1H23M + + + + + + recurring + + monday + + + + + + + once + + + 1234-11-22 + 12:34:00+02:00 + 1234-12-23 + 23:34:00+02:00 + + + + false + false + sixer + ALL:!LOW:@STRENGTH:ROXY + false + + @ml.xdbc-server @@ -887,4 +1013,4 @@ text - \ No newline at end of file + diff --git a/deploy/test/data/ml7-properties/build.properties b/deploy/test/data/ml7-properties/build.properties index 86a9eddc..865e6310 100644 --- a/deploy/test/data/ml7-properties/build.properties +++ b/deploy/test/data/ml7-properties/build.properties @@ -62,7 +62,8 @@ schemas-db=${app-name}-schemas # app-port=8900 xcc-port=8901 -# odbc-port=8043 +dav-port=8902 +#odbc-port=8903 content-forests-per-host=2 From ebd4c012fce583387a7fcf325c579cc780a26e22 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 14:42:02 -0400 Subject: [PATCH 036/113] Added odbc server to the ML7 self-test; fixed a bug in the validate-odbcserver() function --- deploy/lib/xquery/setup.xqy | 2 +- deploy/test/data/ml7-config.xml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index a506f8dc..f19e6632 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -2892,7 +2892,7 @@ declare function setup:create-odbcserver( }; declare function setup:validate-odbcserver( - $server-config as element(gr:http-server)) as item()* + $server-config as element(gr:odbc-server)) as item()* { let $server-name as xs:string? := $server-config/gr:odbc-server-name[fn:string-length(fn:string(.)) > 0] return diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index 78931ce2..88f46c33 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -352,6 +352,9 @@ @ml.xdbc-server + + @ml.odbc-server + From 0f93a48eed01f97a097b4dfc7f1a967dfffe4bdd Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 15:02:00 -0400 Subject: [PATCH 037/113] Added odbc-port to self-test for ML7 --- deploy/test/data/ml7-properties/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/test/data/ml7-properties/build.properties b/deploy/test/data/ml7-properties/build.properties index 865e6310..bb75ceea 100644 --- a/deploy/test/data/ml7-properties/build.properties +++ b/deploy/test/data/ml7-properties/build.properties @@ -63,7 +63,7 @@ schemas-db=${app-name}-schemas app-port=8900 xcc-port=8901 dav-port=8902 -#odbc-port=8903 +odbc-port=8903 content-forests-per-host=2 From d1a58bbc62a16562825e13cfdaf47d3269b9715a Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 15:02:13 -0400 Subject: [PATCH 038/113] rearranged the $*-server-settings --- deploy/lib/xquery/setup.xqy | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index f19e6632..a61215a3 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -116,12 +116,11 @@ declare variable $database-settings := journal-count ; -declare variable $http-server-settings := +declare variable $common-server-settings := enabled root port - modules-database database last-login display-last-login @@ -133,12 +132,10 @@ declare variable $http-server-settings := session-timeout max-time-limit default-time-limit - static-expires pre-commit-trigger-depth pre-commit-trigger-limit collation authentication - default-user privilege concurrent-request-limit log-errors @@ -166,9 +163,6 @@ declare variable $http-server-settings := output-undeclare-prefixes output-version output-include-default-attributes - error-handler - url-rewriter - rewrite-resolves-globally ssl-certificate-template ssl-allow-sslv3 ssl-allow-tls @@ -178,17 +172,32 @@ declare variable $http-server-settings := ; +declare variable $http-server-settings := + + { $common-server-settings/* } + modules-database + error-handler + url-rewriter + rewrite-resolves-globally + static-expires + default-user + +; + declare variable $webdav-server-settings := - { - $http-server-settings/*[fn:not(fn:data(.) = ('modules-database', 'error-handler', 'url-rewriter', 'rewrite-resolves-globally'))], + + { $common-server-settings/* } compute-content-length - } + static-expires + default-user + ; declare variable $xdbc-server-settings := - { - $http-server-settings/*[fn:not(fn:data(.) = ('static-expires', 'default-user', 'error-handler', 'url-rewriter', 'rewrite-resolves-globally'))] - } + + { $common-server-settings/* } + modules-database + ; declare variable $odbc-server-settings := From e8c1230c7596c33e68fda98e7d0c7ea7e68c2762 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 28 Apr 2014 16:04:40 -0400 Subject: [PATCH 039/113] removing some commented-out elements --- deploy/test/data/ml7-config.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index 88f46c33..0c7bcc34 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -81,13 +81,10 @@ - @ml.app-name false /@ml.app-name/ @ml.app-port - - Last-Login @@ -105,12 +102,9 @@ 123 http://marklogic.com/collation/codepoint basic - - @ml.default-user admin-ui 123 - false false false @@ -213,7 +207,6 @@ /roxy/rewrite.xqy true - false false sixer From 84eec3a556c3c47d8a4c527b86d1afac35e8ce5a Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 29 Apr 2014 16:56:18 +0200 Subject: [PATCH 040/113] Added mlcp functionality alike corb and xqsync, first step towards #78 --- deploy/default.properties | 5 ++ deploy/lib/Help.rb | 21 +++++++++ deploy/lib/server_config.rb | 66 +++++++++++++++++++++++++++ deploy/sample/build.sample.properties | 5 ++ 4 files changed, 97 insertions(+) diff --git a/deploy/default.properties b/deploy/default.properties index 5be3fed7..910a7471 100644 --- a/deploy/default.properties +++ b/deploy/default.properties @@ -179,3 +179,8 @@ local-server=localhost #dev-server= #cert-server= #prod-server= + +# +# If you want to use MLCP, make sure mlcp-home is configured correctly. Using a common location as default. +# +mlcp-home=/usr/local/mlcp diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index e66422ec..8be64b18 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -31,6 +31,7 @@ def self.usage corb Runs Corb against the given environment deploy Loads modules, data, cpf configuration into the given environment load Loads a file or folder into the given environment + mlcp Runs MLCP against the given environment recordloader Runs RecordLoader against the given environment test Runs xquery unit tests against the given environment xqsync Runs XQSync against the given environment @@ -347,6 +348,26 @@ def self.corb DOC end + def self.mlcp + <<-DOC.strip_heredoc + Usage: ml {env} mlcp [options] + + Runs MLCP with given command-line options agains selected environment. + MLCP supports options files natively using the -option_file parameter. + The path must a relative or absolute path to a MLCP options file. + See http://docs.marklogic.com/guide/ingestion/content-pump#chapter + + General options: + -v, [--verbose] # Verbose output + -h, [--help] # Shows this help + + Roxy applies variable substitution within option files. You may use variables like: + + -input_file_path + ${ml.data.dir}/ + DOC + end + def self.plugin <<-DOC.strip_heredoc Usage: ml {env} plugin [command] [package] [version] [options] diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 1bfd48f4..ea2a9ec4 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -683,6 +683,72 @@ def corb end end + def mlcp + mlcp_home = @properties['ml.mlcp-home'] + if @properties['ml.mlcp-home'] == nil || ! File.directory?(File.expand_path(mlcp_home)) || ! File.exists?(File.expand_path("#{mlcp_home}/bin/mlcp.sh")) + raise "MLCP not found or mis-configured, please check the mlcp-home setting." + end + + # Find all jars required for running MLCP. At least: + jars = Dir.glob(ServerConfig.expand_path("#{mlcp_home}/lib/*.jar")) + classpath = jars.join(path_separator) + + ARGV.each do |arg| + if arg == "-option_file" + # remove flag from ARGV + index = ARGV.index(arg) + ARGV.slice!(index) + + # capture and remove value from ARGV + option_file = ARGV[index] + ARGV.slice!(index) + + # find and read file if exists + option_file = ServerConfig.expand_path("#{@@path}/#{option_file}") + if File.exist? option_file + logger.debug "Reading options file #{option_file}.." + options = File.read option_file + + # substitute properties + @properties.sort {|x,y| y <=> x}.each do |k, v| + options.gsub!("@#{k}", v) + end + + logger.debug "Options after resolving properties:" + lines = options.split(/[\n\r]+/).reject! { |line| line.empty? || line.match("^#") } + + lines.each do |line| + logger.debug line + end + + # and insert the properties back into ARGV + ARGV[index,0] = lines + else + raise "Option file #{option_file} not found." + end + end + end + + if ARGV.length > 0 + connection_string = %Q{ -username #{@properties['ml.user']} -password #{@properties['ml.password']} -host #{@properties['ml.server']} -port #{@properties['ml.xcc-port']}} + + args = ARGV.join(" ") + + runme = %Q{java -cp #{classpath} com.marklogic.contentpump.ContentPump #{args} #{connection_string}} + else + runme = %Q{java -cp #{classpath} com.marklogic.contentpump.ContentPump} + end + + logger.debug runme + logger.info "" + + system runme + + logger.info "" + + ARGV.clear + end + def credentials logger.info "credentials #{@environment}" # ml will error on invalid environment diff --git a/deploy/sample/build.sample.properties b/deploy/sample/build.sample.properties index d10771f1..375f68d9 100644 --- a/deploy/sample/build.sample.properties +++ b/deploy/sample/build.sample.properties @@ -142,3 +142,8 @@ local-server=localhost #dev-server= #cert-server= #prod-server= + +# +# If you want to use MLCP, make sure mlcp-home is configured correctly. Using a common location as default. +# +mlcp-home=/usr/local/mlcp From 3a333b5aff793d5fa91d15cd566d7258664b283d Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Wed, 30 Apr 2014 09:17:20 +0200 Subject: [PATCH 041/113] Added password prompt for mlcp --- deploy/lib/server_config.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index ea2a9ec4..8776f541 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -730,7 +730,8 @@ def mlcp end if ARGV.length > 0 - connection_string = %Q{ -username #{@properties['ml.user']} -password #{@properties['ml.password']} -host #{@properties['ml.server']} -port #{@properties['ml.xcc-port']}} + password_prompt + connection_string = %Q{ -username #{@properties['ml.user']} -password #{@ml_password} -host #{@properties['ml.server']} -port #{@properties['ml.xcc-port']}} args = ARGV.join(" ") From 5a459257f40ffb38af4ea4ebb89aa0568eb028f1 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Wed, 30 Apr 2014 09:25:21 +0200 Subject: [PATCH 042/113] Added memory settings for mlcp --- deploy/default.properties | 1 + deploy/lib/server_config.rb | 2 +- deploy/sample/build.sample.properties | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/default.properties b/deploy/default.properties index 910a7471..cf174c35 100644 --- a/deploy/default.properties +++ b/deploy/default.properties @@ -184,3 +184,4 @@ local-server=localhost # If you want to use MLCP, make sure mlcp-home is configured correctly. Using a common location as default. # mlcp-home=/usr/local/mlcp +mlcp-vmargs=-Xmx512m diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 8776f541..a649bad8 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -735,7 +735,7 @@ def mlcp args = ARGV.join(" ") - runme = %Q{java -cp #{classpath} com.marklogic.contentpump.ContentPump #{args} #{connection_string}} + runme = %Q{java -cp #{classpath} #{@properties['ml.mlcp-vmargs']} com.marklogic.contentpump.ContentPump #{args} #{connection_string}} else runme = %Q{java -cp #{classpath} com.marklogic.contentpump.ContentPump} end diff --git a/deploy/sample/build.sample.properties b/deploy/sample/build.sample.properties index 375f68d9..fee26d6a 100644 --- a/deploy/sample/build.sample.properties +++ b/deploy/sample/build.sample.properties @@ -147,3 +147,4 @@ local-server=localhost # If you want to use MLCP, make sure mlcp-home is configured correctly. Using a common location as default. # mlcp-home=/usr/local/mlcp +mlcp-vmargs=-Xmx512m From e5a1a3d7808ece0126addc12bf4e1d2eccf119f5 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Wed, 30 Apr 2014 13:36:42 +0200 Subject: [PATCH 043/113] Added test cases for #226, space-separated localnames in fields --- deploy/test/data/ml7-config.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index 0c7bcc34..57e79e41 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -469,7 +469,7 @@ http://www.marklogic.com/ns/sample2 - word-query-include2 + word-query-include2 word-query-include3 1.0 @@ -486,7 +486,7 @@ http://www.marklogic.com/ns/sample2 - word-query-exclude2 + word-query-exclude2 word-query-exclude3 @@ -512,7 +512,7 @@ http://www.marklogic.com/ns/sample2 - sample-included-element2 + sample-included-element2 sample-included-element3 1 @@ -529,7 +529,7 @@ http://www.marklogic.com/ns/sample2 - sample-excluded-element2 + sample-excluded-element2 sample-excluded-element3 @@ -544,7 +544,7 @@ http://www.marklogic.com/ns/sample - sample-included-element + sample2-included-element 1 @@ -552,7 +552,7 @@ http://www.marklogic.com/ns/sample2 - sample-included-element2 + sample2-included-element2 sample2-included-element3 1 @@ -562,14 +562,14 @@ http://www.marklogic.com/ns/sample - sample-excluded-element + sample2-excluded-element http://www.marklogic.com/ns/sample2 - sample-excluded-element2 + sample2-excluded-element2 sample2-excluded-element3 From d87f194d222ee22cc30915140cb3ca60fce7758c Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Wed, 30 Apr 2014 13:37:13 +0200 Subject: [PATCH 044/113] Fix for #226, space-separated localnames in fields --- deploy/lib/xquery/setup.xqy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index a61215a3..49c0f419 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -1279,7 +1279,7 @@ declare function setup:add-field-includes-R( return admin:database-included-element( $e/db:namespace-uri, - $e/db:localname, + $e/db:localname/fn:string(.), ($e/db:weight, 1.0)[1], $e/db:attribute-namespace-uri, ($e/db:attribute-localname, "")[1], @@ -1326,7 +1326,7 @@ declare function setup:add-field-excludes-R( admin:database-excluded-element( $e/db:namespace-uri, - $e/db:localname, + $e/db:localname/fn:string(.), ($e/db:attribute-namespace-uri, "")[1], ($e/db:attribute-localname, "")[1], ($e/db:attribute-value, "")[1])', From fc195a5a9781ece03f0cf740f53dc134f353e43c Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Wed, 30 Apr 2014 15:33:18 +0200 Subject: [PATCH 045/113] Fixed bug in showing usage when running ml without args, logger wasnt initialized yet --- deploy/lib/ml.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/lib/ml.rb b/deploy/lib/ml.rb index 1ef67749..90c5c869 100755 --- a/deploy/lib/ml.rb +++ b/deploy/lib/ml.rb @@ -27,11 +27,6 @@ def need_help? ARGV << '--help' if ARGV.empty? -if ARGV.length == 1 && need_help? - Help.doHelp(@logger, :usage) - exit -end - @profile = find_arg(['-p', '--profile']) if @profile then begin @@ -50,6 +45,11 @@ def need_help? "#{sev}#{msg}\n" } +if ARGV.length == 1 && need_help? + Help.doHelp(@logger, :usage) + exit +end + if RUBY_VERSION < "1.8.7" @logger.warn <<-MSG From 006533bf917481f8056d400bd7b88018e8e39ec0 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 1 May 2014 09:07:10 +0200 Subject: [PATCH 046/113] Bugfix for var substitution in mlcp option files --- deploy/lib/server_config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a649bad8..a8c92bf0 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -715,7 +715,7 @@ def mlcp end logger.debug "Options after resolving properties:" - lines = options.split(/[\n\r]+/).reject! { |line| line.empty? || line.match("^#") } + lines = options.split(/[\n\r]+/).reject { |line| line.empty? || line.match("^#") } lines.each do |line| logger.debug line From 94b6b0d02cb6e7e767bbeefe00d9a1c1e880836d Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 1 May 2014 09:13:53 +0200 Subject: [PATCH 047/113] Improved logging of deploy rest --- deploy/lib/server_config.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a8c92bf0..84f77a47 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -1041,17 +1041,23 @@ def deploy_rest :remove_prefix => @properties['ml.rest-options.dir'], :db => rest_modules_db else - logger.debug "Could not find REST API options directory: #{@properties['ml.rest-options.dir']}\n"; + logger.info "\nNo REST API options found in: #{@properties['ml.rest-options.dir']}"; end + if (@properties.has_key?('ml.rest-ext.dir') && File.exist?(@properties['ml.rest-ext.dir'])) - logger.info "\nLoading REST extensions from #{@properties['ml.rest-ext.dir']}\n" + logger.info "\nLoading REST extensions from #{@properties['ml.rest-ext.dir']}" mlRest.install_extensions(ServerConfig.expand_path(@properties['ml.rest-ext.dir'])) + else + logger.info "\nNo REST extensions found in: #{@properties['ml.rest-ext.dir']}"; end if (@properties.has_key?('ml.rest-transforms.dir') && File.exist?(@properties['ml.rest-transforms.dir'])) - logger.info "\nLoading REST transforms from #{@properties['ml.rest-transforms.dir']}\n" + logger.info "\nLoading REST transforms from #{@properties['ml.rest-transforms.dir']}" mlRest.install_transforms(ServerConfig.expand_path(@properties['ml.rest-transforms.dir'])) + else + logger.info "\nNo REST transforms found in: #{@properties['ml.rest-transforms.dir']}"; end + logger.info("") end end From 6a2c46c715784568d35c783c42f6bb4c9cbc2e38 Mon Sep 17 00:00:00 2001 From: Peter Kester Date: Thu, 1 May 2014 14:52:07 +0200 Subject: [PATCH 048/113] =?UTF-8?q?added=20some=20extra=20options=20to=20d?= =?UTF-8?q?eploy.=20option=20=E2=80=94no-extns=20to=20command=20=E2=80=98d?= =?UTF-8?q?eploy=20modules=E2=80=99=20=20to=20just=20deploy=20modules,=20n?= =?UTF-8?q?o=20rest,=20saves=20time=20when=20there=20are=20a=20lot=20of=20?= =?UTF-8?q?rest=20extensions=20=E2=80=98deploy=20rest=E2=80=99=20will=20on?= =?UTF-8?q?ly=20deploy=20the=20rest=20extensions=20=E2=80=98deploy=20trans?= =?UTF-8?q?form=E2=80=99=20will=20just=20deploy=20the=20transforms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/lib/Help.rb | 14 ++++++++----- deploy/lib/server_config.rb | 39 +++++++++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index 8be64b18..afc80dac 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -235,14 +235,18 @@ def self.deploy --batch=(yes|no) # enable or disable batch commit. By default batch is disabled for the local environment and enabled for all others. + --no-extns # only used when WHAT is modules, will not deploy rest and transform Please choose a WHAT below. - modules # deploys code to your modules db in the given environment - content # deploys content to your content db in the given environment - schemas # deploys schemas to your schemas db in the given environment - cpf # deploys your cpf config to the server in the given environment - rest # deploys your rest extensions to the server in the given environment + modules # deploys code to your modules db in the given environment + content # deploys content to your content db in the given environment + schemas # deploys schemas to your schemas db in the given environment + cpf # deploys your cpf config to the server in the given environment + rest # deploys your rest extensions to the server in the given environment + if a name is specified, then only that extension will be deployed + transform # deploys your rest extensions to the server in the given environment + if a name is specified, then only that transform will be deployed DOC end diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a649bad8..a9fa528e 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -477,6 +477,8 @@ def deploy deploy_modules when 'rest' deploy_rest + when 'transform' + deploy_transform when 'schemas' deploy_schemas when 'cpf' @@ -950,6 +952,7 @@ def modules_databases end def deploy_modules + no_extns = find_arg(['--no-extns']).present? test_dir = @properties['ml.xquery-test.dir'] xquery_dir = @properties['ml.xquery.dir'] # modules_db = @properties['ml.modules-db'] @@ -1017,11 +1020,20 @@ def deploy_modules logger.info "\nLoaded #{total_count} #{pluralize(total_count, "document", "documents")} from #{xquery_dir} to #{xcc.hostname}:#{xcc.port}/#{dest_db} at #{DateTime.now.strftime('%m/%d/%Y %I:%M:%S %P')}\n" end - deploy_rest() + if !no_extns + deploy_rest() + deploy_transforms() + end end def deploy_rest - # Deploy options, extensions, and transforms to the REST API server + extensionname = ARGV.shift + path = @properties['ml.rest-ext.dir'] + if extensionname + path += "/#{extensionname}.xqy" + end + + # Deploy options, extensions to the REST API server if ['rest', 'hybrid'].include? @properties["ml.app-type"] # Figure out where we need to deploy this stuff rest_modules_db = '' @@ -1044,13 +1056,28 @@ def deploy_rest logger.debug "Could not find REST API options directory: #{@properties['ml.rest-options.dir']}\n"; end if (@properties.has_key?('ml.rest-ext.dir') && File.exist?(@properties['ml.rest-ext.dir'])) - logger.info "\nLoading REST extensions from #{@properties['ml.rest-ext.dir']}\n" - mlRest.install_extensions(ServerConfig.expand_path(@properties['ml.rest-ext.dir'])) + logger.info "\nLoading REST extensions from #{path}\n" + mlRest.install_extensions(ServerConfig.expand_path(path)) end + end + end + + def deploy_transform + transformname = ARGV.shift + path = @properties['ml.rest-transforms.dir'] + if transformname + if File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xqy") + path += "/#{transformname}.xqy" + elsif File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xsl") + path += "/#{transformname}.xsl" + end + end + # Deploy transforms to the REST API server + if ['rest', 'hybrid'].include? @properties["ml.app-type"] if (@properties.has_key?('ml.rest-transforms.dir') && File.exist?(@properties['ml.rest-transforms.dir'])) - logger.info "\nLoading REST transforms from #{@properties['ml.rest-transforms.dir']}\n" - mlRest.install_transforms(ServerConfig.expand_path(@properties['ml.rest-transforms.dir'])) + logger.info "\nLoading REST transforms from #{path}\n" + mlRest.install_transforms(ServerConfig.expand_path(path)) end end end From a6d8956cab54ac743bd7e14907ccec20ac58d914 Mon Sep 17 00:00:00 2001 From: Peter Kester Date: Thu, 1 May 2014 15:15:25 +0200 Subject: [PATCH 049/113] Made transform code consistent in use of file extension --- deploy/lib/Help.rb | 4 ++-- deploy/lib/server_config.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index afc80dac..d2bfbd17 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -428,12 +428,12 @@ def self.transform Example: $ ml transform ex:sample - will create a sample.xsl file in your rest-transform directory, + will create a sample.xslt file in your rest-transform directory, using the "ex" namespace prefix. Example: $ ml transform sample - will create a sample.xsl file in your rest-transform directory. + will create a sample.xslt file in your rest-transform directory. Example: $ ml transform sample xqy diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a9fa528e..8e8bcc4b 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -1068,8 +1068,8 @@ def deploy_transform if transformname if File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xqy") path += "/#{transformname}.xqy" - elsif File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xsl") - path += "/#{transformname}.xsl" + elsif File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xslt") + path += "/#{transformname}.xslt" end end From 96c6006fe57f15aa0c2163e2f193d72cbc1c34c1 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Thu, 1 May 2014 11:10:57 -0400 Subject: [PATCH 050/113] Fix #228 - Any properties defined can be overwritten from the command line. For example ml local info --ml.password=mysecretpassword - If the property is not defined a warning is displayed and its value will be ignored. --- deploy/lib/ml.rb | 4 +++- deploy/lib/server_config.rb | 33 ++++++++++++++++++--------------- deploy/lib/util.rb | 18 +++++++++++++++++- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/deploy/lib/ml.rb b/deploy/lib/ml.rb index 1ef67749..3eb78e32 100755 --- a/deploy/lib/ml.rb +++ b/deploy/lib/ml.rb @@ -122,6 +122,8 @@ def need_help? if need_help? && Help.respond_to?(command) Help.doHelp(@logger, command) break + elsif command.start_with?("--ml.") + break elsif ServerConfig.instance_methods.include?(command.to_sym) || ServerConfig.instance_methods.include?(command) raise HelpException.new(command, "Missing environment for #{command}") if @properties["environment"].nil? raise ExitException.new("Missing ml-config.xml file. Check config.file property") if @properties["ml.config.file"].nil? @@ -168,4 +170,4 @@ def need_help? # Print a flat profile to text printer = RubyProf::FlatPrinter.new(result) printer.print(STDOUT) -end \ No newline at end of file +end diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index a649bad8..ae793863 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ############################################################################### +require 'util' require 'uri' require 'net/http' require 'fileutils' @@ -688,39 +689,39 @@ def mlcp if @properties['ml.mlcp-home'] == nil || ! File.directory?(File.expand_path(mlcp_home)) || ! File.exists?(File.expand_path("#{mlcp_home}/bin/mlcp.sh")) raise "MLCP not found or mis-configured, please check the mlcp-home setting." end - + # Find all jars required for running MLCP. At least: jars = Dir.glob(ServerConfig.expand_path("#{mlcp_home}/lib/*.jar")) classpath = jars.join(path_separator) - + ARGV.each do |arg| if arg == "-option_file" # remove flag from ARGV index = ARGV.index(arg) ARGV.slice!(index) - + # capture and remove value from ARGV option_file = ARGV[index] ARGV.slice!(index) - + # find and read file if exists option_file = ServerConfig.expand_path("#{@@path}/#{option_file}") if File.exist? option_file logger.debug "Reading options file #{option_file}.." options = File.read option_file - + # substitute properties @properties.sort {|x,y| y <=> x}.each do |k, v| options.gsub!("@#{k}", v) end - + logger.debug "Options after resolving properties:" lines = options.split(/[\n\r]+/).reject! { |line| line.empty? || line.match("^#") } - + lines.each do |line| logger.debug line end - + # and insert the properties back into ARGV ARGV[index,0] = lines else @@ -728,25 +729,25 @@ def mlcp end end end - + if ARGV.length > 0 password_prompt connection_string = %Q{ -username #{@properties['ml.user']} -password #{@ml_password} -host #{@properties['ml.server']} -port #{@properties['ml.xcc-port']}} - + args = ARGV.join(" ") - + runme = %Q{java -cp #{classpath} #{@properties['ml.mlcp-vmargs']} com.marklogic.contentpump.ContentPump #{args} #{connection_string}} else runme = %Q{java -cp #{classpath} com.marklogic.contentpump.ContentPump} end - + logger.debug runme logger.info "" - + system runme - + logger.info "" - + ARGV.clear end @@ -1602,6 +1603,8 @@ def ServerConfig.properties(prop_file_location = @@path) properties.merge!(ServerConfig.load_properties(env_properties_file, "ml.")) if File.exists? env_properties_file properties = ServerConfig.substitute_properties(properties, properties, "ml.") + + properties = load_prop_from_args(properties) end end diff --git a/deploy/lib/util.rb b/deploy/lib/util.rb index 2ea3f722..97abe42f 100644 --- a/deploy/lib/util.rb +++ b/deploy/lib/util.rb @@ -36,6 +36,22 @@ def find_arg(args = []) nil end +def load_prop_from_args(props) + ARGV.each do |a| + if a.match(/(^--)(ml\..*)(=)(.*)/) + matches = a.match(/(^--)(ml\..*)(=)(.*)/).to_a + ml_key = matches[2] + ml_val = matches[4] + if props.has_key?("#{ml_key}") + props["#{ml_key}"] = ml_val + else + logger.warn "Property #{ml_key} does not exist. It will be skipped." + end + end + end + props +end + def pluralize(count, singular, plural = nil) count == 1 || count =~ /^1(\.0+)?$/ ? singular : plural end @@ -82,4 +98,4 @@ def present? def to_b present? && ['true', 'TRUE', 'yes', 'YES', 'y', 'Y', 't', 'T'].include?(self) end -end \ No newline at end of file +end From 5215755f594f3c1a874557346e1121e9d9cc273c Mon Sep 17 00:00:00 2001 From: Peter Kester Date: Thu, 1 May 2014 17:30:00 +0200 Subject: [PATCH 051/113] Fix for issue #231 added extra command to be able to deploy mvc rest and transform separately --- deploy/lib/Help.rb | 4 ++-- deploy/lib/server_config.rb | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index d2bfbd17..beb56da3 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -235,14 +235,14 @@ def self.deploy --batch=(yes|no) # enable or disable batch commit. By default batch is disabled for the local environment and enabled for all others. - --no-extns # only used when WHAT is modules, will not deploy rest and transform Please choose a WHAT below. - modules # deploys code to your modules db in the given environment + modules # deploys all code to your modules db in the given environment content # deploys content to your content db in the given environment schemas # deploys schemas to your schemas db in the given environment cpf # deploys your cpf config to the server in the given environment + mvc # deploys the mvc code your modules db in the given environment rest # deploys your rest extensions to the server in the given environment if a name is specified, then only that extension will be deployed transform # deploys your rest extensions to the server in the given environment diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 8e8bcc4b..057041ee 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -475,6 +475,8 @@ def deploy deploy_content when 'modules' deploy_modules + when 'mvc' + deploy_mvc when 'rest' deploy_rest when 'transform' @@ -952,7 +954,12 @@ def modules_databases end def deploy_modules - no_extns = find_arg(['--no-extns']).present? + deploy_mvc() + deploy_rest() + deploy_transforms() + end + + def deploy_mvc test_dir = @properties['ml.xquery-test.dir'] xquery_dir = @properties['ml.xquery.dir'] # modules_db = @properties['ml.modules-db'] @@ -1019,11 +1026,6 @@ def deploy_modules logger.info "\nLoaded #{total_count} #{pluralize(total_count, "document", "documents")} from #{xquery_dir} to #{xcc.hostname}:#{xcc.port}/#{dest_db} at #{DateTime.now.strftime('%m/%d/%Y %I:%M:%S %P')}\n" end - - if !no_extns - deploy_rest() - deploy_transforms() - end end def deploy_rest From 477933b3dcf93d3ba7a13c7ce3fd06da3bda8446 Mon Sep 17 00:00:00 2001 From: Peter Kester Date: Thu, 1 May 2014 17:31:47 +0200 Subject: [PATCH 052/113] Fix for issue #230 escape {} in xslt sample --- deploy/sample/rest-transform.sample.xslt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/sample/rest-transform.sample.xslt b/deploy/sample/rest-transform.sample.xslt index 56288c31..eccd682f 100644 --- a/deploy/sample/rest-transform.sample.xslt +++ b/deploy/sample/rest-transform.sample.xslt @@ -21,7 +21,7 @@ For example, an XSL transform named add-attr must be contained in a file named a From 73f213aa70b6fcb53c143c636ab60515c0efcb38 Mon Sep 17 00:00:00 2001 From: Peter Kester Date: Thu, 1 May 2014 18:00:09 +0200 Subject: [PATCH 053/113] Fix for issue #231 conform proposal of Dave --- deploy/lib/Help.rb | 5 ++-- deploy/lib/server_config.rb | 47 ++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/deploy/lib/Help.rb b/deploy/lib/Help.rb index beb56da3..9a390f26 100644 --- a/deploy/lib/Help.rb +++ b/deploy/lib/Help.rb @@ -242,8 +242,9 @@ def self.deploy content # deploys content to your content db in the given environment schemas # deploys schemas to your schemas db in the given environment cpf # deploys your cpf config to the server in the given environment - mvc # deploys the mvc code your modules db in the given environment - rest # deploys your rest extensions to the server in the given environment + src # deploys the src code to your modules db in the given environment + rest # deploys properties, extensions, and transforms to our modules db in the given environment + ext # deploys your rest extensions to the server in the given environment if a name is specified, then only that extension will be deployed transform # deploys your rest extensions to the server in the given environment if a name is specified, then only that transform will be deployed diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 057041ee..d0ec3277 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -475,10 +475,12 @@ def deploy deploy_content when 'modules' deploy_modules - when 'mvc' - deploy_mvc + when 'src' + deploy_src when 'rest' deploy_rest + when 'ext' + deploy_ext when 'transform' deploy_transform when 'schemas' @@ -954,12 +956,11 @@ def modules_databases end def deploy_modules - deploy_mvc() + deploy_src() deploy_rest() - deploy_transforms() end - def deploy_mvc + def deploy_src test_dir = @properties['ml.xquery-test.dir'] xquery_dir = @properties['ml.xquery.dir'] # modules_db = @properties['ml.modules-db'] @@ -1029,12 +1030,6 @@ def deploy_mvc end def deploy_rest - extensionname = ARGV.shift - path = @properties['ml.rest-ext.dir'] - if extensionname - path += "/#{extensionname}.xqy" - end - # Deploy options, extensions to the REST API server if ['rest', 'hybrid'].include? @properties["ml.app-type"] # Figure out where we need to deploy this stuff @@ -1057,22 +1052,30 @@ def deploy_rest else logger.debug "Could not find REST API options directory: #{@properties['ml.rest-options.dir']}\n"; end - if (@properties.has_key?('ml.rest-ext.dir') && File.exist?(@properties['ml.rest-ext.dir'])) - logger.info "\nLoading REST extensions from #{path}\n" - mlRest.install_extensions(ServerConfig.expand_path(path)) - end + deploy_ext() + deploy_transforms() + end + end + + def deploy_ext + extension = find_arg(['--file']) + path = @properties['ml.rest-ext.dir'] + if !extension.blank? + path += "/#{extension}" + end + + # Deploy extensions to the REST API server + if (@properties.has_key?('ml.rest-ext.dir') && File.exist?(@properties['ml.rest-ext.dir'])) + logger.info "\nLoading REST extensions from #{path}\n" + mlRest.install_extensions(ServerConfig.expand_path(path)) end end def deploy_transform - transformname = ARGV.shift + transform = find_arg(['--file']) path = @properties['ml.rest-transforms.dir'] - if transformname - if File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xqy") - path += "/#{transformname}.xqy" - elsif File.exist?(@properties['ml.rest-transforms.dir']+"/#{transformname}.xslt") - path += "/#{transformname}.xslt" - end + if !transform.blank? + path += "/#{transformname}" end # Deploy transforms to the REST API server From 65cb3a96a6312f8db5e29c3d8b6c751322b08a0e Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Thu, 1 May 2014 14:39:26 -0400 Subject: [PATCH 054/113] Fix regex to avoid warning with Ruby 1.8.7 --- deploy/lib/util.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/lib/util.rb b/deploy/lib/util.rb index 97abe42f..1722f607 100644 --- a/deploy/lib/util.rb +++ b/deploy/lib/util.rb @@ -81,7 +81,7 @@ def xquery_safe end def xquery_unsafe - REXML::Text::unnormalize(self).gsub(/\{{/, '{').gsub(/\}}/, '}') + REXML::Text::unnormalize(self).gsub(/\{\{/, '{').gsub(/\}\}/, '}') end end From 699713be2fe0b1cb08c63781f694da80b6fa6f54 Mon Sep 17 00:00:00 2001 From: Richard Louapre Date: Thu, 1 May 2014 14:40:04 -0400 Subject: [PATCH 055/113] Add unit test for #177, #228 --- deploy/test/data/ml7-properties/build.properties | 1 + deploy/test/test_main.rb | 1 + deploy/test/test_server_config.rb | 12 +++++++++++- deploy/test/test_util.rb | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 deploy/test/test_util.rb diff --git a/deploy/test/data/ml7-properties/build.properties b/deploy/test/data/ml7-properties/build.properties index bb75ceea..d1a784a1 100644 --- a/deploy/test/data/ml7-properties/build.properties +++ b/deploy/test/data/ml7-properties/build.properties @@ -145,3 +145,4 @@ local-server=localhost #dev-server= #cert-server= #prod-server= +yoda-age= diff --git a/deploy/test/test_main.rb b/deploy/test/test_main.rb index 2870df8a..124583fc 100644 --- a/deploy/test/test_main.rb +++ b/deploy/test/test_main.rb @@ -1 +1,2 @@ require 'test/test_server_config' +require 'test/test_util' diff --git a/deploy/test/test_server_config.rb b/deploy/test/test_server_config.rb index 38c267da..8e704f48 100644 --- a/deploy/test/test_server_config.rb +++ b/deploy/test/test_server_config.rb @@ -66,4 +66,14 @@ def test_bootstrap Logger.new(STDOUT).info "Testing against MarkLogic version #{version}.." bootstrap_version version end -end \ No newline at end of file + + # issue #228 + def test_load_properties_from_command + ARGV << "--ml.yoda-age=900" + ARGV << "--ml.missing-key=val1" + properties = ServerConfig.properties(File.expand_path("../data/ml7-properties/", __FILE__)) + assert_equal('900', properties['ml.yoda-age']) + assert(!properties.has_key?('missing-key')) + end + +end diff --git a/deploy/test/test_util.rb b/deploy/test/test_util.rb new file mode 100644 index 00000000..3247c7cc --- /dev/null +++ b/deploy/test/test_util.rb @@ -0,0 +1,14 @@ +require 'util' + +class TestProperties < Test::Unit::TestCase + + def teardown + end + + # test for issue #177 + def test_xquery_safe_unsafe + assert_equal('secret', 'secret'.xquery_safe.xquery_unsafe) + assert_equal('crazypassword{123@#$%}},.<>', 'crazypassword{123@#$%}},.<>'.xquery_safe.xquery_unsafe) + end + +end From 1c41183d043cf1400a612c8c01c4949632bdcb70 Mon Sep 17 00:00:00 2001 From: Peter Kester Date: Fri, 2 May 2014 16:23:52 +0200 Subject: [PATCH 056/113] Oops, typo in call to deploy_transform in function deploy_rest --- deploy/lib/server_config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index d0ec3277..585766dd 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -1053,7 +1053,7 @@ def deploy_rest logger.debug "Could not find REST API options directory: #{@properties['ml.rest-options.dir']}\n"; end deploy_ext() - deploy_transforms() + deploy_transform() end end From 387ee48da21f5ec87bcf55592af571f4737fe15f Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Fri, 2 May 2014 14:17:25 -0400 Subject: [PATCH 057/113] updated changelog --- CHANGELOG.mdown | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.mdown b/CHANGELOG.mdown index e13316a3..f432882d 100644 --- a/CHANGELOG.mdown +++ b/CHANGELOG.mdown @@ -109,3 +109,18 @@ This file describes changes made to Roxy per release ## 1.6 (branch: dev) +### New Features +* \#83 Require confirmation before running "ml new" in a Roxy project directory +* \#142 Added confirmation before doing wipe on non-local environments +* \#177 Password now encoded before passing to corb +* \#209 added support for Module Locations config +* \#217 Added --fork parameter to "ml new" +* \#225 Integrated mlcp +* \#228 Added ability to specify properties on the command line + +### Resolved Issues +* \#164 Special characters in passwords now fixed on Windows +* \#211 self-test now validates appserver settings +* \#220 Fixed problem with bootstrapping fields +* \#230 Scaffolded XSLT transform is now valid + From 3f3685b8b38026a5a7d6cae8067f02d8bf3df092 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Sat, 3 May 2014 15:01:04 +0200 Subject: [PATCH 058/113] Added test case with multiple module locations (#209) --- deploy/test/data/ml7-config.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/test/data/ml7-config.xml b/deploy/test/data/ml7-config.xml index 57e79e41..bf81bcc5 100644 --- a/deploy/test/data/ml7-config.xml +++ b/deploy/test/data/ml7-config.xml @@ -157,6 +157,10 @@ http://www.ns.com/ns4 /test-lib.xqy + + http://www.ns.com/ns5 + /test-lib2.xqy + From 147a1679437e5a295f69ec753383984ae79f8aa1 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Sat, 3 May 2014 15:01:40 +0200 Subject: [PATCH 059/113] Fixed bug in support for multiple module locations (#209) --- deploy/lib/xquery/setup.xqy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 49c0f419..1550dac6 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3197,9 +3197,9 @@ declare function setup:configure-server( else $admin-config - let $module-locations := $server-config/gr:module-locations/gr:module-location + let $module-locations := $server-config/gr:module-locations let $admin-config := - if ($module-locations) then + if ($module-locations/*) then if (setup:at-least-version("7.0-0")) then xdmp:eval(' xquery version "1.0-ml"; @@ -3218,7 +3218,7 @@ declare function setup:configure-server( admin:appserver-delete-module-location( $admin-config, $server-id, - for $module-location in $module-locations + for $module-location in $module-locations/* let $same-ns := $old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location ne $module-location/gr:location] return @@ -3229,7 +3229,7 @@ declare function setup:configure-server( admin:appserver-add-module-location( $config, $server-id, - for $module-location in $module-locations + for $module-location in $module-locations/* return if ($old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location = $module-location/gr:location]) then () else From e5a297cec905e6c0a1c74da9fb41625564f3d042 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Sat, 3 May 2014 15:09:51 +0200 Subject: [PATCH 060/113] --ml args aren't consumed, removed them after test, explained warn message is expected (#228) --- deploy/test/test_server_config.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/test/test_server_config.rb b/deploy/test/test_server_config.rb index 8e704f48..64cf6592 100644 --- a/deploy/test/test_server_config.rb +++ b/deploy/test/test_server_config.rb @@ -71,9 +71,12 @@ def test_bootstrap def test_load_properties_from_command ARGV << "--ml.yoda-age=900" ARGV << "--ml.missing-key=val1" + Logger.new(STDOUT).info "A warning about a non-existing property named 'ml.missing-key' will follow, that is supposed to happen.." properties = ServerConfig.properties(File.expand_path("../data/ml7-properties/", __FILE__)) assert_equal('900', properties['ml.yoda-age']) assert(!properties.has_key?('missing-key')) + ARGV.shift + ARGV.shift end end From a442c8fbbc82622a209ee2746ffd1e72b9d6917c Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 11:01:00 +0200 Subject: [PATCH 061/113] Improved support for capturing ml-config using --ml-config (#8) --- deploy/lib/server_config.rb | 40 +++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 5a633153..c14e77dc 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -796,18 +796,19 @@ def credentials def capture full_config = find_arg(['--full-ml-config']) + config = find_arg(['--ml-config']) target_db = find_arg(['--modules-db']) # check params - if full_config == nil && target_db == nil - raise HelpException.new("capture", "either full-ml-config or modules-db is required") + if full_config == nil && config == nil && target_db == nil + raise HelpException.new("capture", "either full-ml-config, ml-config or modules-db is required") elsif target_db != nil && @properties['ml.app-type'] != 'rest' raise ExitException.new("This is a #{@properties['ml.app-type']} application; capture modules only works for app-type=rest") end # retrieve full setup config from environment - if full_config != nil - capture_environment_config + if full_config != nil || config != nil + capture_environment_config(full_config) end # retrieve modules from selected database from environment @@ -901,12 +902,30 @@ def save_files_to_fs(target_db, target_dir) # schemas; CPF configuration; along with users and roles. For users and roles, we probably need an interactive system -- # we don't want or need to capture built-in users and roles. If the application uses app-level security, then we # could start with "Capture user #{default user}?" and then check on each role that user has. - def capture_environment_config + def capture_environment_config(full_config) raise ExitException.new("Capture requires the target environment's hostname to be defined") unless @hostname.present? + if (full_config == nil) + databases = quote_arglist(find_arg(['--databases']) || "#{@properties["ml.content-db"]},#{@properties["ml.modules-db"]},#{@properties["ml.triggers-db"]},#{@properties["ml.schemas-db"]},#{@properties["ml.app-modules-db"]}") + # TODO: take content-forests-per-host into account properly, just taking first by default + forests = quote_arglist(find_arg(['--forests']) || "#{@properties["ml.content-db"]},#{@properties["ml.content-db"]}-001-1,#{@properties["ml.modules-db"]},#{@properties["ml.triggers-db"]},#{@properties["ml.schemas-db"]},,#{@properties["ml.app-modules-db"]}") + # TODO: include dav, xdbc, odbc servers? + servers = quote_arglist(find_arg(['--servers']) || "#{@properties["ml.app-name"]}") + mimes = quote_arglist(find_arg(['--mime-types']) || "##none##") + + # setup.xqy expects ids for users and roles, unfortunately + #users = quote_arglist(find_arg(['--users'])) || '9999999' + #roles = quote_arglist(find_arg(['--roles'])) || '9999999' + # TODO: fix setup.xqy to (also) accept users/roles by name + # TODO: add app-user, default-user, and app-role as defaults + users = '9999999' # non-existing id + roles = '9999999' # non-existing id + end + logger.info "Capturing configuration of MarkLogic on #{@hostname}..." + logger.debug %Q{calling setup:get-configuration((#{databases}), (#{forests}), (#{servers}), (#{users}), (#{roles}), (#{mimes}))..} setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) - r = execute_query %Q{#{setup} setup:get-configuration((), (), (), (), (), ())} + r = execute_query %Q{#{setup} setup:get-configuration((#{databases}), (#{forests}), (#{servers}), (#{users}), (#{roles}), (#{mimes}))} if r.body.match("error log") logger.error r.body @@ -922,6 +941,15 @@ def capture_environment_config return true end end + + def quote_arglist(arglist) + if arglist != nil + # TODO: remove duplicates + args = arglist.split(/[,]+/).reject { |arg| arg.empty? } + arglist = args.join("\",\"") + return "\"#{arglist}\"" + end + end # Build an array of role/capability objects. def permissions(role, capabilities) From 430e6b00f2ffd7ac81bc7080dc44b9db7ef3c6f1 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 11:58:11 +0200 Subject: [PATCH 062/113] Bugfix for #209, changing app-server schemas or module-locations failed --- deploy/lib/xquery/setup.xqy | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 1550dac6..3f39b3c6 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3145,13 +3145,13 @@ declare function setup:configure-server( if ($namespaces) then let $old-ns := admin:appserver-get-namespaces($admin-config, $server-id) let $config := - (: First delete any namespace that matches the prefix :) + (: First delete any namespace that matches the prefix, prefix must be unique :) admin:appserver-delete-namespace( $admin-config, $server-id, for $ns in $namespaces let $same-prefix := - $old-ns[gr:prefix = $ns/gr:prefix][gr:namespace-uri ne $ns/gr:namespace-uri] + $old-ns[gr:prefix eq $ns/gr:prefix][gr:namespace-uri ne $ns/gr:namespace-uri] return if ($same-prefix) then admin:group-namespace($same-prefix/gr:prefix, $same-prefix/gr:namespace-uri) @@ -3174,16 +3174,13 @@ declare function setup:configure-server( if ($schemas) then let $old-schemas := admin:appserver-get-schemas($admin-config, $server-id) let $config := - (: First delete any schema that matches the namespace :) + (: First delete any schema that matches the namespace, namespace must be unique :) admin:appserver-delete-schema( $admin-config, $server-id, for $schema in $schemas - let $same-ns := - $old-schemas[gr:namespace-uri = $schema/gr:namespace-uri][gr:schema-location ne $schema/gr:schema-location] return - if ($same-ns) then $schema - else ()) + $old-schemas[gr:namespace-uri eq $schema/gr:namespace-uri][gr:schema-location ne $schema/gr:schema-location]) return (: Then add in any schema whose namespace isn't already defined :) admin:appserver-add-schema( @@ -3191,7 +3188,7 @@ declare function setup:configure-server( $server-id, for $schema in $schemas return - if ($old-schemas[gr:namespace-uri = $schema/gr:namespace-uri][gr:schema-location = $schema/gr:schema-location]) then () + if ($old-schemas[gr:namespace-uri eq $schema/gr:namespace-uri][gr:schema-location eq $schema/gr:schema-location]) then () else $schema) else @@ -3219,11 +3216,8 @@ declare function setup:configure-server( $admin-config, $server-id, for $module-location in $module-locations/* - let $same-ns := - $old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location ne $module-location/gr:location] return - if ($same-ns) then $module-location - else ()) + $old-module-locations[gr:namespace-uri = $module-location/gr:namespace-uri][gr:location ne $module-location/gr:location]) return (: Then add in any module-location whose namespace isnt already defined :) admin:appserver-add-module-location( @@ -3255,11 +3249,8 @@ declare function setup:configure-server( $admin-config, $server-id, for $request-blackout in $request-blackouts - let $same-blackout := - $old-request-blackouts[setup:get-request-blackout-hash(.) = setup:get-request-blackout-hash($request-blackout)][gr:users ne $request-blackout/gr:users or gr:roles ne $request-blackout/gr:roles] return - if ($same-blackout) then $request-blackout - else ()) + $old-request-blackouts[setup:get-request-blackout-hash(.) = setup:get-request-blackout-hash($request-blackout)][gr:users ne $request-blackout/gr:users or gr:roles ne $request-blackout/gr:roles]) return (: Then add in any request-blackout whose type and period aren't already defined :) admin:appserver-add-request-blackout( From f7cab81460a03cfc64f6c359158b8e94de6fea59 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 12:00:34 +0200 Subject: [PATCH 063/113] Added testing of changed ml-config to self-test (#209), and temporarily fixed for wiping after self-test (#236) --- deploy/test/test_server_config.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/deploy/test/test_server_config.rb b/deploy/test/test_server_config.rb index 64cf6592..d9db820b 100644 --- a/deploy/test/test_server_config.rb +++ b/deploy/test/test_server_config.rb @@ -5,6 +5,7 @@ class TestProperties < Test::Unit::TestCase def teardown + Logger.new(STDOUT).info "Teardown: Wiping self-test deployment.." if @s @s.wipe if @s end @@ -59,6 +60,21 @@ def bootstrap_version(version) assert(@s.bootstrap, "Boostrap should succeeded") assert(@s.validate_install, "Bootstrap passes validation") + + if File.exist? File.expand_path("../data/ml#{version}-config-changed.xml", __FILE__) + @s = ServerConfig.new({ + :config_file => File.expand_path("../data/ml#{version}-config-changed.xml", __FILE__), + :properties => properties, + :logger => Logger.new(STDOUT) + }) + + assert(@s.bootstrap, "Boostrap should succeeded") + assert(@s.validate_install, "Bootstrap passes validation") + end + + # TODO: temporary fix to wipe self-test until teardown is fixed + Logger.new(STDOUT).info "Wiping self-test deployment.." + @s.wipe end def test_bootstrap From 5af3bf562e68b80793d5c97476c94ce84300761b Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 12:01:15 +0200 Subject: [PATCH 064/113] Added parse_json convenience function to util.rb --- deploy/lib/util.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/lib/util.rb b/deploy/lib/util.rb index 1722f607..a2fa5f4f 100644 --- a/deploy/lib/util.rb +++ b/deploy/lib/util.rb @@ -99,3 +99,15 @@ def to_b present? && ['true', 'TRUE', 'yes', 'YES', 'y', 'Y', 't', 'T'].include?(self) end end + +def parse_json(body) + if (body.match('^\[\{"qid":null,')) + items = [] + JSON.parse(body).each do |item| + items.push item['result'] + end + return items.join("\n") + else + return body + end +end \ No newline at end of file From c8f8caefdb1a5f2d8ec31bfd0860919fbb740f50 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 12:03:49 +0200 Subject: [PATCH 065/113] Improved logging of response bodies for bootstrap, wipe, and validate by applying parse_json more generically (#227) --- deploy/lib/server_config.rb | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index c14e77dc..62c2242e 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -393,7 +393,9 @@ def bootstrap logger.info "Bootstrapping your project into MarkLogic on #{@hostname}..." setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) r = execute_query %Q{#{setup} setup:do-setup(#{get_config})} + logger.debug "code: #{r.code.to_i}" + r.body = parse_json(r.body) logger.debug r.body if r.body.match("error log") @@ -415,6 +417,9 @@ def wipe logger.info "Wiping MarkLogic setup for your project on #{@hostname}..." setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) r = execute_query %Q{#{setup} setup:do-wipe(#{get_config})} + logger.debug "code: #{r.code.to_i}" + + r.body = parse_json(r.body) logger.debug r.body if r.body.match(" e - if e.response.body.match('"result":" Date: Thu, 8 May 2014 12:09:29 +0200 Subject: [PATCH 066/113] Added second ml7-config with changes to test data for improved self-test (#209 and #212) --- deploy/test/data/ml7-config-changed.xml | 1016 +++++++++++++++++++++++ 1 file changed, 1016 insertions(+) create mode 100644 deploy/test/data/ml7-config-changed.xml diff --git a/deploy/test/data/ml7-config-changed.xml b/deploy/test/data/ml7-config-changed.xml new file mode 100644 index 00000000..0daf865f --- /dev/null +++ b/deploy/test/data/ml7-config-changed.xml @@ -0,0 +1,1016 @@ + + + 16 + 16 + 3600 + 600 + 100000 + 1000 + 1000 + 10000 + true + true + true + + + /other/daily-task.xqy + / + daily + 2 + 13:00:00-05:00 + + + + + + /other/hourly-task.xqy + / + hourly + 2 + 15 + + + + + + /other/minutely-task.xqy + / + minutely + 3 + + + + + + /other/monthly-task.xqy + / + monthly + 1 + 15 + 13:00:00-05:00 + + + + + + /other/once-task.xqy + / + once + 2019-01-01T13:00:00-05:00 + + + + + + /other/weekly-task.xqy + / + weekly + 1 + + monday + wednesday + friday + + 13:00:00-05:00 + + + + + + + + + + @ml.app-name + false + /@ml.app-name/ + @ml.app-port + + + Last-Login + true +
1.2.3.4
+ 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + http://marklogic.com/collation/codepoint + basic + @ml.default-user + admin-ui + 123 + false + false + false + 1.0 + nonblocking + strict + + normal + ASCII + xml + yes + a + b + c + d + yes + yes + yes + yes + text/xml + NFC + yes + yes + yes + 1.0 + yes + + /roxy/error.xqy + + + http://www.ns.com/ns0 + /other/test.xsd + + + + + ns3 + http://www.ns.com/ns1 + + + ns4 + http://www.ns.com/ns2 + + + http://www.ns.com/ns3 + + + + + http://www.ns.com/ns4 + /other/test-lib.xqy + + + http://www.ns.com/ns5 + /other/test-lib2.xqy + + + + + + + recurring + + monday + + + 12:34:00+02:00 + 23:45:00+02:00 + + + + + + recurring + + monday + + + 12:34:00+02:00 + PT1H23M + + + + + + recurring + + monday + + + + + + + once + + + 1234-11-22 + 12:34:00+02:00 + 1234-12-23 + 23:34:00+02:00 + + + + /roxy/rewrite.xqy + true + false + false + sixer + ALL:!LOW:@STRENGTH:ROXY + false + + + @ml.test-appserver + + @ml.app-name-dav + false + /@ml.app-name/ + @ml.dav-port + true + true + + Last-Login + true +
1.2.3.4
+ 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + 321 + 123 + http://marklogic.com/collation/codepoint + basic + @ml.default-user + admin-ui + 123 + false + false + false + 1.0 + nonblocking + strict + + normal + ASCII + xml + yes + a + b + c + d + yes + yes + yes + yes + text/xml + NFC + yes + yes + yes + 1.0 + yes + + + + http://www.ns.com/ns0 + /test.xsd + + + + + ns1 + http://www.ns.com/ns1 + + + ns2 + http://www.ns.com/ns2 + + + http://www.ns.com/ns3 + + + + + + + recurring + + monday + + + 12:34:00+02:00 + 23:45:00+02:00 + + + + + + recurring + + monday + + + 12:34:00+02:00 + PT1H23M + + + + + + recurring + + monday + + + + + + + once + + + 1234-11-22 + 12:34:00+02:00 + 1234-12-23 + 23:34:00+02:00 + + + + false + false + sixer + ALL:!LOW:@STRENGTH:ROXY + false + + + + + @ml.xdbc-server + + + @ml.odbc-server + + + + + @ml.content-db + + @ml.test-content-db-assignment + @ml.test-modules-db-assignment + + @ml.modules-db + + @ml.schemas-assignment + @ml.triggers-assignment + + + + + @ml.test-content-db-xml + + @ml.content-db + @ml.content-forests-per-host + @ml.schemas-mapping + @ml.triggers-mapping + + + + true + @ml.content-db-security + en + + + http://www.marklogic.com/ns/sample + frag-root + + + http://www.marklogic.com/ns/sample2 + frag-root2 + + + + + http://www.marklogic.com/ns/sample + frag-parents + + + http://www.marklogic.com/ns/sample + frag-parents2 + + + + + http://schemas.microsoft.com/office/word/2003/wordml + p + + + http://schemas.openxmlformats.org/wordprocessingml/2006/main + p + + + + + http://www.w3.org/1999/xhtml + a abbr acronym b big br center cite code dfn em font i kbd q samp small span strong sub sup tt var + + + http://schemas.microsoft.com/office/word/2003/wordml + br cr fldChar fldData fldSimple hlink noBreakHyphen permEnd permStart pgNum proofErr r softHyphen sym t tab + + + http://schemas.microsoft.com/office/word/2003/auxHint + t + + + http://schemas.openxmlformats.org/wordprocessingml/2006/main + r t endnoteReference footnoteReference customXml hyperlink sdt sdtContent commentRangeEnd commentRangeStart bookmarkStart bookmarkEnd fldSimple instrText smartTag ins proofErr + + + http://marklogic.com/entity + person organization location gpe facility religion nationality credit-card-number email coordinate money percent id phone-number url utm date time + + + + + http://schemas.microsoft.com/office/word/2003/wordml + delInstrText delText endnote footnote instrText pict rPr + + + http://schemas.openxmlformats.org/wordprocessingml/2006/main + pPr rPr customXmlPr sdtPr commentReference del + + + + + + false + decompounding + true + true + true + true + true + true + true + true + true + true + + + http://www.marklogic.com/ns/sample + word-query-include + 1.0 + + + + + + http://www.marklogic.com/ns/sample2 + word-query-include2 word-query-include3 + 1.0 + + + + + + + + http://www.marklogic.com/ns/sample + word-query-exclude + + + + + + http://www.marklogic.com/ns/sample2 + word-query-exclude2 word-query-exclude3 + + + + + + + + + test + true + + http://marklogic.com/collation/ + http://marklogic.com/collation/codepoint + + + + http://www.marklogic.com/ns/sample + sample-included-element + 1 + + + + + + http://www.marklogic.com/ns/sample2 + sample-included-element2 sample-included-element3 + 1 + + + + + + + + http://www.marklogic.com/ns/sample + sample-excluded-element + + + + + + http://www.marklogic.com/ns/sample2 + sample-excluded-element2 sample-excluded-element3 + + + + + + + + + test2 + true + + + + http://www.marklogic.com/ns/sample + sample2-included-element + 1 + + + + + + http://www.marklogic.com/ns/sample2 + sample2-included-element2 sample2-included-element3 + 1 + + + + + + + + http://www.marklogic.com/ns/sample + sample2-excluded-element + + + + + + http://www.marklogic.com/ns/sample2 + sample2-excluded-element2 sample2-excluded-element3 + + + + + + + + + test3 + + some/xpath + 1 + + basic + true + true + true + + + + + + + + + string + http://marklogic.com/ns/sample + name + http://marklogic.com/collation/codepoint + false + reject + + + string + http://marklogic.com/ns/sample2 + name2 + http://marklogic.com/collation/codepoint + false + reject + + + + + string + http://marklogic.com/ns/sample + person + + name + http://marklogic.com/collation/codepoint + false + reject + + + string + http://marklogic.com/ns/sample2 + person2 + + name2 + http://marklogic.com/collation/codepoint + false + reject + + + + + date + sample + + false + reject + + + date + sample2 + + false + reject + + + + + sample + http://marklogic.com/ns/sample + + + sample2 + http://marklogic.com/ns/sample2 + + + + + string + http://marklogic.com/collation/codepoint + /sample:root/sample:child + false + reject + + + string + http://marklogic.com/collation/codepoint + /sample:root/sample:child2 + false + reject + + + + + http://www.marklogic.com/ns/sample + geo-element + wgs84 + point + false + reject + + + http://www.marklogic.com/ns/sample + geo-element2 + wgs84 + point + false + reject + + + + + http://marklogic.com/ns/sample + geo + + lat + + lon + wgs84 + false + reject + + + http://marklogic.com/ns/sample + geo2 + + lat2 + + lon2 + wgs84 + false + reject + + + + + http://marklogic.com/ns/sample + geo + http://marklogic.com/ns/sample + lat + http://marklogic.com/ns/sample + lon + wgs84 + true + reject + + + http://marklogic.com/ns/sample + geo2 + http://marklogic.com/ns/sample + lat2 + http://marklogic.com/ns/sample + lon2 + wgs84 + true + reject + + + + + http://marklogic.com/ns/sample + geo + http://marklogic.com/ns/sample + pos + wgs84 + point + false + reject + + + http://marklogic.com/ns/sample + geo2 + http://marklogic.com/ns/sample + pos2 + wgs84 + point + false + reject + + + + + http://www.marklogic.com/ns/sample + sample-element + http://marklogic.com/collation/ + + + http://www.marklogic.com/ns/sample + sample-element2 + http://marklogic.com/collation/ + + + + + http://www.marklogic.com/ns/sample + sample-element + + sample-attribute + http://marklogic.com/collation/ + + + http://www.marklogic.com/ns/sample2 + sample-element2 + + sample-attribute2 + http://marklogic.com/collation/ + + + + + + @ml.modules-db + + + + basic + false + false + true + false + true + true + true + false + true + false + false + false + false + false + false + false + false + false + false + + http://marklogic.com/collation/ + http://marklogic.com/collation/codepoint + + false + false + true + false + true + 5 + 0 + manual + false + false + false + false + false + 32768 + 64 + 16 + 2 + 2 + 1024 + fast + fast + 128 + false + false + false + facet-time + 256 + automatic + automatic + automatic + scaled-log + lower + 0 + 1024 + 2 + 0 + + + @ml.test-modules-db-xml + + @ml.triggers-db-xml + @ml.schemas-db-xml + + + + @ml.app-role + A role for users of the @ml.app-name application + + + + + execute + @ml.app-role + + + update + @ml.app-role + + + insert + @ml.app-role + + + read + @ml.app-role + + + + + + + xdmp:value + + + xdmp:add-response-header + + + xdmp:invoke + + + xdmp:with-namespaces + + + + + @ml.app-role2 + A second role for users of the @ml.app-name application + + + + + execute + @ml.app-role2 + + + update + @ml.app-role2 + + + insert + @ml.app-role2 + + + read + @ml.app-role2 + + + + + + + xdmp:value + + + xdmp:add-response-header + + + xdmp:invoke + + + xdmp:with-namespaces + + + + + + + @ml.app-name-user + A user for the @ml.app-name application + password + + @ml.app-role + + + + + + @ml.app-name-user2 + A second user for the @ml.app-name application + password2 + + @ml.app-role2 + + + + + + + + http://marklogic.com/roxy + sample + /app/models/sample.xqy + @ml.app-modules-db + admin + + + http://marklogic.com/roxy + sample2 + /app/models/sample2.xqy + @ml.app-modules-db + admin + + + + + my-action + http://marklogic.com/custom/privilege/my-action + execute + + + my-action2 + http://marklogic.com/custom/privilege/my-action2 + execute + + + + + application/crazy + crazy stuff + text + + + application/crazy2 + crazy stuff 2 + text + + + From 058383b797ba1210127d87e5caa7d160f9c367f4 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 13:59:41 +0200 Subject: [PATCH 067/113] Added testcases for field tokenizer-overrides (#222) --- deploy/test/data/ml7-config-changed.xml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/deploy/test/data/ml7-config-changed.xml b/deploy/test/data/ml7-config-changed.xml index 0daf865f..da7de5ad 100644 --- a/deploy/test/data/ml7-config-changed.xml +++ b/deploy/test/data/ml7-config-changed.xml @@ -539,7 +539,12 @@ - + + + _ + word + + test2 @@ -579,7 +584,12 @@ - + + + _ + word + + test3 @@ -594,7 +604,12 @@ - + + + _ + word + + From 2dc6e31b10bc7d2d5293892d606b8fdf9f2081d0 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 14:00:07 +0200 Subject: [PATCH 068/113] Added support for field tokenizer-overrides (#222) --- deploy/lib/xquery/setup.xqy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 3f39b3c6..c09efc33 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -1249,6 +1249,28 @@ declare function setup:apply-field-settings( xdmp:set( $admin-config, xdmp:value(fn:concat("admin:database-set-field-", $setting, "($admin-config, $database, $field-name, $value)"))) + + let $add-tokenizers := + if ($db-config/db:fields/db:field/db:tokenizer-overrides/db:tokenizer-override) then + if (setup:at-least-version("7.0-0")) then + for $field in $db-config/db:fields/db:field + let $field-name as xs:string := $field/db:field-name + let $overrides := + for $override in $field/db:tokenizer-overrides/db:tokenizer-override + return + xdmp:value("admin:database-tokenizer-override($override/db:character, $override/db:tokenizer-class)") + where $overrides + return + xdmp:set( + $admin-config, + xdmp:value("admin:database-add-field-tokenizer-override($admin-config, $database, $field-name, $overrides)") + ) + else + fn:error( + xs:QName("VERSION_NOT_SUPPORTED"), + fn:concat("MarkLogic ", xdmp:version(), " does not support field tokenizer-overrides. Use 7.0-0 or higher.")) + else () + return $admin-config }; From 055f2236739b890d314cc4f6a20a46612e92c6c1 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 16:05:21 +0200 Subject: [PATCH 069/113] Capture ml-config (#8) always assumed JSON response (ML7 only), made that agnostic using parse_json util function --- deploy/lib/server_config.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 62c2242e..fed462ea 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -923,18 +923,16 @@ def capture_environment_config(full_config) logger.debug %Q{calling setup:get-configuration((#{databases}), (#{forests}), (#{servers}), (#{users}), (#{roles}), (#{mimes}))..} setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) r = execute_query %Q{#{setup} setup:get-configuration((#{databases}), (#{forests}), (#{servers}), (#{users}), (#{roles}), (#{mimes}))} + r.body = parse_json(r.body) if r.body.match("error log") logger.error r.body logger.error "... Capture FAILED" return false else - JSON.parse(r.body).each do |item| - contents = item['result'] - name = "#{@properties["ml.config.file"].sub( %r{.xml}, '' )}-#{@properties["ml.environment"]}.xml" - File.open(name, 'w') { |file| file.write(contents) } - logger.info("... Captured full configuration into #{name}") - end + name = "#{@properties["ml.config.file"].sub( %r{.xml}, '' )}-#{@properties["ml.environment"]}.xml" + File.open(name, 'w') { |file| file.write(r.body) } + logger.info("... Captured full configuration into #{name}") return true end end From 2b7cb388fcbf6eb23baf1f3847beaf0f2f601bba Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Thu, 8 May 2014 16:07:02 +0200 Subject: [PATCH 070/113] Added support for wiping app-builder deployments (#237) --- deploy/lib/server_config.rb | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index fed462ea..ec031ccc 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -414,9 +414,39 @@ def bootstrap end def wipe - logger.info "Wiping MarkLogic setup for your project on #{@hostname}..." + appbuilder = find_arg(['--app-builder']) + + if (appbuilder != nil) + logger.info "Wiping MarkLogic App-Builder deployment #{appbuilder} from #{@hostname}..." + config = %Q{ + + + + #{appbuilder} + + + + + #{appbuilder}-modules-1 + + + + + #{appbuilder}-modules + + + + + + + } + else + logger.info "Wiping MarkLogic setup for your project from #{@hostname}..." + config = get_config + end + setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) - r = execute_query %Q{#{setup} setup:do-wipe(#{get_config})} + r = execute_query %Q{#{setup} setup:do-wipe(#{config})} logger.debug "code: #{r.code.to_i}" r.body = parse_json(r.body) From 8eb8c8ee22ed8d79d079e1332eed8852e23d5e22 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 9 May 2014 15:29:50 +0200 Subject: [PATCH 071/113] Fix for #239, recognizing REST properties with namespace prefix --- deploy/lib/ml_rest.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/lib/ml_rest.rb b/deploy/lib/ml_rest.rb index 3891531e..fa92336e 100644 --- a/deploy/lib/ml_rest.rb +++ b/deploy/lib/ml_rest.rb @@ -44,7 +44,10 @@ def install_properties(path) file = open(d, "rb") contents = file.read - if contents.match(' 206) @logger.error("code: #{r.code.to_i} body:#{r.body}") end - else - # Properties file needs to be updated - raise ExitException.new "#{d} is in an old format; changes to this file won't take effect. See https://github.com/marklogic/roxy/wiki/REST-properties-format-change" end end else From 965f5d5ac465cce745854d2ca5f032f46e5ce3b4 Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 9 May 2014 15:31:08 +0200 Subject: [PATCH 072/113] Fix for #238, capturing REST properties properly using new --app-builder parameter --- deploy/lib/server_config.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index ec031ccc..06b1e1d2 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -825,6 +825,26 @@ def capture full_config = find_arg(['--full-ml-config']) config = find_arg(['--ml-config']) target_db = find_arg(['--modules-db']) + appbuilder = find_arg(['--app-builder']) + + if (appbuilder != nil) + serverstats = execute_query %Q{ + xquery version "1.0-ml"; + + let $status := xdmp:server-status(xdmp:host(), xdmp:server("#{appbuilder}")) + return ( + string($status//*:port), + $status//*:modules/xdmp:database-name(.) + ) + } + + logger.info parse_json(serverstats.body) + + serverstats.body = parse_json(serverstats.body).split(/[\r\n]+/) + + port = serverstats.body[0] + target_db = serverstats.body[1] + end # check params if full_config == nil && config == nil && target_db == nil @@ -852,6 +872,13 @@ def capture @properties['ml.rest-options.dir'] ) FileUtils.rm_rf("#{tmp_dir}/src/#{@properties['ml.group']}/") + + # Make sure REST properties are in accurate format, so you can directly deploy them again.. + if (port != nil) + r = go "http://#{@hostname}:#{port}/v1/config/properties", "get" + r.body = parse_json(r.body) + File.open("#{@properties['ml.rest-options.dir']}/properties.xml", 'w') { |file| file.write(r.body) } + end # If we have an application/custom directory, we've probably done a capture # before. Don't overwrite that directory. Kill the downloaded custom directory From d14d7ba49fcda293993ac27091640e3ef75d65dc Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Fri, 9 May 2014 15:45:15 +0200 Subject: [PATCH 073/113] Removed some debug info from normal logging --- deploy/lib/server_config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 06b1e1d2..41485576 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -838,7 +838,7 @@ def capture ) } - logger.info parse_json(serverstats.body) + logger.debug parse_json(serverstats.body) serverstats.body = parse_json(serverstats.body).split(/[\r\n]+/) From d264a4c24cb2faec1b5e6a39e8efd5d322b00d5c Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Sat, 10 May 2014 22:10:02 +0200 Subject: [PATCH 074/113] Reverted scheduled-task changes in ml7-config-changed, they add extra ones, while wipe only removes half --- deploy/test/data/ml7-config-changed.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/test/data/ml7-config-changed.xml b/deploy/test/data/ml7-config-changed.xml index da7de5ad..3c930d1f 100644 --- a/deploy/test/data/ml7-config-changed.xml +++ b/deploy/test/data/ml7-config-changed.xml @@ -13,7 +13,7 @@ true - /other/daily-task.xqy + /some/daily-task.xqy / daily 2 @@ -23,7 +23,7 @@ - /other/hourly-task.xqy + /some/hourly-task.xqy / hourly 2 @@ -33,7 +33,7 @@ - /other/minutely-task.xqy + /some/minutely-task.xqy / minutely 3 @@ -42,7 +42,7 @@ - /other/monthly-task.xqy + /some/monthly-task.xqy / monthly 1 @@ -53,7 +53,7 @@ - /other/once-task.xqy + /some/once-task.xqy / once 2019-01-01T13:00:00-05:00 @@ -62,7 +62,7 @@ - /other/weekly-task.xqy + /some/weekly-task.xqy / weekly 1 From 7b0bd7bb57934505b84c4ea39cefeb89af07a4af Mon Sep 17 00:00:00 2001 From: Geert Josten Date: Tue, 13 May 2014 22:04:00 +0200 Subject: [PATCH 075/113] Partial fix for #161, use admin function to create index def instead of copying xml from ml-config --- deploy/lib/xquery/setup.xqy | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index c09efc33..44db9e87 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -1423,7 +1423,16 @@ declare function setup:add-range-element-indexes-R( { if ($index-configs) then setup:add-range-element-indexes-R( - admin:database-add-range-element-index($admin-config, $database, $index-configs[1]), + admin:database-add-range-element-index($admin-config, $database, + admin:database-range-element-index( + $index-configs[1]/db:scalar-type, + $index-configs[1]/db:namespace-uri, + $index-configs[1]/db:localname, + $index-configs[1]/db:collation, + ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1], + ($index-configs[1]/db:invalid-values, "reject")[1] + ) + ), $database, fn:subsequence($index-configs, 2)) else @@ -1473,7 +1482,18 @@ declare function setup:add-range-element-attribute-indexes-R( { if ($index-configs) then setup:add-range-element-attribute-indexes-R( - admin:database-add-range-element-attribute-index($admin-config, $database, $index-configs[1]), + admin:database-add-range-element-attribute-index($admin-config, $database, + admin:database-range-element-attribute-index( + $index-configs[1]/db:scalar-type, + $index-configs[1]/db:parent-namespace-uri, + $index-configs[1]/db:parent-localname, + $index-configs[1]/db:namespace-uri, + $index-configs[1]/db:localname, + $index-configs[1]/db:collation, + ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1], + ($index-configs[1]/db:invalid-values, "reject")[1] + ) + ), $database, fn:subsequence($index-configs, 2)) else From a7bedc694f7986147b2d8bd0899f8618ad8b44e8 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 9 Jun 2014 13:31:51 -0400 Subject: [PATCH 076/113] Some changes to Geert's pull request to make them ML5-friendly. --- deploy/lib/xquery/setup.xqy | 69 ++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 44db9e87..54e5da04 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -1249,7 +1249,7 @@ declare function setup:apply-field-settings( xdmp:set( $admin-config, xdmp:value(fn:concat("admin:database-set-field-", $setting, "($admin-config, $database, $field-name, $value)"))) - + let $add-tokenizers := if ($db-config/db:fields/db:field/db:tokenizer-overrides/db:tokenizer-override) then if (setup:at-least-version("7.0-0")) then @@ -1270,7 +1270,7 @@ declare function setup:apply-field-settings( xs:QName("VERSION_NOT_SUPPORTED"), fn:concat("MarkLogic ", xdmp:version(), " does not support field tokenizer-overrides. Use 7.0-0 or higher.")) else () - + return $admin-config }; @@ -1421,17 +1421,31 @@ declare function setup:add-range-element-indexes-R( $database as xs:unsignedLong, $index-configs as element(db:range-element-index)*) as element(configuration) { + xdmp:log(fn:concat("add-range-element-indexes-R: at-least-version(6.0-1)? ", setup:at-least-version("6.0-1"))), if ($index-configs) then setup:add-range-element-indexes-R( admin:database-add-range-element-index($admin-config, $database, - admin:database-range-element-index( - $index-configs[1]/db:scalar-type, - $index-configs[1]/db:namespace-uri, - $index-configs[1]/db:localname, - $index-configs[1]/db:collation, - ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1], - ($index-configs[1]/db:invalid-values, "reject")[1] - ) + let $function := xdmp:function(xs:QName("admin:database-range-element-index")) + return + if (setup:at-least-version("6.0-1")) then + xdmp:apply( + $function, + $index-configs[1]/db:scalar-type, + $index-configs[1]/db:namespace-uri, + $index-configs[1]/db:localname, + $index-configs[1]/db:collation, + ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1], + ($index-configs[1]/db:invalid-values, "reject")[1] + ) + else + xdmp:apply( + $function, + $index-configs[1]/db:scalar-type, + $index-configs[1]/db:namespace-uri, + $index-configs[1]/db:localname, + $index-configs[1]/db:collation, + ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1] + ) ), $database, fn:subsequence($index-configs, 2)) @@ -1483,16 +1497,31 @@ declare function setup:add-range-element-attribute-indexes-R( if ($index-configs) then setup:add-range-element-attribute-indexes-R( admin:database-add-range-element-attribute-index($admin-config, $database, - admin:database-range-element-attribute-index( - $index-configs[1]/db:scalar-type, - $index-configs[1]/db:parent-namespace-uri, - $index-configs[1]/db:parent-localname, - $index-configs[1]/db:namespace-uri, - $index-configs[1]/db:localname, - $index-configs[1]/db:collation, - ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1], - ($index-configs[1]/db:invalid-values, "reject")[1] - ) + let $function := xdmp:function(xs:QName("admin:database-range-element-attribute-index")) + return + if (setup:at-least-version("6.0-1")) then + xdmp:apply( + $function, + $index-configs[1]/db:scalar-type, + $index-configs[1]/db:parent-namespace-uri, + $index-configs[1]/db:parent-localname, + $index-configs[1]/db:namespace-uri, + $index-configs[1]/db:localname, + $index-configs[1]/db:collation, + ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1], + ($index-configs[1]/db:invalid-values, "reject")[1] + ) + else + xdmp:apply( + $function, + $index-configs[1]/db:scalar-type, + $index-configs[1]/db:parent-namespace-uri, + $index-configs[1]/db:parent-localname, + $index-configs[1]/db:namespace-uri, + $index-configs[1]/db:localname, + $index-configs[1]/db:collation, + ($index-configs[1]/db:range-value-positions/xs:boolean(.), false())[1] + ) ), $database, fn:subsequence($index-configs, 2)) From 2f0c45c052fb7df8babd7ed439d75112dea2c730 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Mon, 7 Jul 2014 11:55:01 -0400 Subject: [PATCH 077/113] #258 allow filesystem for amps --- deploy/lib/xquery/setup.xqy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/lib/xquery/setup.xqy b/deploy/lib/xquery/setup.xqy index 54e5da04..d1e4d34c 100644 --- a/deploy/lib/xquery/setup.xqy +++ b/deploy/lib/xquery/setup.xqy @@ -3927,11 +3927,12 @@ declare function setup:create-amps($import-config) xdmp:eval( 'import module namespace sec="http://marklogic.com/xdmp/security" at "/MarkLogic/security.xqy"; declare variable $amp external; + declare variable $db := if($amp/sec:db-name = "filesystem") then 0 else xdmp:database($amp/sec:db-name); sec:create-amp( $amp/sec:namespace, $amp/sec:local-name, $amp/sec:doc-uri, - xdmp:database($amp/sec:db-name), + $db, $amp/sec:role-name )', (xs:QName("amp"), $amp), From 2ffb32a3708406e9580c934ae753194785d24b13 Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Mon, 21 Jul 2014 10:28:47 -0400 Subject: [PATCH 078/113] fixing binary corruption issue when capturing existing app --- deploy/lib/server_config.rb | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/deploy/lib/server_config.rb b/deploy/lib/server_config.rb index 41485576..32d8595f 100644 --- a/deploy/lib/server_config.rb +++ b/deploy/lib/server_config.rb @@ -415,7 +415,7 @@ def bootstrap def wipe appbuilder = find_arg(['--app-builder']) - + if (appbuilder != nil) logger.info "Wiping MarkLogic App-Builder deployment #{appbuilder} from #{@hostname}..." config = %Q{ @@ -444,7 +444,7 @@ def wipe logger.info "Wiping MarkLogic setup for your project from #{@hostname}..." config = get_config end - + setup = File.read(ServerConfig.expand_path("#{@@path}/lib/xquery/setup.xqy")) r = execute_query %Q{#{setup} setup:do-wipe(#{config})} logger.debug "code: #{r.code.to_i}" @@ -750,7 +750,7 @@ def mlcp logger.debug "Options after resolving properties:" lines = options.split(/[\n\r]+/).reject { |line| line.empty? || line.match("^#") } - + lines.each do |line| logger.debug line end @@ -826,7 +826,7 @@ def capture config = find_arg(['--ml-config']) target_db = find_arg(['--modules-db']) appbuilder = find_arg(['--app-builder']) - + if (appbuilder != nil) serverstats = execute_query %Q{ xquery version "1.0-ml"; @@ -837,11 +837,11 @@ def capture $status//*:modules/xdmp:database-name(.) ) } - + logger.debug parse_json(serverstats.body) - + serverstats.body = parse_json(serverstats.body).split(/[\r\n]+/) - + port = serverstats.body[0] target_db = serverstats.body[1] end @@ -872,12 +872,12 @@ def capture @properties['ml.rest-options.dir'] ) FileUtils.rm_rf("#{tmp_dir}/src/#{@properties['ml.group']}/") - + # Make sure REST properties are in accurate format, so you can directly deploy them again.. if (port != nil) r = go "http://#{@hostname}:#{port}/v1/config/properties", "get" r.body = parse_json(r.body) - File.open("#{@properties['ml.rest-options.dir']}/properties.xml", 'w') { |file| file.write(r.body) } + File.open("#{@properties['ml.rest-options.dir']}/properties.xml", 'wb') { |file| file.write(r.body) } end # If we have an application/custom directory, we've probably done a capture @@ -924,7 +924,7 @@ def save_files_to_fs(target_db, target_dir) }, { :db_name => target_db } - File.open("#{target_dir}#{uri}", 'w') { |file| file.write(r.body) } + File.open("#{target_dir}#{uri}", 'wb') { |file| file.write(r.body) } end end else @@ -942,7 +942,7 @@ def save_files_to_fs(target_db, target_dir) Dir.mkdir("#{target_dir}" + uri) else r = go "http#{@use_https ? 's' : ''}://#{@hostname}:#{@bootstrap_port}/qconsole/endpoints/view.xqy?dbid=#{db_id}&uri=#{uri}", "get" - File.open("#{target_dir}#{uri}", 'w') { |file| file.write(r.body) } + File.open("#{target_dir}#{uri}", 'wb') { |file| file.write(r.body) } end end end @@ -988,12 +988,12 @@ def capture_environment_config(full_config) return false else name = "#{@properties["ml.config.file"].sub( %r{.xml}, '' )}-#{@properties["ml.environment"]}.xml" - File.open(name, 'w') { |file| file.write(r.body) } + File.open(name, 'wb') { |file| file.write(r.body) } logger.info("... Captured full configuration into #{name}") return true end end - + def quote_arglist(arglist) if arglist != nil # TODO: remove duplicates @@ -1133,7 +1133,7 @@ def deploy_rest else logger.info "\nNo REST API options found in: #{@properties['ml.rest-options.dir']}"; end - + deploy_ext() deploy_transform() end @@ -1158,8 +1158,8 @@ def deploy_ext def deploy_transform transform = find_arg(['--file']) path = @properties['ml.rest-transforms.dir'] - if !transform.blank? - path += "/#{transformname}" + if !transform.blank? + path += "/#{transformname}" end # Deploy transforms to the REST API server From 6027b94f63c7dc39d3f2e97b458688de115149c2 Mon Sep 17 00:00:00 2001 From: Brad Mann Date: Fri, 1 Aug 2014 16:02:46 -0400 Subject: [PATCH 079/113] #263 don't double decode values from xdmp:get-request-field() --- src/roxy/lib/request.xqy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/roxy/lib/request.xqy b/src/roxy/lib/request.xqy index eb75ebba..998b0d37 100644 --- a/src/roxy/lib/request.xqy +++ b/src/roxy/lib/request.xqy @@ -38,7 +38,7 @@ declare variable $req:request as map:map := for $val in xdmp:get-request-field($name) return if ($val instance of xs:anySimpleType) then - xdmp:url-decode($val) + $val else ( if ($val instance of binary()) then map:put($map, fn:concat($name, "-filename"), xdmp:get-request-field-filename($name)) From 7fff05a01a810c5884f442e2347561ee459118e0 Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Fri, 8 Aug 2014 13:39:20 -0400 Subject: [PATCH 080/113] adding support for passing through REST extention errors added a helper method to throw http errors from a controller --- src/roxy/error.xqy | 14 ++++++++++++-- src/roxy/lib/controller-helper.xqy | 10 ++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/roxy/error.xqy b/src/roxy/error.xqy index 5e0d7bb7..ceca3c46 100644 --- a/src/roxy/error.xqy +++ b/src/roxy/error.xqy @@ -144,7 +144,12 @@ declare function local:error($title as xs:string?, $heading, $msg) }; -if (($ex/error:code = "XDMP-UNDFUN" and $ex/error:data/error:datum = fn:concat($router:func, "()")) or +if ($ex/error:code = "RESTAPI-EXTNERR") then + xdmp:invoke( + "/MarkLogic/rest-api/error-handler.xqy", + (xs:QName("error:errors"), $error:errors) + ) +else if (($ex/error:code = "XDMP-UNDFUN" and $ex/error:data/error:datum = fn:concat($router:func, "()")) or ($ex/error:code = ("SVC-FILOPN", "XDMP-MODNOTFOUND") and $ex/error:data/error:datum/fn:ends-with(., $router:controller-path))) then local:four-o-four() else if (($ex/error:name, $ex/error:code) = ("XDMP-UNDFUN") and fn:starts-with($ex/error:data/error:datum, "c:")) then @@ -153,8 +158,13 @@ else if ($ex/error:name = $MESSAGES/message/@code) then local:error($MESSAGES/message[@code = $ex/error:name]/@title, fn:string($ex/error:message), $MESSAGES/message[@code = $ex/error:name]/node()) else if ($ex/error:name = "four-o-four" or 404 = xdmp:get-response-code()[1]) then local:four-o-four() +else if ($ex/error:code = "HTTP-ERROR") then +( + xdmp:set-response-code(xs:int($ex/error:data/error:datum[1]), $ex/error:data/error:datum[2]), + $ex/error:data/error:datum[3]/fn:data() +) else ( xdmp:set-response-content-type("text/plain"), $ex - ) \ No newline at end of file + ) diff --git a/src/roxy/lib/controller-helper.xqy b/src/roxy/lib/controller-helper.xqy index fc25090b..91e197f8 100644 --- a/src/roxy/lib/controller-helper.xqy +++ b/src/roxy/lib/controller-helper.xqy @@ -129,3 +129,13 @@ declare function ch:set-format($new-format as xs:string, $formats as xs:string*) ) ) }; + +declare function ch:http-error($error-code as xs:int, $message as xs:string) +{ + ch:http-error($error-code, $message, ()) +}; + +declare function ch:http-error($error-code as xs:int, $message as xs:string, $body) +{ + fn:error((), "HTTP-ERROR", ($error-code, $message, $body)) +}; From 7081fce6e2fadd5da1980a33e11c4ced0fe6fac6 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Tue, 12 Aug 2014 09:33:31 -0400 Subject: [PATCH 081/113] #266 don't rely on the rewriter for unit tests to work --- src/test/js/tests.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/js/tests.js b/src/test/js/tests.js index ab93d96b..5ba06f83 100644 --- a/src/test/js/tests.js +++ b/src/test/js/tests.js @@ -248,9 +248,10 @@ function runSuite(suite) { spinner.show(); $.ajax({ - url: "run", + url: "default.xqy", cache: false, data: { + func: 'run', suite: suite, tests: tests.join(","), runsuiteteardown: suiteTearDown, From 33ddbb1833c079aa11b5e4371cc226046d7c3ece Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Tue, 12 Aug 2014 09:39:51 -0400 Subject: [PATCH 082/113] #266 pass through calls to test/default.xqy --- src/roxy/config/defaults.xqy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/roxy/config/defaults.xqy b/src/roxy/config/defaults.xqy index 21c9fb85..0a96e29d 100644 --- a/src/roxy/config/defaults.xqy +++ b/src/roxy/config/defaults.xqy @@ -62,6 +62,7 @@ declare variable $ROXY-ROUTES := return + $1 @@ -118,4 +119,4 @@ declare variable $LABELS := - ; \ No newline at end of file + ; From 77b132253a345aeac6cae3deb2fa172a8c4d68e2 Mon Sep 17 00:00:00 2001 From: Dave Cassel Date: Tue, 12 Aug 2014 09:58:38 -0400 Subject: [PATCH 083/113] #266 JSHint cleanup --- src/test/js/tests.js | 228 +++++++++++++++++++++++-------------------- 1 file changed, 123 insertions(+), 105 deletions(-) diff --git a/src/test/js/tests.js b/src/test/js/tests.js index 5ba06f83..5fac069c 100644 --- a/src/test/js/tests.js +++ b/src/test/js/tests.js @@ -16,17 +16,21 @@ var queue = []; function disableParent(item, parentClass) { + 'use strict'; + var li = $(item).parents(parentClass); - if ($(item).is(":checked")) { - li.removeClass("disabled"); + if ($(item).is(':checked')) { + li.removeClass('disabled'); } else { - li.addClass("disabled"); + li.addClass('disabled'); } } function runNextTest() { + 'use strict'; + if (queue.length > 0) { runSuite(queue.pop()); } @@ -34,7 +38,7 @@ function runNextTest() { // compute total pass/fails var totalFails = 0; $('td.failed').each(function(){ - fails = parseInt($(this).text(), 10); + var fails = parseInt($(this).text(), 10); if (isNaN(fails)) { fails = 0; @@ -44,28 +48,28 @@ function runNextTest() { var totalPasses = 0; $('td.passed').each(function() { - passes = parseInt($(this).text(), 10); + var passes = parseInt($(this).text(), 10); if (isNaN(passes)) { passes = 0; } totalPasses += passes; }); - var passedText = (totalFails > 0) ? totalPasses : "all"; - passedText += " passed"; - $("#passed-count").text(passedText); + var passedText = (totalFails > 0) ? totalPasses : 'all'; + passedText += ' passed'; + $('#passed-count').text(passedText); if (totalFails > 0) { - $("#failed-count").text(totalFails + " failed"); + $('#failed-count').text(totalFails + ' failed'); } else { - $("#failed-count").text(""); + $('#failed-count').text(''); } - $(".canceltests").hide(); - $(".runtests").show(); + $('.canceltests').hide(); + $('.runtests').show(); - var txt = ""; + var txt = ''; if (totalPasses > 0 && totalFails <= 0) { txt = 'All Passed'; } @@ -89,7 +93,9 @@ function runNextTest() { } function renderError(error) { - result = + 'use strict'; + + var result = '
' + '
Fail
' + '
'; @@ -98,22 +104,22 @@ function renderError(error) { result += '

' + error + '

'; } else { - var formatString = error.find("[nodeName = 'error:format-string']").text(); + var formatString = error.find('[nodeName = "error:format-string"]').text(); if (!formatString) { - formatString = error.find("[nodeName = 'error:code']").text() + ': (' + error.find("[nodeName = 'error:name']").first().text() + ') ' + error.find("[nodeName = 'error:expr']").text(); - if (error.find("[nodeName = 'error:code']").text() !== error.find("[nodeName = 'error:message']").text()) { - formatString += ' -- ' + error.find("[nodeName = 'error:message']").text(); + formatString = error.find('[nodeName = "error:code"]').text() + ': (' + error.find('[nodeName = "error:name"]').first().text() + ') ' + error.find('[nodeName = "error:expr"]').text(); + if (error.find('[nodeName = "error:code"]').text() !== error.find('[nodeName = "error:message"]').text()) { + formatString += ' -- ' + error.find('[nodeName = "error:message"]').text(); } } - var datum = error.find("[nodeName = 'error:datum']").children(); + var datum = error.find('[nodeName = "error:datum"]').children(); if (datum.length <= 0) { datum = ''; } if (formatString && formatString.length > 0) { - result += '

' + formatString.replace(//, ">") + '

'; + result += '

' + formatString.replace(//, '>') + '

'; } result += @@ -124,22 +130,22 @@ function renderError(error) { '' + '