Merge branch release/v8.2.0 into master

.gitignore

@@ -12,3 +12,5 @@ tests/puppeteer/node_modules
 tests/puppeteer/work_directory
 tests/puppeteer/package.json
 tests/puppeteer/package-lock.json
+scripts/sdkjs_common/jsdoc/node_modules
+scripts/sdkjs_common/jsdoc/package-lock.json

configure.py

@@ -21,12 +21,13 @@
 parser.add_option("--external-folder", action="store", type="string", dest="external-folder", default="", help="defines a directory with external folder")
 parser.add_option("--sql-type", action="store", type="string", dest="sql-type", default="postgres", help="defines the sql type wich will be used")
 parser.add_option("--db-port", action="store", type="string", dest="db-port", default="5432", help="defines the sql db-port wich will be used")
+parser.add_option("--db-name", action="store", type="string", dest="db-name", default="onlyoffice", help="defines the sql db-name wich will be used")
 parser.add_option("--db-user", action="store", type="string", dest="db-user", default="onlyoffice", help="defines the sql db-user wich will be used")
 parser.add_option("--db-pass", action="store", type="string", dest="db-pass", default="onlyoffice", help="defines the sql db-pass wich will be used")
 parser.add_option("--compiler", action="store", type="string", dest="compiler", default="", help="defines compiler name. It is not recommended to use it as it's defined automatically (msvc2015, msvc2015_64, gcc, gcc_64, clang, clang_64, etc)")
 parser.add_option("--no-apps", action="store", type="string", dest="no-apps", default="0", help="disables building desktop apps that use qt")
 parser.add_option("--themesparams", action="store", type="string", dest="themesparams", default="", help="provides settings for generating presentation themes thumbnails")
-parser.add_option("--git-protocol", action="store", type="string", dest="git-protocol", default="https", help="can be used only if update is set to true - 'https', 'ssh'")
+parser.add_option("--git-protocol", action="store", type="string", dest="git-protocol", default="auto", help="can be used only if update is set to true - 'https', 'ssh'")
 parser.add_option("--branding", action="store", type="string", dest="branding", default="", help="provides branding path")
 parser.add_option("--branding-name", action="store", type="string", dest="branding-name", default="", help="provides branding name")
 parser.add_option("--branding-url", action="store", type="string", dest="branding-url", default="", help="provides branding url")

make_package.py

@@ -10,15 +10,17 @@
 # parse
 parser = argparse.ArgumentParser(description="Build packages.")
 parser.add_argument("-P", "--platform", dest="platform", type=str,
-                    action="store", help="Defines platform", required=True)
-parser.add_argument("-T", "--targets", dest="targets",   type=str, nargs="+",
-                    action="store", help="Defines targets",  required=True)
-parser.add_argument("-R", "--branding", dest="branding", type=str,
-                    action="store", help="Provides branding path")
+  action="store", help="Defines platform", required=True)
+parser.add_argument("-T", "--targets",  dest="targets",  type=str, nargs="+",
+  action="store", help="Defines targets",  required=True)
 parser.add_argument("-V", "--version",  dest="version",  type=str,
-                    action="store", help="Defines version")
+  action="store", help="Defines version")
 parser.add_argument("-B", "--build",    dest="build",    type=str,
-                    action="store", help="Defines build")
+  action="store", help="Defines build")
+parser.add_argument("-H", "--branch",   dest="branch",   type=str,
+  action="store", help="Defines branch")
+parser.add_argument("-R", "--branding", dest="branding", type=str,
+  action="store", help="Provides branding path")
 args = parser.parse_args()
 
 # vars
@@ -29,8 +31,16 @@
 common.clean = "clean" in args.targets
 common.sign = "sign" in args.targets
 common.deploy = "deploy" in args.targets
-common.version = args.version if args.version else utils.get_env("BUILD_VERSION", "0.0.0")
-common.build = args.build if args.build else utils.get_env("BUILD_NUMBER", "0")
+if args.version: common.version = args.version
+else:            common.version = utils.get_env("PRODUCT_VERSION", "0.0.0")
+utils.set_env("PRODUCT_VERSION", common.version)
+utils.set_env("BUILD_VERSION", common.version)
+if args.build: common.build = args.build
+else:          common.build = utils.get_env("BUILD_NUMBER", "0")
+utils.set_env("BUILD_NUMBER", common.build)
+if args.branch: common.branch = args.branch
+else:           common.branch = utils.get_env("BRANCH_NAME", "null")
+utils.set_env("BRANCH_NAME", common.branch)
 common.branding = args.branding
 common.timestamp = utils.get_timestamp()
 common.workspace_dir = utils.get_abspath(utils.get_script_dir(__file__) + "/..")

scripts/base.py

@@ -39,6 +39,9 @@ def is_os_arm():
     return False
   return True
 
+def get_platform():
+  return platform.machine().lower()
+
 def is_python_64bit():
   return (struct.calcsize("P") == 8)
 
@@ -384,7 +387,7 @@ def cmd2(prog, args=[], is_no_errors=False):
     sys.exit("Error (" + prog + "): " + str(ret))
   return ret
 
-def cmd_exe(prog, args):
+def cmd_exe(prog, args, is_no_errors=False):
   prog_dir = os.path.dirname(prog)
   env_dir = os.environ
   if ("linux" == host_platform()):
@@ -406,7 +409,7 @@ def cmd_exe(prog, args):
       command += (" \"" + arg + "\"")
     process = subprocess.Popen(command, stderr=subprocess.STDOUT, shell=True, env=env_dir)
     ret = process.wait()
-  if ret != 0:
+  if ret != 0 and True != is_no_errors:
     sys.exit("Error (" + prog + "): " + str(ret))
   return ret
 
@@ -426,12 +429,13 @@ def cmd_and_return_cwd(prog, args=[], is_no_errors=False):
 
 def run_command(sCommand):
   popen = subprocess.Popen(sCommand, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-  result = {'stdout' : '', 'stderr' : ''}
+  result = {'stdout' : '', 'stderr' : '', 'returncode' : 0}
   try:
     stdout, stderr = popen.communicate()
     popen.wait()
     result['stdout'] = stdout.strip().decode('utf-8', errors='ignore')
     result['stderr'] = stderr.strip().decode('utf-8', errors='ignore')
+    result['returncode'] = popen.returncode
   finally:
     popen.stdout.close()
     popen.stderr.close()
@@ -496,12 +500,37 @@ def set_cwd(dir):
   return
 
 # git ---------------------------------------------------
+def git_get_origin():
+  cur_dir = os.getcwd()
+  os.chdir(get_script_dir() + "/../")
+  ret = run_command("git config --get remote.origin.url")["stdout"]
+  os.chdir(cur_dir)
+  return ret
+
+def git_is_ssh():
+  git_protocol = config.option("git-protocol")
+  if (git_protocol == "https"):
+    return False
+  if (git_protocol == "ssh"):
+    return True
+  origin = git_get_origin()
+  if (git_protocol == "auto") and (origin.find(":ONLYOFFICE/") != -1):
+    return True
+  return False
+
+def get_ssh_base_url():
+  cur_origin = git_get_origin()
+  ind = cur_origin.find(":ONLYOFFICE/")
+  if (ind == -1):
+    return "git@github.com:ONLYOFFICE/"
+  return cur_origin[:ind+12]
+
 def git_update(repo, is_no_errors=False, is_current_dir=False, git_owner=""):
   print("[git] update: " + repo)
   owner = git_owner if git_owner else "ONLYOFFICE"
   url = "https://github.com/" + owner + "/" + repo + ".git"
-  if config.option("git-protocol") == "ssh":
-    url = "git@github.com:ONLYOFFICE/" + repo + ".git"
+  if git_is_ssh():
+    url = get_ssh_base_url() + repo + ".git"
   folder = get_script_dir() + "/../../" + repo
   if is_current_dir:
     folder = repo
@@ -534,10 +563,12 @@ def get_repositories():
   result.update(get_sdkjs_addons())
   result["onlyoffice.github.io"] = [False, False]
   result["web-apps"] = [False, False]
-  result.update(get_web_apps_addons())
   result["dictionaries"] = [False, False]
   result["core-fonts"] = [False, False]
 
+  if config.check_option("module", "server"):
+    result.update(get_web_apps_addons())
+
   if config.check_option("module", "builder"):
     result["document-templates"] = [False, False]
 
@@ -570,8 +601,8 @@ def get_branding_repositories(checker):
 def create_pull_request(branches_to, repo, is_no_errors=False, is_current_dir=False):
   print("[git] create pull request: " + repo)
   url = "https://github.com/ONLYOFFICE/" + repo + ".git"
-  if config.option("git-protocol") == "ssh":
-    url = "git@github.com:ONLYOFFICE/" + repo + ".git"
+  if git_is_ssh():
+    url = get_ssh_base_url() + repo + ".git"
   folder = get_script_dir() + "/../../" + repo
   if is_current_dir:
     folder = repo
@@ -738,8 +769,9 @@ def qt_config(platform):
   if (-1 != platform.find("xp")):
     config_param += " build_xp"
   if ("ios" == platform):
-    set_env("BITCODE_GENERATION_MODE", "bitcode")
-    set_env("ENABLE_BITCODE", "YES")
+    if (config.check_option("bitcode", "yes")):
+      set_env("BITCODE_GENERATION_MODE", "bitcode")
+      set_env("ENABLE_BITCODE", "YES")
     config_param = config_param.replace("desktop", "")
     config_param += " iphoneos device"
     if (-1 == config_param_lower.find("debug")):
@@ -1046,15 +1078,15 @@ def web_apps_addons_param():
 def download(url, dst):
   return cmd_exe("curl", ["-L", "-o", dst, url])
 
-def extract(src, dst):
+def extract(src, dst, is_no_errors=False):
   app = "7za" if ("mac" == host_platform()) else "7z"
-  return cmd_exe(app, ["x", "-y", src, "-o" + dst])
+  return cmd_exe(app, ["x", "-y", src, "-o" + dst], is_no_errors)
 
-def extract_unicode(src, dst):
+def extract_unicode(src, dst, is_no_errors=False):
   if "windows" == host_platform():
     run_as_bat_win_isolate([u"chcp 65001", u"call 7z.exe x -y \"" + src + u"\" \"-o" + dst + u"\"", u"exit"])
     return
-  return extract(src, dst)
+  return extract(src, dst, is_no_errors)
 
 def archive_folder(src, dst):
   app = "7za" if ("mac" == host_platform()) else "7z"
@@ -1187,7 +1219,7 @@ def mac_correct_rpath_x2t(dir):
   mac_correct_rpath_library("kernel", ["UnicodeConverter"])
   mac_correct_rpath_library("kernel_network", ["UnicodeConverter", "kernel"])
   mac_correct_rpath_library("graphics", ["UnicodeConverter", "kernel"])
-  mac_correct_rpath_library("doctrenderer", ["UnicodeConverter", "kernel", "kernel_network", "graphics"])
+  mac_correct_rpath_library("doctrenderer", ["UnicodeConverter", "kernel", "kernel_network", "graphics", "PdfFile", "XpsFile", "DjVuFile", "DocxRenderer"])
   mac_correct_rpath_library("HtmlFile2", ["UnicodeConverter", "kernel", "kernel_network", "graphics"])
   mac_correct_rpath_library("EpubFile", ["UnicodeConverter", "kernel", "HtmlFile2", "graphics"])
   mac_correct_rpath_library("Fb2File", ["UnicodeConverter", "kernel", "graphics"])
@@ -1206,7 +1238,7 @@ def mac_correct_rpath_x2t(dir):
   if is_file("./allthemesgen"):
     cmd("chmod", ["-v", "+x", "./allthemesgen"])
     cmd("install_name_tool", ["-add_rpath", "@executable_path", "./allthemesgen"], True)
-    mac_correct_rpath_binary("./allthemesgen", ["icudata.58", "icuuc.58", "UnicodeConverter", "kernel", "graphics", "kernel_network", "doctrenderer"])
+    mac_correct_rpath_binary("./allthemesgen", ["icudata.58", "icuuc.58", "UnicodeConverter", "kernel", "graphics", "kernel_network", "doctrenderer", "PdfFile", "XpsFile", "DjVuFile", "DocxRenderer"])
   if is_file("./pluginsmanager"):
     cmd("chmod", ["-v", "+x", "./pluginsmanager"])
     cmd("install_name_tool", ["-add_rpath", "@executable_path", "./pluginsmanager"], True)
@@ -1224,8 +1256,13 @@ def mac_correct_rpath_docbuilder(dir):
   cmd("chmod", ["-v", "+x", "./docbuilder"])
   cmd("install_name_tool", ["-add_rpath", "@executable_path", "./docbuilder"], True)
   mac_correct_rpath_binary("./docbuilder", ["icudata.58", "icuuc.58", "UnicodeConverter", "kernel", "kernel_network", "graphics", "PdfFile", "HtmlRenderer", "XpsFile", "DjVuFile", "HtmlFile2", "Fb2File", "EpubFile", "doctrenderer", "DocxRenderer"])  
-  mac_correct_rpath_library("docbuilder.c", ["icudata.58", "icuuc.58", "UnicodeConverter", "kernel", "kernel_network", "graphics", "doctrenderer"])
-  cmd("install_name_tool", ["-add_rpath", "@loader_path", "libdocbuilder.c.dylib"], True)
+  mac_correct_rpath_library("docbuilder.c", ["icudata.58", "icuuc.58", "UnicodeConverter", "kernel", "kernel_network", "graphics", "doctrenderer", "PdfFile", "XpsFile", "DjVuFile", "DocxRenderer"])
+
+  def add_loader_path_to_rpath(libs):
+    for lib in libs:
+      cmd("install_name_tool", ["-add_rpath", "@loader_path", "lib" + lib + ".dylib"], True)
+
+  add_loader_path_to_rpath(["icuuc.58", "UnicodeConverter", "kernel", "kernel_network", "graphics", "doctrenderer", "PdfFile", "XpsFile", "DjVuFile", "DocxRenderer", "docbuilder.c"])
   os.chdir(cur_dir)
   return
 
@@ -1271,7 +1308,7 @@ def linux_set_origin_rpath_libraries(dir, libs):
   return
 
 def linux_correct_rpath_docbuilder(dir):
-  linux_set_origin_rpath_libraries(dir, ["docbuilder.c.so", "icuuc.so.58", "doctrenderer.so", "graphics.so", "kernel.so", "kernel_network.so", "UnicodeConverter.so"])
+  linux_set_origin_rpath_libraries(dir, ["docbuilder.jni.so", "docbuilder.c.so", "icuuc.so.58", "doctrenderer.so", "graphics.so", "kernel.so", "kernel_network.so", "UnicodeConverter.so", "PdfFile.so", "XpsFile.so", "DjVuFile.so", "DocxRenderer.so"])
   return
 
 def common_check_version(name, good_version, clean_func):
@@ -1327,7 +1364,7 @@ def copy_marketplace_plugin(dst_dir, is_name_as_guid=False, is_desktop_local=Fal
   git_dir = __file__script__path__ + "/../.."
   if False:
     # old version
-    base.copy_sdkjs_plugin(git_dir + "/desktop-sdk/ChromiumBasedEditors/plugins", dst_dir, "manager", is_name_as_guid, is_desktop_local)
+    copy_sdkjs_plugin(git_dir + "/desktop-sdk/ChromiumBasedEditors/plugins", dst_dir, "manager", is_name_as_guid, is_desktop_local)
     return
   src_dir_path = git_dir + "/onlyoffice.github.io/store/plugin"
   name = "marketplace"
@@ -1393,6 +1430,7 @@ def support_old_versions_plugins(out_dir):
 def generate_sdkjs_plugin_list(dst):
   plugins_list = config.option("sdkjs-plugin").rsplit(", ") \
                + config.option("sdkjs-plugin-server").rsplit(", ")
+  plugins_list = list(filter(None, plugins_list))
   with open(get_path(dst), 'w') as file:
     dump = json.dumps(sorted(plugins_list), indent=4)
     file.write(re.sub(r"^(\s{4})", '\t', dump, 0, re.MULTILINE))
@@ -1754,3 +1792,42 @@ def apply_patch(file, patch):
   #file_content_new = "\n#if 0" + file_content_old + "#else" + file_content_new + "#endif\n"
   replaceInFile(file, file_content_old, file_content_new)
   return
+
+def get_autobuild_version(product, platform="", branch="", build=""):
+  download_platform = platform
+  if ("" == download_platform):
+    osType = get_platform()
+    isArm = True if (-1 != osType.find("arm")) else False
+    is64 = True if (osType.endswith("64")) else False
+
+    if ("windows" == host_platform()):
+      download_platform = "win-"
+    elif ("linux" == host_platform()):
+      download_platform = "linux-"
+    else:
+      download_platform = "mac-"
+
+    download_platform += ("arm" if isArm else "")
+    download_platform += ("64" if is64 else "32")
+  else:
+    download_platform = download_platform.replace("_", "-")
+
+  download_build = build
+  if ("" == download_build):
+    download_build = "latest"
+
+  download_branch = branch
+  if ("" == download_branch):
+    download_branch = "develop"
+
+  download_addon = download_branch + "/" + download_build + "/" + product + "-" + download_platform + ".7z"
+  return "http://repo-doc-onlyoffice-com.s3.amazonaws.com/archive/" + download_addon
+
+def create_x2t_js_cache(dir, product):
+  if is_file(dir + "/libdoctrenderer.dylib") and (os.path.getsize(dir + "/libdoctrenderer.dylib") < 5*1024*1024):
+    return
+
+  if (product in ["builder", "server"]):
+    cmd_in_dir(dir, "./x2t", ["-create-js-cache"], True)
+  cmd_in_dir(dir, "./x2t", ["-create-js-snapshots"], True)
+  return

scripts/build_js.py

@@ -146,15 +146,25 @@ def build_sdk_native(directory, minimize=True):
   _run_grunt(directory, get_build_param(minimize) + ["--mobile=true"] + addons)
   return
 
-def build_js_develop(root_dir):
-  #_run_npm_cli(root_dir + "/sdkjs/build")
+
+def build_sdkjs_develop(root_dir):
   external_folder = config.option("--external-folder")
   if (external_folder != ""):
     external_folder = "/" + external_folder
-    
+
   _run_npm_ci(root_dir + external_folder + "/sdkjs/build")
   _run_grunt(root_dir + external_folder + "/sdkjs/build", get_build_param(False) + base.sdkjs_addons_param())
   _run_grunt(root_dir + external_folder + "/sdkjs/build", ["develop"] + base.sdkjs_addons_param())
+
+
+def build_js_develop(root_dir):
+  #_run_npm_cli(root_dir + "/sdkjs/build")
+  external_folder = config.option("--external-folder")
+  if (external_folder != ""):
+    external_folder = "/" + external_folder
+
+  build_sdkjs_develop(root_dir)
+
   _run_npm(root_dir + external_folder + "/web-apps/build")
   _run_npm_ci(root_dir + external_folder + "/web-apps/build/sprites")
   _run_grunt(root_dir + external_folder + "/web-apps/build/sprites", [])

scripts/build_sln.py

@@ -46,4 +46,16 @@ def make(solution=""):
     else:
       base.make_sln_project("../core/DesktopEditor/doctrenderer/docbuilder.com/src", "docbuilder.com.sln")
     base.restorePathForBuilder(new_replace_path)
+
+  # build Java docbuilder wrapper
+  if config.check_option("module", "builder") and "onlyoffice" == config.branding():
+    for platform in platforms:
+      if not platform in config.platforms:
+        continue
+
+      # build JNI library
+      qmake.make(platform, base.get_script_dir() + "/../../core/DesktopEditor/doctrenderer/docbuilder.java/src/jni/docbuilder_jni.pro", "", True)
+      # build Java code to JAR
+      base.cmd_in_dir(base.get_script_dir() + "/../../core/DesktopEditor/doctrenderer/docbuilder.java", "python", ["make.py"])
+
   return

scripts/core_common/modules/hunspell.py

@@ -3,6 +3,11 @@
 import base
 import os
 
+def clean():
+  if base.is_dir("hunspell"):
+    base.delete_dir_with_access_error("hunspell")
+  return
+
 def make(build_js = True):
 
   old_cur_dir = os.getcwd()
@@ -11,6 +16,8 @@ def make(build_js = True):
   core_common_dir = base.get_script_dir() + "/../../core/Common"
 
   os.chdir(core_common_dir + "/3dParty/hunspell")
+
+  base.common_check_version("hunspell", "1", clean)
   base.cmd("python", ["./before.py"])
 
   if (build_js):