diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ffd577d35..144696978b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,8 @@ linux-builder: paths: - build/*.AppImage - build/*.torrent - - build/install-x64/share/ + - build/install-x64/share/*.env + - build/install-x64/share/*.log - build/build-server.log script: - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder" @@ -21,7 +22,7 @@ linux-builder: - cp -r "$CI_PROJECT_DIR/build/install-x64/python/." "$CI_PROJECT_DIR" - export LD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:$LD_LIBRARY_PATH - VERSION=$(python3 src/launch.py -V) - - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME" + - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME.env" - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log" - cd doc; make html SPHINXOPTS="-D html_theme_options.analytics_id=UA-4381101-5"; cd ..; - ~/auto-update-sphinx "$CI_PROJECT_DIR/build" "$CI_COMMIT_REF_NAME" @@ -41,7 +42,8 @@ mac-builder: paths: - build/*.dmg - build/*.torrent - - build/install-x64/share/ + - build/install-x64/share/*.env + - build/install-x64/share/*.log - build/build-server.log script: - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=mac-builder" @@ -65,7 +67,7 @@ mac-builder: - export LD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:$LD_LIBRARY_PATH - export DYLD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:DYLD_LIBRARY_PATH - VERSION=$(python3 src/launch.py -V) - - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME" + - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME.env" - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py bdist_mac --git-branch=$CI_COMMIT_REF_NAME --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME" @@ -82,17 +84,24 @@ windows-builder-x64: paths: - build\*.exe - build\*.torrent - - build\install-x64\share\ + - build\install-x64\share\*.env + - build\install-x64\share\*.log - build\build-server.log script: - try { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } catch { $_.Exception.Response.StatusCode.Value__ } - if (-not (Test-Path "artifacts.zip")) { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/develop/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } - Expand-Archive -Path artifacts.zip -DestinationPath . - Copy-Item "$CI_PROJECT_DIR/build/install-x64/python/*" -Destination "$CI_PROJECT_DIR" - - $env:Path = "$CI_PROJECT_DIR\build\install-x64\lib;$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\bin;C:\msys64\mingw64\lib;C:\msys64\mingw64\lib\python3.7\;C:\msys64\mingw64\lib\python3.7\site-packages\;C:\msys64\mingw64\lib\python3.7\site-packages\PyQt5;C:\msys64\usr\lib\cmake\UnitTest++;C:\msys64\usr;C:\msys64\usr\lib;C:\msys64\usr\local\x64\mingw\bin;C:\msys64\usr\local;" + $env:Path; - - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\usr\lib;C:\msys64\usr\lib\site-packages\;C:\msys64\mingw64\lib\python3.7\;C:\msys64\mingw64\lib\python3.7\site-packages\;C:\msys64\mingw64\lib\python3.7\site-packages\PyQt5;"; + - $env:MSYSTEM = "MINGW64" + - $originalPath = $env:Path + - $NewPath = "$CI_PROJECT_DIR\build\install-x64\bin;$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\bin;" + - $env:Path = $NewPath + $env:Path + - $PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python" + $PY_ABI + "\;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\PyQt5;"; + - $env:Path = $NewPath + $originalPath + - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\PyQt5;" - $VERSION=(python3 src/launch.py -V) - - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force + - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME.env" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME @@ -110,21 +119,29 @@ windows-builder-x86: paths: - build\*.exe - build\*.torrent - - build\install-x86\share\ + - build\install-x86\share\*.env + - build\install-x86\share\*.log - build\build-server.log script: - try { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } catch { $_.Exception.Response.StatusCode.Value__ } - if (-not (Test-Path "artifacts.zip")) { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/develop/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } - Expand-Archive -Path artifacts.zip -DestinationPath . - Copy-Item "$CI_PROJECT_DIR/build/install-x86/python/*" -Destination "$CI_PROJECT_DIR" - - $env:Path = "$CI_PROJECT_DIR\build\install-x86\lib;$CI_PROJECT_DIR\build\install-x86\python;C:\msys32\mingw32\bin;C:\msys32\mingw32\lib;C:\msys32\mingw32\lib\python3.7\;C:\msys32\mingw32\lib\python3.7\site-packages\;C:\msys32\mingw32\lib\python3.7\site-packages\PyQt5;C:\msys32\usr\lib\cmake\UnitTest++;C:\msys32\usr;C:\msys32\usr\lib;C:\msys32\usr\local\x64\mingw\bin;C:\msys32\usr\local;" + $env:Path; - - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;C:\msys32\usr\lib;C:\msys32\usr\lib\site-packages\;C:\msys32\mingw32\lib\python3.7\;C:\msys32\mingw32\lib\python3.7\site-packages\;C:\msys32\mingw32\lib\python3.7\site-packages\PyQt5;"; + - $env:MSYSTEM = "MINGW32" + - $originalPath = $env:Path + - $NewPath = "$CI_PROJECT_DIR\build\install-x86\bin;$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\bin;" + - $env:Path = $NewPath + $env:Path + - $PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python" + $PY_ABI + "\;C:\msys64\mingw32\lib\python" + $PY_ABI + "\site-packages\;C:\msys64\mingw32\lib\python " + $PY_ABI+ "\site-packages\PyQt5;"; + - $env:Path = $NewPath + $originalPath + - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\lib\python" + $PY_ABI + "\site-packages\PyQt5;" - $VERSION=(python3 src/launch.py -V) - - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force + - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME.env" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME - - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-3.7\openshot-qt.exe" + - $EXE_PATH = "$CI_PROJECT_DIR\build\exe.mingw-" + $PY_ABI + "\openshot-qt.exe" + - editbin /LARGEADDRESSAWARE "$EXE_PATH" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME" when: always except: diff --git a/freeze.py b/freeze.py index 36cd55e0a1..c138256986 100644 --- a/freeze.py +++ b/freeze.py @@ -91,7 +91,7 @@ "requests", "zmq", "webbrowser", - "json" + "json", ] # Modules to include @@ -203,7 +203,11 @@ def find_files(directory, patterns): src_files.append((os.path.join(PATH, "installer", "launch-win.bat"), "launch-win.bat")) # Add additional package - python_packages.append('idna') + python_packages.extend([ + "idna", + "OpenGL", + "OpenGL_accelerate", + ]) # Manually add zmq dependency (windows does not freeze it correctly) import zmq diff --git a/installer/build-mac-dmg.sh b/installer/build-mac-dmg.sh index 793f0fc2ae..daf2d66134 100644 --- a/installer/build-mac-dmg.sh +++ b/installer/build-mac-dmg.sh @@ -1,4 +1,7 @@ #!/bin/sh + +# XXX: These paths should be set using `brew prefix` commands, +# for future-proofing against upgrades PATH=/usr/local/Cellar/python@3.7/3.7.9_2/Frameworks/Python.framework/Versions/3.7/bin:/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/qt5/5.5/clang_64/bin:/opt/X11/bin # Get Version diff --git a/installer/build_server.py b/installer/build_server.py index 61020c961a..2bfb178f0f 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -30,10 +30,12 @@ import datetime import platform -import shutil import re +import shutil +import shlex import stat import subprocess +import sysconfig import traceback from github3 import login from requests.auth import HTTPBasicAuth @@ -41,6 +43,7 @@ from version_parser import parse_version_info, parse_build_name PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Primary openshot folder +PY_ABI = sysconfig.get_config_var('py_version_short') # Access info class (for version info) sys.path.append(os.path.join(PATH, 'src', 'classes')) @@ -78,11 +81,14 @@ def output(line): def run_command(command, working_dir=None): """Utility function to return output from command line""" - short_command = command.split('" ')[0] # We don't need to print args + short_command = shlex.split(command)[0] # We don't need to print args output("Running %s... (%s)" % (short_command, working_dir)) - p = subprocess.Popen(command, shell=True, cwd=working_dir, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + p = subprocess.Popen( + command, + shell=True, + cwd=working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) return iter(p.stdout.readline, b"") @@ -148,7 +154,11 @@ def get_release(repo, tag_name): @param repo: github3 repository object @returns: github3 release object or None """ - for release in repo.iter_releases(): + if hasattr(repo, 'releases'): + release_iter = repo.releases() + else: + release_iter = repo.iter_releases() + for release in release_iter: if release.tag_name == tag_name: return release @@ -187,7 +197,7 @@ def upload(file_path, github_release): return url -if __name__ == "__main__": +def main(): # Only run this code when directly executing this script. Parts of this file # are also used in the deploy.py script. try: @@ -215,7 +225,12 @@ def upload(file_path, github_release): git_branch_name = sys.argv[7] # Start log - output("%s Build Log for %s (branch: %s)" % (platform.system(), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), git_branch_name)) + output( + "%s Build Log for %s (branch: %s)" % ( + platform.system(), + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + git_branch_name) + ) # Detect artifact folder (if any) artifact_path = os.path.join(PATH, "build", "install-x64") @@ -227,7 +242,9 @@ def upload(file_path, github_release): # Parse artifact version files (if found) for repo_name in ["libopenshot-audio", "libopenshot", "openshot-qt"]: - version_info.update(parse_version_info(os.path.join(artifact_path, "share", repo_name))) + data_file = f"{repo_name}.env" + version_info.update( + parse_version_info(os.path.join(artifact_path, "share", data_file))) output(str(version_info)) # Get GIT description of openshot-qt-git branch (i.e. v2.0.6-18-ga01a98c) @@ -279,12 +296,13 @@ def upload(file_path, github_release): os.makedirs(os.path.join(app_dir_path, "usr"), exist_ok=True) # XDG Freedesktop icon paths - icons = [ ("scalable", os.path.join(PATH, "xdg", "openshot-qt.svg")), - ("64x64", os.path.join(PATH, "xdg", "icon", "64", "openshot-qt.png")), - ("128x128", os.path.join(PATH, "xdg", "icon", "128", "openshot-qt.png")), - ("256x256", os.path.join(PATH, "xdg", "icon", "256", "openshot-qt.png")), - ("512x512", os.path.join(PATH, "xdg", "icon", "512", "openshot-qt.png")), - ] + icons = [ + ("scalable", os.path.join(PATH, "xdg", "openshot-qt.svg")), + ("64x64", os.path.join(PATH, "xdg", "icon", "64", "openshot-qt.png")), + ("128x128", os.path.join(PATH, "xdg", "icon", "128", "openshot-qt.png")), + ("256x256", os.path.join(PATH, "xdg", "icon", "256", "openshot-qt.png")), + ("512x512", os.path.join(PATH, "xdg", "icon", "512", "openshot-qt.png")), + ] # Copy desktop icons icon_theme_path = os.path.join(app_dir_path, "usr", "share", "icons", "hicolor") @@ -348,7 +366,11 @@ def upload(file_path, github_release): # Create AppImage (OpenShot-%s-x86_64.AppImage) app_image_success = False - for line in run_command('/home/ubuntu/apps/AppImageKit/AppImageAssistant "%s" "%s"' % (app_dir_path, app_build_path)): + for line in run_command(" ".join([ + '/home/ubuntu/apps/AppImageKit/AppImageAssistant', + '"%s"' % app_dir_path, + '"%s"' % app_build_path, + ])): output(line) app_image_success = os.path.exists(app_build_path) @@ -361,23 +383,28 @@ def upload(file_path, github_release): # Delete build (since something failed) os.remove(app_build_path) - if platform.system() == "Darwin": - # Create DMG (OpenShot-%s-x86_64.DMG) app_image_success = False # Build app.bundle and create DMG for line in run_command("bash installer/build-mac-dmg.sh"): output(line) - if ("error".encode("UTF-8") in line and "No errors".encode("UTF-8") not in line) or "rejected".encode("UTF-8") in line: + if ( + ("error".encode("UTF-8") in line + and "No errors".encode("UTF-8") not in line) + or "rejected".encode("UTF-8") in line + ): error("Build-Mac-DMG Error: %s" % line) if "Your image is ready".encode("UTF-8") in line: app_image_success = True # Rename DMG (to be consistent with other OS installers) for dmg_path in os.listdir(os.path.join(PATH, "build")): - if os.path.isfile(os.path.join(PATH, "build", dmg_path)) and dmg_path.endswith(".dmg"): + if ( + os.path.isfile(os.path.join(PATH, "build", dmg_path)) + and dmg_path.endswith(".dmg") + ): os.rename(os.path.join(PATH, "build", dmg_path), app_build_path) # Was the DMG creation successful @@ -389,12 +416,10 @@ def upload(file_path, github_release): # Delete build (since key signing might have failed) os.remove(app_build_path) - if platform.system() == "Windows": - # Move python folder structure, since Cx_Freeze doesn't put it in the correct place - exe_dir = os.path.join(PATH, 'build', 'exe.mingw-3.7') - python_dir = os.path.join(exe_dir, 'lib', 'python3.7') + exe_dir = os.path.join(PATH, 'build', 'exe.mingw-{}'.format(PY_ABI)) + python_dir = os.path.join(exe_dir, 'lib', 'python{}'.format(PY_ABI)) # Remove a redundant openshot_qt module folder (duplicates lots of files) duplicate_openshot_qt_path = os.path.join(python_dir, 'openshot_qt') @@ -402,7 +427,14 @@ def upload(file_path, github_release): shutil.rmtree(duplicate_openshot_qt_path, True) # Remove the following paths. cx_Freeze is including many unneeded files. This prunes them out. - paths_to_delete = ['mediaservice', 'imageformats', 'platforms', 'printsupport', 'lib/openshot_qt', 'resvg.dll'] + paths_to_delete = [ + 'mediaservice', + 'imageformats', + 'platforms', + 'printsupport', + 'lib/openshot_qt', + 'resvg.dll', + ] for delete_path in paths_to_delete: full_delete_path = os.path.join(exe_dir, delete_path) output("Delete path: %s" % full_delete_path) @@ -420,14 +452,24 @@ def upload(file_path, github_release): paths_to_replace = ['imageformats', 'platforms'] for replace_name in paths_to_replace: if windows_32bit: - shutil.copytree(os.path.join('C:\\msys32\\mingw32\\share\\qt5\\plugins', replace_name), os.path.join(exe_dir, replace_name)) + shutil.copytree( + os.path.join('C:\\msys64\\mingw32\\share\\qt5\\plugins', replace_name), + os.path.join(exe_dir, replace_name)) else: - shutil.copytree(os.path.join('C:\\msys64\\mingw64\\share\\qt5\\plugins', replace_name), os.path.join(exe_dir, replace_name)) + shutil.copytree( + os.path.join('C:\\msys64\\mingw64\\share\\qt5\\plugins', replace_name), + os.path.join(exe_dir, replace_name)) # Copy Qt5Core.dll, Qt5Svg.dll to root of frozen directory - paths_to_copy = [("Qt5Core.dll", "C:\\msys64\\mingw64\\bin\\"), ("Qt5Svg.dll", "C:\\msys64\\mingw64\\bin\\")] + paths_to_copy = [ + ("Qt5Core.dll", "C:\\msys64\\mingw64\\bin\\"), + ("Qt5Svg.dll", "C:\\msys64\\mingw64\\bin\\"), + ] if windows_32bit: - paths_to_copy = [("Qt5Core.dll", "C:\\msys32\\mingw32\\bin\\"), ("Qt5Svg.dll", "C:\\msys32\\mingw32\\bin\\")] + paths_to_copy = [ + ("Qt5Core.dll", "C:\\msys64\\mingw32\\bin\\"), + ("Qt5Svg.dll", "C:\\msys64\\mingw32\\bin\\"), + ] for qt_file_name, qt_parent_path in paths_to_copy: qt5_path = os.path.join(qt_parent_path, qt_file_name) new_qt5_path = os.path.join(exe_dir, qt_file_name) @@ -452,7 +494,17 @@ def upload(file_path, github_release): # Add version metadata to frozen app launcher launcher_exe = os.path.join(exe_dir, "openshot-qt.exe") verpatch_success = True - verpatch_command = '"verpatch.exe" "{}" /va /high "{}" /pv "{}" /s product "{}" /s company "{}" /s copyright "{}" /s desc "{}"'.format(launcher_exe, info.VERSION, info.VERSION, info.PRODUCT_NAME, info.COMPANY_NAME, info.COPYRIGHT, info.PRODUCT_NAME) + verpatch_command = " ".join([ + 'verpatch.exe', + '{}'.format(launcher_exe), + '/va', + '/high "{}"'.format(info.VERSION), + '/pv "{}"'.format(info.VERSION), + '/s product "{}"'.format(info.PRODUCT_NAME), + '/s company "{}"'.format(info.COMPANY_NAME), + '/s copyright "{}"'.format(info.COPYRIGHT), + '/s desc "{}"'.format(info.PRODUCT_NAME), + ]) verpatch_output = "" # version-stamp executable for line in run_command(verpatch_command): @@ -472,7 +524,14 @@ def upload(file_path, github_release): # Create Installer (OpenShot-%s-x86_64.exe) inno_success = True - inno_command = '"iscc.exe" /Q /DVERSION=%s /DONLY_64_BIT=%s "%s"' % (version, only_64_bit, os.path.join(PATH, 'installer', 'windows-installer.iss')) + inno_command = " ".join([ + 'iscc.exe', + '/Q', + '/DVERSION=%s' % version, + '/DONLY_64_BIT=%s' % only_64_bit, + '/DPY_EXE_DIR=%s' % "exe.mingw-{}".format(PY_ABI), + '"%s"' % os.path.join(PATH, 'installer', 'windows-installer.iss'), + ]) inno_output = "" # Compile Inno installer for line in run_command(inno_command): @@ -495,7 +554,14 @@ def upload(file_path, github_release): # Sign the installer key_sign_success = True - key_sign_command = '"kSignCMD.exe" /f "%s%s" /p "%s" /d "OpenShot Video Editor" /du "http://www.openshot.org" "%s"' % (windows_key, only_64_bit, windows_key_password, app_build_path) + key_sign_command = " ".join([ + 'kSignCMD.exe', + '/f "%s%s"' % (windows_key, only_64_bit), + '/p "%s"' % windows_key_password, + '/d "OpenShot Video Editor"', + '/du "http://www.openshot.org"', + '"%s"' % app_build_path, + ]) key_sign_output = "" # Sign MSI for line in run_command(key_sign_command): @@ -513,7 +579,6 @@ def upload(file_path, github_release): # Delete build (since key signing might have failed) os.remove(app_build_path) - # Upload Installer to GitHub (if build path exists) if needs_upload and os.path.exists(app_build_path): # Upload file to GitHub @@ -522,7 +587,19 @@ def upload(file_path, github_release): # Create torrent and upload torrent_path = "%s.torrent" % app_build_path - torrent_command = 'mktorrent -a "udp://tracker.openbittorrent.com:80/announce, udp://tracker.publicbt.com:80/announce, udp://tracker.opentrackr.org:1337" -c "OpenShot Video Editor %s" -w "%s" -o "%s" "%s"' % (version, download_url, "%s.torrent" % app_name, app_name) + tracker_list = [ + "udp://tracker.openbittorrent.com:80/announce", + "udp://tracker.publicbt.com:80/announce", + "udp://tracker.opentrackr.org:1337", + ] + torrent_command = " ".join([ + 'mktorrent', + '-a "%s"' % (", ".join(tracker_list)), + '-c "OpenShot Video Editor %s"' % version, + '-w "%s"' % download_url, + '-o "%s"' % ("%s.torrent" % app_name), + '"%s"' % app_name, + ]) torrent_output = "" # Remove existing torrents (if any found) @@ -544,7 +621,10 @@ def upload(file_path, github_release): url = upload(torrent_path, github_release) # Notify Zulip - zulip_upload_log(zulip_token, log, "%s: Build logs for %s" % (platform.system(), app_name), "Successful *%s* build: %s" % (git_branch_name, download_url)) + zulip_upload_log( + zulip_token, log, + "%s: Build logs for %s" % (platform.system(), app_name), + "Successful *%s* build: %s" % (git_branch_name, download_url)) except Exception as ex: tb = traceback.format_exc() @@ -555,5 +635,12 @@ def upload(file_path, github_release): else: # Report any errors detected output("build-server script failed!") - zulip_upload_log(zulip_token, log, "%s: Error log for *%s* build" % (platform.system(), git_branch_name), ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) + zulip_upload_log( + zulip_token, log, + "%s: Error log for *%s* build" % (platform.system(), git_branch_name), + ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) exit(1) + + +if __name__ == "__main__": + main() diff --git a/installer/deploy.py b/installer/deploy.py index 1a9b5ad6ae..9335bf0d70 100644 --- a/installer/deploy.py +++ b/installer/deploy.py @@ -35,8 +35,11 @@ from github3 import login from requests.auth import HTTPBasicAuth from requests import post, get, head -from build_server import output, run_command, error, truncate, zulip_upload_log, get_release, upload, \ - errors_detected, log, version_info, parse_version_info +from build_server import ( + output, run_command, error, truncate, + zulip_upload_log, get_release, upload, + errors_detected, log, + version_info, parse_version_info) PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Primary openshot folder RELEASE_NAME_REGEX = re.compile(r'^OpenShot-v.*?(-.*?)-x86[_64]*') @@ -50,7 +53,7 @@ urllib3.disable_warnings() -if __name__ == "__main__": +def main(): # Only run this code when directly executing this script. zulip_token = None @@ -82,15 +85,19 @@ script_mode = "publish" # Start log - output("%s %s Log for %s" % (platform.system(), script_mode, - datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) + output("%s %s Log for %s" % ( + platform.system(), + script_mode, + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) # Detect artifact folder (if any) artifact_dir = os.path.join(PATH, "build") # Parse artifact version files (if found) for repo_name in repo_names: - version_info.update(parse_version_info(os.path.join(artifact_dir, "install-x64", "share", repo_name))) + data_file = f"{repo_name}.env" + version_info.update(parse_version_info( + os.path.join(artifact_dir, "install-x64", "share", data_file))) output(str(version_info)) # Get version info @@ -104,8 +111,9 @@ original_git_branch = git_branch_name if original_git_branch != git_branch_name: # Branch names do not match - raise Exception("Branch names do not match for all 3 repos: `%s` vs `%s`" % - (original_git_branch, git_branch_name)) + raise Exception( + "Branch names do not match for all 3 repos: `%s` vs `%s`" % + (original_git_branch, git_branch_name)) # Loop through and get and/or create the GitHub Release objects releases = {} @@ -122,14 +130,18 @@ # Get official version release (i.e. v2.1.0, v2.x.x) releases[repo_name] = get_release(repos.get(repo_name), github_release_name) if releases.get(repo_name) and releases.get(repo_name).prerelease is False: - raise Exception("GitHub release for version %s is already released. Did we forget to bump a " - "version? (repo: %s, branch: %s)" % (github_release_name, repo_name, - git_branch_name)) + raise Exception( + "GitHub release for version %s is already released. Did we forget to bump a " + "version? (repo: %s, branch: %s)" % ( + github_release_name, repo_name, + git_branch_name) + ) else: # ignore all branches that don't start with 'release*' - raise Exception("%s only allowed for branch names that start with 'release*'" - " (repo: %s, branch: %s)" % - (script_mode, repo_name, git_branch_name)) + raise Exception( + "%s only allowed for branch names that start with 'release*'" + " (repo: %s, branch: %s)" % ( + script_mode, repo_name, git_branch_name)) if not is_publish: @@ -152,9 +164,11 @@ so_title = "" if so_number: so_title = ", SO: %s" % so_number - log_markdown = "%s Changelog (Version: %s%s)\n---\n%s\n\n" % \ - (repo_name, version_info.get(repo_name, {}).get('VERSION'), - so_title, logs.get(repo_name)) + log_markdown = "%s Changelog (Version: %s%s)\n---\n%s\n\n" % ( + repo_name, + version_info.get(repo_name, {}).get('VERSION'), + so_title, + logs.get(repo_name)) combined_log_markdown += log_markdown if not repo_name == "openshot-qt": formatted_logs[repo_name] = log_title + log_markdown @@ -167,10 +181,11 @@ github_release_name = "v%s" % version_info.get(repo_name, {}).get('VERSION') if not releases.get(repo_name): # Create a new release if one if missing (for each repo) - releases[repo_name] = repos.get(repo_name).create_release(github_release_name, - target_commitish=git_branch_name, - prerelease=True, - body=formatted_logs.get(repo_name)) + releases[repo_name] = repos.get(repo_name).create_release( + github_release_name, + target_commitish=git_branch_name, + prerelease=True, + body=formatted_logs.get(repo_name)) # Upload all deploy artifacts/installers to GitHub # NOTE: ONLY for `openshot-qt` repo @@ -226,9 +241,11 @@ if line: shasum_check_output = line.decode('UTF-8').strip() if shasum_output.split(" ")[0] != shasum_check_output.split(" ")[0]: - raise Exception("shasum validation of %s has failed after downloading " - "the uploaded installer: %s.\n%s\n%s" - % (check_artifact_path, artifact_path, shasum_check_output, shasum_output )) + raise Exception( + "shasum validation of %s has failed after downloading " + "the uploaded installer: %s.\n%s\n%s" % ( + check_artifact_path, artifact_path, shasum_check_output, shasum_output) + ) # Create and upload sha2sum file and instructions output("Hash generated: %s" % shasum_output) @@ -249,7 +266,19 @@ # Create torrent and upload torrent_name = "%s.torrent" % artifact_name torrent_path = os.path.join(artifact_dir, torrent_name) - torrent_command = 'mktorrent -a "udp://tracker.openbittorrent.com:80/announce, udp://tracker.publicbt.com:80/announce, udp://tracker.opentrackr.org:1337" -c "OpenShot Video Editor %s" -w "%s" -o "%s" "%s"' % (github_release.tag_name, download_url, torrent_name, artifact_name) + tracker_list = [ + "udp://tracker.openbittorrent.com:80/announce", + "udp://tracker.publicbt.com:80/announce", + "udp://tracker.opentrackr.org:1337", + ] + torrent_command = " ".join([ + 'mktorrent', + '-a "%s"' % (", ".join(tracker_list)), + '-c "OpenShot Video Editor %s"' % github_release.tag_name, + '-w "%s"' % download_url, + '-o "%s"' % torrent_name, + '"%s"' % artifact_name, + ]) torrent_output = "" for line in run_command(torrent_command, artifact_dir): output(line) @@ -265,18 +294,33 @@ # Submit blog post (if it doesn't already exist) (in draft mode) auth = HTTPBasicAuth(os.getenv('OPENSHOT_ORG_USER'), os.getenv('OPENSHOT_ORG_PASS')) - r = post("https://www.openshot.org/api/release/submit/", auth=auth, data={ "version": openshot_qt_version, - "changelog": log_title + combined_log_markdown }) + r = post( + "https://www.openshot.org/api/release/submit/", + auth=auth, + data={ + "version": openshot_qt_version, + "changelog": log_title + combined_log_markdown + }) if not r.ok: - raise Exception("HTTP post to openshot.org/api/release/submit/ failed: %s (user: %s): %s" % - (r.status_code, os.getenv('OPENSHOT_ORG_USER'), r.content.decode('UTF-8'))) + raise Exception( + "HTTP post to openshot.org/api/release/submit/ failed: %s (user: %s): %s" % ( + r.status_code, + os.getenv('OPENSHOT_ORG_USER'), + r.content.decode('UTF-8')) + ) else: # Publish the release (make new version visible on openshot.org, and make blog post visible) auth = HTTPBasicAuth(os.getenv('OPENSHOT_ORG_USER'), os.getenv('OPENSHOT_ORG_PASS')) - r = post("https://www.openshot.org/api/release/publish/", auth=auth, data={"version": openshot_qt_version }) + r = post( + "https://www.openshot.org/api/release/publish/", + auth=auth, + data={"version": openshot_qt_version}) if not r.ok: - raise Exception("HTTP post to openshot.org/api/release/publish/ failed: %s (user: %s): %s" % - (r.status_code, os.getenv('OPENSHOT_ORG_USER'), r.content.decode('UTF-8'))) + raise Exception( + "HTTP post to openshot.org/api/release/publish/ failed: %s (user: %s): %s" % ( + r.status_code, + os.getenv('OPENSHOT_ORG_USER'), + r.content.decode('UTF-8'))) # Publish GitHub Release objects (in all 3 repos) for repo_name in repo_names: @@ -286,8 +330,10 @@ # Publish github release also github_release.edit(prerelease=False) else: - raise Exception("Cannot publish missing GitHub release: %s, version: %s" % - (repo_name, openshot_qt_version)) + raise Exception( + "Cannot publish missing GitHub release: %s, version: %s" % ( + repo_name, + openshot_qt_version)) # Verify download links on openshot.org are correct (and include the new release version) r = get("https://www.openshot.org/download/") @@ -300,15 +346,23 @@ if r.ok and r.reason == "Found": output("Validation of URL successful: %s" % url) else: - raise Exception("Validation of URL FAILED: %s, %s, %s" % (url, r.status_code, r.reason)) + raise Exception( + "Validation of URL FAILED: %s, %s, %s" % ( + url, r.status_code, r.reason) + ) # Validate the current version is found in each URL if openshot_qt_version not in url: - raise Exception("Validation of URL FAILED. Missing version %s: %s, %s, %s" % - (openshot_qt_version, url, - r.status_code, r.reason)) + raise Exception( + "Validation of URL FAILED. Missing version %s: %s, %s, %s" % ( + openshot_qt_version, + url, + r.status_code, + r.reason) + ) else: - raise Exception("Failed to GET openshot.org/download for URL validation: %s" % r.status_code) + raise Exception( + "Failed to GET openshot.org/download for URL validation: %s" % r.status_code) except Exception as ex: tb = traceback.format_exc() @@ -316,12 +370,19 @@ if not errors_detected: output("Successfully completed %s script!" % script_mode) - zulip_upload_log(zulip_token, log, - "%s: %s **success** log" % (platform.system(), script_mode), - ":congratulations: successful %s" % script_mode) + zulip_upload_log( + zulip_token, log, + "%s: %s **success** log" % (platform.system(), script_mode), + ":congratulations: successful %s" % script_mode) else: # Report any errors detected output("%s script failed!" % script_mode) - zulip_upload_log(zulip_token, log, "%s: %s error log" % (platform.system(), script_mode), - ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) + zulip_upload_log( + zulip_token, log, + "%s: %s error log" % (platform.system(), script_mode), + ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) exit(1) + + +if __name__ == "__main__": + main() diff --git a/installer/fix_qt5_rpath.py b/installer/fix_qt5_rpath.py index 395459edf8..21ccc651c3 100644 --- a/installer/fix_qt5_rpath.py +++ b/installer/fix_qt5_rpath.py @@ -112,7 +112,9 @@ def print_min_versions(PATH): if __name__ == "__main__": - """Run these methods manually for testing""" + # Run these methods manually for testing + + # XXX: This path should be set programmatically, somehow PATH = "/Users/jonathanthomas/apps/openshot-qt/build/exe.macosx-10.15-x86_64-3.7" fix_rpath(PATH) print_min_versions(PATH) diff --git a/installer/prune_daily_builds.py b/installer/prune_daily_builds.py index ad9c0cc4cf..b63532d63a 100644 --- a/installer/prune_daily_builds.py +++ b/installer/prune_daily_builds.py @@ -38,8 +38,7 @@ MAXIMUM_ASSET_AGE_DAYS = 180 # Calculate current date (with timezone) -utc=pytz.UTC -now = utc.localize(datetime.datetime.now()) +now = pytz.UTC.localize(datetime.datetime.now()) def get_release(repo, tag_name): @@ -47,7 +46,11 @@ def get_release(repo, tag_name): @param repo: github3 repository object @returns: github3 release object or None """ - for release in repo.iter_releases(): + if hasattr(repo, 'releases'): + release_iter = repo.releases() + else: + release_iter = repo.iter_releases() + for release in release_iter: if release.tag_name == tag_name: return release @@ -83,4 +86,4 @@ def get_release(repo, tag_name): print("------------------") print("Deleted %s Assets" % delete_count) - print("Skipped %s Assets" % skip_count) \ No newline at end of file + print("Skipped %s Assets" % skip_count) diff --git a/installer/version_parser.py b/installer/version_parser.py index caace3658c..3250918b14 100644 --- a/installer/version_parser.py +++ b/installer/version_parser.py @@ -36,10 +36,11 @@ def parse_version_info(version_path): """Parse version info from gitlab artifacts""" - version_info = { "date": f'{datetime.datetime.now():%Y-%m-%d %H:%M}' } + version_info = {"date": f'{datetime.datetime.now():%Y-%m-%d %H:%M}'} # Get name of version file - version_name = os.path.basename(version_path) + file_name = os.path.basename(version_path) + version_name = os.path.splitext(file_name)[0] version_info[version_name] = { "CI_PROJECT_NAME": None, "CI_COMMIT_REF_NAME": None, @@ -53,10 +54,13 @@ def parse_version_info(version_path): if os.path.exists(version_path): with open(version_path, "r") as f: # Parse each line in f as a 'key:value' string - version_info[version_name].update((l.strip().split(':') for l in f.readlines())) + version_info[version_name].update( + (ln.strip().split(':') for ln in f.readlines()) + ) return version_info + def parse_build_name(version_info, git_branch_name=""): """Calculate the build name used in URLs and Installers from the version_info dict""" # Check for all version info needed to construct the build URL @@ -99,9 +103,9 @@ def parse_build_name(version_info, git_branch_name=""): # Parse all version info (if found) for git_log_filename in os.listdir(settings_path): - git_log_filepath = os.path.join(settings_path, git_log_filename) - if os.path.splitext(git_log_filepath)[1] == "": - # No extension, parse version info + if git_log_filename.endswith(".env"): + # Metadata file, parse version info + git_log_filepath = os.path.join(settings_path, git_log_filename) version_info.update(parse_version_info(git_log_filepath)) # Calculate build name from version info diff --git a/installer/windows-installer.iss b/installer/windows-installer.iss index 3fb40a5a59..118b4c443a 100644 --- a/installer/windows-installer.iss +++ b/installer/windows-installer.iss @@ -7,6 +7,9 @@ #ifndef ONLY_64_BIT #define ONLY_64_BIT "x64" #endif +#ifndef PY_EXE_DIR + #define PY_EXE_DIR "exe.mingw-3.8" +#endif #define MyAppName "OpenShot Video Editor" @@ -138,7 +141,7 @@ Root: HKLM; Subkey: "Software\Classes\OpenShotProject\shell\open\command"; Value [Files] ; Add all frozen files from cx_Freeze build -Source: "..\build\exe.mingw-3.7\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs +Source: "..\build\{#PY_EXE_DIR}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs [Icons] Name: "{commonprograms}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"