From fc6380a6c4e3fec184bb16d6e9d34fd4796259d4 Mon Sep 17 00:00:00 2001 From: Martin Thoma Date: Wed, 24 Mar 2021 02:46:17 +0100 Subject: [PATCH 01/23] Streamline dict access patterns (#4017) * Use 'key in dict' instead of 'key in dict.keys()' * Thank you Frank Dana for the suggestions! * Use get * Fix member variable reference Co-authored-by: Frank Dana --- installer/fix_qt5_rpath.py | 2 +- src/classes/project_data.py | 6 +++--- src/language/generate_translations.py | 4 ++-- src/windows/add_to_timeline.py | 4 ++-- src/windows/cutting.py | 4 ++-- src/windows/file_properties.py | 8 ++------ src/windows/models/add_to_timeline_model.py | 4 +--- src/windows/models/files_model.py | 14 ++++---------- src/windows/views/files_treeview.py | 13 +++---------- src/windows/views/webview.py | 8 ++++---- 10 files changed, 24 insertions(+), 43 deletions(-) diff --git a/installer/fix_qt5_rpath.py b/installer/fix_qt5_rpath.py index 265ad9cc82..395459edf8 100644 --- a/installer/fix_qt5_rpath.py +++ b/installer/fix_qt5_rpath.py @@ -41,7 +41,7 @@ def fix_rpath(PATH): # Loop through all dependencies of each library/executable raw_output = subprocess.Popen(["oTool", "-L", file_path], stdout=subprocess.PIPE).communicate()[0].decode('utf-8') for output in raw_output.split("\n")[1:-1]: - if output and not "is not an object file" in output and ".o):" not in output: + if output and "is not an object file" not in output and ".o):" not in output: dependency_path = output.split('\t')[1].split(' ')[0] dependency_version = output.split('\t')[1].split(' (')[1].replace(')', '') dependency_base_path, dependency_name = os.path.split(dependency_path) diff --git a/src/classes/project_data.py b/src/classes/project_data.py index 672dbe4037..bfbefa9f33 100644 --- a/src/classes/project_data.py +++ b/src/classes/project_data.py @@ -98,7 +98,7 @@ def get(self, key): # True until something disqualifies this as a match match = True # Check each key in key_part dictionary and if not found to be equal as a property in item, move on to next item in list - for subkey in key_part.keys(): + for subkey in key_part: # Get each key in dictionary (i.e. "id", "layer", etc...) subkey = subkey.lower() # If object is missing the key or the values differ, then it doesn't match. @@ -512,7 +512,7 @@ def read_legacy_project_file(self, file_path): for track in reversed(sequence.tracks): for clip in track.clips: # Get associated file for this clip - if clip.file_object.unique_id in file_lookup.keys(): + if clip.file_object.unique_id in file_lookup: file = file_lookup[clip.file_object.unique_id] else: # Skip missing file @@ -840,7 +840,7 @@ def move_temp_paths_to_project_folder(self, file_path, previous_path=None): log.info("Checking clip {} path for file {}".format(clip["id"], file_id)) # Update paths to files stored in our working space or old path structure # (should have already been copied during previous File stage) - if file_id and file_id in reader_paths.keys(): + if file_id and file_id in reader_paths: clip["reader"]["path"] = reader_paths[file_id] log.info("Updated clip {} path for file {}".format(clip["id"], file_id)) diff --git a/src/language/generate_translations.py b/src/language/generate_translations.py index cb26fa68b0..9460bbc719 100755 --- a/src/language/generate_translations.py +++ b/src/language/generate_translations.py @@ -189,9 +189,9 @@ # Loop through props for key in props.keys(): object = props[key] - if "name" in object.keys(): + if "name" in object: effects_text[object["name"]] = "libopenshot (Clip Properties)" - if "choices" in object.keys(): + if "choices" in object: for choice in object["choices"]: effects_text[choice["name"]] = "libopenshot (Clip Properties)" diff --git a/src/windows/add_to_timeline.py b/src/windows/add_to_timeline.py index df4b518aac..1c11ee8a68 100644 --- a/src/windows/add_to_timeline.py +++ b/src/windows/add_to_timeline.py @@ -216,10 +216,10 @@ def accept(self): start_time = 0 end_time = new_clip["reader"]["duration"] - if 'start' in file.data.keys(): + if 'start' in file.data: start_time = file.data['start'] new_clip["start"] = start_time - if 'end' in file.data.keys(): + if 'end' in file.data: end_time = file.data['end'] new_clip["end"] = end_time diff --git a/src/windows/cutting.py b/src/windows/cutting.py index 0ae06038a0..8a987b3825 100644 --- a/src/windows/cutting.py +++ b/src/windows/cutting.py @@ -162,7 +162,7 @@ def __init__(self, file=None, preview=False): # Determine if a start or end attribute is in this file start_frame = 1 - if 'start' in self.file.data.keys(): + if 'start' in self.file.data: start_frame = (float(self.file.data['start']) * self.fps) + 1 # Display start frame (and then the previous frame) @@ -334,7 +334,7 @@ def btnAddClip_clicked(self): log.info('btnAddClip_clicked') # Remove unneeded attributes - if 'name' in self.file.data.keys(): + if 'name' in self.file.data: self.file.data.pop('name') # Save new file diff --git a/src/windows/file_properties.py b/src/windows/file_properties.py index 5102069143..60db13d2e7 100644 --- a/src/windows/file_properties.py +++ b/src/windows/file_properties.py @@ -82,12 +82,8 @@ def __init__(self, file): file_extension = os.path.splitext(filename)[1] fps_float = float(self.file.data["fps"]["num"]) / float(self.file.data["fps"]["den"]) - tags = "" - if "tags" in self.file.data.keys(): - tags = self.file.data["tags"] - name = filename - if "name" in self.file.data.keys(): - name = self.file.data["name"] + tags = self.file.data.get("tags", "") + name = self.file.data.get("name", filename) # Populate fields self.txtFileName.setText(name) diff --git a/src/windows/models/add_to_timeline_model.py b/src/windows/models/add_to_timeline_model.py index 85ee753239..81630dbd28 100644 --- a/src/windows/models/add_to_timeline_model.py +++ b/src/windows/models/add_to_timeline_model.py @@ -72,9 +72,7 @@ def update_model(self, files=[], clear=True): row = [] # Look for friendly name attribute (optional) - name = filename - if 'name' in file.data.keys(): - name = file.data['name'] + name = file.data.get("name", filename) # Append thumbnail col = QStandardItem() diff --git a/src/windows/models/files_model.py b/src/windows/models/files_model.py index 49f7d81e07..903b9ec991 100644 --- a/src/windows/models/files_model.py +++ b/src/windows/models/files_model.py @@ -170,12 +170,8 @@ def update_model(self, clear=True, delete_file_id=None): continue path, filename = os.path.split(file.data["path"]) - tags = "" - if "tags" in file.data.keys(): - tags = file.data["tags"] - name = filename - if "name" in file.data.keys(): - name = file.data["name"] + tags = file.data.get("tags", "") + name = file.data.get("name", filename) media_type = file.data.get("media_type") @@ -183,7 +179,7 @@ def update_model(self, clear=True, delete_file_id=None): if media_type in ["video", "image"]: # Check for start and end attributes (optional) thumbnail_frame = 1 - if 'start' in file.data.keys(): + if 'start' in file.data: fps = file.data["fps"] fps_float = float(fps["num"]) / float(fps["den"]) thumbnail_frame = round(float(file.data['start']) * fps_float) + 1 @@ -492,9 +488,7 @@ def update_file_thumbnail(self, file_id): """Update/re-generate the thumbnail of a specific file""" file = File.get(id=file_id) path, filename = os.path.split(file.data["path"]) - name = filename - if "name" in file.data.keys(): - name = file.data["name"] + name = file.data.get("name", filename) # Refresh thumbnail for updated file self.ignore_updates = True diff --git a/src/windows/views/files_treeview.py b/src/windows/views/files_treeview.py index 2c15ad3a4b..a7e7220ae9 100644 --- a/src/windows/views/files_treeview.py +++ b/src/windows/views/files_treeview.py @@ -183,16 +183,9 @@ def value_updated(self, item): # Get file object and update friendly name and tags attribute f = File.get(id=file_id) - if name and name != os.path.basename(f.data["path"]): - f.data["name"] = name - else: - f.data["name"] = os.path.basename(f.data["path"]) - - if "tags" in f.data.keys(): - if tags != f.data["tags"]: - f.data["tags"] = tags - elif tags: - f.data["tags"] = tags + f.data.update({"name": name or os.path.basename(f.data.get("path"))}) + if "tags" in f.data or tags: + f.data.update({"tags": tags}) # Save File f.save() diff --git a/src/windows/views/webview.py b/src/windows/views/webview.py index cb13c6dfcd..353db0506b 100644 --- a/src/windows/views/webview.py +++ b/src/windows/views/webview.py @@ -2227,7 +2227,7 @@ def Time_Triggered(self, action, clip_ids, speed="1X", playhead_position=0.0): continue # Keep original 'end' and 'duration' - if "original_data" not in clip.data.keys(): + if "original_data" not in clip.data: clip.data["original_data"] = { "end": clip.data["end"], "duration": clip.data["duration"], @@ -2246,7 +2246,7 @@ def Time_Triggered(self, action, clip_ids, speed="1X", playhead_position=0.0): freeze_seconds = float(speed) original_duration = clip.data["duration"] - if "original_data" in clip.data.keys(): + if "original_data" in clip.data: original_duration = clip.data["original_data"]["duration"] log.info('Updating timing for clip ID {}, original duration: {}' @@ -2873,9 +2873,9 @@ def callback(self, data, callback_data): return # Do nothing # Check for optional start and end attributes - if 'start' in file.data.keys(): + if 'start' in file.data: new_clip["start"] = file.data['start'] - if 'end' in file.data.keys(): + if 'end' in file.data: new_clip["end"] = file.data['end'] # Set position and closet track From 2d16072dea8b69863282a6b475db54de8f9758d4 Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Wed, 31 Mar 2021 18:31:28 -0400 Subject: [PATCH 02/23] README: Remove Travis build shields --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 7b2dc7402a..3d1fac9147 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,6 @@ OpenShot Video Editor is an award-winning free and open-source video editor for Linux, Mac, and Windows, and is dedicated to delivering high quality video editing and animation solutions to the world. -## Build Status - -[![Build Status](https://img.shields.io/travis/OpenShot/openshot-qt/develop.svg?label=openshot-qt)](https://travis-ci.org/OpenShot/openshot-qt) [![Build Status](https://img.shields.io/travis/OpenShot/libopenshot/develop.svg?label=libopenshot)](https://travis-ci.org/OpenShot/libopenshot) [![Build Status](https://img.shields.io/travis/OpenShot/libopenshot-audio/develop.svg?label=libopenshot-audio)](https://travis-ci.org/OpenShot/libopenshot-audio) - ## Features * Cross-platform (Linux, Mac, and Windows) From ba63a5a8fe4609ac8b9391765f7c116b21dba9b6 Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Wed, 31 Mar 2021 18:39:41 -0400 Subject: [PATCH 03/23] README: Add Github Actions status badges --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3d1fac9147..72292cda13 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ OpenShot Video Editor is an award-winning free and open-source video editor for Linux, Mac, and Windows, and is dedicated to delivering high quality video editing and animation solutions to the world. +## Build Status + +**openshot-qt** [![CI Build](https://github.com/OpenShot/openshot-qt/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/openshot-qt/actions/workflows/ci.yml) **libopenshot** [![CI Build](https://github.com/OpenShot/libopenshot/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/libopenshot/actions/workflows/ci.yml) **libopenshot-audio** [![CI Build](https://github.com/OpenShot/libopenshot-audio/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/libopenshot-audio/actions/workflows/ci.yml) + ## Features * Cross-platform (Linux, Mac, and Windows) From 076dc243dafd52d0ed5f66d120a4093509c66c0e Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Wed, 31 Mar 2021 18:47:26 -0400 Subject: [PATCH 04/23] README: Add repo name to workflow job title --- .github/workflows/ci.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47f82fb3c0..dbb38c2674 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI Build +name: openshot-qt CI Build on: [push, pull_request] jobs: build: diff --git a/README.md b/README.md index 72292cda13..74daa1a251 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ video editing and animation solutions to the world. ## Build Status -**openshot-qt** [![CI Build](https://github.com/OpenShot/openshot-qt/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/openshot-qt/actions/workflows/ci.yml) **libopenshot** [![CI Build](https://github.com/OpenShot/libopenshot/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/libopenshot/actions/workflows/ci.yml) **libopenshot-audio** [![CI Build](https://github.com/OpenShot/libopenshot-audio/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/libopenshot-audio/actions/workflows/ci.yml) +[![openshot-qt CI Build](https://github.com/OpenShot/openshot-qt/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/openshot-qt/actions/workflows/ci.yml) [![libopenshot CI Build](https://github.com/OpenShot/libopenshot/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/libopenshot/actions/workflows/ci.yml) [![libopenshot-audio CI Build](https://github.com/OpenShot/libopenshot-audio/actions/workflows/ci.yml/badge.svg)](https://github.com/OpenShot/libopenshot-audio/actions/workflows/ci.yml) ## Features From 8c57d3845da8fb6c010b41a1ae0979a94e5a07fd Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Thu, 1 Apr 2021 06:51:10 -0400 Subject: [PATCH 05/23] CI: Ignore Ubuntu 20.04 failures (#4076) --- .github/workflows/ci.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbb38c2674..ed78ee2109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,16 +3,22 @@ on: [push, pull_request] jobs: build: runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.allowed-to-fail }} strategy: + fail-fast: false matrix: - os: [ubuntu-18.04, ubuntu-20.04] + include: + - os: ubuntu-18.04 + allowed-to-fail: false + - os: ubuntu-20.04 + allowed-to-fail: true + defaults: + run: + shell: bash steps: - uses: actions/checkout@v2 - name: Install dependencies - # Some projects don't allow in-source building, so create a separate build directory - # We'll use this as our working directory for all subsequent commands - shell: bash run: | sudo add-apt-repository ppa:openshot.developers/libopenshot-daily sudo apt update @@ -23,11 +29,7 @@ jobs: pip3 install cx_Freeze==6.1 distro defusedxml requests certifi chardet urllib3 - name: Build Python package - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash run: python3 freeze.py build - name: Test - shell: bash run: xvfb-run --auto-servernum --server-num=1 --server-args "-screen 0 1920x1080x24" python3 ./src/tests/query_tests.py From 13ddbc49ec0684c981e35f0cdfd52db502d663c4 Mon Sep 17 00:00:00 2001 From: Lorenzo Spinelli Date: Thu, 1 Apr 2021 13:28:58 +0200 Subject: [PATCH 06/23] keyboard shortcut "R" to toogle the razor tool (#4007) * keyboard shortcut "R" to toogle the razor tool * toogle the razor tool: update docs * Add setting for actionRazorTool shortcut Co-authored-by: FeRD (Frank Dana) --- doc/main_window.rst | 1 + src/settings/_default.settings | 8 ++++++++ src/windows/ui/main-window.ui | 3 +++ 3 files changed, 12 insertions(+) diff --git a/doc/main_window.rst b/doc/main_window.rst index 28e9f97718..5502dbefca 100644 --- a/doc/main_window.rst +++ b/doc/main_window.rst @@ -130,6 +130,7 @@ Ctrl+K Slice All: Keep Both Sides Ctrl+L Slice All: Keep Left Side Ctrl+J Slice All: Keep Right Side Ctrl+G Toggle Snapping +R Toggle Razor tool Ctrl+X Split Clip Ctrl+Shift+D Thumbnail View Ctrl+T Title Editor diff --git a/src/settings/_default.settings b/src/settings/_default.settings index d81b91c9e3..3363c84e59 100644 --- a/src/settings/_default.settings +++ b/src/settings/_default.settings @@ -518,6 +518,14 @@ "value": "Ctrl+M", "type": "text" }, + { + "category": "Keyboard", + "title": "Toggle Razor", + "restart": true, + "setting": "actionRazorTool", + "value": "R", + "type": "text" + }, { "category": "Keyboard", "title": "Previous Marker", diff --git a/src/windows/ui/main-window.ui b/src/windows/ui/main-window.ui index da2c7a6c08..99380a1942 100644 --- a/src/windows/ui/main-window.ui +++ b/src/windows/ui/main-window.ui @@ -884,6 +884,9 @@ Razor Tool + + R + From b9f2e22ce943fd8791d43068688cd7fe8c0add77 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sun, 18 Apr 2021 06:13:05 -0400 Subject: [PATCH 07/23] build_server: Code-formatting/readability fixes - Wrap long code lines - Assemble long command strings from " ".join()-ed lists --- installer/build_server.py | 140 +++++++++++++++++++++++++++++--------- 1 file changed, 109 insertions(+), 31 deletions(-) diff --git a/installer/build_server.py b/installer/build_server.py index 61020c961a..7d83b38491 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -80,9 +80,12 @@ def run_command(command, working_dir=None): """Utility function to return output from command line""" short_command = command.split('" ')[0] # We don't need to print args output("Running %s... (%s)" % (short_command, working_dir)) - p = subprocess.Popen(command, shell=True, cwd=working_dir, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + p = subprocess.Popen( + command, + shell=True, + cwd=working_dir, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) return iter(p.stdout.readline, b"") @@ -187,7 +190,7 @@ def upload(file_path, github_release): return url -if __name__ == "__main__": +def main(): # Only run this code when directly executing this script. Parts of this file # are also used in the deploy.py script. try: @@ -215,7 +218,12 @@ def upload(file_path, github_release): git_branch_name = sys.argv[7] # Start log - output("%s Build Log for %s (branch: %s)" % (platform.system(), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), git_branch_name)) + output( + "%s Build Log for %s (branch: %s)" % ( + platform.system(), + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + git_branch_name) + ) # Detect artifact folder (if any) artifact_path = os.path.join(PATH, "build", "install-x64") @@ -227,7 +235,8 @@ def upload(file_path, github_release): # Parse artifact version files (if found) for repo_name in ["libopenshot-audio", "libopenshot", "openshot-qt"]: - version_info.update(parse_version_info(os.path.join(artifact_path, "share", repo_name))) + version_info.update( + parse_version_info(os.path.join(artifact_path, "share", repo_name))) output(str(version_info)) # Get GIT description of openshot-qt-git branch (i.e. v2.0.6-18-ga01a98c) @@ -279,12 +288,13 @@ def upload(file_path, github_release): os.makedirs(os.path.join(app_dir_path, "usr"), exist_ok=True) # XDG Freedesktop icon paths - icons = [ ("scalable", os.path.join(PATH, "xdg", "openshot-qt.svg")), - ("64x64", os.path.join(PATH, "xdg", "icon", "64", "openshot-qt.png")), - ("128x128", os.path.join(PATH, "xdg", "icon", "128", "openshot-qt.png")), - ("256x256", os.path.join(PATH, "xdg", "icon", "256", "openshot-qt.png")), - ("512x512", os.path.join(PATH, "xdg", "icon", "512", "openshot-qt.png")), - ] + icons = [ + ("scalable", os.path.join(PATH, "xdg", "openshot-qt.svg")), + ("64x64", os.path.join(PATH, "xdg", "icon", "64", "openshot-qt.png")), + ("128x128", os.path.join(PATH, "xdg", "icon", "128", "openshot-qt.png")), + ("256x256", os.path.join(PATH, "xdg", "icon", "256", "openshot-qt.png")), + ("512x512", os.path.join(PATH, "xdg", "icon", "512", "openshot-qt.png")), + ] # Copy desktop icons icon_theme_path = os.path.join(app_dir_path, "usr", "share", "icons", "hicolor") @@ -348,7 +358,11 @@ def upload(file_path, github_release): # Create AppImage (OpenShot-%s-x86_64.AppImage) app_image_success = False - for line in run_command('/home/ubuntu/apps/AppImageKit/AppImageAssistant "%s" "%s"' % (app_dir_path, app_build_path)): + for line in run_command(" ".join([ + '/home/ubuntu/apps/AppImageKit/AppImageAssistant', + '"%s"' % app_dir_path, + '"%s"' % app_build_path, + ])): output(line) app_image_success = os.path.exists(app_build_path) @@ -361,23 +375,28 @@ def upload(file_path, github_release): # Delete build (since something failed) os.remove(app_build_path) - if platform.system() == "Darwin": - # Create DMG (OpenShot-%s-x86_64.DMG) app_image_success = False # Build app.bundle and create DMG for line in run_command("bash installer/build-mac-dmg.sh"): output(line) - if ("error".encode("UTF-8") in line and "No errors".encode("UTF-8") not in line) or "rejected".encode("UTF-8") in line: + if ( + ("error".encode("UTF-8") in line + and "No errors".encode("UTF-8") not in line) + or "rejected".encode("UTF-8") in line + ): error("Build-Mac-DMG Error: %s" % line) if "Your image is ready".encode("UTF-8") in line: app_image_success = True # Rename DMG (to be consistent with other OS installers) for dmg_path in os.listdir(os.path.join(PATH, "build")): - if os.path.isfile(os.path.join(PATH, "build", dmg_path)) and dmg_path.endswith(".dmg"): + if ( + os.path.isfile(os.path.join(PATH, "build", dmg_path)) + and dmg_path.endswith(".dmg") + ): os.rename(os.path.join(PATH, "build", dmg_path), app_build_path) # Was the DMG creation successful @@ -389,9 +408,7 @@ def upload(file_path, github_release): # Delete build (since key signing might have failed) os.remove(app_build_path) - if platform.system() == "Windows": - # Move python folder structure, since Cx_Freeze doesn't put it in the correct place exe_dir = os.path.join(PATH, 'build', 'exe.mingw-3.7') python_dir = os.path.join(exe_dir, 'lib', 'python3.7') @@ -402,7 +419,14 @@ def upload(file_path, github_release): shutil.rmtree(duplicate_openshot_qt_path, True) # Remove the following paths. cx_Freeze is including many unneeded files. This prunes them out. - paths_to_delete = ['mediaservice', 'imageformats', 'platforms', 'printsupport', 'lib/openshot_qt', 'resvg.dll'] + paths_to_delete = [ + 'mediaservice', + 'imageformats', + 'platforms', + 'printsupport', + 'lib/openshot_qt', + 'resvg.dll', + ] for delete_path in paths_to_delete: full_delete_path = os.path.join(exe_dir, delete_path) output("Delete path: %s" % full_delete_path) @@ -420,14 +444,24 @@ def upload(file_path, github_release): paths_to_replace = ['imageformats', 'platforms'] for replace_name in paths_to_replace: if windows_32bit: - shutil.copytree(os.path.join('C:\\msys32\\mingw32\\share\\qt5\\plugins', replace_name), os.path.join(exe_dir, replace_name)) + shutil.copytree( + os.path.join('C:\\msys32\\mingw32\\share\\qt5\\plugins', replace_name), + os.path.join(exe_dir, replace_name)) else: - shutil.copytree(os.path.join('C:\\msys64\\mingw64\\share\\qt5\\plugins', replace_name), os.path.join(exe_dir, replace_name)) + shutil.copytree( + os.path.join('C:\\msys64\\mingw64\\share\\qt5\\plugins', replace_name), + os.path.join(exe_dir, replace_name)) # Copy Qt5Core.dll, Qt5Svg.dll to root of frozen directory - paths_to_copy = [("Qt5Core.dll", "C:\\msys64\\mingw64\\bin\\"), ("Qt5Svg.dll", "C:\\msys64\\mingw64\\bin\\")] + paths_to_copy = [ + ("Qt5Core.dll", "C:\\msys64\\mingw64\\bin\\"), + ("Qt5Svg.dll", "C:\\msys64\\mingw64\\bin\\"), + ] if windows_32bit: - paths_to_copy = [("Qt5Core.dll", "C:\\msys32\\mingw32\\bin\\"), ("Qt5Svg.dll", "C:\\msys32\\mingw32\\bin\\")] + paths_to_copy = [ + ("Qt5Core.dll", "C:\\msys32\\mingw32\\bin\\"), + ("Qt5Svg.dll", "C:\\msys32\\mingw32\\bin\\"), + ] for qt_file_name, qt_parent_path in paths_to_copy: qt5_path = os.path.join(qt_parent_path, qt_file_name) new_qt5_path = os.path.join(exe_dir, qt_file_name) @@ -452,7 +486,17 @@ def upload(file_path, github_release): # Add version metadata to frozen app launcher launcher_exe = os.path.join(exe_dir, "openshot-qt.exe") verpatch_success = True - verpatch_command = '"verpatch.exe" "{}" /va /high "{}" /pv "{}" /s product "{}" /s company "{}" /s copyright "{}" /s desc "{}"'.format(launcher_exe, info.VERSION, info.VERSION, info.PRODUCT_NAME, info.COMPANY_NAME, info.COPYRIGHT, info.PRODUCT_NAME) + verpatch_command = " ".join([ + 'verpatch.exe', + '{}'.format(launcher_exe), + '/va', + '/high "{}"'.format(info.VERSION), + '/pv "{}"'.format(info.VERSION), + '/s product "{}"'.format(info.PRODUCT_NAME), + '/s company "{}"'.format(info.COMPANY_NAME), + '/s copyright "{}"'.format(info.COPYRIGHT), + '/s desc "{}"'.format(info.PRODUCT_NAME), + ]) verpatch_output = "" # version-stamp executable for line in run_command(verpatch_command): @@ -472,7 +516,13 @@ def upload(file_path, github_release): # Create Installer (OpenShot-%s-x86_64.exe) inno_success = True - inno_command = '"iscc.exe" /Q /DVERSION=%s /DONLY_64_BIT=%s "%s"' % (version, only_64_bit, os.path.join(PATH, 'installer', 'windows-installer.iss')) + inno_command = " ".join([ + 'iscc.exe', + '/Q', + '/DVERSION=%s' % version, + '/DONLY_64_BIT=%s' % only_64_bit, + '"%s"' % os.path.join(PATH, 'installer', 'windows-installer.iss'), + ]) inno_output = "" # Compile Inno installer for line in run_command(inno_command): @@ -495,7 +545,14 @@ def upload(file_path, github_release): # Sign the installer key_sign_success = True - key_sign_command = '"kSignCMD.exe" /f "%s%s" /p "%s" /d "OpenShot Video Editor" /du "http://www.openshot.org" "%s"' % (windows_key, only_64_bit, windows_key_password, app_build_path) + key_sign_command = " ".join([ + 'kSignCMD.exe', + '/f "%s%s"' % (windows_key, only_64_bit), + '/p "%s"' % windows_key_password, + '/d "OpenShot Video Editor"', + '/du "http://www.openshot.org"', + '"%s"' % app_build_path, + ]) key_sign_output = "" # Sign MSI for line in run_command(key_sign_command): @@ -513,7 +570,6 @@ def upload(file_path, github_release): # Delete build (since key signing might have failed) os.remove(app_build_path) - # Upload Installer to GitHub (if build path exists) if needs_upload and os.path.exists(app_build_path): # Upload file to GitHub @@ -522,7 +578,19 @@ def upload(file_path, github_release): # Create torrent and upload torrent_path = "%s.torrent" % app_build_path - torrent_command = 'mktorrent -a "udp://tracker.openbittorrent.com:80/announce, udp://tracker.publicbt.com:80/announce, udp://tracker.opentrackr.org:1337" -c "OpenShot Video Editor %s" -w "%s" -o "%s" "%s"' % (version, download_url, "%s.torrent" % app_name, app_name) + tracker_list = [ + "udp://tracker.openbittorrent.com:80/announce", + "udp://tracker.publicbt.com:80/announce", + "udp://tracker.opentrackr.org:1337", + ] + torrent_command = " ".join([ + 'mktorrent', + '-a "%s"' % (", ".join(tracker_list)), + '-c "OpenShot Video Editor %s"' % version, + '-w "%s"' % download_url, + '-o "%s"' % ("%s.torrent" % app_name), + '"%s"' % app_name, + ]) torrent_output = "" # Remove existing torrents (if any found) @@ -544,7 +612,10 @@ def upload(file_path, github_release): url = upload(torrent_path, github_release) # Notify Zulip - zulip_upload_log(zulip_token, log, "%s: Build logs for %s" % (platform.system(), app_name), "Successful *%s* build: %s" % (git_branch_name, download_url)) + zulip_upload_log( + zulip_token, log, + "%s: Build logs for %s" % (platform.system(), app_name), + "Successful *%s* build: %s" % (git_branch_name, download_url)) except Exception as ex: tb = traceback.format_exc() @@ -555,5 +626,12 @@ def upload(file_path, github_release): else: # Report any errors detected output("build-server script failed!") - zulip_upload_log(zulip_token, log, "%s: Error log for *%s* build" % (platform.system(), git_branch_name), ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) + zulip_upload_log( + zulip_token, log, + "%s: Error log for *%s* build" % (platform.system(), git_branch_name), + ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) exit(1) + + +if __name__ == "__main__": + main() From fbd3551dc34a65b18bd2ceaebf51324df40ad9f7 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sun, 18 Apr 2021 06:38:37 -0400 Subject: [PATCH 08/23] Windows builder: mingw32\mingw32\ => mingw64\mingw32 Fix Win32 PATH juggling --- .gitlab-ci.yml | 20 ++++++++++++++++---- installer/build_server.py | 6 +++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6ffd577d35..c175ecdae8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -89,8 +89,14 @@ windows-builder-x64: - if (-not (Test-Path "artifacts.zip")) { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/develop/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } - Expand-Archive -Path artifacts.zip -DestinationPath . - Copy-Item "$CI_PROJECT_DIR/build/install-x64/python/*" -Destination "$CI_PROJECT_DIR" - - $env:Path = "$CI_PROJECT_DIR\build\install-x64\lib;$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\bin;C:\msys64\mingw64\lib;C:\msys64\mingw64\lib\python3.7\;C:\msys64\mingw64\lib\python3.7\site-packages\;C:\msys64\mingw64\lib\python3.7\site-packages\PyQt5;C:\msys64\usr\lib\cmake\UnitTest++;C:\msys64\usr;C:\msys64\usr\lib;C:\msys64\usr\local\x64\mingw\bin;C:\msys64\usr\local;" + $env:Path; - - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\usr\lib;C:\msys64\usr\lib\site-packages\;C:\msys64\mingw64\lib\python3.7\;C:\msys64\mingw64\lib\python3.7\site-packages\;C:\msys64\mingw64\lib\python3.7\site-packages\PyQt5;"; + - $env:MSYSTEM = "MINGW64" + - $originalPath = $env:Path + - $NewPath = "$CI_PROJECT_DIR\build\install-x64\bin;$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\bin;" + - $env:Path = $NewPath + $env:Path + - $PyABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python$PyABI\;C:\msys64\mingw64\lib\python$PyABI\site-packages\;C:\msys64\mingw64\lib\python$PyABI\site-packages\PyQt5;"; + - $env:Path = $NewPath + $originalPath + - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;" - $VERSION=(python3 src/launch.py -V) - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') @@ -117,8 +123,14 @@ windows-builder-x86: - if (-not (Test-Path "artifacts.zip")) { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/develop/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } - Expand-Archive -Path artifacts.zip -DestinationPath . - Copy-Item "$CI_PROJECT_DIR/build/install-x86/python/*" -Destination "$CI_PROJECT_DIR" - - $env:Path = "$CI_PROJECT_DIR\build\install-x86\lib;$CI_PROJECT_DIR\build\install-x86\python;C:\msys32\mingw32\bin;C:\msys32\mingw32\lib;C:\msys32\mingw32\lib\python3.7\;C:\msys32\mingw32\lib\python3.7\site-packages\;C:\msys32\mingw32\lib\python3.7\site-packages\PyQt5;C:\msys32\usr\lib\cmake\UnitTest++;C:\msys32\usr;C:\msys32\usr\lib;C:\msys32\usr\local\x64\mingw\bin;C:\msys32\usr\local;" + $env:Path; - - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;C:\msys32\usr\lib;C:\msys32\usr\lib\site-packages\;C:\msys32\mingw32\lib\python3.7\;C:\msys32\mingw32\lib\python3.7\site-packages\;C:\msys32\mingw32\lib\python3.7\site-packages\PyQt5;"; + - $env:MSYSTEM = "MINGW32" + - $originalPath = $env:Path + - $NewPath = "$CI_PROJECT_DIR\build\install-x86\bin;$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\bin;" + - $env:Path = $NewPath + $env:Path + - $PyABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python$PyABI\;C:\msys64\mingw32\lib\python$PyABI\site-packages\;C:\msys64\mingw32\lib\python$PyABI\site-packages\PyQt5;"; + - $env:Path = $NewPath + $originalPath + - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;" - $VERSION=(python3 src/launch.py -V) - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') diff --git a/installer/build_server.py b/installer/build_server.py index 7d83b38491..a3fdfe2959 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -445,7 +445,7 @@ def main(): for replace_name in paths_to_replace: if windows_32bit: shutil.copytree( - os.path.join('C:\\msys32\\mingw32\\share\\qt5\\plugins', replace_name), + os.path.join('C:\\msys64\\mingw32\\share\\qt5\\plugins', replace_name), os.path.join(exe_dir, replace_name)) else: shutil.copytree( @@ -459,8 +459,8 @@ def main(): ] if windows_32bit: paths_to_copy = [ - ("Qt5Core.dll", "C:\\msys32\\mingw32\\bin\\"), - ("Qt5Svg.dll", "C:\\msys32\\mingw32\\bin\\"), + ("Qt5Core.dll", "C:\\msys64\\mingw32\\bin\\"), + ("Qt5Svg.dll", "C:\\msys64\\mingw32\\bin\\"), ] for qt_file_name, qt_parent_path in paths_to_copy: qt5_path = os.path.join(qt_parent_path, qt_file_name) From 7bfa57cf5b6ce677177af08fd9329e166a5e431a Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sun, 18 Apr 2021 06:44:11 -0400 Subject: [PATCH 09/23] Look up Python ABI version, instead of hardcoding - Hardcoded 'python3.7', 'exe.mingw-3.7', etc. strings have been replaced with one constructed using the value of the sysconfig.py config variable 'py_version_short' - Only exceptions: macOS-only code in fix_qt5_rpath.py/build_mac_dmg.sh (XXX comments placed, warning of their lack of future-proofing) --- .gitlab-ci.yml | 2 +- installer/build-mac-dmg.sh | 3 +++ installer/build_server.py | 7 +++++-- installer/fix_qt5_rpath.py | 4 +++- installer/windows-installer.iss | 5 ++++- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c175ecdae8..c092fbdf55 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,7 +136,7 @@ windows-builder-x86: - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME - - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-3.7\openshot-qt.exe" + - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-$PyABI\openshot-qt.exe" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME" when: always except: diff --git a/installer/build-mac-dmg.sh b/installer/build-mac-dmg.sh index 793f0fc2ae..daf2d66134 100644 --- a/installer/build-mac-dmg.sh +++ b/installer/build-mac-dmg.sh @@ -1,4 +1,7 @@ #!/bin/sh + +# XXX: These paths should be set using `brew prefix` commands, +# for future-proofing against upgrades PATH=/usr/local/Cellar/python@3.7/3.7.9_2/Frameworks/Python.framework/Versions/3.7/bin:/opt/local/bin:/opt/local/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/qt5/5.5/clang_64/bin:/opt/X11/bin # Get Version diff --git a/installer/build_server.py b/installer/build_server.py index a3fdfe2959..87c57ae93a 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -34,6 +34,7 @@ import re import stat import subprocess +import sysconfig import traceback from github3 import login from requests.auth import HTTPBasicAuth @@ -41,6 +42,7 @@ from version_parser import parse_version_info, parse_build_name PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Primary openshot folder +PY_ABI = sysconfig.get_config_var('py_version_short') # Access info class (for version info) sys.path.append(os.path.join(PATH, 'src', 'classes')) @@ -410,8 +412,8 @@ def main(): if platform.system() == "Windows": # Move python folder structure, since Cx_Freeze doesn't put it in the correct place - exe_dir = os.path.join(PATH, 'build', 'exe.mingw-3.7') - python_dir = os.path.join(exe_dir, 'lib', 'python3.7') + exe_dir = os.path.join(PATH, 'build', 'exe.mingw-{}'.format(PY_ABI)) + python_dir = os.path.join(exe_dir, 'lib', 'python{}'.format(PY_ABI)) # Remove a redundant openshot_qt module folder (duplicates lots of files) duplicate_openshot_qt_path = os.path.join(python_dir, 'openshot_qt') @@ -521,6 +523,7 @@ def main(): '/Q', '/DVERSION=%s' % version, '/DONLY_64_BIT=%s' % only_64_bit, + '/DPY_EXE_DIR=%s' % "exe.mingw-{}".format(PY_ABI), '"%s"' % os.path.join(PATH, 'installer', 'windows-installer.iss'), ]) inno_output = "" diff --git a/installer/fix_qt5_rpath.py b/installer/fix_qt5_rpath.py index 395459edf8..21ccc651c3 100644 --- a/installer/fix_qt5_rpath.py +++ b/installer/fix_qt5_rpath.py @@ -112,7 +112,9 @@ def print_min_versions(PATH): if __name__ == "__main__": - """Run these methods manually for testing""" + # Run these methods manually for testing + + # XXX: This path should be set programmatically, somehow PATH = "/Users/jonathanthomas/apps/openshot-qt/build/exe.macosx-10.15-x86_64-3.7" fix_rpath(PATH) print_min_versions(PATH) diff --git a/installer/windows-installer.iss b/installer/windows-installer.iss index 3fb40a5a59..118b4c443a 100644 --- a/installer/windows-installer.iss +++ b/installer/windows-installer.iss @@ -7,6 +7,9 @@ #ifndef ONLY_64_BIT #define ONLY_64_BIT "x64" #endif +#ifndef PY_EXE_DIR + #define PY_EXE_DIR "exe.mingw-3.8" +#endif #define MyAppName "OpenShot Video Editor" @@ -138,7 +141,7 @@ Root: HKLM; Subkey: "Software\Classes\OpenShotProject\shell\open\command"; Value [Files] ; Add all frozen files from cx_Freeze build -Source: "..\build\exe.mingw-3.7\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs +Source: "..\build\{#PY_EXE_DIR}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs [Icons] Name: "{commonprograms}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}" From 9a16b5214fede3918a34a6550cb948769bc6e87e Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sun, 18 Apr 2021 06:51:52 -0400 Subject: [PATCH 10/23] Gitlab-ci: Make PY_ABI an envvar on Windows --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c092fbdf55..381bc17c7b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,8 +93,8 @@ windows-builder-x64: - $originalPath = $env:Path - $NewPath = "$CI_PROJECT_DIR\build\install-x64\bin;$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\bin;" - $env:Path = $NewPath + $env:Path - - $PyABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") - - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python$PyABI\;C:\msys64\mingw64\lib\python$PyABI\site-packages\;C:\msys64\mingw64\lib\python$PyABI\site-packages\PyQt5;"; + - $env:PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python$PY_ABI\;C:\msys64\mingw64\lib\python$PY_ABI\site-packages\;C:\msys64\mingw64\lib\python$PY_ABI\site-packages\PyQt5;"; - $env:Path = $NewPath + $originalPath - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;" - $VERSION=(python3 src/launch.py -V) @@ -127,8 +127,8 @@ windows-builder-x86: - $originalPath = $env:Path - $NewPath = "$CI_PROJECT_DIR\build\install-x86\bin;$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\bin;" - $env:Path = $NewPath + $env:Path - - $PyABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") - - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python$PyABI\;C:\msys64\mingw32\lib\python$PyABI\site-packages\;C:\msys64\mingw32\lib\python$PyABI\site-packages\PyQt5;"; + - $env:PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python$PY_ABI\;C:\msys64\mingw32\lib\python$PY_ABI\site-packages\;C:\msys64\mingw32\lib\python$PY_ABI\site-packages\PyQt5;"; - $env:Path = $NewPath + $originalPath - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;" - $VERSION=(python3 src/launch.py -V) @@ -136,7 +136,7 @@ windows-builder-x86: - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME - - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-$PyABI\openshot-qt.exe" + - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-$PY_ABI\openshot-qt.exe" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME" when: always except: From 1a70c0cdf6b23dd1595bbc2312daca62612837c4 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sun, 18 Apr 2021 16:39:50 -0400 Subject: [PATCH 11/23] Gitlab-CI: Fix use of PowerShell variables --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 381bc17c7b..d19b7356e2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,8 +93,8 @@ windows-builder-x64: - $originalPath = $env:Path - $NewPath = "$CI_PROJECT_DIR\build\install-x64\bin;$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\bin;" - $env:Path = $NewPath + $env:Path - - $env:PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") - - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python$PY_ABI\;C:\msys64\mingw64\lib\python$PY_ABI\site-packages\;C:\msys64\mingw64\lib\python$PY_ABI\site-packages\PyQt5;"; + - $PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python" + $PY_ABI + "\;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\PyQt5;"; - $env:Path = $NewPath + $originalPath - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;" - $VERSION=(python3 src/launch.py -V) @@ -127,8 +127,8 @@ windows-builder-x86: - $originalPath = $env:Path - $NewPath = "$CI_PROJECT_DIR\build\install-x86\bin;$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\bin;" - $env:Path = $NewPath + $env:Path - - $env:PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") - - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python$PY_ABI\;C:\msys64\mingw32\lib\python$PY_ABI\site-packages\;C:\msys64\mingw32\lib\python$PY_ABI\site-packages\PyQt5;"; + - $PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") + - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python" + $PY_ABI + "\;C:\msys64\mingw32\lib\python" + $PY_ABI + "\site-packages\;C:\msys64\mingw32\lib\python " + $PY_ABI+ "\site-packages\PyQt5;"; - $env:Path = $NewPath + $originalPath - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;" - $VERSION=(python3 src/launch.py -V) @@ -136,7 +136,7 @@ windows-builder-x86: - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME - - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-$PY_ABI\openshot-qt.exe" + - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-" + $PY_ABI + "\openshot-qt.exe" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME" when: always except: From 22fda5a8648f02aa5130e8dc38038136137e3153 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Sun, 18 Apr 2021 18:33:41 -0400 Subject: [PATCH 12/23] Gitlab-CI: Add PyQt5 back to PYTHONPATH, crazily - Somehow, having it there makes PyQt5 freezable. That shouldn't be the case, and it seems like a bad sign. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d19b7356e2..0f817baf11 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,7 +96,7 @@ windows-builder-x64: - $PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") - $NewPath = $NewPath + "C:\msys64\mingw64\lib\python" + $PY_ABI + "\;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\PyQt5;"; - $env:Path = $NewPath + $originalPath - - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;" + - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\PyQt5;" - $VERSION=(python3 src/launch.py -V) - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') @@ -130,7 +130,7 @@ windows-builder-x86: - $PY_ABI = (python3 -c "import sysconfig; print(sysconfig.get_config_var('py_version_short'))") - $NewPath = $NewPath + "C:\msys64\mingw32\lib\python" + $PY_ABI + "\;C:\msys64\mingw32\lib\python" + $PY_ABI + "\site-packages\;C:\msys64\mingw32\lib\python " + $PY_ABI+ "\site-packages\PyQt5;"; - $env:Path = $NewPath + $originalPath - - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;" + - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\lib\python" + $PY_ABI + "\site-packages\PyQt5;" - $VERSION=(python3 src/launch.py -V) - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') From a0fca2b08eb059575e303972069ae0f8e2b5de78 Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Mon, 26 Apr 2021 03:30:38 -0700 Subject: [PATCH 13/23] Don't set fractional values for int properties (#4068) --- src/windows/views/properties_tableview.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/windows/views/properties_tableview.py b/src/windows/views/properties_tableview.py index 9898fd4d4e..825d73e392 100644 --- a/src/windows/views/properties_tableview.py +++ b/src/windows/views/properties_tableview.py @@ -272,6 +272,9 @@ def mouseMoveEvent(self, event): self.new_value = max(property_min, self.new_value) self.new_value = min(property_max, self.new_value) + if property_type == "int": + self.new_value = round(self.new_value, 0) + # Update value of this property self.clip_properties_model.value_updated(self.selected_item, -1, self.new_value) From f222cdcd7b1edace1f8b6c27f67c11d607983d04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Apr 2021 06:31:23 -0400 Subject: [PATCH 14/23] Bump actions/upload-artifact from v2.2.2 to v2.2.3 (#4089) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from v2.2.2 to v2.2.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v2.2.2...ee69f02b3dfdecd58bb31b4d133da38ba6fe3700) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/sphinx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sphinx.yml b/.github/workflows/sphinx.yml index 89863bef64..3079dbda46 100644 --- a/.github/workflows/sphinx.yml +++ b/.github/workflows/sphinx.yml @@ -26,7 +26,7 @@ jobs: with: docs-folder: "doc/" # Create an artifact out of the generated HTML - - uses: actions/upload-artifact@v2.2.2 + - uses: actions/upload-artifact@v2.2.3 with: name: UserGuideHTML path: "doc/_build/html/" From 2a89045d4a23f9ef06730e48d113678fea45a56a Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 07:17:36 -0400 Subject: [PATCH 15/23] Small change to github3 API --- installer/build_server.py | 2 +- installer/prune_daily_builds.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/installer/build_server.py b/installer/build_server.py index 87c57ae93a..099482e840 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -153,7 +153,7 @@ def get_release(repo, tag_name): @param repo: github3 repository object @returns: github3 release object or None """ - for release in repo.iter_releases(): + for release in repo.releases(): if release.tag_name == tag_name: return release diff --git a/installer/prune_daily_builds.py b/installer/prune_daily_builds.py index ad9c0cc4cf..a91d01d38f 100644 --- a/installer/prune_daily_builds.py +++ b/installer/prune_daily_builds.py @@ -38,8 +38,7 @@ MAXIMUM_ASSET_AGE_DAYS = 180 # Calculate current date (with timezone) -utc=pytz.UTC -now = utc.localize(datetime.datetime.now()) +now = pytz.UTC.localize(datetime.datetime.now()) def get_release(repo, tag_name): @@ -47,7 +46,7 @@ def get_release(repo, tag_name): @param repo: github3 repository object @returns: github3 release object or None """ - for release in repo.iter_releases(): + for release in repo.releases(): if release.tag_name == tag_name: return release @@ -83,4 +82,4 @@ def get_release(repo, tag_name): print("------------------") print("Deleted %s Assets" % delete_count) - print("Skipped %s Assets" % skip_count) \ No newline at end of file + print("Skipped %s Assets" % skip_count) From 3cf7717c61cbfbf0815c4dad70aa5dfeb4f8aba4 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 07:17:53 -0400 Subject: [PATCH 16/23] deploy.py: Reformat long lines --- installer/deploy.py | 141 +++++++++++++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 42 deletions(-) diff --git a/installer/deploy.py b/installer/deploy.py index 1a9b5ad6ae..3cdbb185e8 100644 --- a/installer/deploy.py +++ b/installer/deploy.py @@ -50,7 +50,7 @@ urllib3.disable_warnings() -if __name__ == "__main__": +def main(): # Only run this code when directly executing this script. zulip_token = None @@ -82,15 +82,18 @@ script_mode = "publish" # Start log - output("%s %s Log for %s" % (platform.system(), script_mode, - datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) + output("%s %s Log for %s" % ( + platform.system(), + script_mode, + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) # Detect artifact folder (if any) artifact_dir = os.path.join(PATH, "build") # Parse artifact version files (if found) for repo_name in repo_names: - version_info.update(parse_version_info(os.path.join(artifact_dir, "install-x64", "share", repo_name))) + version_info.update(parse_version_info( + os.path.join(artifact_dir, "install-x64", "share", repo_name))) output(str(version_info)) # Get version info @@ -104,8 +107,9 @@ original_git_branch = git_branch_name if original_git_branch != git_branch_name: # Branch names do not match - raise Exception("Branch names do not match for all 3 repos: `%s` vs `%s`" % - (original_git_branch, git_branch_name)) + raise Exception( + "Branch names do not match for all 3 repos: `%s` vs `%s`" % + (original_git_branch, git_branch_name)) # Loop through and get and/or create the GitHub Release objects releases = {} @@ -122,14 +126,18 @@ # Get official version release (i.e. v2.1.0, v2.x.x) releases[repo_name] = get_release(repos.get(repo_name), github_release_name) if releases.get(repo_name) and releases.get(repo_name).prerelease is False: - raise Exception("GitHub release for version %s is already released. Did we forget to bump a " - "version? (repo: %s, branch: %s)" % (github_release_name, repo_name, - git_branch_name)) + raise Exception( + "GitHub release for version %s is already released. Did we forget to bump a " + "version? (repo: %s, branch: %s)" % ( + github_release_name, repo_name, + git_branch_name) + ) else: # ignore all branches that don't start with 'release*' - raise Exception("%s only allowed for branch names that start with 'release*'" - " (repo: %s, branch: %s)" % - (script_mode, repo_name, git_branch_name)) + raise Exception( + "%s only allowed for branch names that start with 'release*'" + " (repo: %s, branch: %s)" % ( + script_mode, repo_name, git_branch_name)) if not is_publish: @@ -152,9 +160,11 @@ so_title = "" if so_number: so_title = ", SO: %s" % so_number - log_markdown = "%s Changelog (Version: %s%s)\n---\n%s\n\n" % \ - (repo_name, version_info.get(repo_name, {}).get('VERSION'), - so_title, logs.get(repo_name)) + log_markdown = "%s Changelog (Version: %s%s)\n---\n%s\n\n" % ( + repo_name, + version_info.get(repo_name, {}).get('VERSION'), + so_title, + logs.get(repo_name)) combined_log_markdown += log_markdown if not repo_name == "openshot-qt": formatted_logs[repo_name] = log_title + log_markdown @@ -167,10 +177,11 @@ github_release_name = "v%s" % version_info.get(repo_name, {}).get('VERSION') if not releases.get(repo_name): # Create a new release if one if missing (for each repo) - releases[repo_name] = repos.get(repo_name).create_release(github_release_name, - target_commitish=git_branch_name, - prerelease=True, - body=formatted_logs.get(repo_name)) + releases[repo_name] = repos.get(repo_name).create_release( + github_release_name, + target_commitish=git_branch_name, + prerelease=True, + body=formatted_logs.get(repo_name)) # Upload all deploy artifacts/installers to GitHub # NOTE: ONLY for `openshot-qt` repo @@ -226,9 +237,11 @@ if line: shasum_check_output = line.decode('UTF-8').strip() if shasum_output.split(" ")[0] != shasum_check_output.split(" ")[0]: - raise Exception("shasum validation of %s has failed after downloading " - "the uploaded installer: %s.\n%s\n%s" - % (check_artifact_path, artifact_path, shasum_check_output, shasum_output )) + raise Exception( + "shasum validation of %s has failed after downloading " + "the uploaded installer: %s.\n%s\n%s" % ( + check_artifact_path, artifact_path, shasum_check_output, shasum_output) + ) # Create and upload sha2sum file and instructions output("Hash generated: %s" % shasum_output) @@ -249,7 +262,19 @@ # Create torrent and upload torrent_name = "%s.torrent" % artifact_name torrent_path = os.path.join(artifact_dir, torrent_name) - torrent_command = 'mktorrent -a "udp://tracker.openbittorrent.com:80/announce, udp://tracker.publicbt.com:80/announce, udp://tracker.opentrackr.org:1337" -c "OpenShot Video Editor %s" -w "%s" -o "%s" "%s"' % (github_release.tag_name, download_url, torrent_name, artifact_name) + tracker_list = [ + "udp://tracker.openbittorrent.com:80/announce", + "udp://tracker.publicbt.com:80/announce", + "udp://tracker.opentrackr.org:1337", + ] + torrent_command = " ".join([ + 'mktorrent', + '-a "%s"' % (", ".join(tracker_list)), + '-c "OpenShot Video Editor %s"' % github_release.tag_name, + '-w "%s"' % download_url, + '-o "%s"' % torrent_name, + '"%s"' % artifact_name, + ]) torrent_output = "" for line in run_command(torrent_command, artifact_dir): output(line) @@ -265,18 +290,33 @@ # Submit blog post (if it doesn't already exist) (in draft mode) auth = HTTPBasicAuth(os.getenv('OPENSHOT_ORG_USER'), os.getenv('OPENSHOT_ORG_PASS')) - r = post("https://www.openshot.org/api/release/submit/", auth=auth, data={ "version": openshot_qt_version, - "changelog": log_title + combined_log_markdown }) + r = post( + "https://www.openshot.org/api/release/submit/", + auth=auth, + data={ + "version": openshot_qt_version, + "changelog": log_title + combined_log_markdown + }) if not r.ok: - raise Exception("HTTP post to openshot.org/api/release/submit/ failed: %s (user: %s): %s" % - (r.status_code, os.getenv('OPENSHOT_ORG_USER'), r.content.decode('UTF-8'))) + raise Exception( + "HTTP post to openshot.org/api/release/submit/ failed: %s (user: %s): %s" % ( + r.status_code, + os.getenv('OPENSHOT_ORG_USER'), + r.content.decode('UTF-8')) + ) else: # Publish the release (make new version visible on openshot.org, and make blog post visible) auth = HTTPBasicAuth(os.getenv('OPENSHOT_ORG_USER'), os.getenv('OPENSHOT_ORG_PASS')) - r = post("https://www.openshot.org/api/release/publish/", auth=auth, data={"version": openshot_qt_version }) + r = post( + "https://www.openshot.org/api/release/publish/", + auth=auth, + data={"version": openshot_qt_version}) if not r.ok: - raise Exception("HTTP post to openshot.org/api/release/publish/ failed: %s (user: %s): %s" % - (r.status_code, os.getenv('OPENSHOT_ORG_USER'), r.content.decode('UTF-8'))) + raise Exception( + "HTTP post to openshot.org/api/release/publish/ failed: %s (user: %s): %s" % ( + r.status_code, + os.getenv('OPENSHOT_ORG_USER'), + r.content.decode('UTF-8'))) # Publish GitHub Release objects (in all 3 repos) for repo_name in repo_names: @@ -286,8 +326,10 @@ # Publish github release also github_release.edit(prerelease=False) else: - raise Exception("Cannot publish missing GitHub release: %s, version: %s" % - (repo_name, openshot_qt_version)) + raise Exception( + "Cannot publish missing GitHub release: %s, version: %s" % ( + repo_name, + openshot_qt_version)) # Verify download links on openshot.org are correct (and include the new release version) r = get("https://www.openshot.org/download/") @@ -300,15 +342,23 @@ if r.ok and r.reason == "Found": output("Validation of URL successful: %s" % url) else: - raise Exception("Validation of URL FAILED: %s, %s, %s" % (url, r.status_code, r.reason)) + raise Exception( + "Validation of URL FAILED: %s, %s, %s" % ( + url, r.status_code, r.reason) + ) # Validate the current version is found in each URL if openshot_qt_version not in url: - raise Exception("Validation of URL FAILED. Missing version %s: %s, %s, %s" % - (openshot_qt_version, url, - r.status_code, r.reason)) + raise Exception( + "Validation of URL FAILED. Missing version %s: %s, %s, %s" % ( + openshot_qt_version, + url, + r.status_code, + r.reason) + ) else: - raise Exception("Failed to GET openshot.org/download for URL validation: %s" % r.status_code) + raise Exception( + "Failed to GET openshot.org/download for URL validation: %s" % r.status_code) except Exception as ex: tb = traceback.format_exc() @@ -316,12 +366,19 @@ if not errors_detected: output("Successfully completed %s script!" % script_mode) - zulip_upload_log(zulip_token, log, - "%s: %s **success** log" % (platform.system(), script_mode), - ":congratulations: successful %s" % script_mode) + zulip_upload_log( + zulip_token, log, + "%s: %s **success** log" % (platform.system(), script_mode), + ":congratulations: successful %s" % script_mode) else: # Report any errors detected output("%s script failed!" % script_mode) - zulip_upload_log(zulip_token, log, "%s: %s error log" % (platform.system(), script_mode), - ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) + zulip_upload_log( + zulip_token, log, + "%s: %s error log" % (platform.system(), script_mode), + ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) exit(1) + + +if __name__ == "__main__": + main() From 3c6f7832d511f6506959ef57a4a37ac0c3287d65 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 07:32:53 -0400 Subject: [PATCH 17/23] Munge command strings with shlex --- installer/build_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/installer/build_server.py b/installer/build_server.py index 099482e840..107a76d8d2 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -30,8 +30,9 @@ import datetime import platform -import shutil import re +import shutil +import shlex import stat import subprocess import sysconfig @@ -80,7 +81,7 @@ def output(line): def run_command(command, working_dir=None): """Utility function to return output from command line""" - short_command = command.split('" ')[0] # We don't need to print args + short_command = shlex.split(command)[0] # We don't need to print args output("Running %s... (%s)" % (short_command, working_dir)) p = subprocess.Popen( command, From d9dcd179e52d6f986c621266ffffb0e7f43a47f5 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 07:38:54 -0400 Subject: [PATCH 18/23] Fix editbit command --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0f817baf11..33dd9df9eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,7 +136,8 @@ windows-builder-x86: - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME - - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-" + $PY_ABI + "\openshot-qt.exe" + - $EXE_PATH = "$CI_PROJECT_DIR\build\exe.mingw-" + $PY_ABI + "\openshot-qt.exe" + - editbin /LARGEADDRESSAWARE "$EXE_PATH" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME" when: always except: From 64aec0fb1dfe3a00d189e2c9ff067b3f6d6e7e1b Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 07:42:04 -0400 Subject: [PATCH 19/23] Handle multiple github3 versions --- installer/build_server.py | 6 +++++- installer/prune_daily_builds.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/installer/build_server.py b/installer/build_server.py index 107a76d8d2..5bec294e0c 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -154,7 +154,11 @@ def get_release(repo, tag_name): @param repo: github3 repository object @returns: github3 release object or None """ - for release in repo.releases(): + if hasattr(repo, 'releases'): + release_iter = repo.releases() + else: + release_iter = repo.iter_releases() + for release in release_iter: if release.tag_name == tag_name: return release diff --git a/installer/prune_daily_builds.py b/installer/prune_daily_builds.py index a91d01d38f..b63532d63a 100644 --- a/installer/prune_daily_builds.py +++ b/installer/prune_daily_builds.py @@ -46,7 +46,11 @@ def get_release(repo, tag_name): @param repo: github3 repository object @returns: github3 release object or None """ - for release in repo.releases(): + if hasattr(repo, 'releases'): + release_iter = repo.releases() + else: + release_iter = repo.iter_releases() + for release in release_iter: if release.tag_name == tag_name: return release From b54ae4667e8078e17fd70cb05dcc325aca7fdd2b Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 08:06:34 -0400 Subject: [PATCH 20/23] Rename version_info files to NAME.env --- .gitlab-ci.yml | 20 ++++++++++++-------- installer/build_server.py | 3 ++- installer/deploy.py | 10 +++++++--- installer/version_parser.py | 7 +++++-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33dd9df9eb..144696978b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,8 @@ linux-builder: paths: - build/*.AppImage - build/*.torrent - - build/install-x64/share/ + - build/install-x64/share/*.env + - build/install-x64/share/*.log - build/build-server.log script: - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=linux-builder" @@ -21,7 +22,7 @@ linux-builder: - cp -r "$CI_PROJECT_DIR/build/install-x64/python/." "$CI_PROJECT_DIR" - export LD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:$LD_LIBRARY_PATH - VERSION=$(python3 src/launch.py -V) - - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME" + - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME.env" - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log" - cd doc; make html SPHINXOPTS="-D html_theme_options.analytics_id=UA-4381101-5"; cd ..; - ~/auto-update-sphinx "$CI_PROJECT_DIR/build" "$CI_COMMIT_REF_NAME" @@ -41,7 +42,8 @@ mac-builder: paths: - build/*.dmg - build/*.torrent - - build/install-x64/share/ + - build/install-x64/share/*.env + - build/install-x64/share/*.log - build/build-server.log script: - "curl -O -J -L --header PRIVATE-TOKEN:$ACCESS_TOKEN http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=mac-builder" @@ -65,7 +67,7 @@ mac-builder: - export LD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:$LD_LIBRARY_PATH - export DYLD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:DYLD_LIBRARY_PATH - VERSION=$(python3 src/launch.py -V) - - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME" + - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID\nCI_PIPELINE_ID:$CI_PIPELINE_ID\nVERSION:$VERSION" > "build/install-x64/share/$CI_PROJECT_NAME.env" - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py bdist_mac --git-branch=$CI_COMMIT_REF_NAME --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor" - python3 -u installer/build_server.py "$SLACK_TOKEN" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME" @@ -82,7 +84,8 @@ windows-builder-x64: paths: - build\*.exe - build\*.torrent - - build\install-x64\share\ + - build\install-x64\share\*.env + - build\install-x64\share\*.log - build\build-server.log script: - try { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x64" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } catch { $_.Exception.Response.StatusCode.Value__ } @@ -98,7 +101,7 @@ windows-builder-x64: - $env:Path = $NewPath + $originalPath - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x64\python;C:\msys64\mingw64\lib\python" + $PY_ABI + "\site-packages\PyQt5;" - $VERSION=(python3 src/launch.py -V) - - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force + - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME.env" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME @@ -116,7 +119,8 @@ windows-builder-x86: paths: - build\*.exe - build\*.torrent - - build\install-x86\share\ + - build\install-x86\share\*.env + - build\install-x86\share\*.log - build\build-server.log script: - try { Invoke-WebRequest -Uri "http://gitlab.openshot.org/OpenShot/libopenshot/-/jobs/artifacts/$CI_COMMIT_REF_NAME/download?job=windows-builder-x86" -Headers @{"PRIVATE-TOKEN"="$ACCESS_TOKEN"} -OutFile "artifacts.zip" } catch { $_.Exception.Response.StatusCode.Value__ } @@ -132,7 +136,7 @@ windows-builder-x86: - $env:Path = $NewPath + $originalPath - $env:PYTHONPATH = "$CI_PROJECT_DIR\build\install-x86\python;C:\msys64\mingw32\lib\python" + $PY_ABI + "\site-packages\PyQt5;" - $VERSION=(python3 src/launch.py -V) - - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force + - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME.env" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID`nCI_PIPELINE_ID:$env:CI_PIPELINE_ID`nVERSION:$VERSION" -ItemType file -force - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^') - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"- %C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log" - python3 -u freeze.py build --git-branch=$CI_COMMIT_REF_NAME diff --git a/installer/build_server.py b/installer/build_server.py index 5bec294e0c..2bfb178f0f 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -242,8 +242,9 @@ def main(): # Parse artifact version files (if found) for repo_name in ["libopenshot-audio", "libopenshot", "openshot-qt"]: + data_file = f"{repo_name}.env" version_info.update( - parse_version_info(os.path.join(artifact_path, "share", repo_name))) + parse_version_info(os.path.join(artifact_path, "share", data_file))) output(str(version_info)) # Get GIT description of openshot-qt-git branch (i.e. v2.0.6-18-ga01a98c) diff --git a/installer/deploy.py b/installer/deploy.py index 3cdbb185e8..9335bf0d70 100644 --- a/installer/deploy.py +++ b/installer/deploy.py @@ -35,8 +35,11 @@ from github3 import login from requests.auth import HTTPBasicAuth from requests import post, get, head -from build_server import output, run_command, error, truncate, zulip_upload_log, get_release, upload, \ - errors_detected, log, version_info, parse_version_info +from build_server import ( + output, run_command, error, truncate, + zulip_upload_log, get_release, upload, + errors_detected, log, + version_info, parse_version_info) PATH = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) # Primary openshot folder RELEASE_NAME_REGEX = re.compile(r'^OpenShot-v.*?(-.*?)-x86[_64]*') @@ -92,8 +95,9 @@ def main(): # Parse artifact version files (if found) for repo_name in repo_names: + data_file = f"{repo_name}.env" version_info.update(parse_version_info( - os.path.join(artifact_dir, "install-x64", "share", repo_name))) + os.path.join(artifact_dir, "install-x64", "share", data_file))) output(str(version_info)) # Get version info diff --git a/installer/version_parser.py b/installer/version_parser.py index caace3658c..6421613ed4 100644 --- a/installer/version_parser.py +++ b/installer/version_parser.py @@ -36,7 +36,7 @@ def parse_version_info(version_path): """Parse version info from gitlab artifacts""" - version_info = { "date": f'{datetime.datetime.now():%Y-%m-%d %H:%M}' } + version_info = {"date": f'{datetime.datetime.now():%Y-%m-%d %H:%M}'} # Get name of version file version_name = os.path.basename(version_path) @@ -53,10 +53,13 @@ def parse_version_info(version_path): if os.path.exists(version_path): with open(version_path, "r") as f: # Parse each line in f as a 'key:value' string - version_info[version_name].update((l.strip().split(':') for l in f.readlines())) + version_info[version_name].update( + (ln.strip().split(':') for ln in f.readlines()) + ) return version_info + def parse_build_name(version_info, git_branch_name=""): """Calculate the build name used in URLs and Installers from the version_info dict""" # Check for all version info needed to construct the build URL From 8a868b51043d41e2d618f580f9865ce21289d909 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Mon, 26 Apr 2021 08:11:46 -0400 Subject: [PATCH 21/23] Fix env-file parsing --- installer/version_parser.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/installer/version_parser.py b/installer/version_parser.py index 6421613ed4..3250918b14 100644 --- a/installer/version_parser.py +++ b/installer/version_parser.py @@ -39,7 +39,8 @@ def parse_version_info(version_path): version_info = {"date": f'{datetime.datetime.now():%Y-%m-%d %H:%M}'} # Get name of version file - version_name = os.path.basename(version_path) + file_name = os.path.basename(version_path) + version_name = os.path.splitext(file_name)[0] version_info[version_name] = { "CI_PROJECT_NAME": None, "CI_COMMIT_REF_NAME": None, @@ -102,9 +103,9 @@ def parse_build_name(version_info, git_branch_name=""): # Parse all version info (if found) for git_log_filename in os.listdir(settings_path): - git_log_filepath = os.path.join(settings_path, git_log_filename) - if os.path.splitext(git_log_filepath)[1] == "": - # No extension, parse version info + if git_log_filename.endswith(".env"): + # Metadata file, parse version info + git_log_filepath = os.path.join(settings_path, git_log_filename) version_info.update(parse_version_info(git_log_filepath)) # Calculate build name from version info From 9050d67df0547bdcf6e7b57625493e396ae78060 Mon Sep 17 00:00:00 2001 From: "FeRD (Frank Dana)" Date: Wed, 28 Apr 2021 08:46:00 -0400 Subject: [PATCH 22/23] Freeze: Include OpenGL/OpenGL_accelerate on Win32 --- freeze.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freeze.py b/freeze.py index 36cd55e0a1..c138256986 100644 --- a/freeze.py +++ b/freeze.py @@ -91,7 +91,7 @@ "requests", "zmq", "webbrowser", - "json" + "json", ] # Modules to include @@ -203,7 +203,11 @@ def find_files(directory, patterns): src_files.append((os.path.join(PATH, "installer", "launch-win.bat"), "launch-win.bat")) # Add additional package - python_packages.append('idna') + python_packages.extend([ + "idna", + "OpenGL", + "OpenGL_accelerate", + ]) # Manually add zmq dependency (windows does not freeze it correctly) import zmq From 53c9029cd51b88bfc49992779f7b1f17af1e53d2 Mon Sep 17 00:00:00 2001 From: Frank Dana Date: Wed, 28 Apr 2021 15:59:42 -0700 Subject: [PATCH 23/23] Gitlab builders: work around github3 API differences (#4109) --- installer/build_server.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/installer/build_server.py b/installer/build_server.py index 2bfb178f0f..1f3a064cd3 100644 --- a/installer/build_server.py +++ b/installer/build_server.py @@ -170,10 +170,17 @@ def upload(file_path, github_release): # Delete any matching assets already in this github release # so we don't have any collisions. - for asset in github_release.assets: + if hasattr(github_release, 'original_assets'): + asset_list = github_release.original_assets + else: + asset_list = github_release.assets + for asset in asset_list: if asset.name == file_name: output("GitHub: Removing conflicting installer asset from %s: %s" % (github_release.tag_name, file_name)) - asset._delete(asset._api) + if hasattr(asset, 'delete'): + asset.delete() + else: + asset._delete(asset._api) break for attempt in range(3): @@ -183,7 +190,10 @@ def upload(file_path, github_release): # Upload to GitHub output("GitHub: Uploading asset from %s: %s" % (github_release.tag_name, file_name)) asset = github_release.upload_asset("application/octet-stream", file_name, f) - url = asset.to_json()["browser_download_url"] + if hasattr(asset, 'browser_download_url'): + url = asset.browser_download_url + else: + url = asset.to_json()["browser_download_url"] # Successfully uploaded! break except Exception as ex: