From dd3ee4ae94fbf816b1f60badb46e926bb3d30170 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 4 Dec 2018 02:08:00 -0600 Subject: [PATCH 1/3] Replace Slack messaging during build server with Zulip --- installer/build-server.py | 73 ++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/installer/build-server.py b/installer/build-server.py index 7d1ed8479d..d348d2dd0f 100644 --- a/installer/build-server.py +++ b/installer/build-server.py @@ -33,7 +33,6 @@ import datetime import platform import shutil -from slacker import Slacker import re import stat import subprocess @@ -42,6 +41,8 @@ import time import traceback from github3 import login +from requests.auth import HTTPBasicAuth +from requests import post # Access info class (for version info) sys.path.append(os.path.join(PATH, 'src', 'classes')) @@ -50,8 +51,7 @@ freeze_command = None errors_detected = [] make_command = "make" -slack_token = None -slack_object = None +zulip_token = None s3_access_key = None s3_secret_key = None s3_connection = None @@ -75,6 +75,7 @@ def run_command(command, working_dir=None): stderr=subprocess.STDOUT) return iter(p.stdout.readline, b"") + def output(line): """Append output to list and print it""" print(line) @@ -86,6 +87,7 @@ def output(line): line += "\n" log.write(line) + def error(line): """Append error output to list and print it""" print("Error: %s" % line) @@ -95,6 +97,7 @@ def error(line): else: log.write(line) + def truncate(message, max=256): """Truncate the message with ellipses""" if len(message) < max: @@ -102,33 +105,42 @@ def truncate(message, max=256): else: return "%s..." % message[:max] -def slack(message): - """Append a message to slack #build-server channel""" - print("Slack: %s" % message) - if slack_object: - slack_object.chat.post_message("#build-server", truncate(message[:256])) -def slack_upload_log(log, title, comment=None): - """Upload a file to slack and notify a slack channel""" +def zulip_upload_log(log, title, comment=None): + """Upload a file to zulip and notify a zulip channel""" # Close log file log.close() - print("Slack Upload: %s" % log_path) - if slack_object: - for attempt in range(3): - try: - # Upload build log to slack (and append platform icon to comment [:linux:, :windows:, or :darwin:]) - slack_object.files.upload(log_path, filetype="txt", filename="%s-build-server.txt" % platform.system(), - title=title, initial_comment=':%s: %s' % (platform.system().lower(), comment), channels="#build-server") - # Successfully uploaded! - break - except Exception as ex: - # Quietly fail, and try again - if attempt < 2: - output("Upload log to Slack failed... trying again") - else: - # Throw loud exception - raise Exception('Log upload to Slack failed: %s' % log_path, ex) + print("Zulip Upload: %s" % log_path) + + # Authentication for Zulip + zulip_auth = HTTPBasicAuth('builder-bot@openshot.zulipchat.com', zulip_token) + filename = "%s-build-server.txt" % platform.system() + + # Upload file to Zulip + zulip_url = 'https://openshot.zulipchat.com/api/v1/user_uploads' + zulip_upload_url = '' + resp = post(zulip_url, data={}, auth=zulip_auth, files={filename: (filename, open(log_path, "rb"))}) + if resp.ok: + zulip_upload_url = resp.json().get("uri", "") + + # Determine topic + topic = "Successful Builds" + if "skull" in comment: + topic = "Failed Builds" + + # SEND MESSAGE + zulip_url = 'https://openshot.zulipchat.com/api/v1/messages' + zulip_data = { + "type": "stream", + "to": "build-server", + "subject": topic, + "content": ':%s: %s [Build Log](%s)' % (platform.system().lower(), comment, zulip_upload_url) + } + + resp = post(zulip_url, data=zulip_data, auth=zulip_auth) + if resp.ok: + print(resp) # Re-open the log (for append) log = open(log_path, "a") @@ -191,8 +203,7 @@ def parse_version_info(version_path): try: # Validate command-line arguments if len(sys.argv) >= 2: - slack_token = sys.argv[1] - slack_object = Slacker(slack_token) + zulip_token = sys.argv[1] if len(sys.argv) >= 4: s3_access_key = sys.argv[2] s3_secret_key = sys.argv[3] @@ -492,8 +503,8 @@ def parse_version_info(version_path): # Torrent succeeded! Upload the torrent to github url = upload(torrent_path, github_release) - # Notify Slack - slack_upload_log(log, "%s: Build logs for %s" % (platform.system(), app_name), "Successful *%s* build: %s" % (git_branch_name, download_url)) + # Notify Zulip + zulip_upload_log(log, "%s: Build logs for %s" % (platform.system(), app_name), "Successful *%s* build: %s" % (git_branch_name, download_url)) except Exception as ex: tb = traceback.format_exc() @@ -502,6 +513,6 @@ def parse_version_info(version_path): # Report any errors detected if errors_detected: - slack_upload_log(log, "%s: Error log for *%s* build" % (platform.system(), git_branch_name), ":skull_and_crossbones: %s" % truncate(errors_detected[0], 150)) + zulip_upload_log(log, "%s: Error log for *%s* build" % (platform.system(), git_branch_name), ":skull_and_crossbones: %s" % truncate(errors_detected[0], 100)) exit(1) From e872f349fb83b797f703bfcdbe67491ac1734dff Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 4 Dec 2018 02:50:10 -0600 Subject: [PATCH 2/3] Fix debugging code in build server script --- installer/build-server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/installer/build-server.py b/installer/build-server.py index d348d2dd0f..1c353febba 100644 --- a/installer/build-server.py +++ b/installer/build-server.py @@ -111,7 +111,7 @@ def zulip_upload_log(log, title, comment=None): # Close log file log.close() - print("Zulip Upload: %s" % log_path) + output("Zulip Upload: %s" % log_path) # Authentication for Zulip zulip_auth = HTTPBasicAuth('builder-bot@openshot.zulipchat.com', zulip_token) @@ -140,7 +140,7 @@ def zulip_upload_log(log, title, comment=None): resp = post(zulip_url, data=zulip_data, auth=zulip_auth) if resp.ok: - print(resp) + output(resp) # Re-open the log (for append) log = open(log_path, "a") From d584a4b619a94adc30a297d7e6adfbbfe0a518b5 Mon Sep 17 00:00:00 2001 From: Jonathan Thomas Date: Tue, 4 Dec 2018 03:04:21 -0600 Subject: [PATCH 3/3] More fixing debug code in build-server script --- installer/build-server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installer/build-server.py b/installer/build-server.py index 1c353febba..d80ffa71e8 100644 --- a/installer/build-server.py +++ b/installer/build-server.py @@ -108,11 +108,11 @@ def truncate(message, max=256): def zulip_upload_log(log, title, comment=None): """Upload a file to zulip and notify a zulip channel""" + output("Zulip Upload: %s" % log_path) + # Close log file log.close() - output("Zulip Upload: %s" % log_path) - # Authentication for Zulip zulip_auth = HTTPBasicAuth('builder-bot@openshot.zulipchat.com', zulip_token) filename = "%s-build-server.txt" % platform.system() @@ -123,6 +123,7 @@ def zulip_upload_log(log, title, comment=None): resp = post(zulip_url, data={}, auth=zulip_auth, files={filename: (filename, open(log_path, "rb"))}) if resp.ok: zulip_upload_url = resp.json().get("uri", "") + print(resp) # Determine topic topic = "Successful Builds" @@ -139,11 +140,10 @@ def zulip_upload_log(log, title, comment=None): } resp = post(zulip_url, data=zulip_data, auth=zulip_auth) - if resp.ok: - output(resp) # Re-open the log (for append) log = open(log_path, "a") + print(resp) def get_release(repo, tag_name): """Fetch the GitHub release tagged with the given tag and return it