From 328a10d28d53462571c216868941028d94c25f19 Mon Sep 17 00:00:00 2001
From: Derek Johnston <ross.mcnairn.dev@wordsmith.ai>
Date: Wed, 25 Dec 2024 04:52:55 +0000
Subject: [PATCH 1/3] compress in separate threads
---
build_differ.py | 117 ++++++++++++++++++++++++++++++------------------
1 file changed, 74 insertions(+), 43 deletions(-)
diff --git a/build_differ.py b/build_differ.py
index 0c6ab53..8c2f7d1 100644
--- a/build_differ.py
+++ b/build_differ.py
@@ -1,3 +1,4 @@
+import asyncio
import subprocess
import os
import tarfile
@@ -9,34 +10,48 @@ def get_version():
Extracts the version from the specified __about__.py file.
"""
about = {}
- with open('./src/python_redlines/__about__.py') as f:
+ with open("./src/python_redlines/__about__.py") as f:
exec(f.read(), about)
- return about['__version__']
+ return about["__version__"]
def run_command(command):
"""
Runs a shell command and prints its output.
"""
- process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ process = subprocess.Popen(
+ command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+ )
for line in process.stdout:
print(line.decode().strip())
-def compress_files(source_dir, target_file):
+def _compress_tar(source_dir, target_file):
+ with tarfile.open(target_file, "w:gz") as tar:
+ tar.add(source_dir, arcname=os.path.basename(source_dir))
+
+
+def _compress_zip(source_dir, target_file):
+ with zipfile.ZipFile(target_file, "w", zipfile.ZIP_DEFLATED) as zipf:
+ for root, dirs, files in os.walk(source_dir):
+ for file in files:
+ zipf.write(
+ os.path.join(root, file),
+ os.path.relpath(
+ os.path.join(root, file), os.path.join(source_dir, "..")
+ ),
+ )
+
+
+async def compress_files(source_dir, target_file):
"""
Compresses files in the specified directory into a tar.gz or zip file.
"""
- if target_file.endswith('.tar.gz'):
- with tarfile.open(target_file, "w:gz") as tar:
- tar.add(source_dir, arcname=os.path.basename(source_dir))
- elif target_file.endswith('.zip'):
- with zipfile.ZipFile(target_file, 'w', zipfile.ZIP_DEFLATED) as zipf:
- for root, dirs, files in os.walk(source_dir):
- for file in files:
- zipf.write(os.path.join(root, file),
- os.path.relpath(os.path.join(root, file),
- os.path.join(source_dir, '..')))
+ loop = asyncio.get_event_loop()
+ if target_file.endswith(".tar.gz"):
+ loop.run_in_executor(None, _compress_tar, source_dir, target_file)
+ elif target_file.endswith(".zip"):
+ loop.run_in_executor(None, _compress_zip, source_dir, target_file)
def cleanup_old_builds(dist_dir, current_version):
@@ -44,13 +59,15 @@ def cleanup_old_builds(dist_dir, current_version):
Deletes any build files ending in .zip or .tar.gz in the dist_dir with a different version tag.
"""
for file in os.listdir(dist_dir):
- if not file.endswith((f'{current_version}.zip', f'{current_version}.tar.gz', '.gitignore')):
+ if not file.endswith(
+ (f"{current_version}.zip", f"{current_version}.tar.gz", ".gitignore")
+ ):
file_path = os.path.join(dist_dir, file)
os.remove(file_path)
print(f"Deleted old build file: {file}")
-def main():
+async def main():
version = get_version()
print(f"Version: {version}")
@@ -58,47 +75,61 @@ def main():
# Build for Linux x64
print("Building for Linux x64...")
- run_command('dotnet publish ./csproj -c Release -r linux-x64 --self-contained')
+ run_command("dotnet publish ./csproj -c Release -r linux-x64 --self-contained")
# Build for Linux ARM64
print("Building for Linux ARM64...")
- run_command('dotnet publish ./csproj -c Release -r linux-arm64 --self-contained')
+ run_command("dotnet publish ./csproj -c Release -r linux-arm64 --self-contained")
# Build for Windows x64
print("Building for Windows x64...")
- run_command('dotnet publish ./csproj -c Release -r win-x64 --self-contained')
+ run_command("dotnet publish ./csproj -c Release -r win-x64 --self-contained")
# Build for Windows ARM64
print("Building for Windows ARM64...")
- run_command('dotnet publish ./csproj -c Release -r win-arm64 --self-contained')
+ run_command("dotnet publish ./csproj -c Release -r win-arm64 --self-contained")
# Build for macOS x64
print("Building for macOS x64...")
- run_command('dotnet publish ./csproj -c Release -r osx-x64 --self-contained')
+ run_command("dotnet publish ./csproj -c Release -r osx-x64 --self-contained")
# Build for macOS ARM64
print("Building for macOS ARM64...")
- run_command('dotnet publish ./csproj -c Release -r osx-arm64 --self-contained')
-
- # Compress the Linux x64 build
- linux_x64_build_dir = './csproj/bin/Release/net8.0/linux-x64'
- compress_files(linux_x64_build_dir, f"{dist_dir}/linux-x64-{version}.tar.gz")
-
- # Compress the Linux ARM64 build
- linux_arm64_build_dir = './csproj/bin/Release/net8.0/linux-arm64'
- compress_files(linux_arm64_build_dir, f"{dist_dir}/linux-arm64-{version}.tar.gz")
-
- # Compress the Windows x64 build
- windows_build_dir = './csproj/bin/Release/net8.0/win-x64'
- compress_files(windows_build_dir, f"{dist_dir}/win-x64-{version}.zip")
-
- # Compress the macOS x64 build
- macos_x64_build_dir = './csproj/bin/Release/net8.0/osx-x64'
- compress_files(macos_x64_build_dir, f"{dist_dir}/osx-x64-{version}.tar.gz")
-
- # Compress the macOS ARM64 build
- macos_arm64_build_dir = './csproj/bin/Release/net8.0/osx-arm64'
- compress_files(macos_arm64_build_dir, f"{dist_dir}/osx-arm64-{version}.tar.gz")
+ run_command("dotnet publish ./csproj -c Release -r osx-arm64 --self-contained")
+
+ compression_inputs = [
+ (
+ "./csproj/bin/Release/net8.0/linux-x64",
+ f"{dist_dir}/linux-x64-{version}.tar.gz",
+ ),
+ (
+ "./csproj/bin/Release/net8.0/linux-arm64",
+ f"{dist_dir}/linux-arm64-{version}.tar.gz",
+ ),
+ (
+ "./csproj/bin/Release/net8.0/win-x64",
+ f"{dist_dir}/win-x64-{version}.zip",
+ ),
+ (
+ "./csproj/bin/Release/net8.0/win-arm64",
+ f"{dist_dir}/win-arm64-{version}.zip",
+ ),
+ (
+ "./csproj/bin/Release/net8.0/osx-x64",
+ f"{dist_dir}/osx-x64-{version}.tar.gz",
+ ),
+ (
+ "./csproj/bin/Release/net8.0/osx-arm64",
+ f"{dist_dir}/osx-arm64-{version}.tar.gz",
+ ),
+ ]
+
+ await asyncio.gather(
+ *[
+ compress_files(source_dir, target_file)
+ for source_dir, target_file in compression_inputs
+ ]
+ )
cleanup_old_builds(dist_dir, version)
@@ -106,4 +137,4 @@ def main():
if __name__ == "__main__":
- main()
+ asyncio.run(main())
From cb0bbbf232262b0d449be721a3cb8b78f93b3aa8 Mon Sep 17 00:00:00 2001
From: Derek Johnston <ross.mcnairn.dev@wordsmith.ai>
Date: Wed, 25 Dec 2024 16:49:31 +0000
Subject: [PATCH 2/3] publish in parallel procs
---
build_differ.py | 45 ++++++++++++++++--------------------------
csproj/redlines.csproj | 1 +
2 files changed, 18 insertions(+), 28 deletions(-)
diff --git a/build_differ.py b/build_differ.py
index 8c2f7d1..e837f7b 100644
--- a/build_differ.py
+++ b/build_differ.py
@@ -15,16 +15,18 @@ def get_version():
return about["__version__"]
-def run_command(command):
+async def run_command(command):
"""
Runs a shell command and prints its output.
"""
- process = subprocess.Popen(
- command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+ process = await asyncio.create_subprocess_exec(
+ *command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
)
- for line in process.stdout:
+ async for line in process.stdout:
print(line.decode().strip())
+ await process.wait()
+
def _compress_tar(source_dir, target_file):
with tarfile.open(target_file, "w:gz") as tar:
@@ -49,9 +51,9 @@ async def compress_files(source_dir, target_file):
"""
loop = asyncio.get_event_loop()
if target_file.endswith(".tar.gz"):
- loop.run_in_executor(None, _compress_tar, source_dir, target_file)
+ await loop.run_in_executor(None, _compress_tar, source_dir, target_file)
elif target_file.endswith(".zip"):
- loop.run_in_executor(None, _compress_zip, source_dir, target_file)
+ await loop.run_in_executor(None, _compress_zip, source_dir, target_file)
def cleanup_old_builds(dist_dir, current_version):
@@ -73,29 +75,16 @@ async def main():
dist_dir = "./src/python_redlines/dist/"
- # Build for Linux x64
- print("Building for Linux x64...")
- run_command("dotnet publish ./csproj -c Release -r linux-x64 --self-contained")
-
- # Build for Linux ARM64
- print("Building for Linux ARM64...")
- run_command("dotnet publish ./csproj -c Release -r linux-arm64 --self-contained")
-
- # Build for Windows x64
- print("Building for Windows x64...")
- run_command("dotnet publish ./csproj -c Release -r win-x64 --self-contained")
-
- # Build for Windows ARM64
- print("Building for Windows ARM64...")
- run_command("dotnet publish ./csproj -c Release -r win-arm64 --self-contained")
-
- # Build for macOS x64
- print("Building for macOS x64...")
- run_command("dotnet publish ./csproj -c Release -r osx-x64 --self-contained")
+ run_commands = [
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "linux-x64", "--self-contained"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "linux-arm64", "--self-contained"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "win-x64", "--self-contained"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "win-arm64", "--self-contained"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "osx-x64", "--self-contained"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "osx-arm64", "--self-contained"],
+ ]
- # Build for macOS ARM64
- print("Building for macOS ARM64...")
- run_command("dotnet publish ./csproj -c Release -r osx-arm64 --self-contained")
+ await asyncio.gather(*[run_command(command) for command in run_commands])
compression_inputs = [
(
diff --git a/csproj/redlines.csproj b/csproj/redlines.csproj
index 8b2abdd..0e354c2 100644
--- a/csproj/redlines.csproj
+++ b/csproj/redlines.csproj
@@ -5,6 +5,7 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
+ <RuntimeIdentifiers>win-x64;win-arm64;linux-x64;linux-arm64;osx-x64;osx-arm64</RuntimeIdentifiers>
</PropertyGroup>
<ItemGroup>
From 42d3c82a4c9bba9e6e790a9a0379855fed7bdf96 Mon Sep 17 00:00:00 2001
From: Derek Johnston <ross.mcnairn.dev@wordsmith.ai>
Date: Wed, 25 Dec 2024 19:00:13 +0000
Subject: [PATCH 3/3] manually restore
---
build_differ.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/build_differ.py b/build_differ.py
index e837f7b..c3e4f6d 100644
--- a/build_differ.py
+++ b/build_differ.py
@@ -75,13 +75,15 @@ async def main():
dist_dir = "./src/python_redlines/dist/"
+ await run_command(["dotnet", "restore", "./csproj"])
+
run_commands = [
- ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "linux-x64", "--self-contained"],
- ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "linux-arm64", "--self-contained"],
- ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "win-x64", "--self-contained"],
- ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "win-arm64", "--self-contained"],
- ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "osx-x64", "--self-contained"],
- ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "osx-arm64", "--self-contained"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "linux-x64", "--self-contained", "--no-restore"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "linux-arm64", "--self-contained", "--no-restore"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "win-x64", "--self-contained", "--no-restore"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "win-arm64", "--self-contained", "--no-restore"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "osx-x64", "--self-contained", "--no-restore"],
+ ["dotnet", "publish", "./csproj", "-c", "Release", "-r", "osx-arm64", "--self-contained", "--no-restore"],
]
await asyncio.gather(*[run_command(command) for command in run_commands])