From a515c583dc5b0cb58c7bf388dbf992b55f7ff933 Mon Sep 17 00:00:00 2001 From: George Liaw Date: Wed, 3 Feb 2021 14:08:07 -0800 Subject: [PATCH 1/6] update pex to 2.1.28 --- pex/pex_rules.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 3c9ba592..3460415b 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -531,6 +531,6 @@ def pex_repositories(): http_file( name = "pex_bin", executable = True, - urls = ["https://github.com/pantsbuild/pex/releases/download/v2.1.16/pex"], - sha256 = "38712847654254088a23394728f9a5fb93c6c83631300e7ab427ec780a88f653", + urls = ["https://github.com/pantsbuild/pex/releases/download/v2.1.28/pex"], + sha256 = "bfc0add2649bd2d76043ef4ec07edf28aa3bf2654e8ee9b8e39ff5dcffb37665", ) From 24a527ddd82d3f39a9466d827990855ad64af94e Mon Sep 17 00:00:00 2001 From: George Liaw Date: Wed, 3 Feb 2021 14:12:06 -0800 Subject: [PATCH 2/6] add TODO --- pex/pex_rules.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 3460415b..2bac395d 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -215,6 +215,7 @@ def _pex_binary_impl(ctx): elif script: arguments += ["--script", script] arguments += [ + # TODO set `--tmpdir` option within the build work dir so we stop putting temp files under `/tmp` and filling up root "--resources-directory", "{resources_dir}".format( resources_dir = resources_dir.path, From aee44c1eb3b56258773c3bb5b9d440d08d4ce0ab Mon Sep 17 00:00:00 2001 From: George Liaw Date: Wed, 3 Feb 2021 14:21:23 -0800 Subject: [PATCH 3/6] resource-dir is deprecated in favor of sources-dir --- pex/pex_rules.bzl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 2bac395d..161810f1 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -162,7 +162,7 @@ def _pex_binary_impl(ctx): transitive_files = depset(transitive=transitive_files), ) - resources_dir = ctx.actions.declare_directory("{}.resources".format(ctx.attr.name)) + sources_dir = ctx.actions.declare_directory("{}.sources".format(ctx.attr.name)) # Create resource directory and dump files into it # Relocate files according to `strip_prefix` if necessary and get all files into the base resource directory @@ -170,15 +170,15 @@ def _pex_binary_impl(ctx): # Add `__init__.py` files to make the modules findable by pex execution ctx.actions.run_shell( mnemonic = "CreateResourceDirectory", - outputs = [resources_dir], + outputs = [sources_dir], inputs = runfiles.files.to_list(), - command = 'mkdir -p {resources_dir} && rsync -R {transitive_files} {resources_dir} \ - && if [ "{strip_prefix}" != "" ] && [ -n "$(ls -A {resources_dir}/{strip_prefix})" ]; then cp -R {resources_dir}/{strip_prefix}/* {resources_dir}; fi \ - && if [ "{strip_prefix}" != "" ]; then rm -rf {resources_dir}/{strip_prefix}; fi \ - && if [ -d {resources_dir}/{genfiles_dir}/{strip_prefix} ] && [ -n "$(ls -A {resources_dir}/{genfiles_dir}/{strip_prefix})" ]; then cp -R {resources_dir}/{genfiles_dir}/{strip_prefix}/* {resources_dir}; fi \ - && rm -rf {resources_dir}/{genfiles_parent_dir} \ - && find {resources_dir} -type d -exec touch {{}}/__init__.py \;'.format( - resources_dir = resources_dir.path, + command = 'mkdir -p {sources_dir} && rsync -R {transitive_files} {sources_dir} \ + && if [ "{strip_prefix}" != "" ] && [ -n "$(ls -A {sources_dir}/{strip_prefix})" ]; then cp -R {sources_dir}/{strip_prefix}/* {sources_dir}; fi \ + && if [ "{strip_prefix}" != "" ]; then rm -rf {sources_dir}/{strip_prefix}; fi \ + && if [ -d {sources_dir}/{genfiles_dir}/{strip_prefix} ] && [ -n "$(ls -A {sources_dir}/{genfiles_dir}/{strip_prefix})" ]; then cp -R {sources_dir}/{genfiles_dir}/{strip_prefix}/* {sources_dir}; fi \ + && rm -rf {sources_dir}/{genfiles_parent_dir} \ + && find {sources_dir} -type d -exec touch {{}}/__init__.py \;'.format( + sources_dir = sources_dir.path, transitive_files = " ".join([file.path for file in runfiles.files.to_list()]), genfiles_dir = ctx.configuration.genfiles_dir.path, genfiles_parent_dir = ctx.configuration.genfiles_dir.path.split("/")[0], @@ -216,9 +216,9 @@ def _pex_binary_impl(ctx): arguments += ["--script", script] arguments += [ # TODO set `--tmpdir` option within the build work dir so we stop putting temp files under `/tmp` and filling up root - "--resources-directory", - "{resources_dir}".format( - resources_dir = resources_dir.path, + "--sources-directory", + "{sources_dir}".format( + sources_dir = sources_dir.path, strip_prefix = ctx.attr.strip_prefix.strip("/"), ), "--pex-root", @@ -231,7 +231,7 @@ def _pex_binary_impl(ctx): _inputs = ( runfiles.files.to_list() + py.transitive_eggs.to_list() - ) + [resources_dir] + ) + [sources_dir] ctx.actions.run( mnemonic = "PexPython", From 5948aa32698e8e6333aba568646bc3af0a3dd4dd Mon Sep 17 00:00:00 2001 From: George Liaw Date: Wed, 3 Feb 2021 14:35:52 -0800 Subject: [PATCH 4/6] add TODO --- pex/pex_rules.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 161810f1..341bd469 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -303,6 +303,7 @@ pex_attrs = { flags = ["DIRECT_COMPILE_TIME_INPUT"], allow_files = req_file_types, ), + # TODO support `--pex-repository` option to leverage shared requirements pex files to minimize redo-ing requirements resolutions (https://github.com/pantsbuild/pex/pull/1182) "no_index": attr.bool(default = False), "disable_cache": attr.bool(default = False), "repos": attr.label_list( From 5e7cf9ffe7e193b00acbc13aa9a1554566a32858 Mon Sep 17 00:00:00 2001 From: Avadhani Jonnavithula Date: Wed, 15 Dec 2021 23:14:28 -0800 Subject: [PATCH 5/6] Upgraded pex version to 2.1.56. Re-arranged positional arguments because Pex is now expecting them in strict order. --- pex/pex_rules.bzl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 341bd469..5cebeb16 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -190,6 +190,10 @@ def _pex_binary_impl(ctx): arguments = ["setuptools==44.1.0"] # form the arguments to pex builder + for egg in py.transitive_eggs.to_list(): + arguments += [egg.path] + for req in py.transitive_reqs.to_list(): + arguments += [req] if not ctx.attr.zip_safe: arguments += ["--not-zip-safe"] if not ctx.attr.use_wheels: @@ -206,10 +210,6 @@ def _pex_binary_impl(ctx): arguments += ["--requirement", req_file.path] for repo in repos: arguments += ["--repo", repo] - for egg in py.transitive_eggs.to_list(): - arguments += [egg.path] - for req in py.transitive_reqs.to_list(): - arguments += [req] if main_pkg: arguments += ["--entry-point", main_pkg] elif script: @@ -533,6 +533,6 @@ def pex_repositories(): http_file( name = "pex_bin", executable = True, - urls = ["https://github.com/pantsbuild/pex/releases/download/v2.1.28/pex"], - sha256 = "bfc0add2649bd2d76043ef4ec07edf28aa3bf2654e8ee9b8e39ff5dcffb37665", + urls = ["https://github.com/pantsbuild/pex/releases/download/v2.1.56/pex"], + sha256 = "aff02e2ef0212db4531354e9b7b0d5f61745b3eb49665bc11142f0b603a27db9", ) From c0a6914025260b642d5661788ed96f8311ba4b71 Mon Sep 17 00:00:00 2001 From: Avadhani Jonnavithula Date: Thu, 16 Dec 2021 16:00:33 -0800 Subject: [PATCH 6/6] Added ability to pick up PEX_TMP_DIR from env, to supply as argument to pex through --tmpdir. No longer supply --not-zip-safe argument since its deprecated. The rule will still accept the argument, but it won't do anything. --- pex/pex_rules.bzl | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 5cebeb16..68719250 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -163,16 +163,13 @@ def _pex_binary_impl(ctx): ) sources_dir = ctx.actions.declare_directory("{}.sources".format(ctx.attr.name)) - + pex_tmp_dir_provided = ctx.configuration.default_shell_env.get('PEX_TMP_DIR') # Create resource directory and dump files into it # Relocate files according to `strip_prefix` if necessary and get all files into the base resource directory # Cleanup lingering files to prevent them from being added to the pex + # Create tmpdir for pex, if one is provided # Add `__init__.py` files to make the modules findable by pex execution - ctx.actions.run_shell( - mnemonic = "CreateResourceDirectory", - outputs = [sources_dir], - inputs = runfiles.files.to_list(), - command = 'mkdir -p {sources_dir} && rsync -R {transitive_files} {sources_dir} \ + cmd_to_run = 'mkdir -p {sources_dir} && rsync -R {transitive_files} {sources_dir} \ && if [ "{strip_prefix}" != "" ] && [ -n "$(ls -A {sources_dir}/{strip_prefix})" ]; then cp -R {sources_dir}/{strip_prefix}/* {sources_dir}; fi \ && if [ "{strip_prefix}" != "" ]; then rm -rf {sources_dir}/{strip_prefix}; fi \ && if [ -d {sources_dir}/{genfiles_dir}/{strip_prefix} ] && [ -n "$(ls -A {sources_dir}/{genfiles_dir}/{strip_prefix})" ]; then cp -R {sources_dir}/{genfiles_dir}/{strip_prefix}/* {sources_dir}; fi \ @@ -182,8 +179,14 @@ def _pex_binary_impl(ctx): transitive_files = " ".join([file.path for file in runfiles.files.to_list()]), genfiles_dir = ctx.configuration.genfiles_dir.path, genfiles_parent_dir = ctx.configuration.genfiles_dir.path.split("/")[0], - strip_prefix = ctx.attr.strip_prefix.strip("/"), - ), + strip_prefix = ctx.attr.strip_prefix.strip("/")) + if pex_tmp_dir_provided: + cmd_to_run += " && mkdir -p {}".format(pex_tmp_dir_provided) + ctx.actions.run_shell( + mnemonic = "CreateResourceDirectory", + outputs = [sources_dir], + inputs = runfiles.files.to_list(), + command = cmd_to_run, ) pexbuilder = ctx.executable._pexbuilder @@ -194,14 +197,14 @@ def _pex_binary_impl(ctx): arguments += [egg.path] for req in py.transitive_reqs.to_list(): arguments += [req] - if not ctx.attr.zip_safe: - arguments += ["--not-zip-safe"] if not ctx.attr.use_wheels: arguments += ["--no-use-wheel"] if ctx.attr.no_index: arguments += ["--no-index"] if ctx.attr.disable_cache: arguments += ["--disable-cache"] + if pex_tmp_dir_provided: + arguments += ["--tmpdir", pex_tmp_dir_provided] for interpreter in ctx.attr.interpreters: arguments += ["--python", interpreter] for platform in ctx.attr.platforms: