From 36d7308a756c0926221cf3950dc668132e3f9a47 Mon Sep 17 00:00:00 2001 From: George Liaw Date: Sun, 16 Jul 2017 02:08:01 -0700 Subject: [PATCH 1/6] add support for no-index, requirements files, and repo paths --- pex/pex_rules.bzl | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 8e15ea9a..0a7c4364 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -48,6 +48,7 @@ so that Bazel can find your `prelude_bazel` file. pex_file_types = FileType([".py"]) egg_file_types = FileType([".egg", ".whl"]) +req_file_types = FileType([".txt"]) # As much as I think this test file naming convention is a good thing, it's # probably a bad idea to impose it as a policy to all OSS users of these rules, @@ -82,6 +83,16 @@ def _collect_transitive_reqs(ctx): return transitive_reqs +def _collect_repos(ctx): + repos = {} + for dep in ctx.attr.deps: + if hasattr(dep.py, "repos"): + repos += dep.py.repos + for file in egg_file_types.filter(ctx.files.repos): + repos.update({file.dirname : True}) + return repos.keys() + + def _collect_transitive(ctx): return struct( # These rules don't use transitive_sources internally; it's just here for @@ -159,6 +170,7 @@ def _pex_binary_impl(ctx): ctx.configuration.bin_dir, ctx.outputs.executable, '.pex') py = _collect_transitive(ctx) + repos = _collect_repos(ctx) for dep in ctx.attr.deps: transitive_files += dep.default_runfiles.files @@ -184,6 +196,12 @@ def _pex_binary_impl(ctx): arguments += [] if ctx.attr.pex_use_wheels else ["--no-use-wheel"] if ctx.attr.interpreter: arguments += ["--python", ctx.attr.interpreter] + if ctx.attr.no_index: + arguments += ["--no-index"] + for req_file in ctx.files.req_files: + arguments += ["--requirement", req_file.path] + for repo in repos: + arguments += ["--repo", repo] for egg in py.transitive_eggs: arguments += ["--find-links", egg.dirname] arguments += [ @@ -286,6 +304,11 @@ pex_attrs = { "eggs": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"], allow_files = egg_file_types), "reqs": attr.string_list(), + "req_files": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"], + allow_files = req_file_types), + "no_index": attr.bool(default=False), + "repos": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"], + allow_files = egg_file_types), "data": attr.label_list(allow_files = True, cfg = "data"), @@ -350,6 +373,17 @@ Args: It is recommended that you use `eggs` instead where possible. + req_files: Add requirements from the given requirements files. Must be provided as labels. + + This feature will reduce build determinism! It tells pex to resolve all + the transitive python dependencies and fetch them from pypi. + + It is recommended that you use `eggs` or specify `no_index` instead where possible. + + no_index: If True, don't use pypi to resolve dependencies for `reqs` and `req_files`; Default: False + + repos: Additional repository labels (filegroups of wheel/egg files) to look for requirements. + data: Files to include as resources in the final pex binary. Putting other rules here will cause the *outputs* of those rules to be From 7cba9c323ee3c6712860c113ee689372e88a31ac Mon Sep 17 00:00:00 2001 From: George Liaw Date: Sun, 16 Jul 2017 18:37:10 -0700 Subject: [PATCH 2/6] support repo source package filetypes --- pex/pex_rules.bzl | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index 0a7c4364..d7fdd5f8 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -50,6 +50,20 @@ pex_file_types = FileType([".py"]) egg_file_types = FileType([".egg", ".whl"]) req_file_types = FileType([".txt"]) +# Repos file types according to: https://www.python.org/dev/peps/pep-0527/ +repo_file_types = FileType([ + ".egg", + ".whl", + ".tar.gz", + ".zip", + ".tar", + ".tar.bz2", + ".tar.xz", + ".tar.Z", + ".tgz", + ".tbz" +]) + # As much as I think this test file naming convention is a good thing, it's # probably a bad idea to impose it as a policy to all OSS users of these rules, # so I guess let's skip it. @@ -88,7 +102,7 @@ def _collect_repos(ctx): for dep in ctx.attr.deps: if hasattr(dep.py, "repos"): repos += dep.py.repos - for file in egg_file_types.filter(ctx.files.repos): + for file in repo_file_types.filter(ctx.files.repos): repos.update({file.dirname : True}) return repos.keys() @@ -308,7 +322,7 @@ pex_attrs = { allow_files = req_file_types), "no_index": attr.bool(default=False), "repos": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"], - allow_files = egg_file_types), + allow_files = repo_file_types), "data": attr.label_list(allow_files = True, cfg = "data"), From 2ed08a8b5aa5d79836ace9cc31aff837e31566d5 Mon Sep 17 00:00:00 2001 From: George Liaw Date: Sun, 16 Jul 2017 19:19:19 -0700 Subject: [PATCH 3/6] support cache disabling --- pex/pex_rules.bzl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pex/pex_rules.bzl b/pex/pex_rules.bzl index d7fdd5f8..947fb373 100644 --- a/pex/pex_rules.bzl +++ b/pex/pex_rules.bzl @@ -212,6 +212,8 @@ def _pex_binary_impl(ctx): arguments += ["--python", ctx.attr.interpreter] if ctx.attr.no_index: arguments += ["--no-index"] + if ctx.attr.disable_cache: + arguments += ["--disable-cache"] for req_file in ctx.files.req_files: arguments += ["--requirement", req_file.path] for repo in repos: @@ -321,6 +323,7 @@ pex_attrs = { "req_files": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"], allow_files = req_file_types), "no_index": attr.bool(default=False), + "disable_cache": attr.bool(default=False), "repos": attr.label_list(flags = ["DIRECT_COMPILE_TIME_INPUT"], allow_files = repo_file_types), "data": attr.label_list(allow_files = True, @@ -396,6 +399,8 @@ Args: no_index: If True, don't use pypi to resolve dependencies for `reqs` and `req_files`; Default: False + disable_cache: Disable caching in the pex tool entirely. Default: False + repos: Additional repository labels (filegroups of wheel/egg files) to look for requirements. data: Files to include as resources in the final pex binary. From 70f725fe0c5ed7443a5123dc03892e21ea801c9e Mon Sep 17 00:00:00 2001 From: George Liaw Date: Thu, 20 Jul 2017 13:55:11 -0700 Subject: [PATCH 4/6] fix mktemp to be usable on both linux/mac --- pex/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/BUILD b/pex/BUILD index fff2cd86..d36538b4 100644 --- a/pex/BUILD +++ b/pex/BUILD @@ -22,7 +22,7 @@ genrule( # Workaround really long shebang lines breaking on linux: # Use a /tmp path, but keep the actual venv inside the bazel outdir. # Avoids having to worry about cleanup, even if sandboxing is off. - TMPF=$$(mktemp) + TMPF=$$(mktemp 2>/dev/null || mktemp -t tmp) ln -sf "$$OUTDIR" "$$TMPF" VENV="$${TMPF}/venv" From f6ddac5c6397cd008ca90a3b0d4c055bbb356ea1 Mon Sep 17 00:00:00 2001 From: George Liaw Date: Thu, 20 Jul 2017 13:59:20 -0700 Subject: [PATCH 5/6] add comment --- pex/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/pex/BUILD b/pex/BUILD index d36538b4..87706359 100644 --- a/pex/BUILD +++ b/pex/BUILD @@ -22,6 +22,7 @@ genrule( # Workaround really long shebang lines breaking on linux: # Use a /tmp path, but keep the actual venv inside the bazel outdir. # Avoids having to worry about cleanup, even if sandboxing is off. + # `mktemp -t tmp` is used for OS X. TMPF=$$(mktemp 2>/dev/null || mktemp -t tmp) ln -sf "$$OUTDIR" "$$TMPF" VENV="$${TMPF}/venv" From e4338b27c489be85bce3541f3729116473df4c24 Mon Sep 17 00:00:00 2001 From: George Liaw Date: Thu, 20 Jul 2017 14:02:27 -0700 Subject: [PATCH 6/6] update comment --- pex/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/BUILD b/pex/BUILD index 87706359..dce57010 100644 --- a/pex/BUILD +++ b/pex/BUILD @@ -22,7 +22,7 @@ genrule( # Workaround really long shebang lines breaking on linux: # Use a /tmp path, but keep the actual venv inside the bazel outdir. # Avoids having to worry about cleanup, even if sandboxing is off. - # `mktemp -t tmp` is used for OS X. + # `mktemp -t tmp` is used for older OS X versions. TMPF=$$(mktemp 2>/dev/null || mktemp -t tmp) ln -sf "$$OUTDIR" "$$TMPF" VENV="$${TMPF}/venv"