From 3c3c7d23c1251185b8c9d89c49dc9360f600eb79 Mon Sep 17 00:00:00 2001 From: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu, 22 Mar 2018 04:12:20 -0700 Subject: [PATCH] Introduce Clang, GCC, and Binutils to form a native toolchain This is the first instance in Pants of selectively instantiating a subsystem based on the platform Pants is being invoked on, because gnu binutils (or at least, ld) doesn't support OSX. The XCode cli tools will be relied on instead for linking (more work needed here on effective error messages). Create by squashing 119 commits, most of which had been merged into master: many of these commits were merged with the first python_dist PR, or the PR that introduced the "LLVM" subsystem (now named "Clang"). Squashed commit of the following: commit 9ae7284f362dbd6c40dc2115d8af907f4e1fece3 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Mar 21 15:03:38 2018 -0700 add some more comments to explain what i'm thinking commit 8d48d0a8785cbf56b88b3069a24263e1479b15cd Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Mar 21 14:52:07 2018 -0700 some mild refactoring and more documentation commit 29e962237e6d1595f1068fbe89dae4a39bded26d Merge: ba727b7 a5410b6 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Mar 21 12:52:28 2018 -0700 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit ba727b7a2058ae7b7006361c0a962cb4196533e5 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Mar 20 02:10:40 2018 -0700 gcc is linux specific again (but only for a little bit) commit aaae18cdbb88f1e1cf9ae046a95688177a34074e Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Mar 16 15:26:48 2018 -0700 correct gcc to non-platform specific, but remove from osx for now commit dfe7340d0daf21dbc2265867bd7e011b8d4877c6 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Mar 16 14:38:39 2018 -0700 add platform-specific native toolchain and use to build python dists commit f06c83f67d24aa9e9e6132fef893878da9573931 Merge: f22dfca f9ee6fd Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Mar 16 12:20:11 2018 -0700 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit f22dfca84f0ad088417f6d9ea6aa9121c3cbd556 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Mar 7 16:27:59 2018 -0800 add fixme commit 7d76e706a7b40b0bbdc9ea81d8ee650507ad9fe2 Merge: a35256d d77483e Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Mar 2 15:47:08 2018 -0800 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit a35256d8274e102d6157a34323cf96ecfccd4b39 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Mar 1 11:51:29 2018 -0800 update name of `compiler` package to `clang` commit 529324ef0a82c517555e529c6ab1481dcb9bdd47 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Feb 22 12:07:52 2018 -0800 fix lint errors commit 7763e5df24a51fec719bcb80d3be26c2f2630565 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Feb 22 11:46:58 2018 -0800 rename LLVM -> Compiler commit 91e1353ce50d46bb06496c2aa93d8812b1214d67 Merge: 6a68876 1ed1fdb Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Feb 22 11:28:16 2018 -0800 Merge branch 'dmcclanahan/python-dist-c++-sources' of github.com:cosmicexplorer/pants into dmcclanahan/python-dist-c++-sources commit 6a688768062d97475423bd6afe028a984b815f70 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Feb 21 11:45:30 2018 -0800 refactor out unnecessary interpreter wrapper to use a contextmanager commit 1ed1fdbb43fc967678a6bce476deec3dca779edf Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Feb 21 11:45:30 2018 -0800 refactor out unnecessary interpreter wrapper to use a contextmanager commit 41a1a9a281f50ca30c0f6230d8e9dca3a3dbad9d Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Feb 20 14:17:23 2018 -0800 revert unnecessary python_dist changes commit 1cfd60ac3f7cceedd50262d3cec2c80ce2428652 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Feb 20 14:12:44 2018 -0800 make final simplifications commit a4ebbbd490aa00785f7708c07c32aa92015481b1 Merge: 7783678 71a33d6 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Feb 20 13:57:19 2018 -0800 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit 77836788c4d33b12bb2b5bc1ecb110b1bd19ac04 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Feb 20 13:45:38 2018 -0800 remove more implementation artifacts commit 88851f1dd0a6e4b2ca55feccf9310eecb57b093f Merge: c7c90b2 c014e8d Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Feb 20 10:44:14 2018 -0800 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit c7c90b21836d372fc0cc81ead3b9812e7abf5674 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Feb 16 07:20:37 2018 -0800 refactor unnecessary complexity commit 35899eec325beee93171438838b13cf62e590413 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Feb 15 05:47:49 2018 -0800 try to understand what makes the breaking change commit d9fc2c455d7d5a4f54972ac8696830114e0e4273 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Feb 14 13:55:02 2018 -0800 remove comments in sandboxed_interpreter.py commit ceae8bb82236ffe787ccee9ea2392cd423765090 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Feb 14 13:51:46 2018 -0800 slim down the pull request quite a bit commit 48d7a93fa5522ea5a348ec2496362a1688c94e3a Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Feb 14 13:27:13 2018 -0800 rewrite BinaryTool a little and remove native toolchain subsystem commit cfe5c860bcbde2904d640cc307e9631d06555206 Merge: b1d39ec f55260a Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Feb 14 08:34:39 2018 -0800 Merge branch 'master' of github.com:pantsbuild/pants into dmcclanahan/python-dist-c++-sources commit b1d39ec08ded58b0209268dc0dc2a8a430abf4ae Merge: 55dbd7e 7cdea9a Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue Feb 13 11:54:50 2018 -0800 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit 55dbd7ebfb0a0d96c896eef15641653cb03c4194 Author: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon Feb 12 19:08:59 2018 -0800 suddenly, with very few changes, everything "just works"! commit 608769e9b4d070a76b1a2e9a49b96f1adea63121 Merge: badd80f 569f14c Author: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon Feb 12 19:05:19 2018 -0800 Merge branch 'master' into dmcclanahan/python-dist-c++-sources commit badd80f81d78e8d7e4d596daf99ed64cdc972e8e Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Mon Feb 5 16:09:46 2018 -0800 add llvm distribution support, not just clang commit 0ac2bbe676ddb56884454aeb3659ebe3fe70068a Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Feb 2 16:33:37 2018 -0800 remove source copying contextmanager and set clang arch for setup.py commit 6c5252964e34304ece1873ca420ac7b622792856 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Feb 2 12:16:55 2018 -0800 try cpp module sources and comment out future work commit f33677ba8bf81d558fd072b3a6075da9d55765ec Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Fri Feb 2 11:39:58 2018 -0800 add some more context, leave everything in an inconsistent state commit 5a6ad48d1f83c9d13324693c541ab50ae3785473 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Feb 1 12:07:56 2018 -0800 plumb in a native toolchain subsystem commit 5ed218ebf6f3f527c93defe1d0728d6690a4b41f Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Thu Feb 1 11:11:10 2018 -0800 use a contextmanager to copy source files and edit PYTHONPATH commit 3155138bddf0215bdb41f9a676840b868b2188e3 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Jan 31 20:47:16 2018 -0800 now we can declare cpp_sources in python_dist targets! commit 2591288683aaa7d4ceef69c0f17a2d2e51a39cdc Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Jan 31 03:17:57 2018 -0800 cut off work for now commit 629ff4675fea232b4d4ceff169ed21c0a03056b4 Author: Daniel McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Wed Jan 31 01:02:38 2018 -0800 clean up PythonDistribution and add c_sources field commit 9e3d2ca5d3f6feff3f1809cc9202827cb05d2c5b Author: Chris Livingston Date: Tue Jan 30 14:36:45 2018 -0800 Remove mod to travis yml commit 1a04e80be2845682c65e6734863678bab43cd557 Author: Chris Livingston Date: Tue Jan 30 13:13:37 2018 -0800 Rebase with master commit b4d367ce4edee3a4c32bb5969010d3fccb9738f2 Author: Chris Livingston Date: Wed Jan 24 17:59:30 2018 -0800 Rename superhello to fasthello commit 59164c1d41f3f9c3ff46bff35a21d4316fdacb27 Author: Chris Livingston Date: Wed Jan 24 15:26:03 2018 -0800 Add remove command to travis.yml to remove problematic file from failing CI target commit 8ac832b3cef3a04cf778234e7690cc2291ad89f6 Author: Chris Livingston Date: Wed Jan 24 13:00:12 2018 -0800 Add clean-all statements to integration tests to gauge flakiness commit a7877172f2bbe0938af3087752910503595e2eb2 Author: Chris Livingston Date: Wed Jan 24 10:39:22 2018 -0800 Slightly modify test assertion for conflicting deps test commit 0b82067f6220f18be1dc635f848b68e4b15a74d9 Author: Chris Livingston Date: Tue Jan 23 15:02:47 2018 -0800 Resolve merge conflicts from removal of tasks2 commit 7f757c045b2a13ce656df618c20091ed1eb5e1d7 Author: Chris Livingston Date: Mon Jan 22 12:28:18 2018 -0800 Disallow dependencies on a python_dist target commit 3024a349d4f6571dd7d8e604a43dd892ec4412d9 Author: Chris Livingston Date: Fri Jan 19 11:17:45 2018 -0800 Add xfail test to testprojects tests commit eb8598dd17ff5bba0dfde06d65b6ebbab912158a Author: Chris Livingston Date: Thu Jan 18 17:26:24 2018 -0800 Fix lint commit 654c20ee11957153ad731e719650cd775060ddcc Author: Chris Livingston Date: Thu Jan 18 14:13:42 2018 -0800 Remove unused import commit 22b72f067fce3e83ab4a31e89877187801aa8324 Author: Chris Livingston Date: Thu Jan 18 11:39:49 2018 -0800 Remove add_labels from PythonDistribution object commit a4150d012538ef0895b281f238ed884bdd52d66c Author: Chris Livingston Date: Wed Jan 17 19:17:59 2018 -0800 Remove duplicate functions to enforce DRY commit 178bcbd0dc2c4c5460012877c5d6aafd9eeb484e Author: Chris Livingston Date: Wed Jan 17 14:21:47 2018 -0800 Fix issues with CI and failing testprojects target commit cae065c1da6bffb0170e3e834ba20838bcb2329a Author: Chris Livingston Date: Mon Jan 15 23:44:32 2018 -0800 Add integration tests for targets that conflict with transitive deps listed in the install_requires field of a python distribution's setup.py commit ffae43d78928c5af5d4b18a079ba0ae1125c9907 Author: Chris Livingston Date: Mon Jan 15 22:36:45 2018 -0800 Remove crufty files commit c6edc08d23e2c2f958e13cec086a1c5e3c5a23ba Author: Chris Livingston Date: Fri Jan 12 17:23:46 2018 -0800 remove unneeded dependency test commit 2925530624048540ecea868cbbd94db9046e5095 Author: Chris Livingston Date: Fri Jan 12 17:21:33 2018 -0800 Edge case impl for same setup.py package name/version as a binary dep commit fe8dd55f5f580aa1252ba4bd95ccbf78186eb8ff Author: Chris Livingston Date: Thu Jan 11 22:36:45 2018 -0800 Simplify a few lines, add check for ambiguous python dists, and fix copyright date commit aabf88fb401fb43921d5f7b870d6c8c7efb9eb21 Author: Chris Livingston Date: Thu Jan 11 18:35:34 2018 -0800 Update TODO github issue link commit a6377e4d0188a12f0fa04e001e73e94e6401e3c9 Author: Chris Livingston Date: Thu Jan 11 16:36:12 2018 -0800 Remove tests that break testprojects integration testing commit d41e7eedf225439b72a757133bf5545f61ad04df Author: Chris Livingston Date: Wed Jan 10 16:29:25 2018 -0800 Remove unnecessary targets from BUILD file in superhello_testproject commit 4fd15677a449e3a133a93fa9ee85ec80ed27e0b8 Author: Chris Livingston Date: Wed Jan 10 14:44:23 2018 -0800 Cleanup integration test and move superhello test project to examples/tests due to test breakage when placed in testprojects commit e8f6b9834e00bdcded81a820af5b69e241d2479a Author: Chris Livingston Date: Wed Jan 10 11:59:32 2018 -0800 Fix multiple binary target case and add integration test commit 9e0b2d6b9f82ef6472d6244abda4fffb6344443f Author: Chris Livingston Date: Tue Jan 9 15:57:03 2018 -0800 Add TODO with github link for package conflict case in python dist backend commit a2116c4ba1410d88c11c645e09176eb630902336 Author: Chris Livingston Date: Tue Jan 9 15:20:25 2018 -0800 Remove cruft commit 03bc4c010a0f53914a17f20d58d0fc12ca509249 Author: Chris Livingston Date: Mon Jan 8 16:15:21 2018 -0800 Remove unneccessary checks for invalid targets and streamline method signatures commit 4e9bb6b853d29ce68e81eac74ac0f203126df452 Author: Chris Livingston Date: Fri Jan 5 18:24:44 2018 -0800 Fix install directory clobbering and setup.py positioning commit f1ed6daa6b2637630423eec76c367edd8097e942 Author: Chris Livingston Date: Fri Jan 5 18:23:02 2018 -0800 Fix install directory clobbering and setup.py positioning commit be2583ae10d3815aa20de29909fca6e66057eaa3 Author: Lionel Vital Date: Fri Jan 5 17:48:46 2018 -0800 Addresses a few changes commit 022b1e764e7d136f24c86d42ef538ae98d986d92 Author: Chris Livingston Date: Thu Dec 21 16:50:35 2017 -0800 Add rjiang suggestion for counting setup.py files commit e2bf445f609e8c6065ea3e4c61240a4b14247c5e Author: Chris Livingston Date: Thu Dec 21 15:16:35 2017 -0800 Clean up comments, docstrings, and fix broken testprojects integration tests commit 3e1739f8dff1fa0e0e8952793ad9ea81b19b2625 Author: Chris Livingston Date: Thu Dec 21 12:42:23 2017 -0800 Fix merge conflict commit 365e70892e9ebcafb7e376fbe4f68694e6efba5e Author: Chris Livingston Date: Thu Dec 21 12:30:08 2017 -0800 Style fixes and cruftslaying per rjiang's comments commit 5f4e07c39fdd0fb4f4055e70c1cc0e2c4aa6f534 Author: Chris Livingston Date: Thu Dec 21 12:10:02 2017 -0800 Add detection of multiple setup.py files and throw an error. commit 056e2675fa140831c655c22201591405b524ca0f Author: Chris Livingston Date: Wed Dec 20 17:50:17 2017 -0800 Add integration testing and simple unit test for python create distributions task. Also cleanup code to DRY by creating util helper method and streamline invalid python dist target detection. commit 401a10d28bf66004b3edcada78611647b82d453e Author: Chris Livingston Date: Tue Dec 19 18:12:14 2017 -0800 Working goals using invalidated blocks commit 53a135510db12363a5076cbe494feeae0a7ee23a Author: Chris Livingston Date: Tue Dec 19 16:40:56 2017 -0800 Solid working state based off of vt.results dir caching commit dacd67207cf90a820bd3f42115c1b2be66eab1e6 Author: Chris Livingston Date: Tue Dec 19 15:57:37 2017 -0800 Working caching under vt.results dir. Moving to tests. commit edf110df6f3e8d543d9d8729e3e87ff87e9f9a7b Author: Lionel Vital Date: Tue Dec 19 14:58:12 2017 -0800 More cleanup + move stuff in task execution under invalidated commit 8c7b658457a5d7900bd85892cf43dbd20639ded7 Author: Lionel Vital Date: Fri Dec 15 15:43:56 2017 -0800 Easy nits and whitespace issues commit 31d3a91ada07e96af1c24834a3e8ddec1f43b059 Author: Lionel Vital Date: Fri Dec 15 14:14:19 2017 -0800 Another whitespace error commit 177c379d25001b8a17b8a2eb04d404683061b883 Author: Lionel Vital Date: Fri Dec 15 12:32:06 2017 -0800 Whitespace lint fixes and unused import commit 7b8e8be1969f6ee8e61c166ed4e62ec3f5a167d9 Author: Chris Livingston Date: Thu Dec 14 17:43:54 2017 -0800 Remove unused imports created from refactor commit 51e9c64e97201caa02ed552a4d4beb3e407d1955 Author: Chris Livingston Date: Thu Dec 14 14:44:11 2017 -0800 Remove unused imports and add guard from consuming python dist products commit 59ff560ffdafdbe1240585f24d2ce78c68bc86af Author: Chris Livingston Date: Wed Dec 13 16:52:16 2017 -0800 Add guard statement for case where pants test run does not require data from PythonCreateDistributions task. commit fb1c9034f262a60f803d52b00f59895114f6c32c Author: Chris Livingston Date: Wed Dec 13 14:27:04 2017 -0800 Fix minor BUILD file error to pass CI commit 23f69997d01ae1e96a5fdd4eb57b2370eee9fdcf Author: Chris Livingston Date: Tue Dec 12 16:37:39 2017 -0800 Working distribution create task and integration for pants run/binary/test commit 314ecfbcca275539fbb0b4fdbe2ee32167caa74f Author: Chris Livingston Date: Tue Dec 12 16:05:10 2017 -0800 pants test example commit a859f46eb952f467e0bf8f49b542fffb301897ef Author: Chris Livingston Date: Mon Dec 11 18:07:08 2017 -0800 Remove cruft commit 6881a72b7ab7e8bc8f6a41bd4e65df9bef142d61 Author: Chris Livingston Date: Mon Dec 11 18:01:16 2017 -0800 Revert pants run mods and fix bug in pex build util commit 1c981f71fcea318f6904fcf6644ef3298f67b13a Author: Lionel Vital Date: Mon Dec 11 17:36:58 2017 -0800 Run/test commit 8436dad76ffc3ef5c09db1ab6f8f854714d8120d Author: Lionel Vital Date: Mon Dec 11 17:14:20 2017 -0800 More cleanup of docs commit b6fa385d88252935316e2f21ec3c84f87867d3b8 Author: Lionel Vital Date: Mon Dec 11 16:48:33 2017 -0800 Minor cleanup + rename alias to be standard commit 4b64f2dcbbcbb065078a4fd38571d6de224e9bf7 Author: Chris Livingston Date: Fri Dec 8 16:08:53 2017 -0800 Further iteration on python distribution task, now functioning for python binary create commit 76aeda2af8934245347c8964e7444d9bbbb21b1d Author: Chris Livingston Date: Fri Dec 8 15:20:34 2017 -0800 Working distribution creation task commit 7823d2a2789e8fae39c58c93d13705c7c7ed9c11 Author: Chris Livingston Date: Thu Dec 7 17:37:24 2017 -0800 Progress on pex_build_util commit e4c0556c6d2683175a0552ef59843aa412fffc13 Author: Chris Livingston Date: Thu Dec 7 10:02:26 2017 -0800 Install Python Dist create task commit 4b65461056ac1de082182767291d9acd247c2181 Author: Chris Livingston Date: Tue Dec 5 17:51:57 2017 -0800 Debug session 1 commit c340abf9b891eede1fec81579dcbad1c7a1a06e2 Author: Chris Livingston Date: Tue Dec 5 17:22:32 2017 -0800 Create python distribution task and refactor example commit 9bb228fb929cbf77064efa97d6876cb9c2e4bd9b Author: Lionel Vital Date: Mon Dec 4 16:05:01 2017 -0800 PythonDistribution -> inherit Target instead of PythonTarget commit 72970969893f7f25555f837869b0a3d9ff0605fc Author: Chris Livingston Date: Fri Dec 1 12:41:17 2017 -0800 First pass at kwlzn change suggestions commit e2f84cc134eb8f9201ad2c3e8b75b32413dc495a Author: Chris Livingston Date: Wed Nov 29 10:38:58 2017 -0800 Add python_distribution to backend/targets build file commit 194452ba5c4b2b1e2c9517a6d76ffa960a8dd2f3 Author: Chris Livingston Date: Tue Nov 28 16:50:14 2017 -0800 pants run task implementation - first pass commit 6585142e73bb6ca9ab621fe4a570b04e86629bed Author: Lionel Vital Date: Tue Nov 28 16:11:15 2017 -0800 First stab at run task commit 0ae435e810fff1ded275676cb12ffb8293f2bbde Author: Lionel Vital Date: Tue Nov 28 15:37:57 2017 -0800 Remove excess __init__.py files commit 1940c53f2e399e9685a8686f17098e9c1f2492da Author: Chris Livingston Date: Tue Nov 28 15:22:47 2017 -0800 Clean up misc style issues and delete unnecessary file. commit 9cd32a4e863ce2a5e8532cd4534d66bd0a59c0ac Author: Lionel Vital Date: Tue Nov 28 12:59:37 2017 -0800 Clean up + add some docs commit c8a954e0ca0c9a6100611c1989e57dcbad76aa04 Author: Chris Livingston Date: Tue Nov 28 12:16:26 2017 -0800 Add cleanup of egg-info in pex build util commit 83be228e832e807600e796983b947ea79bea4244 Author: Chris Livingston Date: Fri Nov 24 10:32:26 2017 -0800 Add tensorflow dependency example to superhello; this puts ./pants binary on main:main in a good state for a workable demo commit 74bfb8ce01008f39d2bdc29fe834ae8dbc72a7b4 Author: Chris Livingston Date: Tue Nov 21 16:45:55 2017 -0800 Finalized initial approach at packaging wheels w/ c sources into a pex commit 4e2b1b9a83f0d7d84df03fa06ddd9213c83f25dd Author: Lionel Vital Date: Tue Nov 21 15:02:05 2017 -0800 Minor cleanup commit dd9919447ebb707fce5cff169d755936d5608a01 Author: Chris Livingston Date: Tue Nov 21 14:43:19 2017 -0800 Working superhello project equivalent to backends/tensorflow commit 5837f30153ebf82a549189a3503602af121181ec Author: Lionel Vital Date: Tue Nov 21 14:27:00 2017 -0800 Modify example to use superhello commit 4f9bc98773b7e29c48134e82e43660c45176d11b Author: Chris Livingston Date: Tue Nov 21 13:51:52 2017 -0800 Working python dist example commit cd32058a6f02674d3939ecc52f8a6ece2ca867f6 Author: Chris Livingston Date: Tue Nov 21 12:22:23 2017 -0800 Changes to python binary creation + add new target definition commit 3aa0fe44bed68a51035904ebdf75d68a11e76288 Author: Chris Livingston Date: Tue Nov 21 12:19:55 2017 -0800 Changes to hello2 package commit 46eab744cdafacbd749c56cbdade4e1e8d4156a0 Author: Chris Livingston Date: Tue Nov 21 12:19:22 2017 -0800 Working package example for hello2 commit ca86bf63203641b6987194852ad15425bc4aaef4 Author: Chris Livingston Date: Wed Nov 15 13:01:29 2017 -0800 First pass at distribution task and target commit 3ec11c23c1cc6dff89e455fa2385a7eb12a7f618 Author: Lionel Vital Date: Thu Nov 9 23:27:59 2017 -0800 Python_distribution example --- build-support/docker/centos6/Dockerfile | 6 ++ .../pants/backend/native/subsystems/BUILD | 4 ++ .../pants/backend/native/subsystems/clang.py | 20 +++++++ .../native/subsystems/{llvm.py => gcc.py} | 12 +++- .../native/subsystems/native_toolchain.py | 47 +++++++++++++++ .../subsystems/platform_specific/__init__.py | 0 .../subsystems/platform_specific/linux/BUILD | 9 +++ .../platform_specific/linux/__init__.py | 0 .../platform_specific/linux/binutils.py | 20 +++++++ .../pants/backend/python/subsystems/BUILD | 3 + .../python_dist_build_environment.py | 37 ++++++++++++ .../tasks/build_local_python_distributions.py | 48 +++++++++++---- src/python/pants/binaries/BUILD | 8 +++ .../binaries/execution_environment_mixin.py | 58 +++++++++++++++++++ 14 files changed, 257 insertions(+), 15 deletions(-) create mode 100644 src/python/pants/backend/native/subsystems/clang.py rename src/python/pants/backend/native/subsystems/{llvm.py => gcc.py} (57%) create mode 100644 src/python/pants/backend/native/subsystems/native_toolchain.py create mode 100644 src/python/pants/backend/native/subsystems/platform_specific/__init__.py create mode 100644 src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD create mode 100644 src/python/pants/backend/native/subsystems/platform_specific/linux/__init__.py create mode 100644 src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py create mode 100644 src/python/pants/backend/python/subsystems/python_dist_build_environment.py create mode 100644 src/python/pants/binaries/execution_environment_mixin.py diff --git a/build-support/docker/centos6/Dockerfile b/build-support/docker/centos6/Dockerfile index a733da334f9..632843ea411 100644 --- a/build-support/docker/centos6/Dockerfile +++ b/build-support/docker/centos6/Dockerfile @@ -16,6 +16,12 @@ RUN yum install -y \ libffi \ libffi-devel \ openssl-devel +# LLVM release tarballs are xz-compressed. +RUN yum install -y xz +# GCC requires these to compile. +RUN yum install -y \ + m4 \ + wget # By default, execute in an environment with python27 enabled. ENTRYPOINT ["/usr/bin/scl", "enable", "python27", "devtoolset-7", "--"] diff --git a/src/python/pants/backend/native/subsystems/BUILD b/src/python/pants/backend/native/subsystems/BUILD index 8db0b45aa03..e6f09bd301f 100644 --- a/src/python/pants/backend/native/subsystems/BUILD +++ b/src/python/pants/backend/native/subsystems/BUILD @@ -4,5 +4,9 @@ python_library( dependencies=[ 'src/python/pants/binaries:binary_util', + 'src/python/pants/binaries:execution_environment_mixin', + 'src/python/pants/subsystem', + 'src/python/pants/util:memo', + 'src/python/pants/util:osutil', ], ) diff --git a/src/python/pants/backend/native/subsystems/clang.py b/src/python/pants/backend/native/subsystems/clang.py new file mode 100644 index 00000000000..36458ad6acf --- /dev/null +++ b/src/python/pants/backend/native/subsystems/clang.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pants.binaries.binary_tool import NativeTool +from pants.binaries.execution_environment_mixin import ExecutionPathEnvironment + + +class Clang(NativeTool, ExecutionPathEnvironment): + options_scope = 'clang' + default_version = '6.0.0' + archive_type = 'tgz' + + def get_additional_paths(self): + return [os.path.join(self.select(), 'bin')] diff --git a/src/python/pants/backend/native/subsystems/llvm.py b/src/python/pants/backend/native/subsystems/gcc.py similarity index 57% rename from src/python/pants/backend/native/subsystems/llvm.py rename to src/python/pants/backend/native/subsystems/gcc.py index 9c97eed26ec..18123e0286a 100644 --- a/src/python/pants/backend/native/subsystems/llvm.py +++ b/src/python/pants/backend/native/subsystems/gcc.py @@ -5,10 +5,16 @@ from __future__ import (absolute_import, division, generators, nested_scopes, print_function, unicode_literals, with_statement) +import os + from pants.binaries.binary_tool import NativeTool +from pants.binaries.execution_environment_mixin import ExecutionPathEnvironment -class LLVM(NativeTool): - options_scope = 'llvm' - default_version = '5.0.1' +class GCC(NativeTool, ExecutionPathEnvironment): + options_scope = 'gcc' + default_version = '7.3.0' archive_type = 'tgz' + + def get_additional_paths(self): + return [os.path.join(self.select(), 'bin')] diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py new file mode 100644 index 00000000000..c7d04d3b43f --- /dev/null +++ b/src/python/pants/backend/native/subsystems/native_toolchain.py @@ -0,0 +1,47 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from contextlib import contextmanager + +from pants.backend.native.subsystems.clang import Clang +from pants.backend.native.subsystems.gcc import GCC +from pants.backend.native.subsystems.platform_specific.linux.binutils import Binutils +from pants.binaries.execution_environment_mixin import ExecutionEnvironmentMixin +from pants.subsystem.subsystem import Subsystem +from pants.util.memo import memoized_property +from pants.util.osutil import get_os_name, normalize_os_name + + +class NativeToolchain(Subsystem, ExecutionEnvironmentMixin): + + options_scope = 'native-toolchain' + + PLATFORM_SPECIFIC_TOOLCHAINS = { + # TODO(cosmicexplorer): 'darwin' should have everything here, but there's no + # open-source linker for OSX...yet. + 'darwin': [GCC, Clang], + 'linux': [GCC, Binutils, Clang], + } + + @classmethod + def _get_platform_specific_toolchains(cls): + normed_os_name = normalize_os_name(get_os_name()) + return cls.PLATFORM_SPECIFIC_TOOLCHAINS[normed_os_name] + + @classmethod + def subsystem_dependencies(cls): + prev = super(NativeToolchain, cls).subsystem_dependencies() + cur_platform_subsystems = cls._get_platform_specific_toolchains() + return prev + tuple(sub.scoped(cls) for sub in cur_platform_subsystems) + + @memoized_property + def _toolchain_instances(self): + cur_platform_subsystems = self._get_platform_specific_toolchains() + return [sub.scoped_instance(self) for sub in cur_platform_subsystems] + + def modify_environment(self, env): + return self.apply_successive_env_modifications(env, self._toolchain_instances) diff --git a/src/python/pants/backend/native/subsystems/platform_specific/__init__.py b/src/python/pants/backend/native/subsystems/platform_specific/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD b/src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD new file mode 100644 index 00000000000..3fc007c5e77 --- /dev/null +++ b/src/python/pants/backend/native/subsystems/platform_specific/linux/BUILD @@ -0,0 +1,9 @@ +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library( + dependencies=[ + 'src/python/pants/binaries:binary_util', + 'src/python/pants/binaries:execution_environment_mixin', + ], +) diff --git a/src/python/pants/backend/native/subsystems/platform_specific/linux/__init__.py b/src/python/pants/backend/native/subsystems/platform_specific/linux/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py b/src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py new file mode 100644 index 00000000000..ef6667a8c4d --- /dev/null +++ b/src/python/pants/backend/native/subsystems/platform_specific/linux/binutils.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pants.binaries.binary_tool import NativeTool +from pants.binaries.execution_environment_mixin import ExecutionPathEnvironment + + +class Binutils(NativeTool, ExecutionPathEnvironment): + options_scope = 'binutils' + default_version = '2.30' + archive_type = 'tgz' + + def get_additional_paths(self): + return [os.path.join(self.select(), 'bin')] diff --git a/src/python/pants/backend/python/subsystems/BUILD b/src/python/pants/backend/python/subsystems/BUILD index 67f8417b4b2..bb0f66369af 100644 --- a/src/python/pants/backend/python/subsystems/BUILD +++ b/src/python/pants/backend/python/subsystems/BUILD @@ -4,7 +4,10 @@ python_library( dependencies = [ '3rdparty/python:setuptools', + 'src/python/pants/binaries:execution_environment_mixin', + 'src/python/pants/backend/native', 'src/python/pants/option', 'src/python/pants/subsystem', + 'src/python/pants/util:memo', ], ) diff --git a/src/python/pants/backend/python/subsystems/python_dist_build_environment.py b/src/python/pants/backend/python/subsystems/python_dist_build_environment.py new file mode 100644 index 00000000000..0e8cdd4338d --- /dev/null +++ b/src/python/pants/backend/python/subsystems/python_dist_build_environment.py @@ -0,0 +1,37 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +from pants.backend.native.subsystems.native_toolchain import NativeToolchain +from pants.binaries.execution_environment_mixin import ExecutionEnvironmentMixin +from pants.subsystem.subsystem import Subsystem +from pants.util.memo import memoized_property + + +class PythonDistBuildEnvironment(Subsystem, ExecutionEnvironmentMixin): + + options_scope = 'python-dist-build-environment' + + @classmethod + def subsystem_dependencies(cls): + return super(PythonDistBuildEnvironment, cls).subsystem_dependencies() + (NativeToolchain.scoped(cls),) + + @memoized_property + def _native_toolchain(self): + return NativeToolchain.scoped_instance(self) + + def modify_environment(self, env): + env = self._native_toolchain.modify_environment(env) + # If we're going to be wrapping setup.py-based projects, we really should be + # doing it through a subclass of a distutils UnixCCompiler (in Lib/distutils + # in the CPython source) instead of hoping setup.py knows what to do. The + # default UnixCCompiler from distutils will build a 32/64-bit "fat binary" + # unless you use their undocumented ARCHFLAGS env var, and there may be more + # dragons later on. + env['CC'] = 'gcc' + env['CXX'] = 'g++' + env['ARCHFLAGS'] = '-arch x86_64' + return env diff --git a/src/python/pants/backend/python/tasks/build_local_python_distributions.py b/src/python/pants/backend/python/tasks/build_local_python_distributions.py index b1a93e12dfe..af7a3f78b0c 100644 --- a/src/python/pants/backend/python/tasks/build_local_python_distributions.py +++ b/src/python/pants/backend/python/tasks/build_local_python_distributions.py @@ -8,10 +8,12 @@ import glob import os import shutil +from contextlib import contextmanager from pex.interpreter import PythonInterpreter from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.subsystems.python_dist_build_environment import PythonDistBuildEnvironment from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.tasks.pex_build_util import is_local_python_dist from pants.backend.python.tasks.setup_py import SetupPyRunner @@ -20,7 +22,9 @@ from pants.base.fingerprint_strategy import DefaultFingerprintStrategy from pants.build_graph.address import Address from pants.task.task import Task +from pants.util.contextutil import environment_as from pants.util.dirutil import safe_mkdir +from pants.util.memo import memoized_property class BuildLocalPythonDistributions(Task): @@ -39,18 +43,34 @@ def product_types(cls): def prepare(cls, options, round_manager): round_manager.require_data(PythonInterpreter) + @classmethod + def implementation_version(cls): + return super(BuildLocalPythonDistributions, cls).implementation_version() + [('BuildLocalPythonDistributions', 1)] + + @classmethod + def subsystem_dependencies(cls): + return super(BuildLocalPythonDistributions, cls).subsystem_dependencies() + (PythonDistBuildEnvironment.scoped(cls),) + + # TODO: seriously consider subclassing UnixCCompiler as well as the build_ext + # command from distutils to control the compiler and linker invocations + # transparently instead of hoping distutils does the right thing. + @memoized_property + def _python_dist_build_environment(self): + return PythonDistBuildEnvironment.scoped_instance(self) + @property def cache_target_dirs(self): return True def execute(self): dist_targets = self.context.targets(is_local_python_dist) - build_graph = self.context.build_graph if dist_targets: with self.invalidated(dist_targets, fingerprint_strategy=DefaultFingerprintStrategy(), invalidate_dependents=True) as invalidation_check: + interpreter = self.context.products.get_data(PythonInterpreter) + for vt in invalidation_check.invalid_vts: if vt.target.dependencies: raise TargetDefinitionException( @@ -58,7 +78,7 @@ def execute(self): 'List any 3rd party requirements in the install_requirements argument ' 'of your setup function.' ) - self._create_dist(vt.target, vt.results_dir) + self._create_dist(vt.target, vt.results_dir, interpreter) for vt in invalidation_check.all_vts: dist = self._get_whl_from_dir(os.path.join(vt.results_dir, 'dist')) @@ -66,26 +86,30 @@ def execute(self): self._inject_synthetic_dist_requirements(dist, req_lib_addr) # Make any target that depends on the dist depend on the synthetic req_lib, # for downstream consumption. - for dependent in build_graph.dependents_of(vt.target.address): - build_graph.inject_dependency(dependent, req_lib_addr) - - def _create_dist(self, dist_tgt, dist_target_dir): - """Create a .whl file for the specified python_distribution target.""" - interpreter = self.context.products.get_data(PythonInterpreter) + for dependent in self.context.build_graph.dependents_of(vt.target.address): + self.context.build_graph.inject_dependency(dependent, req_lib_addr) + def _copy_sources(self, dist_tgt, dist_target_dir): # Copy sources and setup.py over to vt results directory for packaging. # NB: The directory structure of the destination directory needs to match 1:1 # with the directory structure that setup.py expects. - for src_relative_to_target_base in dist_tgt.sources_relative_to_target_base(): + all_sources = list(dist_tgt.sources_relative_to_target_base()) + for src_relative_to_target_base in all_sources: src_rel_to_results_dir = os.path.join(dist_target_dir, src_relative_to_target_base) safe_mkdir(os.path.dirname(src_rel_to_results_dir)) abs_src_path = os.path.join(get_buildroot(), dist_tgt.address.spec_path, src_relative_to_target_base) shutil.copyfile(abs_src_path, src_rel_to_results_dir) - # Build a whl using SetupPyRunner and return its absolute path. - setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter) - setup_runner.run() + + def _create_dist(self, dist_tgt, dist_target_dir, interpreter): + """Create a .whl file for the specified python_distribution target.""" + self.context.log.debug("dist_target_dir: '{}'".format(dist_target_dir)) + self._copy_sources(dist_tgt, dist_target_dir) + with self._python_dist_build_environment.execution_environment(): + # Build a whl using SetupPyRunner and return its absolute path. + setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter) + setup_runner.run() def _inject_synthetic_dist_requirements(self, dist, req_lib_addr): """Inject a synthetic requirements library that references a local wheel. diff --git a/src/python/pants/binaries/BUILD b/src/python/pants/binaries/BUILD index 6d618604be6..b888c57ed19 100644 --- a/src/python/pants/binaries/BUILD +++ b/src/python/pants/binaries/BUILD @@ -30,3 +30,11 @@ python_library( 'src/python/pants/util:memo', ], ) + +python_library( + name='execution_environment_mixin', + sources=['execution_environment_mixin.py'], + dependencies=[ + 'src/python/pants/util:contextutil', + ], +) diff --git a/src/python/pants/binaries/execution_environment_mixin.py b/src/python/pants/binaries/execution_environment_mixin.py new file mode 100644 index 00000000000..6e0e6f02b35 --- /dev/null +++ b/src/python/pants/binaries/execution_environment_mixin.py @@ -0,0 +1,58 @@ +# coding=utf-8 +# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os +from contextlib import contextmanager + +from abc import abstractmethod + +from pants.util.contextutil import environment_as + + +def prepend_path(prev_env, path_var, new_entries): + prev_path = prev_env.get(path_var, None) + if prev_path is None: + return ':'.join(new_entries) + return ':'.join(new_entries + prev_path.split(':')) + + +class ExecutionEnvironmentMixin(object): + + @classmethod + def apply_successive_env_modifications(cls, env, environment_mixin_instances): + for env_modifier in environment_mixin_instances: + env = env_modifier.modify_environment(env) + return env + + # There's no reason this has to be just about the shell "environment", really + # just has to be a contextmanager. Implementing this method using a literal + # chroot, or VM image, or something might be really interesting to just + # completely sidestep the installation problem. + @contextmanager + def execution_environment(self, prev_env=None): + if prev_env is None: + prev_env = os.environ + env_copy = prev_env.copy() + new_env = self.modify_environment(env_copy) + + with environment_as(**new_env): + yield + + @abstractmethod + def modify_environment(self, env): pass + + +class ExecutionPathEnvironment(ExecutionEnvironmentMixin): + + @abstractmethod + def get_additional_paths(self): pass + + def modify_environment(self, env): + additional_paths = self.get_additional_paths() + new_path = prepend_path(env, 'PATH', additional_paths) + env['PATH'] = new_path + return env