Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix various setup.py processing bugs #1862

Merged
merged 1 commit into from Jun 15, 2019
Merged

Fix various setup.py processing bugs #1862

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 123 additions & 103 deletions pythonforandroid/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from __future__ import print_function

from os.path import (join, realpath, dirname, expanduser, exists,
split, isdir)
from os.path import (
abspath, join, realpath, dirname, expanduser, exists,
split, isdir
)
from os import environ
import copy
import os
Expand Down Expand Up @@ -590,6 +592,120 @@ def project_has_setup_py(project_dir):
return False


def run_setuppy_install(ctx, project_dir, env=None):
if env is None:
env = dict()

with current_directory(project_dir):
info('got setup.py or similar, running project install. ' +
'(disable this behavior with --ignore-setup-py)')

# Compute & output the constraints we will use:
info('Contents that will be used for constraints.txt:')
constraints = subprocess.check_output([
join(
ctx.build_dir, "venv", "bin", "pip"
),
"freeze"
], env=copy.copy(env))
try:
constraints = constraints.decode("utf-8", "replace")
except AttributeError:
pass
info(constraints)

# Make sure all packages found are fixed in version
# by writing a constraint file, to avoid recipes being
# upgraded & reinstalled:
with open('._tmp_p4a_recipe_constraints.txt', 'wb') as fileh:
fileh.write(constraints.encode("utf-8", "replace"))
try:

info('Populating venv\'s site-packages with '
'ctx.get_site_packages_dir()...')

# Copy dist contents into site-packages for discovery.
# Why this is needed:
# --target is somewhat evil and messes with discovery of
# packages in PYTHONPATH if that also includes the target
# folder. So we need to use the regular virtualenv
# site-packages folder instead.
# Reference:
# https://github.com/pypa/pip/issues/6223
ctx_site_packages_dir = os.path.normpath(
os.path.abspath(ctx.get_site_packages_dir())
)
venv_site_packages_dir = os.path.normpath(os.path.join(
ctx.build_dir, "venv", "lib", [
f for f in os.listdir(os.path.join(
ctx.build_dir, "venv", "lib"
)) if f.startswith("python")
][0], "site-packages"
))
copied_over_contents = []
for f in os.listdir(ctx_site_packages_dir):
full_path = os.path.join(ctx_site_packages_dir, f)
if not os.path.exists(os.path.join(
venv_site_packages_dir, f
)):
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
venv_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
venv_site_packages_dir, f
))
copied_over_contents.append(f)

# Get listing of virtualenv's site-packages, to see the
# newly added things afterwards & copy them back into
# the distribution folder / build context site-packages:
previous_venv_contents = os.listdir(
venv_site_packages_dir
)

# Actually run setup.py:
info('Launching package install...')
shprint(sh.bash, '-c', (
"'" + join(
ctx.build_dir, "venv", "bin", "pip"
).replace("'", "'\"'\"'") + "' " +
"install -c ._tmp_p4a_recipe_constraints.txt -v ."
).format(ctx.get_site_packages_dir().
replace("'", "'\"'\"'")),
_env=copy.copy(env))

# Go over all new additions and copy them back:
info('Copying additions resulting from setup.py back '
'into ctx.get_site_packages_dir()...')
new_venv_additions = []
for f in (set(os.listdir(venv_site_packages_dir)) -
set(previous_venv_contents)):
new_venv_additions.append(f)
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
ctx_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
ctx_site_packages_dir, f
))

# Undo all the changes we did to the venv-site packages:
info('Reverting additions to '
'virtualenv\'s site-packages...')
for f in set(copied_over_contents + new_venv_additions):
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.rmtree(full_path)
else:
os.remove(full_path)
finally:
os.remove("._tmp_p4a_recipe_constraints.txt")


def run_pymodules_install(ctx, modules, project_dir=None,
ignore_setup_py=False):
""" This function will take care of all non-recipe things, by:
Expand All @@ -605,6 +721,10 @@ def run_pymodules_install(ctx, modules, project_dir=None,
info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***')
modules = list(filter(ctx.not_has_package, modules))

# We change current working directory later, so this
# has to be an absolute path:
project_dir = abspath(project_dir)

# Bail out if no python deps and no setup.py to process:
if not modules and (
ignore_setup_py or
Expand Down Expand Up @@ -697,107 +817,7 @@ def run_pymodules_install(ctx, modules, project_dir=None,
if project_dir is not None and (
project_has_setup_py(project_dir) and not ignore_setup_py
):
with current_directory(project_dir):
info('got setup.py or similar, running project install. ' +
'(disable this behavior with --ignore-setup-py)')

# Compute & output the constraints we will use:
info('Contents that will be used for constraints.txt:')
constraints = subprocess.check_output([
join(
ctx.build_dir, "venv", "bin", "pip"
),
"freeze"
], env=copy.copy(env))
try:
constraints = constraints.decode("utf-8", "replace")
except AttributeError:
pass
info(constraints)

# Make sure all packages found are fixed in version
# by writing a constraint file, to avoid recipes being
# upgraded & reinstalled:
with open('constraints.txt', 'wb') as fileh:
fileh.write(constraints.encode("utf-8", "replace"))

info('Populating venv\'s site-packages with '
'ctx.get_site_packages_dir()...')

# Copy dist contents into site-packages for discovery.
# Why this is needed:
# --target is somewhat evil and messes with discovery of
# packages in PYTHONPATH if that also includes the target
# folder. So we need to use the regular virtualenv
# site-packages folder instead.
# Reference:
# https://github.com/pypa/pip/issues/6223
ctx_site_packages_dir = os.path.normpath(
os.path.abspath(ctx.get_site_packages_dir())
)
venv_site_packages_dir = os.path.normpath(os.path.join(
ctx.build_dir, "venv", "lib", [
f for f in os.listdir(os.path.join(
ctx.build_dir, "venv", "lib"
)) if f.startswith("python")
][0], "site-packages"
))
copied_over_contents = []
for f in os.listdir(ctx_site_packages_dir):
full_path = os.path.join(ctx_site_packages_dir, f)
if not os.path.exists(os.path.join(
venv_site_packages_dir, f
)):
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
venv_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
venv_site_packages_dir, f
))
copied_over_contents.append(f)

# Get listing of virtualenv's site-packages, to see the
# newly added things afterwards & copy them back into
# the distribution folder / build context site-packages:
previous_venv_contents = os.listdir(venv_site_packages_dir)

# Actually run setup.py:
info('Launching package install...')
shprint(sh.bash, '-c', (
"'" + join(
ctx.build_dir, "venv", "bin", "pip"
).replace("'", "'\"'\"'") + "' " +
"install -c constraints.txt -v ."
).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
_env=copy.copy(env))

# Go over all new additions and copy them back:
info('Copying additions resulting from setup.py back ' +
'into ctx.get_site_packages_dir()...')
new_venv_additions = []
for f in (set(os.listdir(venv_site_packages_dir)) -
set(previous_venv_contents)):
new_venv_additions.append(f)
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.copytree(full_path, os.path.join(
ctx_site_packages_dir, f
))
else:
shutil.copy2(full_path, os.path.join(
ctx_site_packages_dir, f
))

# Undo all the changes we did to the venv-site packages:
info('Reverting additions to virtualenv\'s site-packages...')
for f in set(copied_over_contents + new_venv_additions):
full_path = os.path.join(venv_site_packages_dir, f)
if os.path.isdir(full_path):
shutil.rmtree(full_path)
else:
os.remove(full_path)
run_setuppy_install(ctx, project_dir, env)
elif not ignore_setup_py:
info("No setup.py found in project directory: " +
str(project_dir)
Expand Down
4 changes: 2 additions & 2 deletions pythonforandroid/pythonpackage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -559,7 +559,7 @@ def parse_as_folder_reference(dep):
# Check if this is either not an url, or a file URL:
if dep.startswith(("/", "file://")) or (
dep.find("/") > 0 and
dep.find("://") < 0):
dep.find("://") < 0) or (dep in ["", "."]):
if dep.startswith("file://"):
dep = urlunquote(urlparse(dep).path)
return dep
Expand Down Expand Up @@ -689,7 +689,7 @@ def get_package_dependencies(package,
for package_dep in current_queue:
new_reqs = set()
if verbose:
print("get_package_dependencies: resolving dependecy "
print("get_package_dependencies: resolving dependency "
"to package name: ".format(package_dep))
package = get_package_name(package_dep)
if package.lower() in packages_processed:
Expand Down
5 changes: 4 additions & 1 deletion pythonforandroid/toolchain.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -631,7 +631,10 @@ def add_parser(subparsers, *args, **kwargs):
args.requirements += u",".join(dependencies)
except ValueError:
# Not a python package, apparently.
pass
warning(
"Processing failed, is this project a valid "
"package? Will continue WITHOUT setup.py deps."
)

# Parse --requirements argument list:
for requirement in split_argument_list(args.requirements):
Expand Down
2 changes: 2 additions & 0 deletions tests/test_pythonpackage_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ def test_is_filesystem_path():
assert not is_filesystem_path("test @ bla")
assert is_filesystem_path("/abc/c@d")
assert not is_filesystem_path("https://user:pw@host/")
assert is_filesystem_path(".")
assert is_filesystem_path("")


def test_parse_as_folder_reference():
Expand Down