Skip to content

Commit c06eba0

Browse files
nmacinnisPeter Bengtsson
nmacinnis
authored and
Peter Bengtsson
committed
Add an optional --index-url argument similar to that in pip (#107)
Adds an optional `--index-url` argument so that hashin can be used with an alternate package index. Pip supports this behavior with the same argument. Hashin was hardcoded to point at pypi.org, so if you happened to be working with an alternate index, hashin couldn't help you. Pip also accepts an `--extra-index-url` argument, but I'm guessing that hashin won't benefit much from supporting the same. I expect the main use case of `--index-url` to be a package or package version that only exists on the alternate index.
1 parent 8da1d5e commit c06eba0

File tree

3 files changed

+122
-6
lines changed

3 files changed

+122
-6
lines changed

‎hashin.py‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@
2525
if sys.version_info >= (3,):
2626
from urllib.request import urlopen
2727
from urllib.error import HTTPError
28+
from urllib.parse import urljoin
2829
else:
2930
from urllib import urlopen
31+
from urlparse import urljoin
3032

3133
input = raw_input # noqa
3234

@@ -44,6 +46,8 @@
4446

4547
DEFAULT_ALGORITHM = "sha256"
4648

49+
DEFAULT_INDEX_URL = os.environ.get("INDEX_URL", "https://pypi.org/")
50+
4751
MAX_WORKERS = None
4852

4953
if sys.version_info >= (3, 4) and sys.version_info < (3, 5):
@@ -153,6 +157,7 @@ def run_packages(
153157
previous_versions=None,
154158
interactive=False,
155159
synchronous=False,
160+
index_url=DEFAULT_INDEX_URL,
156161
):
157162
assert isinstance(specs, list), type(specs)
158163
all_new_lines = []
@@ -161,7 +166,9 @@ def run_packages(
161166

162167
lookup_memory = {}
163168
if not synchronous and len(specs) > 1:
164-
pre_download_packages(lookup_memory, specs, verbose=verbose)
169+
pre_download_packages(
170+
lookup_memory, specs, verbose=verbose, index_url=index_url
171+
)
165172

166173
for spec in specs:
167174
package, version, restriction = _explode_package_spec(spec)
@@ -186,6 +193,7 @@ def run_packages(
186193
algorithm=algorithm,
187194
include_prereleases=include_prereleases,
188195
lookup_memory=lookup_memory,
196+
index_url=index_url,
189197
)
190198
package = data["package"]
191199
# We need to keep this `req` instance for the sake of turning it into a string
@@ -273,14 +281,14 @@ def run_packages(
273281
return 0
274282

275283

276-
def pre_download_packages(memory, specs, verbose=False):
284+
def pre_download_packages(memory, specs, verbose=False, index_url=DEFAULT_INDEX_URL):
277285
futures = {}
278286
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
279287
for spec in specs:
280288
package, _, _ = _explode_package_spec(spec)
281289
req = Requirement(package)
282290
futures[
283-
executor.submit(get_package_data, req.name, verbose=verbose)
291+
executor.submit(get_package_data, req.name, index_url, verbose=verbose)
284292
] = req.name
285293
for future in concurrent.futures.as_completed(futures):
286294
content = future.result()
@@ -567,8 +575,9 @@ def filter_releases(releases, python_versions):
567575
return filtered
568576

569577

570-
def get_package_data(package, verbose=False):
571-
url = "https://pypi.org/pypi/%s/json" % package
578+
def get_package_data(package, index_url, verbose=False):
579+
path = "/pypi/%s/json" % package
580+
url = urljoin(index_url, path)
572581
if verbose:
573582
print(url)
574583
content = json.loads(_download(url))
@@ -615,6 +624,7 @@ def get_package_hashes(
615624
verbose=False,
616625
include_prereleases=False,
617626
lookup_memory=None,
627+
index_url=DEFAULT_INDEX_URL,
618628
):
619629
"""
620630
Gets the hashes for the given package.
@@ -642,7 +652,7 @@ def get_package_hashes(
642652
if lookup_memory is not None and package in lookup_memory:
643653
data = lookup_memory[package]
644654
else:
645-
data = get_package_data(package, verbose)
655+
data = get_package_data(package, index_url, verbose)
646656
if not version:
647657
version = get_latest_version(data, include_prereleases)
648658
assert version
@@ -741,6 +751,11 @@ def get_parser():
741751
action="store_true",
742752
default=False,
743753
)
754+
parser.add_argument(
755+
"--index-url",
756+
help="alternate package index url (default {0})".format(DEFAULT_INDEX_URL),
757+
default=None,
758+
)
744759
return parser
745760

746761

@@ -786,6 +801,7 @@ def main():
786801
dry_run=args.dry_run,
787802
interactive=args.interactive,
788803
synchronous=args.synchronous,
804+
index_url=args.index_url,
789805
)
790806
except PackageError as exception:
791807
print(str(exception), file=sys.stderr)

‎tests/test_arg_parse.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ def test_everything():
1616
"3.5",
1717
"-v",
1818
"--dry-run",
19+
"--index-url",
20+
"https://pypi1.someorg.net/",
1921
]
2022
)
2123
expected = argparse.Namespace(
@@ -30,6 +32,7 @@ def test_everything():
3032
update_all=False,
3133
interactive=False,
3234
synchronous=False,
35+
index_url="https://pypi1.someorg.net/",
3336
)
3437
assert args == (expected, [])
3538

@@ -47,6 +50,8 @@ def test_everything_long():
4750
"3.5",
4851
"--verbose",
4952
"--dry-run",
53+
"--index-url",
54+
"https://pypi1.someorg.net/",
5055
]
5156
)
5257
expected = argparse.Namespace(
@@ -61,6 +66,7 @@ def test_everything_long():
6166
update_all=False,
6267
interactive=False,
6368
synchronous=False,
69+
index_url="https://pypi1.someorg.net/",
6470
)
6571
assert args == (expected, [])
6672

@@ -79,5 +85,6 @@ def test_minimal():
7985
update_all=False,
8086
interactive=False,
8187
synchronous=False,
88+
index_url=None,
8289
)
8390
assert args == (expected, [])

‎tests/test_cli.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ def mock_parse_args(*a, **k):
146146
update_all=False,
147147
interactive=False,
148148
synchronous=False,
149+
index_url=None,
149150
)
150151

151152
mock_get_parser().parse_args.side_effect = mock_parse_args
@@ -1220,6 +1221,53 @@ def mocked_get(url, **options):
12201221
assert output.startswith("hashin==0.10")
12211222

12221223

1224+
def test_run_with_alternate_index_url(murlopen, tmpfile):
1225+
def mocked_get(url, **options):
1226+
if url == "https://pypi.internal.net/pypi/hashin/json":
1227+
return _Response(
1228+
{
1229+
"info": {"version": "0.10", "name": "hashin"},
1230+
"releases": {
1231+
"0.10": [
1232+
{
1233+
"url": "https://pypi.internal.net/packages/2.7/p/hashin/hashin-0.10-py2-none-any.whl",
1234+
"digests": {"sha256": "aaaaa"},
1235+
},
1236+
{
1237+
"url": "https://pypi.internal.net/packages/3.3/p/hashin/hashin-0.10-py3-none-any.whl",
1238+
"digests": {"sha256": "bbbbb"},
1239+
},
1240+
{
1241+
"url": "https://pypi.internal.net/packages/source/p/hashin/hashin-0.10.tar.gz",
1242+
"digests": {"sha256": "ccccc"},
1243+
},
1244+
]
1245+
},
1246+
}
1247+
)
1248+
1249+
raise NotImplementedError(url)
1250+
1251+
murlopen.side_effect = mocked_get
1252+
1253+
with tmpfile() as filename:
1254+
with open(filename, "w") as f:
1255+
f.write("")
1256+
1257+
retcode = hashin.run(
1258+
"hashin",
1259+
filename,
1260+
"sha256",
1261+
verbose=True,
1262+
index_url="https://pypi.internal.net/",
1263+
)
1264+
1265+
assert retcode == 0
1266+
with open(filename) as f:
1267+
output = f.read()
1268+
assert output.startswith("hashin==0.10")
1269+
1270+
12231271
def test_run_contained_names(murlopen, tmpfile):
12241272
"""
12251273
This is based on https://github.com/peterbe/hashin/issues/35
@@ -2005,6 +2053,51 @@ def mocked_get(url, **options):
20052053
assert result == expected
20062054

20072055

2056+
def test_get_package_hashes_from_alternate_index_url(murlopen):
2057+
def mocked_get(url, **options):
2058+
if url == "https://pypi.internal.net/pypi/hashin/json":
2059+
return _Response(
2060+
{
2061+
"info": {"version": "0.10", "name": "hashin"},
2062+
"releases": {
2063+
"0.10": [
2064+
{
2065+
"url": "https://pypi.internal.net/packages/2.7/p/hashin/hashin-0.10-py2-none-any.whl",
2066+
"digests": {"sha256": "ddddd"},
2067+
},
2068+
{
2069+
"url": "https://pypi.internal.net/packages/3.3/p/hashin/hashin-0.10-py3-none-any.whl",
2070+
"digests": {"sha256": "eeeee"},
2071+
},
2072+
{
2073+
"url": "https://pypi.internal.net/packages/source/p/hashin/hashin-0.10.tar.gz",
2074+
"digests": {"sha256": "fffff"},
2075+
},
2076+
]
2077+
},
2078+
}
2079+
)
2080+
2081+
raise NotImplementedError(url)
2082+
2083+
murlopen.side_effect = mocked_get
2084+
2085+
result = hashin.get_package_hashes(
2086+
package="hashin",
2087+
version="0.10",
2088+
algorithm="sha256",
2089+
index_url="https://pypi.internal.net/",
2090+
)
2091+
2092+
expected = {
2093+
"package": "hashin",
2094+
"version": "0.10",
2095+
"hashes": [{"hash": "ddddd"}, {"hash": "eeeee"}, {"hash": "fffff"}],
2096+
}
2097+
2098+
assert result == expected
2099+
2100+
20082101
def test_get_package_hashes_package_not_found(murlopen):
20092102
def mocked_get(url, **options):
20102103
if url == "https://pypi.org/pypi/gobblygook/json":

0 commit comments

Comments
 (0)