From bb9b9f5d001963cbe8a5b701b43a7697d5606179 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Thu, 23 Jul 2020 23:16:59 +0300 Subject: [PATCH 1/6] refactor: remove `get_solc_version_string` --- solcx/__init__.py | 1 - solcx/main.py | 25 ++++++++----------------- tests/test_solc_version.py | 5 ----- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/solcx/__init__.py b/solcx/__init__.py index f55f5f7..dcb256e 100644 --- a/solcx/__init__.py +++ b/solcx/__init__.py @@ -15,7 +15,6 @@ compile_source, compile_standard, get_solc_version, - get_solc_version_string, link_code, ) diff --git a/solcx/main.py b/solcx/main.py index 0785065..af07b84 100644 --- a/solcx/main.py +++ b/solcx/main.py @@ -4,7 +4,7 @@ import json import re -import semantic_version +from semantic_version import Version from .exceptions import ContractsNotFound, SolcError from .wrapper import solc_wrapper @@ -15,11 +15,9 @@ ) -def get_solc_version_string(**kwargs): - kwargs["version"] = True - stdoutdata, stderrdata, command, proc = solc_wrapper(**kwargs) - _, _, version_string = stdoutdata.partition("\n") - if not version_string or not version_string.startswith("Version: "): +def get_solc_version() -> Version: + stdoutdata, stderrdata, command, proc = solc_wrapper(version=True) + if "Version: " not in stdoutdata: raise SolcError( command=command, return_code=proc.returncode, @@ -28,16 +26,9 @@ def get_solc_version_string(**kwargs): stderr_data=stderrdata, message="Unable to extract version string from command output", ) - return version_string.rstrip() - - -def get_solc_version(**kwargs): - # semantic_version as of 2017-5-5 expects only one + to be used in string - return semantic_version.Version( - strip_zeroes_from_month_and_day( - get_solc_version_string(**kwargs)[len("Version: ") :].replace("++", "pp") - ) - ) + version_string = stdoutdata.split("Version: ", maxsplit=1)[1] + version_string = version_string.replace("++", "pp").strip() + return Version(strip_zeroes_from_month_and_day(version_string)) def _parse_compiler_output(stdoutdata): @@ -160,7 +151,7 @@ def link_code(unlinked_bytecode, libraries): (":".join((lib_name, lib_address)) for lib_name, lib_address in libraries.items()) ) stdoutdata, stderrdata, _, _ = solc_wrapper( - stdin=unlinked_bytecode, link=True, libraries=libraries_arg, + stdin=unlinked_bytecode, link=True, libraries=libraries_arg ) return stdoutdata.replace("Linking completed.", "").strip() diff --git a/tests/test_solc_version.py b/tests/test_solc_version.py index 208518c..f651241 100644 --- a/tests/test_solc_version.py +++ b/tests/test_solc_version.py @@ -26,11 +26,6 @@ def test_get_solc_version(all_versions): assert isinstance(v, Version) -def test_get_solc_version_string(all_versions): - v = solcx.get_solc_version_string() - assert isinstance(v, str) - - def test_set_solc_version_pragma(pragmapatch): set_pragma = functools.partial(solcx.set_solc_version_pragma, check_new=True) set_pragma("pragma solidity 0.4.11;") From 682871db2d1624e1935185da9f5ee75fc5bf2c1e Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 24 Jul 2020 00:18:43 +0300 Subject: [PATCH 2/6] style: return `Version` objects from `get_available_solc_versions` --- solcx/install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/solcx/install.py b/solcx/install.py index 1253955..d9d0de9 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -16,6 +16,7 @@ from base64 import b64encode from io import BytesIO from pathlib import Path +from typing import Dict, List, Optional import requests from semantic_version import SimpleSpec, Version @@ -165,9 +166,7 @@ def set_solc_version_pragma(pragma_string, silent=False, check_new=False): def install_solc_pragma(pragma_string, install=True, show_progress=False, solcx_binary_path=None): - version = _select_pragma_version( - pragma_string, [Version(i[1:]) for i in get_available_solc_versions()] - ) + version = _select_pragma_version(pragma_string, get_available_solc_versions()) if not version: raise ValueError("Compatible solc version does not exist") if install: @@ -175,12 +174,12 @@ def install_solc_pragma(pragma_string, install=True, show_progress=False, solcx_ return version -def get_available_solc_versions(headers=None): - versions = [] +def get_available_solc_versions(headers: Optional[Dict] = None) -> List[Version]: + version_list = [] pattern = VERSION_REGEX[_get_platform()] - if not headers and os.getenv("GITHUB_TOKEN"): - auth = b64encode(os.getenv("GITHUB_TOKEN").encode()).decode() + if headers is None and os.getenv("GITHUB_TOKEN") is not None: + auth = b64encode(os.environ["GITHUB_TOKEN"].encode()).decode() headers = {"Authorization": f"Basic {auth}"} data = requests.get(ALL_RELEASES, headers=headers) @@ -200,13 +199,14 @@ def get_available_solc_versions(headers=None): for release in data.json(): asset = next((i for i in release["assets"] if re.match(pattern, i["name"])), False) if asset: - versions.append(release["tag_name"]) + version = Version.coerce(release["tag_name"].lstrip("v")) + version_list.append(version) if release["tag_name"] == MINIMAL_SOLC_VERSION: break - return versions + return sorted(version_list, reverse=True) -def _select_pragma_version(pragma_string, version_list): +def _select_pragma_version(pragma_string: str, version_list: List[Version]) -> Optional[Version]: comparator_set_range = pragma_string.replace(" ", "").split("||") comparator_regex = re.compile(r"(([<>]?=?|\^)\d+\.\d+\.\d+)+") version = None @@ -216,8 +216,8 @@ def _select_pragma_version(pragma_string, version_list): selected = spec.select(version_list) if selected and (not version or version < selected): version = selected - if version: - return str(version) + + return version def get_installed_solc_versions(solcx_binary_path=None): From 363bdc1bdabf4f01e6c043b99058accbbcb47203 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 24 Jul 2020 00:29:06 +0300 Subject: [PATCH 3/6] refactor: use `Version` objects for `get_installed_solc_versions` and `install_solc` --- solcx/install.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/solcx/install.py b/solcx/install.py index d9d0de9..e10ca32 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -16,7 +16,7 @@ from base64 import b64encode from io import BytesIO from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import requests from semantic_version import SimpleSpec, Version @@ -65,6 +65,16 @@ def _get_platform(): ) +def _convert_and_validate_version(version: Union[str, Version]) -> Version: + # take a user-supplied version as a string or Version + # validate the value, and return a Version object + if not isinstance(version, Version): + version = Version(version.lstrip("v")) + if version not in SimpleSpec(">=0.4.11"): + raise ValueError("py-solc-x does not support solc versions <0.4.11") + return version + + def get_solc_folder(solcx_binary_path=None): if os.getenv(SOLCX_BINARY_PATH_VARIABLE): return Path(os.getenv(SOLCX_BINARY_PATH_VARIABLE)) @@ -137,7 +147,7 @@ def get_executable(version=None, solcx_binary_path=None): def set_solc_version(version, silent=False, solcx_binary_path=None): - version = _check_version(version) + version = _convert_and_validate_version(version) get_executable(version, solcx_binary_path) global solc_version solc_version = version @@ -154,7 +164,7 @@ def set_solc_version_pragma(pragma_string, silent=False, check_new=False): f"No compatible solc version installed." f" Use solcx.install_solc_version_pragma('{version}') to install." ) - version = _check_version(version) + version = _convert_and_validate_version(version) global solc_version solc_version = version if not silent: @@ -220,16 +230,21 @@ def _select_pragma_version(pragma_string: str, version_list: List[Version]) -> O return version -def get_installed_solc_versions(solcx_binary_path=None): - return sorted( - i.name[5:] for i in get_solc_folder(solcx_binary_path=solcx_binary_path).glob("solc-v*") - ) +def get_installed_solc_versions(solcx_binary_path=None) -> List[Version]: + install_path = get_solc_folder(solcx_binary_path=solcx_binary_path) + return sorted([Version(i.name[5:]) for i in install_path.glob("solc-v*")], reverse=True) -def install_solc(version, allow_osx=False, show_progress=False, solcx_binary_path=None): +def install_solc( + version: Union[str, Version], + allow_osx: bool = False, + show_progress: bool = False, + solcx_binary_path=None, +) -> None: + arch = _get_arch() platform = _get_platform() - version = _check_version(version) + version = _convert_and_validate_version(version) lock = get_process_lock(version) if not lock.acquire(False): @@ -259,13 +274,6 @@ def install_solc(version, allow_osx=False, show_progress=False, solcx_binary_pat lock.release() -def _check_version(version): - version = Version(version.lstrip("v")) - if version not in SimpleSpec(">=0.4.11"): - raise ValueError("py-solc-x does not support solc versions <0.4.11") - return f"v{version}" - - def _check_subprocess_call(command, message=None, verbose=False, **proc_kwargs): if message: LOGGER.debug(message) @@ -280,8 +288,8 @@ def _chmod_plus_x(executable_path): executable_path.chmod(executable_path.stat().st_mode | stat.S_IEXEC) -def _check_for_installed_version(version, solcx_binary_path=None): - path = get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath("solc-" + version) +def _check_for_installed_version(version: Version, solcx_binary_path=None): + path = get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath(f"solc-v{version}") if path.exists(): LOGGER.info(f"solc {version} already installed at: {path}") return False From fd8c7dc32cd3a269558a23b99182860d0831ab85 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 24 Jul 2020 00:39:02 +0300 Subject: [PATCH 4/6] refactor: use `Version` for internal install functions --- solcx/install.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/solcx/install.py b/solcx/install.py index e10ca32..a3aa634 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -86,9 +86,10 @@ def get_solc_folder(solcx_binary_path=None): return path -def _import_version(path): - version = subprocess.check_output([path, "--version"]).decode() - return "v" + version[version.index("Version: ") + 9 : version.index("+")] +def _get_import_version(path: str) -> Version: + stdout_data = subprocess.check_output([path, "--version"]).decode().strip() + stdout_data = stdout_data[stdout_data.index("Version: ") + 9 : stdout_data.index("+")] + return Version.coerce(stdout_data) def import_installed_solc(solcx_binary_path=None): @@ -112,7 +113,7 @@ def import_installed_solc(solcx_binary_path=None): for path in path_list: try: - version = _import_version(path) + version = _get_import_version(path) assert version not in get_installed_solc_versions() except Exception: continue @@ -122,7 +123,7 @@ def import_installed_solc(solcx_binary_path=None): shutil.copy(path, copy_path) try: # confirm that solc still works after being copied - assert version == _import_version(copy_path) + assert version == _get_import_version(copy_path) except Exception: os.unlink(copy_path) @@ -239,7 +240,7 @@ def install_solc( version: Union[str, Version], allow_osx: bool = False, show_progress: bool = False, - solcx_binary_path=None, + solcx_binary_path: str = None, ) -> None: arch = _get_arch() @@ -330,7 +331,9 @@ def _download_solc(url, show_progress): return content -def _install_solc_linux(version, show_progress, solcx_binary_path=None): +def _install_solc_linux( + version: Version, show_progress: bool, solcx_binary_path: Optional[str] +) -> None: download = DOWNLOAD_BASE.format(version, "solc-static-linux") binary_path = _check_for_installed_version(version, solcx_binary_path=solcx_binary_path) if binary_path: @@ -341,7 +344,9 @@ def _install_solc_linux(version, show_progress, solcx_binary_path=None): _chmod_plus_x(binary_path) -def _install_solc_windows(version, show_progress, solcx_binary_path=None): +def _install_solc_windows( + version: Version, show_progress: bool, solcx_binary_path: Optional[str] +) -> None: download = DOWNLOAD_BASE.format(version, "solidity-windows.zip") install_folder = _check_for_installed_version(version) if install_folder: @@ -355,11 +360,15 @@ def _install_solc_windows(version, show_progress, solcx_binary_path=None): temp_path.rename(install_folder) -def _install_solc_arm(version, show_progress, solcx_binary_path): +def _install_solc_arm( + version: Version, show_progress: bool, solcx_binary_path: Optional[str] +) -> None: _compile_solc(version, show_progress, solcx_binary_path) -def _install_solc_osx(version, allow_osx, show_progress, solcx_binary_path): +def _install_solc_osx( + version: Version, allow_osx: bool, show_progress: bool, solcx_binary_path: Optional[str] +) -> None: if version.startswith("v0.4") and not allow_osx: raise ValueError( "Installing solc {0} on OSX often fails. For suggested installation options:\n" @@ -371,9 +380,9 @@ def _install_solc_osx(version, allow_osx, show_progress, solcx_binary_path): _compile_solc(version, show_progress, solcx_binary_path) -def _compile_solc(version, show_progress, solcx_binary_path): +def _compile_solc(version: Version, show_progress: bool, solcx_binary_path: Optional[str]) -> None: temp_path = _get_temp_folder() - download = DOWNLOAD_BASE.format(version, f"solidity_{version[1:]}.tar.gz") + download = DOWNLOAD_BASE.format(version, f"solidity_{version}.tar.gz") binary_path = _check_for_installed_version(version) if not binary_path: return From 1786ec781a8138263c227c96bd74470558a2af4e Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 24 Jul 2020 00:58:50 +0300 Subject: [PATCH 5/6] fix: minor Version related fixes --- solcx/install.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/solcx/install.py b/solcx/install.py index a3aa634..6e8d376 100644 --- a/solcx/install.py +++ b/solcx/install.py @@ -30,7 +30,7 @@ tqdm = None -DOWNLOAD_BASE = "https://github.com/ethereum/solidity/releases/download/{}/{}" +DOWNLOAD_BASE = "https://github.com/ethereum/solidity/releases/download/v{}/{}" ALL_RELEASES = "https://api.github.com/repos/ethereum/solidity/releases?per_page=100" MINIMAL_SOLC_VERSION = "v0.4.11" @@ -118,7 +118,7 @@ def import_installed_solc(solcx_binary_path=None): except Exception: continue copy_path = str( - get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath("solc-" + version) + get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath(f"solc-v{version}") ) shutil.copy(path, copy_path) try: @@ -131,12 +131,14 @@ def import_installed_solc(solcx_binary_path=None): def get_executable(version=None, solcx_binary_path=None): if not version: version = solc_version + else: + version = _convert_and_validate_version(version) if not version: raise SolcNotInstalled( "Solc is not installed. Call solcx.get_available_solc_versions()" " to view for available versions and solcx.install_solc() to install." ) - solc_bin = get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath("solc-" + version) + solc_bin = get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath(f"solc-v{version}") if sys.platform == "win32": solc_bin = solc_bin.joinpath("solc.exe") if not solc_bin.exists(): @@ -157,9 +159,7 @@ def set_solc_version(version, silent=False, solcx_binary_path=None): def set_solc_version_pragma(pragma_string, silent=False, check_new=False): - version = _select_pragma_version( - pragma_string, [Version(i[1:]) for i in get_installed_solc_versions()] - ) + version = _select_pragma_version(pragma_string, get_installed_solc_versions()) if not version: raise SolcNotInstalled( f"No compatible solc version installed." @@ -172,7 +172,7 @@ def set_solc_version_pragma(pragma_string, silent=False, check_new=False): LOGGER.info(f"Using solc version {solc_version}") if check_new: latest = install_solc_pragma(pragma_string, False) - if Version(latest) > Version(version[1:]): + if latest > version: LOGGER.info(f"Newer compatible solc version exists: {latest}") @@ -233,7 +233,7 @@ def _select_pragma_version(pragma_string: str, version_list: List[Version]) -> O def get_installed_solc_versions(solcx_binary_path=None) -> List[Version]: install_path = get_solc_folder(solcx_binary_path=solcx_binary_path) - return sorted([Version(i.name[5:]) for i in install_path.glob("solc-v*")], reverse=True) + return sorted([Version(i.name[6:]) for i in install_path.glob("solc-v*")], reverse=True) def install_solc( @@ -355,7 +355,7 @@ def _install_solc_windows( with zipfile.ZipFile(BytesIO(content)) as zf: zf.extractall(str(temp_path)) install_folder = get_solc_folder(solcx_binary_path=solcx_binary_path).joinpath( - "solc-" + version + f"solc-v{version}" ) temp_path.rename(install_folder) @@ -369,7 +369,7 @@ def _install_solc_arm( def _install_solc_osx( version: Version, allow_osx: bool, show_progress: bool, solcx_binary_path: Optional[str] ) -> None: - if version.startswith("v0.4") and not allow_osx: + if version < Version("0.5.0") and not allow_osx: raise ValueError( "Installing solc {0} on OSX often fails. For suggested installation options:\n" "https://github.com/iamdefinitelyahuman/py-solc-x/wiki/Installing-Solidity-on-OSX\n\n" From 7b4fb9acd70a2a43dd36a5c8a662a82c9df44a96 Mon Sep 17 00:00:00 2001 From: Ben Hauser Date: Fri, 24 Jul 2020 00:59:01 +0300 Subject: [PATCH 6/6] test: update tests to use `Version` object --- tests/test_install.py | 6 +++--- tests/test_solc_version.py | 44 +++++++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index f01082c..0e2cf95 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -19,7 +19,7 @@ def isolation(): def test_not_installed(): solcx.install.get_executable() - with pytest.raises(SolcNotInstalled): + with pytest.raises(ValueError): solcx.install.get_executable("v0.4.0") solcx.install.solc_version = None with pytest.raises(SolcNotInstalled): @@ -27,9 +27,9 @@ def test_not_installed(): def test_unsupported_version(): - solcx.install._check_version("0.4.11") + solcx.install._convert_and_validate_version("0.4.11") with pytest.raises(ValueError): - solcx.install._check_version("0.4.10") + solcx.install._convert_and_validate_version("0.4.10") def test_unknown_platform(): diff --git a/tests/test_solc_version.py b/tests/test_solc_version.py index f651241..cacf60a 100644 --- a/tests/test_solc_version.py +++ b/tests/test_solc_version.py @@ -13,11 +13,25 @@ def pragmapatch(monkeypatch): monkeypatch.setattr( "solcx.install.get_installed_solc_versions", - lambda: ["v0.4.2", "v0.4.11", "v0.4.25", "v0.5.0", "v0.5.4", "v0.5.7", "v1.2.3"], + lambda: [ + Version("0.4.2"), + Version("0.4.11"), + Version("0.4.25"), + Version("0.5.0"), + Version("0.5.4"), + Version("0.5.7"), + Version("1.2.3"), + ], ) monkeypatch.setattr( "solcx.install.get_available_solc_versions", - lambda: ["v0.4.11", "v0.4.24", "v0.4.26", "v0.5.3", "v0.6.0"], + lambda: [ + Version("0.4.11"), + Version("0.4.24"), + Version("0.4.26"), + Version("0.5.3"), + Version("0.6.0"), + ], ) @@ -29,30 +43,30 @@ def test_get_solc_version(all_versions): def test_set_solc_version_pragma(pragmapatch): set_pragma = functools.partial(solcx.set_solc_version_pragma, check_new=True) set_pragma("pragma solidity 0.4.11;") - assert solcx.install.solc_version == "v0.4.11" + assert solcx.install.solc_version == Version("0.4.11") set_pragma("pragma solidity ^0.4.11;") - assert solcx.install.solc_version == "v0.4.25" + assert solcx.install.solc_version == Version("0.4.25") set_pragma("pragma solidity >=0.4.0<0.4.25;") - assert solcx.install.solc_version == "v0.4.11" + assert solcx.install.solc_version == Version("0.4.11") set_pragma("pragma solidity >=0.4.2;") - assert solcx.install.solc_version == "v1.2.3" + assert solcx.install.solc_version == Version("1.2.3") set_pragma("pragma solidity >=0.4.2<0.5.5;") - assert solcx.install.solc_version == "v0.5.4" + assert solcx.install.solc_version == Version("0.5.4") set_pragma("pragma solidity ^0.4.2 || 0.5.5;") - assert solcx.install.solc_version == "v0.4.25" + assert solcx.install.solc_version == Version("0.4.25") set_pragma("pragma solidity ^0.4.2 || >=0.5.4<0.7.0;") - assert solcx.install.solc_version == "v0.5.7" + assert solcx.install.solc_version == Version("0.5.7") with pytest.raises(SolcNotInstalled): set_pragma("pragma solidity ^0.7.1;") def test_install_solc_version_pragma(pragmapatch): install_pragma = functools.partial(solcx.install_solc_pragma, install=False) - assert install_pragma("pragma solidity ^0.4.11;") == "0.4.26" - assert install_pragma("pragma solidity >=0.4.0<0.4.25;") == "0.4.24" - assert install_pragma("pragma solidity >=0.4.2;") == "0.6.0" - assert install_pragma("pragma solidity >=0.4.2<0.5.5;") == "0.5.3" - assert install_pragma("pragma solidity ^0.4.2 || 0.5.5;") == "0.4.26" - assert install_pragma("pragma solidity ^0.4.2 || >=0.5.4<0.7.0;") == "0.6.0" + assert install_pragma("pragma solidity ^0.4.11;") == Version("0.4.26") + assert install_pragma("pragma solidity >=0.4.0<0.4.25;") == Version("0.4.24") + assert install_pragma("pragma solidity >=0.4.2;") == Version("0.6.0") + assert install_pragma("pragma solidity >=0.4.2<0.5.5;") == Version("0.5.3") + assert install_pragma("pragma solidity ^0.4.2 || 0.5.5;") == Version("0.4.26") + assert install_pragma("pragma solidity ^0.4.2 || >=0.5.4<0.7.0;") == Version("0.6.0") with pytest.raises(ValueError): install_pragma("pragma solidity ^0.7.1;")