Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: algo to function in substratools #303

Merged
merged 13 commits into from
Oct 11, 2022
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,16 @@ compute_plan = client.add_compute_plan(
- link_dataset_with_data_samples return value in remote mode
- Fix the compute plan rank calculation in local mode (#299)

### Changed

- Apply changes from algo to function in substratools (#303)

## [0.39.0](https://github.com/Substra/substra/releases/tag/0.39.0) - 2022-10-03

### Removed

- BREAKING CHANGE: remove category from substra.schema.AlgoSpec and substra.models.Algo

### Added

- Prevent use of `__` in asset metadata keys in local mode
Expand Down
6 changes: 3 additions & 3 deletions references/sdk.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Client
```text
Client(url: Union[str, NoneType] = None, token: Union[str, NoneType] = None, retry_timeout: int = 300, insecure: bool = False, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
Client(url: Optional[str] = None, token: Optional[str] = None, retry_timeout: int = 300, insecure: bool = False, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
```

Create a client
Expand Down Expand Up @@ -255,7 +255,7 @@ algorithm.
- `pathlib.Path`: Path of the downloaded model
## download_model_from_task
```text
download_model_from_task(self, task_key: str, identifier: str, folder: os.PathLike) -> None
download_model_from_task(self, task_key: str, identifier: str, folder: os.PathLike) -> pathlib.Path
```

Download task model to destination file.
Expand All @@ -273,7 +273,7 @@ algorithm.
- `pathlib.Path`: Path of the downloaded model
## from_config_file
```text
from_config_file(profile_name: str = 'default', config_path: Union[str, pathlib.Path] = '~/.substra', tokens_path: Union[str, pathlib.Path] = '~/.substra-tokens', token: Union[str, NoneType] = None, retry_timeout: int = 300, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
from_config_file(profile_name: str = 'default', config_path: Union[str, pathlib.Path] = '~/.substra', tokens_path: Union[str, pathlib.Path] = '~/.substra-tokens', token: Optional[str] = None, retry_timeout: int = 300, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
```

Returns a new Client configured with profile data from configuration files.
Expand Down
20 changes: 10 additions & 10 deletions substra/sdk/backends/local/compute/spawner/subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,19 @@
logger = logging.getLogger(__name__)

PYTHON_SCRIPT_REGEX = r"(?<=\")([^\"]*\.py)(?=\")"
METHOD_REGEX = r"\"\-\-method-name\"\,\s*\"([^\"]*)\""
METHOD_REGEX = r"\"\-\-function-name\"\,\s*\"([^\"]*)\""


def _get_entrypoint_from_dockerfile(tmpdir):
"""
Extracts the .py script and the function name to execute in
an ENTRYPOINT line of a Dockerfile, located in tmpdir.
For instance if the line `ENTRYPOINT ["python3", "algo.py", "--method-name", "train"]` is in the Dockerfile,
For instance if the line `ENTRYPOINT ["python3", "algo.py", "--function-name", "train"]` is in the Dockerfile,
`algo.py`, `train` is extracted.
"""
valid_example = (
"""The entry point should be specified as follow: """
"""``ENTRYPOINT ["<executor>", "<algo_file.py>", "--method-name", "<method name>"]"""
"""``ENTRYPOINT ["<executor>", "<algo_file.py>", "--function-name", "<method name>"]"""
)
with open(tmpdir / "Dockerfile") as f:
for line in f:
Expand All @@ -41,19 +41,19 @@ def _get_entrypoint_from_dockerfile(tmpdir):
if len(script_name) != 1:
raise ExecutionError("Couldn't extract script from ENTRYPOINT line in Dockerfile", valid_example)

method_name = re.findall(METHOD_REGEX, line)
if len(method_name) != 1:
function_name = re.findall(METHOD_REGEX, line)
if len(function_name) != 1:
raise ExecutionError("Couldn't extract method name from ENTRYPOINT line in Dockerfile", valid_example)

return script_name[0], method_name[0]
return script_name[0], function_name[0]

raise ExecutionError("Couldn't get entrypoint in Dockerfile", valid_example)


def _get_command_args(
method_name: str, args_template: typing.List[string.Template], local_volumes: typing.Dict[str, str]
function_name: str, args_template: typing.List[string.Template], local_volumes: typing.Dict[str, str]
) -> typing.List[str]:
args = ["--method-name", str(method_name)]
args = ["--function-name", str(function_name)]
args += [tpl.substitute(**local_volumes) for tpl in args_template]
return args

Expand Down Expand Up @@ -98,12 +98,12 @@ def spawn(
algo_dir = pathlib.Path(algo_dir)
args_dir = pathlib.Path(args_dir)
uncompress(archive_path, algo_dir)
script_name, method_name = _get_entrypoint_from_dockerfile(algo_dir)
script_name, function_name = _get_entrypoint_from_dockerfile(algo_dir)

args_file = args_dir / "arguments.txt"

py_command = [sys.executable, str(algo_dir / script_name), f"@{args_file}"]
py_command_args = _get_command_args(method_name, command_args_tpl, local_volumes)
py_command_args = _get_command_args(function_name, command_args_tpl, local_volumes)
write_command_args_file(args_file, py_command_args)

if data_sample_paths is not None and len(data_sample_paths) > 0:
Expand Down
Loading