Skip to content

Commit

Permalink
test: algo to function in substratools (#303)
Browse files Browse the repository at this point in the history
Signed-off-by: ThibaultFy <50656860+ThibaultFy@users.noreply.github.com>
  • Loading branch information
ThibaultFy authored Oct 11, 2022
1 parent 6137bee commit 15f7e1c
Show file tree
Hide file tree
Showing 6 changed files with 144 additions and 142 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,16 @@ compute_plan = client.add_compute_plan(
- link_dataset_with_data_samples return value in remote mode
- Fix the compute plan rank calculation in local mode (#299)

### Changed

- Apply changes from algo to function in substratools (#303)

## [0.39.0](https://github.com/Substra/substra/releases/tag/0.39.0) - 2022-10-03

### Removed

- BREAKING CHANGE: remove category from substra.schema.AlgoSpec and substra.models.Algo

### Added

- Prevent use of `__` in asset metadata keys in local mode
Expand Down
6 changes: 3 additions & 3 deletions references/sdk.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# Client
```text
Client(url: Union[str, NoneType] = None, token: Union[str, NoneType] = None, retry_timeout: int = 300, insecure: bool = False, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
Client(url: Optional[str] = None, token: Optional[str] = None, retry_timeout: int = 300, insecure: bool = False, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
```

Create a client
Expand Down Expand Up @@ -255,7 +255,7 @@ algorithm.
- `pathlib.Path`: Path of the downloaded model
## download_model_from_task
```text
download_model_from_task(self, task_key: str, identifier: str, folder: os.PathLike) -> None
download_model_from_task(self, task_key: str, identifier: str, folder: os.PathLike) -> pathlib.Path
```

Download task model to destination file.
Expand All @@ -273,7 +273,7 @@ algorithm.
- `pathlib.Path`: Path of the downloaded model
## from_config_file
```text
from_config_file(profile_name: str = 'default', config_path: Union[str, pathlib.Path] = '~/.substra', tokens_path: Union[str, pathlib.Path] = '~/.substra-tokens', token: Union[str, NoneType] = None, retry_timeout: int = 300, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
from_config_file(profile_name: str = 'default', config_path: Union[str, pathlib.Path] = '~/.substra', tokens_path: Union[str, pathlib.Path] = '~/.substra-tokens', token: Optional[str] = None, retry_timeout: int = 300, backend_type: substra.sdk.schemas.BackendType = <BackendType.REMOTE: 'remote'>)
```

Returns a new Client configured with profile data from configuration files.
Expand Down
20 changes: 10 additions & 10 deletions substra/sdk/backends/local/compute/spawner/subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,19 @@
logger = logging.getLogger(__name__)

PYTHON_SCRIPT_REGEX = r"(?<=\")([^\"]*\.py)(?=\")"
METHOD_REGEX = r"\"\-\-method-name\"\,\s*\"([^\"]*)\""
METHOD_REGEX = r"\"\-\-function-name\"\,\s*\"([^\"]*)\""


def _get_entrypoint_from_dockerfile(tmpdir):
"""
Extracts the .py script and the function name to execute in
an ENTRYPOINT line of a Dockerfile, located in tmpdir.
For instance if the line `ENTRYPOINT ["python3", "algo.py", "--method-name", "train"]` is in the Dockerfile,
For instance if the line `ENTRYPOINT ["python3", "algo.py", "--function-name", "train"]` is in the Dockerfile,
`algo.py`, `train` is extracted.
"""
valid_example = (
"""The entry point should be specified as follow: """
"""``ENTRYPOINT ["<executor>", "<algo_file.py>", "--method-name", "<method name>"]"""
"""``ENTRYPOINT ["<executor>", "<algo_file.py>", "--function-name", "<method name>"]"""
)
with open(tmpdir / "Dockerfile") as f:
for line in f:
Expand All @@ -41,19 +41,19 @@ def _get_entrypoint_from_dockerfile(tmpdir):
if len(script_name) != 1:
raise ExecutionError("Couldn't extract script from ENTRYPOINT line in Dockerfile", valid_example)

method_name = re.findall(METHOD_REGEX, line)
if len(method_name) != 1:
function_name = re.findall(METHOD_REGEX, line)
if len(function_name) != 1:
raise ExecutionError("Couldn't extract method name from ENTRYPOINT line in Dockerfile", valid_example)

return script_name[0], method_name[0]
return script_name[0], function_name[0]

raise ExecutionError("Couldn't get entrypoint in Dockerfile", valid_example)


def _get_command_args(
method_name: str, args_template: typing.List[string.Template], local_volumes: typing.Dict[str, str]
function_name: str, args_template: typing.List[string.Template], local_volumes: typing.Dict[str, str]
) -> typing.List[str]:
args = ["--method-name", str(method_name)]
args = ["--function-name", str(function_name)]
args += [tpl.substitute(**local_volumes) for tpl in args_template]
return args

Expand Down Expand Up @@ -98,12 +98,12 @@ def spawn(
algo_dir = pathlib.Path(algo_dir)
args_dir = pathlib.Path(args_dir)
uncompress(archive_path, algo_dir)
script_name, method_name = _get_entrypoint_from_dockerfile(algo_dir)
script_name, function_name = _get_entrypoint_from_dockerfile(algo_dir)

args_file = args_dir / "arguments.txt"

py_command = [sys.executable, str(algo_dir / script_name), f"@{args_file}"]
py_command_args = _get_command_args(method_name, command_args_tpl, local_volumes)
py_command_args = _get_command_args(function_name, command_args_tpl, local_volumes)
write_command_args_file(args_file, py_command_args)

if data_sample_paths is not None and len(data_sample_paths) > 0:
Expand Down
Loading

0 comments on commit 15f7e1c

Please sign in to comment.