Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Custom base images for functions #1297

Merged
merged 14 commits into from
May 7, 2024
152 changes: 109 additions & 43 deletions client/quantum_serverless/core/job.py
Original file line number Diff line number Diff line change
Expand Up @@ -455,54 +455,22 @@ def upload(self, program: QiskitFunction):
tracer = trace.get_tracer("client.tracer")
with tracer.start_as_current_span("job.run") as span:
span.set_attribute("program", program.title)

url = f"{self.host}/api/{self.version}/programs/upload/"
artifact_file_path = os.path.join(program.working_dir, "artifact.tar")

# check if entrypoint exists
if not os.path.exists(
os.path.join(program.working_dir, program.entrypoint)
):
raise QuantumServerlessException(
f"Entrypoint file [{program.entrypoint}] does not exist "
f"in [{program.working_dir}] working directory."
if program.image is not None:
# upload function with custom image
program_title = _upload_with_docker_image(
program=program, url=url, token=self._token, span=span
)

with tarfile.open(artifact_file_path, "w") as tar:
for filename in os.listdir(program.working_dir):
fpath = os.path.join(program.working_dir, filename)
tar.add(fpath, arcname=filename)

# check file size
size_in_mb = Path(artifact_file_path).stat().st_size / 1024**2
if size_in_mb > MAX_ARTIFACT_FILE_SIZE_MB:
raise QuantumServerlessException(
f"{artifact_file_path} is {int(size_in_mb)} Mb, "
f"which is greater than {MAX_ARTIFACT_FILE_SIZE_MB} allowed. "
f"Try to reduce size of `working_dir`."
elif program.entrypoint is not None:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it good to put a waring message when both image and entrypoint are set?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be better to move all validation to QiskitFunction itself to reduce boilderplate code. Created an iissue for that. #1315

Thanks for suggestion!

# upload funciton with artifact
program_title = _upload_with_artifact(
program=program, url=url, token=self._token, span=span
)

with open(artifact_file_path, "rb") as file:
response_data = safe_json_request(
request=lambda: requests.post(
url=url,
data={
"title": program.title,
"entrypoint": program.entrypoint,
"arguments": json.dumps({}),
"dependencies": json.dumps(program.dependencies or []),
"env_vars": json.dumps(program.env_vars or {}),
},
files={"artifact": file},
headers={"Authorization": f"Bearer {self._token}"},
timeout=REQUESTS_TIMEOUT,
)
else:
raise QuantumServerlessException(
"Function must either have `entryoint` or `image` specified."
)
program_title = response_data.get("title", "na")
span.set_attribute("program.title", program_title)

if os.path.exists(artifact_file_path):
os.remove(artifact_file_path)

return program_title

Expand Down Expand Up @@ -835,3 +803,101 @@ def _map_status_to_serverless(status: str) -> str:
return status_map[status]
except KeyError:
return status


def _upload_with_docker_image(
program: QiskitFunction, url: str, token: str, span: Any
) -> str:
"""Uploads function with custom docker image.

Args:
program (QiskitFunction): function instance
url (str): upload gateway url
token (str): auth token
span (Any): tracing span

Returns:
str: uploaded function name
"""
response_data = safe_json_request(
request=lambda: requests.post(
url=url,
data={
"title": program.title,
"image": program.image,
"arguments": json.dumps({}),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unrelated question. Is the arguments for the program used?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

probably you are right and it is not used anymore. I will look it up and try to fix it in following PRs

"dependencies": json.dumps(program.dependencies or []),
"env_vars": json.dumps(program.env_vars or {}),
},
headers={"Authorization": f"Bearer {token}"},
timeout=REQUESTS_TIMEOUT,
)
)
program_title = response_data.get("title", "na")
span.set_attribute("program.title", program_title)
return program_title


def _upload_with_artifact(
program: QiskitFunction, url: str, token: str, span: Any
) -> str:
"""Uploads function with artifact.

Args:
program (QiskitFunction): function instance
url (str): endpoint for gateway upload
token (str): auth token
span (Any): tracing span

Raises:
QuantumServerlessException: if no entrypoint or size of artifact is too large.

Returns:
str: uploaded function name
"""
artifact_file_path = os.path.join(program.working_dir, "artifact.tar")

# check if entrypoint exists
if not os.path.exists(os.path.join(program.working_dir, program.entrypoint)):
raise QuantumServerlessException(
f"Entrypoint file [{program.entrypoint}] does not exist "
f"in [{program.working_dir}] working directory."
)

with tarfile.open(artifact_file_path, "w") as tar:
for filename in os.listdir(program.working_dir):
fpath = os.path.join(program.working_dir, filename)
tar.add(fpath, arcname=filename)

# check file size
size_in_mb = Path(artifact_file_path).stat().st_size / 1024**2
if size_in_mb > MAX_ARTIFACT_FILE_SIZE_MB:
raise QuantumServerlessException(
f"{artifact_file_path} is {int(size_in_mb)} Mb, "
f"which is greater than {MAX_ARTIFACT_FILE_SIZE_MB} allowed. "
f"Try to reduce size of `working_dir`."
)

with open(artifact_file_path, "rb") as file:
response_data = safe_json_request(
request=lambda: requests.post(
url=url,
data={
"title": program.title,
"entrypoint": program.entrypoint,
"arguments": json.dumps({}),
"dependencies": json.dumps(program.dependencies or []),
"env_vars": json.dumps(program.env_vars or {}),
},
files={"artifact": file},
headers={"Authorization": f"Bearer {token}"},
timeout=REQUESTS_TIMEOUT,
)
)
program_title = response_data.get("title", "na")
span.set_attribute("program.title", program_title)

if os.path.exists(artifact_file_path):
os.remove(artifact_file_path)

return program_title
Loading
Loading