Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add upload to attic cache #79

Merged
merged 3 commits into from
Oct 30, 2024
Merged

add upload to attic cache #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6101c80
add upload to attic cache
techknowlogick Oct 25, 2024
6b119b8
only push 'out'
techknowlogick Oct 25, 2024
f41bcd6
add docs
techknowlogick Oct 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,19 @@ set. These environment variables are currently not propagated to ssh when using
the `--remote` flag, instead the user is expected that cachix credentials are
configured on the remote machine.

## Attic support

nix-fast-build can upload to attic like this:

```console
$ nix-fast-build --attic-cache mic92
```

nix-fast-build assumes that your current machine is either logged in to attic.
Authentication is not propagated to ssh when using the `--remote` flag, instead
the user is expected that attic credentials are configured on the remote
machine.

## Machine-readable builds results

nix-fast-build supports both its own json format and junit:
Expand Down Expand Up @@ -175,7 +188,8 @@ nix-shell -p python3Packages.junit2html --run 'junit2html result.xml result.html
```console
usage: nix-fast-build [-h] [-f FLAKE] [-j MAX_JOBS] [--option name value]
[--remote-ssh-option name value]
[--cachix-cache CACHIX_CACHE] [--no-nom]
[--cachix-cache CACHIX_CACHE]
[--attic-cache ATTIC_CACHE] [--no-nom]
[--systems SYSTEMS] [--retries RETRIES] [--no-link]
[--out-link OUT_LINK] [--remote REMOTE]
[--always-upload-source] [--no-download] [--skip-cached]
Expand All @@ -197,6 +211,8 @@ options:
ssh option when accessing remote
--cachix-cache CACHIX_CACHE
Cachix cache to upload to
--attic-cache ATTIC_CACHE
Attic cache to upload to
--no-nom Don't use nix-output-monitor to print build output
(default: false)
--systems SYSTEMS Space-separated list of systems to build for (default:
Expand Down
67 changes: 67 additions & 0 deletions nix_fast_build/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ class Options:

cachix_cache: str | None = None

attic_cache: str | None = None

@property
def remote_url(self) -> None | str:
if self.remote is None:
Expand All @@ -97,6 +99,7 @@ class ResultType(enum.Enum):
UPLOAD = enum.auto()
DOWNLOAD = enum.auto()
CACHIX = enum.auto()
ATTIC = enum.auto()


@dataclass
Expand Down Expand Up @@ -183,6 +186,11 @@ async def parse_args(args: list[str]) -> Options:
help="Cachix cache to upload to",
default=None,
)
parser.add_argument(
"--attic-cache",
help="Attic cache to upload to",
default=None,
)
parser.add_argument(
"--no-nom",
help="Don't use nix-output-monitor to print build output (default: false)",
Expand Down Expand Up @@ -318,6 +326,7 @@ async def parse_args(args: list[str]) -> Options:
eval_workers=a.eval_workers,
copy_to=a.copy_to,
cachix_cache=a.cachix_cache,
attic_cache=a.attic_cache,
no_link=a.no_link,
out_link=a.out_link,
result_format=ResultFormat[a.result_format.upper()],
Expand Down Expand Up @@ -633,6 +642,22 @@ async def upload_cachix(
proc = await asyncio.create_subprocess_exec(*cmd)
return await proc.wait()

async def upload_attic(self, opts: Options) -> int:
if opts.attic_cache is None:
return 0
cmd = maybe_remote(
[
*nix_shell("nixpkgs#attic-client", "attic"),
"push",
opts.attic_cache,
self.outputs["out"],
],
opts,
)
logger.debug("run %s", shlex.join(cmd))
proc = await asyncio.create_subprocess_exec(*cmd)
return await proc.wait()

async def download(self, exit_stack: AsyncExitStack, opts: Options) -> int:
if not opts.remote_url or not opts.download:
return 0
Expand Down Expand Up @@ -757,6 +782,7 @@ async def run_builds(
build_queue: QueueWithContext[Job | StopTask],
upload_queue: QueueWithContext[Build | StopTask],
cachix_queue: QueueWithContext[Build | StopTask],
attic_queue: QueueWithContext[Build | StopTask],
download_queue: QueueWithContext[Build | StopTask],
results: list[Result],
opts: Options,
Expand Down Expand Up @@ -791,6 +817,7 @@ async def run_builds(
upload_queue.put_nowait(build)
download_queue.put_nowait(build)
cachix_queue.put_nowait(build)
attic_queue.put_nowait(build)


async def run_uploads(
Expand Down Expand Up @@ -842,6 +869,29 @@ async def run_cachix_upload(
)


async def run_attic_upload(
attic_queue: QueueWithContext[Build | StopTask],
results: list[Result],
opts: Options,
) -> int:
while True:
async with attic_queue.get_context() as build:
if isinstance(build, StopTask):
logger.debug("finish attic upload task")
return 0
start_time = timeit.default_timer()
rc = await build.upload_attic(opts)
results.append(
Result(
result_type=ResultType.ATTIC,
attr=build.attr,
success=rc == 0,
duration=start_time - timeit.default_timer(),
error=f"attic upload exited with {rc}" if rc != 0 else None,
)
)


async def run_downloads(
stack: AsyncExitStack,
download_queue: QueueWithContext[Build | StopTask],
Expand Down Expand Up @@ -915,6 +965,7 @@ async def run(stack: AsyncExitStack, opts: Options) -> int:
results: list[Result] = []
build_queue: QueueWithContext[Job | StopTask] = QueueWithContext()
cachix_queue: QueueWithContext[Build | StopTask] = QueueWithContext()
attic_queue: QueueWithContext[Build | StopTask] = QueueWithContext()
upload_queue: QueueWithContext[Build | StopTask] = QueueWithContext()
download_queue: QueueWithContext[Build | StopTask] = QueueWithContext()

Expand All @@ -937,6 +988,7 @@ async def run(stack: AsyncExitStack, opts: Options) -> int:
build_queue,
upload_queue,
cachix_queue,
attic_queue,
download_queue,
results,
opts,
Expand All @@ -961,6 +1013,16 @@ async def run(stack: AsyncExitStack, opts: Options) -> int:
name=f"cachix-{i}",
)
)
tasks.append(
tg.create_task(
run_attic_upload(
attic_queue,
results,
opts,
),
name=f"attic-{i}",
)
)
tasks.append(
tg.create_task(
run_downloads(stack, download_queue, results, opts),
Expand Down Expand Up @@ -993,6 +1055,11 @@ async def run(stack: AsyncExitStack, opts: Options) -> int:
cachix_queue.put_nowait(StopTask())
await cachix_queue.join()

logger.debug("Uploads finished, waiting for attic uploads to finish...")
for _ in range(opts.max_jobs):
attic_queue.put_nowait(StopTask())
await attic_queue.join()

logger.debug("Uploads finished, waiting for downloads to finish...")
for _ in range(opts.max_jobs):
download_queue.put_nowait(StopTask())
Expand Down