Skip to content

MarQS reserve concurrency system & queue priority for resuming/retrying #1715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 19, 2025
Merged

MarQS reserve concurrency system & queue priority for resuming/retrying #1715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
511a97c
run engine v1: orgs are no longer considered for concurrency
ericallam Feb 10, 2025
ac8fa75
Add reserve concurrency concept to allow waiting to resume parent tas…
ericallam Feb 10, 2025
7061d3a
child tasks inherit the queue timestamp from their parent tasks to pr…
ericallam Feb 10, 2025
0587b5a
handle reserve concurrency with recursive deadlocks
ericallam Feb 11, 2025
67f2f8b
Finish docs update for concurrency
ericallam Feb 11, 2025
bb53fc5
Some fixes from badge conflict resolution
ericallam Feb 13, 2025
61fab3d
WIP priority queues
ericallam Feb 15, 2025
32817f9
Implement MarQS priority queues
ericallam Feb 19, 2025
84f5563
Fix the migrations
ericallam Feb 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
340 changes: 340 additions & 0 deletions apps/webapp/app/components/admin/debugRun.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,340 @@
import { useIsImpersonating } from "~/hooks/useOrganizations";
import { useHasAdminAccess } from "~/hooks/useUser";
import { Button } from "../primitives/Buttons";
import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "../primitives/Dialog";
import { Cog6ToothIcon } from "@heroicons/react/20/solid";
import { type loader } from "~/routes/resources.taskruns.$runParam.debug";
import { UseDataFunctionReturn, useTypedFetcher } from "remix-typedjson";
import { useEffect } from "react";
import { Spinner } from "../primitives/Spinner";
import * as Property from "~/components/primitives/PropertyTable";
import { ClipboardField } from "../primitives/ClipboardField";
import { MarQSShortKeyProducer } from "~/v3/marqs/marqsKeyProducer";

export function AdminDebugRun({ friendlyId }: { friendlyId: string }) {
const hasAdminAccess = useHasAdminAccess();
const isImpersonating = useIsImpersonating();

if (!hasAdminAccess && !isImpersonating) {
return null;
}

return (
<Dialog key={`debug-${friendlyId}`}>
<DialogTrigger asChild>
<Button variant="tertiary/small" LeadingIcon={Cog6ToothIcon}>
Debug run
</Button>
</DialogTrigger>
<DebugRunDialog friendlyId={friendlyId} />
</Dialog>
);
}

export function DebugRunDialog({ friendlyId }: { friendlyId: string }) {
return (
<DialogContent
key={`debug`}
className="overflow-y-auto sm:h-[80vh] sm:max-h-[80vh] sm:max-w-[50vw]"
>
<DebugRunContent friendlyId={friendlyId} />
</DialogContent>
);
}

function DebugRunContent({ friendlyId }: { friendlyId: string }) {
const fetcher = useTypedFetcher<typeof loader>();
const isLoading = fetcher.state === "loading";

useEffect(() => {
fetcher.load(`/resources/taskruns/${friendlyId}/debug`);
}, [friendlyId]);

return (
<>
<DialogHeader>Debugging run</DialogHeader>
{isLoading ? (
<div className="grid place-items-center p-6">
<Spinner />
</div>
) : fetcher.data ? (
<DebugRunData {...fetcher.data} />
) : (
<>Failed to get run debug data</>
)}
</>
);
}

function DebugRunData({
run,
queueConcurrencyLimit,
queueCurrentConcurrency,
envConcurrencyLimit,
envCurrentConcurrency,
queueReserveConcurrency,
envReserveConcurrency,
}: UseDataFunctionReturn<typeof loader>) {
const keys = new MarQSShortKeyProducer("marqs:");

const withPrefix = (key: string) => `marqs:${key}`;

return (
<Property.Table>
<Property.Item>
<Property.Label>ID</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField value={run.id} variant="tertiary/small" iconButton />
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Message key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(keys.messageKey(run.id))}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>GET message</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`GET ${withPrefix(keys.messageKey(run.id))}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(
keys.queueKey(run.runtimeEnvironment, run.queue, run.concurrencyKey ?? undefined)
)}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Get queue set</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`ZRANGE ${withPrefix(
keys.queueKey(run.runtimeEnvironment, run.queue, run.concurrencyKey ?? undefined)
)} 0 -1`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue current concurrency key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(
keys.queueCurrentConcurrencyKey(
run.runtimeEnvironment,
run.queue,
run.concurrencyKey ?? undefined
)
)}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>

<Property.Item>
<Property.Label>Get queue current concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`SMEMBERS ${withPrefix(
keys.queueCurrentConcurrencyKey(
run.runtimeEnvironment,
run.queue,
run.concurrencyKey ?? undefined
)
)}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue current concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<span>{queueCurrentConcurrency ?? "0"}</span>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue reserve concurrency key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(
keys.queueReserveConcurrencyKeyFromQueue(
keys.queueKey(run.runtimeEnvironment, run.queue, run.concurrencyKey ?? undefined)
)
)}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>

<Property.Item>
<Property.Label>Get queue reserve concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`SMEMBERS ${withPrefix(
keys.queueReserveConcurrencyKeyFromQueue(
keys.queueKey(run.runtimeEnvironment, run.queue, run.concurrencyKey ?? undefined)
)
)}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue reserve concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<span>{queueReserveConcurrency ?? "0"}</span>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue concurrency limit key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(keys.queueConcurrencyLimitKey(run.runtimeEnvironment, run.queue))}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>GET queue concurrency limit</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`GET ${withPrefix(
keys.queueConcurrencyLimitKey(run.runtimeEnvironment, run.queue)
)}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Queue concurrency limit</Property.Label>
<Property.Value className="flex items-center gap-2">
<span>{queueConcurrencyLimit ?? "Not set"}</span>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Env current concurrency key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(keys.envCurrentConcurrencyKey(run.runtimeEnvironment))}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Get env current concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`SMEMBERS ${withPrefix(keys.envCurrentConcurrencyKey(run.runtimeEnvironment))}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Env current concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<span>{envCurrentConcurrency ?? "0"}</span>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Env reserve concurrency key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(keys.envReserveConcurrencyKey(run.runtimeEnvironment.id))}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Get env reserve concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`SMEMBERS ${withPrefix(
keys.envReserveConcurrencyKey(run.runtimeEnvironment.id)
)}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Env reserve concurrency</Property.Label>
<Property.Value className="flex items-center gap-2">
<span>{envReserveConcurrency ?? "0"}</span>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Env concurrency limit key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={withPrefix(keys.envConcurrencyLimitKey(run.runtimeEnvironment))}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>GET env concurrency limit</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`GET ${withPrefix(keys.envConcurrencyLimitKey(run.runtimeEnvironment))}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Env concurrency limit</Property.Label>
<Property.Value className="flex items-center gap-2">
<span>{envConcurrencyLimit ?? "Not set"}</span>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Shared queue key</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`GET ${withPrefix(keys.envSharedQueueKey(run.runtimeEnvironment))}`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
<Property.Item>
<Property.Label>Get shared queue set</Property.Label>
<Property.Value className="flex items-center gap-2">
<ClipboardField
value={`ZRANGEBYSCORE ${withPrefix(
keys.envSharedQueueKey(run.runtimeEnvironment)
)} -inf ${Date.now()} WITHSCORES`}
variant="tertiary/small"
iconButton
/>
</Property.Value>
</Property.Item>
</Property.Table>
);
}
11 changes: 9 additions & 2 deletions apps/webapp/app/components/runs/v3/BatchStatus.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
import { CheckCircleIcon } from "@heroicons/react/20/solid";
import { CheckCircleIcon, XCircleIcon } from "@heroicons/react/20/solid";
import { BatchTaskRunStatus } from "@trigger.dev/database";
import assertNever from "assert-never";
import { Spinner } from "~/components/primitives/Spinner";
import { cn } from "~/utils/cn";

export const allBatchStatuses = ["PENDING", "COMPLETED"] as const satisfies Readonly<
export const allBatchStatuses = ["PENDING", "COMPLETED", "ABORTED"] as const satisfies Readonly<
Array<BatchTaskRunStatus>
>;

const descriptions: Record<BatchTaskRunStatus, string> = {
PENDING: "The batch has child runs that have not yet completed.",
COMPLETED: "All the batch child runs have finished.",
ABORTED: "The batch was aborted because some child tasks could not be triggered.",
};

export function descriptionForBatchStatus(status: BatchTaskRunStatus): string {
Expand Down Expand Up @@ -50,6 +51,8 @@ export function BatchStatusIcon({
return <Spinner className={cn(batchStatusColor(status), className)} />;
case "COMPLETED":
return <CheckCircleIcon className={cn(batchStatusColor(status), className)} />;
case "ABORTED":
return <XCircleIcon className={cn(batchStatusColor(status), className)} />;
default: {
assertNever(status);
}
Expand All @@ -62,6 +65,8 @@ export function batchStatusColor(status: BatchTaskRunStatus): string {
return "text-pending";
case "COMPLETED":
return "text-success";
case "ABORTED":
return "text-error";
default: {
assertNever(status);
}
Expand All @@ -74,6 +79,8 @@ export function batchStatusTitle(status: BatchTaskRunStatus): string {
return "In progress";
case "COMPLETED":
return "Completed";
case "ABORTED":
return "Aborted";
default: {
assertNever(status);
}
Expand Down
2 changes: 1 addition & 1 deletion apps/webapp/app/env.server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ const EnvironmentSchema = z.object({
MARQS_AVAILABLE_CAPACITY_BIAS: z.coerce.number().default(0.3),
MARQS_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25),
MARQS_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0),
MARQS_MAXIMUM_ORG_COUNT: z.coerce.number().int().optional(),
MARQS_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(),

PROD_TASK_HEARTBEAT_INTERVAL_MS: z.coerce.number().int().optional(),

Expand Down
Loading
Loading