Skip to content

Commit

Permalink
Merge pull request #188 from runlightyear/incremental-sync
Browse files Browse the repository at this point in the history
Logging + getting prod ready
  • Loading branch information
ebouck authored Sep 12, 2024
2 parents 996ff8f + 3c144f9 commit bc0699c
Showing 39 changed files with 209 additions and 35 deletions.
7 changes: 7 additions & 0 deletions packages/@runlightyear/airtable/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @runlightyear/airtable

## 1.2.5

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 1.2.4

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/airtable/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/airtable",
"version": "1.2.4",
"version": "1.2.5",
"publishConfig": {
"access": "public"
},
11 changes: 11 additions & 0 deletions packages/@runlightyear/cli/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
# @runlightyear/cli

## 1.5.0

### Minor Changes

- 2d9ee36f2: Prepare to support streaming logs

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 1.4.4

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/cli/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/cli",
"version": "1.4.4",
"version": "1.5.0",
"publishConfig": {
"access": "public"
},
2 changes: 1 addition & 1 deletion packages/@runlightyear/cli/src/shared/runAction.ts
Original file line number Diff line number Diff line change
@@ -92,7 +92,7 @@ export default async function runAction({
},
body: JSON.stringify({
status,
logs,
// logs,
endedAt: "now",
// deliveryId,
}),
7 changes: 7 additions & 0 deletions packages/@runlightyear/gcal/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @runlightyear/gcal

## 0.11.5

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 0.11.4

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/gcal/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/gcal",
"version": "0.11.4",
"version": "0.11.5",
"publishConfig": {
"access": "public"
},
7 changes: 7 additions & 0 deletions packages/@runlightyear/github/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @runlightyear/github

## 1.3.5

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 1.3.4

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/github/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/github",
"version": "1.3.4",
"version": "1.3.5",
"publishConfig": {
"access": "public"
},
7 changes: 7 additions & 0 deletions packages/@runlightyear/gmail/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @runlightyear/gmail

## 0.6.6

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 0.6.5

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/gmail/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/gmail",
"version": "0.6.5",
"version": "0.6.6",
"publishConfig": {
"access": "public"
},
7 changes: 7 additions & 0 deletions packages/@runlightyear/gsheets/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @runlightyear/gsheets

## 0.8.5

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 0.8.4

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/gsheets/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/gsheets",
"version": "0.8.4",
"version": "0.8.5",
"publishConfig": {
"access": "public"
},
6 changes: 6 additions & 0 deletions packages/@runlightyear/lightyear/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# @runlightyear/lightyear

## 1.11.0

### Minor Changes

- 2d9ee36f2: Prepare to support streaming logs

## 1.10.0

### Minor Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/lightyear/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/lightyear",
"version": "1.10.0",
"version": "1.11.0",
"publishConfig": {
"access": "public"
},
21 changes: 13 additions & 8 deletions packages/@runlightyear/lightyear/src/base/baseRequest.ts
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ export interface BaseRequestProps {
data?: {
[key: string]: any;
};
suppressLogs?: boolean;
}

/**
@@ -29,8 +30,10 @@ export default async function baseRequest({
uri,
params,
data,
suppressLogs = false,
}: BaseRequestProps): Promise<Response> {
console.debug("in baseRequest");
if (!suppressLogs) console.debug("in baseRequest");

const baseUrl = getBaseUrl();
invariant(baseUrl, "Missing BASE_URL");

@@ -52,20 +55,22 @@ export default async function baseRequest({
body: data && JSON.stringify(data),
};

console.debug(`baseRequest url: ${url}`);
console.debug("baseRequest props", props);
if (!suppressLogs) console.debug(`baseRequest url: ${url}`);
if (!suppressLogs) console.debug("baseRequest props", props);

const response = await fetch(url, props);

if (!response.ok) {
console.error(
`Base request error: ${response.status} ${response.statusText}`
);
console.error(JSON.stringify(await response.json(), null, 2));
if (!suppressLogs)
console.error(
`Base request error: ${response.status} ${response.statusText}`
);
if (!suppressLogs)
console.error(JSON.stringify(await response.json(), null, 2));
throw new BaseRequestError(response);
}

console.debug("about to return from baseRequest");
if (!suppressLogs) console.debug("about to return from baseRequest");

return response;
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { prefixedRedactedConsole } from "../logging";

export function handlerResult(
export async function handlerResult(
statusCode: number,
message: string,
additionalData?: { [key: string]: any }
@@ -13,6 +13,8 @@ export function handlerResult(
// emptyLogs();
// emptySecrets();

await prefixedRedactedConsole.flushQueue();

return {
statusCode,
body: JSON.stringify(
1 change: 1 addition & 0 deletions packages/@runlightyear/lightyear/src/handler/index.ts
Original file line number Diff line number Diff line change
@@ -124,6 +124,7 @@ export async function handler(
} else if (operation === "refreshSubscription") {
return handleRefreshSubscription({ webhookName });
} else if (operation === "run") {
// prefixedRedactedConsole.setStreamLogsTo({ runId });
return handleRun({ actionName, runId, data, context });
} else if (operation === "getAuthRequestUrl") {
if (!customAppName) {
Original file line number Diff line number Diff line change
@@ -2,6 +2,9 @@ import argsToStr from "./argsToStr";
import redactSecrets from "./redactSecrets";
import isString from "../util/isString";
import * as process from "process";
import { dayjsUtc } from "../util/dayjsUtc";
import baseRequest from "../base/baseRequest";
import { getEnvName } from "../util/getEnvName";

export type LogDisplayLevel = "DEBUG" | "INFO";

@@ -13,14 +16,26 @@ export class PrefixedRedactedConsole {
secrets: Array<string> = [];
history: Array<string> = [];
logDisplayLevel: LogDisplayLevel = "DEBUG";
streamLogsTo: { runId?: string } | null = null;
streamCursor: number = 0;
logQueue: Array<{
message: string;
level: "INFO" | "DEBUG" | "WARN" | "ERROR" | "LOG" | "TRACE";
timestamp: string;
}> = [];

setGlobalPrefix(prefix: string) {
this.globalPrefix = prefix;
}

setStreamLogsTo(props: { runId?: string }) {
this.streamLogsTo = props;
}

initialize() {
this.secrets = [];
this.history = [];
this.logQueue = [];
}

addSecrets(secrets: Array<string | null>) {
@@ -112,6 +127,11 @@ export class PrefixedRedactedConsole {
const { params, color, prefix, stream, display } = props;
const message = `[${prefix}]: ${this._redactParams(params)}`;
this.history.push(message);
this._enqueue({
message: this._redactParams(params),
prefix,
timestamp: dayjsUtc().toISOString(),
});
if (display) {
stream.write(
`${
@@ -120,4 +140,33 @@ export class PrefixedRedactedConsole {
);
}
}

_enqueue(props: { message: string; prefix: string; timestamp: string }) {
if (this.streamLogsTo) {
this.logQueue.push({
message: props.message,
level: props.prefix as any,
timestamp: props.timestamp,
});
}

if (this.logQueue.length === 100) {
// do not await this so we can stream in the background
this.flushQueue();
}
}

async flushQueue() {
if (this.logQueue.length > 0) {
const envName = getEnvName();
const logsToStream = [...this.logQueue];
this.logQueue = [];
await baseRequest({
method: "POST",
uri: `/api/v1/envs/${envName}/logs`,
data: { ...this.streamLogsTo, logs: logsToStream },
suppressLogs: true,
});
}
}
}
Original file line number Diff line number Diff line change
@@ -120,18 +120,13 @@ export abstract class ModelSynchronizer<T> {
}
}

console.debug("about to map object data");
if (this.toObjectData) {
console.debug("using this.toObjectData", this.toObjectData);
for (const [objectFieldName, transform] of Object.entries(
this.toObjectData
)) {
console.debug("objectFieldName", objectFieldName);
console.debug("transform", transform);
if (typeof transform === "function") {
object.data[objectFieldName] = transform(source);
} else {
console.debug("doing the get on source", source);
object.data[objectFieldName] = get(source, transform);
}
}
7 changes: 7 additions & 0 deletions packages/@runlightyear/linear/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
# @runlightyear/linear

## 0.10.5

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 0.10.4

### Patch Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/linear/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/linear",
"version": "0.10.4",
"version": "0.10.5",
"publishConfig": {
"access": "public"
},
11 changes: 11 additions & 0 deletions packages/@runlightyear/node/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
# @runlightyear/node

## 0.4.0

### Minor Changes

- 016a33620: Require an env on init

### Patch Changes

- Updated dependencies [2d9ee36f2]
- @runlightyear/lightyear@1.11.0

## 0.3.0

### Minor Changes
2 changes: 1 addition & 1 deletion packages/@runlightyear/node/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@runlightyear/node",
"version": "0.3.0",
"version": "0.4.0",
"publishConfig": {
"access": "public"
},
Loading

0 comments on commit bc0699c

Please sign in to comment.