Skip to content

Commit 3b9776d

Browse files
LasimLasim
committed
refactor(satellite): extract tarball operations for improved maintainability
1 parent 7e6a437 commit 3b9776d

File tree

2 files changed

+206
-145
lines changed

2 files changed

+206
-145
lines changed

‎services/satellite/src/process/github-deployment.ts‎

Lines changed: 8 additions & 145 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
import { spawn } from 'child_process';
22
import { mkdir } from 'fs/promises';
3-
import * as tar from 'tar';
43
import * as path from 'path';
54
import * as fs from 'fs';
6-
import { Octokit } from '@octokit/rest';
75
import { Logger } from 'pino';
86
import { MCPServerConfig } from './types';
97
import { LogBuffer } from './log-buffer';
@@ -24,6 +22,10 @@ import {
2422
detectRuntime,
2523
type GitHubInfo
2624
} from '../utils/node-helpers';
25+
import {
26+
downloadRepository,
27+
extractTarball
28+
} from '../utils/tarball-operations';
2729

2830
// Re-export GitHubInfo for backward compatibility
2931
export type { GitHubInfo };
@@ -70,147 +72,7 @@ export class GitHubDeploymentHandler {
7072
!!this.backendClient;
7173
}
7274

73-
/**
74-
* Download GitHub repository as tarball using Octokit
75-
* Includes retry logic with exponential backoff
76-
*/
77-
async downloadRepository(
78-
owner: string,
79-
repo: string,
80-
ref: string,
81-
token: string,
82-
maxRetries = 3
83-
): Promise<Buffer> {
84-
const octokit = new Octokit({ auth: token });
85-
86-
for (let attempt = 1; attempt <= maxRetries; attempt++) {
87-
try {
88-
this.logger.debug({
89-
operation: 'github_tarball_download_start',
90-
owner,
91-
repo,
92-
ref,
93-
attempt,
94-
max_retries: maxRetries
95-
}, `Downloading GitHub repository tarball (attempt ${attempt}/${maxRetries})`);
96-
97-
const response = await octokit.request('GET /repos/{owner}/{repo}/tarball/{ref}', {
98-
owner,
99-
repo,
100-
ref,
101-
request: {
102-
parseSuccessResponseBody: false // Get raw response
103-
}
104-
});
105-
106-
// Response.data is a ReadableStream - convert to Buffer
107-
let buffer: Buffer;
108-
109-
if (response.data instanceof ReadableStream) {
110-
// Convert ReadableStream to Buffer
111-
const reader = response.data.getReader();
112-
const chunks: Uint8Array[] = [];
113-
114-
while (true) {
115-
const { done, value } = await reader.read();
116-
if (done) break;
117-
chunks.push(value);
118-
}
119-
120-
buffer = Buffer.concat(chunks);
121-
} else if (response.data instanceof ArrayBuffer) {
122-
buffer = Buffer.from(response.data);
123-
} else if (Buffer.isBuffer(response.data)) {
124-
buffer = response.data;
125-
} else {
126-
throw new Error(`Unexpected response data type: ${typeof response.data}`);
127-
}
128-
129-
this.logger.info({
130-
operation: 'github_tarball_download_success',
131-
owner,
132-
repo,
133-
ref,
134-
size_bytes: buffer.length,
135-
attempt
136-
}, `Downloaded GitHub repository tarball (${(buffer.length / 1024).toFixed(2)} KB)`);
137-
138-
return buffer;
139-
140-
} catch (error) {
141-
const errorMessage = error instanceof Error ? error.message : String(error);
142-
143-
this.logger.error({
144-
operation: 'github_tarball_download_failed',
145-
owner,
146-
repo,
147-
ref,
148-
attempt,
149-
max_retries: maxRetries,
150-
error: errorMessage
151-
}, `Failed to download GitHub repository tarball (attempt ${attempt}/${maxRetries})`);
152-
153-
if (attempt === maxRetries) {
154-
throw new Error(`Failed to download GitHub repository after ${maxRetries} attempts: ${errorMessage}`);
155-
}
156-
157-
// Exponential backoff: 1s, 2s, 4s
158-
const backoffMs = Math.pow(2, attempt - 1) * 1000;
159-
await new Promise(resolve => setTimeout(resolve, backoffMs));
160-
}
161-
}
162-
163-
throw new Error('Unreachable: maxRetries exhausted');
164-
}
165-
166-
/**
167-
* Extract tarball to temporary directory
168-
*/
169-
async extractTarball(tarballBuffer: Buffer, tempDir: string): Promise<void> {
170-
const isTmpfs = await this.tmpfsManager.isTmpfs(tempDir);
171-
172-
this.logger.debug({
173-
operation: 'tarball_extract_start',
174-
temp_dir: tempDir,
175-
tarball_size: tarballBuffer.length,
176-
is_tmpfs: isTmpfs
177-
}, `Extracting tarball to ${isTmpfs ? 'tmpfs' : 'filesystem'} directory`);
178-
179-
try {
180-
// Create temp directory
181-
await mkdir(tempDir, { recursive: true });
182-
183-
// Write tarball to temp file (tar.extract needs a file path)
184-
const tarballPath = path.join(tempDir, 'repo.tar.gz');
185-
await fs.promises.writeFile(tarballPath, tarballBuffer);
186-
187-
// Extract tarball (GitHub tarballs have a root directory, so strip it)
188-
await tar.extract({
189-
file: tarballPath,
190-
cwd: tempDir,
191-
strip: 1 // Remove the root directory from GitHub tarball
192-
});
193-
194-
// Remove the tarball file
195-
await fs.promises.unlink(tarballPath);
196-
197-
this.logger.debug({
198-
operation: 'tarball_extract_success',
199-
temp_dir: tempDir
200-
}, 'Tarball extracted successfully');
201-
202-
} catch (error) {
203-
const errorMessage = error instanceof Error ? error.message : String(error);
20475

205-
this.logger.error({
206-
operation: 'tarball_extract_failed',
207-
temp_dir: tempDir,
208-
error: errorMessage
209-
}, 'Failed to extract tarball');
210-
211-
throw new Error(`Failed to extract tarball: ${errorMessage}`);
212-
}
213-
}
21476

21577
/**
21678
* Install dependencies in extracted repository
@@ -984,11 +846,12 @@ export class GitHubDeploymentHandler {
984846
}, 'GitHub token fetched successfully');
985847

986848
// Download repository as tarball
987-
const tarballBuffer = await this.downloadRepository(
849+
const tarballBuffer = await downloadRepository(
988850
githubInfo.owner,
989851
githubInfo.repo,
990852
githubInfo.ref,
991-
tokenResult.token
853+
tokenResult.token,
854+
this.logger
992855
);
993856

994857
// Create deployment directory
@@ -1039,7 +902,7 @@ export class GitHubDeploymentHandler {
1039902
}
1040903

1041904
// Extract tarball to deployment directory
1042-
await this.extractTarball(tarballBuffer, deploymentDir);
905+
await extractTarball(tarballBuffer, deploymentDir, this.logger, this.tmpfsManager);
1043906

1044907
// Detect runtime from extracted files or use config runtime
1045908
const runtime = config.runtime || await detectRuntime(deploymentDir);

‎services/satellite/src/utils/tarball-operations.ts‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/**
2+
* Tarball operations for GitHub repository deployments
3+
*
4+
* Extracted from github-deployment.ts to reduce file size and improve maintainability.
5+
* Contains download and extraction logic for GitHub repository tarballs.
6+
*/
7+
8+
import { Octokit } from '@octokit/rest';
9+
import * as tar from 'tar';
10+
import * as path from 'path';
11+
import * as fs from 'fs';
12+
import { mkdir } from 'fs/promises';
13+
14+
/**
15+
* Optional logger interface for dependency injection
16+
*/
17+
interface Logger {
18+
trace?: (obj: unknown, msg?: string) => void;
19+
debug?: (obj: unknown, msg?: string) => void;
20+
info?: (obj: unknown, msg?: string) => void;
21+
warn?: (obj: unknown, msg?: string) => void;
22+
error?: (obj: unknown, msg?: string) => void;
23+
fatal?: (obj: unknown, msg?: string) => void;
24+
}
25+
26+
/**
27+
* Optional TmpfsManager interface for tmpfs detection
28+
*/
29+
interface TmpfsManager {
30+
isTmpfs(dirPath: string): Promise<boolean>;
31+
}
32+
33+
/**
34+
* Download GitHub repository as tarball using Octokit
35+
*
36+
* Includes retry logic with exponential backoff to handle transient network failures.
37+
*
38+
* @param owner - GitHub repository owner
39+
* @param repo - GitHub repository name
40+
* @param ref - Git reference (commit hash, branch, or tag)
41+
* @param token - GitHub authentication token
42+
* @param logger - Optional logger for debug/info/error messages
43+
* @param maxRetries - Maximum number of retry attempts (default: 3)
44+
* @returns Buffer containing the tarball data
45+
* @throws Error if download fails after all retries
46+
*/
47+
export async function downloadRepository(
48+
owner: string,
49+
repo: string,
50+
ref: string,
51+
token: string,
52+
logger?: Logger,
53+
maxRetries = 3
54+
): Promise<Buffer> {
55+
const octokit = new Octokit({ auth: token });
56+
57+
for (let attempt = 1; attempt <= maxRetries; attempt++) {
58+
try {
59+
logger?.debug?.({
60+
operation: 'github_tarball_download_start',
61+
owner,
62+
repo,
63+
ref,
64+
attempt,
65+
max_retries: maxRetries
66+
}, `Downloading GitHub repository tarball (attempt ${attempt}/${maxRetries})`);
67+
68+
const response = await octokit.request('GET /repos/{owner}/{repo}/tarball/{ref}', {
69+
owner,
70+
repo,
71+
ref,
72+
request: {
73+
parseSuccessResponseBody: false // Get raw response
74+
}
75+
});
76+
77+
// Response.data is a ReadableStream - convert to Buffer
78+
let buffer: Buffer;
79+
80+
if (response.data instanceof ReadableStream) {
81+
// Convert ReadableStream to Buffer
82+
const reader = response.data.getReader();
83+
const chunks: Uint8Array[] = [];
84+
85+
while (true) {
86+
const { done, value } = await reader.read();
87+
if (done) break;
88+
chunks.push(value);
89+
}
90+
91+
buffer = Buffer.concat(chunks);
92+
} else if (response.data instanceof ArrayBuffer) {
93+
buffer = Buffer.from(response.data);
94+
} else if (Buffer.isBuffer(response.data)) {
95+
buffer = response.data;
96+
} else {
97+
throw new Error(`Unexpected response data type: ${typeof response.data}`);
98+
}
99+
100+
logger?.info?.({
101+
operation: 'github_tarball_download_success',
102+
owner,
103+
repo,
104+
ref,
105+
size_bytes: buffer.length,
106+
attempt
107+
}, `Downloaded GitHub repository tarball (${(buffer.length / 1024).toFixed(2)} KB)`);
108+
109+
return buffer;
110+
111+
} catch (error) {
112+
const errorMessage = error instanceof Error ? error.message : String(error);
113+
114+
logger?.error?.({
115+
operation: 'github_tarball_download_failed',
116+
owner,
117+
repo,
118+
ref,
119+
attempt,
120+
max_retries: maxRetries,
121+
error: errorMessage
122+
}, `Failed to download GitHub repository tarball (attempt ${attempt}/${maxRetries})`);
123+
124+
if (attempt === maxRetries) {
125+
throw new Error(`Failed to download GitHub repository after ${maxRetries} attempts: ${errorMessage}`);
126+
}
127+
128+
// Exponential backoff: 1s, 2s, 4s
129+
const backoffMs = Math.pow(2, attempt - 1) * 1000;
130+
await new Promise(resolve => setTimeout(resolve, backoffMs));
131+
}
132+
}
133+
134+
throw new Error('Unreachable: maxRetries exhausted');
135+
}
136+
137+
/**
138+
* Extract tarball to deployment directory
139+
*
140+
* Handles GitHub tarball structure (with root directory) and supports both
141+
* tmpfs and regular filesystem deployments.
142+
*
143+
* @param tarballBuffer - Buffer containing the tarball data
144+
* @param tempDir - Deployment directory path
145+
* @param logger - Optional logger for debug/error messages
146+
* @param tmpfsManager - Optional tmpfs manager for tmpfs detection
147+
* @throws Error if extraction fails
148+
*/
149+
export async function extractTarball(
150+
tarballBuffer: Buffer,
151+
tempDir: string,
152+
logger?: Logger,
153+
tmpfsManager?: TmpfsManager
154+
): Promise<void> {
155+
const isTmpfs = tmpfsManager ? await tmpfsManager.isTmpfs(tempDir) : false;
156+
157+
logger?.debug?.({
158+
operation: 'tarball_extract_start',
159+
temp_dir: tempDir,
160+
tarball_size: tarballBuffer.length,
161+
is_tmpfs: isTmpfs
162+
}, `Extracting tarball to ${isTmpfs ? 'tmpfs' : 'filesystem'} directory`);
163+
164+
try {
165+
// Create temp directory
166+
await mkdir(tempDir, { recursive: true });
167+
168+
// Write tarball to temp file (tar.extract needs a file path)
169+
const tarballPath = path.join(tempDir, 'repo.tar.gz');
170+
await fs.promises.writeFile(tarballPath, tarballBuffer);
171+
172+
// Extract tarball (GitHub tarballs have a root directory, so strip it)
173+
await tar.extract({
174+
file: tarballPath,
175+
cwd: tempDir,
176+
strip: 1 // Remove the root directory from GitHub tarball
177+
});
178+
179+
// Remove the tarball file
180+
await fs.promises.unlink(tarballPath);
181+
182+
logger?.debug?.({
183+
operation: 'tarball_extract_success',
184+
temp_dir: tempDir
185+
}, 'Tarball extracted successfully');
186+
187+
} catch (error) {
188+
const errorMessage = error instanceof Error ? error.message : String(error);
189+
190+
logger?.error?.({
191+
operation: 'tarball_extract_failed',
192+
temp_dir: tempDir,
193+
error: errorMessage
194+
}, 'Failed to extract tarball');
195+
196+
throw new Error(`Failed to extract tarball: ${errorMessage}`);
197+
}
198+
}

0 commit comments

Comments
 (0)