Skip to content

Commit

Permalink
Merge pull request #2555 from iclanton/ianc/s3-rest
Browse files Browse the repository at this point in the history 
[rush] Replace the AWS SDK packages with the S3 REST API.
  • Loading branch information
@iclanton
iclanton authored Mar 17, 2021
2 parents 68c1ef9 + 16d1c39 commit 24185b8
Show file tree
Hide file tree
Showing 17 changed files with 431 additions and 899 deletions.
1 change: 0 additions & 1 deletion apps/rush-lib/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
},
"license": "MIT",
"dependencies": {
"@aws-sdk/client-s3": "~3.3.0",
"@azure/identity": "~1.2.0",
"@azure/storage-blob": "~12.3.0",
"@pnpm/link-bins": "~5.3.7",
Expand Down
6 changes: 3 additions & 3 deletions apps/rush-lib/src/api/BuildCacheConfiguration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ import type {
AzureEnvironmentNames,
AzureStorageBuildCacheProvider
} from '../logic/buildCache/AzureStorageBuildCacheProvider';
const AmazonS3BuildCacheProviderModule: typeof import('../logic/buildCache/AmazonS3BuildCacheProvider') = Import.lazy(
'../logic/buildCache/AmazonS3BuildCacheProvider',
const AmazonS3BuildCacheProviderModule: typeof import('../logic/buildCache/AmazonS3/AmazonS3BuildCacheProvider') = Import.lazy(
'../logic/buildCache/AmazonS3/AmazonS3BuildCacheProvider',
require
);
import type { AmazonS3BuildCacheProvider } from '../logic/buildCache/AmazonS3BuildCacheProvider';
import type { AmazonS3BuildCacheProvider } from '../logic/buildCache/AmazonS3/AmazonS3BuildCacheProvider';

/**
* Describes the file structure for the "common/config/rush/build-cache.json" config file.
Expand Down
4 changes: 2 additions & 2 deletions apps/rush-lib/src/logic/base/BaseInstallManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,7 @@ export abstract class BaseInstallManager {
webClient.userAgent = `pnpm/? npm/? node/${process.version} ${os.platform()} ${os.arch()}`;
webClient.accept = 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*';

const response: WebClientResponse = await webClient.fetch(queryUrl);
const response: WebClientResponse = await webClient.fetchAsync(queryUrl);
if (!response.ok) {
throw new Error('Failed to query');
}
Expand All @@ -634,7 +634,7 @@ export abstract class BaseInstallManager {
// Make sure the tarball wasn't deleted from the CDN
webClient.accept = '*/*';

const response2: fetch.Response = await webClient.fetch(url);
const response2: fetch.Response = await webClient.fetchAsync(url);

if (!response2.ok) {
if (response2.status === 404) {
Expand Down
97 changes: 20 additions & 77 deletions .../buildCache/AmazonS3BuildCacheProvider.ts → ...he/AmazonS3/AmazonS3BuildCacheProvider.ts
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import { Readable } from 'stream';
import { Terminal } from '@rushstack/node-core-library';
import { S3Client, GetObjectCommand, PutObjectCommand, GetObjectCommandOutput } from '@aws-sdk/client-s3';
// import { defaultProvider as awsCredentialsProvider } from '@aws-sdk/credential-provider-node';

import { EnvironmentConfiguration, EnvironmentVariableNames } from '../../api/EnvironmentConfiguration';
import { CloudBuildCacheProviderBase } from './CloudBuildCacheProviderBase';
import { CredentialCache, ICredentialCacheEntry } from '../CredentialCache';
import { RushConstants } from '../RushConstants';
import { Utilities } from '../../utilities/Utilities';

interface IAmazonS3Credentials {
accessKeyId: string;
secretAccessKey: string;
}

import { EnvironmentConfiguration, EnvironmentVariableNames } from '../../../api/EnvironmentConfiguration';
import { CloudBuildCacheProviderBase } from '../CloudBuildCacheProviderBase';
import { CredentialCache, ICredentialCacheEntry } from '../../CredentialCache';
import { RushConstants } from '../../RushConstants';
import { AmazonS3Client, IAmazonS3Credentials } from './AmazonS3Client';

export interface IAmazonS3BuildCacheProviderOptions {
s3Bucket: string;
Expand All @@ -25,8 +17,7 @@ export interface IAmazonS3BuildCacheProviderOptions {
}

export class AmazonS3BuildCacheProvider extends CloudBuildCacheProviderBase {
private readonly _s3Bucket: string;
private readonly _s3Region: string;
private readonly _options: IAmazonS3BuildCacheProviderOptions;
private readonly _s3Prefix: string | undefined;
private readonly _environmentWriteCredential: string | undefined;
private readonly _isCacheWriteAllowedByConfiguration: boolean;
Expand All @@ -36,36 +27,19 @@ export class AmazonS3BuildCacheProvider extends CloudBuildCacheProviderBase {
return this._isCacheWriteAllowedByConfiguration || !!this._environmentWriteCredential;
}

private __s3Client: S3Client | undefined;
private __s3Client: AmazonS3Client | undefined;

public constructor(options: IAmazonS3BuildCacheProviderOptions) {
super();
this._s3Bucket = options.s3Bucket;
this._s3Region = options.s3Region;
this._options = options;
this._s3Prefix = options.s3Prefix;
this._environmentWriteCredential = EnvironmentConfiguration.buildCacheWriteCredential;
this._isCacheWriteAllowedByConfiguration = options.isCacheWriteAllowed;
}

private _deserializeCredentials(credentialString: string | undefined): IAmazonS3Credentials | undefined {
if (!credentialString) {
return undefined;
}

const splitIndex: number = credentialString.indexOf(':');
if (splitIndex === -1) {
throw new Error('Amazon S3 credential is in an unexpected format.');
}

return {
accessKeyId: credentialString.substring(0, splitIndex),
secretAccessKey: credentialString.substring(splitIndex + 1)
};
}

private get _credentialCacheId(): string {
if (!this.__credentialCacheId) {
const cacheIdParts: string[] = ['aws-s3', this._s3Region, this._s3Bucket];
const cacheIdParts: string[] = ['aws-s3', this._options.s3Region, this._options.s3Bucket];

if (this._isCacheWriteAllowedByConfiguration) {
cacheIdParts.push('cacheWriteAllowed');
Expand All @@ -77,9 +51,9 @@ export class AmazonS3BuildCacheProvider extends CloudBuildCacheProviderBase {
return this.__credentialCacheId;
}

private async _getS3ClientAsync(): Promise<S3Client> {
private async _getS3ClientAsync(): Promise<AmazonS3Client> {
if (!this.__s3Client) {
let credentials: IAmazonS3Credentials | undefined = this._deserializeCredentials(
let credentials: IAmazonS3Credentials | undefined = AmazonS3Client.tryDeserializeCredentials(
this._environmentWriteCredential
);
if (!credentials) {
Expand All @@ -101,29 +75,19 @@ export class AmazonS3BuildCacheProvider extends CloudBuildCacheProviderBase {
`Update the credentials by running "rush ${RushConstants.updateCloudCredentialsCommandName}".`
);
} else {
credentials = this._deserializeCredentials(cacheEntry?.credential);
credentials = AmazonS3Client.tryDeserializeCredentials(cacheEntry?.credential);
}
} else {
// This logic was temporarily disabled to eliminate the dependency on @aws-sdk/credential-provider-node
// which caused this issue:
//
// "[rush] Broken peer dependency error when installing @microsoft/rush-lib"
// https://github.com/microsoft/rushstack/issues/2547

// try {
// credentials = await awsCredentialsProvider()();
// } catch {
} else if (this._isCacheWriteAllowedByConfiguration) {
throw new Error(
"An Amazon S3 credential hasn't been provided, or has expired. " +
`Update the credentials by running "rush ${RushConstants.updateCloudCredentialsCommandName}", ` +
`or provide an <AccessKeyId>:<SecretAccessKey> pair in the ` +
`${EnvironmentVariableNames.RUSH_BUILD_CACHE_WRITE_CREDENTIAL} environment variable`
);
// }
}
}

this.__s3Client = new S3Client({ region: this._s3Region, credentials });
this.__s3Client = new AmazonS3Client(credentials, this._options);
}

return this.__s3Client;
Expand All @@ -134,24 +98,9 @@ export class AmazonS3BuildCacheProvider extends CloudBuildCacheProviderBase {
cacheId: string
): Promise<Buffer | undefined> {
try {
const client: S3Client = await this._getS3ClientAsync();
const fetchResult: GetObjectCommandOutput | undefined = await client.send(
new GetObjectCommand({
Bucket: this._s3Bucket,
Key: this._s3Prefix ? `${this._s3Prefix}/${cacheId}` : cacheId
})
);
if (fetchResult === undefined) {
return undefined;
}

return await Utilities.readStreamToBufferAsync(fetchResult.Body as Readable);
const client: AmazonS3Client = await this._getS3ClientAsync();
return await client.getObjectAsync(this._s3Prefix ? `${this._s3Prefix}/${cacheId}` : cacheId);
} catch (e) {
if (e.name === 'NoSuchKey') {
// No object was uploaded with that name/key
return undefined;
}

terminal.writeWarningLine(`Error getting cache entry from S3: ${e}`);
return undefined;
}
Expand All @@ -160,22 +109,16 @@ export class AmazonS3BuildCacheProvider extends CloudBuildCacheProviderBase {
public async trySetCacheEntryBufferAsync(
terminal: Terminal,
cacheId: string,
entryStream: Buffer
objectBuffer: Buffer
): Promise<boolean> {
if (!this.isCacheWriteAllowed) {
terminal.writeErrorLine('Writing to S3 cache is not allowed in the current configuration.');
return false;
}

try {
const client: S3Client = await this._getS3ClientAsync();
await client.send(
new PutObjectCommand({
Bucket: this._s3Bucket,
Key: this._s3Prefix ? `${this._s3Prefix}/${cacheId}` : cacheId,
Body: entryStream
})
);
const client: AmazonS3Client = await this._getS3ClientAsync();
await client.uploadObjectAsync(this._s3Prefix ? `${this._s3Prefix}/${cacheId}` : cacheId, objectBuffer);
return true;
} catch (e) {
terminal.writeWarningLine(`Error uploading cache entry to S3: ${e}`);
Expand Down
Loading

0 comments on commit 24185b8

Please sign in to comment.