Make it work with amazon s3 presigned urls

[TobiSell]

Is your feature request related to a problem? Please describe.
Uploading files bigger than 5 gb directly to S3 buckets using ngx-uploadx

Describe the solution you'd like
On projects with S3 Storage, you can create a presigned url in order to let the angular client upload their files directly to S3. This saves a lot of time and ressources, since the webserver itself does nothing but creating this presigned url.
(Details: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html#query-string-auth-v4-signing-example)

I would like to use Multipartuploads with ngx-uploadx in that scenario https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html

Describe alternatives you've considered
no alternatives yet

Additional context

[kukhariev]

possible implementation

import { Metadata, Uploader } from 'ngx-uploadx';
export interface S3SigObject {
  chunkSize: number;
  url: string;
  uploadId: string;
  urls: string[];
  parts: Part[];
  metadata: Metadata;
}
interface Part {
  ETag: string;
  PartNumber: number;
}
export class S3 extends Uploader {
  s3Sig = { metadata: this.metadata } as S3SigObject;

  protected async getFileUrl(): Promise<string> {
    this.offset = 0;
    await this._serverRequest(this.endpoint);
    this.chunkSize = this.s3Sig.chunkSize;
    return this.s3Sig.url;
  }

  protected async sendFileContent(): Promise<number> {
    const index = this.s3Sig.parts.length;
    const { body, end } = this.getChunk();
    await this.request({ method: 'PUT', body, url: this.s3Sig.urls[index] });
    const etag = this.getValueFromResponse('etag') || '';
    const part: Part = { ETag: etag, PartNumber: index + 1 };
    this.s3Sig.parts = [...this.s3Sig.parts, part];
    if (end === this.size) {
      await this._serverRequest(this.url);
      return end;
    }
    return end + 1;
  }

  protected async getOffset(): Promise<number | undefined> {
    await this._serverRequest(this.url);
    return Math.min(this.s3Sig.parts.length * this.chunkSize, this.size);
  }

  private async _serverRequest(url: string): Promise<S3SigObject> {
    const body: FormData = new FormData();
    body.set('s3Sig', JSON.stringify(this.s3Sig));
    await this.request({
      method: 'POST',
      body,
      url
    });
    return (this.response as object) as S3SigObject;
  }
}

But first we need a smart server API...

[reyx]

possible implementation

import { Metadata, Uploader } from 'ngx-uploadx';
export interface S3SigObject {
  chunkSize: number;
  url: string;
  uploadId: string;
  urls: string[];
  parts: Part[];
  metadata: Metadata;
}
interface Part {
  ETag: string;
  PartNumber: number;
}
export class S3 extends Uploader {
  s3Sig = { metadata: this.metadata } as S3SigObject;

  protected async getFileUrl(): Promise<string> {
    this.offset = 0;
    await this._serverRequest(this.endpoint);
    this.chunkSize = this.s3Sig.chunkSize;
    return this.s3Sig.url;
  }

  protected async sendFileContent(): Promise<number> {
    const index = this.s3Sig.parts.length;
    const { body, end } = this.getChunk();
    await this.request({ method: 'PUT', body, url: this.s3Sig.urls[index] });
    const etag = this.getValueFromResponse('etag') || '';
    const part: Part = { ETag: etag, PartNumber: index + 1 };
    this.s3Sig.parts = [...this.s3Sig.parts, part];
    if (end === this.size) {
      await this._serverRequest(this.url);
      return end;
    }
    return end + 1;
  }

  protected async getOffset(): Promise<number | undefined> {
    await this._serverRequest(this.url);
    return Math.min(this.s3Sig.parts.length * this.chunkSize, this.size);
  }

  private async _serverRequest(url: string): Promise<S3SigObject> {
    const body: FormData = new FormData();
    body.set('s3Sig', JSON.stringify(this.s3Sig));
    await this.request({
      method: 'POST',
      body,
      url
    });
    return (this.response as object) as S3SigObject;
  }
}

But first we need a smart server API...

Just remove the "+ 1" on "return end + 1"

[kukhariev]

updated version:

import { store, UploaderX } from 'ngx-uploadx';
export interface S3Multipart {
  partSize: number;
  name: string;
  UploadId: string;
  partsUrls: string[];
  Parts?: Part[];
}
interface Part {
  ETag: string;
  PartNumber: number;
}
export class UploaderXS3 extends UploaderX {
  s3 = {} as S3Multipart;

  async getFileUrl(): Promise<string> {
    const url = await super.getFileUrl();
    if (this.response?.partSize) {
      this.s3 = { ...this.response };
      this.s3.Parts ??= [];
      store.set(url, JSON.stringify(this.s3));
      this.offset = this.s3.Parts.length * this.s3.partSize;
      if (this.s3?.partsUrls.length === this.s3?.Parts.length) {
        await this.setMetadata(this.url);
      }
    }
    return url;
  }

  async sendFileContent(): Promise<number | undefined> {
    if (this.s3.partsUrls) {
      this.s3.Parts ??= [];
      const i = this.s3.Parts.length;
      const { body, end } = this.getChunk(this.offset, this.s3.partSize);
      await this.request({
        method: 'PUT',
        body,
        url: this.s3.partsUrls[i],
        skipAuthorization: true
      });

      const ETag = this.getValueFromResponse('etag');
      if (!ETag) {
        return this.offset;
      }
      const part: Part = { ETag, PartNumber: i + 1 };
      this.s3.Parts.push(part);
      if (end === this.size) {
        await this.setMetadata(this.url);
      }
      return end;
    } else {
      return super.sendFileContent();
    }
  }

  async getOffset(): Promise<number | undefined> {
    const _s3 = store.get(this.url);
    if (_s3) {
      this.s3 = JSON.parse(_s3);
      await this.setMetadata(this.url);
      if (this.response.UploadId) {
        this.s3 = { ...this.response };
      }
      this.s3.Parts ??= [];
      return Math.min(this.s3.Parts.length * this.s3.partSize, this.size);
    }
    return super.getOffset();
  }

  private async setMetadata(url: string): Promise<S3Multipart> {
    const body = JSON.stringify(this.s3);
    await this.request({
      method: 'PATCH',
      headers: { 'Content-Type': 'application/json; charset=utf-8' },
      body,
      url
    });
    if (this.response?.partsUrls) {
      this.s3 = { ...this.response };
    }
    return this.s3;
  }
}

[kukhariev]

For proper operation, at least "exposeHeaders" : ["etag"] and "AllowedMethods": ["PUT"] must be set in the cors configuration.

b2 example:

[
  {
    "corsRuleName": "ngx-uploadx",
    "allowedOrigins": [
      "*"
    ],
    "allowedHeaders": [
      "*"
    ],
    "allowedOperations": [
      "s3_put"
    ],
    "exposeHeaders": ["etag"],
    "maxAgeSeconds": 3600
  }
]