feat(release): update c4b0cf0

.env.sample

@@ -2,3 +2,4 @@ NODE_ENV=
 PAYMENT_SERVICE_BASE_URL=
 ALLOW_LISTED_ADDRESSES=
 PRIVATE_ROUTE_SECRET=
+MIGRATE_ON_STARTUP=

.eslintrc

@@ -9,5 +9,12 @@
  "@typescript-eslint/no-unused-vars": ["error"],
  "header/header": [2, "./resources/license.header.js"]
  },
- "ignorePatterns": ["lib/*", ".vscode/*", "bundle-jobs.cjs", "scripts"]
+ "ignorePatterns": [
+ "lib/*",
+ ".vscode/*",
+ "bundle-jobs.cjs",
+ "scripts",
+ "package.json",
+ "testWallet.json"
+ ]
 }

.mocharc.js

@@ -2,7 +2,7 @@
 
 process.env.NODE_ENV ??= "test";
 process.env.PORT ??= 1234;
-process.env.ARWEAVE_GATEWAY ??= "http://localhost:1984";
+process.env.ARWEAVE_GATEWAY ??= "http://arlocal:1984";
 
 // Mocha configuration file
 // Reference for options: https://github.com/mochajs/mocha/blob/master/example/config/.mocharc.js

.nvmrc

@@ -1 +1 @@
-v18.16.0
+v18.17.0

.prettierignore

@@ -0,0 +1 @@
+CODEOWNERS

.vscode/settings.json

@@ -7,6 +7,7 @@
  "cSpell.words": [
  "alexsasharegan",
  "Aptos",
+ "ardriveapp",
  "arlocal",
  "avsc",
  "Backoff",
@@ -19,6 +20,8 @@
  "knexfile",
  "multistream",
  "nvmrc",
+ "oracledb",
+ "outdir",
  "Readables",
  "schemaname",
  "solana",
@@ -29,6 +32,9 @@
  "throughputs",
  "trivago",
  "typecheck",
+ "unbundle",
+ "unbundling",
+ "Winc",
  "winstons"
  ],
  "[terraform]": {

.yarnrc.yml

@@ -1,3 +1,7 @@
 nodeLinker: node-modules
 
+plugins:
+ - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
+ spec: "@yarnpkg/plugin-workspace-tools"
+
 yarnPath: .yarn/releases/yarn-3.2.1.cjs

CODEOWNERS

@@ -2,4 +2,4 @@
 
 # The @ardriveapp/services team is the code owner for the entire repository.
 
-- @ardriveapp/services
+* @ardriveapp/services

Dockerfile

@@ -1,4 +1,4 @@
-ARG NODE_VERSION=18.16.0
+ARG NODE_VERSION=18.17.0
 ARG NODE_VERSION_SHORT=18
 
 FROM node:${NODE_VERSION}-bullseye-slim AS builder
@@ -8,7 +8,10 @@ WORKDIR /usr/src/app
 COPY . .
 RUN yarn && yarn build
 
-FROM gcr.io/distroless/nodejs:${NODE_VERSION_SHORT}
+# Clean out dependencies
+RUN yarn workspaces focus --production
+
+FROM gcr.io/distroless/nodejs${NODE_VERSION_SHORT}-debian11
 WORKDIR /usr/src/app
 
 # Add shell

Dockerfile.fulfillment

@@ -0,0 +1,35 @@
+ARG NODE_VERSION=18.17.0
+ARG NODE_VERSION_SHORT=18
+
+FROM node:${NODE_VERSION}-bullseye-slim AS builder
+
+# Build
+WORKDIR /usr/src/app
+COPY . .
+RUN yarn && yarn build
+
+RUN cd ecs/fulfillment-pipeline/ && yarn install && yarn build
+
+# Clean out dependencies
+RUN yarn workspaces focus --production
+
+FROM gcr.io/distroless/nodejs${NODE_VERSION_SHORT}-debian11
+WORKDIR /usr/src/app
+
+# Add shell
+COPY --from=busybox:1.35.0-uclibc /bin/sh /bin/sh
+COPY --from=busybox:1.35.0-uclibc /bin/addgroup /bin/addgroup
+COPY --from=busybox:1.35.0-uclibc /bin/adduser /bin/adduser
+COPY --from=busybox:1.35.0-uclibc /bin/chown /bin/chown
+
+# Create user
+RUN addgroup -g 1000 node \
+ && adduser -u 1000 -G node -s /bin/sh -D node
+RUN chown -R node ./
+USER node
+
+# Copy build files
+COPY --from=builder --chown=node /usr/src/app/ecs/fulfillment-pipeline/dist ./dist
+
+EXPOSE 3000
+CMD ["./dist/app.js"]

Dockerfile.migration

@@ -1,4 +1,4 @@
-ARG NODE_VERSION=18.16.0
+ARG NODE_VERSION=18.17.0
 
 FROM node:${NODE_VERSION}-bullseye-slim
 

README.md

@@ -2,18 +2,18 @@
 
 Welcome to the Turbo Upload Service 👋
 
-## Local Development
+## Setting up the development environment
 
-### Requirements
+### System Package Installation
 
 For a compatible development environment, we require the following packages installed on the system:
 
+- `nvm`
 - `yarn`
-- `nvm` (optional)
-- `husky` (optional)
-- `docker` (optional)
+- `husky`
+- `docker`
 
-### Running locally
+### Running the Upload Service locally
 
 With a compatible system, follow these steps to start the upload service:
 
@@ -22,7 +22,8 @@ With a compatible system, follow these steps to start the upload service:
 - `yarn build`
 - `yarn db:up && yarn db:migrate:latest`
 - `yarn start`
- - alternatively use `yarn start:watch` to run the app in development mode with hot reloading provided by `nodemon`
+
+Developers can alternatively use `yarn start:watch` to run the app in development mode with hot reloading provided by `nodemon`
 
 ## Database
 
@@ -58,20 +59,28 @@ You can rollback knex migrations using the following command:
 
 Additional `knex` documentation can be found [here](https://knexjs.org/guide/migrations.html).
 
-### Docker
+## Docker
 
-To run this service and a connected postgres database, fully migrated.
+### Building Image
 
-Run the container:
+To build the container:
 
 ```shell
-yarn start:docker
+docker build --build-arg NODE_VERSION=$(cat .nvmrc |cut -c2-8) --build-arg NODE_VERSION_SHORT=$(cat .nvmrc |cut -c2-3) .
 ```
 
-To build the image:
+### Docker Compose
+
+Runs this service, against the most recent version of `payment-service` and `arlocal`, and local postgres instances.
 
 ```shell
-docker build --build-arg NODE_VERSION=$(cat .nvmrc |cut -c2-8) --build-arg NODE_VERSION_SHORT=$(cat .nvmrc |cut -c2-3) .
+docker compose up -d
+```
+
+Run just the upload service against migrated local postgres instance.
+
+```shell
+docker compose up upload-service --build
 ```
 
 ## Tests

docker-compose.yml

@@ -10,15 +10,17 @@ services:
  build:
  context: .
  args:
- NODE_VERSION: ${NODE_VERSION:-18.16.0}
+ NODE_VERSION: ${NODE_VERSION:-18.17.0}
  NODE_VERSION_SHORT: ${NODE_VERSION_SHORT:-18}
  environment:
  NODE_ENV: ${NODE_ENV:-test}
  DB_HOST: upload-service-pg
  DB_PORT: 5432
  DB_PASSWORD: postgres
  PAYMENT_SERVICE_BASE_URL: ${PAYMENT_SERVICE_BASE_URL:-payment.ardrive.dev}
+ MAX_DATA_ITEM_SIZE: ${MAX_DATA_ITEM_SIZE:-10737418240}
  ALLOW_LISTED_ADDRESSES: ${ALLOW_LISTED_ADDRESSES:-}
+ MIGRATE_ON_STARTUP: true
  ports:
  - "${PORT:-3000}:${PORT:-3000}"
  volumes:
@@ -40,9 +42,9 @@ services:
  test-runner:
  build:
  context: .
- dockerfile: tests/Dockerfile.integration
+ dockerfile: tests/integration.Dockerfile
  args:
- NODE_VERSION: ${NODE_VERSION:-18.16.0}
+ NODE_VERSION: ${NODE_VERSION:-18.17.0}
  environment:
  DB_HOST: upload-service-pg
  DB_PASSWORD: postgres

docs/openapi.yaml

@@ -16,19 +16,45 @@ components:
  id:
  type: string
  description: The id of the data item.
+ example: "QpmY8mZmFEC8RxNsgbxSV6e36OF6quIYaPRKzvUco0o"
  owner:
  type: string
- description: The wallet that signed data item.
+ description: The normalized wallet address that signed data item.
+ example: "8wgRDgvYOrtSaWEIV21g0lTuWDUnTu4_iYj4hmA7PI0"
  dataCaches:
  type: array
  description: The Arweave data caches the data item has been posted to.
  items:
  type: string
- example:
- id: 7j-sF0lsslGVZ8lhEGXe5CtueB4iRYM3_oZ9m4GY_40
- owner: xsi06LVwuRe2SaNFo0Yc1UtF3GSyi-GtzUtTkjLXrEw
- dataCaches:
- - arweave.net
+ example:
+ - arweave.net
+ fastFinalityIndexes:
+ type: array
+ description: The Arweave fast finality indexes the data item has been posted to.
+ items:
+ type: string
+ example:
+ - arweave.net
+ deadlineHeight:
+ type: integer
+ description: The deadline block height for the data item to be posted to Arweave.
+ example: 1310000
+ timestamp:
+ type: integer
+ description: The timestamp in millisecond of when the data item was fully uploaded and receipt was signed.
+ example: 1700590909589
+ version:
+ type: string
+ description: The version of the receipt
+ example: "0.1.0"
+ signature:
+ type: string
+ description: The base64URL signature of the receipt
+ example: "iU_S6uuG1OD8k0XqMGOmbcKfysDckEMUy4R9-ODPXiQjKXuT4lTngRFBFKO5NQT1iIfqSDKcbTRL6gJowM_L7bBQZRGkojzXD0PNNU2F0bNNJ80VtUktHifGTXbCgz5kiFciL19n0P3nX6ZfXnOn-H8ALZzRJV69apdvwqitpNKxLMPyc-QA0QBxmC3CKPz_7fy2Qg0QHr5g_ZT2Of-YJ_RsZTEoc3g1fgzsEmMBPOsx4XtPrhV6llnA3pncngzHbPdFvypdWiO8Bvr0EWmazNsoanuwK5uKJ_ROIGXW_dBBGN8Vrfv5U6dJnhJVn5IE7JFlixpFTluF_ICRzbUq2pk_re6jEtW1H3ItH2iN0UeFUw1uDbq3HJW6lDc8aOwDwDspJI11KEI6uCz5QmQy2V8DvRknoqcxmuihF6XmmJIZgTVeo6LNufEis9kFxqtc3Dh_gn8z0cDXKEKFycudckmcHP7vkWD68uSssMMJIdVgwvPZss06svfRnI-E33j3MrQI9FzMIv-7Df8iYATyeyldM1v3gexG0kQm0AMG1_8_SLqwu2QlqzM41mrK5vNmQxOVdIQSOPWPvzbF-YGRwpCjlveRBuARGC9JNC4UipvDYri2gRWuBx2uDL7dmVFv1gRll3dYNMYaMULHYngtrrCynB3Cyfhh7cyPlwuNlk0"
+ public:
+ type: string
+ description: The public key of the wallet that signed the receipt
+ example: "qREovbmD6oxgHYNCzOeTei07lSz0-YLcjnvgSDzqptsCiqNOtB3RKUToSX5hkPD45fJDY4057XkkcsQRuGsU8y9rgm37i-Kiyd5Z_iy6pJXrwi9XnAgGL118lIV790GZ7xe5o3DvPV3Px74C0ABsfL9lW86D4t_qClJ6wSQksKNd7rnUImIvHW0vxLswST7dfUngevzKt4kv48VTub4951XdUHjb45Uurf7xFYSCizAGtGqr5GYDFrVk-mNrzFH5bXt06PJfxe9E5ujIE5Uq1Az6vqEOO0E1mWmXqdTPluAxcjmgktkoNLHZnnU-BsYuFaTWW5NU3aS-RgJKXYs9O6Dc1-2SITl-H_wtdGNSj31fj72UkaAbkpA1mionK-8bOIkSpYgKCyTC42oHh1Fw4SLXPyLxBj1w6F32LSLjpse5dmKymj4fJPezCMdi709uIiVT7XOm4LZBCzFOxS6-UNxgE57dBFpcWkcqNO3p00biYqH5d5bzvK3bwd-4j0KyKlqPYTProSnd3P6ROZuuJaLoLD_Or5-L_dUjawHz-DFlzmckYaf8l3XdzPM4JSsE8CEDZl0NCN0AidXt_wjbr6k9JsO7cnLB226AjxhcyuxZOlhgkIn7EbxpVXx-O2mXkcXF8PixCP0k5brtriLeF1MAdspgd_S-LKQeGVr7-mk"
 
 paths:
  /tx:
@@ -47,8 +73,8 @@ paths:
  schema:
  type: integer
  format: int64
- maximum: 2147483648 # 2GB in bytes
- required: true
+ maximum: 10737418240 # 10GiB
+ required: false
  Content-Type:
  schema:
  type: string
@@ -72,6 +98,15 @@ paths:
  examples:
  DataItemExists:
  value: Data Item Exists
+
+ "402":
+ description: Insufficient balance
+ content:
+ text/plain:
+ schema:
+ type: string
+ default: "Insufficient balance"
+
  "400":
  description: Invalid data item post
  content:

ecs/fulfillment-pipeline/package.json

@@ -0,0 +1,25 @@
+{
+ "name": "@ardriveapp/fulfillment-pipeline",
+ "version": "1.0.0",
+ "description": "",
+ "main": "src/app.ts",
+ "type": "commonjs",
+ "scripts": {
+ "build": "rimraf ./dist; esbuild --bundle src/app.ts --outdir=dist --format=cjs --platform=node --external:better-sqlite3 --external:mysql2 --external:mysql --external:tedious --external:oracledb --external:mock-aws-s3 --external:nock --external:\"*.html\"",
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "author": "Permanent Data Solutions Inc.",
+ "license": "AGPL-3.0",
+ "dependencies": {
+ "@aws-sdk/client-sqs": "^3.427.0",
+ "@aws-sdk/node-http-handler": "^3.374.0",
+ "dotenv": "^16.0.1",
+ "sqs-consumer": "^7.3.0"
+ },
+ "devDependencies": {
+ "esbuild": "^0.19.4",
+ "rimraf": "^3.0.2",
+ "ts-node": "^10.9.1",
+ "typescript": "^4.7.4"
+ }
+}