Update build scripts

.dockerignore

@@ -5,3 +5,5 @@ target/
 query-node/generated
 query-node/**/dist
 query-node/lib
+cli/
+tests/

.github/workflows/content-directory-schemas.yml

@@ -17,4 +17,7 @@ jobs:
     - name: validate
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/cd-schemas checks --quiet

.github/workflows/joystream-cli.yml

@@ -17,6 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace @joystream/cli checks --quiet
     - name: yarn pack test
       run: |
@@ -39,6 +42,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build        
         yarn workspace @joystream/cli checks --quiet
     - name: yarn pack test
       run: |

.github/workflows/joystream-types.yml

@@ -17,6 +17,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace @joystream/types checks --quiet
     - name: npm pack test
       run: |
@@ -40,6 +41,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace @joystream/types checks --quiet
     - name: npm pack test
       run: |

.github/workflows/network-tests.yml

@@ -17,6 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace network-tests checks --quiet
 
   network_build_osx:
@@ -34,4 +37,7 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
+        yarn workspace @joystream/cd-schemas generate:all
+        yarn workspace @joystream/cd-schemas build
         yarn workspace network-tests checks --quiet

.github/workflows/pioneer.yml

@@ -17,6 +17,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace pioneer build
 
   pioneer_build_osx:
@@ -34,6 +35,7 @@ jobs:
     - name: build
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace pioneer build
 
   pioneer_lint_ubuntu:
@@ -51,6 +53,7 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace pioneer lint --quiet
 
   pioneer_lint_osx:
@@ -68,4 +71,5 @@ jobs:
     - name: lint
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace pioneer lint --quiet

.github/workflows/run-network-tests.yml

@@ -97,7 +97,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn build:packages
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -121,7 +123,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn build:packages
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       - name: Execute network tests
@@ -145,7 +149,11 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/cd-schemas generate:all
+          yarn workspace @joystream/cd-schemas build
       - name: Ensure tests are runnable
         run: yarn workspace @joystream/cd-schemas checks --quiet
       - name: Start chain
@@ -171,9 +179,12 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
-      - name: Ensure query-node builds
-        run: yarn workspace query-node-root build
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
+          yarn workspace @joystream/cd-schemas generate:all
+          yarn workspace @joystream/cd-schemas build
+          yarn workspace query-node-root build
       - name: Ensure tests are runnable
         run: yarn workspace network-tests build
       # Bring up hydra query-node development instance, then run content directory
@@ -199,7 +210,9 @@ jobs:
           docker load --input joystream-node-docker-image.tar.gz
           docker images
       - name: Install packages and dependencies
-        run: yarn install --frozen-lockfile
+        run: |
+          yarn install --frozen-lockfile
+          yarn workspace @joystream/types build
       - name: Build storage node
         run: yarn workspace storage-node build
       - name: Start Services

.github/workflows/storage-node.yml

@@ -17,7 +17,9 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile
+        yarn workspace @joystream/types build
         yarn workspace storage-node checks --quiet
+        yarn workspace storage-node build
 
   storage_node_build_osx:
     name: MacOS Checks
@@ -34,4 +36,6 @@ jobs:
     - name: checks
       run: |
         yarn install --frozen-lockfile --network-timeout 120000
+        yarn workspace @joystream/types build
         yarn workspace storage-node checks --quiet
+        yarn workspace storage-node build

README.md

@@ -25,12 +25,11 @@ After cloning the repo run the following initialization scripts:
 # Install rust toolchain
 ./setup.sh
 
-# Install npm package dependencies
-# Also good habit to run this when switching between branches
-yarn install
+# Install npm package dependencies, build packages and docker images
+yarn build
 
-# run some tests
-yarn cargo-checks
+# start a local development network
+yarn start
 ```
 
 ## Software
@@ -75,6 +74,7 @@ The HEAD of the master branch should always be used for the correct version of t
 ```sh
 git checkout master
 yarn install
+yarn build:packages
 yarn workspace pioneer start
 ```
 
@@ -89,7 +89,7 @@ You can also run your our own joystream-node:
 
 ```sh
 git checkout master
-WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo build --release
 ./target/release/joystream-node -- --pruning archive --chain testnets/joy-testnet-4.json
 ```
 

apps.Dockerfile

@@ -2,13 +2,31 @@ FROM node:12 as builder
 
 WORKDIR /joystream
 COPY . /joystream
+RUN  rm -fr /joystream/pioneer
 
 # Do not set NODE_ENV=production until after running yarn install
 # to ensure dev dependencies are installed.
-RUN yarn install --frozen-lockfile
+RUN yarn --forzen-lockfile
 
-RUN yarn workspace pioneer build
-RUN yarn workspace storage-node build
+RUN yarn workspace @joystream/types build
 RUN yarn workspace query-node-root build
+RUN yarn workspace storage-node build
+
+# Second stage to reduce image size, enable it when
+# all packages have correctly identified what is a devDependency and what is not.
+# It will reduce the image size by about 500MB (down from 2.2GB to 1.7GB)
+
+# # Remove files that are not needed after build.
+# # We will re-fetch only dependencies needed for running the apps.
+# RUN rm -fr node_modules/
+# RUN rm -fr .git/
+
+# FROM node:12
+# WORKDIR /joystream
+# COPY --from=builder /joystream/ /joystream/
+
+# # Skip installing devDependencies, since we have already built the packages.
+# ENV NODE_ENV=production
+# RUN yarn install --forzen-lockfile --production
 
 ENTRYPOINT [ "yarn" ]

build-docker-images.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e
+
+if ! command -v docker-compose &> /dev/null
+then
+  echo "docker-compose not found. Skipping docker image builds."
+  exit 0
+fi
+
+# Build or fetch cached joystream/node docker image
+if [[ "$SKIP_JOYSTREAM_NODE" = 1 || "$SKIP_JOYSTREAM_NODE" = "true" ]]; then
+  echo "Skipping build of joystream/node docker image."
+else
+  # Fetch a cached joystream/node image if one is found matching code shasum instead of building
+  CODE_HASH=`scripts/runtime-code-shasum.sh`
+  IMAGE=joystream/node:${CODE_HASH}
+  echo "Trying to fetch cached ${IMAGE} image"
+  docker pull ${IMAGE} || :
+
+  if ! docker inspect ${IMAGE} > /dev/null;
+  then
+    echo "Fetch failed, building image locally"
+    docker-compose build joystream-node
+  else
+    echo "Tagging cached image as 'latest'"
+    docker image tag ${IMAGE} joystream/node:latest
+  fi
+fi
+
+# Build joystream/apps docker image
+echo "Building 'joystream/apps' docker image..."
+docker-compose build colossus
+
+# Build the pioneer docker image
+echo "Building pioneer docker image"
+docker-compose build pioneer

build-npm-packages.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+yarn
+yarn workspace @joystream/types build
+yarn workspace @joystream/cd-schemas generate:all
+yarn workspace @joystream/cd-schemas build
+yarn workspace query-node-root build
+yarn workspace @joystream/cli build
+yarn workspace storage-node build
+yarn workspace pioneer build

docker-compose.yml

@@ -154,10 +154,9 @@ services:
       - "127.0.0.1:6379:6379"
 
   pioneer:
-    image: joystream/apps
+    image: joystream/pioneer
     build:
       context: .
-      dockerfile: apps.Dockerfile
+      dockerfile: pioneer.Dockerfile
     ports:
-      - "127.0.0.1:3000:3000"
-    command: workspace pioneer start
+      - "127.0.0.1:3000:80"

node/README.md

@@ -26,7 +26,7 @@ cd joystream/
 Compile the node and runtime:
 
 ```bash
-WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo build --release
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo build --release
 ```
 
 This produces the binary in `./target/release/joystream-node`
@@ -79,7 +79,7 @@ If you are building a tagged release from `master` branch and want to install th
 This will install the executable `joystream-node` to your `~/.cargo/bin` folder, which you would normally have in your `$PATH` environment.
 
 ```bash
-WASM_BUILD_TOOLCHAIN=nightly-2020-05-23 cargo install joystream-node --path node/ --locked
+WASM_BUILD_TOOLCHAIN=nightly-2021-02-20 cargo install joystream-node --path node/ --locked
 ```
 
 Now you can run and connect to the testnet:

package.json

@@ -4,8 +4,10 @@
   "version": "1.0.0",
   "license": "GPL-3.0-only",
   "scripts": {
-    "postinstall": "yarn workspace @joystream/types build && yarn workspace @joystream/cd-schemas generate:all && yarn workspace @joystream/cd-schemas build && yarn workspace @joystream/cli build",
-    "build": "./build.sh",
+    "build": "./build-npm-packages.sh && ./build-docker-images.sh",
+    "build:packages": "./build-npm-packages.sh",
+    "build:docker": "./build-docker-images.sh",
+    "setup": "./setup.sh",
     "start": "./start.sh",
     "cargo-checks": "devops/git-hooks/pre-commit && devops/git-hooks/pre-push",
     "cargo-build": "scripts/cargo-build.sh"