diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index 60035dba0fac..59c92b4c179f 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -16,7 +16,7 @@ on: types: - crawlee-docker-image-bump schedule: - - cron: 0 */1 * * * + - cron: 0 */1 * * * env: ACTIONS_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} @@ -31,38 +31,38 @@ jobs: if: github.repository == 'apify/crawlee' steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - fetch-depth: 0 + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + fetch-depth: 0 - - name: Use Bun - uses: oven-sh/setup-bun@v2 + - name: Use Bun + uses: oven-sh/setup-bun@v2 - - name: Install Bun Dependencies - run: | - pushd ./scripts/actions/docker-images - bun install - popd + - name: Install Bun Dependencies + run: | + pushd ./scripts/actions/docker-images + bun install + popd - - name: Setup git user - run: | - git config --global user.name "Apify Release Bot" - git config --global user.email "noreply@apify.com" + - name: Setup git user + run: | + git config --global user.name "Apify Release Bot" + git config --global user.email "noreply@apify.com" - - name: Run Docker Action - run: | - pushd ./scripts/actions/docker-images - bun run-ci - popd - env: - ACTIONS_TOKEN: ${{ env.ACTIONS_TOKEN }} - CRAWLEE_BETA_VERSION: ${{ env.CRAWLEE_BETA_VERSION }} - TRIGGER_LATEST: ${{ env.TRIGGER_LATEST }} + - name: Run Docker Action + run: | + pushd ./scripts/actions/docker-images + bun run-ci + popd + env: + ACTIONS_TOKEN: ${{ env.ACTIONS_TOKEN }} + CRAWLEE_BETA_VERSION: ${{ env.CRAWLEE_BETA_VERSION }} + TRIGGER_LATEST: ${{ env.TRIGGER_LATEST }} - - name: Update Docker state - run: | - git add . - git diff-index --quiet HEAD || git commit -m "chore(docker): update docker state [skip ci]" - git push + - name: Update Docker state + run: | + git add . + git diff-index --quiet HEAD || git commit -m "chore(docker): update docker state [skip ci]" + git push diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cfb1871198ec..4ac8898b88ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,55 +32,55 @@ jobs: fail-fast: true matrix: # We don't test on Windows as the tests are flaky - os: [ubuntu-latest] - node-version: [16, 18, 20, 22] + os: [ ubuntu-latest ] + node-version: [ 16, 18, 20, 22 ] runs-on: ${{ matrix.os }} steps: - - name: Cancel Workflow Action - uses: styfle/cancel-workflow-action@0.12.1 - with: - access_token: ${{ github.token }} - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Activate cache for Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - cache: 'yarn' - - - name: Setup Playwright - uses: microsoft/playwright-github-action@v1 - - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- - - - name: Install Dependencies - run: yarn - - - name: Build - run: yarn ci:build - - - name: Tests - run: yarn test + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.12.1 + with: + access_token: ${{ github.token }} + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + cache: 'yarn' + + - name: Setup Playwright + uses: microsoft/playwright-github-action@v1 + + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- + + - name: Install Dependencies + run: yarn + + - name: Build + run: yarn ci:build + + - name: Tests + run: yarn test release: name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)" @@ -89,137 +89,137 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - fetch-depth: 0 - - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Activate cache for Node.js 20 - uses: actions/setup-node@v4 - with: - cache: 'yarn' - - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- - - - name: Install Dependencies - run: yarn - - - name: Build - run: yarn ci:build - - - name: Setup git user and npm - run: | - git config --global user.name "Apify Release Bot" - git config --global user.email "noreply@apify.com" - - echo "access=public" > .npmrc - echo "//registry.npmjs.org/:_authToken=${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}" >> .npmrc - - - name: Bump version to custom version - if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }} - run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes - env: - NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} - GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - - name: Bump version to ${{ github.event.inputs.version }} version - if: ${{ github.event.inputs.version != 'custom' }} - run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes - env: - NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} - GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - - name: Pin versions in internal dependencies and update lockfile - run: | - yarn release:pin-versions - yarn install --no-immutable - git add . - git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]' - git push - - - name: Publish packages - run: yarn publish:prod --yes --no-verify-access - env: - NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} - GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - - name: Trigger Docker image builds - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - repository: apify/crawlee - event-type: crawlee-docker-image-bump + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js 20 + uses: actions/setup-node@v4 + with: + cache: 'yarn' + + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- + + - name: Install Dependencies + run: yarn + + - name: Build + run: yarn ci:build + + - name: Setup git user and npm + run: | + git config --global user.name "Apify Release Bot" + git config --global user.email "noreply@apify.com" + + echo "access=public" > .npmrc + echo "//registry.npmjs.org/:_authToken=${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}" >> .npmrc + + - name: Bump version to custom version + if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }} + run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes + env: + NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} + GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + - name: Bump version to ${{ github.event.inputs.version }} version + if: ${{ github.event.inputs.version != 'custom' }} + run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes + env: + NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} + GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + - name: Pin versions in internal dependencies and update lockfile + run: | + yarn release:pin-versions + yarn install --no-immutable + git add . + git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]' + git push + + - name: Publish packages + run: yarn publish:prod --yes --no-verify-access + env: + NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} + GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + - name: Trigger Docker image builds + uses: peter-evans/repository-dispatch@v3 + with: + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + repository: apify/crawlee + event-type: crawlee-docker-image-bump version-docs: - needs: release - runs-on: ubuntu-latest - if: (github.event.inputs.version == 'minor' || github.event.inputs.version == 'major') - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - fetch-depth: 0 - - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Install jq - run: sudo apt-get install jq - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Activate cache for Node.js 20 - uses: actions/setup-node@v4 - with: - cache: 'yarn' - - - name: Install dependencies - run: | - # install project deps - yarn - # install website deps - cd website - yarn - - - name: Snapshot the current version - run: | - cd website - yarn docusaurus docs:version $(jq .version ../lerna.json | sed s/\"//g | head -c 3) - yarn docusaurus api:version $(jq .version ../lerna.json | sed s/\"//g | head -c 3) - - - name: Commit and push the version snapshot - run: | - git config --global user.name "Apify Release Bot" - git config --global user.email "noreply@apify.com" - git add . - git diff-index --quiet HEAD || git commit -m 'chore(release): update docs for ${{ inputs.version }} version [skip ci]' - git push + needs: release + runs-on: ubuntu-latest + if: (github.event.inputs.version == 'minor' || github.event.inputs.version == 'major') + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install jq + run: sudo apt-get install jq + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js 20 + uses: actions/setup-node@v4 + with: + cache: 'yarn' + + - name: Install dependencies + run: | + # install project deps + yarn + # install website deps + cd website + yarn + + - name: Snapshot the current version + run: | + cd website + yarn docusaurus docs:version $(jq .version ../lerna.json | sed s/\"//g | head -c 3) + yarn docusaurus api:version $(jq .version ../lerna.json | sed s/\"//g | head -c 3) + + - name: Commit and push the version snapshot + run: | + git config --global user.name "Apify Release Bot" + git config --global user.email "noreply@apify.com" + git add . + git diff-index --quiet HEAD || git commit -m 'chore(release): update docs for ${{ inputs.version }} version [skip ci]' + git push diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 9d1c20adeb1c..59094e047ca0 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -2,9 +2,9 @@ name: Check on: push: - branches: [master, renovate/**] + branches: [ master, renovate/** ] pull_request: - branches: [master] + branches: [ master ] env: YARN_IGNORE_NODE: 1 @@ -22,136 +22,136 @@ jobs: matrix: # tests on windows are extremely unstable # os: [ ubuntu-latest, windows-2019 ] - os: [ubuntu-latest] - node-version: [16, 18, 20, 22] + os: [ ubuntu-latest ] + node-version: [ 16, 18, 20, 22 ] steps: - - name: Cancel Workflow Action - uses: styfle/cancel-workflow-action@0.12.1 - with: - access_token: ${{ github.token }} + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.12.1 + with: + access_token: ${{ github.token }} - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate - - name: Activate cache for Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - cache: 'yarn' + - name: Activate cache for Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + cache: 'yarn' - - uses: microsoft/playwright-github-action@v1 + - uses: microsoft/playwright-github-action@v1 - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- - - name: Install Dependencies - run: yarn + - name: Install Dependencies + run: yarn - - name: Build - run: yarn ci:build + - name: Build + run: yarn ci:build - - name: Tests - run: yarn test + - name: Tests + run: yarn test docs: name: Docs build if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master') runs-on: ubuntu-latest steps: - - name: Checkout Source code - uses: actions/checkout@v4 - - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Activate cache for Node.js 20 - uses: actions/setup-node@v4 - with: - cache: 'yarn' - - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- - - - name: Install Dependencies - run: yarn - - - name: Build & deploy docs - run: | - cd website - yarn - yarn build - env: - APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} + - name: Checkout Source code + uses: actions/checkout@v4 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js 20 + uses: actions/setup-node@v4 + with: + cache: 'yarn' + + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- + + - name: Install Dependencies + run: yarn + + - name: Build & deploy docs + run: | + cd website + yarn + yarn build + env: + APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} lint: name: Lint runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Activate cache for Node.js 20 - uses: actions/setup-node@v4 - with: - cache: 'yarn' - - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- - - - name: Install Dependencies - run: yarn - - - name: ESLint - run: yarn lint - - - name: Biome format - run: yarn format:check + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js 20 + uses: actions/setup-node@v4 + with: + cache: 'yarn' + + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- + + - name: Install Dependencies + run: yarn + + - name: ESLint + run: yarn lint + + - name: Biome format + run: yarn format:check release_next: name: Release @next @@ -160,79 +160,79 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: 20 - - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate - - - name: Activate cache for Node.js 20 - uses: actions/setup-node@v4 - with: - cache: 'yarn' - - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- - - - name: Install Dependencies - run: yarn - - - name: Build - run: yarn ci:build - - - name: Generate changed packages list - id: changed-packages - run: | - echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" >> $GITHUB_OUTPUT - - - name: Report nothing to release - if: steps.changed-packages.outputs.changed_packages == '0' - run: echo "Nothing to release" - - - name: Release @next - if: steps.changed-packages.outputs.changed_packages != '0' - run: | - git config --global user.name 'Apify Release Bot' - git config --global user.email 'noreply@apify.com' - yarn turbo copy --force -- --canary --preid=beta - git commit -am "chore: bump canary versions [skip ci]" - - echo "access=public" > .npmrc - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc - yarn publish:next --yes --no-verify-access - env: - NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} - GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' - GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - - - name: Collect versions for Docker images - id: versions - run: | - crawlee=`node -p "require('./packages/crawlee/package.json').version"` - echo "crawlee=$crawlee" >> $GITHUB_OUTPUT - - - name: Trigger Docker image builds - uses: peter-evans/repository-dispatch@v3 - # Trigger next images only if we have something new pushed - if: steps.changed-packages.outputs.changed_packages != '0' - with: - token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} - repository: apify/crawlee - event-type: crawlee-docker-image-bump - client-payload: > - { - "crawlee_version": "${{ steps.versions.outputs.crawlee }}" - } + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate + + - name: Activate cache for Node.js 20 + uses: actions/setup-node@v4 + with: + cache: 'yarn' + + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- + + - name: Install Dependencies + run: yarn + + - name: Build + run: yarn ci:build + + - name: Generate changed packages list + id: changed-packages + run: | + echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" >> $GITHUB_OUTPUT + + - name: Report nothing to release + if: steps.changed-packages.outputs.changed_packages == '0' + run: echo "Nothing to release" + + - name: Release @next + if: steps.changed-packages.outputs.changed_packages != '0' + run: | + git config --global user.name 'Apify Release Bot' + git config --global user.email 'noreply@apify.com' + yarn turbo copy --force -- --canary --preid=beta + git commit -am "chore: bump canary versions [skip ci]" + + echo "access=public" > .npmrc + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc + yarn publish:next --yes --no-verify-access + env: + NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }} + GIT_USER: 'noreply@apify.com:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}' + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + - name: Collect versions for Docker images + id: versions + run: | + crawlee=`node -p "require('./packages/crawlee/package.json').version"` + echo "crawlee=$crawlee" >> $GITHUB_OUTPUT + + - name: Trigger Docker image builds + uses: peter-evans/repository-dispatch@v3 + # Trigger next images only if we have something new pushed + if: steps.changed-packages.outputs.changed_packages != '0' + with: + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + repository: apify/crawlee + event-type: crawlee-docker-image-bump + client-payload: > + { + "crawlee_version": "${{ steps.versions.outputs.crawlee }}" + } diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 59ae8169badf..8d8b241dd256 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: schedule: # Runs at 2 am every day - - cron: '0 2 * * *' + - cron: '0 2 * * *' env: YARN_IGNORE_NODE: 1 @@ -18,53 +18,53 @@ jobs: strategy: fail-fast: false matrix: - storage: [LOCAL, MEMORY, PLATFORM] + storage: [ LOCAL, MEMORY, PLATFORM ] steps: - - name: Cancel Workflow Action - uses: styfle/cancel-workflow-action@0.12.1 - with: - access_token: ${{ github.token }} + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.12.1 + with: + access_token: ${{ github.token }} - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Use Node.js 20 - uses: actions/setup-node@v4 - with: - node-version: 20 + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 - - name: Enable corepack - run: | - corepack enable - corepack prepare yarn@stable --activate + - name: Enable corepack + run: | + corepack enable + corepack prepare yarn@stable --activate - - name: Activate cache for Node.js 20 - uses: actions/setup-node@v4 - with: - cache: 'yarn' + - name: Activate cache for Node.js 20 + uses: actions/setup-node@v4 + with: + cache: 'yarn' - - uses: microsoft/playwright-github-action@v1 + - uses: microsoft/playwright-github-action@v1 - - name: Turbo cache - id: turbo-cache - uses: actions/cache@v4 - with: - path: .turbo - key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} - restore-keys: | - turbo-${{ github.job }}-${{ github.ref_name }}- + - name: Turbo cache + id: turbo-cache + uses: actions/cache@v4 + with: + path: .turbo + key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + turbo-${{ github.job }}-${{ github.ref_name }}- - - name: Login to Apify - run: npx -y apify-cli@beta login -t ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }} + - name: Login to Apify + run: npx -y apify-cli@beta login -t ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }} - - name: Install Dependencies - run: yarn + - name: Install Dependencies + run: yarn - - name: Build - run: yarn ci:build + - name: Build + run: yarn ci:build - - name: Test with storage ${{ matrix.storage }} - run: yarn test:e2e - env: - STORAGE_IMPLEMENTATION: ${{ matrix.storage }} + - name: Test with storage ${{ matrix.storage }} + run: yarn test:e2e + env: + STORAGE_IMPLEMENTATION: ${{ matrix.storage }} diff --git a/.github/workflows/update_new_issue.yml b/.github/workflows/update_new_issue.yml new file mode 100644 index 000000000000..f5bd18a1d8e5 --- /dev/null +++ b/.github/workflows/update_new_issue.yml @@ -0,0 +1,25 @@ +name: Update new issue + +on: + issues: + types: + - opened + +jobs: + label_issues: + name: Label issues + runs-on: ubuntu-latest + permissions: + issues: write + + steps: + # Add the "t-tooling" label to all new issues + - uses: actions/github-script@v7 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ["t-tooling"] + }) diff --git a/package.json b/package.json index 2cc771b467ac..b1ec740c6ec5 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "@types/lodash.isequal": "^4.5.8", "@types/lodash.merge": "^4.6.7", "@types/mime-types": "^2.1.1", - "@types/node": "^20.0.0", + "@types/node": "^22.5.1", "@types/proper-lockfile": "^4.1.2", "@types/ps-tree": "^1.1.2", "@types/rimraf": "^4.0.0", @@ -109,14 +109,14 @@ "puppeteer": "22.12.0", "rimraf": "^6.0.0", "tsx": "^4.4.0", - "turbo": "1.13.3", - "typescript": "^5.4.3", + "turbo": "^2.1.0", + "typescript": "^5.5.4", "vite-tsconfig-paths": "^4.3.2", "vitest": "^2.0.0" }, - "packageManager": "yarn@4.3.1", + "packageManager": "yarn@4.4.1", "volta": { "node": "20.15.0", - "yarn": "4.3.1" + "yarn": "4.4.1" } } diff --git a/turbo.json b/turbo.json index 43a29aeb0068..90a5a974869f 100644 --- a/turbo.json +++ b/turbo.json @@ -1,5 +1,5 @@ { - "pipeline": { + "tasks": { "build": { "dependsOn": ["^build"], "outputs": ["dist/**"] diff --git a/website/package.json b/website/package.json index 2a193d6cf3e3..bbfc93b40867 100644 --- a/website/package.json +++ b/website/package.json @@ -56,5 +56,5 @@ "stream-browserify": "^3.0.0", "unist-util-visit": "^5.0.0" }, - "packageManager": "yarn@4.3.1" + "packageManager": "yarn@4.4.1" } diff --git a/yarn.lock b/yarn.lock index 38e8a2eede6d..71227f3e105e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -755,7 +755,7 @@ __metadata: "@types/lodash.isequal": "npm:^4.5.8" "@types/lodash.merge": "npm:^4.6.7" "@types/mime-types": "npm:^2.1.1" - "@types/node": "npm:^20.0.0" + "@types/node": "npm:^22.5.1" "@types/proper-lockfile": "npm:^4.1.2" "@types/ps-tree": "npm:^1.1.2" "@types/rimraf": "npm:^4.0.0" @@ -791,8 +791,8 @@ __metadata: puppeteer: "npm:22.12.0" rimraf: "npm:^6.0.0" tsx: "npm:^4.4.0" - turbo: "npm:1.13.3" - typescript: "npm:^5.4.3" + turbo: "npm:^2.1.0" + typescript: "npm:^5.5.4" vite-tsconfig-paths: "npm:^4.3.2" vitest: "npm:^2.0.0" languageName: unknown @@ -2484,7 +2484,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*": +"@types/node@npm:*, @types/node@npm:^22.5.1": version: 22.5.1 resolution: "@types/node@npm:22.5.1" dependencies: @@ -2493,15 +2493,6 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^20.0.0": - version: 20.16.2 - resolution: "@types/node@npm:20.16.2" - dependencies: - undici-types: "npm:~6.19.2" - checksum: 10c0/74fac185dbd2c6b301cde75747c1cd6c9ca1ad7266147ea107a798837f40b8a05991177ebd7925fc7450d51334096259d186afad6e0d22864b7c97f8afef6ba7 - languageName: node - linkType: hard - "@types/normalize-package-data@npm:^2.4.0": version: 2.4.4 resolution: "@types/normalize-package-data@npm:2.4.4" @@ -12093,58 +12084,58 @@ __metadata: languageName: node linkType: hard -"turbo-darwin-64@npm:1.13.3": - version: 1.13.3 - resolution: "turbo-darwin-64@npm:1.13.3" +"turbo-darwin-64@npm:2.1.0": + version: 2.1.0 + resolution: "turbo-darwin-64@npm:2.1.0" conditions: os=darwin & cpu=x64 languageName: node linkType: hard -"turbo-darwin-arm64@npm:1.13.3": - version: 1.13.3 - resolution: "turbo-darwin-arm64@npm:1.13.3" +"turbo-darwin-arm64@npm:2.1.0": + version: 2.1.0 + resolution: "turbo-darwin-arm64@npm:2.1.0" conditions: os=darwin & cpu=arm64 languageName: node linkType: hard -"turbo-linux-64@npm:1.13.3": - version: 1.13.3 - resolution: "turbo-linux-64@npm:1.13.3" +"turbo-linux-64@npm:2.1.0": + version: 2.1.0 + resolution: "turbo-linux-64@npm:2.1.0" conditions: os=linux & cpu=x64 languageName: node linkType: hard -"turbo-linux-arm64@npm:1.13.3": - version: 1.13.3 - resolution: "turbo-linux-arm64@npm:1.13.3" +"turbo-linux-arm64@npm:2.1.0": + version: 2.1.0 + resolution: "turbo-linux-arm64@npm:2.1.0" conditions: os=linux & cpu=arm64 languageName: node linkType: hard -"turbo-windows-64@npm:1.13.3": - version: 1.13.3 - resolution: "turbo-windows-64@npm:1.13.3" +"turbo-windows-64@npm:2.1.0": + version: 2.1.0 + resolution: "turbo-windows-64@npm:2.1.0" conditions: os=win32 & cpu=x64 languageName: node linkType: hard -"turbo-windows-arm64@npm:1.13.3": - version: 1.13.3 - resolution: "turbo-windows-arm64@npm:1.13.3" +"turbo-windows-arm64@npm:2.1.0": + version: 2.1.0 + resolution: "turbo-windows-arm64@npm:2.1.0" conditions: os=win32 & cpu=arm64 languageName: node linkType: hard -"turbo@npm:1.13.3": - version: 1.13.3 - resolution: "turbo@npm:1.13.3" - dependencies: - turbo-darwin-64: "npm:1.13.3" - turbo-darwin-arm64: "npm:1.13.3" - turbo-linux-64: "npm:1.13.3" - turbo-linux-arm64: "npm:1.13.3" - turbo-windows-64: "npm:1.13.3" - turbo-windows-arm64: "npm:1.13.3" +"turbo@npm:^2.1.0": + version: 2.1.0 + resolution: "turbo@npm:2.1.0" + dependencies: + turbo-darwin-64: "npm:2.1.0" + turbo-darwin-arm64: "npm:2.1.0" + turbo-linux-64: "npm:2.1.0" + turbo-linux-arm64: "npm:2.1.0" + turbo-windows-64: "npm:2.1.0" + turbo-windows-arm64: "npm:2.1.0" dependenciesMeta: turbo-darwin-64: optional: true @@ -12160,7 +12151,7 @@ __metadata: optional: true bin: turbo: bin/turbo - checksum: 10c0/0382cc88f65a6690e97d30a6ad5d9b9ede7705f5f57edea27629408b166eff4a5938824746ce2cbcd50d2b64ebdbd359160e2cbe009f0bfd6f144997f9f6705b + checksum: 10c0/e57c58f07424f2f49d8d3a603a7338474f124299fbdfdaf034ca82fe2f08c686de5e050d33b175cf2b5a6b8a38117778a7e9bc6a78ec588ea3f708c3ab864acb languageName: node linkType: hard @@ -12298,7 +12289,7 @@ __metadata: languageName: node linkType: hard -"typescript@npm:>=3 < 6, typescript@npm:^5.4.3": +"typescript@npm:>=3 < 6, typescript@npm:^5.5.4": version: 5.5.4 resolution: "typescript@npm:5.5.4" bin: @@ -12308,7 +12299,7 @@ __metadata: languageName: node linkType: hard -"typescript@patch:typescript@npm%3A>=3 < 6#optional!builtin, typescript@patch:typescript@npm%3A^5.4.3#optional!builtin": +"typescript@patch:typescript@npm%3A>=3 < 6#optional!builtin, typescript@patch:typescript@npm%3A^5.5.4#optional!builtin": version: 5.5.4 resolution: "typescript@patch:typescript@npm%3A5.5.4#optional!builtin::version=5.5.4&hash=379a07" bin: