Implement S3 leasing #220
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| env: | |
| GO_VERSION: "1.21" | |
| name: Commit | |
| jobs: | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: | | |
| go install golang.org/x/tools/cmd/goimports@latest | |
| go install honnef.co/go/tools/cmd/staticcheck@latest | |
| export PATH="$HOME/go/bin:$PATH" | |
| - uses: pre-commit/action@v3.0.0 | |
| build-windows: | |
| name: Build Windows | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: sudo apt-get install -y mingw-w64 | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: | | |
| go build ./cmd/litestream/ | |
| file ./litestream.exe | |
| env: | |
| CGO_ENABLED: "1" | |
| GOOS: windows | |
| GOARCH: amd64 | |
| CC: x86_64-w64-mingw32-gcc | |
| build: | |
| name: Build & Unit Test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: go env | |
| - run: go install ./cmd/litestream | |
| - run: go test -v ./... | |
| # long-running-test: | |
| # name: Run Long Running Unit Test | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - uses: actions/setup-go@v2 | |
| # with: | |
| # go-version: '1.20' | |
| # - uses: actions/cache@v2 | |
| # with: | |
| # path: ~/go/pkg/mod | |
| # key: ${{ inputs.os }}-go-${{ hashFiles('**/go.sum') }} | |
| # restore-keys: ${{ inputs.os }}-go- | |
| # | |
| # - run: go install ./cmd/litestream | |
| # - run: go test -v -run=TestCmd_Replicate_LongRunning ./integration -long-running-duration 1m | |
| s3-mock-test: | |
| name: Run S3 Mock Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| # cache: 'pip' | |
| - run: pip install moto[s3,server] | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: go env | |
| - run: go install ./cmd/litestream | |
| - run: ./etc/s3_mock.py go test -v ./replica_client_test.go -integration s3 | |
| s3-integration-test: | |
| name: Run S3 Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: github.ref == 'refs/heads/main' | |
| concurrency: | |
| group: integration-test-s3 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: go env | |
| - run: go install ./cmd/litestream | |
| - run: go test -v ./replica_client_test.go -integration s3 | |
| env: | |
| LITESTREAM_S3_ACCESS_KEY_ID: ${{ secrets.LITESTREAM_S3_ACCESS_KEY_ID }} | |
| LITESTREAM_S3_SECRET_ACCESS_KEY: ${{ secrets.LITESTREAM_S3_SECRET_ACCESS_KEY }} | |
| LITESTREAM_S3_REGION: us-east-1 | |
| LITESTREAM_S3_BUCKET: integration.litestream.io | |
| gcp-integration-test: | |
| name: Run GCP Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: github.ref == 'refs/heads/main' | |
| concurrency: | |
| group: integration-test-gcp | |
| steps: | |
| - name: Extract GCP credentials | |
| run: 'echo "$GOOGLE_APPLICATION_CREDENTIALS" > /opt/gcp.json' | |
| shell: bash | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{secrets.GOOGLE_APPLICATION_CREDENTIALS}} | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: go env | |
| - run: go install ./cmd/litestream | |
| - run: go test -v ./replica_client_test.go -integration gcs | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS: /opt/gcp.json | |
| LITESTREAM_GCS_BUCKET: integration.litestream.io | |
| abs-integration-test: | |
| name: Run Azure Blob Store Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| if: github.ref == 'refs/heads/main' | |
| concurrency: | |
| group: integration-test-abs | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: go env | |
| - run: go install ./cmd/litestream | |
| - run: go test -v ./replica_client_test.go -integration abs | |
| env: | |
| LITESTREAM_ABS_ACCOUNT_NAME: ${{ secrets.LITESTREAM_ABS_ACCOUNT_NAME }} | |
| LITESTREAM_ABS_ACCOUNT_KEY: ${{ secrets.LITESTREAM_ABS_ACCOUNT_KEY }} | |
| LITESTREAM_ABS_BUCKET: integration | |
| sftp-integration-test: | |
| name: Run SFTP Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - name: Prepare OpenSSH server | |
| run: |- | |
| sudo mkdir -p /test/etc/ssh /test/home /run/sshd /test/data/ | |
| sudo ssh-keygen -t ed25519 -f /test/etc/ssh/id_ed25519_host -N "" | |
| sudo ssh-keygen -t ed25519 -f /test/etc/ssh/id_ed25519 -N "" | |
| sudo chmod 0600 /test/etc/ssh/id_ed25519_host /test/etc/ssh/id_ed25519 | |
| sudo chmod 0644 /test/etc/ssh/id_ed25519_host.pub /test/etc/ssh/id_ed25519.pub | |
| sudo cp /test/etc/ssh/id_ed25519 /test/id_ed25519 | |
| sudo chown $USER /test/id_ed25519 | |
| sudo tee /test/etc/ssh/sshd_config <<EOF | |
| Port 2222 | |
| HostKey /test/etc/ssh/id_ed25519_host | |
| AuthorizedKeysFile /test/etc/ssh/id_ed25519.pub | |
| AuthenticationMethods publickey | |
| Subsystem sftp internal-sftp | |
| UsePAM no | |
| LogLevel DEBUG | |
| EOF | |
| sudo /usr/sbin/sshd -e -f /test/etc/ssh/sshd_config -E /test/debug.log | |
| - name: Test OpenSSH server works with pubkey auth | |
| run: ssh -v -i /test/id_ed25519 -o StrictHostKeyChecking=accept-new -p 2222 root@localhost whoami || (sudo cat /test/debug.log && exit 1) | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - run: go env | |
| - run: go install ./cmd/litestream | |
| - run: go test -v ./replica_client_test.go -integration sftp | |
| env: | |
| LITESTREAM_SFTP_HOST: "localhost:2222" | |
| LITESTREAM_SFTP_USER: "root" | |
| LITESTREAM_SFTP_KEY_PATH: /test/id_ed25519 | |
| LITESTREAM_SFTP_PATH: /test/data |