Skip to content

Commit

Permalink
Update construction of StemcellAutomation.zip
Browse files Browse the repository at this point in the history 
This updates the construction of StemcellAutomation.zip to require
user-provided files. This is closer to how the official binaries for
`stembuild` were built. A script has been added to build the zip file
when provided with the appropriate assets.
  • Loading branch information
@ystros
ystros committed Feb 21, 2025
1 parent 2a075fa commit 7e71135
Show file tree
Hide file tree
Showing 4 changed files with 92 additions and 158 deletions.
122 changes: 6 additions & 116 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,6 @@ GOSRC = $(shell find . -name "*.go" ! -name "*test.go" ! -name "*fake*" ! -path
STEMCELL_VERSION = $(shell echo "$${STEMBUILD_VERSION}")
LD_FLAGS = "-w -s -X github.com/cloudfoundry/stembuild/version.Version=${STEMCELL_VERSION}"

# These are the sources for StemcellAutomation.zip
STEMCELL_AUTOMATION_PS1 := $(shell ls stemcell-automation/*ps1 | grep -iv Test)
BOSH_AGENT_REPO ?= ${HOME}/go/src/github.com/cloudfoundry/bosh-agent
LGPO_URL = 'https://download.microsoft.com/download/8/5/C/85C25433-A1B0-4FFA-9429-7E023E7DA8D8/LGPO.zip'
BOSH_GCS_URL = 'https://s3.amazonaws.com/bosh-gcscli/bosh-gcscli-0.0.6-windows-amd64.exe'
BOSH_BLOBSTORE_DAV_URL = http://bosh-davcli-artifacts.s3.amazonaws.com
BOSH_BLOBSTORE_S3_URL = http://bosh-s3cli-artifacts.s3.amazonaws.com
BOSH_WINDOWS_DEPENDENCIES_URL = http://bosh-windows-dependencies.s3.amazonaws.com
# Ignore things under cis-merge* directory because the paths contain spaces and make doesn't like
# that
PSMODULES_SOURCES = $(shell find ./modules | grep -v .git | grep -vi "test" | grep -v cis-merge)
BOSH_AGENT_SOURCES = $(shell find $(BOSH_AGENT_REPO) | egrep -v ".git|test.go|fake|.md")

ifeq ($(OS),Windows_NT)
COMMAND = out/stembuild.exe
CP = cp
Expand Down Expand Up @@ -54,12 +41,10 @@ integration-badger : generate-fake-stemcell-automation
generate-fake-stemcell-automation:
$(CP) integration/construct/assets/StemcellAutomation.zip assets/

generate: assets/StemcellAutomation.zip

out/stembuild : generate $(GOSRC)
out/stembuild : assets/StemcellAutomation.zip $(GOSRC)
CGO_ENABLED=0 go build -o $(COMMAND) -ldflags $(LD_FLAGS) .

out/stembuild.exe : generate $(GOSRC)
out/stembuild.exe : assets/StemcellAutomation.zip $(GOSRC)
GOOS=windows CGO_ENABLED=0 go build -o out/stembuild.exe -ldflags $(LD_FLAGS) .

test : units
Expand All @@ -73,104 +58,9 @@ units : format generate-fake-stemcell-automation
contract :
go run github.com/onsi/ginkgo/v2/ginkgo -r --randomize-all --randomize-suites --keep-going --flake-attempts 2 iaas_cli

.PHONY : all build build-integration clean format generate generate-fake-stemcell-automation
.PHONY : all build build-integration clean format generate-fake-stemcell-automation
.PHONY : test units units-full integration integration-tests-full

# ===============================================================================
# The following to create the StemcellAutomation.zip that's packaged in stembuild
# ===============================================================================

assets/local/bosh-agent.exe: $(BOSH_AGENT_SOURCES)
cd $(BOSH_AGENT_REPO) && \
GOOS=windows GOARCH=amd64 bin/build && \
cd -
mv $(BOSH_AGENT_REPO)/out/bosh-agent assets/local/bosh-agent.exe

assets/local/bosh-blobstore-dav.exe:
@echo "### Creating assets/local/bosh-blobstore-dav.exe"
$(eval BOSH_BLOBSTORE_DAV_FILE=$(shell curl -s $(BOSH_BLOBSTORE_DAV_URL) | xq --xpath '//Key' | sort --version-sort | tail -1))
curl -o assets/local/bosh-blobstore-dav.exe -L $(BOSH_BLOBSTORE_DAV_URL)/$(BOSH_BLOBSTORE_DAV_FILE)

assets/local/bosh-blobstore-gcs.exe:
@echo "### Creating assets/local/bosh-blobstore-gcs.exe"
curl -o assets/local/bosh-blobstore-gcs.exe -L $(BOSH_GCS_URL)

assets/local/bosh-blobstore-s3.exe:
@echo "### Creating assets/local/bosh-blobstore-s3.exe"
$(eval BOSH_BLOBSTORE_S3_FILE=$(shell curl -s $(BOSH_BLOBSTORE_S3_URL) | xq --xpath '//Key' | sort --version-sort | tail -1))
curl -o assets/local/bosh-blobstore-s3.exe -L $(BOSH_BLOBSTORE_S3_URL)/$(BOSH_BLOBSTORE_S3_FILE)

assets/local/bosh-psmodules.zip: $(PSMODULES_SOURCES)
@echo "### Creating/Updating assets/local/bosh-psmodules.zip"
cd modules && zip -r ../bosh-psmodules.zip . && cd ..
mv bosh-psmodules.zip assets/local/bosh-psmodules.zip

assets/local/job-service-wrapper.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/job-service-wrapper.exe
@echo "### Creating/Updating assets/local/job-service-wrapper.exe"
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/job-service-wrapper.exe assets/local

assets/local/pipe.exe: $(BOSH_AGENT_SOURCES)
cd $(BOSH_AGENT_REPO) && \
GOOS=windows GOARCH=amd64 bin/build && \
cd -
mv $(BOSH_AGENT_REPO)/out/bosh-agent-pipe assets/local/pipe.exe

assets/local/service_wrapper.exe: $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.exe
@echo "### Creating/Updating assets/local/service_wrapper.exe"
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.exe assets/local

assets/local/service_wrapper.xml: $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.xml
@echo "### Creating/Updating assets/local/service_wrapper.xml"
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/service_wrapper.xml assets/local

assets/local/tar.exe:
@echo "### Creating assets/local/tar.exe"
$(eval BOSH_WINDOWS_DEPENDENCIES_FILE=$(shell curl -s $(BOSH_WINDOWS_DEPENDENCIES_URL) | xq --xpath '//Key[contains(text(), "tar")]' | sort --version-sort | tail -1))
curl -o assets/local/tar.exe -L $(BOSH_WINDOWS_DEPENDENCIES_URL)/$(BOSH_WINDOWS_DEPENDENCIES_FILE)

assets/local/agent.zip: assets/local/bosh-agent.exe assets/local/pipe.exe assets/local/service_wrapper.xml assets/local/service_wrapper.exe assets/local/bosh-blobstore-dav.exe assets/local/bosh-blobstore-gcs.exe assets/local/bosh-blobstore-s3.exe assets/local/job-service-wrapper.exe assets/local/tar.exe
@echo "### Creating/Updating assets/local/agent.zip"
mkdir -p assets/temp/deps
$(CP) assets/local/service_wrapper.exe \
assets/local/service_wrapper.xml \
assets/local/bosh-agent.exe \
assets/temp
$(CP) assets/local/bosh-blobstore-dav.exe \
assets/local/bosh-blobstore-gcs.exe \
assets/local/bosh-blobstore-s3.exe \
assets/local/job-service-wrapper.exe \
assets/local/pipe.exe \
assets/local/tar.exe \
assets/temp/deps
cd assets/temp && zip -r ../local/agent.zip * && cd -
rm -rf assets/temp

assets/local/LGPO.zip:
@echo "### Creating assets/local/LGPO.zip"
curl -o assets/local/LGPO.zip -L $(LGPO_URL)

assets/local/OpenSSH-Win64.zip: $(BOSH_AGENT_REPO)/integration/windows/fixtures/OpenSSH-Win64.zip
@echo "### Creating/Updating assets/local/OpenSSH-Win64.zip"
$(CP) $(BOSH_AGENT_REPO)/integration/windows/fixtures/OpenSSH-Win64.zip assets/local

assets/local/deps.json: assets/local/agent.zip assets/local/bosh-psmodules.zip assets/local/LGPO.zip assets/local/OpenSSH-Win64.zip
@echo "### Creating/Updating assets/local/deps.json"
@#Note: The order of the following matters, change the script before changing these
stemcell-automation/generate-dep-json.bash \
assets/local/OpenSSH-Win64.zip \
assets/local/bosh-psmodules.zip \
assets/local/agent.zip \
assets/local/LGPO.zip \
> assets/local/deps.json

assets/StemcellAutomation.zip: $(STEMCELL_AUTOMATION_PS1) assets/local/OpenSSH-Win64.zip assets/local/bosh-psmodules.zip assets/local/deps.json assets/local/agent.zip
@echo "### Creating/Updating assets/StemcellAutomation.zip"
mkdir -p assets/temp
cp -a $(STEMCELL_AUTOMATION_PS1) \
assets/local/OpenSSH-Win64.zip \
assets/local/bosh-psmodules.zip \
assets/local/deps.json \
assets/local/agent.zip \
assets/temp
cd assets/temp && zip ../StemcellAutomation.zip * && cd -
rm -rf assets/temp
assets/StemcellAutomation.zip:
@echo "Unable to locate assets/StemcellAutomation.zip. This file is bundled into the final stembuild executable and must be present. Please see README.md on how to construct this file."
exit 1
79 changes: 62 additions & 17 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ stembuild construct -vm-ip <IP of VM> -vm-username <vm username> -vm-password <v
```

### Requirements
- LGPO.zip in current working directory
- LGPO.zip in current working directory. This is a zip of LGPO.exe from Microsoft that is compatible with the current version of Windows. https://www.microsoft.com/en-us/download/details.aspx?id=55319
- Constructed assets/StemcellAutomation.zip (contents described below)
- Running Windows VM with:
- Up-to-date Operating System
- Reachable by IP over port 5985
Expand Down Expand Up @@ -113,14 +114,14 @@ Flags:
```

### Compiling & Running Stembuild Locally
### Running Stembuild Locally

Assuming you've followed [these instructions](https://bosh.io/docs/windows-stemcell-create/) and you've created a Windows VM at 10.9.9.115 whose Administrator's password is "c1oudc0w".

```bash
export TARGET_VM_PASSWORD=c1oudc0w VCENTER_PASSWORD='Admin!23'
BOSH_AGENT_REPO=~/workspace/bosh-agent STEMBUILD_VERSION=2019.2 make
GOVC_INSECURE=true out/stembuild -debug \
export TARGET_VM_PASSWORD=c1oudc0w VCENTER_PASSWORD='Admin!23'

GOVC_INSECURE=true stembuild -debug \
construct \
-vm-ip 10.9.9.115 \
-vm-username Administrator \
Expand All @@ -129,7 +130,8 @@ GOVC_INSECURE=true out/stembuild -debug \
-vcenter-username administrator@vsphere.local \
-vcenter-password $VCENTER_PASSWORD \
-vm-inventory-path "/dc/vm/Discovered virtual machine/w2019-stemcell"
GOVC_INSECURE=true out/stembuild -debug \

GOVC_INSECURE=true stembuild -debug \
package \
-vcenter-url vcenter-70.nono.io \
-vcenter-username administrator@vsphere.local \
Expand Down Expand Up @@ -227,23 +229,66 @@ The output should be nothing if there are no out-of-sync dependencies.

## Compile stembuild locally

Some of the make targets require the use of curl and [xq](https://github.com/sibprogrammer/xq?tab=readme-ov-file#installation)
to download the latest dependencies from S3 to create the embedded StemcellAutomation.zip resource. The dependencies
downloaded from s3 are the same ones used by [stembuild CI](https://github.com/cloudfoundry/greenhouse-ci/).

Download or clone the bosh-agent repository
```
git clone https://github.com/cloudfoundry/bosh-agent.git
```

Download or clone the stembuild repository and navigate to it
```
git clone https://github.com/cloudfoundry/stembuild.git
cd stembuild
```

Use `make build` to build stembuild, providing the corresponding values for the bosh-agent path and stemcell version you
You will need to construct `assets/StemcellAutomation.zip`. This file represents various BOSH executables necessary to build a working stemcell.

**assets/StemcellAutomation.zip files:**
| File | Source / Description |
|-|-|
| OpenSSH-Win64.zip | https://github.com/PowerShell/Win32-OpenSSH/releases |
| bosh-psmodules.zip | https://github.com/cloudfoundry/bosh-psmodules/tree/master/modules |
| agent.zip | A zip constructed using various BOSH executables. See list of necessary files below. |
| deps.json | A JSON file with the SHA256 checksums and optionally the version for each component in this zip. See format below. |

**agent.zip files:**
| File | Source / Description |
|-|-|
| bosh-agent.exe | https://github.com/cloudfoundry/bosh-agent/ |
| sha | Git commit SHA from the bosh-agent repo |
| deps/pipe.exe | https://github.com/cloudfoundry/bosh-agent/tree/main/jobsupervisor/pipe |
| deps/bosh-blobstore-gcs.exe | https://github.com/cloudfoundry/bosh-gcscli |
| deps/bosh-blobstore-dav.exe | https://github.com/cloudfoundry/bosh-davcli |
| deps/bosh-blobstore-s3.exe | https://github.com/cloudfoundry/bosh-s3cli |
| deps/tar.exe | https://github.com/cloudfoundry/bsdtar/ |
| deps/job-service-wrapper.exe | https://github.com/bosh-dep-forks/winsw |
| service_wrapper.exe | https://github.com/bosh-dep-forks/winsw |
| service_wrapper.xml | https://github.com/cloudfoundry/bosh-agent/blob/main/integration/windows/fixtures/service_wrapper.xml |

**deps.json format:**
```json
{
"OpenSSH-Win64.zip": {
"sha": "SOME-SHA256",
},
"bosh-psmodules.zip": {
"sha": "SOME-SHA256",
},
"agent.zip": {
"sha": "SOME-SHA256",
},
"LGPO.zip": {
"sha": "SOME-SHA256",
"version": "3"
}
}
```

Once you have these files, run `./bin/build-stemcell-automation-zip.sh`:
```bash
OPENSSH_ZIP=OpenSSH-Win64.zip \
BOSH_PSMODULES_ZIP="bosh-psmodules.zip" \
AGENT_ZIP="agent.zip" \
DEPS_JSON="deps.json" \
./bin/build-stemcell-automation-zip.sh
```

Use `make build` to build stembuild, providing the corresponding values for the stemcell version you
would like to build, for example:
```
BOSH_AGENT_REPO=../bosh-agent STEMBUILD_VERSION=2019.70 make build
STEMBUILD_VERSION=2019.70 make build
```
24 changes: 24 additions & 0 deletions bin/build-stemcell-automation-zip.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -eu -o pipefail

ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
STEMCELL_AUTOMATION_PS1=$(ls "${ROOT_DIR}"/stemcell-automation/*ps1 | grep -iv Test)

: ${OPENSSH_ZIP?"Please see README.md on where to obtain this."}
: ${BOSH_PSMODULES_ZIP?"Please see README.md on where to obtain this."}
: ${AGENT_ZIP?"Please see README.md on how to construct this."}
: ${DEPS_JSON?"Please see README.md on how to construct this."}

TEMP_DIR=$(mktemp -d)

cp "${OPENSSH_ZIP}" "${TEMP_DIR}/OpenSSH-Win64.zip"
cp "${BOSH_PSMODULES_ZIP}" "${TEMP_DIR}/bosh-psmodules.zip"
cp "${AGENT_ZIP}" "${TEMP_DIR}/agent.zip"
cp "${DEPS_JSON}" "${TEMP_DIR}/deps.json"
cp ${STEMCELL_AUTOMATION_PS1} "$TEMP_DIR"

rm "${ROOT_DIR}/assets/StemcellAutomation.zip"

zip -rj "${ROOT_DIR}/assets/StemcellAutomation.zip" "$TEMP_DIR"

rm -r "$TEMP_DIR"
25 changes: 0 additions & 25 deletions stemcell-automation/generate-dep-json.bash
View file

This file was deleted.

0 comments on commit 7e71135

Please sign in to comment.