build: use SFTP for launchpad uploads

.travis.yml

@@ -68,8 +68,10 @@ matrix:
  - debhelper
  - dput
  - fakeroot
+ - python-bzrlib
+ - python-paramiko
  script:
- - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+ - go run build/ci.go debsrc -upload ppa:ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
 
  # This builder does the Linux Azure uploads
  - if: type = push

build/ci-notes.md

@@ -7,11 +7,18 @@ Canonical.
 Packages of develop branch commits have suffix -unstable and cannot be installed alongside
 the stable version. Switching between release streams requires user intervention.
 
+## Launchpad
+
 The packages are built and served by launchpad.net. We generate a Debian source package
 for each distribution and upload it. Their builder picks up the source package, builds it
 and installs the new version into the PPA repository. Launchpad requires a valid signature
-by a team member for source package uploads. The signing key is stored in an environment
-variable which Travis CI makes available to certain builds.
+by a team member for source package uploads.
+
+The signing key is stored in an environment variable which Travis CI makes available to
+certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the
+packages. To set this up yourself, you need to create a Launchpad user and add a GPG key
+and SSH key to it. Then encode both keys as base64 and configure 'secret' environment
+variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis.
 
 We want to build go-ethereum with the most recent version of Go, irrespective of the Go
 version that is available in the main Ubuntu repository. In order to make this possible,
@@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools:
 
  $ sudo apt-add-repository ppa:gophers/ubuntu/archive
  $ sudo apt-get update
- $ sudo apt-get install build-essential golang-1.10 devscripts debhelper
+ $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko
 
 Create the source packages:
 

build/ci.go

@@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string {
 func archiveUpload(archive string, blobstore string, signer string) error {
  // If signing was requested, generate the signature files
  if signer != "" {
- pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
- if err != nil {
- return fmt.Errorf("invalid base64 %s", signer)
- }
- if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
+ key := getenvBase64(signer)
+ if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil {
  return err
  }
  }
@@ -489,6 +486,7 @@ func doDebianSource(cmdline []string) {
  var (
  signer = flag.String("signer", "", `Signing key name, also used as package author`)
  upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`)
+ sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`)
  workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`)
  now = time.Now()
  )
@@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) {
  maybeSkipArchive(env)
 
  // Import the signing key.
- if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" {
- key, err := base64.StdEncoding.DecodeString(b64key)
- if err != nil {
- log.Fatal("invalid base64 PPA_SIGNING_KEY")
- }
+ if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 {
  gpg := exec.Command("gpg", "--import")
  gpg.Stdin = bytes.NewReader(key)
  build.MustRun(gpg)
@@ -523,12 +517,45 @@ func doDebianSource(cmdline []string) {
  build.MustRunCommand("debsign", changes)
  }
  if *upload != "" {
- build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes)
+ uploadDebianSource(*workdir, *upload, *sshUser, changes)
  }
  }
  }
 }
 
+func uploadDebianSource(workdir, ppa, sshUser, changes string) {
+ // Create the dput config file.
+ dputConfig := filepath.Join(workdir, "dput.cf")
+ p := strings.Split(ppa, "/")
+ if len(p) != 2 {
+ log.Fatal("-upload PPA name must contain single /")
+ }
+ templateData := map[string]string{
+ "LaunchpadUser": p[0],
+ "LaunchpadPPA": p[1],
+ "LaunchpadSSH": sshUser,
+ }
+ if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 {
+ idfile := filepath.Join(workdir, "sshkey")
+ ioutil.WriteFile(idfile, sshkey, 0600)
+ templateData["IdentityFile"] = idfile
+ }
+ build.Render("build/dput-launchpad.cf", dputConfig, 0644, templateData)
+
+ // Run dput to do the upload.
+ dput := exec.Command("dput", "-c", dputConfig, "--no-upload-log", ppa, changes)
+ dput.Stdin = strings.NewReader("Yes\n") // accept SSH host key
+ build.MustRun(dput)
+}
+
+func getenvBase64(variable string) []byte {
+ dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable))
+ if err != nil {
+ log.Fatal("invalid base64 " + variable)
+ }
+ return []byte(dec)
+}
+
 func makeWorkdir(wdflag string) string {
  var err error
  if wdflag != "" {
@@ -800,15 +827,10 @@ func doAndroidArchive(cmdline []string) {
  os.Rename(archive, meta.Package+".aar")
  if *signer != "" && *deploy != "" {
  // Import the signing key into the local GPG instance
- b64key := os.Getenv(*signer)
- key, err := base64.StdEncoding.DecodeString(b64key)
- if err != nil {
- log.Fatalf("invalid base64 %s", *signer)
- }
+ key := getenvBase64(*signer)
  gpg := exec.Command("gpg", "--import")
  gpg.Stdin = bytes.NewReader(key)
  build.MustRun(gpg)
-
  keyID, err := build.PGPKeyID(string(key))
  if err != nil {
  log.Fatal(err)

build/dput-launchpad.cf

@@ -0,0 +1,8 @@
+[{{.LaunchpadUser}}/{{.LaunchpadPPA}}]
+fqdn = ppa.launchpad.net
+method = sftp
+incoming = ~{{.LaunchpadUser}}/ubuntu/{{.LaunchpadPPA}}/
+login = {{.LaunchpadSSH}}
+{{ if .IdentityFile }}
+ ssh_options = IdentityFile {{.IdentityFile}}
+{{ end }}