Skip to content

Commit

Permalink
Merge branch 'master' into partial-check
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored Jan 4, 2021
2 parents b45eb50 + 9a09991 commit faaa390
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 1,063 deletions.
85 changes: 0 additions & 85 deletions pkg/repository/v1_repository_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -824,88 +824,3 @@ func hash(s string) string {

return hex.EncodeToString(shaWriter.Sum(nil))
}

// Test we can correctly load manifests generate by tools/migrate
// which generate the v1manifest from the v0manifest.
//func TestWithMigrate(t *testing.T) {
// // generate using tools/migrate
// mdir := "./testdata/manifests"
//
// // create a repo using the manifests as a mirror.
// // profileDir will contains the only trusted root.
// repo, profileDir := createMigrateRepo(t, mdir)
// _, err := repo.loadRoot()
// assert.Nil(t, err)
// defer os.RemoveAll(profileDir)
// _ = profileDir
//
// err = repo.updateLocalRoot()
// assert.Nil(t, err)
//
// _, err = repo.ensureManifests()
// assert.Nil(t, err)
//
// // after ensureManifests we should can load index/timestamp/snapshot
// var snap v1manifest.Snapshot
// var index v1manifest.Index
// var root v1manifest.Root
// {
// exists, err := repo.local.LoadManifest(&index)
// assert.Nil(t, err)
// assert.True(t, exists)
// exists, err = repo.local.LoadManifest(&root)
// assert.Nil(t, err)
// assert.True(t, exists)
// exists, err = repo.local.LoadManifest(&snap)
// assert.Nil(t, err)
// assert.True(t, exists)
// }
//
// // check can load component manifests.
// assert.NotZero(t, len(snap.Meta))
// assert.NotZero(t, len(index.Components))
// {
// // Every component should in snapshot's meta
// t.Logf("snap meta: %+v", snap.Meta)
// for _, item := range index.Components {
// _, ok := snap.Meta[item.URL]
// assert.True(t, ok, "component url: %s", item.URL)
// }
//
// // Test after updateComponentManifest we can load it locally.
// for id, item := range index.Components {
// _, err := repo.updateComponentManifest(id)
// assert.Nil(t, err)
//
// filename := v1manifest.ComponentManifestFilename(id)
// _, _, err = snap.VersionedURL(item.URL)
// assert.Nil(t, err)
// _, err = repo.local.LoadComponentManifest(&index, filename)
// assert.Nil(t, err)
// }
// }
//}

/*
func createMigrateRepo(t *testing.T, mdir string) (repo *V1Repository, profileDir string) {
var err error
profileDir, err = ioutil.TempDir("", "tiup-*")
assert.Nil(t, err)
t.Logf("using profile dir: %s", profileDir)
// copy root.json from mdir to profileDir
data, err := ioutil.ReadFile(filepath.Join(mdir, "root.json"))
assert.Nil(t, err)
err = ioutil.WriteFile(filepath.Join(profileDir, "root.json"), data, 0644)
assert.Nil(t, err)
localdata.DefaultTiupHome = profileDir
profile := localdata.InitProfile()
options := Options{}
mirror := NewMirror(mdir, MirrorOptions{})
local, err := v1manifest.NewManifests(profile)
assert.Nil(t, err)
repo = NewV1Repo(mirror, options, local)
return
}
*/
243 changes: 7 additions & 236 deletions server/package/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,34 +14,15 @@
package main

import (
"crypto/sha1"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"

"github.com/juju/errors"
"github.com/pingcap/tiup/pkg/environment"
"github.com/pingcap/tiup/pkg/localdata"
"github.com/pingcap/tiup/pkg/repository/v0manifest"
pkgver "github.com/pingcap/tiup/pkg/repository/version"
"github.com/pingcap/tiup/pkg/utils"
"github.com/pingcap/tiup/pkg/version"
"github.com/spf13/cobra"
)

var mirror = environment.Mirror()
var errNotFound = fmt.Errorf("resource not found")

func main() {
if err := execute(); err != nil {
fmt.Println("Packaging component failed:", err)
Expand Down Expand Up @@ -80,19 +61,20 @@ func execute() error {
},
}

// some arguments are not used anymore, we keep them to make it compatible
// with legacy CI jobs
rootCmd.Flags().StringVar(&options.goos, "os", runtime.GOOS, "Target OS of the package")
rootCmd.Flags().StringVar(&options.goarch, "arch", runtime.GOARCH, "Target ARCH of the package")
rootCmd.Flags().StringVarP(&options.dir, "", "C", "", "Change directory before compress")
rootCmd.Flags().StringVar(&options.name, "name", "", "Name of the package")
rootCmd.Flags().StringVar(&options.version, "release", "", "Version of the package")
rootCmd.Flags().StringVar(&options.entry, "entry", "", "Entry point of the package")
rootCmd.Flags().StringVar(&options.desc, "desc", "", "Description of the package")
rootCmd.Flags().BoolVar(&options.standalone, "standalone", false, "Can the component run standalone")
rootCmd.Flags().BoolVar(&options.hide, "hide", false, "Don't show the component in `tiup list`")
rootCmd.Flags().StringVar(&options.entry, "entry", "", "(deprecated) Entry point of the package")
rootCmd.Flags().StringVar(&options.desc, "desc", "", "(deprecated) Description of the package")
rootCmd.Flags().BoolVar(&options.standalone, "standalone", false, "(deprecated) Can the component run standalone")
rootCmd.Flags().BoolVar(&options.hide, "hide", false, "(deprecated) Don't show the component in `tiup list`")

_ = rootCmd.MarkFlagRequired("name")
_ = rootCmd.MarkFlagRequired("release")
_ = rootCmd.MarkFlagRequired("entry")

return rootCmd.Execute()
}
Expand All @@ -103,200 +85,7 @@ func pack(targets []string, options packageOptions) error {
}

// tar -czf package/{name}-{version}-{goos}-{goarch}.tar.gz target
if err := packTarget(targets, options); err != nil {
return err
}

if err := checksum(options); err != nil {
return err
}

if err := manifestIndex(options); err != nil {
return err
}

return componentIndex(options)
}

func current(fname string, target interface{}) error {
err := load("package/"+fname, target)
if err != nil && !os.IsNotExist(err) {
return err
}
if os.IsNotExist(err) {
if strings.HasPrefix(mirror, "http") {
url := join(mirror, fname+"?t="+strconv.Itoa(int(time.Now().Unix())))
if err := get(url, target); err != nil {
return err
}
} else {
if utils.IsNotExist(filepath.Join(mirror, fname)) {
return errNotFound
}
f, err := os.Open(filepath.Join(mirror, fname))
if err != nil {
return err
}
defer f.Close()
return json.NewDecoder(f).Decode(target)
}
}
return nil
}

func manifestIndex(options packageOptions) error {
mIndex := v0manifest.ComponentManifest{}
err := current("tiup-manifest.index", &mIndex)
if err != nil && err != errNotFound {
return err
}
if err == errNotFound {
mIndex.Modified = time.Now().Format(time.RFC3339)
mIndex.TiUPVersion = pkgver.Version(version.NewTiUPVersion().SemVer())
mIndex.Description = "TiDB components manager"
}

pair := options.goos + "/" + options.goarch

for idx := range mIndex.Components {
if mIndex.Components[idx].Name == options.name {
if options.desc != "" {
mIndex.Components[idx].Desc = options.desc
}
if mIndex.Components[idx].Standalone != options.standalone {
mIndex.Components[idx].Standalone = options.standalone
}
if mIndex.Components[idx].Hide != options.hide {
mIndex.Components[idx].Hide = options.hide
}
for _, p := range mIndex.Components[idx].Platforms {
if p == pair {
return write("package/tiup-manifest.index", mIndex)
}
}
mIndex.Components[idx].Platforms = append(mIndex.Components[idx].Platforms, pair)
return write("package/tiup-manifest.index", mIndex)
}
}
mIndex.Components = append(mIndex.Components, v0manifest.ComponentInfo{
Name: options.name,
Desc: options.desc,
Standalone: options.standalone,
Hide: options.hide,
Platforms: []string{pair},
})

return write("package/tiup-manifest.index", mIndex)
}

func componentIndex(options packageOptions) error {
fname := fmt.Sprintf("tiup-component-%s.index", options.name)
pair := options.goos + "/" + options.goarch

cIndex := v0manifest.VersionManifest{}
err := current(fname, &cIndex)
if err != nil && err != errNotFound {
return err
}

version := pkgver.Version(options.version)

v := v0manifest.VersionInfo{
Version: version,
Date: time.Now().Format(time.RFC3339),
Entry: options.entry,
Platforms: []string{pair},
}

// Generate a new component index
if err == errNotFound {
cIndex = v0manifest.VersionManifest{
Description: options.desc,
Modified: time.Now().Format(time.RFC3339),
}
if version.IsNightly() {
cIndex.Nightly = &v
} else {
cIndex.Versions = append(cIndex.Versions, v)
}
return write("package/"+fname, cIndex)
}

for idx := range cIndex.Versions {
if cIndex.Versions[idx].Version == version {
cIndex.Versions[idx].Date = time.Now().Format(time.RFC3339)
cIndex.Versions[idx].Entry = options.entry
for _, p := range cIndex.Versions[idx].Platforms {
if p == pair {
return write("package/"+fname, cIndex)
}
}
cIndex.Versions[idx].Platforms = append(cIndex.Versions[idx].Platforms, pair)
return write("package/"+fname, cIndex)
}
}

if version.IsNightly() {
if cIndex.Nightly == nil {
cIndex.Nightly = &v
return write("package/"+fname, cIndex)
}
cIndex.Nightly.Date = time.Now().Format(time.RFC3339)
cIndex.Nightly.Entry = options.entry
for _, p := range cIndex.Nightly.Platforms {
if p == pair {
return write("package/"+fname, cIndex)
}
}
cIndex.Nightly.Platforms = append(cIndex.Nightly.Platforms, pair)
return write("package/"+fname, cIndex)
}

cIndex.Versions = append(cIndex.Versions, v)
return write("package/"+fname, cIndex)
}

func join(url, file string) string {
if strings.HasSuffix(url, "/") {
return url + file
}
return url + "/" + file
}

func write(file string, data interface{}) error {
f, err := os.Create(file)
if err != nil {
return err
}
defer f.Close()

enc := json.NewEncoder(f)
enc.SetIndent("", " ")
return enc.Encode(data)
}

func load(file string, target interface{}) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()

return json.NewDecoder(f).Decode(target)
}

func get(url string, target interface{}) error {
r, err := http.Get(url)
if err != nil {
return err
}
defer r.Body.Close()

if r.StatusCode == http.StatusNotFound {
return errNotFound
}

return json.NewDecoder(r.Body).Decode(target)
return packTarget(targets, options)
}

func packTarget(targets []string, options packageOptions) error {
Expand All @@ -315,24 +104,6 @@ func packTarget(targets []string, options packageOptions) error {
return nil
}

func checksum(options packageOptions) error {
tarball, err := os.OpenFile(fmt.Sprintf("package/%s-%s-%s-%s.tar.gz", options.name, options.version, options.goos, options.goarch), os.O_RDONLY, 0)
if err != nil {
return errors.Trace(err)
}
defer tarball.Close()

sha1Writter := sha1.New()
if _, err := io.Copy(sha1Writter, tarball); err != nil {
return errors.Trace(err)
}

checksum := hex.EncodeToString(sha1Writter.Sum(nil))
file := fmt.Sprintf("package/%s-%s-%s-%s.sha1", options.name, options.version, options.goos, options.goarch)

return ioutil.WriteFile(file, []byte(checksum), 0664)
}

func chwd() error {
pwd, found := os.LookupEnv(localdata.EnvNameWorkDir)
if !found {
Expand Down
Loading

0 comments on commit faaa390

Please sign in to comment.