diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b1f6de8..c4b326f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,25 +14,18 @@ jobs: goreleaser: runs-on: ubuntu-latest steps: - - - name: Checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - - name: Fetch all tags - run: git fetch --force --tags - - - name: Set up Go - uses: actions/setup-go@v2 + - run: git fetch --force --tags + - uses: actions/setup-go@v4 with: - go-version: 1.16 - - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + go-version: stable + - uses: goreleaser/goreleaser-action@v4 with: + # either 'goreleaser' (default) or 'goreleaser-pro': distribution: goreleaser version: latest - args: release --rm-dist + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fdad08e..96bc464 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -13,21 +13,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v4 with: - go-version: '^1.16' - - - name: Cache go modules - uses: actions/cache@v2 - with: - path: | - ~/.cache/go-build - ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + go-version: '^1.19' - name: Run tests run: go test ./... @@ -35,8 +25,8 @@ jobs: name: Linting runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: golangci/golangci-lint-action@v2 + - uses: actions/checkout@v3 + - uses: golangci/golangci-lint-action@v3 with: version: latest # Enable additional linters (see: https://golangci-lint.run/usage/linters/) diff --git a/.goreleaser.yml b/.goreleaser.yml index 31f8db4..300a66f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -12,4 +12,4 @@ archives: release: draft: true - prerelease: true + prerelease: "true" diff --git a/README.md b/README.md index 4fd046d..47ba56c 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,44 @@ +# Veidemannctl + [![License Apache](https://img.shields.io/github/license/nlnwa/veidemannctl.svg)](https://github.com/nlnwa/veidemannctl/blob/master/LICENSE) [![GitHub release](https://img.shields.io/github/release/nlnwa/veidemannctl.svg)](https://github.com/nlnwa/veidemannctl/releases/latest) - -# veidemannctl +[![Go Report Card](https://goreportcard.com/badge/github.com/nlnwa/veidemannctl?style=flat-square)](https://goreportcard.com/report/github.com/nlnwa/veidemannctl) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/nlnwa/veidemannctl)](https://pkg.go.dev/github.com/nlnwa/veidemannctl) ## Install -Install the latest release version + +Install the latest release version (linux/amd64) + ```console curl -sL https://raw.githubusercontent.com/nlnwa/veidemannctl/master/install.sh | bash ``` ## Usage + To get a list of available commands and configuration flags: + ```console veidemanctl -h ``` + ## Documentation -Usage documentation: https://nlnwa.github.io/veidemannctl -## Build instructions +Usage documentation: + +## Build + ```console go build ``` -## Generate documentation and templates +## Test + +```console +go test ./... +``` + +## Generate documentation + ```console go generate ``` diff --git a/src/apiutil/apiutil.go b/apiutil/apiutil.go similarity index 69% rename from src/apiutil/apiutil.go rename to apiutil/apiutil.go index 6d5aeaf..1ba73a5 100644 --- a/src/apiutil/apiutil.go +++ b/apiutil/apiutil.go @@ -14,12 +14,18 @@ package apiutil import ( + "context" + "errors" "fmt" + "io" "strconv" "strings" + "github.com/nlnwa/veidemann-api/go/commons/v1" commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" ) @@ -47,7 +53,7 @@ func stringSliceContains(slice []string, s string) bool { // The filter string should have the format: [.]*[+|-]=. // The message should be a pointer to a proto message. // The mask will be updated with the fields that are set. -func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV1.FieldMask) error { +func CreateTemplateFilter(filterString string, msg proto.Message, mask *commonsV1.FieldMask) error { path, value, ok := strings.Cut(filterString, "=") if !ok { return fmt.Errorf("invalid filter: %s", filterString) @@ -59,7 +65,7 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV tokens := strings.Split(path, ".") var fieldType protoreflect.FieldDescriptor - msgType := obj.ProtoReflect().Descriptor() + msgType := msg.ProtoReflect().Descriptor() for _, token := range tokens { fieldType = msgType.Fields().ByJSONName(token) @@ -76,7 +82,7 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV if err != nil { return fmt.Errorf("error converting %v to boolean: %w", value, err) } - setValue(protoreflect.ValueOfBool(v), obj, fieldType) + setValue(protoreflect.ValueOfBool(v), msg, fieldType) case protoreflect.Int32Kind: fallthrough @@ -87,7 +93,7 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV if err != nil { return fmt.Errorf("error converting %v to int: %w", value, err) } - setValue(protoreflect.ValueOfInt32(int32(v)), obj, fieldType) + setValue(protoreflect.ValueOfInt32(int32(v)), msg, fieldType) case protoreflect.Int64Kind: fallthrough @@ -98,7 +104,7 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV if err != nil { return fmt.Errorf("error converting %v to int: %w", value, err) } - setValue(protoreflect.ValueOfInt64(v), obj, fieldType) + setValue(protoreflect.ValueOfInt64(v), msg, fieldType) case protoreflect.Uint32Kind: fallthrough @@ -107,7 +113,7 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV if err != nil { return fmt.Errorf("error converting %v to uint: %w", value, err) } - setValue(protoreflect.ValueOfUint32(uint32(v)), obj, fieldType) + setValue(protoreflect.ValueOfUint32(uint32(v)), msg, fieldType) case protoreflect.Uint64Kind: fallthrough @@ -116,27 +122,27 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV if err != nil { return fmt.Errorf("error converting %v to uint: %w", value, err) } - setValue(protoreflect.ValueOfUint64(v), obj, fieldType) + setValue(protoreflect.ValueOfUint64(v), msg, fieldType) case protoreflect.FloatKind: v, err := strconv.ParseFloat(value, 32) if err != nil { return fmt.Errorf("error converting %v to float: %w", value, err) } - setValue(protoreflect.ValueOfFloat32(float32(v)), obj, fieldType) + setValue(protoreflect.ValueOfFloat32(float32(v)), msg, fieldType) case protoreflect.DoubleKind: v, err := strconv.ParseFloat(value, 64) if err != nil { return fmt.Errorf("error converting %v to float: %w", value, err) } - setValue(protoreflect.ValueOfFloat64(v), obj, fieldType) + setValue(protoreflect.ValueOfFloat64(v), msg, fieldType) case protoreflect.StringKind: - setValue(protoreflect.ValueOfString(value), obj, fieldType) + setValue(protoreflect.ValueOfString(value), msg, fieldType) case protoreflect.BytesKind: - setValue(protoreflect.ValueOfBytes([]byte(value)), obj, fieldType) + setValue(protoreflect.ValueOfBytes([]byte(value)), msg, fieldType) case protoreflect.EnumKind: enumVal := fieldType.Enum().Values().ByName(protoreflect.Name(value)) @@ -147,17 +153,17 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV } return fmt.Errorf("not a valid enum value '%s' for '%s'. Valid values: %s", value, fieldType.FullName(), strings.Join(sa, ", ")) } - setValue(protoreflect.ValueOfEnum(enumVal.Number()), obj, fieldType) + setValue(protoreflect.ValueOfEnum(enumVal.Number()), msg, fieldType) case protoreflect.GroupKind: case protoreflect.MessageKind: var item protoreflect.Value isNewFieldValue := false - if obj.ProtoReflect().Has(fieldType) { - item = obj.ProtoReflect().Mutable(fieldType) + if msg.ProtoReflect().Has(fieldType) { + item = msg.ProtoReflect().Mutable(fieldType) } else { isNewFieldValue = true - item = obj.ProtoReflect().NewField(fieldType) + item = msg.ProtoReflect().NewField(fieldType) } var message protoreflect.Message @@ -187,15 +193,16 @@ func CreateTemplateFilter(filterString string, obj proto.Message, mask *commonsV } // Set the field value if it is a new field value if isNewFieldValue { - obj.ProtoReflect().Set(fieldType, item) + msg.ProtoReflect().Set(fieldType, item) } - obj = message.Interface() + msg = message.Interface() } } return nil } +// setValue sets the value of a field in a proto message. func setValue(v protoreflect.Value, obj proto.Message, fieldType protoreflect.FieldDescriptor) { if fieldType.IsList() { list := obj.ProtoReflect().NewField(fieldType) @@ -205,25 +212,72 @@ func setValue(v protoreflect.Value, obj proto.Message, fieldType protoreflect.Fi obj.ProtoReflect().Set(fieldType, v) } -func CreateListRequest(kind configV1.Kind, ids []string, name string, labelString string, filterString string, pageSize int32, page int32) (*configV1.ListRequest, error) { - request := &configV1.ListRequest{ +// CreateListRequest creates a list request from the given parameters. +func CreateListRequest(kind configV1.Kind, ids []string, name string, labelString string, filters []string, pageSize int32, page int32) (*configV1.ListRequest, error) { + var queryMask *commonsV1.FieldMask + var queryTemplate *configV1.ConfigObject + + if filters != nil { + queryMask = new(commonsV1.FieldMask) + queryTemplate = new(configV1.ConfigObject) + + for _, filter := range filters { + if err := CreateTemplateFilter(filter, queryTemplate, queryMask); err != nil { + return nil, err + } + } + } + + return &configV1.ListRequest{ Kind: kind, Id: ids, NameRegex: name, LabelSelector: CreateSelector(labelString), Offset: page, PageSize: pageSize, + QueryTemplate: queryTemplate, + QueryMask: queryMask, + }, nil +} + +// CompleteName returns a list of names that matches the given name. +func CompleteName(kind string, name string) ([]string, error) { + k := format.GetKind(kind) + if k == configV1.Kind_undefined { + return nil, errors.New("undefined kind") + } + + request, err := CreateListRequest(k, nil, name, "", nil, 10, 0) + if err != nil { + return nil, err + } + + request.ReturnedFieldsMask = &commons.FieldMask{ + Paths: []string{"meta.name"}, + } + + conn, err := connection.Connect() + if err != nil { + return nil, err } + defer conn.Close() - if filterString != "" { - queryMask := new(commonsV1.FieldMask) - queryTemplate := new(configV1.ConfigObject) - if err := CreateTemplateFilter(filterString, queryTemplate, queryMask); err != nil { + client := configV1.NewConfigClient(conn) + r, err := client.ListConfigObjects(context.Background(), request) + if err != nil { + return nil, err + } + + var names []string + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { return nil, err } - request.QueryMask = queryMask - request.QueryTemplate = queryTemplate + names = append(names, msg.Meta.Name) } - - return request, nil + return names, nil } diff --git a/src/apiutil/apiutil_test.go b/apiutil/apiutil_test.go similarity index 97% rename from src/apiutil/apiutil_test.go rename to apiutil/apiutil_test.go index 7e6b37f..9129c76 100644 --- a/src/apiutil/apiutil_test.go +++ b/apiutil/apiutil_test.go @@ -13,13 +13,14 @@ package apiutil -import ( +import ( + "reflect" + "testing" + commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" configV1 "github.com/nlnwa/veidemann-api/go/config/v1" frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" "google.golang.org/protobuf/proto" - "reflect" - "testing" api "github.com/nlnwa/veidemann-api/go/config/v1" ) @@ -85,7 +86,7 @@ func TestCreateListRequest(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := CreateListRequest(api.Kind_browserConfig, tt.args.ids, tt.args.name, tt.args.labelString, "", tt.args.pageSize, tt.args.page) + got, err := CreateListRequest(api.Kind_browserConfig, tt.args.ids, tt.args.name, tt.args.labelString, nil, tt.args.pageSize, tt.args.page) if err != nil { t.Errorf("Error in CreateListRequest(): %v", err) } @@ -135,7 +136,7 @@ func TestCreateTemplateFilter(t *testing.T) { {"oneof/int", args{filterString: "browserConfig.pageLoadTimeoutMs=100", templateObj: &configV1.ConfigObject{}}, &commonsV1.FieldMask{Paths: []string{"browserConfig.pageLoadTimeoutMs"}}, - &configV1.ConfigObject{Spec: &configV1.ConfigObject_BrowserConfig{&configV1.BrowserConfig{PageLoadTimeoutMs: 100}}}, + &configV1.ConfigObject{Spec: &configV1.ConfigObject_BrowserConfig{BrowserConfig: &configV1.BrowserConfig{PageLoadTimeoutMs: 100}}}, false, }, {"illegal filter for template", diff --git a/bindata/asset.go b/bindata/asset.go deleted file mode 100644 index 963a876..0000000 --- a/bindata/asset.go +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package bindata - -import ( - "github.com/pkg/errors" - "io/ioutil" -) - -// Asset returns an embedded asset -func Asset(name string) ([]byte, error) { - file, err := Assets.Open(name) - if err != nil { - return nil, err - } - defer file.Close() - - info, err := file.Stat() - if err != nil { - return nil, err - } - - if info.IsDir() { - return nil, errors.Errorf("Asset %s is a Directory, not a File", name) - } - - return ioutil.ReadAll(file) -} diff --git a/bindata/assets_generate.go b/bindata/assets_generate.go deleted file mode 100644 index 3a36d51..0000000 --- a/bindata/assets_generate.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build ignore - -/* - * Copyright 2019 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "github.com/nlnwa/veidemannctl/bindata" - "github.com/shurcooL/vfsgen" - "log" -) - -func main() { - err := vfsgen.Generate(bindata.Assets, vfsgen.Options{ - PackageName: "bindata", - BuildTags: "!dev", - VariableName: "Assets", - Filename: "bindata/bindata.go", - }) - if err != nil { - log.Fatalln(err) - } -} diff --git a/bindata/bindata.go b/bindata/bindata.go deleted file mode 100644 index 6aee93d..0000000 --- a/bindata/bindata.go +++ /dev/null @@ -1,468 +0,0 @@ -// Code generated by vfsgen; DO NOT EDIT. - -// +build !dev - -package bindata - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - pathpkg "path" - "time" -) - -// Assets statically implements the virtual filesystem provided to vfsgen. -var Assets = func() http.FileSystem { - fs := vfsgen۰FS{ - "/": &vfsgen۰DirInfo{ - name: "/", - modTime: time.Date(2020, 11, 23, 13, 13, 50, 745327392, time.UTC), - }, - "/CrawlExecutionStatus_table.template": &vfsgen۰CompressedFileInfo{ - name: "CrawlExecutionStatus_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 927983425, time.UTC), - uncompressedSize: 324, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4c\x90\xc1\x4a\x03\x31\x10\x86\xef\xfb\x14\x3f\x01\x8f\x0d\x6a\xb5\x07\x6f\xe2\x06\xac\x47\xf5\x01\xb6\x36\x53\x0c\xb8\x59\x99\x8c\xf6\x30\xcc\xbb\x4b\xb2\x95\xdd\xcb\x64\xf2\xf1\x87\xef\x27\xaa\x91\x4e\x29\x13\xdc\x73\x78\xec\xc3\xab\xc3\xc6\xac\x03\xa0\x9a\xf2\x2f\x71\x21\x33\xd5\x6f\x4e\x59\x4e\x18\xae\x36\xdb\x9d\xdf\xee\x0a\x96\xe5\xe6\xbe\x8e\xeb\x8a\x6e\xef\x2e\x73\x80\xdb\x47\x07\xf7\x32\x7d\xb4\xf3\x4d\x0e\x42\x0e\xae\x9f\x8e\x65\xbe\xb2\x40\xd2\x58\x59\xc8\x71\x5e\xab\x87\xa9\x90\x98\x75\xaa\x94\x63\x6b\xd2\xad\xe4\xcd\xb7\x96\xc6\x01\x7e\x1f\xe1\x9b\x07\xbe\x69\xe0\xfb\xe9\xf8\x33\x52\x96\xf2\xc4\x87\xf3\x17\x45\x33\xa8\x56\x45\x4b\xb0\xbc\xa7\x91\x56\x2c\xe4\x38\x13\xd5\x73\x92\x4f\xf8\xc0\x3c\xf1\xff\x27\x70\x7d\xdf\xc8\x03\x54\x7d\x4d\x5d\xaa\x2d\x6d\xff\x02\x00\x00\xff\xff\xdf\xc1\xb9\xcc\x44\x01\x00\x00"), - }, - "/CrawlExecutionStatus_wide.template": &vfsgen۰CompressedFileInfo{ - name: "CrawlExecutionStatus_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 927983425, time.UTC), - uncompressedSize: 324, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4c\x90\xc1\x4a\x03\x31\x10\x86\xef\xfb\x14\x3f\x01\x8f\x0d\x6a\xb5\x07\x6f\xe2\x06\xac\x47\xf5\x01\xb6\x36\x53\x0c\xb8\x59\x99\x8c\xf6\x30\xcc\xbb\x4b\xb2\x95\xdd\xcb\x64\xf2\xf1\x87\xef\x27\xaa\x91\x4e\x29\x13\xdc\x73\x78\xec\xc3\xab\xc3\xc6\xac\x03\xa0\x9a\xf2\x2f\x71\x21\x33\xd5\x6f\x4e\x59\x4e\x18\xae\x36\xdb\x9d\xdf\xee\x0a\x96\xe5\xe6\xbe\x8e\xeb\x8a\x6e\xef\x2e\x73\x80\xdb\x47\x07\xf7\x32\x7d\xb4\xf3\x4d\x0e\x42\x0e\xae\x9f\x8e\x65\xbe\xb2\x40\xd2\x58\x59\xc8\x71\x5e\xab\x87\xa9\x90\x98\x75\xaa\x94\x63\x6b\xd2\xad\xe4\xcd\xb7\x96\xc6\x01\x7e\x1f\xe1\x9b\x07\xbe\x69\xe0\xfb\xe9\xf8\x33\x52\x96\xf2\xc4\x87\xf3\x17\x45\x33\xa8\x56\x45\x4b\xb0\xbc\xa7\x91\x56\x2c\xe4\x38\x13\xd5\x73\x92\x4f\xf8\xc0\x3c\xf1\xff\x27\x70\x7d\xdf\xc8\x03\x54\x7d\x4d\x5d\xaa\x2d\x6d\xff\x02\x00\x00\xff\xff\xdf\xc1\xb9\xcc\x44\x01\x00\x00"), - }, - "/CrawlLog_table.template": &vfsgen۰CompressedFileInfo{ - name: "CrawlLog_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 927983425, time.UTC), - uncompressedSize: 237, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x64\xcd\x3d\x6b\x86\x30\x14\xc5\xf1\x3d\x9f\xe2\x70\xe1\x19\x0d\x2d\xcf\x0b\x5d\x45\x85\xba\x15\xdb\xe2\xaa\xe4\x5e\x69\x16\xb5\x49\x2c\x48\xc8\x77\x2f\x91\x0e\x42\x97\x3f\xe7\x4c\xbf\x18\x59\x26\x3b\x0b\xe8\xb5\x29\xeb\xa6\x23\x14\x29\x29\x00\x88\x71\x75\x76\x0e\x13\x86\x4b\x71\x7d\xe8\xeb\xc3\xe3\x52\x3c\xdf\x73\xef\x4f\xb9\x2f\x39\x37\x3f\x80\xfa\xb2\xab\xd0\x32\x81\xde\xc6\xf0\x45\xa0\x4e\xbe\x37\xf1\x41\xf8\xd3\xd9\xe3\x9a\x8f\x7d\x95\x63\x79\x4a\x49\xc5\x28\x33\x1f\x92\x3a\x31\xff\x14\xfd\x07\xe9\x4c\xdd\x78\x80\xee\x47\x67\x5a\x86\xae\xad\x37\xcb\x8f\xb8\x3d\x83\xd0\x67\x2f\x3f\xb3\x38\xce\x22\xf4\x7b\x18\xc3\xe6\xab\x85\x25\x25\xf5\x1b\x00\x00\xff\xff\x44\x9e\xca\xfe\xed\x00\x00\x00"), - }, - "/CrawlLog_wide.template": &vfsgen۰CompressedFileInfo{ - name: "CrawlLog_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 927983425, time.UTC), - uncompressedSize: 878, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x8c\x91\x31\x6f\xdb\x30\x10\x85\xf7\xfc\x8a\x03\x01\x8d\x26\x22\xdb\x51\xd2\xd1\x70\x5c\xd4\x43\x80\x42\x72\x91\x55\xaa\x78\x8a\x39\x98\x54\x8f\x74\x5a\x85\xd0\x7f\x2f\x8e\x4c\x55\x19\x2e\xda\x2e\x0f\x3a\x52\xbc\xf7\xf0\xbe\x10\x14\x76\xda\x20\x88\x4f\xbb\xcd\xe3\xae\x14\xb0\x18\xc7\x1b\x00\x80\x10\xb4\x79\x45\x72\x38\x8e\x21\xf4\xa4\x8d\xef\xa0\xce\x16\xab\x42\xae\x0a\x07\xd9\x22\xbf\x63\xfd\xb0\x64\x7d\x60\x59\x2f\x5d\x0d\xe2\x79\x53\x6e\x61\xaf\x04\x88\xcf\x8d\x3f\x0a\x10\x25\x7e\x3b\xa3\xf3\xa8\xbe\x90\x8e\x63\x7b\x18\x7a\x14\x20\x9e\x1b\x6a\x4b\xec\x90\xdc\xc1\x0a\x36\x21\x74\xe8\x27\xf7\xaf\xa4\x5f\x8e\xbe\x1d\x1a\xc3\x77\xff\x9f\x25\x7e\xaf\x1c\x64\x79\xee\x20\xbb\x8f\x47\x39\x27\xdb\xfd\xc0\xf6\xec\xb5\x35\x29\xde\xbe\xdf\x28\x45\xe8\x5c\x0c\xe5\x7a\x6b\x1c\xa6\x88\x5b\x6b\x3c\x1a\xff\x1e\xb3\x44\xfe\xa3\xd2\x6f\x3c\x7c\x44\xdf\x1e\x9f\xd2\x13\x4f\xc3\x1f\x62\xff\x35\x70\x71\x2b\x8b\xdb\x58\xd8\x9a\x75\x99\xb4\xe0\x74\x95\xb7\xd4\xbc\x60\x89\x5d\x5c\xde\x21\x11\x92\x00\x71\xd0\x27\xac\x7c\x73\xea\x05\x88\x1d\x91\xa5\x0b\xcf\x10\xd0\xa8\x88\xec\x66\xe6\x73\xd5\x8b\x7c\xc7\x24\x19\x54\xba\xad\x41\x32\x80\xbd\x02\xf9\xa8\x5d\x6b\x5f\x91\x06\x26\x06\x72\x0e\x8c\xa7\xd6\x92\xe2\x2e\xd2\x83\x5f\xc4\xa2\xf9\x25\xa3\x7f\xfa\xe7\x77\x32\xd1\x51\x4c\x47\x41\x76\xaf\x20\x5b\xaa\x1a\xe4\xc4\x86\xf3\x4c\x64\xd8\x7d\x02\x03\x72\xc6\x05\x64\xe5\x1b\x7f\x76\x5b\xab\xf8\x5b\xbf\x21\xc8\xc8\x86\xeb\x7a\x8a\x0f\x3d\x69\x74\x97\x5d\x5d\x05\x9d\x03\x91\x0f\x6b\x2e\xe5\x37\x07\x5e\x92\x30\x8c\x23\x84\xe0\xf5\x09\x41\x4e\x38\xe2\xd9\x77\xcd\x85\x45\x2c\xbc\x37\xd5\x41\xa8\x78\x90\x2c\x68\xd4\x3c\xc2\xcf\x00\x00\x00\xff\xff\x5f\x36\x34\xb0\x6e\x03\x00\x00"), - }, - "/JobExecutionStatus_table.template": &vfsgen۰CompressedFileInfo{ - name: "JobExecutionStatus_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 927983425, time.UTC), - uncompressedSize: 246, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4c\x8d\x41\x6b\x84\x30\x10\x85\xef\xf9\x15\x8f\x01\x8f\x06\x5a\x5b\xe9\xb5\xd4\x40\xbd\xb6\xbb\x77\xc5\x89\x10\x58\xe3\x92\x64\xd9\xc3\x90\xff\xbe\x44\x3d\x78\x78\x21\xf3\xcd\xf0\x3e\x11\xb6\xb3\xf3\x16\xf4\x6b\xbe\x3b\xf3\x47\xa8\x73\x56\x00\x20\x72\x0f\xce\xa7\x19\x43\x55\x37\xad\x6e\xda\x88\xea\xed\x33\xa2\xfa\x3a\x52\xbf\x7f\x1c\xef\x00\xea\x99\x40\xff\x69\x4c\x96\x40\xdd\x3a\x45\x02\x5d\x83\x8b\x3b\x0d\x09\xc9\x2d\x65\x65\x3c\xef\xdf\x9c\x95\x88\xf5\xbc\xf9\xd4\x49\x76\x12\x71\xc9\x00\xdd\x33\xf4\xd6\x0d\xdd\xad\xd3\x63\xb1\x3e\xc5\x9f\x30\x3e\x6f\x96\xa1\x8b\xe5\x18\x72\x86\x48\x69\xdf\xce\x43\xba\xb8\xc5\x9e\x98\xf1\xbc\x13\xf5\x0a\x00\x00\xff\xff\xa0\xf2\x67\xf1\xf6\x00\x00\x00"), - }, - "/JobExecutionStatus_wide.template": &vfsgen۰CompressedFileInfo{ - name: "JobExecutionStatus_wide.template", - modTime: time.Date(2020, 11, 23, 7, 48, 38, 665766977, time.UTC), - uncompressedSize: 467, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\x90\x4f\x6f\xc2\x30\x0c\xc5\xef\xfd\x14\x56\x04\x17\xb4\xa6\x62\x0c\x34\x21\xed\x30\x8d\x4a\x83\xe3\xfe\xdc\x5b\x1a\xd3\x45\xa3\x0e\x4a\xd2\x32\x64\xe5\xbb\x4f\x0d\x1d\x83\x83\x13\xe7\x25\xd6\xef\xe5\x31\xa7\x90\x4d\x6a\xe3\x4f\x07\x5c\x42\xad\xfd\x57\xbb\x95\x95\x69\x32\xda\xd3\xb1\xcc\x3a\xd4\x0a\x9b\x92\x28\x2d\x0f\x3a\xab\x4d\xb6\xb3\x86\xbc\x46\x9b\x75\x53\xb9\x31\xdb\xfc\x07\xab\xd6\x6b\x43\xef\xbe\xf4\xad\x9b\x64\x90\x86\x90\x24\xcc\x0a\x77\x9a\x10\xc4\x6b\xfe\xbc\xca\xdf\x44\x94\x01\x00\x98\x35\x75\x68\x1d\x86\xc0\x7c\xb0\x9a\xfc\x0e\x8a\x71\x3a\x5b\xc8\xd9\xc2\xc1\x7f\x33\x9d\x3b\x18\x3f\x0e\x95\xde\x3f\x0c\x6b\x01\x62\xad\x04\x88\x8d\xd9\xc6\xbd\xc7\xa2\x00\xb1\x32\x95\x13\x20\x3e\xad\x76\x67\xd5\x7a\xf0\xba\xe9\xaf\x72\x52\xe7\xb6\x27\x5a\x74\xe8\x43\x48\x98\x91\xd4\x9f\xd7\x8b\x8d\x48\xbe\xc2\xab\xbe\x0a\x90\x6b\x05\x32\x12\x41\x46\x20\xc8\x95\xa9\xda\x06\xc9\xbb\x17\x5b\x1e\xf7\xa8\x40\xf6\xe8\xe1\x10\x02\x30\xf7\xc8\xf8\xdc\xfa\x0f\xdd\xe0\x95\x96\x93\x3a\x2b\x09\xb3\x2d\xa9\x46\x18\x7d\xe3\xe9\x0e\x46\x5d\xb9\x6f\x11\x96\x4f\x20\x2f\xb1\xba\xc8\xbb\xc9\x6a\xda\xfb\x9b\xab\x22\x4e\x0d\x43\x37\x3f\xfa\x0d\x00\x00\xff\xff\xa4\xaa\xad\x86\xd3\x01\x00\x00"), - }, - "/PageLog_table.template": &vfsgen۰CompressedFileInfo{ - name: "PageLog_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 868, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x84\x92\x4f\x8f\xda\x30\x10\xc5\xef\x7c\x8a\x91\xa5\x1c\x33\xda\x38\x62\x11\x7b\x5b\x51\x5a\xa1\xfe\x61\x95\x5d\xb4\x57\x4c\x3c\x0b\x56\x83\x83\x6c\xb3\x2d\x8a\xf2\xdd\xab\xf1\x96\x90\x20\xaa\x1e\x22\xd9\xe3\x68\xe6\xbd\xf7\x9b\xa6\xd1\xf4\x66\x2c\x81\x28\xe6\xcf\xcb\x55\x31\x9b\x0b\x48\xdb\x76\xd4\x34\x1b\x67\xb6\xbb\xb0\x57\x5b\xb2\x41\xb5\x6d\xd3\x1c\x9c\xb1\xe1\x0d\xd6\x49\x7e\x8f\xf9\xbd\x87\x24\x1d\x07\x48\x72\x0d\x49\x36\xe6\xdb\x54\xe2\x54\xf2\x21\x93\x98\xc9\xf3\x7b\x2a\x25\x4a\xe9\xd7\x80\xaf\xca\x95\x0b\x0d\xf8\xd9\xd5\xfb\x99\x2a\x77\x04\xf8\x1c\x54\x38\xfa\x59\xad\x09\xf0\x93\xf1\x65\xfd\x4e\xee\xf4\xa4\xc2\x0e\x70\xe5\x0c\x60\x41\xbe\x3e\xba\x92\x5e\x4e\x07\xe2\x9b\xd5\xe4\xd4\xa6\x22\xc0\xef\x66\x1f\xab\x2c\xcc\x91\xa7\x10\x35\x93\xd5\x51\xfd\xe8\x62\x6b\xb9\x7a\xf9\xb6\xf8\xf1\x75\xe8\xaa\x3c\x29\xdb\xb7\x04\x29\xcb\x9e\xdc\x61\x36\xb9\x63\xa9\xff\xed\xfa\xf4\xf8\xa5\x0b\xca\xd8\x77\x72\x9e\x06\x11\xa5\x5d\x46\xdd\x21\x93\xb9\x5f\x83\x78\x7d\x2c\x66\xb0\xd0\x02\xc4\xfc\x37\x95\xc7\x60\x6a\x1b\x6f\x2b\x67\xc4\x70\xec\xa6\xae\x34\x57\x4e\x54\x55\xf5\xaf\x9b\x04\xba\xe6\xfd\x7c\x7b\x7d\x63\x8c\xc3\xae\x3d\xb5\xff\x24\x7c\x8e\xdd\x3f\x40\x22\xc7\x28\x19\x2f\x7f\xb9\xbf\xb0\xfe\xb0\x14\x29\x63\x2c\x49\x89\xe3\xa1\xc1\x08\x59\x80\x28\xc8\x0b\x10\x8c\x95\x7d\x16\x8b\x58\x2a\x99\x5e\x3c\x59\xfe\x97\x79\x5e\xf9\x77\xca\x6e\xe9\xb2\x03\xfc\x18\x68\x7f\xa8\x54\x18\x6c\x6b\x84\x75\x46\x74\xc3\xdf\x35\xeb\x24\xcd\xa6\x13\x16\xba\x3c\x86\xca\xd8\x9f\xfe\xe1\xf6\xdc\xbf\xcf\xc3\xb1\xdd\x36\x9d\xa7\x5e\x2f\xc8\xe5\xd7\x8f\x15\xc1\xb6\x1d\xfd\x09\x00\x00\xff\xff\x0e\x9d\x9e\xc9\x64\x03\x00\x00"), - }, - "/PageLog_wide.template": &vfsgen۰CompressedFileInfo{ - name: "PageLog_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 868, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x84\x92\x4f\x8f\xda\x30\x10\xc5\xef\x7c\x8a\x91\xa5\x1c\x33\xda\x38\x62\x11\x7b\x5b\x51\x5a\xa1\xfe\x61\x95\x5d\xb4\x57\x4c\x3c\x0b\x56\x83\x83\x6c\xb3\x2d\x8a\xf2\xdd\xab\xf1\x96\x90\x20\xaa\x1e\x22\xd9\xe3\x68\xe6\xbd\xf7\x9b\xa6\xd1\xf4\x66\x2c\x81\x28\xe6\xcf\xcb\x55\x31\x9b\x0b\x48\xdb\x76\xd4\x34\x1b\x67\xb6\xbb\xb0\x57\x5b\xb2\x41\xb5\x6d\xd3\x1c\x9c\xb1\xe1\x0d\xd6\x49\x7e\x8f\xf9\xbd\x87\x24\x1d\x07\x48\x72\x0d\x49\x36\xe6\xdb\x54\xe2\x54\xf2\x21\x93\x98\xc9\xf3\x7b\x2a\x25\x4a\xe9\xd7\x80\xaf\xca\x95\x0b\x0d\xf8\xd9\xd5\xfb\x99\x2a\x77\x04\xf8\x1c\x54\x38\xfa\x59\xad\x09\xf0\x93\xf1\x65\xfd\x4e\xee\xf4\xa4\xc2\x0e\x70\xe5\x0c\x60\x41\xbe\x3e\xba\x92\x5e\x4e\x07\xe2\x9b\xd5\xe4\xd4\xa6\x22\xc0\xef\x66\x1f\xab\x2c\xcc\x91\xa7\x10\x35\x93\xd5\x51\xfd\xe8\x62\x6b\xb9\x7a\xf9\xb6\xf8\xf1\x75\xe8\xaa\x3c\x29\xdb\xb7\x04\x29\xcb\x9e\xdc\x61\x36\xb9\x63\xa9\xff\xed\xfa\xf4\xf8\xa5\x0b\xca\xd8\x77\x72\x9e\x06\x11\xa5\x5d\x46\xdd\x21\x93\xb9\x5f\x83\x78\x7d\x2c\x66\xb0\xd0\x02\xc4\xfc\x37\x95\xc7\x60\x6a\x1b\x6f\x2b\x67\xc4\x70\xec\xa6\xae\x34\x57\x4e\x54\x55\xf5\xaf\x9b\x04\xba\xe6\xfd\x7c\x7b\x7d\x63\x8c\xc3\xae\x3d\xb5\xff\x24\x7c\x8e\xdd\x3f\x40\x22\xc7\x28\x19\x2f\x7f\xb9\xbf\xb0\xfe\xb0\x14\x29\x63\x2c\x49\x89\xe3\xa1\xc1\x08\x59\x80\x28\xc8\x0b\x10\x8c\x95\x7d\x16\x8b\x58\x2a\x99\x5e\x3c\x59\xfe\x97\x79\x5e\xf9\x77\xca\x6e\xe9\xb2\x03\xfc\x18\x68\x7f\xa8\x54\x18\x6c\x6b\x84\x75\x46\x74\xc3\xdf\x35\xeb\x24\xcd\xa6\x13\x16\xba\x3c\x86\xca\xd8\x9f\xfe\xe1\xf6\xdc\xbf\xcf\xc3\xb1\xdd\x36\x9d\xa7\x5e\x2f\xc8\xe5\xd7\x8f\x15\xc1\xb6\x1d\xfd\x09\x00\x00\xff\xff\x0e\x9d\x9e\xc9\x64\x03\x00\x00"), - }, - "/browserConfig_table.template": &vfsgen۰CompressedFileInfo{ - name: "browserConfig_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 388, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x7c\x8f\x41\x4b\xc3\x40\x10\x85\xef\xf9\x15\xc3\x40\x8e\x59\x88\xb5\xc5\x6b\xd5\x42\x03\x46\xc4\x2a\xbd\x66\xe9\x4c\xd2\x39\x74\xb7\xec\xae\x6d\x75\xd9\xff\x2e\xdb\x8a\xa8\x04\x2f\xef\xf2\x0d\xf3\xbe\x17\x23\x71\x2f\x86\x01\x97\x8b\xf9\xfd\xe2\x19\xa1\x4a\xa9\x00\x00\x88\x71\xef\xc4\x84\x1e\xba\xb2\x9a\xcc\xd4\x64\xe6\xa1\xac\xea\xa9\x87\xb2\xbe\xc9\x51\xe7\xb8\xba\x44\x07\xd8\x10\x02\x3e\xea\x1d\x23\xe0\xab\x67\x37\x1f\xd8\x04\x04\x5c\x8b\x21\x7b\x5c\xc9\x47\x06\x4f\x7a\xe0\x17\xd9\xb1\x7d\xcb\xa8\xd5\xa7\xc6\xe8\x4d\x90\x83\x84\x77\x4c\xa9\x88\x91\x0d\x9d\x05\x8a\x1f\xed\x7f\x9b\xaf\x09\x4e\xe5\x94\x72\xf1\xe1\x1c\x1d\xa8\x86\x40\xb5\x1c\xb4\xca\x06\xa0\x56\x7b\xde\xa8\x5b\x67\x8f\x9e\xdd\x9d\x35\xbd\x0c\xea\xdb\x69\x94\x5e\x2c\xd7\x42\x61\xfb\x0f\x5f\xb2\x0c\xdb\xf1\x07\x79\xd9\x83\xd5\xf4\xb5\xae\xf5\xa3\x57\xbf\x16\xe7\xd3\xd6\xa7\x54\x7c\x06\x00\x00\xff\xff\xbc\xab\x2d\x1b\x84\x01\x00\x00"), - }, - "/browserConfig_wide.template": &vfsgen۰CompressedFileInfo{ - name: "browserConfig_wide.template", - modTime: time.Date(2020, 11, 23, 7, 48, 38, 573763871, time.UTC), - uncompressedSize: 466, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x7c\x8f\x51\x4b\xfb\x30\x14\xc5\xdf\xfb\x29\x2e\x81\xbe\x0c\x9a\xd0\xff\xfe\x1b\xe2\xdb\xd4\xc1\x06\x56\xc5\x29\x7b\x5d\xd6\xdc\x65\x57\x6c\x52\xda\xac\xdb\x0c\xf9\xee\x92\x4d\x07\x4a\xf1\xe5\x10\x72\x0e\xf7\x9c\x9f\xf7\x19\x88\x81\xb6\xee\x58\xe3\x35\x68\x72\xdb\xdd\x9a\x97\xb6\x12\xe6\xdd\xec\xa5\xe8\x90\x14\x56\xd2\x98\x4c\xd6\x24\xb4\x15\xa5\x35\x1b\xd2\xa2\xcb\xf9\xed\xe9\xf5\xb8\x7e\xc3\xd2\x0d\x04\x64\x21\x24\x89\xf7\x0a\x37\x64\x10\xd8\x6c\x3a\xb9\x9b\x3e\xb3\xd3\x37\x00\x80\xf7\x75\x43\xc6\x6d\x60\x95\x66\xc3\x31\x1f\x8e\x5b\x48\xb3\x7c\xd4\x42\x9a\x5f\x45\xc9\xa3\xfc\x3b\xcb\x0a\xd8\x5c\x31\x60\x0f\xb2\x42\x06\xec\xb5\xc5\x66\xa2\xd1\x38\x06\x6c\x49\x46\xd9\xfd\x82\x3e\xa2\xf1\x24\x35\xbe\x50\x85\x76\x17\xad\x42\x1e\xe6\x46\x96\x8e\x3a\x72\x47\x16\x42\xe2\x3d\x1a\xf5\xbd\xeb\xd2\xfe\xbb\xf9\xbf\x82\x43\x3a\x52\xb1\xb8\x3b\xc9\x0a\xf8\x5c\x01\x2f\xd0\x49\x1e\x17\x00\x5f\xd4\x58\xf2\x9b\xc6\xee\x5b\x6c\xce\xd4\xfc\xb2\xa9\xd7\x3d\xaf\x5c\x92\x72\xdb\x3f\xfc\x19\x92\xde\xf6\x1f\x88\x64\xf7\x56\xaa\x2f\xba\xa2\xed\x4d\xfd\x20\x8e\xd1\xa2\x0d\x21\xf9\x0c\x00\x00\xff\xff\x93\x9c\x1f\xb5\xd2\x01\x00\x00"), - }, - "/browserScript_table.template": &vfsgen۰CompressedFileInfo{ - name: "browserScript_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/browserScript_wide.template": &vfsgen۰CompressedFileInfo{ - name: "browserScript_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/collection_table.template": &vfsgen۰CompressedFileInfo{ - name: "collection_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/collection_wide.template": &vfsgen۰CompressedFileInfo{ - name: "collection_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/completion.sh": &vfsgen۰CompressedFileInfo{ - name: "completion.sh", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 2363, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xdc\x55\xcd\x6f\x9c\x46\x14\x3f\xc3\x5f\xf1\x34\xa5\x02\x5a\xb3\xbb\xce\xb1\x0e\x6d\xd3\xc8\xaa\x2a\xb5\x4d\x94\x1e\xaa\x0a\xaf\x11\x3b\xbc\x5d\x4f\x32\xcc\xe0\x99\xc1\xae\x43\xf9\xdf\xab\x61\xd8\x5d\xd8\x0f\xf7\x50\x9f\x72\x01\xde\xf7\xe7\xef\x91\xe7\x0f\xc8\x4a\xac\x0a\x21\xa8\xe1\x79\x5d\x28\x8d\xf9\x06\x4d\x14\x43\xeb\x03\x00\x94\x48\x79\xa1\x10\x12\xf5\x06\x4c\xb1\xe2\xa8\xd3\x08\x32\x82\xc2\x30\xf3\x44\x96\x29\xa1\x52\xac\xd9\x26\xa7\xaa\x78\xe4\x79\xcf\x66\xa8\x09\x64\x64\xa5\xe4\xa3\x46\x35\xd2\x19\x38\xb9\x23\x35\x81\x9b\x3e\x04\x40\x46\x7a\x73\xc7\x3f\x72\xba\x53\xcf\xc8\x46\xc9\xa6\x3e\x52\xb8\x93\xda\xe4\xbd\x68\xac\xfb\x51\xae\x8e\x34\x3f\xca\xd5\x24\x6c\x2d\x39\x33\x28\x50\xeb\x91\xea\x9e\x39\x76\xa7\x24\x47\xab\x54\xca\x8d\x25\x35\xbd\xc3\xb2\x71\xac\x49\x88\xad\xe0\x54\x95\x9a\x2a\x56\x9b\x13\x1d\x71\x82\x3e\x8e\x46\x2c\x47\x1a\x96\xd4\x04\xe2\xde\x07\x97\xb4\xe0\x6e\x0c\x69\xd0\xba\x71\x64\xc1\xe5\xb2\x83\xfb\x06\xd5\x13\x18\xac\x6a\x5e\x18\x84\xc9\x50\x65\x63\x7c\xdf\x63\x6b\xc8\x20\xf9\x0c\x24\xe8\xed\x08\x2c\xaf\xc0\xdc\xa1\x18\x92\x53\x68\x1a\x25\x60\xd1\x93\x6b\xe6\xf7\xef\xde\x6b\x4a\xd4\xac\x37\x89\xc2\x21\x66\x17\xc6\x64\xeb\x50\x00\x09\x68\xa3\x0e\xdd\x0d\x96\x41\xdb\x7f\x74\xb3\x15\x9a\x47\x44\x61\x5d\xd0\x46\x75\xe1\x05\x0c\x5f\x9f\xc3\x0b\x68\x99\x28\xf1\xef\xef\x42\x56\x86\x5d\x4c\xb6\x19\x78\x87\x3e\xa4\x2a\x51\xfd\xf4\x14\x4d\xd4\x67\x35\x6f\xe8\xa7\xc8\x12\x83\xe9\xb6\x09\x29\x69\xdb\x5a\x31\x61\xb8\x80\x19\x2b\xbb\xce\x89\xd9\x1a\xaa\xa2\x5e\x33\x8e\x90\x98\xa3\x46\xc1\x6b\x78\x1d\x4d\xb8\xa0\xb0\x96\xca\x0c\x1d\xde\x65\x43\x20\x91\xfb\x7e\x27\x86\x04\xed\x96\xb2\x32\xfd\x6a\x01\xaf\xbe\x9f\x97\xf8\x30\x17\x0d\xe7\x10\x4f\xba\x03\xe3\x14\xde\xbe\xfb\xed\xfd\x87\xeb\xf7\xbf\xfe\xe5\x62\x53\x59\xd5\x1b\x14\x90\xfc\x09\x24\x88\xa0\xaf\x60\x0d\xe1\xd7\xf7\x10\xda\xe8\x87\x09\x67\x3f\x2e\x3b\x02\x31\x81\x24\xd9\x4e\xe2\x1f\x28\x1e\x3f\x41\x38\x87\x39\xb4\xce\x1e\xc8\x0d\x21\xc1\xc2\x3e\xa1\x83\xf9\x6d\x76\x0b\xcb\x6f\x83\xbd\x38\x58\x40\x17\x0e\x3b\xb6\x66\x7e\xe7\xfb\x07\x77\x61\x83\x26\x57\xa8\x65\xa3\x28\xee\x4e\x83\x5d\x80\x0c\x82\xf6\x2b\x21\x1b\xa1\x6d\x22\x90\xe0\x3d\x2c\x60\xb9\x3c\xa8\x76\x58\xae\xcb\x6d\x00\xfb\x3a\x77\x79\x20\x68\x9d\xbf\xa9\xe3\xcb\x65\x37\x0e\xfa\xc3\x7f\x84\x5a\x9c\xaf\xa5\x9f\xdf\x71\x35\x0e\x5b\x47\xc0\xf9\xff\x2b\x93\x24\x76\xa4\x5f\xd2\x36\xd0\x46\x1b\x59\xe5\xeb\x46\xd0\x5d\xfb\x68\xa1\x11\x82\x96\x17\xda\xe4\x54\x56\x55\x21\xca\x0e\xd8\xbe\xc4\xc3\x75\x8a\x77\x92\x13\xcb\x30\x5e\xb7\x89\x9e\x9b\xee\x84\x75\x75\x75\x3a\x86\x9b\x81\x9b\x76\xec\x7b\x9e\xf7\xec\x1a\x58\x85\xc1\xb9\xe7\x79\x23\x9f\xdf\xc4\xa7\xa2\xa1\x2e\xe8\x19\x98\x88\xa2\x7a\x09\x88\x78\x6e\x1f\xad\x9d\xef\xd9\x67\xfa\x3c\x2e\x86\x5f\xc3\x70\x82\xce\x1f\xc2\x0a\x4d\x31\xb3\x39\x6e\xef\xe1\x4b\xee\xbd\x85\x2f\x09\x6c\x76\xa4\xff\x33\xdc\xba\x1b\x3f\xdc\xc3\xe9\xc1\x84\xe9\xc5\x3c\x05\x0f\xef\x79\x6c\xec\xc1\x70\x23\x9e\x41\x83\x03\x40\xf2\x00\xbf\xfc\xfc\xfb\xbb\x0f\xd7\x6f\xdf\xfc\x71\x9d\x5e\x5a\xba\x4e\x07\x88\x84\x6d\x9d\xa6\xba\x59\x69\xa3\xa2\x60\x71\xb1\xb8\xe0\x28\x36\xe6\x2e\xaa\xe3\xb8\x3b\x8f\x8a\x7f\x03\x00\x00\xff\xff\x6b\x7f\x63\x1f\x3b\x09\x00\x00"), - }, - "/crawlConfig_table.template": &vfsgen۰CompressedFileInfo{ - name: "crawlConfig_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/crawlConfig_wide.template": &vfsgen۰CompressedFileInfo{ - name: "crawlConfig_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/crawlEntity_table.template": &vfsgen۰CompressedFileInfo{ - name: "crawlEntity_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 122, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xaa\xae\x4e\x49\x4d\xcb\xcc\x4b\x55\x50\xf2\x70\x75\x74\x71\x0d\x52\x52\xd0\xad\xad\xe5\x52\x50\x50\x50\xa8\xae\x2e\x28\xca\xcc\x2b\x49\x53\x48\x50\xd5\x35\x36\xd3\x33\x36\x2b\x56\x50\xd5\x35\x31\xd0\x33\x31\x28\x4e\x50\x50\xf2\x4c\x51\x52\x50\xf2\x4b\xcc\x4d\x55\xaa\xad\xe5\xaa\xae\x4e\xcd\x4b\x01\x6b\xe4\x42\xd2\x85\xaa\x43\xcf\x33\x45\x41\xcf\x37\xb5\x24\x51\x0f\xa4\xab\xb6\x96\x0b\x10\x00\x00\xff\xff\x58\xd6\xbb\x4d\x7a\x00\x00\x00"), - }, - "/crawlEntity_wide.template": &vfsgen۰CompressedFileInfo{ - name: "crawlEntity_wide.template", - modTime: time.Date(2020, 11, 23, 7, 48, 38, 610765120, time.UTC), - uncompressedSize: 250, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x64\xcd\xcf\x4a\xc3\x40\x10\x06\xf0\x7b\x9e\xe2\x63\xa1\x97\xc2\xce\x46\x5a\x72\xf0\x26\x5a\xb0\xe0\x1f\xf0\x09\xba\xc9\x4e\xe2\x4a\x33\x09\x66\xad\xc8\xb8\xef\x2e\x5d\xff\x5c\xbc\x7d\x33\xc3\xfc\x3e\x55\x0b\xb7\x1e\xa6\xf4\x31\xf3\x25\x86\x98\x9e\xdf\x5a\xea\xa6\xd1\xc9\x51\xde\xbd\x3b\x71\x0c\x3c\x7a\x11\xeb\xe7\xe8\x86\xc9\x75\x93\xf4\x71\x70\xa7\x0b\xba\x2e\xe9\xb1\x7d\xe1\x2e\xad\x1d\x6c\xce\x55\xa5\x1a\xb8\x8f\xc2\x30\xb7\xbb\xab\x9b\xdd\x93\x29\x6b\x00\x50\x9d\x5f\xa3\xa4\x1e\x87\x95\xdd\x34\xb4\x69\x16\xac\xec\xb6\xa6\x6d\x7d\x0e\x4d\xbd\x1c\x60\xf6\xc1\xc0\x3c\xf8\x91\x0d\xcc\x9d\x6f\xf9\xb8\x98\x9c\x2b\x55\x96\xf0\xcb\x17\xe4\xfb\x06\xba\xe7\xe4\xa9\x0c\xf8\xc4\x1f\xff\x8f\xa6\xa2\xd3\x3e\xfc\x7c\x9c\x1b\x72\xae\xbe\x02\x00\x00\xff\xff\x03\xd6\xd8\x7c\xfa\x00\x00\x00"), - }, - "/crawlHostGroupConfig_table.template": &vfsgen۰CompressedFileInfo{ - name: "crawlHostGroupConfig_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/crawlHostGroupConfig_wide.template": &vfsgen۰CompressedFileInfo{ - name: "crawlHostGroupConfig_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/crawlJob_table.template": &vfsgen۰CompressedFileInfo{ - name: "crawlJob_table.template", - modTime: time.Date(2020, 11, 23, 13, 13, 50, 741327257, time.UTC), - uncompressedSize: 348, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x74\xce\xc1\x4a\xc3\x40\x10\x06\xe0\x7b\x9e\x62\x58\xc8\xa5\x90\x59\xd2\x6a\x11\x6f\xda\x14\xac\x58\x05\xfb\x02\xd9\x24\x93\x38\x92\xec\x86\xec\x36\xb5\x2c\xfb\xee\x92\x4a\x44\x84\x5e\x86\x9f\x7f\x0e\xdf\xef\x7d\x02\x72\xd1\x18\x77\xee\xe9\x1e\x1a\x76\x1f\xc7\x02\x4b\xd3\x49\xdd\xea\x93\x92\x23\x71\x45\x9d\xd2\x3a\x51\x3d\xcb\xc6\xc8\xd2\xe8\x9a\x1b\x39\xa6\xb8\xb9\xa4\xb7\xe2\x93\x4a\xb7\x90\x90\x84\x10\x45\xde\x57\x54\xb3\x26\x10\x4f\xdb\x87\x6c\xfb\x2e\x2e\x35\x00\x80\xf7\xfd\xc0\xda\xd5\x90\xc7\xc9\x6a\x8d\xab\xb5\x85\x38\x49\x6f\x2d\xc4\x77\x16\xe2\xf4\x66\x3a\x4b\x9b\x83\xd8\x55\x02\xc4\xab\xea\x48\x80\xc8\xd8\xaa\xa2\xa5\xa9\xd9\xab\xaf\xc7\xb3\x23\xfb\x13\xb3\xe3\xa0\x1c\x1b\x2d\x42\x88\xbc\x27\x5d\xcd\xfc\x2f\xf2\x17\x18\x27\x60\x3a\xcb\x31\x07\xdc\x55\x80\x7b\x72\x0a\x27\x04\xf0\xd0\x53\x89\x9b\x41\x9d\xda\x67\x53\xe0\x2c\xfe\xef\x5f\xb8\x63\x67\x71\x5e\x71\xfd\x3d\x2f\x3b\x84\x10\x7d\x07\x00\x00\xff\xff\x6b\xb7\x3d\x0a\x5c\x01\x00\x00"), - }, - "/crawlJob_wide.template": &vfsgen۰CompressedFileInfo{ - name: "crawlJob_wide.template", - modTime: time.Date(2020, 11, 23, 13, 13, 50, 704326004, time.UTC), - uncompressedSize: 348, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x74\xce\xc1\x4a\xc3\x40\x10\x06\xe0\x7b\x9e\x62\x58\xc8\xa5\x90\x59\xd2\x6a\x11\x6f\xda\x14\xac\x58\x05\xfb\x02\xd9\x24\x93\x38\x92\xec\x86\xec\x36\xb5\x2c\xfb\xee\x92\x4a\x44\x84\x5e\x86\x9f\x7f\x0e\xdf\xef\x7d\x02\x72\xd1\x18\x77\xee\xe9\x1e\x1a\x76\x1f\xc7\x02\x4b\xd3\x49\xdd\xea\x93\x92\x23\x71\x45\x9d\xd2\x3a\x51\x3d\xcb\xc6\xc8\xd2\xe8\x9a\x1b\x39\xa6\xb8\xb9\xa4\xb7\xe2\x93\x4a\xb7\x90\x90\x84\x10\x45\xde\x57\x54\xb3\x26\x10\x4f\xdb\x87\x6c\xfb\x2e\x2e\x35\x00\x80\xf7\xfd\xc0\xda\xd5\x90\xc7\xc9\x6a\x8d\xab\xb5\x85\x38\x49\x6f\x2d\xc4\x77\x16\xe2\xf4\x66\x3a\x4b\x9b\x83\xd8\x55\x02\xc4\xab\xea\x48\x80\xc8\xd8\xaa\xa2\xa5\xa9\xd9\xab\xaf\xc7\xb3\x23\xfb\x13\xb3\xe3\xa0\x1c\x1b\x2d\x42\x88\xbc\x27\x5d\xcd\xfc\x2f\xf2\x17\x18\x27\x60\x3a\xcb\x31\x07\xdc\x55\x80\x7b\x72\x0a\x27\x04\xf0\xd0\x53\x89\x9b\x41\x9d\xda\x67\x53\xe0\x2c\xfe\xef\x5f\xb8\x63\x67\x71\x5e\x71\xfd\x3d\x2f\x3b\x84\x10\x7d\x07\x00\x00\xff\xff\x6b\xb7\x3d\x0a\x5c\x01\x00\x00"), - }, - "/crawlScheduleConfig_table.template": &vfsgen۰CompressedFileInfo{ - name: "crawlScheduleConfig_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/crawlScheduleConfig_wide.template": &vfsgen۰CompressedFileInfo{ - name: "crawlScheduleConfig_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/json.template": &vfsgen۰FileInfo{ - name: "json.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - content: []byte("\x7b\x7b\x70\x72\x65\x74\x74\x79\x4a\x73\x6f\x6e\x20\x2e\x7d\x7d\x0a"), - }, - "/politenessConfig_table.template": &vfsgen۰CompressedFileInfo{ - name: "politenessConfig_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/politenessConfig_wide.template": &vfsgen۰CompressedFileInfo{ - name: "politenessConfig_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 154, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xb1\x0a\xc2\x30\x10\x87\xf1\x3d\x4f\xf1\xe7\xa0\x63\x8e\xda\x42\x71\x15\x2c\x58\x50\x07\x9f\xa0\x91\x5c\xa1\xa0\x41\x4c\xb7\xe3\xde\x5d\x12\x1c\x74\xfb\x96\xef\xa7\x1a\x65\x59\x93\x80\x4e\xe3\xe1\x38\xde\x08\xde\xcc\x01\x80\xea\xeb\xbd\xa6\x6d\xc1\xdc\xf8\x7e\xe0\x7e\xc8\x68\x7c\xd7\x72\xd7\x96\xd8\xed\xf3\x0c\x9a\x22\x81\xae\xe1\x29\x04\x3a\x87\xbb\x3c\x32\x99\x39\x55\x49\xb1\x3a\xee\x07\xf9\x07\xea\xcf\x53\x04\x5f\x64\x0b\x5c\x8c\x6f\x56\xc7\xcc\x7d\x02\x00\x00\xff\xff\x9a\xc7\x3e\x46\x9a\x00\x00\x00"), - }, - "/roleMapping_table.template": &vfsgen۰CompressedFileInfo{ - name: "roleMapping_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 207, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xcd\xca\x82\x40\x14\x87\xf1\xfd\x5c\xc5\x9f\x03\x2e\x3d\xf8\x01\xe2\xf6\x85\x57\x48\xc8\x16\x76\x03\x4e\xcd\x31\x04\x9b\x86\x94\x36\xc3\xdc\x7b\x38\x14\xd5\xea\x79\x56\x3f\xef\x8d\x8c\x93\x15\xd0\xae\xf9\xfb\x6f\x7a\x42\x1a\x82\x02\x00\xef\xdd\x7d\xb2\xeb\x88\x21\x49\xcb\x8a\xcb\x6a\x41\x92\x16\x19\x17\xd9\x36\x79\xcd\x79\xfd\x99\x01\xd4\x1a\x02\x1d\xf4\x55\x08\xb4\xd7\x27\x99\x17\x02\xf5\xb7\x59\x28\x04\xe5\xbd\x58\x13\x69\xf5\xe5\xfe\x9a\x51\x7a\xbc\x3b\x80\x5b\x03\xee\x64\xd5\xbc\xa9\xaf\x8d\x32\xf8\xe8\xe4\xcc\x1b\xde\x69\xe7\x26\x7b\x89\x1f\x82\x7a\x06\x00\x00\xff\xff\xc0\x98\xe6\xb7\xcf\x00\x00\x00"), - }, - "/roleMapping_wide.template": &vfsgen۰CompressedFileInfo{ - name: "roleMapping_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 207, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x54\xcc\xcd\xca\x82\x40\x14\x87\xf1\xfd\x5c\xc5\x9f\x03\x2e\x3d\xf8\x01\xe2\xf6\x85\x57\x48\xc8\x16\x76\x03\x4e\xcd\x31\x04\x9b\x86\x94\x36\xc3\xdc\x7b\x38\x14\xd5\xea\x79\x56\x3f\xef\x8d\x8c\x93\x15\xd0\xae\xf9\xfb\x6f\x7a\x42\x1a\x82\x02\x00\xef\xdd\x7d\xb2\xeb\x88\x21\x49\xcb\x8a\xcb\x6a\x41\x92\x16\x19\x17\xd9\x36\x79\xcd\x79\xfd\x99\x01\xd4\x1a\x02\x1d\xf4\x55\x08\xb4\xd7\x27\x99\x17\x02\xf5\xb7\x59\x28\x04\xe5\xbd\x58\x13\x69\xf5\xe5\xfe\x9a\x51\x7a\xbc\x3b\x80\x5b\x03\xee\x64\xd5\xbc\xa9\xaf\x8d\x32\xf8\xe8\xe4\xcc\x1b\xde\x69\xe7\x26\x7b\x89\x1f\x82\x7a\x06\x00\x00\xff\xff\xc0\x98\xe6\xb7\xcf\x00\x00\x00"), - }, - "/seed_table.template": &vfsgen۰CompressedFileInfo{ - name: "seed_table.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 207, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4c\xcd\x41\x0b\x82\x40\x10\x05\xe0\xfb\xfe\x8a\xc7\x80\x47\x87\xc0\x10\xaf\x81\x42\x1e\xea\xa0\x74\xd7\xda\x11\x16\x6c\x11\x5d\x82\x58\xf6\xbf\x47\x2b\x4b\x5d\x86\xe1\xf1\xf8\x9e\xf7\x5a\x26\x63\x05\x74\x6e\x4e\x75\xd3\x11\xf2\x10\x14\x00\x78\xbf\xac\xc6\xba\x09\x43\x96\x17\x25\x17\xe5\x86\x2c\x3f\x1e\xbe\x77\xff\xab\x6d\x00\xb5\x9a\x40\xb7\x75\x26\x50\x63\x9d\x71\xef\x18\xd4\x66\x1b\xef\xb3\x68\x0a\x41\x79\x2f\x56\x47\x55\xfd\x91\x89\xe3\x28\x26\xbf\x7a\x0d\xe0\x56\x83\x2f\xe2\x46\xbe\x8e\x4f\x01\xf7\x8b\x3c\xb8\x17\xd1\xbc\x0f\x74\x32\xc5\xca\x2f\x4f\x6b\x21\xa8\x4f\x00\x00\x00\xff\xff\xe7\x39\x7e\x79\xcf\x00\x00\x00"), - }, - "/seed_wide.template": &vfsgen۰CompressedFileInfo{ - name: "seed_wide.template", - modTime: time.Date(2020, 3, 17, 21, 41, 8, 928983452, time.UTC), - uncompressedSize: 335, - - compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\x4c\x8f\x41\x4b\xc4\x30\x10\x85\xef\xfd\x15\x8f\x81\xbd\xd9\xb0\xb0\x52\xf6\x2a\x6c\xc1\x2e\xae\x87\x2e\x9e\x44\x48\x6a\x26\x10\xa9\x69\xd9\x06\x41\x66\xf3\xdf\xa5\xe9\x56\xbd\x0c\x93\xc7\x17\xde\x37\x22\x96\x9d\x0f\x0c\x7a\xac\x1f\x0e\x75\x4b\x28\x53\x2a\x00\x40\x64\xbc\xf8\x10\x1d\xf4\xa6\xdc\x55\x6a\x57\x4d\xd8\x94\xf7\xdb\x79\x2e\xfb\x7e\x1e\xd5\x1a\x68\x50\x63\x09\xf4\x72\xe9\x09\x54\x87\xe8\xe3\x77\x0e\x0e\x7e\x32\x5d\xcf\xf3\xfa\x64\x3a\xee\x27\x02\x1d\x87\xae\xb1\x94\x52\x21\xc2\xc1\xe6\xca\xe2\xd6\xb7\x30\x50\x27\x8e\x46\xe5\x07\xae\xf8\x35\x59\x2d\x54\x16\x59\xb5\xf6\x5f\xd9\x44\x55\xdb\x49\x43\x35\xf6\xf6\xfb\xd9\x7c\x32\xd4\x79\xe4\x77\x75\x66\xb6\x6a\x91\x6a\xd9\x65\xe4\x2f\x5f\x0d\xb3\x87\x88\xc6\xab\x4e\x49\xc4\xf5\x26\x9e\xcc\xf8\x9f\x3c\x0e\x5d\xcb\x6e\xb9\xf4\x8a\x8f\xc1\x07\xe8\x3b\x64\x5a\xbf\xe9\x94\x8a\x9f\x00\x00\x00\xff\xff\xc2\x01\x68\xdd\x4f\x01\x00\x00"), - }, - } - fs["/"].(*vfsgen۰DirInfo).entries = []os.FileInfo{ - fs["/CrawlExecutionStatus_table.template"].(os.FileInfo), - fs["/CrawlExecutionStatus_wide.template"].(os.FileInfo), - fs["/CrawlLog_table.template"].(os.FileInfo), - fs["/CrawlLog_wide.template"].(os.FileInfo), - fs["/JobExecutionStatus_table.template"].(os.FileInfo), - fs["/JobExecutionStatus_wide.template"].(os.FileInfo), - fs["/PageLog_table.template"].(os.FileInfo), - fs["/PageLog_wide.template"].(os.FileInfo), - fs["/browserConfig_table.template"].(os.FileInfo), - fs["/browserConfig_wide.template"].(os.FileInfo), - fs["/browserScript_table.template"].(os.FileInfo), - fs["/browserScript_wide.template"].(os.FileInfo), - fs["/collection_table.template"].(os.FileInfo), - fs["/collection_wide.template"].(os.FileInfo), - fs["/completion.sh"].(os.FileInfo), - fs["/crawlConfig_table.template"].(os.FileInfo), - fs["/crawlConfig_wide.template"].(os.FileInfo), - fs["/crawlEntity_table.template"].(os.FileInfo), - fs["/crawlEntity_wide.template"].(os.FileInfo), - fs["/crawlHostGroupConfig_table.template"].(os.FileInfo), - fs["/crawlHostGroupConfig_wide.template"].(os.FileInfo), - fs["/crawlJob_table.template"].(os.FileInfo), - fs["/crawlJob_wide.template"].(os.FileInfo), - fs["/crawlScheduleConfig_table.template"].(os.FileInfo), - fs["/crawlScheduleConfig_wide.template"].(os.FileInfo), - fs["/json.template"].(os.FileInfo), - fs["/politenessConfig_table.template"].(os.FileInfo), - fs["/politenessConfig_wide.template"].(os.FileInfo), - fs["/roleMapping_table.template"].(os.FileInfo), - fs["/roleMapping_wide.template"].(os.FileInfo), - fs["/seed_table.template"].(os.FileInfo), - fs["/seed_wide.template"].(os.FileInfo), - } - - return fs -}() - -type vfsgen۰FS map[string]interface{} - -func (fs vfsgen۰FS) Open(path string) (http.File, error) { - path = pathpkg.Clean("/" + path) - f, ok := fs[path] - if !ok { - return nil, &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist} - } - - switch f := f.(type) { - case *vfsgen۰CompressedFileInfo: - gr, err := gzip.NewReader(bytes.NewReader(f.compressedContent)) - if err != nil { - // This should never happen because we generate the gzip bytes such that they are always valid. - panic("unexpected error reading own gzip compressed bytes: " + err.Error()) - } - return &vfsgen۰CompressedFile{ - vfsgen۰CompressedFileInfo: f, - gr: gr, - }, nil - case *vfsgen۰FileInfo: - return &vfsgen۰File{ - vfsgen۰FileInfo: f, - Reader: bytes.NewReader(f.content), - }, nil - case *vfsgen۰DirInfo: - return &vfsgen۰Dir{ - vfsgen۰DirInfo: f, - }, nil - default: - // This should never happen because we generate only the above types. - panic(fmt.Sprintf("unexpected type %T", f)) - } -} - -// vfsgen۰CompressedFileInfo is a static definition of a gzip compressed file. -type vfsgen۰CompressedFileInfo struct { - name string - modTime time.Time - compressedContent []byte - uncompressedSize int64 -} - -func (f *vfsgen۰CompressedFileInfo) Readdir(count int) ([]os.FileInfo, error) { - return nil, fmt.Errorf("cannot Readdir from file %s", f.name) -} -func (f *vfsgen۰CompressedFileInfo) Stat() (os.FileInfo, error) { return f, nil } - -func (f *vfsgen۰CompressedFileInfo) GzipBytes() []byte { - return f.compressedContent -} - -func (f *vfsgen۰CompressedFileInfo) Name() string { return f.name } -func (f *vfsgen۰CompressedFileInfo) Size() int64 { return f.uncompressedSize } -func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode { return 0444 } -func (f *vfsgen۰CompressedFileInfo) ModTime() time.Time { return f.modTime } -func (f *vfsgen۰CompressedFileInfo) IsDir() bool { return false } -func (f *vfsgen۰CompressedFileInfo) Sys() interface{} { return nil } - -// vfsgen۰CompressedFile is an opened compressedFile instance. -type vfsgen۰CompressedFile struct { - *vfsgen۰CompressedFileInfo - gr *gzip.Reader - grPos int64 // Actual gr uncompressed position. - seekPos int64 // Seek uncompressed position. -} - -func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) { - if f.grPos > f.seekPos { - // Rewind to beginning. - err = f.gr.Reset(bytes.NewReader(f.compressedContent)) - if err != nil { - return 0, err - } - f.grPos = 0 - } - if f.grPos < f.seekPos { - // Fast-forward. - _, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos) - if err != nil { - return 0, err - } - f.grPos = f.seekPos - } - n, err = f.gr.Read(p) - f.grPos += int64(n) - f.seekPos = f.grPos - return n, err -} -func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - f.seekPos = 0 + offset - case io.SeekCurrent: - f.seekPos += offset - case io.SeekEnd: - f.seekPos = f.uncompressedSize + offset - default: - panic(fmt.Errorf("invalid whence value: %v", whence)) - } - return f.seekPos, nil -} -func (f *vfsgen۰CompressedFile) Close() error { - return f.gr.Close() -} - -// vfsgen۰FileInfo is a static definition of an uncompressed file (because it's not worth gzip compressing). -type vfsgen۰FileInfo struct { - name string - modTime time.Time - content []byte -} - -func (f *vfsgen۰FileInfo) Readdir(count int) ([]os.FileInfo, error) { - return nil, fmt.Errorf("cannot Readdir from file %s", f.name) -} -func (f *vfsgen۰FileInfo) Stat() (os.FileInfo, error) { return f, nil } - -func (f *vfsgen۰FileInfo) NotWorthGzipCompressing() {} - -func (f *vfsgen۰FileInfo) Name() string { return f.name } -func (f *vfsgen۰FileInfo) Size() int64 { return int64(len(f.content)) } -func (f *vfsgen۰FileInfo) Mode() os.FileMode { return 0444 } -func (f *vfsgen۰FileInfo) ModTime() time.Time { return f.modTime } -func (f *vfsgen۰FileInfo) IsDir() bool { return false } -func (f *vfsgen۰FileInfo) Sys() interface{} { return nil } - -// vfsgen۰File is an opened file instance. -type vfsgen۰File struct { - *vfsgen۰FileInfo - *bytes.Reader -} - -func (f *vfsgen۰File) Close() error { - return nil -} - -// vfsgen۰DirInfo is a static definition of a directory. -type vfsgen۰DirInfo struct { - name string - modTime time.Time - entries []os.FileInfo -} - -func (d *vfsgen۰DirInfo) Read([]byte) (int, error) { - return 0, fmt.Errorf("cannot Read from directory %s", d.name) -} -func (d *vfsgen۰DirInfo) Close() error { return nil } -func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil } - -func (d *vfsgen۰DirInfo) Name() string { return d.name } -func (d *vfsgen۰DirInfo) Size() int64 { return 0 } -func (d *vfsgen۰DirInfo) Mode() os.FileMode { return 0755 | os.ModeDir } -func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime } -func (d *vfsgen۰DirInfo) IsDir() bool { return true } -func (d *vfsgen۰DirInfo) Sys() interface{} { return nil } - -// vfsgen۰Dir is an opened dir instance. -type vfsgen۰Dir struct { - *vfsgen۰DirInfo - pos int // Position within entries for Seek and Readdir. -} - -func (d *vfsgen۰Dir) Seek(offset int64, whence int) (int64, error) { - if offset == 0 && whence == io.SeekStart { - d.pos = 0 - return 0, nil - } - return 0, fmt.Errorf("unsupported Seek in directory %s", d.name) -} - -func (d *vfsgen۰Dir) Readdir(count int) ([]os.FileInfo, error) { - if d.pos >= len(d.entries) && count > 0 { - return nil, io.EOF - } - if count <= 0 || count > len(d.entries)-d.pos { - count = len(d.entries) - d.pos - } - e := d.entries[d.pos : d.pos+count] - d.pos += count - return e, nil -} diff --git a/bindata/bindata_dev.go b/bindata/bindata_dev.go deleted file mode 100644 index 593620d..0000000 --- a/bindata/bindata_dev.go +++ /dev/null @@ -1,32 +0,0 @@ -// +build dev - -/* - * Copyright 2019 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package bindata - -import ( - "net/http" - "path/filepath" - "runtime" -) - -// Find the 'res' directory relative to this file to allow callers to be in any package -var _, b, _, _ = runtime.Caller(0) -var dir = filepath.Join(filepath.Dir(b), "../res") - -// Assets contains project assets. -var Assets http.FileSystem = http.Dir(dir) diff --git a/cmd/abort/abort.go b/cmd/abort/abort.go new file mode 100644 index 0000000..667dcae --- /dev/null +++ b/cmd/abort/abort.go @@ -0,0 +1,72 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abort + +import ( + "context" + "fmt" + + controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +type abortCmdOptions struct { + // ceids is a list of crawl execution ids to abort + ceids []string +} + +func (o *abortCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.ceids = args + return nil +} + +func (o *abortCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := controllerV1.NewControllerClient(conn) + + for _, ceid := range o.ceids { + request := controllerV1.ExecutionId{Id: ceid} + _, err := client.AbortCrawlExecution(context.Background(), &request) + if err != nil { + return fmt.Errorf("failed to abort execution '%v': %w", ceid, err) + } + } + return nil +} + +func NewAbortCmd() *cobra.Command { + o := &abortCmdOptions{} + + return &cobra.Command{ + GroupID: "run", + Use: "abort CRAWL-EXECUTION-ID ...", + Short: "Abort crawl executions", + Long: `Abort one or many crawl executions.`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + _ = o.complete(cmd, args) + + // silence usage to avoid showing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } +} diff --git a/cmd/abortjobexecution/abortjobexecution.go b/cmd/abortjobexecution/abortjobexecution.go new file mode 100644 index 0000000..4859c7b --- /dev/null +++ b/cmd/abortjobexecution/abortjobexecution.go @@ -0,0 +1,75 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package abortjobexecution + +import ( + "context" + "fmt" + + controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +type abortJobExecutionCmdOptions struct { + // jeids is a list of job execution ids to abort + jeids []string +} + +func (opt *abortJobExecutionCmdOptions) complete(cmd *cobra.Command, args []string) error { + opt.jeids = args + return nil +} + +func (opt *abortJobExecutionCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := controllerV1.NewControllerClient(conn) + + for _, jeid := range opt.jeids { + request := controllerV1.ExecutionId{Id: jeid} + _, err := client.AbortJobExecution(context.Background(), &request) + if err != nil { + return fmt.Errorf("failed to abort job execution '%v': %w", jeid, err) + } + } + return nil +} + +func NewAbortJobExecutionCmd() *cobra.Command { + o := &abortJobExecutionCmdOptions{} + + return &cobra.Command{ + GroupID: "run", + Use: "abortjobexecution JOB-EXECUTION-ID ...", + Short: "Abort job executions", + Long: `Abort one or many job executions.`, + Aliases: []string{"abortjob"}, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + err := o.complete(cmd, args) + if err != nil { + return err + } + // silence usage to prevent printing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } +} diff --git a/cmd/activeroles/activeroles.go b/cmd/activeroles/activeroles.go new file mode 100644 index 0000000..d9c4239 --- /dev/null +++ b/cmd/activeroles/activeroles.go @@ -0,0 +1,62 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package activeroles + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/ptypes/empty" + controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +type activeRolesCmdOptions struct{} + +func (o *activeRolesCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := controllerV1.NewControllerClient(conn) + + r, err := client.GetRolesForActiveUser(context.Background(), &empty.Empty{}) + if err != nil { + return err + } + + for _, role := range r.Role { + fmt.Println(role) + } + return nil +} + +func NewActiveRolesCmd() *cobra.Command { + o := &activeRolesCmdOptions{} + + return &cobra.Command{ + GroupID: "debug", + Use: "activeroles", + Short: "Get the active roles for the currently logged in user", + Long: `Get the active roles for the currently logged in user.`, + Args: cobra.NoArgs, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + return o.run() + }, + } +} diff --git a/cmd/cmd.go b/cmd/cmd.go new file mode 100644 index 0000000..490cf51 --- /dev/null +++ b/cmd/cmd.go @@ -0,0 +1,141 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, GitVersion 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "os" + + "github.com/nlnwa/veidemannctl/cmd/abort" + "github.com/nlnwa/veidemannctl/cmd/abortjobexecution" + "github.com/nlnwa/veidemannctl/cmd/activeroles" + configcmd "github.com/nlnwa/veidemannctl/cmd/config" + "github.com/nlnwa/veidemannctl/cmd/create" + deletecmd "github.com/nlnwa/veidemannctl/cmd/delete" + "github.com/nlnwa/veidemannctl/cmd/get" + importcmd "github.com/nlnwa/veidemannctl/cmd/import" + "github.com/nlnwa/veidemannctl/cmd/logconfig" + "github.com/nlnwa/veidemannctl/cmd/login" + "github.com/nlnwa/veidemannctl/cmd/logout" + "github.com/nlnwa/veidemannctl/cmd/pause" + "github.com/nlnwa/veidemannctl/cmd/report" + "github.com/nlnwa/veidemannctl/cmd/run" + "github.com/nlnwa/veidemannctl/cmd/script_parameters" + "github.com/nlnwa/veidemannctl/cmd/status" + "github.com/nlnwa/veidemannctl/cmd/unpause" + "github.com/nlnwa/veidemannctl/cmd/update" + "github.com/nlnwa/veidemannctl/cmd/version" + "github.com/nlnwa/veidemannctl/config" + + "github.com/spf13/cobra" +) + +// NewRootCmd returns the root command. +func NewRootCmd() *cobra.Command { + cobra.EnableCommandSorting = false + + cmd := &cobra.Command{ + Use: "veidemannctl", + Short: "veidemannctl controls the Veidemann web crawler", + Long: "veidemannctl controls the Veidemann web crawler", + DisableAutoGenTag: true, + } + + // Add global flags + cmd.PersistentFlags().String("config", "", "Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/") + cmd.PersistentFlags().String("context", "", "The name of the context to use") + cmd.PersistentFlags().String("server", "", "The address of the Veidemann server to use") + cmd.PersistentFlags().String("server-name-override", "", + "If set, it will override the virtual host name of authority (e.g. :authority header field) in requests") + cmd.PersistentFlags().String("api-key", "", + "If set, it will be used as the bearer token for authentication") + cmd.PersistentFlags().String("log-level", "info", `set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace"`) + cmd.PersistentFlags().String("log-format", "pretty", `set log format, available formats are: "pretty" or "json"`) + cmd.PersistentFlags().Bool("log-caller", false, "include information about caller in log output") + + // Add subcommands + cmd.AddCommand(configcmd.NewConfigCmd()) // config + + cmd.AddGroup(&cobra.Group{ + ID: "basic", + Title: "Basic Commands:", + }) + cmd.AddCommand(get.NewGetCmd()) // get + cmd.AddCommand(create.NewCreateCmd()) // create + cmd.AddCommand(update.NewUpdateCmd()) // update + cmd.AddCommand(deletecmd.NewDeleteCmd()) // delete + + cmd.AddGroup(&cobra.Group{ + ID: "advanced", + Title: "Advanced Commands:", + }) + cmd.AddCommand(report.NewReportCmd()) // report + cmd.AddCommand(importcmd.NewImportCmd()) // import + + cmd.AddGroup(&cobra.Group{ + ID: "run", + Title: "Crawl Commands:", + }) + cmd.AddCommand(run.NewRunCmd()) // run + cmd.AddCommand(abort.NewAbortCmd()) // abort + cmd.AddCommand(abortjobexecution.NewAbortJobExecutionCmd()) // abortjobexecution + + cmd.AddGroup(&cobra.Group{ + ID: "status", + Title: "Management Commands:", + }) + cmd.AddCommand(status.NewStatusCmd()) // status + cmd.AddCommand(pause.NewPauseCmd()) // pause + cmd.AddCommand(unpause.NewUnpauseCmd()) // unpause + + cmd.AddGroup(&cobra.Group{ + ID: "login", + Title: "Authentication Commands:", + }) + cmd.AddCommand(login.NewLoginCmd()) // login + cmd.AddCommand(logout.NewLogoutCmd()) // logout + + cmd.AddCommand(version.NewVersionCmd()) // version + + cmd.AddGroup(&cobra.Group{ + ID: "debug", + Title: "Troubleshooting and Debug Commands:", + }) + cmd.AddCommand(scriptparameters.NewScriptParametersCmd()) // script-parameters + cmd.AddCommand(logconfig.NewLogConfigCmd()) // logconfig + cmd.AddCommand(activeroles.NewActiveRolesCmd()) // activeroles + + return cmd +} + +// Execute initializes the root command and executes it. +func Execute() { + // Initialize root command + cmd := NewRootCmd() + + // Register function to run after command is initialized + cobra.OnInitialize(func() { + // Initialize config from flags + err := config.Init(cmd.PersistentFlags()) + if err != nil { + fmt.Printf("Initialization failed: %v\n", err) + os.Exit(1) + } + }) + + // Execute root command + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/config/config.go b/cmd/config/config.go new file mode 100644 index 0000000..e054269 --- /dev/null +++ b/cmd/config/config.go @@ -0,0 +1,40 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/spf13/cobra" +) + +func NewConfigCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Modify or view configuration files", + } + + cmd.AddCommand(newUseContextCmd()) // use-context + cmd.AddCommand(newCreateContextCmd()) // create-context + cmd.AddCommand(newCurrentContextCmd()) // current-context + cmd.AddCommand(newListContextsCmd()) // list-contexts + cmd.AddCommand(newImportCaCmd()) // import-ca + cmd.AddCommand(newSetServerNameOverrideCmd()) // set-server-name-override + cmd.AddCommand(newGetServerNameOverrideCmd()) // get-server-name-override + cmd.AddCommand(newSetApiKeyCmd()) // set-apikey + cmd.AddCommand(newGetApiKeyCmd()) // get-apikey + cmd.AddCommand(newSetAddressCmd()) // set-address + cmd.AddCommand(newGetAddressCmd()) // get-address + cmd.AddCommand(newViewConfigCmd()) // view + + return cmd +} diff --git a/cmd/config/create_context.go b/cmd/config/create_context.go new file mode 100644 index 0000000..1625566 --- /dev/null +++ b/cmd/config/create_context.go @@ -0,0 +1,49 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +// createContextCmd represents the create-context command +func newCreateContextCmd() *cobra.Command { + return &cobra.Command{ + Use: "create-context NAME", + Short: "Create a new context", + Long: `Create a new context + +Examples: + # Create context for the prod cluster + veidemannctl config create-context prod +`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + + name := args[0] + err := config.CreateContext(name) + if err != nil { + return err + } + fmt.Printf("Created context \"%v\"\n", name) + + return nil + }, + } +} diff --git a/cmd/config/current_context.go b/cmd/config/current_context.go new file mode 100644 index 0000000..ed12f84 --- /dev/null +++ b/cmd/config/current_context.go @@ -0,0 +1,39 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newCurrentContextCmd() *cobra.Command { + return &cobra.Command{ + Use: "current-context", + Short: "Display the current context", + Long: `Display the current context + +Examples: + # Display the current context + veidemannctl config current-context +`, + Aliases: []string{"context"}, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(config.GetContext()) + }, + } +} diff --git a/cmd/config/get_address.go b/cmd/config/get_address.go new file mode 100644 index 0000000..b2fbc6b --- /dev/null +++ b/cmd/config/get_address.go @@ -0,0 +1,39 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newGetAddressCmd() *cobra.Command { + return &cobra.Command{ + Use: "get-address", + Short: "Display Veidemann controller service address", + Long: `Display Veidemann controller service address + +Examples: + # Display Veidemann controller service address + veidemannctl config get-address +`, + Aliases: []string{"address", "get-server", "server"}, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(config.GetServer()) + }, + } +} diff --git a/cmd/config/get_apikey.go b/cmd/config/get_apikey.go new file mode 100644 index 0000000..4584dea --- /dev/null +++ b/cmd/config/get_apikey.go @@ -0,0 +1,39 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newGetApiKeyCmd() *cobra.Command { + return &cobra.Command{ + Use: "get-apikey", + Short: "Display Veidemann authentication api-key", + Long: `Display Veidemann authentication api-key + +Examples: + # Display Veidemann authentication api-key + veidemannctl config get-apikey +`, + Aliases: []string{"get-apikey", "apikey", "get-api-key", "api-key"}, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(config.GetApiKey()) + }, + } +} diff --git a/cmd/config/get_server_name_override.go b/cmd/config/get_server_name_override.go new file mode 100644 index 0000000..79b5be8 --- /dev/null +++ b/cmd/config/get_server_name_override.go @@ -0,0 +1,39 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newGetServerNameOverrideCmd() *cobra.Command { + return &cobra.Command{ + Use: "get-server-name-override", + Short: "Display the server name override", + Long: `Display the server name override + +Examples: + # Display the server name override + veidemannctl config get-server-name-override +`, + Aliases: []string{"get-servername"}, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(config.GetServerNameOverride()) + }, + } +} diff --git a/cmd/config/import_ca.go b/cmd/config/import_ca.go new file mode 100644 index 0000000..cbc3276 --- /dev/null +++ b/cmd/config/import_ca.go @@ -0,0 +1,46 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "os" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newImportCaCmd() *cobra.Command { + return &cobra.Command{ + Use: "import-ca FILENAME", + Short: "Import file with trusted certificate chains for the idp and controller.", + Long: `Import file with trusted certificate chains for the idp and controller. These are in addition to the default certs configured for the OS.`, + Aliases: []string{"ca"}, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + rootCA := args[0] + rootCABytes, err := os.ReadFile(rootCA) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + if err := config.SetCaCert(string(rootCABytes)); err != nil { + return err + } + fmt.Printf("Successfully imported root CA cert from file %s\n", rootCA) + return nil + }, + } +} diff --git a/cmd/config/list_contexts.go b/cmd/config/list_contexts.go new file mode 100644 index 0000000..7e4a3d7 --- /dev/null +++ b/cmd/config/list_contexts.go @@ -0,0 +1,49 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newListContextsCmd() *cobra.Command { + return &cobra.Command{ + Use: "list-contexts", + Short: "Display the known contexts", + Long: `Display the known contexts. + +Examples: + # Get a list of known contexts + veidemannctl config list-contexts +`, + Aliases: []string{"contexts"}, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + + cs, err := config.ListContexts() + if err != nil { + return fmt.Errorf("failed to list contexts: %w", err) + } + for _, c := range cs { + fmt.Println(c) + } + return nil + }, + } +} diff --git a/cmd/config/set_address.go b/cmd/config/set_address.go new file mode 100644 index 0000000..9efaac0 --- /dev/null +++ b/cmd/config/set_address.go @@ -0,0 +1,40 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newSetAddressCmd() *cobra.Command { + return &cobra.Command{ + Use: "set-address HOST:PORT", + Short: "Set the address to the Veidemann controller service", + Long: `Set the address to the Veidemann controller service + +Examples: + # Sets the address to Veidemann controller service to localhost:50051 + veidemannctl config set-address localhost:50051 +`, + Aliases: []string{"set-server"}, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + + return config.SetServerAddress(args[0]) + }, + } +} diff --git a/cmd/config/set_apikey.go b/cmd/config/set_apikey.go new file mode 100644 index 0000000..4f5b203 --- /dev/null +++ b/cmd/config/set_apikey.go @@ -0,0 +1,41 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +// newSetApiKeyCmd represents the set-apikey command +func newSetApiKeyCmd() *cobra.Command { + return &cobra.Command{ + Use: "set-apikey API-KEY", + Short: "Set the api-key to use for authentication", + Long: `Set the api-key to use for authentication + +Examples: + # Set the api-key to use for authentication to Veidemann controller service to myKey + veidemannctl config set-apikey myKey +`, + Aliases: []string{"set-apikey", "set-api-key"}, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + + return config.SetApiKey(args[0]) + }, + } +} diff --git a/cmd/config/set_server_name_override.go b/cmd/config/set_server_name_override.go new file mode 100644 index 0000000..b8d1328 --- /dev/null +++ b/cmd/config/set_server_name_override.go @@ -0,0 +1,43 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +func newSetServerNameOverrideCmd() *cobra.Command { + return &cobra.Command{ + Use: "set-server-name-override HOSTNAME", + Short: "Set the server name override", + Long: `Set the server name override. + +Use this when there is a mismatch between the exposed server name for the cluster and the certificate. The use is a security +risk and is only recommended for testing. + +Examples: + # Sets the server name override to test.local + veidemannctl config set-server-name-override test.local +`, + Aliases: []string{"set-servername"}, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + + return config.SetServerNameOverride(args[0]) + }, + } +} diff --git a/cmd/config/use_context.go b/cmd/config/use_context.go new file mode 100644 index 0000000..8ab05e7 --- /dev/null +++ b/cmd/config/use_context.go @@ -0,0 +1,49 @@ +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" +) + +// newUseContextCmd returns the use-context subcommand. +func newUseContextCmd() *cobra.Command { + return &cobra.Command{ + Use: "use-context CONTEXT", + Short: "Set the current context", + Long: `Set the current context + +Examples: + # Use the context for the prod cluster + veidemannctl config use-context prod +`, + Aliases: []string{"use"}, + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + c, err := config.ListContexts() + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + return c, cobra.ShellCompDirectiveNoSpace + }, + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to avoid printing usage when returning an error + cmd.SilenceUsage = true + + name := args[0] + ok, err := config.ContextExists(name) + if err != nil { + return fmt.Errorf("failed switching context to '%s': %w", name, err) + } + if !ok { + return fmt.Errorf("non existing context '%s'", name) + } + if err := config.SetCurrentContext(name); err != nil { + return fmt.Errorf("failed switching context to '%s': %w", name, err) + } + fmt.Printf("Switched to context \"%v\"\n", name) + return nil + }, + } +} diff --git a/cmd/config/view.go b/cmd/config/view.go new file mode 100644 index 0000000..5da3257 --- /dev/null +++ b/cmd/config/view.go @@ -0,0 +1,38 @@ +// Copyright © 2023 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/config" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +func newViewConfigCmd() *cobra.Command { + return &cobra.Command{ + Use: "view", + Short: "Display the current config", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + b, err := yaml.Marshal(config.GetConfig()) + if err != nil { + return err + } + fmt.Print(string(b)) + return nil + }, + } +} diff --git a/cmd/create/create.go b/cmd/create/create.go new file mode 100644 index 0000000..dad4095 --- /dev/null +++ b/cmd/create/create.go @@ -0,0 +1,148 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package create + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type createCmdOptions struct { + filename string + concurrency int +} + +func (o *createCmdOptions) run() error { + if o.filename == "-" { + o.filename = "" + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + sigs := make(chan os.Signal, 2) + signal.Notify(sigs, os.Interrupt) + sig := <-sigs + log.Debug().Msgf("Received %s signal, aborting...", sig) + cancel() + }() + + result := make(chan *configV1.ConfigObject, 256) + err := format.Unmarshal(ctx, o.filename, result) + if err != nil { + return fmt.Errorf("failed to parse input: %w", err) + } + + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + validate := func(co *configV1.ConfigObject) error { + if co.ApiVersion == "" { + return fmt.Errorf("missing apiVersion") + } + if co.Kind == configV1.Kind_undefined { + return fmt.Errorf("missing kind") + } + if co.GetMeta().GetName() == "" { + return fmt.Errorf("missing metadata.name") + } + return nil + } + + handleError := func(co *configV1.ConfigObject, err error) { + log.Error().Err(err). + Str("kind", co.GetKind().String()). + Str("meta.name", co.GetMeta().Name). + Str("id", co.GetId()). + Msg("Failed to save config object") + } + + var wg sync.WaitGroup + for i := 0; i < o.concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for co := range result { + // validate + err = validate(co) + if err != nil { + handleError(co, fmt.Errorf("validation failed: %w", err)) + continue + } + // save + for attempts := 0; attempts < 3; attempts++ { + r, err := client.SaveConfigObject(context.Background(), co) + s, ok := status.FromError(err) + if ok && s.Code() == codes.Unauthenticated { + // retry if unauthenticated + log.Debug().Int("attempts", attempts).Msg("Unauthenticated, retrying...") + continue + } + if err != nil { + handleError(co, err) + break + } + log.Info().Str("kind", r.GetKind().String()).Str("meta.name", r.Meta.Name).Str("id", r.Id).Msg("Saved config object") + break + } + } + }() + } + wg.Wait() + + return nil +} + +func NewCreateCmd() *cobra.Command { + o := &createCmdOptions{} + + cmd := &cobra.Command{ + GroupID: "basic", + Use: "create", + Short: "Create or update config objects", + Long: `Create or update one or many config objects`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to prevent printing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } + + // filename is a required flag + cmd.Flags().StringVarP(&o.filename, "filename", "f", "", "Filename or directory to read from. "+ + "If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin.") + _ = cmd.MarkFlagRequired("filename") + cmd.Flags().IntVarP(&o.concurrency, "concurrency", "c", 32, "Number of concurrent requests") + + return cmd +} diff --git a/cmd/delete/delete.go b/cmd/delete/delete.go new file mode 100644 index 0000000..4460de7 --- /dev/null +++ b/cmd/delete/delete.go @@ -0,0 +1,161 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package del + +import ( + "context" + "errors" + "fmt" + "io" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/apiutil" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +type deleteCmdOptions struct { + kind configV1.Kind + ids []string + label string + filters []string + dryRun bool +} + +// complete completes the delete command by parsing the arguments passed to the command +func (o *deleteCmdOptions) complete(cmd *cobra.Command, args []string) error { + k := args[0] + + o.kind = format.GetKind(k) + o.ids = args[1:] + + if o.kind == configV1.Kind_undefined { + return fmt.Errorf("undefined kind '%s'", k) + } + + if len(o.ids) == 0 && o.filters == nil && o.label == "" { + return fmt.Errorf("Either the -f or -q flags, or one or more id's must be provided\n") + } + + return nil +} + +// run runs the delete command which deletes one or more config objects +func (o *deleteCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + selector, err := apiutil.CreateListRequest(o.kind, o.ids, "", o.label, o.filters, 0, 0) + if err != nil { + return fmt.Errorf("could not create request: %w", err) + } + + r, err := client.ListConfigObjects(context.Background(), selector) + if err != nil { + return fmt.Errorf("could not list objects: %w", err) + } + + count, err := client.CountConfigObjects(context.Background(), selector) + if err != nil { + return fmt.Errorf("could not count objects: %w", err) + } + + if o.dryRun { + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("error getting object: %w", err) + } + log.Debug().Msgf("Outputing record of kind '%s' with name '%s'", msg.Kind, msg.Meta.Name) + fmt.Printf("%s\n", msg.Meta.Name) + } + fmt.Printf("Requested count: %v\nTo actually delete, add: --dry-run=false\n", count.Count) + + return nil + } + + var deleted int + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("error getting object: %w", err) + } + log.Debug().Msgf("Deleting record of kind '%s' with name '%s'", msg.Kind, msg.Meta.Name) + + request := &configV1.ConfigObject{ + ApiVersion: "v1", + Kind: o.kind, + Id: msg.Id, + } + + r, err := client.DeleteConfigObject(context.Background(), request) + if err != nil { + log.Error().Err(err).Str("id", msg.Id).Msgf("Could not delete object") + } + if r.Deleted { + deleted++ + } + } + log.Info().Msgf("Deleted %d objects of %d selected", deleted, count.Count) + + return nil +} + +func NewDeleteCmd() *cobra.Command { + o := &deleteCmdOptions{} + + cmd := &cobra.Command{ + GroupID: "basic", + Use: "delete KIND (ID ...)", + Short: "Delete config objects", + Long: `Delete one or many config objects. + +` + + format.ListObjectNames() + + `Examples: + # Delete a seed. + veidemannctl delete seed 407a9600-4f25-4f17-8cff-ee1b8ee950f6`, + Args: cobra.MatchAll( + cobra.MinimumNArgs(1), + func(cmd *cobra.Command, args []string) error { + return cobra.OnlyValidArgs(cmd, args[:1]) + }), + ValidArgs: format.GetObjectNames(), + PreRunE: o.complete, + RunE: func(cmd *cobra.Command, args []string) error { + // set silence usage to true to avoid printing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } + cmd.PersistentFlags().StringVarP(&o.label, "label", "l", "", "Delete objects by label (: | )") + cmd.PersistentFlags().StringArrayVarP(&o.filters, "filter", "q", nil, "Delete objects by field (i.e. meta.description=foo)") + cmd.PersistentFlags().BoolVarP(&o.dryRun, "dry-run", "", true, "Set to false to execute delete") + + return cmd +} diff --git a/cmd/get/get.go b/cmd/get/get.go new file mode 100644 index 0000000..ebbd397 --- /dev/null +++ b/cmd/get/get.go @@ -0,0 +1,153 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package get + +import ( + "errors" + "fmt" + "io" + + "github.com/nlnwa/veidemannctl/apiutil" + "github.com/spf13/cobra" + + "context" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" +) + +type getCmdOptions struct { + kind configV1.Kind + ids []string + label string + name string + filters []string + filename string + format string + goTemplate string + pageSize int32 + page int32 +} + +func (o *getCmdOptions) complete(cmd *cobra.Command, args []string) error { + kind := args[0] + o.ids = args[1:] + + o.kind = format.GetKind(kind) + + if o.kind == configV1.Kind_undefined { + return fmt.Errorf(`undefined kind "%v"`, kind) + } + return nil +} + +func (o *getCmdOptions) run() error { + request, err := apiutil.CreateListRequest(o.kind, o.ids, o.name, o.label, o.filters, o.pageSize, o.page) + if err != nil { + return fmt.Errorf("error creating request: %w", err) + } + + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + r, err := client.ListConfigObjects(context.Background(), request) + if err != nil { + return fmt.Errorf("error from controller: %w", err) + } + + out, err := format.ResolveWriter(o.filename) + if err != nil { + return fmt.Errorf("could not resolve output file '%v': %w", o.filename, err) + } + + s, err := format.NewFormatter(o.kind.String(), out, o.format, o.goTemplate) + if err != nil { + return err + } + defer s.Close() + + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + if err := s.WriteRecord(msg); err != nil { + return err + } + } + return nil +} + +func NewGetCmd() *cobra.Command { + o := &getCmdOptions{} + + cmd := &cobra.Command{ + GroupID: "basic", + Use: "get KIND [ID ...]", + Short: "Display config objects", + Long: `Display one or many config objects. + +` + + `Valid object types: +` + + format.ListObjectNames() + + `Examples: + # List all seeds. + veidemannctl get seed + + # List all seeds in yaml output format. + veidemannctl get seed -o yaml`, + ValidArgs: format.GetObjectNames(), + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.complete(cmd, args); err != nil { + return err + } + // set SilenceUsage to true to prevent printing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } + + cmd.Flags().StringVarP(&o.label, "label", "l", "", "List objects by label (: | )") + cmd.Flags().StringVarP(&o.name, "name", "n", "", "List objects by name (accepts regular expressions)") + _ = cmd.RegisterFlagCompletionFunc("name", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + names, err := apiutil.CompleteName(args[0], toComplete) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + return names, cobra.ShellCompDirectiveDefault + }) + cmd.Flags().StringArrayVarP(&o.filters, "filter", "q", nil, "Filter objects by field (i.e. meta.description=foo)") + cmd.Flags().StringVarP(&o.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") + _ = cmd.RegisterFlagCompletionFunc("output", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return []string{"json", "table", "yaml", "wide", "template", "template-file"}, cobra.ShellCompDirectiveDefault + }) + cmd.Flags().StringVarP(&o.goTemplate, "template", "t", "", "A Go template used to format the output") + cmd.Flags().StringVarP(&o.filename, "filename", "f", "", "Filename to write to") + cmd.Flags().Int32VarP(&o.pageSize, "pagesize", "s", 10, "Number of objects to get") + cmd.Flags().Int32VarP(&o.page, "page", "p", 0, "The page number") + + return cmd +} diff --git a/cmd/import/convertoos.go b/cmd/import/convertoos.go new file mode 100644 index 0000000..12cc067 --- /dev/null +++ b/cmd/import/convertoos.go @@ -0,0 +1,274 @@ +// Copyright © 2023 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package importcmd + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/config" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/nlnwa/veidemannctl/importutil" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +// ConvertOosCmdOptions is the options for the convert oos command +type ConvertOosCmdOptions struct { + Filename string + ErrorFile string + OutFile string + Toplevel bool + IgnoreScheme bool + CheckUri bool + CheckUriTimeout time.Duration + DbDir string + ResetDb bool + Concurrency int + SkipImport bool + EntityId string + EntityName string + EntityLabels []string + SeedLabels []string +} + +// run runs the convert oos command +func (o *ConvertOosCmdOptions) run() error { + // Create output writer (file or stdout) + out, err := format.ResolveWriter(o.OutFile) + if err != nil { + return fmt.Errorf("unable to open output file: %v: %w", o.OutFile, err) + } + defer out.Close() + + // Create error writer (file or stderr) + var errFile io.WriteCloser + if o.ErrorFile == "" || o.ErrorFile == "-" { + errFile = os.Stderr + } else { + f, err := os.Create(o.ErrorFile) + if err != nil { + return fmt.Errorf("unable to open error file: %v: %w", o.ErrorFile, err) + } + defer f.Close() + } + + // Connect to Veidemann API server + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } + defer conn.Close() + + // Create Veidemann config client + client := configV1.NewConfigClient(conn) + + // Create uri checker for checking liveness of uri + var uriChecker *UriChecker + if o.CheckUri { + uriChecker = &UriChecker{ + Client: NewHttpClient(o.CheckUriTimeout, false), + } + } + + // Create key normalizer for state database + uriNormalizer := &UriKeyNormalizer{ignoreScheme: o.IgnoreScheme, toplevel: o.Toplevel} + + dbDir := path.Join(o.DbDir, config.GetContext(), configV1.Kind_seed.String()) + + // Create database for storing state + seedDb, err := importutil.NewImportDb(dbDir, o.ResetDb) + if err != nil { + return fmt.Errorf("failed to initialize import db: %w", err) + } + defer seedDb.Close() + + // Import existing seeds into state database + if !o.SkipImport { + log.Info().Msg("Importing existing seeds...") + err = importutil.ImportExisting(seedDb, client, configV1.Kind_seed, uriNormalizer) + if err != nil { + return fmt.Errorf("failed to import existing seeds: %w", err) + } + } + + // Create Record reader for file input + rr, err := importutil.NewRecordReader(o.Filename, &LineAsStringDecoder{}, "*.txt") + if err != nil { + return fmt.Errorf("unable to open file '%v': %w", o.Filename, err) + } + + entityId := o.EntityId + entityName := o.EntityName + var entityLabels []*configV1.Label + if len(o.EntityLabels) > 0 { + for _, l := range o.EntityLabels { + kv := strings.SplitN(l, ":", 2) + if len(kv) != 2 { + return fmt.Errorf("invalid entity label: %s", l) + } + entityLabels = append(entityLabels, &configV1.Label{Key: kv[0], Value: kv[1]}) + } + } + + // If entityId is set, ignore entityName and entityLabels + if entityId != "" { + entityName = "" + entityLabels = nil + } + + var seedLabels []*configV1.Label + if len(o.SeedLabels) > 0 { + for _, l := range o.SeedLabels { + kv := strings.SplitN(l, ":", 2) + if len(kv) != 2 { + return fmt.Errorf("invalid seed label: %s", l) + } + seedLabels = append(seedLabels, &configV1.Label{Key: kv[0], Value: kv[1]}) + } + } + + // Processor for converting oos records into import records + proc := func(uri string) error { + seed := &seedDesc{ + EntityId: entityId, + EntityName: entityName, + EntityLabel: entityLabels, + Uri: uri, + SeedLabel: seedLabels, + } + // If entityId or entityName is not set, use uri as entityName + if entityId == "" && entityName == "" { + seed.EntityName = uri + } + + if uriChecker != nil { + uri, err := uriChecker.Check(uri) + if err != nil { + return fmt.Errorf("failed URL check: %w", err) + } + seed.Uri = uri + } + + normalizedUri, err := uriNormalizer.Normalize(seed.Uri) + if err != nil { + return fmt.Errorf("failed to normalize URL '%s': %w", uri, err) + } + + if ids, err := seedDb.Get(normalizedUri); err != nil { + return err + } else if len(ids) > 0 { + return errAlreadyExists{key: normalizedUri} + } + + json, err := json.Marshal(seed) + if err != nil { + return err + } + if _, err = fmt.Fprintf(out, "%s\n", json); err != nil { + return err + } + return nil + } + + // Create error logger + errorLog := log.Output(zerolog.ConsoleWriter{Out: errFile, TimeFormat: time.RFC3339}) + + // Error handler for executor + errHandler := func(state importutil.Job[string]) { + l := errorLog.With(). + Str("uri", state.Val). + Str("filename", state.GetFilename()). + Int("recNum", state.GetRecordNum()).Logger() + + var err errAlreadyExists + if errors.As(state.GetError(), &err) { + l.Info().Msg(err.Error()) + } else { + l.Error().Err(state.GetError()).Msg("") + } + } + + // Create cuncurrent executor for processing records + executor := importutil.NewExecutor(o.Concurrency, proc, errHandler) + + // Read records from file and queue for processing + for { + var uri string + state, err := rr.Next(&uri) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + errorLog.Error().Err(err).Msgf("error decoding record: %v", state) + continue + } + // ignore empty lines + if uri == "" { + continue + } + executor.Queue <- importutil.Job[string]{State: state, Val: uri} + } + + // Wait for all records to be processed + count, success, failed := executor.Wait() + + errorLog.Info().Int("total", count).Int("success", success).Int("failed", failed).Msg("Finished converting records") + + return nil +} + +// newConvertOosCmd creates the convert oos command +func newConvertOosCmd() *cobra.Command { + o := &ConvertOosCmdOptions{} + + var cmd = &cobra.Command{ + Use: "convertoos", + Short: "Convert Out of Scope file(s) to seed import file", + Long: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return o.run() + }, + } + + cmd.Flags().StringVarP(&o.Filename, "filename", "f", "", "Filename or directory to read from. "+ + "If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin.") + _ = cmd.MarkFlagRequired("filename") + cmd.Flags().StringVarP(&o.ErrorFile, "err-file", "e", "-", "File to write errors to. '-' writes to stderr.") + cmd.Flags().StringVarP(&o.OutFile, "out-file", "o", "-", "File to write result to. '-' writes to stdout.") + cmd.Flags().BoolVar(&o.Toplevel, "toplevel", true, "Convert URI to toplevel by removing path") + cmd.Flags().BoolVar(&o.IgnoreScheme, "ignore-scheme", true, "Ignore the URL's scheme when checking if this URL is already imported.") + cmd.Flags().BoolVarP(&o.CheckUri, "check-uri", "", true, "Check the uri for liveness and follow 301") + cmd.Flags().DurationVarP(&o.CheckUriTimeout, "check-uri-timeout", "", 2*time.Second, "Timeout when checking uri for liveness") + cmd.Flags().StringVarP(&o.DbDir, "db-dir", "b", "/tmp/veidemannctl", "Directory for storing state db") + cmd.Flags().BoolVar(&o.ResetDb, "truncate", false, "Truncate state database") + cmd.Flags().IntVarP(&o.Concurrency, "concurrency", "c", 16, "Number of concurrent workers") + cmd.Flags().BoolVar(&o.SkipImport, "skip-import", false, "Do not import existing seeds into state database") + cmd.Flags().StringVar(&o.EntityId, "entity-id", "", "Entity id to use for all seeds (overrides entity-name and entity-label)") + cmd.Flags().StringVar(&o.EntityName, "entity-name", "", "Entity name to use for all seeds") + cmd.Flags().StringSliceVar(&o.EntityLabels, "entity-label", []string{"source:oos"}, "Entity labels to use for all seeds") + cmd.Flags().StringSliceVar(&o.SeedLabels, "seed-label", []string{"source:oos"}, "Seed labels to use for all seeds") + + return cmd +} diff --git a/cmd/import/decode.go b/cmd/import/decode.go new file mode 100644 index 0000000..470efd3 --- /dev/null +++ b/cmd/import/decode.go @@ -0,0 +1,77 @@ +// Copyright © 2023 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package importcmd + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "reflect" + "strings" + + "gopkg.in/yaml.v3" +) + +// LineAsStringDecoder is a decoder that reads a line from the input as a string +type LineAsStringDecoder struct { + r *bufio.Reader +} + +func (l *LineAsStringDecoder) Init(r io.Reader, suffix string) { + l.r = bufio.NewReader(r) +} + +func (l *LineAsStringDecoder) Read(v interface{}) error { + s, err := l.r.ReadString('\n') + if err != nil { + return err + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return fmt.Errorf("invalid target: %v", reflect.TypeOf(v)) + } + + s = strings.TrimSpace(s) + rv.Elem().Set(reflect.ValueOf(&s).Elem()) + + return nil +} + +type decoder interface { + Decode(v interface{}) error +} + +// JsonYamlDecoder is a decoder that reads json or yaml from the input and decodes it into a struct +type JsonYamlDecoder struct { + decoder +} + +func (j *JsonYamlDecoder) Init(r io.Reader, suffix string) { + if suffix == ".yaml" || suffix == ".yml" { + dec := yaml.NewDecoder(r) + // TODO check this + dec.KnownFields(true) + j.decoder = dec + } else { + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + j.decoder = dec + } +} + +func (j *JsonYamlDecoder) Read(v interface{}) error { + return j.Decode(v) +} diff --git a/cmd/import/duplicatereport.go b/cmd/import/duplicatereport.go new file mode 100644 index 0000000..21a14e6 --- /dev/null +++ b/cmd/import/duplicatereport.go @@ -0,0 +1,147 @@ +// Copyright © 2023 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package importcmd + +import ( + "fmt" + "io" + "os" + "path" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/config" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/nlnwa/veidemannctl/importutil" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +// duplicateReportCmdOptions is the options for the convert oos command +type duplicateReportCmdOptions struct { + Kind configV1.Kind + OutFile string + DbDir string + ResetDb bool + Toplevel bool + IgnoreScheme bool + SkipImport bool +} + +// complete completes the duplicate report command options +func (o *duplicateReportCmdOptions) complete(cmd *cobra.Command, args []string) error { + kind := format.GetKind(args[0]) + if kind == configV1.Kind_undefined { + return fmt.Errorf("undefined kind: %v", kind) + } + o.Kind = kind + return nil +} + +type DuplicateReporter interface { + Report(w io.Writer) error +} + +// run runs the convert oos command with the given options +func (o *duplicateReportCmdOptions) run() error { + // Create output writer (file or stdout) + var out io.Writer + if o.OutFile == "" { + out = os.Stdout + } else { + f, err := os.Create(o.OutFile) + if err != nil { + return fmt.Errorf("unable to open output file: %v: %w", o.OutFile, err) + } + defer f.Close() + out = f + } + + // Connect to Veidemann + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + // Create key normalizer + var keyNormalizer importutil.KeyNormalizer + if o.Kind == configV1.Kind_seed { + keyNormalizer = &UriKeyNormalizer{toplevel: o.Toplevel, ignoreScheme: o.IgnoreScheme} + } + + dbDir := path.Join(o.DbDir, config.GetContext(), o.Kind.String()) + + // Create state Database of kind seed or kind crawlEntity from Veidemann + stateDb, err := importutil.NewImportDb(dbDir, o.ResetDb) + if err != nil { + return fmt.Errorf("failed to initialize import db: %w", err) + } + defer stateDb.Close() + + // Import existing into state database + if !o.SkipImport { + log.Info().Str("kind", o.Kind.String()).Msg("Importing from Veidemann...") + err = importutil.ImportExisting(stateDb, client, o.Kind, keyNormalizer) + if err != nil { + return fmt.Errorf("failed to import '%v': %w", o.Kind, err) + } + } + + var duplicateReporter DuplicateReporter + + if o.Kind == configV1.Kind_seed { + duplicateReporter = importutil.SeedReporter{ImportDb: stateDb, Client: client} + } else { + duplicateReporter = importutil.DuplicateKindReporter{ImportDb: stateDb} + } + + return duplicateReporter.Report(out) +} + +func newDuplicateReportCmd() *cobra.Command { + o := &duplicateReportCmdOptions{} + + cmd := &cobra.Command{ + Use: "duplicatereport KIND", + Short: "List duplicated seeds or crawl entities in Veidemann", + Long: ``, + Args: cobra.ExactArgs(1), + ValidArgs: []string{ + configV1.Kind_seed.String(), + configV1.Kind_crawlEntity.String(), + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.complete(cmd, args); err != nil { + return err + } + + // silence usage to prevent printing usage when error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } + + cmd.Flags().StringVarP(&o.OutFile, "out-file", "o", "", "File to write output.") + cmd.Flags().StringVarP(&o.DbDir, "db-dir", "b", "/tmp/veidemannctl", "Directory for storing state db") + cmd.Flags().BoolVarP(&o.Toplevel, "toplevel", "", false, "Convert URI to toplevel by removing path before checking for duplicates.") + cmd.Flags().BoolVarP(&o.IgnoreScheme, "ignore-scheme", "", false, "Ignore the URL's scheme when checking for duplicates.") + cmd.Flags().BoolVar(&o.ResetDb, "truncate", false, "Truncate state database") + cmd.Flags().BoolVar(&o.SkipImport, "skip-import", false, "Do not import existing seeds into state database") + + return cmd +} diff --git a/src/cmd/logconfig/logconfig.go b/cmd/import/error.go similarity index 68% rename from src/cmd/logconfig/logconfig.go rename to cmd/import/error.go index 5f56771..0997666 100644 --- a/src/cmd/logconfig/logconfig.go +++ b/cmd/import/error.go @@ -1,4 +1,4 @@ -// Copyright © 2017 National Library of Norway +// Copyright © 2023 National Library of Norway // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -11,15 +11,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package logconfig +package importcmd -import ( - "github.com/spf13/cobra" -) +import "fmt" -// reportCmd represents the report command -var LogconfigCmd = &cobra.Command{ - Use: "logconfig", - Short: "Configure logging", - Long: `Configure logging.`, +type errAlreadyExists struct { + key string +} + +func (e errAlreadyExists) Error() string { + return fmt.Sprintf("already exists: %s", e.key) } diff --git a/src/importutil/httpclient.go b/cmd/import/httpclient.go similarity index 70% rename from src/importutil/httpclient.go rename to cmd/import/httpclient.go index c084746..12bb6d0 100644 --- a/src/importutil/httpclient.go +++ b/cmd/import/httpclient.go @@ -11,24 +11,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -package importutil +package importcmd import ( "net/http" "time" ) -func NewHttpClient(timeout int64) *http.Client { - httpTimeout := time.Duration(timeout) * time.Millisecond - httpClient := &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, +func NewHttpClient(timeout time.Duration, followRedirects bool) *http.Client { + c := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, - ResponseHeaderTimeout: httpTimeout, + ResponseHeaderTimeout: timeout, }, - Timeout: httpTimeout, + Timeout: timeout, + } + if !followRedirects { + c.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } } - return httpClient + return c } diff --git a/src/cmd/importcmd/import.go b/cmd/import/import.go similarity index 65% rename from src/cmd/importcmd/import.go rename to cmd/import/import.go index 2280ca9..a71e146 100644 --- a/src/cmd/importcmd/import.go +++ b/cmd/import/import.go @@ -17,15 +17,17 @@ import ( "github.com/spf13/cobra" ) -// ImportCmd represents the import command -var ImportCmd = &cobra.Command{ - Use: "import", - Short: "Import data into Veidemann using subcommands", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - cmd.Help() - }, -} +func NewImportCmd() *cobra.Command { + cmd := &cobra.Command{ + GroupID: "advanced", + Use: "import", + Short: "Import data into Veidemann using subcommands", + Long: ``, + } + + cmd.AddCommand(newConvertOosCmd()) // convertoos + cmd.AddCommand(newImportSeedCmd()) // seed + cmd.AddCommand(newDuplicateReportCmd()) // duplicate -func init() { + return cmd } diff --git a/cmd/import/importseeds.go b/cmd/import/importseeds.go new file mode 100644 index 0000000..de27d81 --- /dev/null +++ b/cmd/import/importseeds.go @@ -0,0 +1,291 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package importcmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path" + "sync" + "time" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/config" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/importutil" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +type ImportSeedCmdOptions struct { + Toplevel bool + IgnoreScheme bool + CheckUri bool + Truncate bool + DryRun bool + SkipImport bool + CheckUriTimeout time.Duration + Filename string + ErrorFile string + CrawlJobId string + DbDir string + Concurrency int +} + +func (o *ImportSeedCmdOptions) run() error { + // Create error writer (file or stderr) + var errFile io.Writer + if o.ErrorFile == "" || o.ErrorFile == "-" { + errFile = os.Stderr + } else { + f, err := os.Create(o.ErrorFile) + if err != nil { + return fmt.Errorf("unable to open error file '%v': %w", o.ErrorFile, err) + } + defer f.Close() + errFile = f + } + + // Create Veidemann config client + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect %w", err) + } + defer conn.Close() + client := configV1.NewConfigClient(conn) + + // Create/open state database for entities + entityDbDir := path.Join(o.DbDir, config.GetContext(), configV1.Kind_crawlEntity.String()) + entityDb, err := importutil.NewImportDb(entityDbDir, o.Truncate) + if err != nil { + return fmt.Errorf("failed to initialize entity state db: %w", err) + } + defer entityDb.Close() + + uriNormalizer := &UriKeyNormalizer{ignoreScheme: o.IgnoreScheme, toplevel: o.Toplevel} + + // Create/open state database for seeds + seedDbDir := path.Join(o.DbDir, config.GetContext(), configV1.Kind_seed.String()) + seedDb, err := importutil.NewImportDb(seedDbDir, o.Truncate) + if err != nil { + return fmt.Errorf("failed to initialize seed state db: %w", err) + } + defer seedDb.Close() + + if !o.SkipImport { + // Import entities + err = importutil.ImportExisting(entityDb, client, configV1.Kind_crawlEntity, nil) + if err != nil { + return fmt.Errorf("failed to import entities: %w", err) + } + + // Import seeds + err = importutil.ImportExisting(seedDb, client, configV1.Kind_seed, uriNormalizer) + if err != nil { + return fmt.Errorf("failed to import seeds: %w", err) + } + } + + // Create Record reader for file input + rr, err := importutil.NewRecordReader(o.Filename, &JsonYamlDecoder{}, "*.json") + if err != nil { + return fmt.Errorf("failed to initialize reader: %w", err) + } + + var uriChecker *UriChecker + if o.CheckUri { + uriChecker = &UriChecker{ + Client: NewHttpClient(o.CheckUriTimeout, false), + } + } + + crawlJobRef := []*configV1.ConfigRef{{Kind: configV1.Kind_crawlJob, Id: o.CrawlJobId}} + + var m sync.Mutex + + // Create error logger + errorLog := log.Output(zerolog.ConsoleWriter{Out: errFile, TimeFormat: time.RFC3339}) + + // Create processor function for each record in input file + proc := func(sd *seedDesc) error { + if o.CrawlJobId != "" { + sd.crawlJobRef = crawlJobRef + } + + if uriChecker != nil { + // check liveness of uri and follow permanent redirects + uri, err := uriChecker.Check(sd.Uri) + if err != nil { + return err + } + sd.Uri = uri + } + + normalizedUri, err := uriNormalizer.Normalize(sd.Uri) + if err != nil { + return fmt.Errorf("failed to normalize URL '%s': %w", sd.Uri, err) + } + + // Ensure every concurrent process can see the same state by locking + // while checking and updating state database and Veidemann. + m.Lock() + defer m.Unlock() + + // Check if seed already exists in state database + seedIds, err := seedDb.Get(normalizedUri) + if err != nil { + return err + } else if len(seedIds) > 0 { + return errAlreadyExists{key: normalizedUri} + } + + if o.DryRun { + return nil + } + + var entity *configV1.ConfigObject + + if sd.EntityId == "" { + entityIds, err := entityDb.Get(sd.EntityName) + if err != nil { + return fmt.Errorf("failed to get entity: %w", err) + } + + if len(entityIds) > 0 { + sd.EntityId = entityIds[0] + } else { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + entity, err = client.SaveConfigObject(ctx, sd.toEntity()) + if err != nil { + return fmt.Errorf("failed to create entity in Veidemann: %w", err) + } + + _, _, err := entityDb.Set(sd.EntityName, entity.Id) + if err != nil { + return fmt.Errorf("failed to save new entity to import db: %w", err) + } + + sd.EntityId = entity.Id + errorLog.Info().Str("entityId", entity.Id).Str("entityName", entity.Meta.Name).Msg("Created new entity in Veidemann") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + seed, err := client.SaveConfigObject(ctx, sd.toSeed()) + if err != nil { + if entity != nil { // Delete created entity if seed creation failed + if _, err := client.DeleteConfigObject(ctx, entity); err != nil { + errorLog.Error().Err(err). + Str("uri", sd.Uri). + Str("entityId", entity.Id). + Str("entityName", entity.Meta.Name). + Msg("Failed to delete new entity from Veidemann after seed creation error") + } + } + return fmt.Errorf("failed to create seed in Veidemann: %w", err) + } + errorLog.Info().Str("key", normalizedUri).Str("seedId", seed.Id).Str("uri", sd.Uri).Msg("Created new seed in Veidemann") + + _, _, err = seedDb.Set(normalizedUri, seed.Id) + if err != nil { + return fmt.Errorf("failed to save new seed to import db: %w", err) + } + + return nil + } + + errHandler := func(state importutil.Job[*seedDesc]) { + l := errorLog.With(). + Str("uri", state.Val.Uri). + Str("filename", state.GetFilename()). + Int("recNum", state.GetRecordNum()).Logger() + + var err errAlreadyExists + if errors.As(state.GetError(), &err) { + l.Warn().Msgf("Skipping: %v", err.Error()) + } else { + l.Error().Err(state.GetError()).Msg("") + } + } + + executor := importutil.NewExecutor(o.Concurrency, proc, errHandler) + + // Process each record in input file and add to import db if not already present + for { + var sd seedDesc + state, err := rr.Next(&sd) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + errorLog.Error().Err(err).Msgf("error decoding record: %v", state) + continue + } + executor.Queue <- importutil.Job[*seedDesc]{State: state, Val: &sd} + } + + count, success, failed := executor.Wait() + + errorLog.Info().Int("processed", count).Int("imported", success).Int("errors", failed).Msg("Import completed") + + return err +} + +func newImportSeedCmd() *cobra.Command { + o := &ImportSeedCmdOptions{} + + // cmd represents the import command + cmd := &cobra.Command{ + Use: "seed", + Short: "Import seeds", + Long: `Import new seeds and entities from a line oriented JSON file on the following format: + +{"entityName":"foo","uri":"https://www.example.com/","entityDescription":"desc","entityLabel":[{"key":"foo","value":"bar"}],"seedLabel":[{"key":"foo","value":"bar"},{"key":"foo2","value":"bar"}],"seedDescription":"foo"} + +Every record must be formatted on a single line. + + +`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return o.run() + }, + } + + // filename is required + cmd.Flags().StringVarP(&o.Filename, "filename", "f", "", "Filename or directory to read from. "+ + "If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin.") + _ = cmd.MarkFlagRequired("filename") + cmd.Flags().StringVarP(&o.ErrorFile, "err-file", "e", "-", "File to write errors to. \"-\" writes to stderr") + cmd.Flags().BoolVarP(&o.Toplevel, "toplevel", "", false, "Convert URI by removing path") + cmd.Flags().BoolVarP(&o.IgnoreScheme, "ignore-scheme", "", true, "Ignore the URL's scheme when checking if this URL is already imported") + cmd.Flags().BoolVarP(&o.CheckUri, "check-uri", "", false, "Check the uri for liveness and follow permanent redirects") + cmd.Flags().DurationVarP(&o.CheckUriTimeout, "check-uri-timeout", "", 2*time.Second, "Timeout duration when checking uri for liveness") + cmd.Flags().StringVarP(&o.CrawlJobId, "crawljob-id", "", "", "Set crawlJob ID for new seeds") + cmd.Flags().StringVarP(&o.DbDir, "db-dir", "b", "/tmp/veidemannctl", "Directory for storing state db") + cmd.Flags().BoolVar(&o.Truncate, "truncate", false, "Truncate state database") + cmd.Flags().BoolVarP(&o.DryRun, "dry-run", "", false, "Run without actually writing anything to Veidemann") + cmd.Flags().BoolVar(&o.SkipImport, "skip-import", false, "Do not import existing seeds into state database") + cmd.Flags().IntVarP(&o.Concurrency, "concurrency", "c", 16, "Number of concurrent workers") + + return cmd +} diff --git a/src/cmd/importcmd/normalize.go b/cmd/import/normalize.go similarity index 80% rename from src/cmd/importcmd/normalize.go rename to cmd/import/normalize.go index 1aa240f..275c11e 100644 --- a/src/cmd/importcmd/normalize.go +++ b/cmd/import/normalize.go @@ -18,7 +18,6 @@ package importcmd import ( "errors" - "fmt" "net/url" "strings" ) @@ -28,19 +27,19 @@ type UriKeyNormalizer struct { ignoreScheme bool } -func (u *UriKeyNormalizer) Normalize(s string) (key string, err error) { +func (u *UriKeyNormalizer) Normalize(s string) (string, error) { uri, err := url.Parse(s) if err != nil { - return "", fmt.Errorf("unparseable URL '%v', cause: %v", s, err) - } - - if uri.Host == "" { - return "", errors.New("unparseable URL") + return "", err } uri.Fragment = "" uri.Host = strings.ToLower(uri.Host) + if uri.Hostname() == "" { + return "", errors.New("missing hostname") + } + if u.toplevel { uri.Path = "/" uri.RawQuery = "" @@ -54,7 +53,5 @@ func (u *UriKeyNormalizer) Normalize(s string) (key string, err error) { uri.Scheme = "" } - key = uri.String() - - return + return uri.String(), nil } diff --git a/src/cmd/importcmd/importseed_test.go b/cmd/import/normalize_test.go similarity index 63% rename from src/cmd/importcmd/importseed_test.go rename to cmd/import/normalize_test.go index b52ee1a..47b2499 100644 --- a/src/cmd/importcmd/importseed_test.go +++ b/cmd/import/normalize_test.go @@ -22,40 +22,40 @@ import ( func Test_importer_normalizeUri(t *testing.T) { tests := []struct { - name string - seedDescriptor *seedDesc - wantUri string - wantErr bool + name string + uri string + wantUri string + wantErr bool }{ - {"1", &seedDesc{Uri: "http://www.example.com"}, "http://www.example.com/", false}, - {"2", &seedDesc{Uri: "http://www.example.com#hash"}, "http://www.example.com/", false}, - {"3", &seedDesc{Uri: "http://www.example.com?query"}, "http://www.example.com/?query", false}, - {"4", &seedDesc{Uri: "http://www.example.com?query#hash"}, "http://www.example.com/?query", false}, - {"5", &seedDesc{Uri: "http://www.example.com/"}, "http://www.example.com/", false}, - {"6", &seedDesc{Uri: "http://www.example.com/#hash"}, "http://www.example.com/", false}, - {"7", &seedDesc{Uri: "http://www.example.com/?query"}, "http://www.example.com/?query", false}, - {"8", &seedDesc{Uri: "http://www.example.com/?query#hash"}, "http://www.example.com/?query", false}, - {"9", &seedDesc{Uri: "http://www.example.com/foo/bar"}, "http://www.example.com/foo/bar", false}, - {"10", &seedDesc{Uri: "http://www.example.com/foo/bar#hash"}, "http://www.example.com/foo/bar", false}, - {"11", &seedDesc{Uri: "http://www.example.com/foo/bar?query"}, "http://www.example.com/foo/bar?query", false}, - {"12", &seedDesc{Uri: "http://www.example.com/foo/bar?query#hash"}, "http://www.example.com/foo/bar?query", false}, - {"13", &seedDesc{Uri: "https://www.example.com/foo/bar#hash"}, "https://www.example.com/foo/bar", false}, - {"14", &seedDesc{Uri: "HTTPS://www.example.com/foo/bar#hash"}, "https://www.example.com/foo/bar", false}, - {"15", &seedDesc{Uri: "https://www.Example.Com/foo/bar#hash"}, "https://www.example.com/foo/bar", false}, - {"16", &seedDesc{Uri: "http://www.example.com/foo/"}, "http://www.example.com/foo/", false}, - {"17", &seedDesc{Uri: "http://www.example.com/foo"}, "http://www.example.com/foo", false}, - {"18", &seedDesc{Uri: "https://www.example.com/foo/"}, "https://www.example.com/foo/", false}, - {"19", &seedDesc{Uri: "https://www.example.com/foo"}, "https://www.example.com/foo", false}, + {"1", "http://www.example.com", "http://www.example.com/", false}, + {"2", "http://www.example.com#hash", "http://www.example.com/", false}, + {"3", "http://www.example.com?query", "http://www.example.com/?query", false}, + {"4", "http://www.example.com?query#hash", "http://www.example.com/?query", false}, + {"5", "http://www.example.com/", "http://www.example.com/", false}, + {"6", "http://www.example.com/#hash", "http://www.example.com/", false}, + {"7", "http://www.example.com/?query", "http://www.example.com/?query", false}, + {"8", "http://www.example.com/?query#hash", "http://www.example.com/?query", false}, + {"9", "http://www.example.com/foo/bar", "http://www.example.com/foo/bar", false}, + {"10", "http://www.example.com/foo/bar#hash", "http://www.example.com/foo/bar", false}, + {"11", "http://www.example.com/foo/bar?query", "http://www.example.com/foo/bar?query", false}, + {"12", "http://www.example.com/foo/bar?query#hash", "http://www.example.com/foo/bar?query", false}, + {"13", "https://www.example.com/foo/bar#hash", "https://www.example.com/foo/bar", false}, + {"14", "HTTPS://www.example.com/foo/bar#hash", "https://www.example.com/foo/bar", false}, + {"15", "https://www.Example.Com/foo/bar#hash", "https://www.example.com/foo/bar", false}, + {"16", "http://www.example.com/foo/", "http://www.example.com/foo/", false}, + {"17", "http://www.example.com/foo", "http://www.example.com/foo", false}, + {"18", "https://www.example.com/foo/", "https://www.example.com/foo/", false}, + {"19", "https://www.example.com/foo", "https://www.example.com/foo", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - i := &importer{} - importFlags.toplevel = false - if err := i.normalizeUri(tt.seedDescriptor); (err != nil) != tt.wantErr { + n := &UriKeyNormalizer{} + got, err := n.Normalize(tt.uri) + if (err != nil) != tt.wantErr { t.Errorf("normalizeUri() error = %v, wantErr %v", err, tt.wantErr) } - if tt.seedDescriptor.Uri != tt.wantUri { - t.Errorf("normalizeUri() uri = %v, want %v", tt.seedDescriptor.Uri, tt.wantUri) + if got != tt.wantUri { + t.Errorf("normalizeUri() uri = %v, want %v", got, tt.wantUri) } }) } @@ -90,13 +90,13 @@ func Test_UriKeyNormalizer_toplevelFlag(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - i := &UriKeyNormalizer{toplevel: true, ignoreScheme: false} - key, err := i.Normalize(tt.uri) + n := &UriKeyNormalizer{toplevel: true} + got, err := n.Normalize(tt.uri) if (err != nil) != tt.wantErr { t.Errorf("Normalize() error = %v, wantErr %v", err, tt.wantErr) } - if key != tt.wantKey { - t.Errorf("Normalize() key = %v, want %v", key, tt.wantKey) + if got != tt.wantKey { + t.Errorf("Normalize() key = %v, want %v", got, tt.wantKey) } }) } @@ -131,13 +131,13 @@ func Test_UriKeyNormalizer_ignoreSchemeFlag(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - i := &UriKeyNormalizer{toplevel: false, ignoreScheme: true} - key, err := i.Normalize(tt.uri) + i := &UriKeyNormalizer{ignoreScheme: true} + got, err := i.Normalize(tt.uri) if (err != nil) != tt.wantErr { t.Errorf("Normalize() error = %v, wantErr %v", err, tt.wantErr) } - if key != tt.wantKey { - t.Errorf("Normalize() key = %v, want %v", key, tt.wantKey) + if got != tt.wantKey { + t.Errorf("Normalize() key = %v, want %v", got, tt.wantKey) } }) } diff --git a/cmd/import/util.go b/cmd/import/util.go new file mode 100644 index 0000000..0268132 --- /dev/null +++ b/cmd/import/util.go @@ -0,0 +1,143 @@ +package importcmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "golang.org/x/net/html" +) + +// UriChecker checks if a uri is reachable +type UriChecker struct { + *http.Client +} + +// Check checks if a uri is reachable and returns the uri if it is reachable +// If the uri is not reachable, it returns an error +// If the uri is redirected with 301, it returns the redirected uri +func (uc *UriChecker) Check(uri string) (string, error) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodHead, uri, nil) + if err != nil { + return "", err + } + resp, err := uc.Client.Do(req) + if err != nil { + var uerr *url.Error + if errors.As(err, &uerr) && uerr.Timeout() { + return "", fmt.Errorf("timeout") + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return "", fmt.Errorf("no such host: %s", dnsErr.Name) + } + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusMovedPermanently { + u, err := resp.Location() + if err != nil { + return "", err + } + return u.String(), nil + } + + if resp.StatusCode < 400 { + return uri, nil + } + + return uri, fmt.Errorf("%s", resp.Status) +} + +// GetTitle returns the title of the uri +func (uc *UriChecker) GetTitle(uri string) string { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, uri, nil) + if err != nil { + return "" + } + resp, err := uc.Client.Do(req) + if err != nil { + return "" + } + defer resp.Body.Close() + + doc, err := html.Parse(resp.Body) + if err != nil { + return "" + } + var title string + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { + title = strings.TrimSpace(n.FirstChild.Data) + return + } + if n.Type == html.ElementNode && n.Data == "body" { + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + return title +} + +type seedDesc struct { + EntityId string `json:"entityId,omitempty" yaml:"entityId,omitempty"` + EntityName string `json:"entityName,omitempty" yaml:"entityName,omitempty"` + EntityDescription string `json:"entityDescription,omitempty" yaml:"entityDescription,omitempty"` + EntityLabel []*configV1.Label `json:"entityLabel,omitempty" yaml:"entityLabel,omitempty"` + Uri string `json:"uri,omitempty" yaml:"uri,omitempty"` + SeedDescription string `json:"seedDescription,omitempty" yaml:"seedDescription,omitempty"` + SeedLabel []*configV1.Label `json:"seedLabel,omitempty" yaml:"seedLabel,omitempty"` + + Description string `json:"description,omitempty" yaml:"description,omitempty"` + crawlJobRef []*configV1.ConfigRef +} + +func (sd *seedDesc) String() string { + b, _ := json.Marshal(sd) + return string(b) +} + +func (sd *seedDesc) toEntity() *configV1.ConfigObject { + return &configV1.ConfigObject{ + ApiVersion: "v1", + Kind: configV1.Kind_crawlEntity, + Id: sd.EntityId, + Meta: &configV1.Meta{ + Name: sd.EntityName, + Description: sd.EntityDescription, + Label: sd.EntityLabel, + }, + } +} + +func (sd *seedDesc) toSeed() *configV1.ConfigObject { + return &configV1.ConfigObject{ + ApiVersion: "v1", + Kind: configV1.Kind_seed, + Meta: &configV1.Meta{ + Name: sd.Uri, + Description: sd.SeedDescription, + Label: sd.SeedLabel, + }, + Spec: &configV1.ConfigObject_Seed{ + Seed: &configV1.Seed{ + EntityRef: &configV1.ConfigRef{ + Kind: configV1.Kind_crawlEntity, + Id: sd.EntityId, + }, + JobRef: sd.crawlJobRef, + }, + }, + } +} diff --git a/cmd/import/util_test.go b/cmd/import/util_test.go new file mode 100644 index 0000000..ca3277b --- /dev/null +++ b/cmd/import/util_test.go @@ -0,0 +1,32 @@ +package importcmd + +import ( + "testing" + "time" +) + +func TestUriChecker(t *testing.T) { + + uriChecker := &UriChecker{ + Client: NewHttpClient(1*time.Second, false), + } + + tests := []struct { + uri string + want string + }{ + {"https://www.nb.no/", "https://www.nb.no/"}, + {"https://lokalhistoriewiki.no", "https://lokalhistoriewiki.no/wiki/Lokalhistoriewiki:Hovedside"}, + } + + for _, tt := range tests { + got, err := uriChecker.Check(tt.uri) + if err != nil { + t.Error(err) + } + + if got != tt.want { + t.Errorf("Want %s, got %s", tt.want, got) + } + } +} diff --git a/cmd/logconfig/delete.go b/cmd/logconfig/delete.go new file mode 100644 index 0000000..6c1d62e --- /dev/null +++ b/cmd/logconfig/delete.go @@ -0,0 +1,87 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logconfig + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/ptypes/empty" + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +type deleteLoggerCmdOptions struct { + logger string +} + +func (o *deleteLoggerCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.logger = args[0] + + return nil +} + +func (o *deleteLoggerCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + r, err := client.GetLogConfig(context.Background(), &empty.Empty{}) + if err != nil { + return fmt.Errorf("failed to get log config: %w", err) + } + + loggers := make(map[string]configV1.LogLevels_Level) + for _, l := range r.LogLevel { + if l.Logger != "" && l.Logger != o.logger { + loggers[l.Logger] = l.Level + } + } + + n := &configV1.LogLevels{} + for k, v := range loggers { + n.LogLevel = append(n.LogLevel, &configV1.LogLevels_LogLevel{Logger: k, Level: v}) + } + + _, err = client.SaveLogConfig(context.Background(), n) + if err != nil { + return fmt.Errorf("failed to save log config: %w", err) + } + return nil +} + +func newDeleteLoggerCmd() *cobra.Command { + o := &deleteLoggerCmdOptions{} + + return &cobra.Command{ + Use: "delete LOGGER", + Short: "Delete a logger", + Long: `Delete a logger.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.complete(cmd, args); err != nil { + return err + } + // silence usage to prevent printing usage when error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } +} diff --git a/cmd/logconfig/list.go b/cmd/logconfig/list.go new file mode 100644 index 0000000..fc39f7c --- /dev/null +++ b/cmd/logconfig/list.go @@ -0,0 +1,53 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logconfig + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/ptypes/empty" + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +func newListLoggersCmd() *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List configured loggers", + Long: `List configured loggers.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + r, err := client.GetLogConfig(context.Background(), &empty.Empty{}) + if err != nil { + return fmt.Errorf("could not get log config: %w", err) + } + + fmt.Printf("%-45s %s\n", "LOGGER", "LEVEL") + for _, l := range r.LogLevel { + fmt.Printf("%-45s %s\n", l.Logger, l.Level) + } + return nil + }, + } +} diff --git a/src/cmd/reports/report.go b/cmd/logconfig/logconfig.go similarity index 64% rename from src/cmd/reports/report.go rename to cmd/logconfig/logconfig.go index e563b6e..10b9146 100644 --- a/src/cmd/reports/report.go +++ b/cmd/logconfig/logconfig.go @@ -11,25 +11,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -package reports +package logconfig import ( "github.com/spf13/cobra" ) -func NewReportCmd() *cobra.Command { - // reportCmd represents the report command - var cmd = &cobra.Command{ - Use: "report", - Short: "Get report", - Long: `Request a report.`, +func NewLogConfigCmd() *cobra.Command { + cmd := &cobra.Command{ + GroupID: "debug", + Use: "logconfig", + Short: "Configure logging", + Long: `Configure logging.`, } - cmd.AddCommand(newJobExecutionCmd()) - cmd.AddCommand(newCrawlExecutionCmd()) - cmd.AddCommand(crawllogCmd) - cmd.AddCommand(queryCmd) - cmd.AddCommand(pagelogCmd) + cmd.AddCommand(newDeleteLoggerCmd()) // delete + cmd.AddCommand(newListLoggersCmd()) // list + cmd.AddCommand(newSetLoggerCmd()) // set return cmd } diff --git a/cmd/logconfig/set.go b/cmd/logconfig/set.go new file mode 100644 index 0000000..65cf46e --- /dev/null +++ b/cmd/logconfig/set.go @@ -0,0 +1,100 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logconfig + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/ptypes/empty" + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +type setLoggerCmdOptions struct { + logger string + level configV1.LogLevels_Level +} + +func (o *setLoggerCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.logger = args[0] + + l, ok := configV1.LogLevels_Level_value[args[1]] + if !ok { + return fmt.Errorf("invalid log level: %s", args[1]) + } + + level := configV1.LogLevels_Level(l) + if level == configV1.LogLevels_UNDEFINED { + return fmt.Errorf("invalid log level: %s", args[1]) + } + + return nil +} + +func (o *setLoggerCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + r, err := client.GetLogConfig(context.Background(), &empty.Empty{}) + if err != nil { + return fmt.Errorf("could not get log config: %w", err) + } + + loggers := make(map[string]configV1.LogLevels_Level) + for _, l := range r.LogLevel { + if l.Logger != "" { + loggers[l.Logger] = l.Level + } + } + + loggers[o.logger] = o.level + n := &configV1.LogLevels{} + for k, v := range loggers { + n.LogLevel = append(n.LogLevel, &configV1.LogLevels_LogLevel{Logger: k, Level: v}) + } + + _, err = client.SaveLogConfig(context.Background(), n) + if err != nil { + return fmt.Errorf("failed to save log config: %w", err) + } + return nil +} + +func newSetLoggerCmd() *cobra.Command { + + return &cobra.Command{ + Use: "set LOGGER LEVEL", + Short: "Configure logger", + Long: `Configure logger.`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + o := &setLoggerCmdOptions{} + if err := o.complete(cmd, args); err != nil { + return err + } + + // silence usage to avoid printing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } +} diff --git a/cmd/login/login.go b/cmd/login/login.go new file mode 100644 index 0000000..c9ade8c --- /dev/null +++ b/cmd/login/login.go @@ -0,0 +1,39 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package login + +import ( + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +func NewLoginCmd() *cobra.Command { + var manualLogin bool + + var cmd = &cobra.Command{ + GroupID: "login", + Use: "login", + Short: "Log in to Veidemann", + Long: ``, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + return connection.Login(manualLogin) + }, + } + + cmd.Flags().BoolVarP(&manualLogin, "manual", "m", false, + "Manually copy and paste login url and code. Used to log in from a remote terminal.") + + return cmd +} diff --git a/src/cmd/config/config.go b/cmd/logout/logout.go similarity index 66% rename from src/cmd/config/config.go rename to cmd/logout/logout.go index f0bb19d..5531431 100644 --- a/src/cmd/config/config.go +++ b/cmd/logout/logout.go @@ -11,21 +11,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package config +package logout import ( + "github.com/nlnwa/veidemannctl/connection" "github.com/spf13/cobra" ) -// configCmd represents the config command -var ConfigCmd = &cobra.Command{ - Use: "config", - Short: "Modify veidemannctl config files using subcommands", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - cmd.Help() - }, -} - -func init() { +func NewLogoutCmd() *cobra.Command { + return &cobra.Command{ + GroupID: "login", + Use: "logout", + Short: "Log out of Veidemann", + Long: `Log out of Veidemann.`, + RunE: func(cmd *cobra.Command, args []string) error { + return connection.Logout() + }, + } } diff --git a/src/cmd/pause.go b/cmd/pause/pause.go similarity index 53% rename from src/cmd/pause.go rename to cmd/pause/pause.go index 8f40354..d2b3091 100644 --- a/src/cmd/pause.go +++ b/cmd/pause/pause.go @@ -11,40 +11,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cmd +package pause import ( "context" - "log" "github.com/golang/protobuf/ptypes/empty" - "github.com/nlnwa/veidemannctl/src/connection" + controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/connection" "github.com/spf13/cobra" ) -// pauseCmd represents the pause command -var pauseCmd = &cobra.Command{ - Use: "pause", - Short: "Pause crawler", - Long: `Pause crawler`, - Run: func(cmd *cobra.Command, args []string) { - err := pauseCrawler() - if err != nil { - log.Fatalf("could not pause crawler: %v", err) - } - }, -} - -func pauseCrawler() error { - client, conn := connection.NewControllerClient() - defer func() { - _ = conn.Close() - }() +func NewPauseCmd() *cobra.Command { + return &cobra.Command{ + GroupID: "status", + Use: "pause", + Short: "Request crawler to pause", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() - _, err := client.PauseCrawler(context.Background(), &empty.Empty{}) - return err -} + client := controllerV1.NewControllerClient(conn) -func init() { - RootCmd.AddCommand(pauseCmd) + _, err = client.PauseCrawler(context.Background(), &empty.Empty{}) + return err + }, + } } diff --git a/cmd/report/crawlexecution.go b/cmd/report/crawlexecution.go new file mode 100644 index 0000000..c5d2ec8 --- /dev/null +++ b/cmd/report/crawlexecution.go @@ -0,0 +1,200 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" + frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" + reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" + "github.com/nlnwa/veidemannctl/apiutil" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// crawlExecCmdOptions holds the crawl execution command line options +type crawlExecCmdOptions struct { + ids []string + filters []string + pageSize int32 + page int32 + goTemplate string + format string + file string + orderByPath string + orderDesc bool + to *time.Time + from *time.Time + watch bool + states []string +} + +// complete completes the crawl execution command options +func (o *crawlExecCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.ids = args + + // use viper to parse time for the flags "to" and "from" + v := viper.New() + + if err := v.BindPFlag("to", cmd.Flag("to")); err != nil { + return fmt.Errorf("failed to bind flag: %w", err) + } else if v.IsSet("to") { + to := v.GetTime("to") + o.to = &to + } + if err := v.BindPFlag("from", cmd.Flag("from")); err != nil { + return fmt.Errorf("failed to bind flag: %w", err) + } else if v.IsSet("from") { + from := v.GetTime("from") + o.from = &from + } + return nil +} + +// run runs the crawl execution command +func (o *crawlExecCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := reportV1.NewReportClient(conn) + + request, err := o.createCrawlExecutionsListRequest() + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + r, err := client.ListExecutions(context.Background(), request) + if err != nil { + return fmt.Errorf("failed to list crawl executions: %w", err) + } + out, err := format.ResolveWriter(o.file) + if err != nil { + return fmt.Errorf("unable to open output file: %v: %w", o.file, err) + } + s, err := format.NewFormatter("CrawlExecutionStatus", out, o.format, o.goTemplate) + if err != nil { + return err + } + defer s.Close() + + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + if err := s.WriteRecord(msg); err != nil { + return err + } + } + return nil +} + +// newCrawlExecutionCmd creates the crawl execution command +func newCrawlExecutionCmd() *cobra.Command { + o := &crawlExecCmdOptions{} + + cmd := &cobra.Command{ + Use: "crawlexecution [ID ...]", + Short: "Get current status for crawl executions", + Long: `Get current status for crawl executions.`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.complete(cmd, args); err != nil { + return err + } + cmd.SilenceUsage = true + + return o.run() + }, + } + cmd.Flags().Int32VarP(&o.pageSize, "pagesize", "s", 10, "Number of objects to get") + cmd.Flags().Int32VarP(&o.page, "page", "p", 0, "The page number") + cmd.Flags().StringVarP(&o.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") + cmd.Flags().StringVarP(&o.goTemplate, "template", "t", "", "A Go template used to format the output") + cmd.Flags().StringSliceVarP(&o.filters, "filter", "q", nil, "Filter objects by field (i.e. meta.description=foo)") + cmd.Flags().StringSliceVar(&o.states, "state", nil, "Filter objects by state. Valid states are UNDEFINED, FETCHING, SLEEPING, FINISHED or FAILED") + cmd.Flags().StringVarP(&o.file, "filename", "f", "", "Filename to write to") + cmd.Flags().StringVar(&o.orderByPath, "order-by", "", "Order by path") + cmd.Flags().String("to", "", "To start time") + cmd.Flags().String("from", "", "From start time") + cmd.Flags().BoolVar(&o.orderDesc, "desc", false, "Order descending") + cmd.Flags().BoolVarP(&o.watch, "watch", "w", false, "Get a continous stream of changes") + + return cmd +} + +// createCrawlExecutionsListRequest creates a crawl execution list request +func (o *crawlExecCmdOptions) createCrawlExecutionsListRequest() (*reportV1.CrawlExecutionsListRequest, error) { + request := &reportV1.CrawlExecutionsListRequest{ + Id: o.ids, + Watch: o.watch, + PageSize: o.pageSize, + Offset: o.page, + OrderByPath: o.orderByPath, + OrderDescending: o.orderDesc, + } + + if o.watch { + request.PageSize = 0 + } + + if o.from != nil { + fmt.Println(o.from) + request.StartTimeFrom = timestamppb.New(*o.from) + } + + if o.to != nil { + request.StartTimeTo = timestamppb.New(*o.to) + } + + if len(o.states) > 0 { + for _, state := range o.states { + if s, ok := frontierV1.CrawlExecutionStatus_State_value[state]; !ok { + return nil, fmt.Errorf("not a crawlexecution state: %s", state) + } else { + request.State = append(request.State, frontierV1.CrawlExecutionStatus_State(s)) + } + } + } + + if len(o.filters) > 0 { + queryTemplate := new(frontierV1.CrawlExecutionStatus) + queryMask := new(commonsV1.FieldMask) + + for _, filter := range o.filters { + err := apiutil.CreateTemplateFilter(filter, queryTemplate, queryMask) + if err != nil { + return nil, err + } + } + + request.QueryMask = queryMask + request.QueryTemplate = queryTemplate + } + + return request, nil +} diff --git a/cmd/report/crawllog.go b/cmd/report/crawllog.go new file mode 100644 index 0000000..36884e3 --- /dev/null +++ b/cmd/report/crawllog.go @@ -0,0 +1,138 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "io" + + commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" + logV1 "github.com/nlnwa/veidemann-api/go/log/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +type crawlLogCmdOptions struct { + ids []string + executionId string + pageSize int32 + page int32 + goTemplate string + format string + file string +} + +func (o *crawlLogCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.ids = args + + if len(o.ids) == 0 && o.executionId == "" { + return fmt.Errorf("request must provide either warcId or executionId") + } + // set silence usage to true to avoid printing usage when an error occurs + cmd.SilenceUsage = true + + return nil +} + +func (o *crawlLogCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := logV1.NewLogClient(conn) + + request, err := createCrawlLogListRequest(o) + if err != nil { + return fmt.Errorf("error creating request: %w", err) + } + + r, err := client.ListCrawlLogs(context.Background(), request) + if err != nil { + return fmt.Errorf("error from controller: %w", err) + } + + out, err := format.ResolveWriter(o.file) + if err != nil { + return fmt.Errorf("unable to open output file: %v: %w", o.file, err) + } + s, err := format.NewFormatter("CrawlLog", out, o.format, o.goTemplate) + if err != nil { + return err + } + defer s.Close() + + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + log.Debug().Msgf("Outputting crawl log record with WARC id '%s'", msg.WarcId) + if err := s.WriteRecord(msg); err != nil { + return err + } + } + return nil +} + +func newCrawlLogCmd() *cobra.Command { + o := &crawlLogCmdOptions{} + cmd := &cobra.Command{ + Use: "crawllog [ID ...]", + Short: "View crawl log", + Long: `View crawl log.`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.complete(cmd, args); err != nil { + return err + } + return o.run() + }, + } + + cmd.Flags().Int32VarP(&o.pageSize, "pagesize", "s", 10, "Number of objects to get") + cmd.Flags().Int32VarP(&o.page, "page", "p", 0, "The page number") + cmd.Flags().StringVarP(&o.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") + cmd.Flags().StringVarP(&o.goTemplate, "template", "t", "", "A Go template used to format the output") + cmd.Flags().StringVarP(&o.file, "filename", "f", "", "Filename to write to") + cmd.Flags().StringVar(&o.executionId, "execution-id", "", "Execution ID") + + return cmd +} + +func createCrawlLogListRequest(o *crawlLogCmdOptions) (*logV1.CrawlLogListRequest, error) { + request := &logV1.CrawlLogListRequest{} + request.WarcId = o.ids + request.Offset = o.page + request.PageSize = o.pageSize + + if o.executionId != "" { + queryMask := new(commonsV1.FieldMask) + queryMask.Paths = append(queryMask.Paths, "executionId") + queryTemplate := new(logV1.CrawlLog) + queryTemplate.ExecutionId = o.executionId + + request.QueryMask = queryMask + request.QueryTemplate = queryTemplate + } + + return request, nil +} diff --git a/cmd/report/jobexecution.go b/cmd/report/jobexecution.go new file mode 100644 index 0000000..7bdca84 --- /dev/null +++ b/cmd/report/jobexecution.go @@ -0,0 +1,183 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" + frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" + reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" + "github.com/nlnwa/veidemannctl/apiutil" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +type jobExecutionCmdOptions struct { + ids []string + filters []string + states []string + pageSize int32 + page int32 + orderByPath string + orderDesc bool + to *time.Time + from *time.Time + goTemplate string + format string + file string + watch bool +} + +func (o *jobExecutionCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.ids = args + + v := viper.New() + + if err := v.BindPFlag("to", cmd.Flag("to")); err != nil { + return fmt.Errorf("failed to bind flag: %w", err) + } + if v.IsSet("to") { + to := v.GetTime("to") + o.to = &to + } + if err := v.BindPFlag("from", cmd.Flag("from")); err != nil { + return fmt.Errorf("failed to bind flag: %w", err) + } + if v.IsSet("from") { + from := v.GetTime("from") + o.from = &from + } + + return nil +} + +func (o *jobExecutionCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := reportV1.NewReportClient(conn) + + request, err := o.createJobExecutionsListRequest() + if err != nil { + return fmt.Errorf("error creating request: %w", err) + } + + r, err := client.ListJobExecutions(context.Background(), request) + if err != nil { + return fmt.Errorf("error from controller: %w", err) + } + + w, err := format.ResolveWriter(o.file) + if err != nil { + return fmt.Errorf("unable to open output file: %v: %w", o.file, err) + } + s, err := format.NewFormatter("JobExecutionStatus", w, o.format, o.goTemplate) + if err != nil { + return fmt.Errorf("error creating formatter: %w", err) + } + defer s.Close() + + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + if err := s.WriteRecord(msg); err != nil { + return err + } + } + return nil +} + +func newJobExecutionCmd() *cobra.Command { + o := &jobExecutionCmdOptions{} + var cmd = &cobra.Command{ + Use: "jobexecution [ID ...]", + Short: "Get current status for job executions", + Long: `Get current status for job executions.`, + PreRunE: o.complete, + RunE: func(cmd *cobra.Command, args []string) error { + // set silence usage to true to avoid printing usage when an error occurs + cmd.SilenceUsage = true + return o.run() + }, + } + + cmd.Flags().Int32VarP(&o.pageSize, "pagesize", "s", 10, "Number of objects to get") + cmd.Flags().Int32VarP(&o.page, "page", "p", 0, "The page number") + cmd.Flags().StringVarP(&o.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") + cmd.Flags().StringVarP(&o.goTemplate, "template", "t", "", "A Go template used to format the output") + cmd.Flags().StringSliceVarP(&o.filters, "filter", "q", nil, "Filter objects by field (i.e. meta.description=foo") + cmd.Flags().StringSliceVar(&o.states, "state", nil, "Filter objects by state(s)") + cmd.Flags().StringVarP(&o.file, "filename", "f", "", "Filename to write to") + cmd.Flags().StringVar(&o.orderByPath, "order-by", "", "Order by path") + cmd.Flags().BoolVar(&o.orderDesc, "desc", false, "Order descending") + cmd.Flags().String("to", "", "To start time") + cmd.Flags().String("from", "", "From start time") + cmd.Flags().BoolVarP(&o.watch, "watch", "w", false, "Get a continous stream of changes") + + return cmd +} + +func (o *jobExecutionCmdOptions) createJobExecutionsListRequest() (*reportV1.JobExecutionsListRequest, error) { + request := &reportV1.JobExecutionsListRequest{ + Id: o.ids, + Watch: o.watch, + PageSize: o.pageSize, + Offset: o.page, + OrderByPath: o.orderByPath, + OrderDescending: o.orderDesc, + } + if o.watch { + request.PageSize = 0 + } + + if len(o.states) > 0 { + for _, state := range o.states { + if s, ok := frontierV1.JobExecutionStatus_State_value[state]; !ok { + return nil, fmt.Errorf("not a jobexecution state: %s", state) + } else { + request.State = append(request.State, frontierV1.JobExecutionStatus_State(s)) + } + } + } + + if len(o.filters) > 0 { + queryMask := new(commonsV1.FieldMask) + queryTemplate := new(frontierV1.JobExecutionStatus) + for _, filter := range o.filters { + err := apiutil.CreateTemplateFilter(filter, queryTemplate, queryMask) + if err != nil { + return nil, err + } + } + request.QueryMask = queryMask + request.QueryTemplate = queryTemplate + } + + return request, nil +} diff --git a/src/cmd/reports/jobexecution_test.go b/cmd/report/jobexecution_test.go similarity index 78% rename from src/cmd/reports/jobexecution_test.go rename to cmd/report/jobexecution_test.go index 4d0ad8e..479e88e 100644 --- a/src/cmd/reports/jobexecution_test.go +++ b/cmd/report/jobexecution_test.go @@ -1,4 +1,4 @@ -// Copyright © 2018 NAME HERE +// Copyright © 2018 National Library of Norway // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package reports +package report import ( + "testing" + "github.com/nlnwa/veidemann-api/go/commons/v1" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/proto" - "testing" frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" @@ -27,40 +28,46 @@ import ( func TestCreateJobExecutionsListRequest(t *testing.T) { tests := []struct { name string - ids []string - flags jobExecFlags + opt jobExecutionCmdOptions want *reportV1.JobExecutionsListRequest wantErr bool }{ - {"1", nil, - jobExecFlags{pageSize: 10}, + { + "1", + jobExecutionCmdOptions{pageSize: 10}, &reportV1.JobExecutionsListRequest{PageSize: 10}, - false}, - {"2", []string{"id1", "id2"}, - jobExecFlags{pageSize: 10}, + false, + }, + { + "2", + jobExecutionCmdOptions{ids: []string{"id1", "id2"}, pageSize: 10}, &reportV1.JobExecutionsListRequest{Id: []string{"id1", "id2"}, PageSize: 10}, - false}, - {"3", nil, - jobExecFlags{filters: []string{"jobId=jobId1"}, pageSize: 10}, + false, + }, + { + "3", + jobExecutionCmdOptions{filters: []string{"jobId=jobId1"}, pageSize: 10}, &reportV1.JobExecutionsListRequest{ QueryTemplate: &frontierV1.JobExecutionStatus{JobId: "jobId1"}, QueryMask: &commons.FieldMask{Paths: []string{"jobId"}}, PageSize: 10, }, - false}, - {"4", nil, - jobExecFlags{filters: []string{"jobId=jobId1"}, pageSize: 10, watch: true}, + false, + }, + { + "4", + jobExecutionCmdOptions{filters: []string{"jobId=jobId1"}, pageSize: 10, watch: true}, &reportV1.JobExecutionsListRequest{ QueryTemplate: &frontierV1.JobExecutionStatus{JobId: "jobId1"}, QueryMask: &commons.FieldMask{Paths: []string{"jobId"}}, Watch: true, }, - false}, + false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - jobExecConf = tt.flags - got, err := createJobExecutionsListRequest(tt.ids) + got, err := tt.opt.createJobExecutionsListRequest() if (err != nil) != tt.wantErr { t.Errorf("CreateJobExecutionsListRequest() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/cmd/report/pagelog.go b/cmd/report/pagelog.go new file mode 100644 index 0000000..0008345 --- /dev/null +++ b/cmd/report/pagelog.go @@ -0,0 +1,135 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "io" + + commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" + logV1 "github.com/nlnwa/veidemann-api/go/log/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/spf13/cobra" +) + +type pageLogCmdOptions struct { + ids []string + executionId string + pageSize int32 + page int32 + goTemplate string + format string + file string +} + +func (o *pageLogCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.ids = args + if len(o.ids) == 0 && o.executionId == "" { + return fmt.Errorf("request must provide either warcId or executionId") + } + // set silence usage to true to avoid printing usage when an error occurs + cmd.SilenceUsage = true + return nil +} + +func (o *pageLogCmdOptions) run() error { + // initialize output writer + out, err := format.ResolveWriter(o.file) + if err != nil { + return fmt.Errorf("error opening output file: %w", err) + } + s, err := format.NewFormatter("PageLog", out, o.format, o.goTemplate) + if err != nil { + return err + } + defer s.Close() + + // connect to grpc server + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := logV1.NewLogClient(conn) + + // create request + request := o.createPageLogListRequest() + + r, err := client.ListPageLogs(context.Background(), request) + if err != nil { + return fmt.Errorf("could not get page log: %w", err) + } + + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("error getting object: %w", err) + } + if err := s.WriteRecord(msg); err != nil { + return err + } + } + return nil +} + +func newPageLogCmd() *cobra.Command { + o := &pageLogCmdOptions{} + + cmd := &cobra.Command{ + Use: "pagelog [ID ...]", + Short: "View page log", + Long: `View page log.`, + PreRunE: o.complete, + RunE: func(cmd *cobra.Command, args []string) error { + // set silence usage to true to avoid printing usage when an error occurs + cmd.SilenceUsage = true + return o.run() + }, + } + + cmd.Flags().Int32VarP(&o.pageSize, "pagesize", "s", 10, "Number of objects to get") + cmd.Flags().Int32VarP(&o.page, "page", "p", 0, "The page number") + cmd.Flags().StringVarP(&o.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") + cmd.Flags().StringVarP(&o.goTemplate, "template", "t", "", "A Go template used to format the output") + cmd.Flags().StringVarP(&o.file, "filename", "f", "", "Filename to write to") + cmd.Flags().StringVar(&o.executionId, "execution-id", "", "Execution ID") + + return cmd +} + +func (o *pageLogCmdOptions) createPageLogListRequest() *logV1.PageLogListRequest { + request := &logV1.PageLogListRequest{} + request.WarcId = o.ids + request.Offset = o.page + request.PageSize = o.pageSize + + if o.executionId != "" { + queryMask := new(commonsV1.FieldMask) + queryMask.Paths = append(queryMask.Paths, "executionId") + queryTemplate := new(logV1.PageLog) + queryTemplate.ExecutionId = o.executionId + + request.QueryMask = queryMask + request.QueryTemplate = queryTemplate + } + + return request +} diff --git a/cmd/report/query.go b/cmd/report/query.go new file mode 100644 index 0000000..255bb47 --- /dev/null +++ b/cmd/report/query.go @@ -0,0 +1,255 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package report + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/invopop/yaml" + reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" + "github.com/nlnwa/veidemannctl/config" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" +) + +type queryCmdOptions struct { + queryOrFile string + queryArgs []any + pageSize int32 + page int32 + goTemplate string + file string + format string +} + +func (o *queryCmdOptions) complete(cmd *cobra.Command, args []string) error { + o.queryOrFile = args[0] + + for _, arg := range args[1:] { + o.queryArgs = append(o.queryArgs, arg) + } + + return nil +} + +// run runs the query command +func (o *queryCmdOptions) run() error { + q, err := o.parseQuery() + if err != nil { + return fmt.Errorf("failed to parse query: %w", err) + } + + request := reportV1.ExecuteDbQueryRequest{ + Query: q.query, + Limit: o.pageSize, + } + + log.Debug().Msgf("Executing query: %s", request.GetQuery()) + + var w io.Writer + + if o.file == "" || o.file == "-" { + w = os.Stdout + } else { + w, err = os.Create(o.file) + if err != nil { + return fmt.Errorf("failed to create file: %s: %w", o.file, err) + } + } + + var formatter format.Formatter + if q.template == "" { + formatter, err = format.NewFormatter("", w, o.format, o.goTemplate) + } else { + formatter, err = format.NewFormatter("", w, "template", q.template) + } + if err != nil { + return fmt.Errorf("failed to create formatter: %w", err) + } + + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := reportV1.NewReportClient(conn) + + stream, err := client.ExecuteDbQuery(context.Background(), &request) + if err != nil { + return fmt.Errorf("failed executing query: %w", err) + } + + for { + value, err := stream.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("failed to receive a value: %w", err) + } + if err := formatter.WriteRecord(value.GetRecord()); err != nil { + return err + } + } + + return nil +} + +func newQueryCmd() *cobra.Command { + o := &queryCmdOptions{} + + cmd := &cobra.Command{ + Use: "query (QUERY-STRING | FILENAME) [ARGS ...]", + Short: "Run a database query", + Long: `Run a database query. The query should be a JavaScript string like the ones used by RethinkDB JavaScript driver.`, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + d, err := config.GetConfigPath("query") + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + q := listStoredQueries(d) + var names []string + for _, s := range q { + if strings.HasPrefix(s.name, toComplete) { + names = append(names, s.name+"\t"+s.description) + } + } + return names, cobra.ShellCompDirectiveDefault + }, + PreRunE: o.complete, + RunE: func(cmd *cobra.Command, args []string) error { + // set silence usage to true to avoid printing usage when an error occurs + cmd.SilenceUsage = true + + return o.run() + }, + } + + cmd.Flags().Int32VarP(&o.pageSize, "pagesize", "s", 10, "Number of objects to get") + cmd.Flags().Int32VarP(&o.page, "page", "p", 0, "The page number") + cmd.Flags().StringVarP(&o.format, "output", "o", "json", "Output format (json|yaml|template|template-file)") + cmd.Flags().StringVarP(&o.goTemplate, "template", "t", "", "A Go template used to format the output") + cmd.Flags().StringVarP(&o.file, "filename", "f", "", "Filename to write to") + + return cmd +} + +// query is a struct for holding query definitions +type query struct { + name string + description string + query string + template string +} + +func (o *queryCmdOptions) parseQuery() (*query, error) { + var q *query + + if strings.HasPrefix(o.queryOrFile, "r.") { + q = &query{ + query: o.queryOrFile, + } + } else { + filename, err := findQueryFile(o.queryOrFile) + if err != nil { + return nil, err + } + log.Debug().Msgf("Using query definition from file '%s'", filename) + q, err = readQuery(filename) + if err != nil { + return nil, err + } + } + + q.query = fmt.Sprintf(q.query, o.queryArgs...) + + return q, nil +} + +// findQueryFile finds a query file +func findQueryFile(name string) (string, error) { + filename := name + if _, err := os.Stat(filename); !os.IsNotExist(err) { + return filename, nil + } + + queryDir, err := config.GetConfigPath("query") + if err != nil { + return "", fmt.Errorf("failed to resolve query dir: %w", err) + } + + filename = filepath.Join(queryDir, name) + if _, err := os.Stat(filename); !os.IsNotExist(err) { + return filename, nil + } + filename = filepath.Join(queryDir, name) + ".yml" + if _, err := os.Stat(filename); !os.IsNotExist(err) { + return filename, nil + } + filename = filepath.Join(queryDir, name) + ".yaml" + if _, err := os.Stat(filename); !os.IsNotExist(err) { + return filename, nil + } + return "", fmt.Errorf("query not found: %s", name) +} + +// readQuery reads a query definition from a file +func readQuery(name string) (*query, error) { + data, err := os.ReadFile(name) + if err != nil { + return nil, fmt.Errorf("failed to read file: %s: %w", name, err) + } + qd := new(query) + // Found file + if strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml") { + err := yaml.Unmarshal(data, qd) + if err != nil { + return nil, err + } + } else { + qd.query = string(data) + } + qd.name = strings.TrimSuffix(filepath.Base(name), filepath.Ext(name)) + + return qd, nil +} + +// listStoredQueries returns a list of query definitions stored in the query directory +func listStoredQueries(path string) []*query { + var r []*query + + if files, err := os.ReadDir(path); err == nil { + for _, f := range files { + if !f.IsDir() { + q, err := readQuery(filepath.Join(path, f.Name())) + if err != nil { + return nil + } + r = append(r, q) + } + } + } + + return r +} diff --git a/src/cmd/logout.go b/cmd/report/report.go similarity index 59% rename from src/cmd/logout.go rename to cmd/report/report.go index 4fa8f35..08f6fad 100644 --- a/src/cmd/logout.go +++ b/cmd/report/report.go @@ -11,26 +11,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cmd +package report import ( - "github.com/nlnwa/veidemannctl/src/configutil" "github.com/spf13/cobra" - "github.com/spf13/viper" ) -// logoutCmd represents the logout command -var logoutCmd = &cobra.Command{ - Use: "logout", - Short: "Log out of Veidemann", - Long: `Log out of Veidemann.`, - Run: func(cmd *cobra.Command, args []string) { - viper.Set("accessToken", "") - viper.Set("nonce", "") - configutil.WriteConfig() - }, -} +func NewReportCmd() *cobra.Command { + cmd := &cobra.Command{ + GroupID: "advanced", + Use: "report", + Short: "Request a report", + } + + cmd.AddCommand(newJobExecutionCmd()) // jobexecution + cmd.AddCommand(newCrawlExecutionCmd()) // crawlexecution + cmd.AddCommand(newCrawlLogCmd()) // crawllog + cmd.AddCommand(newQueryCmd()) // query + cmd.AddCommand(newPageLogCmd()) // pagelog -func init() { - RootCmd.AddCommand(logoutCmd) + return cmd } diff --git a/src/cmd/run.go b/cmd/run/run.go similarity index 57% rename from src/cmd/run.go rename to cmd/run/run.go index ea3d0eb..ff007b6 100644 --- a/src/cmd/run.go +++ b/cmd/run/run.go @@ -11,58 +11,59 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cmd +package run import ( + "context" "fmt" - "log" controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" - "golang.org/x/net/context" - "github.com/nlnwa/veidemannctl/src/connection" + "github.com/nlnwa/veidemannctl/connection" "github.com/spf13/cobra" ) -// runCmd represents the run command -var runCmd = &cobra.Command{ - Use: "run jobId [seedId]", - Short: "Immediately run a crawl", - Long: `Run a crawl. If seedId is submitted only this seed will be run using the configuration -from the submitted jobId. This will run even if the seed is not configured to use the jobId. -If seedId is not submitted then all the seeds wich are configured to use the submitted jobId will be crawled.`, +func NewRunCmd() *cobra.Command { + // runCmd represents the run command + return &cobra.Command{ + GroupID: "run", + Use: "run JOB-ID [SEED-ID]", + Short: "Run a crawl job", + Long: `Run a crawl job. + +If a seed is provided, the job will be created and started with the seed only. +If the job is already running, the seed will be added to the running job.`, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true - Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - client, conn := connection.NewControllerClient() + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect") + } defer conn.Close() + client := controllerV1.NewControllerClient(conn) + switch len(args) { - case 1: - // One argument (only jobId) + case 1: // One argument (only jobId) request := controllerV1.RunCrawlRequest{JobId: args[0]} r, err := client.RunCrawl(context.Background(), &request) if err != nil { - log.Fatalf("could not run job: %v", err) + return fmt.Errorf("could not run job: %w", err) } fmt.Printf("Job Execution ID: %v\n", r.GetJobExecutionId()) - case 2: - // Two arguments (jobId and seedId) + case 2: // Two arguments (jobId and seedId) request := controllerV1.RunCrawlRequest{JobId: args[0], SeedId: args[1]} r, err := client.RunCrawl(context.Background(), &request) if err != nil { - log.Fatalf("could not run job: %v", err) + return fmt.Errorf("could not run job: %w", err) } fmt.Printf("Job Execution ID: %v\n", r.GetJobExecutionId()) } - } else { - cmd.Usage() - } - }, -} - -func init() { - RootCmd.AddCommand(runCmd) + return nil + }, + } } diff --git a/src/cmd/script_parameters.go b/cmd/script_parameters/script_parameters.go similarity index 58% rename from src/cmd/script_parameters.go rename to cmd/script_parameters/script_parameters.go index 19ce191..4edb2cc 100644 --- a/src/cmd/script_parameters.go +++ b/cmd/script_parameters/script_parameters.go @@ -11,40 +11,46 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cmd +package scriptparameters import ( "context" "fmt" + "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - log "github.com/sirupsen/logrus" + "github.com/nlnwa/veidemannctl/connection" "github.com/spf13/cobra" ) -// scriptParametersCmd represents the script-parameters command -var scriptParametersCmd = &cobra.Command{ - Use: "script-parameters CRAWLJOB_CONFIG_ID [SEED_ID]", - Short: "Get the active script parameters for a Crawl Job", - Long: `Get the active script parameters for a Crawl Job +func NewScriptParametersCmd() *cobra.Command { + // scriptParametersCmd represents the script-parameters command + return &cobra.Command{ + GroupID: "debug", + Use: "script-parameters JOB-ID [SEED-ID]", + Short: "Get the effective script parameters for a crawl job", + Long: `Get the effective script parameters for a crawl job and optionally a seed in the context of the crawl job. Examples: # See active script parameters for a Crawl Job veidemannctl script-parameters 5604f0cc-315d-4091-8d6e-1b17a7eb990b - # See active script parameters for a Crawl Job and eventual overrides from Seed and Entity + # Get effective script parameters for a Seed in the context of a Crawl Job veidemannctl script-parameters 5604f0cc-315d-4091-8d6e-1b17a7eb990b 9f89ca44-afe0-4f8f-808f-9df1a0fe64c9 `, - Aliases: []string{"params"}, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - fmt.Println("Missing CrawlJobConfig id") - cmd.Usage() - } - if len(args) <= 2 { - configClient, conn := connection.NewConfigClient() + Aliases: []string{"params"}, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + // silence usage to prevent printing usage when an error occurs + cmd.SilenceUsage = true + + conn, err := connection.Connect() + if err != nil { + return fmt.Errorf("failed to connect: %w", err) + } defer conn.Close() + configClient := config.NewConfigClient(conn) + request := &config.GetScriptAnnotationsRequest{ Job: &config.ConfigRef{Kind: config.Kind_crawlJob, Id: args[0]}, } @@ -54,19 +60,13 @@ Examples: response, err := configClient.GetScriptAnnotations(context.Background(), request) if err != nil { - log.Fatalf("Failed getting parameters for %v. Cause: %v", args[0], err) + return fmt.Errorf("failed getting parameters for %v: %w", args[0], err) } for _, a := range response.GetAnnotation() { fmt.Printf("Param: %s = '%s'\n", a.Key, a.Value) } - } else { - fmt.Println("Too many arguments") - cmd.Usage() - } - }, -} - -func init() { - RootCmd.AddCommand(scriptParametersCmd) + return nil + }, + } } diff --git a/cmd/status/status.go b/cmd/status/status.go new file mode 100644 index 0000000..fff9b82 --- /dev/null +++ b/cmd/status/status.go @@ -0,0 +1,51 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package status + +import ( + "context" + "fmt" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/connection" + "github.com/spf13/cobra" +) + +func NewStatusCmd() *cobra.Command { + return &cobra.Command{ + GroupID: "status", + Use: "status", + Short: "Display crawler status", + Long: `Display crawler status.`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := controller.NewControllerClient(conn) + + crawlerStatus, err := client.Status(context.Background(), &empty.Empty{}) + if err != nil { + return fmt.Errorf("failed to get crawler status: %w", err) + } + fmt.Printf("Status: %v, Url queue size: %v, Busy crawl host groups: %v\n", + crawlerStatus.RunStatus, crawlerStatus.QueueSize, crawlerStatus.BusyCrawlHostGroupCount) + return nil + }, + } +} diff --git a/src/cmd/unpause.go b/cmd/unpause/unpause.go similarity index 52% rename from src/cmd/unpause.go rename to cmd/unpause/unpause.go index 19a23a3..3c2734a 100644 --- a/src/cmd/unpause.go +++ b/cmd/unpause/unpause.go @@ -11,40 +11,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cmd +package unpause import ( "context" - "log" "github.com/golang/protobuf/ptypes/empty" - "github.com/nlnwa/veidemannctl/src/connection" + controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/connection" "github.com/spf13/cobra" ) -// unpauseCmd represents the unpause command -var unpauseCmd = &cobra.Command{ - Use: "unpause", - Short: "Unpause crawler", - Long: `Unpause crawler`, - Run: func(cmd *cobra.Command, args []string) { - err := unpauseCrawler() - if err != nil { - log.Fatalf("could not unpause crawler: %v", err) - } - }, -} - -func unpauseCrawler() error { - client, conn := connection.NewControllerClient() - defer func() { - _ = conn.Close() - }() +func NewUnpauseCmd() *cobra.Command { + return &cobra.Command{ + GroupID: "status", + Use: "unpause", + Short: "Request crawler to unpause", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() - _, err := client.UnPauseCrawler(context.Background(), &empty.Empty{}) - return err -} + client := controllerV1.NewControllerClient(conn) -func init() { - RootCmd.AddCommand(unpauseCmd) + _, err = client.UnPauseCrawler(context.Background(), &empty.Empty{}) + return err + }, + } } diff --git a/cmd/update/update.go b/cmd/update/update.go new file mode 100644 index 0000000..89c8df3 --- /dev/null +++ b/cmd/update/update.go @@ -0,0 +1,144 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package update + +import ( + "context" + "fmt" + + commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/nlnwa/veidemannctl/apiutil" + "github.com/nlnwa/veidemannctl/connection" + "github.com/nlnwa/veidemannctl/format" + "github.com/spf13/cobra" +) + +type updateCmdOptions struct { + kind configV1.Kind + ids []string + name string + label string + filters []string + updateField string + pageSize int32 +} + +func (o *updateCmdOptions) complete(cmd *cobra.Command, args []string) error { + // first arg is kind + k := args[0] + o.kind = format.GetKind(k) + + if o.kind == configV1.Kind_undefined { + return fmt.Errorf("undefined kind '%s'", k) + } + + // rest of args are ids + o.ids = args[1:] + + return nil +} + +// run runs the update command +func (o *updateCmdOptions) run() error { + conn, err := connection.Connect() + if err != nil { + return err + } + defer conn.Close() + + client := configV1.NewConfigClient(conn) + + selector, err := apiutil.CreateListRequest(o.kind, o.ids, o.name, o.label, o.filters, o.pageSize, 0) + if err != nil { + return fmt.Errorf("error creating request: %w", err) + } + + updateMask := new(commonsV1.FieldMask) + updateTemplate := new(configV1.ConfigObject) + if err := apiutil.CreateTemplateFilter(o.updateField, updateTemplate, updateMask); err != nil { + return fmt.Errorf("error creating request: %w", err) + } + + updateRequest := &configV1.UpdateRequest{ + ListRequest: selector, + UpdateMask: updateMask, + UpdateTemplate: updateTemplate, + } + + u, err := client.UpdateConfigObjects(context.Background(), updateRequest) + if err != nil { + return fmt.Errorf("error from controller: %w", err) + } + fmt.Printf("Objects updated: %v\n", u.Updated) + return nil +} + +func NewUpdateCmd() *cobra.Command { + o := &updateCmdOptions{} + + cmd := &cobra.Command{ + GroupID: "basic", + Use: "update KIND [ID ...]", + Short: "Update fields of config objects of the same kind", + Long: `Update a field of one or many config objects of the same kind`, + Example: `# Add CrawlJob for a seed. +veidemannctl update seed -n "https://www.gwpda.org/" -u seed.jobRef+=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709 + +# Replace all configured CrawlJobs for a seed with a new one. +veidemannctl update seed -n "https://www.gwpda.org/" -u seed.jobRef=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709`, + + Args: cobra.MatchAll( + cobra.MinimumNArgs(1), + func(cmd *cobra.Command, args []string) error { + return cobra.OnlyValidArgs(cmd, args[:1]) + }, + ), + ValidArgs: format.GetObjectNames(), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.complete(cmd, args); err != nil { + return err + } + // silence usage to prevent printing usage when an error occurs + cmd.SilenceUsage = true + return o.run() + }, + } + + // update-field is required + cmd.Flags().StringVarP(&o.updateField, "update-field", "u", "", "Which field to update (i.e. meta.description=foo)") + _ = cmd.MarkFlagRequired("update-field") + + // label is optional + cmd.Flags().StringVarP(&o.label, "label", "l", "", "Filter objects by label (: | )") + + // name is optional + cmd.Flags().StringVarP(&o.name, "name", "n", "", "Filter objects by name (accepts regular expressions)") + // register name flag completion func + _ = cmd.RegisterFlagCompletionFunc("name", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + names, err := apiutil.CompleteName(args[0], toComplete) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + return names, cobra.ShellCompDirectiveDefault + }) + + // filters is optional + cmd.Flags().StringArrayVarP(&o.filters, "filter", "q", nil, "Filter objects by field (i.e. meta.description=foo)") + + // limit is optional + cmd.Flags().Int32VarP(&o.pageSize, "limit", "s", 0, "Limit the number of objects to update. 0 = no limit") + + return cmd +} diff --git a/cmd/version/version.go b/cmd/version/version.go new file mode 100644 index 0000000..0a8c2a7 --- /dev/null +++ b/cmd/version/version.go @@ -0,0 +1,31 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, GitVersion 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" + + "github.com/nlnwa/veidemannctl/version" + "github.com/spf13/cobra" +) + +func NewVersionCmd() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the client version information", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(version.String()) + }, + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..184a8ee --- /dev/null +++ b/config/config.go @@ -0,0 +1,410 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/mitchellh/go-homedir" + "github.com/mitchellh/mapstructure" + "github.com/nlnwa/veidemannctl/logger" + "github.com/rs/zerolog/log" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "gopkg.in/yaml.v3" +) + +// Config represents the persistent configuration. +type Config struct { + Server string `yaml:"server,omitempty" mapstructure:"server"` + RootCAs string `yaml:"certificate-authority-data,omitempty" mastructure:"certificate-authority-data"` + ServerNameOverride string `yaml:"server-name-override,omitempty" mapstructure:"server-name-override"` + AuthProvider *AuthProvider `yaml:"auth-provider,omitempty" mapstructure:"auth-provider"` +} + +// cfg is the persistent configuration. +var cfg Config + +// AuthProvider is the authentication provider configuration. +type AuthProvider struct { + // Name is the type of authentication provider. + Name string `yaml:"name" mapstructure:"name"` + + // Config is the configuration for the authentication provider. + Config any `yaml:"config" mapstructure:"config"` +} + +const ( + ProviderOIDC = "oidc" + ProviderApiKey = "apikey" +) + +// ApiKeyConfig is the configuration for the apikey authentication provider. +type ApiKeyConfig struct { + ApiKey string `yaml:"api-key,omitempty" mapstructure:"api-key"` +} + +// GetApiKeyConfig returns the apikey configuration. +func GetApiKeyConfig() (*ApiKeyConfig, error) { + // If api-key is set as a flag, use that as the api-key + if apiKey := GetApiKey(); apiKey != "" { + return &ApiKeyConfig{ApiKey: apiKey}, nil + } + // If no auth provider in config, return nil + if cfg.AuthProvider == nil { + return nil, nil + } + // Use the api-key from the config file + ap := new(ApiKeyConfig) + // decode the config into the api-key config struct + err := mapstructure.Decode(cfg.AuthProvider.Config, ap) + return ap, err +} + +// OIDCConfig is the configuration for the oidc authentication provider. +type OIDCConfig struct { + ClientID string `yaml:"client-id" mapstructure:"client-id"` + ClientSecret string `yaml:"client-secret" mapstructure:"client-secret"` + IdToken string `yaml:"id-token" mapstructure:"id-token"` + RefreshToken string `yaml:"refresh-token,omitempty" mapstructure:"refresh-token"` + IdpIssuerUrl string `yaml:"idp-issuer-url" mapstructure:"idp-issuer-url"` +} + +// GetOIDCConfig returns the oidc configuration. +func GetOIDCConfig() (*OIDCConfig, error) { + if cfg.AuthProvider == nil { + return nil, nil + } + ap := new(OIDCConfig) + err := mapstructure.Decode(cfg.AuthProvider.Config, ap) + return ap, err +} + +// Init initializes configuration from config file, flags and environment variables. +func Init(flags *pflag.FlagSet) error { + if flags == nil { + return nil + } + + logLevel, _ := flags.GetString("log-level") + logFormat, _ := flags.GetString("log-format") + logCaller, _ := flags.GetBool("log-caller") + + logger.InitLogger(logLevel, logFormat, logCaller) + + defer func() { + log.Debug().Msgf("Using config file: %s", viper.ConfigFileUsed()) + }() + + // resolve path to context directory + ctxDir, err := GetConfigPath("contexts") + if err != nil { + return fmt.Errorf("failed to resolve config directory path: %w", err) + } + + // create context directory + log.Debug().Msgf("Creating context directory: %s", ctxDir) + if err := os.MkdirAll(ctxDir, 0777); err != nil { + return fmt.Errorf("failed to create context directory: %w", err) + } + // resolve context + ctxName, _ := flags.GetString("context") + ctxName, err = resolveContext(ctxName) + if err != nil { + return fmt.Errorf("failed to resolve context: %w", err) + } + // set runtime context + viper.Set("context", ctxName) + + // resolve path to config file + configFile, _ := flags.GetString("config") + if configFile != "" { + viper.SetConfigFile(configFile) + } else { + viper.AddConfigPath(ctxDir) + viper.SetConfigName(ctxName) + } + + // read config file + err = viper.ReadInConfig() + if errors.As(err, new(viper.ConfigFileNotFoundError)) { + path := filepath.Join(ctxDir, ctxName+".yaml") + viper.SetConfigFile(path) + } else if err != nil { + return err + } + + // store config in global variable + err = viper.Unmarshal(&cfg) + if err != nil { + return err + } + + // bind flags to viper to + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() // read in environment variables that match + if err := viper.BindPFlags(flags); err != nil { + return fmt.Errorf("failed to bind flags: %w", err) + } + + return nil +} + +// GetContext returns the effective context. +func GetContext() string { + return viper.GetString("context") +} + +// GetServer returns the server address. +func GetServer() string { + return viper.GetString("server") +} + +// GetRootCAs returns certificate authority data. +func GetRootCAs() string { + return viper.GetString("certificate-authority-data") +} + +// GetServerNameOverride retuns +func GetServerNameOverride() string { + return viper.GetString("server-name-override") +} + +func GetApiKey() string { + return viper.GetString("api-key") +} + +// GetAuthProviderName returns the authentication provider name. +func GetAuthProviderName() string { + ap := GetAuthProvider() + if ap == nil { + return "" + } + return ap.Name +} + +// GetAuthProvider returns the authentication provider. +func GetAuthProvider() *AuthProvider { + if apiKey := GetApiKey(); apiKey != "" { + c, _ := GetApiKeyConfig() + return &AuthProvider{ + Name: ProviderApiKey, + Config: c, + } + } + return cfg.AuthProvider +} + +// SetServerAddress sets the server address. +func SetServerAddress(server string) error { + cfg.Server = server + return writeConfig() +} + +// SetAuthProvider sets the authentication provider. +func SetApiKey(apiKey string) error { + cfg.AuthProvider = &AuthProvider{ + Name: ProviderApiKey, + Config: ApiKeyConfig{ + ApiKey: apiKey, + }, + } + return writeConfig() +} + +// SetCaCert sets the certificate authority data. +func SetCaCert(cert string) error { + cfg.RootCAs = cert + return writeConfig() +} + +// SetServerNameOverride sets the server name override. +func SetServerNameOverride(name string) error { + cfg.ServerNameOverride = name + return writeConfig() +} + +// SetAuthProvider sets the authentication provider. +func SetAuthProvider(authProvider *AuthProvider) error { + cfg.AuthProvider = authProvider + return writeConfig() +} + +// CreateContext creates a new context. +func CreateContext(name string) error { + ok, err := ContextExists(name) + if err != nil { + return fmt.Errorf("failed to check if context '%s' already exists: %w", name, err) + } + if ok { + return fmt.Errorf("context already exists: %s", name) + } + + contextDir, err := GetConfigPath("contexts") + if err != nil { + return err + } + + _, err = os.Create(filepath.Join(contextDir, name+".yaml")) + return err +} + +// writeConfig writes the config to file. +func writeConfig() error { + y, err := yaml.Marshal(cfg) + if err != nil { + return fmt.Errorf("failed to marshal config: %w", err) + } + + configFile := viper.ConfigFileUsed() + + file, err := os.Create(configFile) + if err != nil { + return fmt.Errorf("failed to create config file \"%s\": %w", configFile, err) + } + defer file.Close() + + if err = file.Chmod(0600); err != nil { + return fmt.Errorf("failed to change access mode on config file \"%s\": %w", configFile, err) + } + + if _, err = file.Write(y); err != nil { + return fmt.Errorf("failed to write config file \"%s\": %w", configFile, err) + } + return nil +} + +// GetConfigPath returns the full path of a config directory or config file. +func GetConfigPath(subdirOrFile string) (string, error) { + // Find home directory. + home, err := homedir.Dir() + if err != nil { + return "", err + } + + return filepath.Join(home, ".veidemann", subdirOrFile), nil +} + +// context represents the context file. +type context struct { + Context string +} + +// resolveContext resolves the effective context. +func resolveContext(name string) (string, error) { + switch name { + case "": + contextFile, err := GetConfigPath("context.yaml") + if err != nil { + return "", err + } + _, err = os.Stat(contextFile) + if err != nil { + err = SetCurrentContext("default") + return "default", err + } + f, err := os.Open(contextFile) + if err != nil { + return "", err + } + defer f.Close() + dec := yaml.NewDecoder(f) + c := new(context) + err = dec.Decode(c) + if err != nil { + return "", err + } + + return c.Context, err + case "kubectl": + output, err := exec.Command("kubectl", "config", "current-context").CombinedOutput() + if err != nil { + _, _ = os.Stderr.WriteString(err.Error()) + return "", err + } + return strings.TrimSpace(string(output)), nil + default: + return name, nil + } +} + +// SetCurrentContext sets the current context. +func SetCurrentContext(ctxName string) error { + contextFile, err := GetConfigPath("context.yaml") + if err != nil { + return fmt.Errorf("failed to resolve config dir': %w", err) + } + w, err := os.Create(contextFile) + if err != nil { + return fmt.Errorf("failed to create or open '%s': %w", contextFile, err) + } + defer w.Close() + + enc := yaml.NewEncoder(w) + err = enc.Encode(context{ctxName}) + if err != nil { + return fmt.Errorf("failed to write context to '%s': %w", contextFile, err) + } + defer enc.Close() + + return nil +} + +// ListContexts lists all contexts. +func ListContexts() ([]string, error) { + var files []string + contextDir, err := GetConfigPath("contexts") + if err != nil { + return nil, err + } + fileInfo, err := os.ReadDir(contextDir) + if err != nil { + return files, err + } + + for _, file := range fileInfo { + if !file.IsDir() { + sufIdx := strings.LastIndex(file.Name(), ".") + if sufIdx > 0 { + files = append(files, file.Name()[:sufIdx]) + } + } + } + return files, nil +} + +// ContextExists checks if a context exists. +func ContextExists(name string) (bool, error) { + cs, err := ListContexts() + if err != nil { + return false, err + } + + for _, c := range cs { + if name == c { + return true, nil + } + } + return false, nil +} + +// GetConfig returns the persistent config. +func GetConfig() Config { + return cfg +} diff --git a/connection/auth.go b/connection/auth.go new file mode 100644 index 0000000..8a2ba5d --- /dev/null +++ b/connection/auth.go @@ -0,0 +1,401 @@ +// Copyright © 2017 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connection + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "math/rand" + "net" + "net/http" + "os/exec" + "runtime" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "github.com/golang/protobuf/ptypes/empty" + controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" + "github.com/nlnwa/veidemannctl/config" + "github.com/rs/zerolog/log" + "golang.org/x/oauth2" +) + +const ( + // manualRedirectURI is the redirect URI used when manual login is used. + manualRedirectURI = "urn:ietf:wg:oauth:2.0:oob" + // autoRedirectURI is the redirect URI used when automatic login is used. + autoRedirectURI = "http://localhost:9876" +) + +// Provider is the type of authentication provider. +type Provider string + +// Login logs in using the configured authentication provider. +// If manualLogin is true, the user will be given a URL to paste in a browser window, +// else a browser window will be opened automatically. +func Login(manualLogin bool) error { + p := config.GetAuthProviderName() + if p == "" { + p = config.ProviderOIDC + } + switch p { + case config.ProviderOIDC: + c, err := config.GetOIDCConfig() + if err != nil { + return err + } + if c == nil { + c = &config.OIDCConfig{} + } + claims, err := loginOIDC(c, manualLogin) + if err != nil { + return err + } + fmt.Printf("Hello, %s!\n", claims.Name) + case config.ProviderApiKey: + // no login procedure for apikey + } + return nil +} + +// loginOIDC logs in using the OIDC authentication flow. +// If manualLogin is true, the user will be given a URL to paste in a browser window, +// else a browser window will be opened automatically. +func loginOIDC(oidcConfig *config.OIDCConfig, manualLogin bool) (*claims, error) { + clientID := oidcConfig.ClientID + if clientID == "" { + clientID = "veidemann-cli" + } + clientSecret := oidcConfig.ClientSecret + if clientSecret == "" { + clientSecret = "cli-app-secret" + } + // Does the provider use "offline_access" scope to request a refresh token + // or does it use "access_type=offline" (e.g. Google)? + offlineAsScope := false + scopes := []string{oidc.ScopeOpenID, "profile", "email", "groups", "audience:server:client_id:veidemann-api"} + if offlineAsScope { + scopes = append(scopes, "offline_access") + } + idpIssuerUrl := oidcConfig.IdpIssuerUrl + if idpIssuerUrl == "" { + idp, err := getIdpIssuer() + if err != nil { + return nil, err + } else if idp == "" { + return nil, nil + } else { + idpIssuerUrl = idp + } + } + + log.Debug().Msgf("Using identity provider: %s", idpIssuerUrl) + + o := oidcProvider{ + idpIssuerUrl: idpIssuerUrl, + clientID: clientID, + clientSecret: clientSecret, + scopes: scopes, + } + + claims, err := o.login(manualLogin) + if err != nil { + return nil, fmt.Errorf("login failed: %w", err) + } + + // Set the auth provider in the config + err = config.SetAuthProvider(&config.AuthProvider{ + Name: config.ProviderOIDC, + Config: config.OIDCConfig{ + ClientID: o.clientID, + ClientSecret: o.clientSecret, + IdToken: o.rawIdToken, + RefreshToken: o.refreshToken, + IdpIssuerUrl: o.idpIssuerUrl, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to save auth provider: %w", err) + } + + return claims, nil +} + +// Logout removes the auth provider from the config. Effectively logging out. +func Logout() error { + return config.SetAuthProvider(nil) +} + +// getIdpIssuer resolves the OIDC issuer from the server. +func getIdpIssuer() (string, error) { + conn, err := connect() + if err != nil { + return "", err + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + reply, err := controllerV1.NewControllerClient(conn).GetOpenIdConnectIssuer(ctx, &empty.Empty{}) + if err != nil { + return "", fmt.Errorf("failed to get oidc issuer: %w", err) + } + + idp := reply.GetOpenIdConnectIssuer() + if idp == "" { + log.Warn().Msg("Server is configured without an identity provider - proceeding without authentication.") + } else { + log.Debug().Msgf(`Using idp issuer address : "%v"`, idp) + } + + return idp, nil +} + +// oidcProvider implements the oidc authentication flow. +type oidcProvider struct { + clientID string + clientSecret string + rawIdToken string + refreshToken string + idpIssuerUrl string + scopes []string +} + +// login using oidc code flow. +// If manual is true, the user will be given a URL to paste in a browser window, +// else a browser window will be opened automatically. +func (op *oidcProvider) login(manual bool) (*claims, error) { + // get http client with configured CAs + client := httpClientForRootCAs() + if client == nil { + client = http.DefaultClient + } + + // initialize OIDC ID Token verifier + var idTokenVerifier *oidc.IDTokenVerifier + ctx := oidc.ClientContext(context.Background(), client) + p, err := oidc.NewProvider(ctx, op.idpIssuerUrl) + if err != nil { + log.Warn().Msgf("Could not connect to authentication server '%s': %v. Proceeding without authentication", op.idpIssuerUrl, err) + } else { + oc := oidc.Config{ClientID: op.clientID} + idTokenVerifier = p.Verifier(&oc) + } + + // create nonce and use nonce as state + nonce := randStringBytesMaskImprSrc(16) + state := nonce + + var redirectURI string + if manual { + redirectURI = manualRedirectURI + } else { + redirectURI = autoRedirectURI + } + + oauth2Config := &oauth2.Config{ + ClientID: op.clientID, + ClientSecret: op.clientSecret, + Endpoint: p.Endpoint(), + Scopes: op.scopes, + RedirectURL: redirectURI, + } + + authCodeURL := oauth2Config.AuthCodeURL(nonce, oidc.Nonce(nonce)) + + var code string + + if manual { + fmt.Println("Paste this uri in a browser window. Follow the login steps and paste the code here.") + fmt.Println(authCodeURL) + + fmt.Print("Code: ") + if _, err := fmt.Scan(&code); err != nil { + return nil, err + } + } else { + var gotState string + err := openBrowser(authCodeURL) + if err != nil { + return nil, err + } + code, gotState, err = listenAndWaitForAuthorizationCode(autoRedirectURI) + if err != nil { + return nil, err + } + if gotState != state { + return nil, errors.New("state is not equal") + } + } + + oauth2Token, err := oauth2Config.Exchange(ctx, code) + if err != nil { + return nil, err + } + + op.refreshToken = oauth2Token.RefreshToken + + // Extract the ID Token from OAuth2 token. + rawIDToken, ok := oauth2Token.Extra("id_token").(string) + if !ok { + return nil, errors.New("token not found") + } + op.rawIdToken = rawIDToken + + // Parse and verify ID Token payload. + idToken, err := idTokenVerifier.Verify(ctx, op.rawIdToken) + if err != nil { + return nil, err + } + + if idToken.Nonce != nonce { + return nil, errors.New("nonce did not match") + } + + claims := new(claims) + if err := idToken.Claims(claims); err != nil { + return nil, err + } + + return claims, err +} + +// oidcCredentials implements credentials.PerRPCCredentials for oidc authentication. +type oidcCredentials struct { + idToken string +} + +// GetRequestMetadata implements credentials.PerRPCCredentials. +func (oc oidcCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + return map[string]string{ + "authorization": "Bearer" + " " + oc.idToken, + }, nil +} + +// RequireTransportSecurity implements credentials.PerRPCCredentials. +func (oc oidcCredentials) RequireTransportSecurity() bool { + return true +} + +// claims represent custom claims. +type claims struct { + Email string `json:"email"` + Verified bool `json:"email_verified"` + Groups []string `json:"groups"` + Name string `json:"name"` +} + +// openBrowser tries to open the URL in a browser. +func openBrowser(authCodeURL string) error { + var err error + + switch runtime.GOOS { + case "linux": + err = exec.Command("xdg-open", authCodeURL).Start() + case "windows": + err = exec.Command("rundll32", "url.dll,FileProtocolHandler", authCodeURL).Start() + case "darwin": + err = exec.Command("open", authCodeURL).Start() + default: + err = fmt.Errorf("unsupported platform") + } + if err != nil { + return fmt.Errorf("failed to open browser: %w", err) + } + + return nil +} + +// httpClientForRootCAs returns an HTTP client that trusts the provided root CAs. +func httpClientForRootCAs() *http.Client { + // Create a certificate pool with systems CAs + certPool, err := x509.SystemCertPool() + if err != nil { + log.Warn().Msg("Could not read system trusted certificates, using only the configured ones") + certPool = x509.NewCertPool() + } + tlsConfig := tls.Config{RootCAs: certPool} + + // Add CAs from config + if config.GetRootCAs() != "" { + rootCABytes := []byte(config.GetRootCAs()) + if !tlsConfig.RootCAs.AppendCertsFromPEM(rootCABytes) { + log.Warn().Msg("No certs found in root CA file") + } + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tlsConfig, + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } +} + +// letterBytes is used to generate a random string. +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + + return string(b) +} + +// apiKeyCredentials implements credentials.PerRPCCredentials for apikey authentication. +type apiKeyCredentials struct { + apiKey string +} + +// GetRequestMetadata implements PerRPCCredentials +func (a apiKeyCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + return map[string]string{ + "authorization": "ApiKey" + " " + a.apiKey, + }, nil +} + +// RequireTransportSecurity implements PerRPCCredentials +func (a apiKeyCredentials) RequireTransportSecurity() bool { + return true +} diff --git a/connection/connection.go b/connection/connection.go new file mode 100644 index 0000000..3a8041e --- /dev/null +++ b/connection/connection.go @@ -0,0 +1,105 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connection + +import ( + "context" + "crypto/x509" + "fmt" + "strings" + + "github.com/nlnwa/veidemannctl/config" + "github.com/rs/zerolog/log" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// Connect returns a connection to the server. +// If there is an auth provider configured, the connection will be authenticated with the configured credentials. +func Connect() (*grpc.ClientConn, error) { + ap := config.GetAuthProvider() + if ap == nil { + return connect() + } + var creds credentials.PerRPCCredentials + switch ap.Name { + case config.ProviderApiKey: + apiKeyConfig, err := config.GetApiKeyConfig() + if err != nil { + return nil, err + } + creds = apiKeyCredentials{apiKey: apiKeyConfig.ApiKey} + case config.ProviderOIDC: + oidcConfig, err := config.GetOIDCConfig() + if err != nil { + return nil, err + } + creds = oidcCredentials{idToken: oidcConfig.IdToken} + default: + return nil, fmt.Errorf("unknown auth provider: %s", ap.Name) + } + + return connect(grpc.WithPerRPCCredentials(creds)) +} + +// connect returns a connection to the server. +// If tls is true, the connection will be encrypted with TLS. +func connect(opts ...grpc.DialOption) (*grpc.ClientConn, error) { + address := config.GetServer() + dialOptions := append(opts, + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + grpc.WithBlock(), + grpc.FailOnNonTempDialError(true), + ) + + log.Debug().Msgf("Connecting to %v", address) + conn, err := grpc.DialContext(context.Background(), address, append(dialOptions, grpc.WithTransportCredentials(clientTransportCredentials()))...) + if err != nil { + if strings.Contains(err.Error(), "first record does not look like a TLS handshake") { + log.Debug().Msg("Failed to connect with TLS, retrying with insecure transport credentials") + conn, err = grpc.DialContext(context.Background(), address, append(dialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))...) + if err != nil { + return nil, err + } + log.Warn().Msg("Connected with insecure transport. Server does not support TLS") + } else { + return nil, err + } + } + return conn, nil +} + +// clientTransportCredentials returns the transport credentials to use for the client +func clientTransportCredentials() credentials.TransportCredentials { + // Create a pool with systems CAs + certPool, err := x509.SystemCertPool() + if err != nil { + log.Warn().Msg("Could not read system trusted certificates, using only the configured ones") + certPool = x509.NewCertPool() + } + + // Add configured CAs + if config.GetRootCAs() != "" { + rootCABytes := []byte(config.GetRootCAs()) + if !certPool.AppendCertsFromPEM(rootCABytes) { + log.Warn().Msg("no certs found in root CA file") + } + } + + serverNameOverride := config.GetServerNameOverride() + log.Debug().Msgf("Using server name override: %s", serverNameOverride) + + return credentials.NewClientTLSFromCert(certPool, serverNameOverride) +} diff --git a/connection/loginserver.go b/connection/loginserver.go new file mode 100644 index 0000000..77ab635 --- /dev/null +++ b/connection/loginserver.go @@ -0,0 +1,62 @@ +package connection + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// response is the response from the login server +type response struct { + code string + state string +} + +func handleLoginResponse(resp chan<- *response) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + fmt.Fprintf(w, "

%s

", "Window can safely be closed") + resp <- &response{ + query.Get("code"), + query.Get("state"), + } + } +} + +// listenAndWaitForAuthorizationCode starts a http server and waits for the authorization code and state +func listenAndWaitForAuthorizationCode(uri string) (string, string, error) { + var addr string + if u, err := url.Parse(uri); err != nil { + return "", "", err + } else { + addr = fmt.Sprintf("%s:%s", u.Hostname(), u.Port()) + } + server := &http.Server{ + Addr: addr, + } + defer func() { + go func() { + _ = server.Shutdown(context.Background()) + }() + }() + + response := make(chan *response) + http.HandleFunc("/", handleLoginResponse(response)) + var err error + + go func() { + err = server.ListenAndServe() + if err != nil { + close(response) + } + }() + + // wait for response + resp := <-response + if resp == nil { + return "", "", err + } + + return resp.code, resp.state, nil +} diff --git a/connection/loginserver_test.go b/connection/loginserver_test.go new file mode 100644 index 0000000..1875d62 --- /dev/null +++ b/connection/loginserver_test.go @@ -0,0 +1,51 @@ +package connection + +import ( + "context" + "net/http" + "testing" + "time" +) + +func TestLoginServer(t *testing.T) { + done := make(chan struct{}) + + timeout := time.NewTimer(time.Second) + const testCode = "abcd" + const testState = "1234" + + go func() { + defer close(done) + code, state, err := listenAndWaitForAuthorizationCode(autoRedirectURI) + if err != nil { + t.Error(err) + } + if code != testCode { + t.Errorf("want %s, got %s", testCode, code) + } + if state != testState { + t.Errorf("want %s, get %s", testState, state) + } + }() + + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, autoRedirectURI, nil) + if err != nil { + t.Error(err) + } + + q := req.URL.Query() + q.Add("code", testCode) + q.Add("state", testState) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() + + select { + case <-done: + case <-timeout.C: + t.Error("timed out waiting for code") + } +} diff --git a/gendoc/md_docs.go b/docs/generate.go similarity index 78% rename from gendoc/md_docs.go rename to docs/generate.go index f483649..6b20b31 100644 --- a/gendoc/md_docs.go +++ b/docs/generate.go @@ -18,20 +18,22 @@ package main import ( "fmt" - "github.com/nlnwa/veidemannctl/src/cmd" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra/doc" + "os" "path/filepath" "runtime" + + "github.com/nlnwa/veidemannctl/cmd" + "github.com/spf13/cobra/doc" ) func main() { // Find 'this' directory relative to this file to allow callers to be in any package var _, b, _, _ = runtime.Caller(0) - var dir = filepath.Join(filepath.Dir(b), "../docs") - fmt.Println("generating documentation") - err := doc.GenMarkdownTree(cmd.RootCmd, dir) + var dir = filepath.Dir(b) + fmt.Println("Generating documentation...") + err := doc.GenMarkdownTree(cmd.NewRootCmd(), dir) if err != nil { - log.Fatal(err) + _, _ = fmt.Fprintln(os.Stderr, err) + os.Exit(1) } } diff --git a/docs/index.md b/docs/index.md index 1d47405..b3480fa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,15 @@ # veidemannctl + A command line client for Veidemann ## Install -Install the latest release version + +Install the latest release version (currently only for linux/amd64) + ```console curl -sL https://raw.githubusercontent.com/nlnwa/veidemannctl/master/install.sh | bash ``` ## Documentation + Documentation for available commands and flags can be found [here](https://nlnwa.github.io/veidemannctl/veidemannctl.html) diff --git a/docs/veidemannctl.md b/docs/veidemannctl.md index 3afe131..61b6c0c 100644 --- a/docs/veidemannctl.md +++ b/docs/veidemannctl.md @@ -1,46 +1,44 @@ ## veidemannctl -Veidemann command line client +veidemannctl controls the Veidemann web crawler ### Synopsis -A command line client for Veidemann which can manipulate configs and request status of the crawler. - -``` -veidemannctl [flags] -``` +veidemannctl controls the Veidemann web crawler ### Options ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - -h, --help help for veidemannctl - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + -h, --help help for veidemannctl + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl abort](veidemannctl_abort.md) - Abort one or more crawl executions -* [veidemannctl abortjobexecution](veidemannctl_abortjobexecution.md) - Abort one or more job executions +* [veidemannctl abort](veidemannctl_abort.md) - Abort crawl executions +* [veidemannctl abortjobexecution](veidemannctl_abortjobexecution.md) - Abort job executions * [veidemannctl activeroles](veidemannctl_activeroles.md) - Get the active roles for the currently logged in user -* [veidemannctl completion](veidemannctl_completion.md) - Output bash completion code -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands -* [veidemannctl create](veidemannctl_create.md) - Create or update a config object -* [veidemannctl delete](veidemannctl_delete.md) - Delete a config object -* [veidemannctl get](veidemannctl_get.md) - Get the value(s) for an object type +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files +* [veidemannctl create](veidemannctl_create.md) - Create or update config objects +* [veidemannctl delete](veidemannctl_delete.md) - Delete config objects +* [veidemannctl get](veidemannctl_get.md) - Display config objects * [veidemannctl import](veidemannctl_import.md) - Import data into Veidemann using subcommands * [veidemannctl logconfig](veidemannctl_logconfig.md) - Configure logging -* [veidemannctl login](veidemannctl_login.md) - Initiate browser session for logging in to Veidemann +* [veidemannctl login](veidemannctl_login.md) - Log in to Veidemann * [veidemannctl logout](veidemannctl_logout.md) - Log out of Veidemann -* [veidemannctl pause](veidemannctl_pause.md) - Pause crawler -* [veidemannctl report](veidemannctl_report.md) - Get report -* [veidemannctl run](veidemannctl_run.md) - Immediately run a crawl -* [veidemannctl script-parameters](veidemannctl_script-parameters.md) - Get the active script parameters for a Crawl Job -* [veidemannctl status](veidemannctl_status.md) - Get crawler status -* [veidemannctl unpause](veidemannctl_unpause.md) - Unpause crawler -* [veidemannctl update](veidemannctl_update.md) - Update the value(s) for an object type +* [veidemannctl pause](veidemannctl_pause.md) - Request crawler to pause +* [veidemannctl report](veidemannctl_report.md) - Request a report +* [veidemannctl run](veidemannctl_run.md) - Run a crawl job +* [veidemannctl script-parameters](veidemannctl_script-parameters.md) - Get the effective script parameters for a crawl job +* [veidemannctl status](veidemannctl_status.md) - Display crawler status +* [veidemannctl unpause](veidemannctl_unpause.md) - Request crawler to unpause +* [veidemannctl update](veidemannctl_update.md) - Update fields of config objects of the same kind +* [veidemannctl version](veidemannctl_version.md) - Print the client version information diff --git a/docs/veidemannctl_abort.md b/docs/veidemannctl_abort.md index 2db6059..9c81ddb 100644 --- a/docs/veidemannctl_abort.md +++ b/docs/veidemannctl_abort.md @@ -1,13 +1,13 @@ ## veidemannctl abort -Abort one or more crawl executions +Abort crawl executions ### Synopsis -Abort one or more crawl executions. +Abort one or many crawl executions. ``` -veidemannctl abort [flags] +veidemannctl abort CRAWL-EXECUTION-ID ... [flags] ``` ### Options @@ -19,15 +19,17 @@ veidemannctl abort [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_abortjobexecution.md b/docs/veidemannctl_abortjobexecution.md index ff63d01..52dea7a 100644 --- a/docs/veidemannctl_abortjobexecution.md +++ b/docs/veidemannctl_abortjobexecution.md @@ -1,13 +1,13 @@ ## veidemannctl abortjobexecution -Abort one or more job executions +Abort job executions ### Synopsis -Abort one or more job executions. +Abort one or many job executions. ``` -veidemannctl abortjobexecution [flags] +veidemannctl abortjobexecution JOB-EXECUTION-ID ... [flags] ``` ### Options @@ -19,15 +19,17 @@ veidemannctl abortjobexecution [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_activeroles.md b/docs/veidemannctl_activeroles.md index 2a9a72c..84ab788 100644 --- a/docs/veidemannctl_activeroles.md +++ b/docs/veidemannctl_activeroles.md @@ -19,15 +19,17 @@ veidemannctl activeroles [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_completion.md b/docs/veidemannctl_completion.md deleted file mode 100644 index e6d67c2..0000000 --- a/docs/veidemannctl_completion.md +++ /dev/null @@ -1,39 +0,0 @@ -## veidemannctl completion - -Output bash completion code - -### Synopsis - -Output bash completion code. The shell code must be evalutated to provide -interactive completion of veidemannctl commands. This can be done by sourcing it from the .bash _profile. - -Example: - ## Load the kubectl completion code for bash into the current shell - source <(veidemannctl completion) - - -``` -veidemannctl completion [flags] -``` - -### Options - -``` - -h, --help help for completion -``` - -### Options inherited from parent commands - -``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. -``` - -### SEE ALSO - -* [veidemannctl](veidemannctl.md) - Veidemann command line client - diff --git a/docs/veidemannctl_config.md b/docs/veidemannctl_config.md index 5afb963..b981de5 100644 --- a/docs/veidemannctl_config.md +++ b/docs/veidemannctl_config.md @@ -1,10 +1,6 @@ ## veidemannctl config -Modify veidemannctl config files using subcommands - -``` -veidemannctl config [flags] -``` +Modify or view configuration files ### Options @@ -15,26 +11,29 @@ veidemannctl config [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client -* [veidemannctl config create-context](veidemannctl_config_create-context.md) - Creates a new context -* [veidemannctl config current-context](veidemannctl_config_current-context.md) - Displays the current-context -* [veidemannctl config get-address](veidemannctl_config_get-address.md) - Displays Veidemann controller service address -* [veidemannctl config get-apikey](veidemannctl_config_get-apikey.md) - Displays Veidemann authentication api-key -* [veidemannctl config get-server-name-override](veidemannctl_config_get-server-name-override.md) - Displays the server name override +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler +* [veidemannctl config create-context](veidemannctl_config_create-context.md) - Create a new context +* [veidemannctl config current-context](veidemannctl_config_current-context.md) - Display the current context +* [veidemannctl config get-address](veidemannctl_config_get-address.md) - Display Veidemann controller service address +* [veidemannctl config get-apikey](veidemannctl_config_get-apikey.md) - Display Veidemann authentication api-key +* [veidemannctl config get-server-name-override](veidemannctl_config_get-server-name-override.md) - Display the server name override * [veidemannctl config import-ca](veidemannctl_config_import-ca.md) - Import file with trusted certificate chains for the idp and controller. -* [veidemannctl config list-contexts](veidemannctl_config_list-contexts.md) - Displays the known contexts -* [veidemannctl config set-address](veidemannctl_config_set-address.md) - Sets the address to Veidemann controller service -* [veidemannctl config set-apikey](veidemannctl_config_set-apikey.md) - Sets the api-key to use for authentication -* [veidemannctl config set-server-name-override](veidemannctl_config_set-server-name-override.md) - Sets the server name override -* [veidemannctl config use-context](veidemannctl_config_use-context.md) - Sets the current-context +* [veidemannctl config list-contexts](veidemannctl_config_list-contexts.md) - Display the known contexts +* [veidemannctl config set-address](veidemannctl_config_set-address.md) - Set the address to the Veidemann controller service +* [veidemannctl config set-apikey](veidemannctl_config_set-apikey.md) - Set the api-key to use for authentication +* [veidemannctl config set-server-name-override](veidemannctl_config_set-server-name-override.md) - Set the server name override +* [veidemannctl config use-context](veidemannctl_config_use-context.md) - Set the current context +* [veidemannctl config view](veidemannctl_config_view.md) - Display the current config diff --git a/docs/veidemannctl_config_create-context.md b/docs/veidemannctl_config_create-context.md index eb6773b..c531fc4 100644 --- a/docs/veidemannctl_config_create-context.md +++ b/docs/veidemannctl_config_create-context.md @@ -1,10 +1,10 @@ ## veidemannctl config create-context -Creates a new context +Create a new context ### Synopsis -Creates a new context +Create a new context Examples: # Create context for the prod cluster @@ -12,7 +12,7 @@ Examples: ``` -veidemannctl config create-context CONTEXT_NAME [flags] +veidemannctl config create-context NAME [flags] ``` ### Options @@ -24,15 +24,17 @@ veidemannctl config create-context CONTEXT_NAME [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_current-context.md b/docs/veidemannctl_config_current-context.md index 9f88370..502eba5 100644 --- a/docs/veidemannctl_config_current-context.md +++ b/docs/veidemannctl_config_current-context.md @@ -1,10 +1,10 @@ ## veidemannctl config current-context -Displays the current-context +Display the current context ### Synopsis -Displays the current-context +Display the current context Examples: # Display the current context @@ -24,15 +24,17 @@ veidemannctl config current-context [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_get-address.md b/docs/veidemannctl_config_get-address.md index 58f4652..aa93a49 100644 --- a/docs/veidemannctl_config_get-address.md +++ b/docs/veidemannctl_config_get-address.md @@ -1,10 +1,10 @@ ## veidemannctl config get-address -Displays Veidemann controller service address +Display Veidemann controller service address ### Synopsis -Displays Veidemann controller service address +Display Veidemann controller service address Examples: # Display Veidemann controller service address @@ -24,15 +24,17 @@ veidemannctl config get-address [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_get-apikey.md b/docs/veidemannctl_config_get-apikey.md index 8c131fb..2ba47e8 100644 --- a/docs/veidemannctl_config_get-apikey.md +++ b/docs/veidemannctl_config_get-apikey.md @@ -1,10 +1,10 @@ ## veidemannctl config get-apikey -Displays Veidemann authentication api-key +Display Veidemann authentication api-key ### Synopsis -Displays Veidemann authentication api-key +Display Veidemann authentication api-key Examples: # Display Veidemann authentication api-key @@ -24,15 +24,17 @@ veidemannctl config get-apikey [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_get-server-name-override.md b/docs/veidemannctl_config_get-server-name-override.md index 4e20f2f..03f2441 100644 --- a/docs/veidemannctl_config_get-server-name-override.md +++ b/docs/veidemannctl_config_get-server-name-override.md @@ -1,10 +1,10 @@ ## veidemannctl config get-server-name-override -Displays the server name override +Display the server name override ### Synopsis -Displays the server name override +Display the server name override Examples: # Display the server name override @@ -12,7 +12,7 @@ Examples: ``` -veidemannctl config get-server-name-override HOST [flags] +veidemannctl config get-server-name-override [flags] ``` ### Options @@ -24,15 +24,17 @@ veidemannctl config get-server-name-override HOST [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_import-ca.md b/docs/veidemannctl_config_import-ca.md index e31c8b0..5dbfffb 100644 --- a/docs/veidemannctl_config_import-ca.md +++ b/docs/veidemannctl_config_import-ca.md @@ -7,7 +7,7 @@ Import file with trusted certificate chains for the idp and controller. Import file with trusted certificate chains for the idp and controller. These are in addition to the default certs configured for the OS. ``` -veidemannctl config import-ca CA_CERT_FILE_NAME [flags] +veidemannctl config import-ca FILENAME [flags] ``` ### Options @@ -19,15 +19,17 @@ veidemannctl config import-ca CA_CERT_FILE_NAME [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_list-contexts.md b/docs/veidemannctl_config_list-contexts.md index 97afcfb..7c82ee9 100644 --- a/docs/veidemannctl_config_list-contexts.md +++ b/docs/veidemannctl_config_list-contexts.md @@ -1,10 +1,10 @@ ## veidemannctl config list-contexts -Displays the known contexts +Display the known contexts ### Synopsis -Displays the known contexts. +Display the known contexts. Examples: # Get a list of known contexts @@ -24,15 +24,17 @@ veidemannctl config list-contexts [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_set-address.md b/docs/veidemannctl_config_set-address.md index 8091e78..fcee2b3 100644 --- a/docs/veidemannctl_config_set-address.md +++ b/docs/veidemannctl_config_set-address.md @@ -1,10 +1,10 @@ ## veidemannctl config set-address -Sets the address to Veidemann controller service +Set the address to the Veidemann controller service ### Synopsis -Sets the address to Veidemann controller service +Set the address to the Veidemann controller service Examples: # Sets the address to Veidemann controller service to localhost:50051 @@ -24,15 +24,17 @@ veidemannctl config set-address HOST:PORT [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_set-apikey.md b/docs/veidemannctl_config_set-apikey.md index 60a1a79..6218fe1 100644 --- a/docs/veidemannctl_config_set-apikey.md +++ b/docs/veidemannctl_config_set-apikey.md @@ -1,10 +1,10 @@ ## veidemannctl config set-apikey -Sets the api-key to use for authentication +Set the api-key to use for authentication ### Synopsis -Sets the api-key to use for authentication +Set the api-key to use for authentication Examples: # Set the api-key to use for authentication to Veidemann controller service to myKey @@ -12,7 +12,7 @@ Examples: ``` -veidemannctl config set-apikey API_KEY [flags] +veidemannctl config set-apikey API-KEY [flags] ``` ### Options @@ -24,15 +24,17 @@ veidemannctl config set-apikey API_KEY [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_set-server-name-override.md b/docs/veidemannctl_config_set-server-name-override.md index 0032ad5..b9f8bd2 100644 --- a/docs/veidemannctl_config_set-server-name-override.md +++ b/docs/veidemannctl_config_set-server-name-override.md @@ -1,12 +1,12 @@ ## veidemannctl config set-server-name-override -Sets the server name override +Set the server name override ### Synopsis -Sets the server name override. +Set the server name override. -Use this when there is a mismatch between exposed server name for the cluster and the certificate. The use is a security +Use this when there is a mismatch between the exposed server name for the cluster and the certificate. The use is a security risk and is only recommended for testing. Examples: @@ -15,7 +15,7 @@ Examples: ``` -veidemannctl config set-server-name-override HOST [flags] +veidemannctl config set-server-name-override HOSTNAME [flags] ``` ### Options @@ -27,15 +27,17 @@ veidemannctl config set-server-name-override HOST [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_use-context.md b/docs/veidemannctl_config_use-context.md index 0ca5be7..91345b6 100644 --- a/docs/veidemannctl_config_use-context.md +++ b/docs/veidemannctl_config_use-context.md @@ -1,10 +1,10 @@ ## veidemannctl config use-context -Sets the current-context +Set the current context ### Synopsis -Sets the current-context +Set the current context Examples: # Use the context for the prod cluster @@ -12,7 +12,7 @@ Examples: ``` -veidemannctl config use-context CONTEXT_NAME [flags] +veidemannctl config use-context CONTEXT [flags] ``` ### Options @@ -24,15 +24,17 @@ veidemannctl config use-context CONTEXT_NAME [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl config](veidemannctl_config.md) - Modify veidemannctl config files using subcommands +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files diff --git a/docs/veidemannctl_config_view.md b/docs/veidemannctl_config_view.md new file mode 100644 index 0000000..6a8106b --- /dev/null +++ b/docs/veidemannctl_config_view.md @@ -0,0 +1,31 @@ +## veidemannctl config view + +Display the current config + +``` +veidemannctl config view [flags] +``` + +### Options + +``` + -h, --help help for view +``` + +### Options inherited from parent commands + +``` + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests +``` + +### SEE ALSO + +* [veidemannctl config](veidemannctl_config.md) - Modify or view configuration files + diff --git a/docs/veidemannctl_create.md b/docs/veidemannctl_create.md index 3efb80b..27adf5a 100644 --- a/docs/veidemannctl_create.md +++ b/docs/veidemannctl_create.md @@ -1,6 +1,10 @@ ## veidemannctl create -Create or update a config object +Create or update config objects + +### Synopsis + +Create or update one or many config objects ``` veidemannctl create [flags] @@ -9,6 +13,7 @@ veidemannctl create [flags] ### Options ``` + -c, --concurrency int Number of concurrent requests (default 32) -f, --filename string Filename or directory to read from. If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin. -h, --help help for create ``` @@ -16,15 +21,17 @@ veidemannctl create [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_delete.md b/docs/veidemannctl_delete.md index 2cb16c6..a4dc20c 100644 --- a/docs/veidemannctl_delete.md +++ b/docs/veidemannctl_delete.md @@ -1,53 +1,54 @@ ## veidemannctl delete -Delete a config object +Delete config objects ### Synopsis -Delete a config object. - -Valid object types include: - * browserConfig - * browserScript - * collection - * crawlConfig - * crawlEntity - * crawlHostGroupConfig - * crawlJob - * crawlScheduleConfig - * politenessConfig - * roleMapping - * seed +Delete one or many config objects. + + - browserConfig + - browserScript + - collection + - crawlConfig + - crawlEntity + - crawlHostGroupConfig + - crawlJob + - crawlScheduleConfig + - politenessConfig + - roleMapping + - seed Examples: - #Delete a seed. + # Delete a seed. veidemannctl delete seed 407a9600-4f25-4f17-8cff-ee1b8ee950f6 ``` -veidemannctl delete [id] ... [flags] +veidemannctl delete KIND (ID ...) [flags] ``` ### Options ``` - --dry-run Set to false to execute delete (default true) - -q, --filter string Delete objects by field (i.e. meta.description=foo) - -h, --help help for delete - -l, --label string Delete objects by label (: | ) + --dry-run Set to false to execute delete (default true) + -q, --filter stringArray Delete objects by field (i.e. meta.description=foo) + -h, --help help for delete + -l, --label string Delete objects by label (: | ) ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_get.md b/docs/veidemannctl_get.md index 4ae4236..018b5ef 100644 --- a/docs/veidemannctl_get.md +++ b/docs/veidemannctl_get.md @@ -1,61 +1,63 @@ ## veidemannctl get -Get the value(s) for an object type +Display config objects ### Synopsis -Display one or many objects. - -Valid object types include: - * browserConfig - * browserScript - * collection - * crawlConfig - * crawlEntity - * crawlHostGroupConfig - * crawlJob - * crawlScheduleConfig - * politenessConfig - * roleMapping - * seed +Display one or many config objects. + +Valid object types: + - browserConfig + - browserScript + - collection + - crawlConfig + - crawlEntity + - crawlHostGroupConfig + - crawlJob + - crawlScheduleConfig + - politenessConfig + - roleMapping + - seed Examples: - #List all seeds. + # List all seeds. veidemannctl get seed - #List all seeds in yaml output format. - veidemannctl get seed -f yaml + # List all seeds in yaml output format. + veidemannctl get seed -o yaml ``` -veidemannctl get [flags] +veidemannctl get KIND [ID ...] [flags] ``` ### Options ``` - -f, --filename string File name to write to - -q, --filter string Filter objects by field (i.e. meta.description=foo - -h, --help help for get - -l, --label string List objects by label (: | ) - -n, --name string List objects by name (accepts regular expressions) - -o, --output string Output format (table|wide|json|yaml|template|template-file) (default "table") - -p, --page int32 The page number - -s, --pagesize int32 Number of objects to get (default 10) - -t, --template string A Go template used to format the output + -f, --filename string Filename to write to + -q, --filter stringArray Filter objects by field (i.e. meta.description=foo) + -h, --help help for get + -l, --label string List objects by label (: | ) + -n, --name string List objects by name (accepts regular expressions) + -o, --output string Output format (table|wide|json|yaml|template|template-file) (default "table") + -p, --page int32 The page number + -s, --pagesize int32 Number of objects to get (default 10) + -t, --template string A Go template used to format the output ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_import.md b/docs/veidemannctl_import.md index 5dd2bc1..42c21e0 100644 --- a/docs/veidemannctl_import.md +++ b/docs/veidemannctl_import.md @@ -2,10 +2,6 @@ Import data into Veidemann using subcommands -``` -veidemannctl import [flags] -``` - ### Options ``` @@ -15,18 +11,20 @@ veidemannctl import [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler * [veidemannctl import convertoos](veidemannctl_import_convertoos.md) - Convert Out of Scope file(s) to seed import file -* [veidemannctl import duplicatereport](veidemannctl_import_duplicatereport.md) - List duplicated seeds in Veidemann +* [veidemannctl import duplicatereport](veidemannctl_import_duplicatereport.md) - List duplicated seeds or crawl entities in Veidemann * [veidemannctl import seed](veidemannctl_import_seed.md) - Import seeds diff --git a/docs/veidemannctl_import_convertoos.md b/docs/veidemannctl_import_convertoos.md index 10063b8..0a5eb07 100644 --- a/docs/veidemannctl_import_convertoos.md +++ b/docs/veidemannctl_import_convertoos.md @@ -9,27 +9,35 @@ veidemannctl import convertoos [flags] ### Options ``` - --checkuri Check the uri for liveness and follow 301 (default true) - --checkuri-timeout int Timeout in ms when checking uri for liveness. (default 2000) - -b, --db-directory string Directory for storing state db (default "/tmp/veidemannctl") - -e, --errorfile string File to write errors to. - -f, --filename string Filename or directory to read from. If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin. (required) - -h, --help help for convertoos - --ignore-scheme Ignore the URL's scheme when checking if this URL is already imported. - -o, --outfile string File to write result to. (required) - -r, --reset-db Clean state db - --toplevel Convert URI to toplevel by removing path. (default true) + --check-uri Check the uri for liveness and follow 301 (default true) + --check-uri-timeout duration Timeout when checking uri for liveness (default 2s) + -c, --concurrency int Number of concurrent workers (default 16) + -b, --db-dir string Directory for storing state db (default "/tmp/veidemannctl") + --entity-id string Entity id to use for all seeds (overrides entity-name and entity-label) + --entity-label strings Entity labels to use for all seeds (default [source:oos]) + --entity-name string Entity name to use for all seeds + -e, --err-file string File to write errors to. '-' writes to stderr. (default "-") + -f, --filename string Filename or directory to read from. If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin. + -h, --help help for convertoos + --ignore-scheme Ignore the URL's scheme when checking if this URL is already imported. (default true) + -o, --out-file string File to write result to. '-' writes to stdout. (default "-") + --seed-label strings Seed labels to use for all seeds (default [source:oos]) + --skip-import Do not import existing seeds into state database + --toplevel Convert URI to toplevel by removing path (default true) + --truncate Truncate state database ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO diff --git a/docs/veidemannctl_import_duplicatereport.md b/docs/veidemannctl_import_duplicatereport.md index 1eb3bc5..46160cb 100644 --- a/docs/veidemannctl_import_duplicatereport.md +++ b/docs/veidemannctl_import_duplicatereport.md @@ -1,31 +1,34 @@ ## veidemannctl import duplicatereport -List duplicated seeds in Veidemann +List duplicated seeds or crawl entities in Veidemann ``` -veidemannctl import duplicatereport [kind] [flags] +veidemannctl import duplicatereport KIND [flags] ``` ### Options ``` - -b, --db-directory string Directory for storing state db (default "/tmp/veidemannctl") - -h, --help help for duplicatereport - --ignore-scheme Ignore the URL's scheme when checking for duplicates. - -o, --outFile string File to write output. - -r, --reset-db Clean state db - --toplevel Convert URI to toplevel by removing path before checking for duplicates. + -b, --db-dir string Directory for storing state db (default "/tmp/veidemannctl") + -h, --help help for duplicatereport + --ignore-scheme Ignore the URL's scheme when checking for duplicates. + -o, --out-file string File to write output. + --skip-import Do not import existing seeds into state database + --toplevel Convert URI to toplevel by removing path before checking for duplicates. + --truncate Truncate state database ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO diff --git a/docs/veidemannctl_import_seed.md b/docs/veidemannctl_import_seed.md index b222c39..b504942 100644 --- a/docs/veidemannctl_import_seed.md +++ b/docs/veidemannctl_import_seed.md @@ -20,28 +20,32 @@ veidemannctl import seed [flags] ### Options ``` - --checkuri Check the uri for liveness and follow 301 - --checkuri-timeout int Timeout in ms when checking uri for liveness. (default 500) - --crawljob-id string Set crawlJob ID for new seeds. - -b, --db-directory string Directory for storing state db (default "/tmp/veidemannctl") - --dry-run Run the import without writing anything to Veidemann - -e, --errorfile string File to write errors to. - -f, --filename string Filename or directory to read from. If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin. - -h, --help help for seed - --ignore-scheme Ignore the URL's scheme when checking if this URL is already imported. - -r, --reset-db Clean state db - --toplevel Convert URI to toplevel by removing path. + --check-uri Check the uri for liveness and follow permanent redirects + --check-uri-timeout duration Timeout duration when checking uri for liveness (default 2s) + -c, --concurrency int Number of concurrent workers (default 16) + --crawljob-id string Set crawlJob ID for new seeds + -b, --db-dir string Directory for storing state db (default "/tmp/veidemannctl") + --dry-run Run without actually writing anything to Veidemann + -e, --err-file string File to write errors to. "-" writes to stderr (default "-") + -f, --filename string Filename or directory to read from. If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin. + -h, --help help for seed + --ignore-scheme Ignore the URL's scheme when checking if this URL is already imported (default true) + --skip-import Do not import existing seeds into state database + --toplevel Convert URI by removing path + --truncate Truncate state database ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO diff --git a/docs/veidemannctl_logconfig.md b/docs/veidemannctl_logconfig.md index 8ccbb77..6bfcdce 100644 --- a/docs/veidemannctl_logconfig.md +++ b/docs/veidemannctl_logconfig.md @@ -15,17 +15,19 @@ Configure logging. ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler * [veidemannctl logconfig delete](veidemannctl_logconfig_delete.md) - Delete a logger * [veidemannctl logconfig list](veidemannctl_logconfig_list.md) - List configured loggers * [veidemannctl logconfig set](veidemannctl_logconfig_set.md) - Configure logger diff --git a/docs/veidemannctl_logconfig_delete.md b/docs/veidemannctl_logconfig_delete.md index 5ccbfdd..4bfc844 100644 --- a/docs/veidemannctl_logconfig_delete.md +++ b/docs/veidemannctl_logconfig_delete.md @@ -7,7 +7,7 @@ Delete a logger Delete a logger. ``` -veidemannctl logconfig delete [logger] [flags] +veidemannctl logconfig delete LOGGER [flags] ``` ### Options @@ -19,12 +19,14 @@ veidemannctl logconfig delete [logger] [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO diff --git a/docs/veidemannctl_logconfig_list.md b/docs/veidemannctl_logconfig_list.md index b7e2139..ef628b8 100644 --- a/docs/veidemannctl_logconfig_list.md +++ b/docs/veidemannctl_logconfig_list.md @@ -19,12 +19,14 @@ veidemannctl logconfig list [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO diff --git a/docs/veidemannctl_logconfig_set.md b/docs/veidemannctl_logconfig_set.md index 98baaba..c513eae 100644 --- a/docs/veidemannctl_logconfig_set.md +++ b/docs/veidemannctl_logconfig_set.md @@ -7,7 +7,7 @@ Configure logger Configure logger. ``` -veidemannctl logconfig set [logger] [level] [flags] +veidemannctl logconfig set LOGGER LEVEL [flags] ``` ### Options @@ -19,12 +19,14 @@ veidemannctl logconfig set [logger] [level] [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO diff --git a/docs/veidemannctl_login.md b/docs/veidemannctl_login.md index 8988c71..02b901b 100644 --- a/docs/veidemannctl_login.md +++ b/docs/veidemannctl_login.md @@ -1,6 +1,6 @@ ## veidemannctl login -Initiate browser session for logging in to Veidemann +Log in to Veidemann ``` veidemannctl login [flags] @@ -10,21 +10,23 @@ veidemannctl login [flags] ``` -h, --help help for login - -m, --manual Manually copy and paste login url and code. Use this to log in from a remote terminal. + -m, --manual Manually copy and paste login url and code. Used to log in from a remote terminal. ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_logout.md b/docs/veidemannctl_logout.md index 125b537..81e93d9 100644 --- a/docs/veidemannctl_logout.md +++ b/docs/veidemannctl_logout.md @@ -19,15 +19,17 @@ veidemannctl logout [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_pause.md b/docs/veidemannctl_pause.md index 6fafedb..4804c64 100644 --- a/docs/veidemannctl_pause.md +++ b/docs/veidemannctl_pause.md @@ -1,10 +1,6 @@ ## veidemannctl pause -Pause crawler - -### Synopsis - -Pause crawler +Request crawler to pause ``` veidemannctl pause [flags] @@ -19,15 +15,17 @@ veidemannctl pause [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_report.md b/docs/veidemannctl_report.md index 3f4b546..2d08e46 100644 --- a/docs/veidemannctl_report.md +++ b/docs/veidemannctl_report.md @@ -1,10 +1,6 @@ ## veidemannctl report -Get report - -### Synopsis - -Request a report. +Request a report ### Options @@ -15,17 +11,19 @@ Request a report. ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler * [veidemannctl report crawlexecution](veidemannctl_report_crawlexecution.md) - Get current status for crawl executions * [veidemannctl report crawllog](veidemannctl_report_crawllog.md) - View crawl log * [veidemannctl report jobexecution](veidemannctl_report_jobexecution.md) - Get current status for job executions diff --git a/docs/veidemannctl_report_crawlexecution.md b/docs/veidemannctl_report_crawlexecution.md index 4134c09..11a4056 100644 --- a/docs/veidemannctl_report_crawlexecution.md +++ b/docs/veidemannctl_report_crawlexecution.md @@ -7,14 +7,14 @@ Get current status for crawl executions Get current status for crawl executions. ``` -veidemannctl report crawlexecution [flags] +veidemannctl report crawlexecution [ID ...] [flags] ``` ### Options ``` --desc Order descending - -f, --filename string File name to write to + -f, --filename string Filename to write to -q, --filter strings Filter objects by field (i.e. meta.description=foo) --from string From start time -h, --help help for crawlexecution @@ -31,15 +31,17 @@ veidemannctl report crawlexecution [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl report](veidemannctl_report.md) - Get report +* [veidemannctl report](veidemannctl_report.md) - Request a report diff --git a/docs/veidemannctl_report_crawllog.md b/docs/veidemannctl_report_crawllog.md index 6eb2f42..66b7df6 100644 --- a/docs/veidemannctl_report_crawllog.md +++ b/docs/veidemannctl_report_crawllog.md @@ -7,34 +7,35 @@ View crawl log View crawl log. ``` -veidemannctl report crawllog [flags] +veidemannctl report crawllog [ID ...] [flags] ``` ### Options ``` - -f, --filename string File name to write to - -q, --filter string Filter objects by field (i.e. meta.description=foo - -h, --help help for crawllog - -o, --output string Output format (table|wide|json|yaml|template|template-file) (default "table") - -p, --page int32 The page number - -s, --pagesize int32 Number of objects to get (default 10) - -t, --template string A Go template used to format the output - -w, --watch Get a continous stream of changes + --execution-id string Execution ID + -f, --filename string Filename to write to + -h, --help help for crawllog + -o, --output string Output format (table|wide|json|yaml|template|template-file) (default "table") + -p, --page int32 The page number + -s, --pagesize int32 Number of objects to get (default 10) + -t, --template string A Go template used to format the output ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl report](veidemannctl_report.md) - Get report +* [veidemannctl report](veidemannctl_report.md) - Request a report diff --git a/docs/veidemannctl_report_jobexecution.md b/docs/veidemannctl_report_jobexecution.md index 523b18a..e55c378 100644 --- a/docs/veidemannctl_report_jobexecution.md +++ b/docs/veidemannctl_report_jobexecution.md @@ -7,14 +7,14 @@ Get current status for job executions Get current status for job executions. ``` -veidemannctl report jobexecution [flags] +veidemannctl report jobexecution [ID ...] [flags] ``` ### Options ``` --desc Order descending - -f, --filename string File name to write to + -f, --filename string Filename to write to -q, --filter strings Filter objects by field (i.e. meta.description=foo --from string From start time -h, --help help for jobexecution @@ -31,15 +31,17 @@ veidemannctl report jobexecution [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl report](veidemannctl_report.md) - Get report +* [veidemannctl report](veidemannctl_report.md) - Request a report diff --git a/docs/veidemannctl_report_pagelog.md b/docs/veidemannctl_report_pagelog.md index 916bf42..78b097f 100644 --- a/docs/veidemannctl_report_pagelog.md +++ b/docs/veidemannctl_report_pagelog.md @@ -7,34 +7,35 @@ View page log View page log. ``` -veidemannctl report pagelog [flags] +veidemannctl report pagelog [ID ...] [flags] ``` ### Options ``` - -f, --filename string File name to write to - -q, --filter string Filter objects by field (i.e. meta.description=foo - -h, --help help for pagelog - -o, --output string Output format (table|wide|json|yaml|template|template-file) (default "table") - -p, --page int32 The page number - -s, --pagesize int32 Number of objects to get (default 10) - -t, --template string A Go template used to format the output - -w, --watch Get a continous stream of changes + --execution-id string Execution ID + -f, --filename string Filename to write to + -h, --help help for pagelog + -o, --output string Output format (table|wide|json|yaml|template|template-file) (default "table") + -p, --page int32 The page number + -s, --pagesize int32 Number of objects to get (default 10) + -t, --template string A Go template used to format the output ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl report](veidemannctl_report.md) - Get report +* [veidemannctl report](veidemannctl_report.md) - Request a report diff --git a/docs/veidemannctl_report_query.md b/docs/veidemannctl_report_query.md index 008993a..f633407 100644 --- a/docs/veidemannctl_report_query.md +++ b/docs/veidemannctl_report_query.md @@ -4,16 +4,16 @@ Run a database query ### Synopsis -Run a database query. The query should be a java script string like the ones used by RethinkDb javascript driver. +Run a database query. The query should be a JavaScript string like the ones used by RethinkDB JavaScript driver. ``` -veidemannctl report query [queryString|file] args... [flags] +veidemannctl report query (QUERY-STRING | FILENAME) [ARGS ...] [flags] ``` ### Options ``` - -f, --filename string File name to write to + -f, --filename string Filename to write to -h, --help help for query -o, --output string Output format (json|yaml|template|template-file) (default "json") -p, --page int32 The page number @@ -24,15 +24,17 @@ veidemannctl report query [queryString|file] args... [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl report](veidemannctl_report.md) - Get report +* [veidemannctl report](veidemannctl_report.md) - Request a report diff --git a/docs/veidemannctl_run.md b/docs/veidemannctl_run.md index 711841a..2964b85 100644 --- a/docs/veidemannctl_run.md +++ b/docs/veidemannctl_run.md @@ -1,15 +1,16 @@ ## veidemannctl run -Immediately run a crawl +Run a crawl job ### Synopsis -Run a crawl. If seedId is submitted only this seed will be run using the configuration -from the submitted jobId. This will run even if the seed is not configured to use the jobId. -If seedId is not submitted then all the seeds wich are configured to use the submitted jobId will be crawled. +Run a crawl job. + +If a seed is provided, the job will be created and started with the seed only. +If the job is already running, the seed will be added to the running job. ``` -veidemannctl run jobId [seedId] [flags] +veidemannctl run JOB-ID [SEED-ID] [flags] ``` ### Options @@ -21,15 +22,17 @@ veidemannctl run jobId [seedId] [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_script-parameters.md b/docs/veidemannctl_script-parameters.md index 8347891..8bd84b3 100644 --- a/docs/veidemannctl_script-parameters.md +++ b/docs/veidemannctl_script-parameters.md @@ -1,21 +1,21 @@ ## veidemannctl script-parameters -Get the active script parameters for a Crawl Job +Get the effective script parameters for a crawl job ### Synopsis -Get the active script parameters for a Crawl Job +Get the effective script parameters for a crawl job and optionally a seed in the context of the crawl job. Examples: # See active script parameters for a Crawl Job veidemannctl script-parameters 5604f0cc-315d-4091-8d6e-1b17a7eb990b - # See active script parameters for a Crawl Job and eventual overrides from Seed and Entity + # Get effective script parameters for a Seed in the context of a Crawl Job veidemannctl script-parameters 5604f0cc-315d-4091-8d6e-1b17a7eb990b 9f89ca44-afe0-4f8f-808f-9df1a0fe64c9 ``` -veidemannctl script-parameters CRAWLJOB_CONFIG_ID [SEED_ID] [flags] +veidemannctl script-parameters JOB-ID [SEED-ID] [flags] ``` ### Options @@ -27,15 +27,17 @@ veidemannctl script-parameters CRAWLJOB_CONFIG_ID [SEED_ID] [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_status.md b/docs/veidemannctl_status.md index 0fcae45..f020767 100644 --- a/docs/veidemannctl_status.md +++ b/docs/veidemannctl_status.md @@ -1,10 +1,10 @@ ## veidemannctl status -Get crawler status +Display crawler status ### Synopsis -Get crawler status. +Display crawler status. ``` veidemannctl status [flags] @@ -19,15 +19,17 @@ veidemannctl status [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_unpause.md b/docs/veidemannctl_unpause.md index 1d9806b..ccf3f77 100644 --- a/docs/veidemannctl_unpause.md +++ b/docs/veidemannctl_unpause.md @@ -1,10 +1,6 @@ ## veidemannctl unpause -Unpause crawler - -### Synopsis - -Unpause crawler +Request crawler to unpause ``` veidemannctl unpause [flags] @@ -19,15 +15,17 @@ veidemannctl unpause [flags] ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_update.md b/docs/veidemannctl_update.md index 2681c55..05aafc9 100644 --- a/docs/veidemannctl_update.md +++ b/docs/veidemannctl_update.md @@ -1,48 +1,50 @@ ## veidemannctl update -Update the value(s) for an object type +Update fields of config objects of the same kind ### Synopsis -Update one or many objects with new values. +Update a field of one or many config objects of the same kind ``` -veidemannctl update [object_type] [flags] +veidemannctl update KIND [ID ...] [flags] ``` ### Examples ``` # Add CrawlJob for a seed. -veidemannctl update seed -n "http://www.gwpda.org/" -u seed.jobRef+=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709 +veidemannctl update seed -n "https://www.gwpda.org/" -u seed.jobRef+=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709 # Replace all configured CrawlJobs for a seed with a new one. -veidemannctl update seed -n "http://www.gwpda.org/" -u seed.jobRef=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709 +veidemannctl update seed -n "https://www.gwpda.org/" -u seed.jobRef=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709 ``` ### Options ``` - -q, --filter string Filter objects by field (i.e. meta.description=foo) - -h, --help help for update - -l, --label string List objects by label (: | ) - -s, --limit int32 Limit the number of objects to update. 0 = no limit - -n, --name string List objects by name (accepts regular expressions) - -u, --updateField string Filter objects by field (i.e. meta.description=foo) + -q, --filter stringArray Filter objects by field (i.e. meta.description=foo) + -h, --help help for update + -l, --label string Filter objects by label (: | ) + -s, --limit int32 Limit the number of objects to update. 0 = no limit + -n, --name string Filter objects by name (accepts regular expressions) + -u, --update-field string Which field to update (i.e. meta.description=foo) ``` ### Options inherited from parent commands ``` - --apiKey string Api-key used for authentication instead of interactive logon trough IDP. - --config string config file (default is $HOME/.veidemannctl.yaml) - --context string The name of the veidemannconfig context to use. - -c, --controllerAddress string Address to the Controller service (default "localhost:50051") - -d, --debug Turn on debugging - --serverNameOverride string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests. + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests ``` ### SEE ALSO -* [veidemannctl](veidemannctl.md) - Veidemann command line client +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler diff --git a/docs/veidemannctl_version.md b/docs/veidemannctl_version.md new file mode 100644 index 0000000..0a0ffac --- /dev/null +++ b/docs/veidemannctl_version.md @@ -0,0 +1,31 @@ +## veidemannctl version + +Print the client version information + +``` +veidemannctl version [flags] +``` + +### Options + +``` + -h, --help help for version +``` + +### Options inherited from parent commands + +``` + --api-key string If set, it will be used as the bearer token for authentication + --config string Path to the config file to use (By default configuration file is stored under $HOME/.veidemann/contexts/ + --context string The name of the context to use + --log-caller include information about caller in log output + --log-format string set log format, available formats are: "pretty" or "json" (default "pretty") + --log-level string set log level, available levels are "panic", "fatal", "error", "warn", "info", "debug" and "trace" (default "info") + --server string The address of the Veidemann server to use + --server-name-override string If set, it will override the virtual host name of authority (e.g. :authority header field) in requests +``` + +### SEE ALSO + +* [veidemannctl](veidemannctl.md) - veidemannctl controls the Veidemann web crawler + diff --git a/src/format/JsonFormatter.go b/format/JsonFormatter.go similarity index 70% rename from src/format/JsonFormatter.go rename to format/JsonFormatter.go index f205213..86434a3 100644 --- a/src/format/JsonFormatter.go +++ b/format/JsonFormatter.go @@ -14,27 +14,31 @@ package format import ( - "errors" "fmt" - "google.golang.org/protobuf/proto" "io" "reflect" + + "google.golang.org/protobuf/proto" ) +// jsonFormatter is a formatter that writes records as json type jsonFormatter struct { *MarshalSpec } +// newJsonFormatter creates a new json formatter func newJsonFormatter(s *MarshalSpec) Formatter { return &jsonFormatter{ MarshalSpec: s, } } +// WriteRecord writes a record to the formatters writer func (jf *jsonFormatter) WriteRecord(record interface{}) error { switch v := record.(type) { case string: - fmt.Fprint(jf.rWriter, v) + _, err := fmt.Fprint(jf.rWriter, v) + return err case proto.Message: var values reflect.Value values = reflect.ValueOf(v).Elem().FieldByName("Value") @@ -45,32 +49,33 @@ func (jf *jsonFormatter) WriteRecord(record interface{}) error { } for i := 0; i < values.Len(); i++ { - m := values.Index(i).Interface().(proto.Message) - + m, ok := values.Index(i).Interface().(proto.Message) + if !ok { + return fmt.Errorf("illegal record type '%T'", record) + } err := marshalElementJson(jf.rWriter, m) if err != nil { return err } } } else { - m := reflect.ValueOf(v).Interface().(proto.Message) - - err := marshalElementJson(jf.rWriter, m) + err := marshalElementJson(jf.rWriter, v) if err != nil { return err } } default: - return errors.New(fmt.Sprintf("Illegal record type '%T'", record)) + return fmt.Errorf("illegal record type '%T'", record) } return nil } +// marshalElementJson marshals a proto message to json func marshalElementJson(w io.Writer, msg proto.Message) error { if b, err := jsonMarshaler.Marshal(msg); err != nil { - return errors.New(fmt.Sprintf("Could not convert %v to JSON: %v", msg, err)) + return fmt.Errorf("could not convert %v to JSON: %w", msg, err) } else { - fmt.Fprintln(w, string(b)) - return nil + _, err := fmt.Fprintln(w, string(b)) + return err } } diff --git a/src/format/TemplateFormatter.go b/format/TemplateFormatter.go similarity index 83% rename from src/format/TemplateFormatter.go rename to format/TemplateFormatter.go index 4a35557..d123078 100644 --- a/src/format/TemplateFormatter.go +++ b/format/TemplateFormatter.go @@ -20,21 +20,23 @@ import ( "bytes" "encoding/json" "fmt" - "google.golang.org/protobuf/types/known/timestamppb" - "github.com/nlnwa/veidemann-api/go/config/v1" - log "github.com/sirupsen/logrus" "reflect" "strings" "text/template" "time" + + "github.com/nlnwa/veidemann-api/go/config/v1" + "google.golang.org/protobuf/types/known/timestamppb" ) +// templateFormatter is a formatter that uses a template to format the output type templateFormatter struct { *MarshalSpec headerWritten bool parsedTemplate *template.Template } +// newTemplateFormatter creates a new template formatter func newTemplateFormatter(s *MarshalSpec) (Formatter, error) { t := &templateFormatter{ MarshalSpec: s, @@ -47,6 +49,7 @@ func newTemplateFormatter(s *MarshalSpec) (Formatter, error) { return t, nil } +// WriteRecord writes a record to the formatters writer func (tf *templateFormatter) WriteRecord(record interface{}) error { if !tf.headerWritten { tf.headerWritten = true @@ -54,7 +57,7 @@ func (tf *templateFormatter) WriteRecord(record interface{}) error { if tpl != nil { err := tpl.Execute(tf.rWriter, nil) if err != nil { - log.Fatal("Failed applying header template: ", err) + return fmt.Errorf("failed applying header template: %w", err) } } } @@ -65,18 +68,19 @@ func (tf *templateFormatter) WriteRecord(record interface{}) error { var j interface{} err := json.Unmarshal([]byte(r), &j) if err != nil { - return fmt.Errorf("failed to parse json: %v", err) + return fmt.Errorf("failed to parse json: %w", err) } record = j } err := tpl.Execute(tf.rWriter, record) if err != nil { - return fmt.Errorf("failed applying template to '%v': %v", record, err) + return fmt.Errorf("failed applying template to '%v': %w", record, err) } } return nil } +// parseTemplate parses a template string and returns a template func parseTemplate(templateString string) (*template.Template, error) { ESC := string(rune(0x1b)) funcMap := template.FuncMap{ @@ -108,29 +112,29 @@ func parseTemplate(templateString string) (*template.Template, error) { if ts == nil { return " " } else { - dateTime := ts["dateTime"].(map[string]interface{}) - date := dateTime["date"].(map[string]interface{}) - time := dateTime["time"].(map[string]interface{}) + dateTime, _ := ts["dateTime"].(map[string]interface{}) + date, _ := dateTime["date"].(map[string]interface{}) + time, _ := dateTime["time"].(map[string]interface{}) return fmt.Sprintf("%04.f-%02.f-%02.fT%02.f:%02.f:%02.f", date["year"], date["month"], date["day"], time["hour"], time["minute"], time["second"]) } }, - "json": func(v interface{}) string { + "json": func(v interface{}) (string, error) { if v == nil { - return "" + return "", nil } else { var buf bytes.Buffer encoder := json.NewEncoder(&buf) encoder.SetEscapeHTML(false) err := encoder.Encode(v) if err != nil { - log.Fatal(err) + return "", err } - return string(buf.Bytes()) + return buf.String(), nil } }, - "prettyJson": func(v interface{}) string { + "prettyJson": func(v interface{}) (string, error) { if v == nil { - return "" + return "", nil } else { var buf bytes.Buffer encoder := json.NewEncoder(&buf) @@ -138,16 +142,17 @@ func parseTemplate(templateString string) (*template.Template, error) { encoder.SetIndent("", " ") err := encoder.Encode(v) if err != nil { - log.Fatal(err) + return "", err } - return buf.String() + return buf.String(), nil } }, "nl": func() string { return "\n" }, "join": func(sep string, v interface{}) string { a := reflect.ValueOf(v) if a.Kind() != reflect.Slice { - return v.(string) + s, _ := v.(string) + return s } var b strings.Builder if a.Len() == 0 { @@ -191,17 +196,17 @@ func parseTemplate(templateString string) (*template.Template, error) { "printLabels": func(v interface{}) string { if labels, ok := v.([]*config.Label); ok { b := strings.Builder{} - b.WriteString("[") + // b.WriteString("[") for i, l := range labels { if i > 0 { b.WriteString(", ") } b.WriteString(l.Key) - b.WriteString(":'") + b.WriteString(":") b.WriteString(l.Value) - b.WriteString("'") + // b.WriteString("'") } - b.WriteString("]") + // b.WriteString("]") return b.String() } return "" diff --git a/src/format/YamlFormatter.go b/format/YamlFormatter.go similarity index 61% rename from src/format/YamlFormatter.go rename to format/YamlFormatter.go index caf541d..1b3db15 100644 --- a/src/format/YamlFormatter.go +++ b/format/YamlFormatter.go @@ -14,24 +14,27 @@ package format import ( - "errors" "fmt" - "github.com/ghodss/yaml" - "google.golang.org/protobuf/proto" "io" "reflect" + + "github.com/invopop/yaml" + "google.golang.org/protobuf/proto" ) +// yamlFormatter is a formatter that writes records as yaml type yamlFormatter struct { *MarshalSpec } +// newYamlFormatter creates a new yaml formatter func newYamlFormatter(s *MarshalSpec) Formatter { return &yamlFormatter{ MarshalSpec: s, } } +// WriteRecord writes a record to the formatters writer func (yf *yamlFormatter) WriteRecord(record interface{}) error { switch v := record.(type) { case string: @@ -41,8 +44,14 @@ func (yf *yamlFormatter) WriteRecord(record interface{}) error { return err } - fmt.Fprint(yf.rWriter, string(final)) - fmt.Fprintln(yf.rWriter, "---") + _, err = fmt.Fprint(yf.rWriter, string(final)) + if err != nil { + return err + } + _, err = fmt.Fprintln(yf.rWriter, "---") + if err != nil { + return err + } case proto.Message: var values reflect.Value values = reflect.ValueOf(v).Elem().FieldByName("Value") @@ -53,44 +62,49 @@ func (yf *yamlFormatter) WriteRecord(record interface{}) error { } for i := 0; i < values.Len(); i++ { + var err error if i > 0 { - yf.rWriter.Write([]byte("---\n")) + _, err = yf.rWriter.Write([]byte("---\n")) + } + if err != nil { + return err } - m := values.Index(i).Interface().(proto.Message) - - err := marshalElementYaml(yf.rWriter, m) + m, ok := values.Index(i).Interface().(proto.Message) + if !ok { + return fmt.Errorf("illegal record type '%T'", record) + } + err = marshalElementYaml(yf.rWriter, m) if err != nil { return err } } } else { - m := reflect.ValueOf(v).Interface().(proto.Message) - - err := marshalElementYaml(yf.rWriter, m) + err := marshalElementYaml(yf.rWriter, v) if err != nil { return err } - yf.rWriter.Write([]byte("---\n")) + _, err = yf.rWriter.Write([]byte("---\n")) + return err } default: - return errors.New(fmt.Sprintf("Illegal record type '%T'", record)) + return fmt.Errorf("illegal record type '%T'", record) } return nil } +// marshalElementYaml marshals a proto message to yaml func marshalElementYaml(w io.Writer, msg proto.Message) error { r, err := jsonMarshaler.Marshal(msg) if err != nil { - return errors.New(fmt.Sprintf("Could not convert %v to JSON: %v", msg, err)) + return fmt.Errorf("failed to convert %v to JSON: %w", msg, err) } final, err := yaml.JSONToYAML(r) if err != nil { - return errors.New(fmt.Sprintf("Could not convert %v to YAML: %v", r, err)) + return fmt.Errorf("failed to convert %v to YAML: %w", msg, err) } - fmt.Fprintln(w, string(final)) - - return nil + _, err = fmt.Fprintln(w, string(final)) + return err } diff --git a/format/formatter.go b/format/formatter.go new file mode 100644 index 0000000..e87fcc7 --- /dev/null +++ b/format/formatter.go @@ -0,0 +1,419 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package format + +import ( + "bufio" + "bytes" + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + + "github.com/invopop/yaml" + "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/rs/zerolog/log" + "google.golang.org/protobuf/encoding/protojson" +) + +var jsonMarshaler = &protojson.MarshalOptions{EmitUnpopulated: true} +var jsonUnMarshaler = &protojson.UnmarshalOptions{DiscardUnknown: true} + +//go:embed res +var res embed.FS + +// templateDir is the directory where the templates are located +const templateDir = "res/" + +// Formatter is the interface for formatters +type Formatter interface { + WriteRecord(interface{}) error + Close() error +} + +// MarshalSpec is the specification for a formatter +type MarshalSpec struct { + ObjectType string + Format string + Template string + defaultTemplateName string + + rFormat string + rTemplate string + rWriter io.Writer + resolved bool +} + +// NewFormatter creates a new formatter +func NewFormatter(objectType string, out io.Writer, format string, template string) (formatter Formatter, err error) { + s := &MarshalSpec{ + ObjectType: objectType, + Format: format, + Template: template, + rWriter: out, + } + + if err := s.resolve(); err != nil { + return nil, fmt.Errorf("failed to resolve formatter: %w", err) + } + + switch s.rFormat { + case "json": + formatter = newJsonFormatter(s) + case "yaml": + formatter = newYamlFormatter(s) + case "template": + formatter, err = newTemplateFormatter(s) + case "table": + formatter, err = newTemplateFormatter(s) + case "wide": + formatter, err = newTemplateFormatter(s) + default: + return nil, fmt.Errorf("illegal or missing format '%s'", s.rFormat) + } + return +} + +// ResolveWriter creates a file and returns an io.Writer for the file. +// If filename is empty, os.StdOut is returned +func ResolveWriter(filename string) (io.WriteCloser, error) { + if filename == "" || filename == "-" { + return os.Stdout, nil + } else { + f, err := os.Create(filename) + if err != nil { + return nil, fmt.Errorf("failed to create file '%s': %w", filename, err) + } + return f, nil + } +} + +// resolve resolves the template and format +func (s *MarshalSpec) resolve() (err error) { + if !s.resolved { + switch s.Format { + case "template": + if s.Template == "" { + if s.defaultTemplateName != "" { + data, err := res.ReadFile(templateDir + s.defaultTemplateName) + if err != nil { + return err + } + s.rTemplate = string(data) + } else { + return errors.New("Format is 'template', but template is missing") + } + } else { + s.rTemplate = s.Template + } + s.rFormat = s.Format + case "template-file": + if s.Template == "" { + return errors.New("format is 'template-file', but template is missing") + } + data, err := os.ReadFile(s.Template) + if err != nil { + return fmt.Errorf("template not found: %w", err) + } + s.rTemplate = string(data) + s.rFormat = s.Format + case "json": + s.rTemplate = "" + s.rFormat = s.Format + case "yaml": + s.rTemplate = "" + s.rFormat = s.Format + case "table": + if s.ObjectType == "" { + return fmt.Errorf("format is table, but object type is missing") + } + + templateName := s.ObjectType + "_table.template" + data, err := res.ReadFile(templateDir + templateName) + if err != nil { + return err + } + s.rTemplate = string(data) + s.rFormat = s.Format + case "wide": + if s.ObjectType == "" { + return fmt.Errorf("format is wide, but object type is missing") + } + + templateName := s.ObjectType + "_wide.template" + data, err := res.ReadFile(templateDir + templateName) + if err != nil { + return err + } + s.rTemplate = string(data) + s.rFormat = s.Format + default: + s.rTemplate = s.Template + s.rFormat = s.Format + } + } + s.resolved = true + return nil +} + +// Close closes the formatter +func (s *MarshalSpec) Close() error { + if s == nil { + return nil + } + if closer, ok := s.rWriter.(io.Closer); ok && closer != os.Stdout { + return closer.Close() + } + return nil +} + +type fileType string + +const jsonFile fileType = "json" +const yamlFile fileType = "yaml" + +// unmarshal unmarshals a file into a channel of ConfigObjects based on the file type +func unmarshal(r io.Reader, result chan<- *config.ConfigObject, done <-chan struct{}, t fileType) error { + var err error + + switch t { + case jsonFile: + err = unmarshalJson(r, result, done) + case yamlFile: + err = unmarshalYaml(r, result, done) + default: + err = fmt.Errorf("unknown file type '%s'", t) + } + + if err != nil { + return err + } + return nil +} + +// Unmarshal unmarshals a file into a channel of ConfigObjects +func Unmarshal(ctx context.Context, filename string, result chan<- *config.ConfigObject) error { + var err error + var f *os.File + var t fileType + + if filename == "" { + r := bufio.NewReader(os.Stdin) + + // read one byte to determine if json or yaml + b, err := r.Peek(1) + if err != nil { + return err + } + if b[0] == '{' { + t = jsonFile + } else { + t = yamlFile + } + + go func() { + defer close(result) + err := unmarshal(r, result, ctx.Done(), t) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal") + } + }() + + return nil + } + + f, err = os.Open(filename) + if err != nil { + return fmt.Errorf("failed to open file '%s': %w", filename, err) + } + + if fi, _ := f.Stat(); !fi.IsDir() { + if HasSuffix(f.Name(), ".yaml", ".yml") { + t = yamlFile + } else if HasSuffix(f.Name(), ".json", ".jsonl") { + t = jsonFile + } + + go func() { + defer close(result) + err := unmarshal(f, result, ctx.Done(), t) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal") + } + }() + + return nil + } + + des, err := f.ReadDir(0) + if err != nil { + return fmt.Errorf("failed to read directory '%s': %w", filename, err) + } + + var wg sync.WaitGroup + for _, fi := range des { + if fi.IsDir() { + continue + } + + if HasSuffix(fi.Name(), ".yaml", ".yml") { + t = yamlFile + } else if HasSuffix(fi.Name(), ".json", ".jsonl") { + t = jsonFile + } + + f, err = os.Open(fi.Name()) + if err != nil { + log.Error().Err(err).Msg("failed to open file") + continue + } + + wg.Add(1) + go func() { + f := f + defer wg.Done() + defer f.Close() + err := unmarshal(f, result, ctx.Done(), t) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal") + } + }() + } + // wait for all goroutines to finish + wg.Wait() + // close the channel to end the loop in the caller + close(result) + + return nil +} + +type yamlReader struct { + *bufio.Reader +} + +func newYamlReader(r io.Reader) yamlReader { + return yamlReader{Reader: bufio.NewReader(r)} +} + +// readYaml reads a yaml document from the reader and returns it as a byte array +func (yr yamlReader) readYaml() ([]byte, error) { + delim := []byte{'-', '-', '-'} + var inDoc bool = true + var err error + var l, doc []byte + + for inDoc && err == nil { + isPrefix := true + ln := []byte{} + for isPrefix && err == nil { + l, isPrefix, err = yr.ReadLine() + ln = append(ln, l...) + } + + if len(ln) >= 3 && bytes.Equal(delim, ln[:3]) { + inDoc = false + } else if len(ln) > 0 { + doc = append(doc, ln...) + doc = append(doc, '\n') + } + } + + return doc, err +} + +// readJson reads a yaml document from the reader and returns it as a json byte array +func (yr yamlReader) readJson() ([]byte, error) { + data, err := yr.readYaml() + if err != nil { + return nil, err + } + + data = bytes.TrimSpace(data) + if len(data) == 0 { + return nil, nil + } + + // convert yaml to json before unmarshaling because protojson doesn't support yaml + return yaml.YAMLToJSON(data) +} + +// unmarshalYaml unmarshals a yaml stream into ConfigObjects and sends them to the result channel +func unmarshalYaml(r io.Reader, result chan<- *config.ConfigObject, done <-chan struct{}) error { + yr := newYamlReader(r) + + var b []byte + var err error + + for { + b, err = yr.readJson() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + target := &config.ConfigObject{} + err = jsonUnMarshaler.Unmarshal(b, target) + if err != nil { + return err + } + select { + case <-done: + return nil + case result <- target: + } + } +} + +// unmarshalJson unmarshals a json stream into ConfigObjects and sends them to the result channel +func unmarshalJson(r io.Reader, result chan<- *config.ConfigObject, done <-chan struct{}) error { + dec := json.NewDecoder(r) + var err error + var msg json.RawMessage + + for { + err = dec.Decode(&msg) + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return err + } + target := &config.ConfigObject{} + err = jsonUnMarshaler.Unmarshal(msg, target) + if err != nil { + return err + } + select { + case <-done: + return nil + case result <- target: + } + } +} + +// HasSuffix checks if a string has one of the suffixes given as input +func HasSuffix(s string, suffix ...string) bool { + for _, suf := range suffix { + if strings.HasSuffix(s, suf) { + return true + } + } + return false +} diff --git a/src/format/formatter_test.go b/format/formatter_test.go similarity index 99% rename from src/format/formatter_test.go rename to format/formatter_test.go index 23a6e31..8daa46f 100644 --- a/src/format/formatter_test.go +++ b/format/formatter_test.go @@ -15,12 +15,13 @@ package format import ( "bytes" + "testing" + "time" + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/timestamppb" - "testing" - "time" ) func TestHasSuffix(t *testing.T) { diff --git a/src/format/objecttypes.go b/format/objecttypes.go similarity index 70% rename from src/format/objecttypes.go rename to format/objecttypes.go index a41cf8d..d535d9f 100644 --- a/src/format/objecttypes.go +++ b/format/objecttypes.go @@ -14,22 +14,25 @@ package format import ( - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "fmt" "sort" "strings" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" ) -// Get kind for string -func GetKind(Name string) configV1.Kind { - Name = strings.ToLower(Name) +// GetKind returns the Kind for the given name. +func GetKind(name string) configV1.Kind { + name = strings.ToLower(name) for _, k := range configV1.Kind_name { - if strings.ToLower(k) == Name { + if strings.ToLower(k) == name { return configV1.Kind(configV1.Kind_value[k]) } } return configV1.Kind_undefined } +// GetObjectNames returns a sorted list of object names. func GetObjectNames() []string { result := make([]string, len(configV1.Kind_name)-1) idx := 0 @@ -42,3 +45,12 @@ func GetObjectNames() []string { sort.Strings(result) return result } + +// ListObjectNames returns a string with a list of object names. +func ListObjectNames() string { + var names string + for _, v := range GetObjectNames() { + names += fmt.Sprintf(" - %s\n", v) + } + return fmt.Sprintln(names) +} diff --git a/src/format/objecttypes_test.go b/format/objecttypes_test.go similarity index 100% rename from src/format/objecttypes_test.go rename to format/objecttypes_test.go diff --git a/res/CrawlExecutionStatus_table.template b/format/res/CrawlExecutionStatus_table.template similarity index 100% rename from res/CrawlExecutionStatus_table.template rename to format/res/CrawlExecutionStatus_table.template diff --git a/res/CrawlExecutionStatus_wide.template b/format/res/CrawlExecutionStatus_wide.template similarity index 100% rename from res/CrawlExecutionStatus_wide.template rename to format/res/CrawlExecutionStatus_wide.template diff --git a/res/CrawlLog_table.template b/format/res/CrawlLog_table.template similarity index 100% rename from res/CrawlLog_table.template rename to format/res/CrawlLog_table.template diff --git a/res/CrawlLog_wide.template b/format/res/CrawlLog_wide.template similarity index 100% rename from res/CrawlLog_wide.template rename to format/res/CrawlLog_wide.template diff --git a/res/JobExecutionStatus_table.template b/format/res/JobExecutionStatus_table.template similarity index 100% rename from res/JobExecutionStatus_table.template rename to format/res/JobExecutionStatus_table.template diff --git a/res/JobExecutionStatus_wide.template b/format/res/JobExecutionStatus_wide.template similarity index 100% rename from res/JobExecutionStatus_wide.template rename to format/res/JobExecutionStatus_wide.template diff --git a/res/PageLog_table.template b/format/res/PageLog_table.template similarity index 93% rename from res/PageLog_table.template rename to format/res/PageLog_table.template index 50ba197..b637b7a 100644 --- a/res/PageLog_table.template +++ b/format/res/PageLog_table.template @@ -1,5 +1,5 @@ {{define "RESOURCE" -}} -{{brightmagenta}}{{printf `%36.36s %-5t %3d %15s %-92.92s %-12.12s %-5t %-22.22s` .WarcId .FromCache .StatusCode .DiscoveryPath .Uri .ResourceType .Renderable .MimeType}}{{reset}} +{{brightmagenta}}{{printf `%36.36s %-5t %3d %15s %-92.92s %-12.12s %-5t %-22.22s` .WarcId .FromCache .StatusCode .DiscoveryPath .Uri .ResourceType .Renderable .ContentType}}{{reset}} {{end -}} {{define "OUTLINK" -}} diff --git a/res/PageLog_wide.template b/format/res/PageLog_wide.template similarity index 93% rename from res/PageLog_wide.template rename to format/res/PageLog_wide.template index 50ba197..b637b7a 100644 --- a/res/PageLog_wide.template +++ b/format/res/PageLog_wide.template @@ -1,5 +1,5 @@ {{define "RESOURCE" -}} -{{brightmagenta}}{{printf `%36.36s %-5t %3d %15s %-92.92s %-12.12s %-5t %-22.22s` .WarcId .FromCache .StatusCode .DiscoveryPath .Uri .ResourceType .Renderable .MimeType}}{{reset}} +{{brightmagenta}}{{printf `%36.36s %-5t %3d %15s %-92.92s %-12.12s %-5t %-22.22s` .WarcId .FromCache .StatusCode .DiscoveryPath .Uri .ResourceType .Renderable .ContentType}}{{reset}} {{end -}} {{define "OUTLINK" -}} diff --git a/format/res/browserConfig_table.template b/format/res/browserConfig_table.template new file mode 100644 index 0000000..4544748 --- /dev/null +++ b/format/res/browserConfig_table.template @@ -0,0 +1,5 @@ +{{define "HEADER" -}} + {{printf `%-36.36s %-25s %-10s %-12s %-12s` "Id" "Name" "WindowSize" "PageTimeout" "MaxInactivity"}} +{{end -}} + +{{printf `%36s %-25s %-4dx%-5d %-12v %-12v` .Id .Meta.Name .Spec.BrowserConfig.WindowWidth .Spec.BrowserConfig.WindowHeight .Spec.BrowserConfig.PageLoadTimeoutMs .Spec.BrowserConfig.MaxInactivityTimeMs}} diff --git a/res/browserConfig_wide.template b/format/res/browserConfig_wide.template similarity index 100% rename from res/browserConfig_wide.template rename to format/res/browserConfig_wide.template diff --git a/res/browserScript_table.template b/format/res/browserScript_table.template similarity index 100% rename from res/browserScript_table.template rename to format/res/browserScript_table.template diff --git a/res/browserScript_wide.template b/format/res/browserScript_wide.template similarity index 100% rename from res/browserScript_wide.template rename to format/res/browserScript_wide.template diff --git a/res/collection_table.template b/format/res/collection_table.template similarity index 100% rename from res/collection_table.template rename to format/res/collection_table.template diff --git a/res/collection_wide.template b/format/res/collection_wide.template similarity index 100% rename from res/collection_wide.template rename to format/res/collection_wide.template diff --git a/res/crawlConfig_table.template b/format/res/crawlConfig_table.template similarity index 100% rename from res/crawlConfig_table.template rename to format/res/crawlConfig_table.template diff --git a/res/crawlConfig_wide.template b/format/res/crawlConfig_wide.template similarity index 100% rename from res/crawlConfig_wide.template rename to format/res/crawlConfig_wide.template diff --git a/res/crawlEntity_table.template b/format/res/crawlEntity_table.template similarity index 100% rename from res/crawlEntity_table.template rename to format/res/crawlEntity_table.template diff --git a/format/res/crawlEntity_wide.template b/format/res/crawlEntity_wide.template new file mode 100644 index 0000000..344d1c3 --- /dev/null +++ b/format/res/crawlEntity_wide.template @@ -0,0 +1,7 @@ +{{- /*gotype: github.com/nlnwa/veidemann-api/go/config/v1.ConfigObject*/ -}} + +{{define "HEADER" -}} + {{printf `%-36.36s %-40.40s %s` "Id" "Name" "Labels"}} +{{end -}} + +{{printLabels .Meta.Label | printf `%36s %-40.40s %s` .Id .Meta.Name}} diff --git a/res/crawlHostGroupConfig_table.template b/format/res/crawlHostGroupConfig_table.template similarity index 100% rename from res/crawlHostGroupConfig_table.template rename to format/res/crawlHostGroupConfig_table.template diff --git a/res/crawlHostGroupConfig_wide.template b/format/res/crawlHostGroupConfig_wide.template similarity index 100% rename from res/crawlHostGroupConfig_wide.template rename to format/res/crawlHostGroupConfig_wide.template diff --git a/res/crawlJob_table.template b/format/res/crawlJob_table.template similarity index 100% rename from res/crawlJob_table.template rename to format/res/crawlJob_table.template diff --git a/res/crawlJob_wide.template b/format/res/crawlJob_wide.template similarity index 100% rename from res/crawlJob_wide.template rename to format/res/crawlJob_wide.template diff --git a/res/crawlScheduleConfig_table.template b/format/res/crawlScheduleConfig_table.template similarity index 100% rename from res/crawlScheduleConfig_table.template rename to format/res/crawlScheduleConfig_table.template diff --git a/res/crawlScheduleConfig_wide.template b/format/res/crawlScheduleConfig_wide.template similarity index 59% rename from res/crawlScheduleConfig_wide.template rename to format/res/crawlScheduleConfig_wide.template index 8176bb2..98010cf 100644 --- a/res/crawlScheduleConfig_wide.template +++ b/format/res/crawlScheduleConfig_wide.template @@ -1,5 +1,5 @@ {{define "HEADER" -}} - {{printf `%-36.36s %-20.20s %-18s` "Id" "Name" "Labels"}} + {{printf `%-36.36s %-20.20s %-18s` "Id" "name" "Labels"}} {{end -}} {{printf `%36s %-20.20s %18s` .Id .Meta.Name .Meta.Label}} diff --git a/res/json.template b/format/res/json.template similarity index 100% rename from res/json.template rename to format/res/json.template diff --git a/res/politenessConfig_table.template b/format/res/politenessConfig_table.template similarity index 59% rename from res/politenessConfig_table.template rename to format/res/politenessConfig_table.template index 8176bb2..98010cf 100644 --- a/res/politenessConfig_table.template +++ b/format/res/politenessConfig_table.template @@ -1,5 +1,5 @@ {{define "HEADER" -}} - {{printf `%-36.36s %-20.20s %-18s` "Id" "Name" "Labels"}} + {{printf `%-36.36s %-20.20s %-18s` "Id" "name" "Labels"}} {{end -}} {{printf `%36s %-20.20s %18s` .Id .Meta.Name .Meta.Label}} diff --git a/res/politenessConfig_wide.template b/format/res/politenessConfig_wide.template similarity index 59% rename from res/politenessConfig_wide.template rename to format/res/politenessConfig_wide.template index 8176bb2..98010cf 100644 --- a/res/politenessConfig_wide.template +++ b/format/res/politenessConfig_wide.template @@ -1,5 +1,5 @@ {{define "HEADER" -}} - {{printf `%-36.36s %-20.20s %-18s` "Id" "Name" "Labels"}} + {{printf `%-36.36s %-20.20s %-18s` "Id" "name" "Labels"}} {{end -}} {{printf `%36s %-20.20s %18s` .Id .Meta.Name .Meta.Label}} diff --git a/res/roleMapping_table.template b/format/res/roleMapping_table.template similarity index 69% rename from res/roleMapping_table.template rename to format/res/roleMapping_table.template index abddd15..b443039 100644 --- a/res/roleMapping_table.template +++ b/format/res/roleMapping_table.template @@ -1,5 +1,5 @@ {{define "HEADER" -}} - {{printf `%-36.36s %-20.20s %-18.18s %-18.18s` "Id" "Name" "Labels" "Role"}} + {{printf `%-36.36s %-20.20s %-18.18s %-18.18s` "Id" "name" "Labels" "Role"}} {{end -}} {{printf `%36s %-20.20s %18.18v %18.18v` .Id .Meta.Name .Meta.Label .Spec.RoleMapping.Role}} diff --git a/format/res/roleMapping_wide.template b/format/res/roleMapping_wide.template new file mode 100644 index 0000000..773f20e --- /dev/null +++ b/format/res/roleMapping_wide.template @@ -0,0 +1,5 @@ +{{define "HEADER" -}} + {{printf `%-36.36s %-20.20s %-18.18s` "Id" "name" "Role"}} +{{end -}} + +{{printf `%36s %-20.20s %18.18v` .Id .Meta.Name .Spec.RoleMapping.Role}} diff --git a/res/seed_table.template b/format/res/seed_table.template similarity index 100% rename from res/seed_table.template rename to format/res/seed_table.template diff --git a/res/seed_wide.template b/format/res/seed_wide.template similarity index 100% rename from res/seed_wide.template rename to format/res/seed_wide.template diff --git a/go.mod b/go.mod index 700bf8f..211524e 100644 --- a/go.mod +++ b/go.mod @@ -1,37 +1,60 @@ module github.com/nlnwa/veidemannctl -go 1.16 +go 1.19 require ( - github.com/coreos/go-oidc v2.2.1+incompatible - github.com/dgraph-io/badger/v2 v2.2007.4 - github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/ghodss/yaml v1.0.0 - github.com/golang/glog v1.0.0 // indirect - github.com/golang/protobuf v1.5.2 - github.com/golang/snappy v0.0.4 // indirect - github.com/klauspost/compress v1.15.1 // indirect - github.com/magiconair/properties v1.8.6 + github.com/coreos/go-oidc/v3 v3.6.0 + github.com/dgraph-io/badger/v4 v4.1.0 + github.com/dgraph-io/ristretto v0.1.1 + github.com/golang/protobuf v1.5.3 + github.com/invopop/yaml v0.2.0 + github.com/magiconair/properties v1.8.7 github.com/mitchellh/go-homedir v1.1.0 + github.com/mitchellh/mapstructure v1.5.0 github.com/nlnwa/veidemann-api/go v0.0.0-20220110104816-ea13deeb9671 - github.com/pkg/errors v0.9.1 - github.com/pquerna/cachecontrol v0.1.0 // indirect - github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect - github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 - github.com/sirupsen/logrus v1.8.1 - github.com/spf13/afero v1.8.2 // indirect - github.com/spf13/cobra v1.4.0 - github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.7.0 - golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect - golang.org/x/net v0.0.0-20220403103023-749bd193bc2b - golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a - golang.org/x/sys v0.0.0-20220403205710-6acee93ad0eb // indirect - google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de // indirect - google.golang.org/grpc v1.45.0 - google.golang.org/protobuf v1.28.0 - gopkg.in/ini.v1 v1.66.4 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v2 v2.4.0 + github.com/rs/zerolog v1.29.1 + github.com/spf13/cobra v1.7.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.16.0 + github.com/stretchr/testify v1.8.4 + golang.org/x/net v0.10.0 + golang.org/x/oauth2 v0.8.0 + google.golang.org/grpc v1.55.0 + google.golang.org/protobuf v1.30.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.1.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spf13/afero v1.9.5 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index fbb63cf..3de6fbf 100644 --- a/go.sum +++ b/go.sum @@ -17,17 +17,6 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -36,7 +25,6 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -50,107 +38,59 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= -github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/coreos/go-oidc/v3 v3.6.0 h1:AKVxfYw1Gmkn/w96z0DbT/B/xFnzTd3MkZvWLjF4n/o= +github.com/coreos/go-oidc/v3 v3.6.0/go.mod h1:ZpHUsHBucTUj6WOkrP4E20UPynbLZzhTQ1XKCXkxyPc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= -github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= +github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw= +github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -159,8 +99,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -176,14 +114,14 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -195,13 +133,10 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -212,224 +147,113 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= -github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= -github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= +github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= -github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/nlnwa/veidemann-api/go v0.0.0-20220110104816-ea13deeb9671 h1:cfr+JnoQfYSeSRcZSqvbDm6h2o+x3ZQ0Qd8gJ5UXdzA= github.com/nlnwa/veidemann-api/go v0.0.0-20220110104816-ea13deeb9671/go.mod h1:Hc8kF9WOmwggqSJyXs/aUo2ocim92FrVtW/otmIgtk4= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= -github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= +github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU= -github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= -github.com/spf13/afero v1.8.2/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= +github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= -github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= +github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= +github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= -golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -453,7 +277,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -464,11 +287,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -476,11 +296,9 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -501,17 +319,10 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220403103023-749bd193bc2b h1:vI32FkLJNAWtGD4BwkThwEy6XS7ZLLMHkSkYfF8M0W0= -golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -521,16 +332,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -541,34 +344,20 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -587,45 +376,29 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220403205710-6acee93ad0eb h1:PVGECzEo9Y3uOidtkHGdd347NjLtITfJFO9BxFpmRoo= -golang.org/x/sys v0.0.0-20220403205710-6acee93ad0eb/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -642,7 +415,6 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -679,16 +451,9 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -709,19 +474,6 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -753,7 +505,6 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -767,35 +518,9 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de h1:9Ti5SG2U4cAcluryUo/sFay3TQKoxiFMfaT0pbizU7k= -google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -813,19 +538,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0/go.mod h1:DNq5QpG7LJqD2AamLZ7zvKE0DEpVl2BSEVjFycAAjRY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -840,30 +554,19 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= -gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/importutil/db.go b/importutil/db.go new file mode 100644 index 0000000..aba76b1 --- /dev/null +++ b/importutil/db.go @@ -0,0 +1,311 @@ +// Copyright © 2019 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package importutil + +import ( + "bytes" + "context" + "encoding/gob" + "errors" + "fmt" + "io" + "os" + "time" + + "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/ristretto/z" + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/rs/zerolog/log" +) + +// badgerLogger is a log adapter that implements badger.Logger +type badgerLogger struct { + prefix string +} + +func (l badgerLogger) Errorf(fmt string, args ...interface{}) { + log.Error().Msgf(l.prefix+fmt, args...) +} + +func (l badgerLogger) Warningf(fmt string, args ...interface{}) { + log.Warn().Msgf(l.prefix+fmt, args...) +} + +func (l badgerLogger) Infof(fmt string, args ...interface{}) { + log.Debug().Msgf(l.prefix+fmt, args...) +} + +func (l badgerLogger) Debugf(fmt string, args ...interface{}) { + log.Trace().Msgf(l.prefix+fmt, args...) +} + +type ExistsCode int + +const ( + Undefined ExistsCode = iota + NewKey + NewId + Exists +) + +func (e ExistsCode) ExistsInVeidemann() bool { + return e > NewKey +} + +func (e ExistsCode) String() string { + if e < Undefined || e > Exists { + return "UNKNOWN" + } + names := []string{ + "UNDEFINED", + "NEW_KEY", + "NEW_ID", + "EXISTS"} + + return names[e] +} + +type KeyNormalizer interface { + Normalize(key string) (string, error) +} + +type ImportDb struct { + db *badger.DB + gc *time.Ticker +} + +func NewImportDb(dbDir string, truncate bool) (*ImportDb, error) { + if err := os.MkdirAll(dbDir, 0777); err != nil { + return nil, fmt.Errorf("failed to create db dir %s: %w", dbDir, err) + } + opts := badger.DefaultOptions(dbDir) + opts.Logger = badgerLogger{prefix: "Badger: "} + + db, err := badger.Open(opts) + if err != nil { + return nil, fmt.Errorf("could not open db %s: %w", dbDir, err) + } + + if truncate { + err = db.DropAll() + if err != nil { + return nil, fmt.Errorf("failed to reset db %s: %w", dbDir, err) + } + } + + d := &ImportDb{ + db: db, + gc: time.NewTicker(5 * time.Minute), + } + + // Run GC in background + go func() { + for range d.gc.C { + d.RunValueLogGC(0.7) + } + }() + + return d, nil +} + +func (d *ImportDb) RunValueLogGC(discardRatio float64) { + var err error + for err == nil { + err = d.db.RunValueLogGC(discardRatio) + } +} + +// Close closes the database, stops the GC ticker and waits for +func (d *ImportDb) Close() { + _ = d.db.RunValueLogGC(0.7) + d.gc.Stop() + _ = d.db.Close() +} + +func ImportExisting(db *ImportDb, client configV1.ConfigClient, kind configV1.Kind, keyNormalizer KeyNormalizer) error { + req := &configV1.ListRequest{ + Kind: kind, + } + r, err := client.ListConfigObjects(context.Background(), req) + if err != nil { + return fmt.Errorf("failed to list %s from Veidemann: %w", kind.String(), err) + } + + var count, imported, failed int + + start := time.Now() + for { + msg, err := r.Recv() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return fmt.Errorf("error reading %s from Veidemann: %w", kind.String(), err) + } + + id := msg.GetId() + key := msg.GetMeta().GetName() + + if keyNormalizer != nil { + key, err = keyNormalizer.Normalize(key) + if err != nil { + failed++ + log.Error().Err(err).Str("key", msg.GetMeta().GetName()).Str("id", id).Msg("Normalization failed") + continue + } + } + + code, _, err := db.Set(key, id) + if err != nil { + return fmt.Errorf("error writing to db: %w", err) + } + + l := log.With().Str("key", key).Str("id", id).Str("kind", kind.String()).Logger() + + count++ + switch code { + case Undefined: + failed++ + case NewKey: + imported++ + l.Info().Msg("New key imported from Veidemann") + case NewId: + l.Info().Msg("New id imported from Veidemann") + case Exists: + l.Info().Msg("Already imported from Veidemann") + } + } + elapsed := time.Since(start) + log.Info().Str("kind", kind.String()).Int("total", count).Int("imported", imported).Int("errors", failed).Str("elapsed", elapsed.String()).Msg("Import from Veidemann complete") + + return nil +} + +// Iterate iterates over all keys in the db and calls the function with the key and value. +// The function is not called in parallel. +func (d *ImportDb) Iterate(fn func([]byte, []byte)) error { + stream := d.db.NewStream() + + // -- Optional settings + stream.NumGo = 16 // Set number of goroutines to use for iteration. + stream.LogPrefix = "Badger.Streaming" // For identifying stream logs. Outputs to Logger. + // -- End of optional settings. + + // Send is called serially, while Stream.Orchestrate is running. + stream.Send = func(buf *z.Buffer) error { + list, err := badger.BufferToKVList(buf) + if err != nil { + return err + } + for _, kv := range list.GetKv() { + fn(kv.Key, kv.Value) + } + return nil + } + + // Run the stream + return stream.Orchestrate(context.Background()) +} + +// Get returns the ids for the key +func (d *ImportDb) Get(key string) (ids []string, err error) { + for { + err = d.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if errors.Is(err, badger.ErrKeyNotFound) { + return nil + } + if err != nil { + return err + } + + return item.Value(func(v []byte) error { + ids = bytesToStringArray(v) + return nil + }) + }) + if !errors.Is(err, badger.ErrConflict) { + break + } + } + return ids, err +} + +// Set sets the id as a value for the key. +func (d *ImportDb) Set(key string, id string) (code ExistsCode, ids []string, err error) { + for { + err = d.db.Update(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if errors.Is(err, badger.ErrKeyNotFound) { + code = NewKey + ids = append(ids, id) + v := stringArrayToBytes(ids) + return txn.Set([]byte(key), v) + } + if err != nil { + return err + } + + err = item.Value(func(v []byte) error { + ids = bytesToStringArray(v) + return nil + }) + if err != nil { + return err + } + if !stringArrayContains(ids, id) { + code = NewId + ids = append(ids, id) + v := stringArrayToBytes(ids) + return txn.Set([]byte(key), v) + } else { + code = Exists + } + return nil + }) + if !errors.Is(err, badger.ErrConflict) { + break + } + } + return +} + +// stringArrayToBytes returns a byte array from a string array +func stringArrayToBytes(v []string) []byte { + buf := &bytes.Buffer{} + if err := gob.NewEncoder(buf).Encode(v); err != nil { + panic(err) + } + return buf.Bytes() +} + +// bytesToStringArray returns a string array from a byte array +func bytesToStringArray(v []byte) []string { + buf := bytes.NewBuffer(v) + strs := []string{} + if err := gob.NewDecoder(buf).Decode(&strs); err != nil && !errors.Is(err, io.EOF) { + panic(err) + } + return strs +} + +// stringArrayContains returns true if the string array contains the string +func stringArrayContains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} diff --git a/src/importutil/db_test.go b/importutil/db_test.go similarity index 50% rename from src/importutil/db_test.go rename to importutil/db_test.go index b10ba67..5ef7659 100644 --- a/src/importutil/db_test.go +++ b/importutil/db_test.go @@ -14,12 +14,12 @@ package importutil import ( - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" "reflect" "testing" + // configV1 "github.com/nlnwa/veidemann-api/go/config/v1" ) -func TestImportDb_stringArrayToBytes(t *testing.T) { +func TestStringArrayToBytes(t *testing.T) { tests := []struct { name string args []string @@ -32,15 +32,14 @@ func TestImportDb_stringArrayToBytes(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &ImportDb{} - if got := d.stringArrayToBytes(tt.args); !reflect.DeepEqual(got, tt.want) { + if got := stringArrayToBytes(tt.args); !reflect.DeepEqual(got, tt.want) { t.Errorf("ImportDb.stringArrayToBytes() = %v, want %v", got, tt.want) } }) } } -func TestImportDb_bytesToStringArray(t *testing.T) { +func TestBytesToStringArray(t *testing.T) { tests := []struct { name string args []byte @@ -54,8 +53,7 @@ func TestImportDb_bytesToStringArray(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &ImportDb{} - if got := d.bytesToStringArray(tt.args); !reflect.DeepEqual(got, tt.want) { + if got := bytesToStringArray(tt.args); !reflect.DeepEqual(got, tt.want) { t.Errorf("ImportDb.bytesToStringArray() = %v, want %v", got, tt.want) } }) @@ -68,11 +66,10 @@ func TestExistsCode_ExistsInVeidemann(t *testing.T) { e ExistsCode want bool }{ - {"error", ERROR, false}, - {"new", NEW, false}, - {"duplicate_new", DUPLICATE_NEW, false}, - {"exists_veidemann", EXISTS_VEIDEMANN, true}, - {"duplicate_veidemann", DUPLICATE_VEIDEMANN, true}, + {"error", Undefined, false}, + {"new", NewKey, false}, + {"exists_veidemann", NewId, true}, + {"duplicate_veidemann", Exists, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -82,60 +79,3 @@ func TestExistsCode_ExistsInVeidemann(t *testing.T) { }) } } - -func TestImportDb_CheckAndUpdateVeidemann(t *testing.T) { - s1u := "https://www.eiksenteret.no" - s2u := "https://www.eiksenteret.no" - s4u := "https://www.foo.no" - s4d := "https://www.foo.no" - s5u := "https://www.foo.no" - s5d := "http://www.foo.no" - s6u := "https://www.foo.no" - s6d := "http://www.foo.no" - f := func(client configV1.ConfigClient, data interface{}) (id string, err error) { - switch data { - case s1u: - return "s1", nil - case s2u: - return "s2", nil - case s5d: - return "s5", nil - case s6d: - return "s6", nil - default: - return "", nil - } - } - - type args struct { - uri string - data interface{} - createFunc func(client configV1.ConfigClient, data interface{}) (id string, err error) - } - tests := []struct { - name string - args args - want *ExistsResponse - wantErr bool - }{ - {"first", args{s1u, s1u, f}, &ExistsResponse{Code: NEW, NormalizedKey: s1u}, false}, - {"duplicate", args{s2u, s2u, f}, &ExistsResponse{Code: EXISTS_VEIDEMANN, NormalizedKey: s2u, KnownIds: []string{"s1"}}, false}, - {"no_id_first", args{s4u, s4d, f}, &ExistsResponse{Code: NEW, NormalizedKey: s4u}, false}, - {"no_id_duplicate", args{s5u, s5d, f}, &ExistsResponse{Code: DUPLICATE_NEW, NormalizedKey: s5u}, false}, - {"no_id_duplicate_with_path", args{s6u, s6d, f}, &ExistsResponse{Code: EXISTS_VEIDEMANN, NormalizedKey: s6u, KnownIds: []string{"s5"}}, false}, - } - - d := NewImportDb(nil, "/tmp/vmtest", configV1.Kind_seed, &NoopKeyNormalizer{}, true) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := d.CheckAndUpdateVeidemann(tt.args.uri, tt.args.data, tt.args.createFunc) - if (err != nil) != tt.wantErr { - t.Errorf("ImportDb.CheckAndUpdateVeidemann() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ImportDb.CheckAndUpdateVeidemann() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/importutil/duplicates.go b/importutil/duplicates.go new file mode 100644 index 0000000..1ef8ecf --- /dev/null +++ b/importutil/duplicates.go @@ -0,0 +1,170 @@ +// Copyright © 2023 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package importutil + +import ( + "context" + "encoding/json" + "io" + + configV1 "github.com/nlnwa/veidemann-api/go/config/v1" + "github.com/rs/zerolog/log" +) + +type DuplicateReportRecord struct { + Name string + Records []Record +} + +type Record struct { + Id string +} + +type DuplicateKindReporter struct { + *ImportDb +} + +func (d DuplicateKindReporter) Report(w io.Writer) error { + var nrOfDuplicateKeys int32 + var nrOfDuplicateValues int32 + var rec DuplicateReportRecord + + checkDuplicate := func(k []byte, v []byte) { + ids := bytesToStringArray(v) + // If there is only one id, it is not a duplicate + if len(ids) < 2 { + return + } + nrOfDuplicateKeys++ + nrOfDuplicateValues += int32(len(ids)) + + rec.Name = string(k) + rec.Records = rec.Records[:0] + for _, id := range ids { + rec.Records = append(rec.Records, Record{Id: id}) + } + + l := log.With().Str("key", rec.Name).Logger() + + b, err := json.Marshal(&rec) + if err != nil { + l.Error().Err(err).Msg("failed to marshal record to json") + return + } + if _, err := w.Write(b); err != nil { + l.Error().Err(err).Msg("failed to write record") + return + } + if _, err := w.Write([]byte{'\n'}); err != nil { + log.Error().Err(err).Msg("failed to write record") + return + } + } + + err := d.Iterate(checkDuplicate) + if err != nil { + return err + } + + log.Info().Int32("duplicate keys", nrOfDuplicateKeys).Int32("duplicate values", nrOfDuplicateValues).Msg("Duplicate report completed") + return nil +} + +type SeedDuplicateReportRecord struct { + Host string + Seeds []SeedRecord +} + +type SeedRecord struct { + SeedId string + Uri string + SeedDescription string + EntityId string + EntityName string + EntityDescription string +} + +type SeedReporter struct { + *ImportDb + Client configV1.ConfigClient +} + +func (d SeedReporter) Report(w io.Writer) error { + var nrOfDuplicateKeys int32 + var nrOfDuplicateValues int32 + var rec *SeedDuplicateReportRecord + + checkDuplicate := func(k []byte, v []byte) { + ids := bytesToStringArray(v) + // If there is only one id, it is not a duplicate + if len(ids) < 2 { + return + } + nrOfDuplicateKeys++ + nrOfDuplicateValues += int32(len(ids)) + + rec.Host = string(k) + // Avoid memory allocation by reusing the same slice + rec.Seeds = rec.Seeds[:0] + + l := log.With().Str("key", rec.Host).Logger() + + for _, id := range ids { + l = l.With().Str("id", id).Logger() + ref := &configV1.ConfigRef{Id: id, Kind: configV1.Kind_seed} + seed, err := d.Client.GetConfigObject(context.Background(), ref) + if err != nil { + l.Error().Err(err).Msg("failed to get seed from Veidemann") + continue + } + sr := SeedRecord{ + SeedId: seed.GetId(), + Uri: seed.GetMeta().GetName(), + SeedDescription: seed.GetMeta().GetDescription(), + EntityId: seed.GetSeed().GetEntityRef().GetId(), + } + rec.Seeds = append(rec.Seeds, sr) + + entity, err := d.Client.GetConfigObject(context.Background(), seed.GetSeed().GetEntityRef()) + if err != nil { + l.Warn().Err(err).Str("key", rec.Host).Str("entityId", sr.EntityId).Msg("failed to get entity from Veidemann") + continue + } + sr.EntityName = entity.GetMeta().GetName() + sr.EntityDescription = entity.GetMeta().GetDescription() + } + + b, err := json.Marshal(rec) + if err != nil { + l.Error().Err(err).Msg("failed to marshal record to json") + return + } + + if _, err := w.Write(b); err != nil { + l.Error().Err(err).Msg("failed to write record") + return + } + if _, err := w.Write([]byte{'\n'}); err != nil { + l.Error().Err(err).Msg("failed to write record") + return + } + } + err := d.Iterate(checkDuplicate) + if err != nil { + return err + } + + log.Info().Int32("duplicate keys", nrOfDuplicateKeys).Int32("duplicate values", nrOfDuplicateValues).Msg("Duplicate report completed") + return nil +} diff --git a/src/importutil/recordreader.go b/importutil/recordreader.go similarity index 54% rename from src/importutil/recordreader.go rename to importutil/recordreader.go index 11827e8..f6ae634 100644 --- a/src/importutil/recordreader.go +++ b/importutil/recordreader.go @@ -1,17 +1,28 @@ +// Copyright © 2019 National Library of Norway +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package importutil import ( - "bufio" - "encoding/json" + "errors" "fmt" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" "io" "os" "path/filepath" - "reflect" "runtime" "strings" + + "github.com/rs/zerolog/log" ) type State struct { @@ -54,15 +65,17 @@ func NewRecordReader(fileOrDir string, decoder RecordDecoder, filePattern string var f *os.File f, err = os.Open(fileOrDir) if err != nil { - log.Fatalf("Could not open file '%s': %v", fileOrDir, err) - return + return nil, fmt.Errorf("could not open file '%s': %w", fileOrDir, err) } if fi, _ := f.Stat(); fi.IsDir() { l.dir = f - l.initRecordReader() + err = l.initRecordReader() + if err != nil { + return nil, fmt.Errorf("could not open file '%s': %w", fileOrDir, err) + } } else { l.curFileName = f.Name() - log.Infof("Reading file %s", l.curFileName) + log.Info().Str("filename", l.curFileName).Msg("Reading file") l.recordDecoder.Init(f, filepath.Ext(f.Name())) } } @@ -73,15 +86,15 @@ func NewRecordReader(fileOrDir string, decoder RecordDecoder, filePattern string // recognized by filepath.Match. func hasMeta(path string) bool { magicChars := `*?[` - if runtime.GOOS != "windows" { + if strings.HasPrefix(runtime.GOOS, "windows") { magicChars = `*?[\` } return strings.ContainsAny(path, magicChars) } -func (l *recordReader) initRecordReader() (err error) { +func (l *recordReader) initRecordReader() error { if l.curFile != nil { - l.curFile.Close() + _ = l.curFile.Close() } if l.dir == nil { @@ -89,10 +102,11 @@ func (l *recordReader) initRecordReader() (err error) { } var f []os.FileInfo + var err error for { f, err = l.dir.Readdir(1) if err != nil { - return + return err } fi := f[0] @@ -101,85 +115,33 @@ func (l *recordReader) initRecordReader() (err error) { if match, err = filepath.Match(l.filePattern, fi.Name()); match && err == nil { l.curFile, err = os.Open(filepath.Join(l.dir.Name(), fi.Name())) if err != nil { - log.Fatalf("Could not open file '%s': %v", fi.Name(), err) - return + return fmt.Errorf("failed to open file \"%s\": %w", fi.Name(), err) } l.curRecNum = 0 l.curFileName = l.curFile.Name() - log.Infof("Reading file %s", l.curFileName) + log.Info().Str("filename", l.curFileName).Msg("Reading file") l.recordDecoder.Init(l.curFile, filepath.Ext(l.curFile.Name())) - return + break } } } - return + return nil } -func (l *recordReader) Next(target interface{}) (s *State, err error) { - err = l.recordDecoder.Read(target) - if err == io.EOF { +func (l *recordReader) Next(v interface{}) (*State, error) { + err := l.recordDecoder.Read(v) + if errors.Is(err, io.EOF) { if err = l.initRecordReader(); err != nil { - return + return nil, err } - return l.Next(target) + return l.Next(v) } l.curRecNum++ - s = &State{} - s.fileName = l.curFileName - s.recNum = l.curRecNum - s.err = err - return -} - -type LineAsStringDecoder struct { - r *bufio.Reader -} - -func (l *LineAsStringDecoder) Init(r io.Reader, suffix string) { - l.r = bufio.NewReader(r) -} - -func (l *LineAsStringDecoder) Read(v interface{}) (err error) { - s, err := l.r.ReadString('\n') - if err != nil { - return - } - - rv := reflect.ValueOf(v) - if rv.Kind() != reflect.Ptr || rv.IsNil() { - return fmt.Errorf("invalid target: %v", reflect.TypeOf(v)) - } - - s = strings.TrimSpace(s) - rv.Elem().Set(reflect.ValueOf(&s).Elem()) - - return -} - -type JsonYamlDecoder struct { - dec jyDecoder -} - -func (j *JsonYamlDecoder) Init(r io.Reader, suffix string) { - if suffix == ".yaml" || suffix == ".yml" { - dec := yaml.NewDecoder(r) - dec.SetStrict(true) - j.dec = dec - return - } else { - dec := json.NewDecoder(r) - dec.DisallowUnknownFields() - j.dec = dec - return - } -} - -func (j *JsonYamlDecoder) Read(v interface{}) (err error) { - return j.dec.Decode(v) -} - -type jyDecoder interface { - Decode(v interface{}) (err error) + return &State{ + fileName: l.curFileName, + recNum: l.curRecNum, + err: err, + }, nil } diff --git a/importutil/worker.go b/importutil/worker.go new file mode 100644 index 0000000..6e3945b --- /dev/null +++ b/importutil/worker.go @@ -0,0 +1,106 @@ +/* + * Copyright 2021 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package importutil + +import ( + "sync" +) + +// Payload is an interface for the payload of a job in a work queue +type Payload interface { + any +} + +// Job represents a piece of work in the work queue +type Job[P Payload] struct { + *State + Val P +} + +// Executor is a work queue that executes jobs in concurrent workers. +type Executor[P Payload] struct { + Queue chan Job[P] + stats chan error + done chan struct{} + wg sync.WaitGroup + + // Keep track of the number of jobs completed and the number of jobs that failed. + count int + success int + failed int +} + +// NewExecutor creates a work queue with nrOfWorkers workers. +// +// do is the function that will be called for each job. +// onError is the function that will be called for each job that fails. +// +// To close the work queue, call Wait() after all jobs have been queued. +// Writing to the Queue channel after Wait() has been called will panic. +func NewExecutor[P Payload](nrOfWorkers int, do func(P) error, onError func(Job[P])) *Executor[P] { + e := &Executor[P]{ + Queue: make(chan Job[P], nrOfWorkers), + done: make(chan struct{}), + stats: make(chan error, nrOfWorkers), + } + + // start error handler + go func() { + defer close(e.done) + for err := range e.stats { + e.count++ + if err == nil { + e.success++ + } else { + e.failed++ + } + } + }() + + // start workers + for i := 0; i < nrOfWorkers; i++ { + e.wg.Add(1) + go func() { + defer e.wg.Done() + for job := range e.Queue { + err := do(job.Val) + e.stats <- err + if err != nil { + job.err = err + onError(job) + } + } + }() + } + + return e +} + +// Wait waits for all jobs to complete. +// It returns the number of jobs completed, the number of jobs that succeeded and the number of jobs that failed. +func (e *Executor[P]) Wait() (int, int, int) { + // stop workers + close(e.Queue) + // wait for workers to finish + e.wg.Wait() + // stop counting + close(e.stats) + // wait for counting to complete + <-e.done + // return stats + return e.count, e.success, e.failed +} diff --git a/install.sh b/install.sh index 091d459..5c4cf66 100755 --- a/install.sh +++ b/install.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -VERSION=`curl -s -I https://github.com/nlnwa/veidemannctl/releases/latest | grep location | sed -e 's/.*tag\/\([0-9\.]\+\).*$/\1/'` -echo Installing veidemannctl v${VERSION} +VERSION=$(curl -s -I https://github.com/nlnwa/veidemannctl/releases/latest | grep location | sed -e 's/.*tag\/\([0-9\.]\+\).*$/\1/') +echo "Installing veidemannctl v${VERSION}" -curl -L#o veidemannctl https://github.com/nlnwa/veidemannctl/releases/download/${VERSION}/veidemannctl_${VERSION}_linux_amd64 -sudo mv veidemannctl /usr/local/bin/veidemannctl -sudo chmod +x /usr/local/bin/veidemannctl +curl -Lo veidemannctl https://github.com/nlnwa/veidemannctl/releases/download/${VERSION}/veidemannctl_${VERSION}_linux_amd64 +sudo install veidemannctl /usr/local/bin/veidemannctl +rm veidemannctl -# Install command completion -sudo sh -c "/usr/local/bin/veidemannctl completion > /etc/bash_completion.d/veidemannctl" +# Install command completion for bash +sudo sh -c "/usr/local/bin/veidemannctl completion bash > /etc/bash_completion.d/veidemannctl" diff --git a/logger/initlog.go b/logger/initlog.go new file mode 100644 index 0000000..74991b7 --- /dev/null +++ b/logger/initlog.go @@ -0,0 +1,70 @@ +/* + * Copyright 2021 National Library of Norway. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package logger + +import ( + stdlog "log" + "os" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +// Log formats +const ( + formatJson = "json" + formatPretty = "pretty" +) + +// InitLogger initializes the logger with the given level and format. +// If logCaller is true, the caller is logged. +func InitLogger(level string, format string, logCaller bool) { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + + switch strings.ToLower(level) { + case "panic": + log.Logger = log.Level(zerolog.PanicLevel) + case "fatal": + log.Logger = log.Level(zerolog.FatalLevel) + case "error": + log.Logger = log.Level(zerolog.ErrorLevel) + case "warn": + log.Logger = log.Level(zerolog.WarnLevel) + case "info": + log.Logger = log.Level(zerolog.InfoLevel) + case "debug": + log.Logger = log.Level(zerolog.DebugLevel) + case "trace": + log.Logger = log.Level(zerolog.TraceLevel) + default: + log.Logger = log.Level(zerolog.Disabled) + } + + if strings.ToLower(format) != formatJson { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) + } + + if logCaller { + log.Logger = log.With().Caller().Logger() + } + + // Set stdlog to use zerolog + stdlog.SetFlags(0) + stdlog.SetOutput(log.Logger) +} diff --git a/main.go b/main.go index 3151ab2..2048aa3 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,5 @@ // Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); +// Licensed under the Apache License, GitVersion 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // @@ -14,23 +14,20 @@ package main import ( - "github.com/nlnwa/veidemannctl/src/cmd" - v "github.com/nlnwa/veidemannctl/src/version" - log "github.com/sirupsen/logrus" - stdlogger "log" + "time" + + "github.com/nlnwa/veidemannctl/cmd" + v "github.com/nlnwa/veidemannctl/version" ) -var version = "master" +var ( + version = "" + commit = "" + date = time.Now().Format(time.RFC3339) +) -//go:generate go run -tags=dev bindata/assets_generate.go -//go:generate go run gendoc/md_docs.go +//go:generate go run ./docs func main() { - v.Version.SetGitVersion(version) + v.Set(version, commit, date) cmd.Execute() } - -func init() { - stdlogger.SetOutput(log.StandardLogger().Writer()) - - log.SetLevel(log.WarnLevel) -} diff --git a/res/browserConfig_table.template b/res/browserConfig_table.template deleted file mode 100644 index 7541fc2..0000000 --- a/res/browserConfig_table.template +++ /dev/null @@ -1,5 +0,0 @@ -{{define "HEADER" -}} - {{printf `%-36.36s %-15s %18s %11s %12s %12s` "Id" "Name" "UserAgent" "WindowSize" "PageTimeout" "MaxInactivity"}} -{{end -}} - -{{printf `%36s %-15s %18s %4d x%5d %12v %12v` .Id .Meta.Name .Spec.BrowserConfig.UserAgent .Spec.BrowserConfig.WindowWidth .Spec.BrowserConfig.WindowHeight .Spec.BrowserConfig.PageLoadTimeoutMs .Spec.BrowserConfig.MaxInactivityTimeMs}} diff --git a/res/completion.sh b/res/completion.sh deleted file mode 100644 index 1460501..0000000 --- a/res/completion.sh +++ /dev/null @@ -1,67 +0,0 @@ -__veidemannctl_parse_get() { - declare -rA tables=( ["entity"]="config_crawl_entities" ["browser"]="config_browser_configs" \ - ["crawlconfig"]="config_crawl_configs" ["group"]="config_crawl_host_group_configs" ["job"]="config_crawl_jobs" \ - ["politeness"]="config_politeness_configs" ["role"]="dog" ["schedule"]="config_crawl_schedule_configs" \ - ["script"]="config_browser_scripts" ["seed"]="config_seeds" ) - local table=${tables[$1]} query template veidemannctl_out - - if [ -z "$table" ]; then - return 0 - fi - - query="r.table('${table}')" - if [ -n "$cur" ]; then - query="${query}.between('${cur}', '${cur}z', {index:'id'})" - fi - query="${query}.orderBy({index:'id'}).pluck('id')" - template="{{println .id}}" - if mapfile -t veidemannctl_out < <( veidemannctl report query "${query}" -o template -t"${template}" -s20 2>/dev/null ); then - mapfile -t COMPREPLY < <( compgen -W "$( printf '%q ' "${veidemannctl_out[@]}" )" -- "$cur" | awk '/ / { print "\""$0"\"" } /^[^ ]+$/ { print $0 }' ) - fi -} - -__veidemannctl_get_resource() { - if [[ ${#nouns[@]} -eq 0 ]]; then - return 1 - fi - __veidemannctl_parse_get ${nouns[${#nouns[@]} -1]} - if [[ $? -eq 0 ]]; then - return 0 - fi -} - -__veidemannctl_query_resource() { - local veidemannctl_out - if mapfile -t veidemannctl_out < <( veidemannctl report query --comp 2>/dev/null ); then - mapfile -t COMPREPLY < <( compgen -W "$( printf '%q ' "${veidemannctl_out[@]}" )" -- "$cur" | awk '/ / { print "\""$0"\"" } /^[^ ]+$/ { print $0 }' ) - fi -} - -__veidemannctl_custom_func() { - case ${last_command} in - veidemannctl_get) - __veidemannctl_get_resource - return - ;; - veidemannctl_report_query) - __veidemannctl_query_resource - return - ;; - *) - ;; - esac -} - -__veidemannctl_get_name() { - if [[ ${#nouns[@]} -eq 0 ]]; then - return 1 - fi - local noun - noun=${nouns[${#nouns[@]} -1]} - local template - template="{{println .meta.name}}" - local veidemannctl_out - if mapfile -t veidemannctl_out < <( veidemannctl get "$noun" -n "^${cur}" -s20 -o template -t "${template}" 2>/dev/null ); then - mapfile -t COMPREPLY < <( printf '%q\n' "${veidemannctl_out[@]}" | awk -v IGNORECASE=1 -v p="$cur" '{p==substr($0,0,length(p))} { print $0 }' ) - fi -} diff --git a/res/crawlEntity_wide.template b/res/crawlEntity_wide.template deleted file mode 100644 index dd1cd07..0000000 --- a/res/crawlEntity_wide.template +++ /dev/null @@ -1,7 +0,0 @@ -{{- /*gotype: github.com/nlnwa/veidemann-api/go/config/v1.ConfigObject*/ -}} - -{{define "HEADER" -}} - {{printf `%-36.36s %-40.40s %-60s` "Id" "Name" "Labels"}} -{{end -}} - -{{printLabels .Meta.Label | printf `%36s %-40.40s %-60.60s` .Id .Meta.Name}} diff --git a/res/roleMapping_wide.template b/res/roleMapping_wide.template deleted file mode 100644 index abddd15..0000000 --- a/res/roleMapping_wide.template +++ /dev/null @@ -1,5 +0,0 @@ -{{define "HEADER" -}} - {{printf `%-36.36s %-20.20s %-18.18s %-18.18s` "Id" "Name" "Labels" "Role"}} -{{end -}} - -{{printf `%36s %-20.20s %18.18v %18.18v` .Id .Meta.Name .Meta.Label .Spec.RoleMapping.Role}} diff --git a/src/cmd/abort.go b/src/cmd/abort.go deleted file mode 100644 index d82c041..0000000 --- a/src/cmd/abort.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" - - "context" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/spf13/cobra" - "log" -) - -// abortCmd represents the abort command -var abortCmd = &cobra.Command{ - Use: "abort", - Short: "Abort one or more crawl executions", - Long: `Abort one or more crawl executions.`, - Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - client, conn := connection.NewControllerClient() - defer conn.Close() - - for _, arg := range args { - request := controllerV1.ExecutionId{Id: arg} - _, err := client.AbortCrawlExecution(context.Background(), &request) - if err != nil { - log.Fatalf("could not abort execution '%v': %v", arg, err) - } - } - } else { - cmd.Usage() - } - }, -} - -func init() { - RootCmd.AddCommand(abortCmd) -} diff --git a/src/cmd/abortJobExecution.go b/src/cmd/abortJobExecution.go deleted file mode 100644 index 8195861..0000000 --- a/src/cmd/abortJobExecution.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" - - "context" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/spf13/cobra" - "log" -) - -// abortCmd represents the abort command -var abortJobExecutionCmd = &cobra.Command{ - Use: "abortjobexecution", - Short: "Abort one or more job executions", - Long: `Abort one or more job executions.`, - Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - client, conn := connection.NewControllerClient() - defer conn.Close() - - for _, arg := range args { - request := controllerV1.ExecutionId{Id: arg} - _, err := client.AbortJobExecution(context.Background(), &request) - if err != nil { - log.Fatalf("could not abort job execution '%v': %v", arg, err) - } - } - } else { - cmd.Usage() - } - }, -} - -func init() { - RootCmd.AddCommand(abortJobExecutionCmd) -} diff --git a/src/cmd/activeroles.go b/src/cmd/activeroles.go deleted file mode 100644 index 3b7b7e2..0000000 --- a/src/cmd/activeroles.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/nlnwa/veidemannctl/src/connection" - "golang.org/x/net/context" -) - -// activerolesCmd represents the activeroles command -var activerolesCmd = &cobra.Command{ - Use: "activeroles", - Short: "Get the active roles for the currently logged in user", - Long: `Get the active roles for the currently logged in user.`, - - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 0 { - client, conn := connection.NewControllerClient() - defer conn.Close() - - r, err := client.GetRolesForActiveUser(context.Background(), &empty.Empty{}) - if err != nil { - log.Fatalf("could not get active role: %v", err) - } - - for _, role := range r.Role { - fmt.Println(role) - } - } else { - fmt.Println("activeroles takes no arguments.") - fmt.Println("See 'veidemannctl activeroles -h' for help") - } - }, -} - -func init() { - RootCmd.AddCommand(activerolesCmd) -} diff --git a/src/cmd/completion.go b/src/cmd/completion.go deleted file mode 100644 index 3cba1cc..0000000 --- a/src/cmd/completion.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "github.com/spf13/cobra" - "os" -) - -// completionCmd represents the completion command -var completionCmd = &cobra.Command{ - Use: "completion", - Short: "Output bash completion code", - Long: `Output bash completion code. The shell code must be evalutated to provide -interactive completion of veidemannctl commands. This can be done by sourcing it from the .bash _profile. - -Example: - ## Load the kubectl completion code for bash into the current shell - source <(veidemannctl completion) -`, - Run: func(cmd *cobra.Command, args []string) { - RootCmd.GenBashCompletion(os.Stdout) - }, -} - -func init() { - RootCmd.AddCommand(completionCmd) -} diff --git a/src/cmd/config/address.go b/src/cmd/config/address.go deleted file mode 100644 index 58b6873..0000000 --- a/src/cmd/config/address.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "github.com/nlnwa/veidemannctl/src/configutil" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -// setAddrCmd represents the report command -var setAddrCmd = &cobra.Command{ - Use: "set-address HOST:PORT", - Short: "Sets the address to Veidemann controller service", - Long: `Sets the address to Veidemann controller service - -Examples: - # Sets the address to Veidemann controller service to localhost:50051 - veidemannctl config set-address localhost:50051 -`, - Aliases: []string{"set-addr", "set-controller"}, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 1 { - viper.Set("controllerAddress", args[0]) - configutil.WriteConfig() - } else { - cmd.Usage() - } - }, -} - -// useContextCmd represents the report command -var getAddrCmd = &cobra.Command{ - Use: "get-address", - Short: "Displays Veidemann controller service address", - Long: `Displays Veidemann controller service address - -Examples: - # Display Veidemann controller service address - veidemannctl config get-address -`, - Aliases: []string{"get-addr", "address", "controller"}, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Controller address: %s\n", viper.Get("controllerAddress")) - }, -} - -func init() { - ConfigCmd.AddCommand(setAddrCmd) - ConfigCmd.AddCommand(getAddrCmd) -} diff --git a/src/cmd/config/apikey.go b/src/cmd/config/apikey.go deleted file mode 100644 index 535c8de..0000000 --- a/src/cmd/config/apikey.go +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2019 National Library of Norway. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package config - -import ( - "fmt" - "github.com/nlnwa/veidemannctl/src/configutil" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -// setApiKeyCmd represents the set-apikey command -var setApiKeyCmd = &cobra.Command{ - Use: "set-apikey API_KEY", - Short: "Sets the api-key to use for authentication", - Long: `Sets the api-key to use for authentication - -Examples: - # Set the api-key to use for authentication to Veidemann controller service to myKey - veidemannctl config set-apikey myKey -`, - Aliases: []string{"set-apikey", "set-api-key"}, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 1 { - viper.Set("apiKey", args[0]) - configutil.WriteConfig() - } else { - cmd.Usage() - } - }, -} - -// getApiKeyCmd represents the get-apikey command -var getApiKeyCmd = &cobra.Command{ - Use: "get-apikey", - Short: "Displays Veidemann authentication api-key", - Long: `Displays Veidemann authentication api-key - -Examples: - # Display Veidemann authentication api-key - veidemannctl config get-apikey -`, - Aliases: []string{"get-apikey", "apikey", "get-api-key", "api-key"}, - Run: func(cmd *cobra.Command, args []string) { - apiKey := viper.Get("apiKey") - if apiKey == nil { - fmt.Printf("No api-key configured\n") - } else { - fmt.Printf("api-key: %s\n", apiKey) - } - }, -} - -func init() { - ConfigCmd.AddCommand(setApiKeyCmd) - ConfigCmd.AddCommand(getApiKeyCmd) -} diff --git a/src/cmd/config/context.go b/src/cmd/config/context.go deleted file mode 100644 index a664687..0000000 --- a/src/cmd/config/context.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "github.com/nlnwa/veidemannctl/src/configutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "path/filepath" -) - -// useContextCmd represents the use-context command -var useContextCmd = &cobra.Command{ - Use: "use-context CONTEXT_NAME", - Short: "Sets the current-context", - Long: `Sets the current-context - -Examples: - # Use the context for the prod cluster - veidemannctl config use-context prod -`, - Aliases: []string{"use"}, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 1 { - ok, err := configutil.ContextExists(args[0]) - if err != nil { - log.Fatalf("Failed switching context to %v. Cause: %v", args[0], err) - } - if !ok { - fmt.Printf("Non existing context '%v'\n", args[0]) - return - } - if err := configutil.SetCurrentContext(args[0]); err != nil { - log.Fatalf("Failed switching context to %v. Cause: %v", args[0], err) - } - fmt.Printf("Switched to context: '%v'\n", args[0]) - } else { - fmt.Println("Missing context name") - cmd.Usage() - } - }, -} - -// createContextCmd represents the create-context command -var createContextCmd = &cobra.Command{ - Use: "create-context CONTEXT_NAME", - Short: "Creates a new context", - Long: `Creates a new context - -Examples: - # Create context for the prod cluster - veidemannctl config create-context prod -`, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 1 { - ok, err := configutil.ContextExists(args[0]) - if err != nil { - log.Fatalf("Failed creating context %v. Cause: %v", args[0], err) - } - if ok { - fmt.Printf("Context '%v' already exists\n", args[0]) - return - } - - contextDir := configutil.GetConfigDir("contexts") - viper.SetConfigFile(filepath.Join(contextDir, args[0]+".yaml")) - configutil.WriteConfig() - - if err := configutil.SetCurrentContext(args[0]); err != nil { - log.Fatalf("Failed creating context %v. Cause: %v", args[0], err) - } - fmt.Printf("Created context: '%v'\n", args[0]) - } else { - fmt.Println("Missing context name") - cmd.Usage() - } - }, -} - -// currentContextCmd represents the current-context command -var currentContextCmd = &cobra.Command{ - Use: "current-context", - Short: "Displays the current-context", - Long: `Displays the current-context - -Examples: - # Display the current context - veidemannctl config current-context -`, - Aliases: []string{"context"}, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Using context:", configutil.GlobalFlags.Context) - }, -} - -// listContextsCmd represents the list-contexts command -var listContextsCmd = &cobra.Command{ - Use: "list-contexts", - Short: "Displays the known contexts", - Long: `Displays the known contexts. - -Examples: - # Get a list of known contexts - veidemannctl config list-contexts -`, - Aliases: []string{"contexts"}, - Run: func(cmd *cobra.Command, args []string) { - cs, err := configutil.ListContexts() - if err != nil { - log.Fatalf("Failed listing contexts to %v. Cause: %v", args[0], err) - } - fmt.Println("Known contexts:") - for _, c := range cs { - fmt.Println(c) - } - }, -} - -func init() { - ConfigCmd.AddCommand(useContextCmd) - ConfigCmd.AddCommand(createContextCmd) - ConfigCmd.AddCommand(currentContextCmd) - ConfigCmd.AddCommand(listContextsCmd) -} diff --git a/src/cmd/config/importCa.go b/src/cmd/config/importCa.go deleted file mode 100644 index aae4394..0000000 --- a/src/cmd/config/importCa.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "github.com/nlnwa/veidemannctl/src/configutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "io/ioutil" -) - -// importCaCmd represents the import-ca command -var importCaCmd = &cobra.Command{ - Use: "import-ca CA_CERT_FILE_NAME", - Short: "Import file with trusted certificate chains for the idp and controller.", - Long: `Import file with trusted certificate chains for the idp and controller. These are in addition to the default certs configured for the OS.`, - Aliases: []string{"ca"}, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 1 { - rootCAs := args[0] - if rootCAs != "" { - rootCABytes, err := ioutil.ReadFile(rootCAs) - if err != nil { - log.Fatalf("failed to read root-ca: %v", err) - } - viper.Set("rootCAs", string(rootCABytes)) - configutil.WriteConfig() - } - fmt.Printf("Successfully imported root CA cert from file %s\n", rootCAs) - } else { - cmd.Usage() - } - }, -} - -func init() { - ConfigCmd.AddCommand(importCaCmd) -} diff --git a/src/cmd/config/servername.go b/src/cmd/config/servername.go deleted file mode 100644 index f31cd7b..0000000 --- a/src/cmd/config/servername.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "github.com/nlnwa/veidemannctl/src/configutil" - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -// setServerNameOverrideCmd represents the set-server-name-override command -var setServerNameOverrideCmd = &cobra.Command{ - Use: "set-server-name-override HOST", - Short: "Sets the server name override", - Long: `Sets the server name override. - -Use this when there is a mismatch between exposed server name for the cluster and the certificate. The use is a security -risk and is only recommended for testing. - -Examples: - # Sets the server name override to test.local - veidemannctl config set-server-name-override test.local -`, - Aliases: []string{"set-servername"}, - Run: func(cmd *cobra.Command, args []string) { - if len(args) == 1 { - viper.Set("serverNameOverride", args[0]) - configutil.WriteConfig() - } else { - cmd.Usage() - } - }, -} - -// getServerNameOverrideCmd represents the get-server-name-override command -var getServerNameOverrideCmd = &cobra.Command{ - Use: "get-server-name-override HOST", - Short: "Displays the server name override", - Long: `Displays the server name override - -Examples: - # Display the server name override - veidemannctl config get-server-name-override -`, - Aliases: []string{"get-servername"}, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Server name override: %s\n", viper.Get("serverNameOverride")) - }, -} - -func init() { - ConfigCmd.AddCommand(setServerNameOverrideCmd) - ConfigCmd.AddCommand(getServerNameOverrideCmd) -} diff --git a/src/cmd/create.go b/src/cmd/create.go deleted file mode 100644 index 48772ab..0000000 --- a/src/cmd/create.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "fmt" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - "github.com/spf13/cobra" - "log" - "os" -) - -var filename string - -// createCmd represents the create command -var createCmd = &cobra.Command{ - Use: "create", - Short: "Create or update a config object", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - - if filename == "" { - cmd.Usage() - os.Exit(1) - } else if filename == "-" { - filename = "" - } - result, err := format.Unmarshal(filename) - if err != nil { - log.Fatalf("Parse error: %v", err) - os.Exit(1) - } - - client, conn := connection.NewConfigClient() - defer conn.Close() - - for _, co := range result { - if co.ApiVersion == "" { - handleError(co, fmt.Errorf("Missing apiVersion")) - } - if co.Kind == configV1.Kind_undefined { - handleError(co, fmt.Errorf("Missing kind")) - } - r, err := client.SaveConfigObject(context.Background(), co) - handleError(co, err) - fmt.Printf("Saved %v: %v %v\n", co.Kind, r.Meta.Name, r.Id) - } - }, -} - -func handleError(msg *configV1.ConfigObject, err error) { - if err != nil { - fmt.Printf("Could not save %v: %v. Cause: %v\n", msg.Kind, msg.Meta.Name, err) - os.Exit(2) - } -} - -func init() { - RootCmd.AddCommand(createCmd) - - createCmd.PersistentFlags().StringVarP(&filename, "filename", "f", "", "Filename or directory to read from. "+ - "If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin.") -} diff --git a/src/cmd/delete.go b/src/cmd/delete.go deleted file mode 100644 index 239b419..0000000 --- a/src/cmd/delete.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/apiutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "golang.org/x/net/context" - "io" -) - -var deleteFlags struct { - label string - filter string - dryRun bool -} - -// deleteCmd represents the delete command -var deleteCmd = &cobra.Command{ - Use: "delete [id] ...", - Short: "Delete a config object", - Long: `Delete a config object. - -` + - printValidObjectTypes() + - `Examples: - #Delete a seed. - veidemannctl delete seed 407a9600-4f25-4f17-8cff-ee1b8ee950f6`, - - Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - configClient, conn := connection.NewConfigClient() - defer conn.Close() - - k := format.GetKind(args[0]) - - var ids []string - - if len(args) > 1 { - ids = args[1:] - } - - if k == configV1.Kind_undefined { - fmt.Printf("Unknown object type\n") - cmd.Usage() - return - } - - if len(ids) == 0 && deleteFlags.filter == "" && deleteFlags.label == "" { - fmt.Printf("At least one of the -f or -l flags must be set or one or more id's\n") - cmd.Usage() - return - } - - selector, err := apiutil.CreateListRequest(k, ids, "", deleteFlags.label, deleteFlags.filter, 0, 0) - if err != nil { - log.Fatalf("Error creating request: %v", err) - } - if err != nil { - log.Fatalf("Error creating request: %v", err) - } - - r, err := configClient.ListConfigObjects(context.Background(), selector) - if err != nil { - log.Fatalf("Error from controller: %v", err) - } - - count, err := configClient.CountConfigObjects(context.Background(), selector) - if err != nil { - log.Fatalf("Error from controller: %v", err) - } - - if deleteFlags.dryRun { - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - log.Fatalf("Error getting object: %v", err) - } - log.Debugf("Outputing record of kind '%s' with name '%s'", msg.Kind, msg.Meta.Name) - fmt.Printf("%s\n", msg.Meta.Name) - } - fmt.Printf("Requested count: %v\nTo actually delete, add: --dry-run=false\n", count.Count) - } else { - var deleted int - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - log.Fatalf("Error getting object: %v", err) - } - log.Debugf("Deleting record of kind '%s' with name '%s'", msg.Kind, msg.Meta.Name) - - request := &configV1.ConfigObject{ - ApiVersion: "v1", - Kind: k, - Id: msg.Id, - } - - r, err := configClient.DeleteConfigObject(context.Background(), request) - if err != nil { - log.Fatalf("could not delete '%v': %v\n", msg.Id, err) - } - if r.Deleted { - deleted++ - fmt.Print(".") - } else { - fmt.Printf("\nCould not delete %v: %v\n", k, msg.Id) - } - } - fmt.Printf("\nRequested count: %v\n", count.Count) - fmt.Printf("Deleted count: %v\n", deleted) - } - } else { - fmt.Println("You must specify the object type to delete. ") - for _, k := range configV1.Kind_name { - fmt.Println(k) - } - fmt.Println("See 'veidemannctl get -h' for help") - } - }, - ValidArgs: format.GetObjectNames(), -} - -func init() { - RootCmd.AddCommand(deleteCmd) - - deleteCmd.PersistentFlags().StringVarP(&deleteFlags.label, "label", "l", "", "Delete objects by label (: | )") - deleteCmd.PersistentFlags().StringVarP(&deleteFlags.filter, "filter", "q", "", "Delete objects by field (i.e. meta.description=foo)") - deleteCmd.PersistentFlags().BoolVarP(&deleteFlags.dryRun, "dry-run", "", true, "Set to false to execute delete") -} diff --git a/src/cmd/get.go b/src/cmd/get.go deleted file mode 100644 index 6839177..0000000 --- a/src/cmd/get.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "github.com/nlnwa/veidemannctl/src/apiutil" - log "github.com/sirupsen/logrus" - "io" - "os" - - "github.com/spf13/cobra" - - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - "golang.org/x/net/context" -) - -var flags struct { - label string - name string - filter string - file string - format string - goTemplate string - pageSize int32 - page int32 -} - -// getCmd represents the get command -var getCmd = &cobra.Command{ - Use: "get ", - Short: "Get the value(s) for an object type", - Long: `Display one or many objects. - -` + - printValidObjectTypes() + - `Examples: - #List all seeds. - veidemannctl get seed - - #List all seeds in yaml output format. - veidemannctl get seed -f yaml`, - - Run: func(cmd *cobra.Command, args []string) { - if len(args) > 0 { - configClient, conn := connection.NewConfigClient() - defer conn.Close() - - k := format.GetKind(args[0]) - - var ids []string - - if len(args) > 1 { - ids = args[1:] - } - - if k == configV1.Kind_undefined { - fmt.Printf("Unknown object type\n") - cmd.Usage() - return - } - - request, err := apiutil.CreateListRequest(k, ids, flags.name, flags.label, flags.filter, flags.pageSize, flags.page) - if err != nil { - log.Fatalf("Error creating request: %v", err) - } - - r, err := configClient.ListConfigObjects(context.Background(), request) - if err != nil { - log.Fatalf("Error from controller: %v", err) - } - - out, err := format.ResolveWriter(flags.file) - if err != nil { - log.Fatalf("Could not resolve output '%v': %v", flags.file, err) - } - s, err := format.NewFormatter(args[0], out, flags.format, flags.goTemplate) - if err != nil { - log.Fatal(err) - } - defer s.Close() - - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - log.Fatalf("Error getting object: %v", err) - } - log.Debugf("Outputting record of kind '%s' with name '%s'", msg.Kind, msg.Meta.Name) - if s.WriteRecord(msg) != nil { - os.Exit(1) - } - } - } else { - fmt.Print("You must specify the object type to get. ") - fmt.Println(printValidObjectTypes()) - fmt.Println("See 'veidemannctl get -h' for help") - } - }, - ValidArgs: format.GetObjectNames(), -} - -func printValidObjectTypes() string { - var names string - for _, v := range format.GetObjectNames() { - names += fmt.Sprintf(" * %s\n", v) - } - return fmt.Sprintf("Valid object types include:\n%s\n", names) -} - -func init() { - RootCmd.AddCommand(getCmd) - - // Here you will define your flags and configuration settings. - - getCmd.PersistentFlags().StringVarP(&flags.label, "label", "l", "", "List objects by label (: | )") - - getCmd.PersistentFlags().StringVarP(&flags.name, "name", "n", "", "List objects by name (accepts regular expressions)") - annotation := make(map[string][]string) - annotation[cobra.BashCompCustom] = []string{"__veidemannctl_get_name"} - getCmd.PersistentFlags().Lookup("name").Annotations = annotation - - getCmd.PersistentFlags().StringVarP(&flags.filter, "filter", "q", "", "Filter objects by field (i.e. meta.description=foo") - getCmd.PersistentFlags().StringVarP(&flags.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") - getCmd.PersistentFlags().StringVarP(&flags.goTemplate, "template", "t", "", "A Go template used to format the output") - getCmd.PersistentFlags().StringVarP(&flags.file, "filename", "f", "", "File name to write to") - getCmd.PersistentFlags().Int32VarP(&flags.pageSize, "pagesize", "s", 10, "Number of objects to get") - getCmd.PersistentFlags().Int32VarP(&flags.page, "page", "p", 0, "The page number") -} diff --git a/src/cmd/get_test.go b/src/cmd/get_test.go deleted file mode 100644 index 86557d1..0000000 --- a/src/cmd/get_test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "strings" - "testing" -) - -func Test_printValidObjectTypes(t *testing.T) { - prefix := "Valid object types" - - if got := printValidObjectTypes(); !strings.HasPrefix(got, prefix) { - t.Errorf("printValidObjectTypes() = '%v', but should start with '%v'", got, prefix) - } -} diff --git a/src/cmd/importcmd/convertoos.go b/src/cmd/importcmd/convertoos.go deleted file mode 100644 index ce11f7a..0000000 --- a/src/cmd/importcmd/convertoos.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package importcmd - -import ( - "crypto/x509" - "encoding/json" - "errors" - "fmt" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/importutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "golang.org/x/net/html" - "io" - "net" - "net/http" - "net/url" - "os" - "strings" -) - -var convertFlags struct { - filename string - errorFile string - outFile string - toplevel bool - ignoreScheme bool - checkUri bool - checkUriTimeout int64 - dbDir string - resetDb bool -} - -// convertOosCmd represents the convertoos command -var convertOosCmd = &cobra.Command{ - Use: "convertoos", - Short: "Convert Out of Scope file(s) to seed import file", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - c := &converter{} - var err error - - // Check inputflag - if convertFlags.filename == "" { - fmt.Printf("Import file is required. See --filename\n") - _ = cmd.Usage() - os.Exit(1) - } - - // Check ouput flag - if convertFlags.outFile == "" { - fmt.Printf("Output file is required. See --outfile\n") - _ = cmd.Usage() - os.Exit(1) - } - - // Create output file - var out io.Writer - out, err = os.Create(convertFlags.outFile) - defer out.(io.Closer).Close() - if err != nil { - log.Fatalf("Unable to open out file: %v, cause: %v", convertFlags.outFile, err) - os.Exit(1) - } - - // Create error writer (file or stdout) - var errFile io.Writer - if convertFlags.errorFile == "" { - errFile = os.Stdout - } else { - errFile, err = os.Create(convertFlags.errorFile) - defer errFile.(io.Closer).Close() - if err != nil { - log.Fatalf("Unable to open error file: %v, cause: %v", convertFlags.errorFile, err) - os.Exit(1) - } - } - - // Create Veidemann config client - client, conn := connection.NewConfigClient() - defer conn.Close() - - // Create http client - c.httpClient = importutil.NewHttpClient(convertFlags.checkUriTimeout) - - // Create state Database based on seeds in Veidemann - keyNormalizer := &UriKeyNormalizer{ignoreScheme: convertFlags.ignoreScheme, toplevel: convertFlags.toplevel} - impf := importutil.NewImportDb(client, convertFlags.dbDir, configV1.Kind_seed, keyNormalizer, convertFlags.resetDb) - impf.ImportExisting() - defer impf.Close() - - // Create Record reader for file input - rr, err := importutil.NewRecordReader(convertFlags.filename, &importutil.LineAsStringDecoder{}, "*.txt") - if err != nil { - log.Fatalf("Parse error: %v", err) - os.Exit(1) - } - - // Processor for converting oos records into import records - proc := func(value interface{}, state *importutil.State) error { - v := value.(string) - seed := &seedDesc{ - Uri: v, - SeedLabel: []*configV1.Label{{Key: "source", Value: "oosh"}}, - EntityLabel: []*configV1.Label{{Key: "source", Value: "oosh"}}, - } - - err := c.checkUri(seed) - if err != nil { - return err - } - - if exists, err := impf.Check(seed.Uri); err != nil { - return err - } else if exists.Code > importutil.NEW { - return fmt.Errorf("%v already exists", v) - } - - if seed.EntityName == "" { - seed.EntityName = seed.Uri - } - - json, err := json.Marshal(seed) - if err != nil { - return err - } - if _, err = fmt.Fprintf(out, "%s\n", json); err != nil { - return err - } - - return nil - } - - // Error handler - errorHandler := func(state *importutil.StateVal) { - _, _ = fmt.Fprintf(errFile, "ERR: %v %v %v\n", state.GetFilename(), state.GetRecordNum(), state.GetError()) - } - - // Create multithreaded executor - conv := importutil.NewExecutor(512, proc, errorHandler) - - // Process - var ts string - for { - state, err := rr.Next(&ts) - if err == io.EOF { - break - } - if err != nil { - _, _ = fmt.Fprintf(errFile, "Error decoding record: %v, cause: %v", state, err) - os.Exit(1) - } - conv.Do(state, ts) - } - - conv.Finish() - _, _ = fmt.Fprintf(os.Stderr, "\nRecords read: %v, imported: %v, Failed: %v\n", conv.Count, conv.Success, conv.Failed) - }, -} - -func init() { - ImportCmd.AddCommand(convertOosCmd) - - convertOosCmd.PersistentFlags().StringVarP(&convertFlags.filename, "filename", "f", "", "Filename or directory to read from. "+ - "If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin. (required)") - convertOosCmd.PersistentFlags().StringVarP(&convertFlags.errorFile, "errorfile", "e", "", "File to write errors to.") - convertOosCmd.PersistentFlags().StringVarP(&convertFlags.outFile, "outfile", "o", "", "File to write result to. (required)") - convertOosCmd.PersistentFlags().BoolVarP(&convertFlags.toplevel, "toplevel", "", true, "Convert URI to toplevel by removing path.") - convertOosCmd.PersistentFlags().BoolVarP(&convertFlags.ignoreScheme, "ignore-scheme", "", false, "Ignore the URL's scheme when checking if this URL is already imported.") - convertOosCmd.PersistentFlags().BoolVarP(&convertFlags.checkUri, "checkuri", "", true, "Check the uri for liveness and follow 301") - convertOosCmd.PersistentFlags().Int64VarP(&convertFlags.checkUriTimeout, "checkuri-timeout", "", 2000, "Timeout in ms when checking uri for liveness.") - convertOosCmd.PersistentFlags().StringVarP(&convertFlags.dbDir, "db-directory", "b", "/tmp/veidemannctl", "Directory for storing state db") - convertOosCmd.PersistentFlags().BoolVarP(&convertFlags.resetDb, "reset-db", "r", false, "Clean state db") -} - -type converter struct { - httpClient *http.Client -} - -func (c *converter) checkUri(s *seedDesc) (err error) { - uri, err := url.Parse(s.Uri) - if err != nil { - return fmt.Errorf("unparseable URL '%v', cause: %v", s.Uri, err) - } - - if uri.Host == "" { - return errors.New("unparseable URL") - } - - uri.Fragment = "" - if convertFlags.toplevel { - uri.Path = "" - uri.RawQuery = "" - s.Uri = uri.Scheme + "://" + uri.Host - } else { - s.Uri = uri.String() - } - - if convertFlags.checkUri { - c.checkRedirect(s.Uri, s, 0) - } - return -} - -func (c *converter) checkRedirect(uri string, s *seedDesc, count int) { - if count > 5 { - return - } - count++ - - resp, err := c.httpClient.Head(uri) - if err != nil { - uerr := err.(*url.Error) - if uerr.Timeout() { - err = nil - } else { - switch v := uerr.Err.(type) { - case *net.OpError: - if t, ok := v.Err.(*net.DNSError); ok { - err = fmt.Errorf("no such host %s", t.Name) - } - return - case x509.HostnameError: - case x509.UnknownAuthorityError: - case x509.CertificateInvalidError: - return - default: - return - } - } - } else { - _ = resp.Body.Close() - if resp.StatusCode == 301 { - uri = resp.Header.Get("Location") - if uri != "" { - c.checkRedirect(uri, s, count) - } - } else { - s.EntityName = c.getTitle(uri) - } - } -} - -func (c *converter) getTitle(uri string) string { - resp, err := c.httpClient.Get(uri) - if err != nil { - return "" - } - defer resp.Body.Close() - - doc, err := html.Parse(resp.Body) - if err != nil { - return "" - } - var title string - var f func(*html.Node) - f = func(n *html.Node) { - if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { - title = strings.TrimSpace(n.FirstChild.Data) - return - } - for c := n.FirstChild; c != nil; c = c.NextSibling { - f(c) - } - } - f(doc) - return title -} diff --git a/src/cmd/importcmd/duplicatereport.go b/src/cmd/importcmd/duplicatereport.go deleted file mode 100644 index 06a0a5d..0000000 --- a/src/cmd/importcmd/duplicatereport.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package importcmd - -import ( - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/importutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "io" - "os" -) - -var dupFlags struct { - outFile string - dbDir string - resetDb bool - toplevel bool - ignoreScheme bool -} - -// duplicateReportCmd represents the duplicatereport command -var duplicateReportCmd = &cobra.Command{ - Use: "duplicatereport [kind]", - Short: "List duplicated seeds in Veidemann", - Long: ``, - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - var err error - - kind := configV1.Kind(configV1.Kind_value[args[0]]) - - // Create output writer (file or stdout) - var out io.Writer - if dupFlags.outFile == "" { - out = os.Stdout - } else { - out, err = os.Create(dupFlags.outFile) - defer out.(io.Closer).Close() - if err != nil { - log.Fatalf("Unable to open out file: %v, cause: %v", dupFlags.outFile, err) - os.Exit(1) - } - } - - // Create Veidemann config client - client, conn := connection.NewConfigClient() - defer conn.Close() - - // Create state Database based on seeds in Veidemann - var keyNormalizer importutil.KeyNormalizer - if kind == configV1.Kind_seed { - keyNormalizer = &UriKeyNormalizer{toplevel: dupFlags.toplevel, ignoreScheme: dupFlags.ignoreScheme} - } else { - keyNormalizer = &importutil.NoopKeyNormalizer{} - } - impf := importutil.NewImportDb(client, dupFlags.dbDir, kind, keyNormalizer, dupFlags.resetDb) - impf.ImportExisting() - defer impf.Close() - - switch kind { - case configV1.Kind_seed: - if err = impf.SeedDuplicateReport(out); err != nil { - log.Errorf("failed creating seed duplicate report: %v", err) - } - case configV1.Kind_crawlEntity: - if err = impf.CrawlEntityDuplicateReport(out); err != nil { - log.Errorf("failed creating crawl entity duplicate report: %v", err) - } - } - }, -} - -func init() { - ImportCmd.AddCommand(duplicateReportCmd) - - duplicateReportCmd.PersistentFlags().StringVarP(&dupFlags.outFile, "outFile", "o", "", "File to write output.") - duplicateReportCmd.PersistentFlags().StringVarP(&dupFlags.dbDir, "db-directory", "b", "/tmp/veidemannctl", "Directory for storing state db") - duplicateReportCmd.PersistentFlags().BoolVarP(&dupFlags.resetDb, "reset-db", "r", false, "Clean state db") - duplicateReportCmd.PersistentFlags().BoolVarP(&dupFlags.toplevel, "toplevel", "", false, "Convert URI to toplevel by removing path before checking for duplicates.") - duplicateReportCmd.PersistentFlags().BoolVarP(&dupFlags.toplevel, "ignore-scheme", "", false, "Ignore the URL's scheme when checking for duplicates.") -} diff --git a/src/cmd/importcmd/importseed.go b/src/cmd/importcmd/importseed.go deleted file mode 100644 index 52fa091..0000000 --- a/src/cmd/importcmd/importseed.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package importcmd - -import ( - "crypto/x509" - "errors" - "fmt" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/importutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "golang.org/x/net/context" - "io" - "net" - "net/http" - "net/url" - "os" - "strings" -) - -type seedDesc struct { - EntityName string - Uri string - EntityLabel []*configV1.Label - SeedLabel []*configV1.Label - EntityDescription string - SeedDescription string - Description string - fileName string - recNum int - err error - crawlJobRef []*configV1.ConfigRef -} - -var importFlags struct { - filename string - errorFile string - toplevel bool - ignoreScheme bool - checkUri bool - checkUriTimeout int64 - crawlJobId string - dbDir string - resetDb bool - dryRun bool -} - -// importSeedCmd represents the import command -var importSeedCmd = &cobra.Command{ - Use: "seed", - Short: "Import seeds", - Long: `Import new seeds and entities from a line oriented JSON file on the following format: - -{"entityName":"foo","uri":"https://www.example.com/","entityDescription":"desc","entityLabel":[{"key":"foo","value":"bar"}],"seedLabel":[{"key":"foo","value":"bar"},{"key":"foo2","value":"bar"}],"seedDescription":"foo"} - -Every record must be formatted on a single line. - - -`, - - Run: func(cmd *cobra.Command, args []string) { - i := &importer{} - var err error - - // Check inputflag - if importFlags.filename == "" { - fmt.Printf("Import file is required. See --filename\n") - _ = cmd.Usage() - os.Exit(1) - } - - // Create error writer (file or stdout) - var errFile io.Writer - if importFlags.errorFile == "" { - errFile = os.Stdout - } else { - errFile, err = os.Create(importFlags.errorFile) - defer errFile.(io.Closer).Close() - if err != nil { - log.Fatalf("Unable to open error file: %v, cause: %v", importFlags.errorFile, err) - os.Exit(1) - } - } - - processorLogger := &log.Logger{ - Out: os.Stderr, - Formatter: new(log.TextFormatter), - Hooks: make(log.LevelHooks), - Level: log.InfoLevel, - } - - // Create Veidemann config client - client, conn := connection.NewConfigClient() - defer conn.Close() - i.configClient = client - - // Create http client - i.httpClient = importutil.NewHttpClient(importFlags.checkUriTimeout) - - // Create state Database based on seeds in Veidemann - impEntity := importutil.NewImportDb(client, importFlags.dbDir, configV1.Kind_crawlEntity, &importutil.NoopKeyNormalizer{}, importFlags.resetDb) - impEntity.ImportExisting() - defer impEntity.Close() - - // Create state Database based on seeds in Veidemann - keyNormalizer := &UriKeyNormalizer{ignoreScheme: importFlags.ignoreScheme, toplevel: importFlags.toplevel} - impSeed := importutil.NewImportDb(client, importFlags.dbDir, configV1.Kind_seed, keyNormalizer, importFlags.resetDb) - impSeed.ImportExisting() - defer impSeed.Close() - - // Create Record reader for file input - rr, err := importutil.NewRecordReader(importFlags.filename, &importutil.JsonYamlDecoder{}, "*.json") - if err != nil { - log.Fatalf("Unable to create RecordReader: %v", err) - os.Exit(1) - } - - // Processor for converting oos records into import records - proc := func(value interface{}, readerState *importutil.State) error { - sd := value.(*seedDesc) - if err := i.normalizeUri(sd); err != nil { - return err - } - exists, err := impSeed.Check(sd.Uri) - if err != nil { - return err - } - if exists.Code.ExistsInVeidemann() { - return fmt.Errorf("seed already exists: %v", sd.Uri) - } - if err := i.checkUri(sd); err != nil { - return err - } - - exists, err = impSeed.CheckAndUpdateVeidemann(sd.Uri, sd, func(client configV1.ConfigClient, data interface{}) (id string, err error) { - if importFlags.dryRun { - return "", nil - } else { - obj := data.(*seedDesc) - - var eId string - entityExists, err := impEntity.Check(obj.EntityName) - if err != nil { - return "", err - } - if entityExists.Code.ExistsInVeidemann() { - processorLogger.Infof("entity already exists: %v", obj.EntityName) - eId = entityExists.KnownIds[0] - } else { - e := &configV1.ConfigObject{ - ApiVersion: "v1", - Kind: configV1.Kind_crawlEntity, - Meta: &configV1.Meta{ - Name: obj.EntityName, - Description: obj.EntityDescription, - Label: obj.EntityLabel, - }, - } - ctx := context.Background() - processorLogger.Debugf("store entity: %v", e) - e, err = client.SaveConfigObject(ctx, e) - - if err != nil { - return "", fmt.Errorf("Error writing crawl entity: %v", err) - } - eId = e.Id - } - - s := &configV1.ConfigObject{ - ApiVersion: "v1", - Kind: configV1.Kind_seed, - Meta: &configV1.Meta{ - Name: obj.Uri, - Description: obj.SeedDescription, - Label: obj.SeedLabel, - }, - Spec: &configV1.ConfigObject_Seed{ - Seed: &configV1.Seed{ - EntityRef: &configV1.ConfigRef{ - Kind: configV1.Kind_crawlEntity, - Id: eId, - }, - JobRef: obj.crawlJobRef, - }, - }, - } - processorLogger.Debugf("store seed: %v", s) - ctx := context.Background() - s, err = client.SaveConfigObject(ctx, s) - if err != nil && !entityExists.Code.ExistsInVeidemann() { - if d, err := client.DeleteConfigObject(ctx, &configV1.ConfigObject{Kind: configV1.Kind_crawlEntity, Id: eId}); err == nil { - fmt.Println("Delete entity: ", d) - } else { - fmt.Println("Failed deletion of entity: ", err) - } - return "", fmt.Errorf("Error writing seed: %v", err) - } - return s.Id, nil - } - }) - if err != nil { - return err - } - if exists.Code.ExistsInVeidemann() { - return fmt.Errorf("seed already exists: %v", sd.Uri) - } - - return nil - } - - // Error handler - errorHandler := func(state *importutil.StateVal) { - var uri string - if state.Val != nil { - uri = state.Val.(*seedDesc).Uri - } - _, _ = fmt.Fprintf(errFile, "{\"uri\": \"%s\", \"err\": \"%s\", \"file\": \"%s\", \"recNum\": %v}\n", uri, state.GetError(), state.GetFilename(), state.GetRecordNum()) - } - - // Create multithreaded executor - conv := importutil.NewExecutor(32, proc, errorHandler) - - crawlJobRef := []*configV1.ConfigRef{ - { - Kind: configV1.Kind_crawlJob, - Id: importFlags.crawlJobId, - }, - } - - // Process - for { - var sd seedDesc - state, err := rr.Next(&sd) - if err == io.EOF { - break - } - if err != nil { - _, _ = fmt.Fprintf(errFile, "Error decoding record: %v, cause: %v", state, err) - os.Exit(1) - } - if importFlags.crawlJobId != "" { - sd.crawlJobRef = crawlJobRef - } - - conv.Do(state, &sd) - } - - conv.Finish() - _, _ = fmt.Fprintf(os.Stderr, "\nRecords read: %v, imported: %v, Failed: %v\n", conv.Count, conv.Success, conv.Failed) - }, -} - -func init() { - ImportCmd.AddCommand(importSeedCmd) - - importSeedCmd.PersistentFlags().StringVarP(&importFlags.filename, "filename", "f", "", "Filename or directory to read from. "+ - "If input is a directory, all files ending in .yaml or .json will be tried. An input of '-' will read from stdin.") - importSeedCmd.PersistentFlags().StringVarP(&importFlags.errorFile, "errorfile", "e", "", "File to write errors to.") - importSeedCmd.PersistentFlags().BoolVarP(&importFlags.toplevel, "toplevel", "", false, "Convert URI to toplevel by removing path.") - importSeedCmd.PersistentFlags().BoolVarP(&importFlags.ignoreScheme, "ignore-scheme", "", false, "Ignore the URL's scheme when checking if this URL is already imported.") - importSeedCmd.PersistentFlags().BoolVarP(&importFlags.checkUri, "checkuri", "", false, "Check the uri for liveness and follow 301") - importSeedCmd.PersistentFlags().Int64VarP(&importFlags.checkUriTimeout, "checkuri-timeout", "", 500, "Timeout in ms when checking uri for liveness.") - importSeedCmd.PersistentFlags().StringVarP(&importFlags.crawlJobId, "crawljob-id", "", "", "Set crawlJob ID for new seeds.") - importSeedCmd.PersistentFlags().StringVarP(&importFlags.dbDir, "db-directory", "b", "/tmp/veidemannctl", "Directory for storing state db") - importSeedCmd.PersistentFlags().BoolVarP(&importFlags.resetDb, "reset-db", "r", false, "Clean state db") - importSeedCmd.PersistentFlags().BoolVarP(&importFlags.dryRun, "dry-run", "", false, "Run the import without writing anything to Veidemann") -} - -type importer struct { - httpClient *http.Client - configClient configV1.ConfigClient -} - -func (i *importer) normalizeUri(s *seedDesc) (err error) { - uri, err := url.Parse(s.Uri) - if err != nil { - return fmt.Errorf("unparseable URL '%v', cause: %v", s.Uri, err) - } - - if uri.Host == "" { - return errors.New("unparseable URL") - } - - uri.Fragment = "" - uri.Host = strings.ToLower(uri.Host) - - if importFlags.toplevel { - uri.Path = "/" - uri.RawQuery = "" - } - - if uri.Path == "" { - uri.Path = "/" - } - - s.Uri = uri.String() - - return -} - -func (i *importer) checkUri(s *seedDesc) (err error) { - if importFlags.checkUri { - i.checkRedirect(s.Uri, s, 0) - } - return -} - -func (i *importer) checkRedirect(uri string, s *seedDesc, count int) { - if count > 5 { - return - } - count++ - - resp, err := i.httpClient.Head(uri) - if err != nil { - uerr := err.(*url.Error) - if uerr.Timeout() { - err = nil - } else { - switch v := uerr.Err.(type) { - case *net.OpError: - if t, ok := v.Err.(*net.DNSError); ok { - err = fmt.Errorf("no such host %s", t.Name) - } - return - case x509.HostnameError: - case x509.UnknownAuthorityError: - case x509.CertificateInvalidError: - return - default: - return - } - } - } else { - _ = resp.Body.Close() - if resp.StatusCode == 301 { - uri = resp.Header.Get("Location") - if uri != "" { - i.checkRedirect(uri, s, count) - } - } else { - s.Uri = uri - } - } -} diff --git a/src/cmd/logconfig/deletelogger.go b/src/cmd/logconfig/deletelogger.go deleted file mode 100644 index 1106783..0000000 --- a/src/cmd/logconfig/deletelogger.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logconfig - -import ( - "context" - "fmt" - "github.com/golang/protobuf/ptypes/empty" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "os" -) - -// DeleteCmd represents the delete command -var DeleteLoggerCmd = &cobra.Command{ - Use: "delete [logger]", - Short: "Delete a logger", - Long: `Delete a logger.`, - Run: func(cmd *cobra.Command, args []string) { - if len(args) != 1 { - fmt.Printf("Wrong number of arguments\n") - cmd.Help() - os.Exit(1) - } - - logger := args[0] - - client, conn := connection.NewConfigClient() - defer conn.Close() - - r, err := client.GetLogConfig(context.Background(), &empty.Empty{}) - if err != nil { - log.Fatalf("could not get log config: %v", err) - } - - var loggers map[string]configV1.LogLevels_Level - loggers = make(map[string]configV1.LogLevels_Level) - for _, l := range r.LogLevel { - if l.Logger != "" && l.Logger != logger { - loggers[l.Logger] = l.Level - } - } - - n := &configV1.LogLevels{} - for k, v := range loggers { - n.LogLevel = append(n.LogLevel, &configV1.LogLevels_LogLevel{Logger: k, Level: v}) - } - - _, err = client.SaveLogConfig(context.Background(), n) - if err != nil { - log.Fatalf("could not get log config: %v", err) - } - }, -} - -func init() { - LogconfigCmd.AddCommand(DeleteLoggerCmd) -} diff --git a/src/cmd/logconfig/listloggers.go b/src/cmd/logconfig/listloggers.go deleted file mode 100644 index 26af120..0000000 --- a/src/cmd/logconfig/listloggers.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logconfig - -import ( - "context" - "fmt" - "github.com/golang/protobuf/ptypes/empty" - "github.com/nlnwa/veidemannctl/src/connection" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -// reportCmd represents the report command -var ListLoggersCmd = &cobra.Command{ - Use: "list", - Short: "List configured loggers", - Long: `List configured loggers.`, - Run: func(cmd *cobra.Command, args []string) { - client, conn := connection.NewConfigClient() - defer conn.Close() - - r, err := client.GetLogConfig(context.Background(), &empty.Empty{}) - if err != nil { - log.Fatalf("could not get log config: %v", err) - } - - fmt.Printf("%-45s %s\n", "Logger", "Level") - fmt.Println("---------------------------------------------------") - for _, l := range r.LogLevel { - fmt.Printf("%-45s %s\n", l.Logger, l.Level) - } - }, -} - -func init() { - LogconfigCmd.AddCommand(ListLoggersCmd) -} diff --git a/src/cmd/logconfig/setlogger.go b/src/cmd/logconfig/setlogger.go deleted file mode 100644 index 7f89157..0000000 --- a/src/cmd/logconfig/setlogger.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logconfig - -import ( - "context" - "fmt" - "github.com/golang/protobuf/ptypes/empty" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "os" -) - -// reportCmd represents the report command -var SetLoggerCmd = &cobra.Command{ - Use: "set [logger] [level]", - Short: "Configure logger", - Long: `Configure logger.`, - Run: func(cmd *cobra.Command, args []string) { - if len(args) != 2 { - fmt.Printf("Wrong number of arguments\n") - cmd.Help() - os.Exit(1) - } - - logger := args[0] - level := configV1.LogLevels_Level(configV1.LogLevels_Level_value[args[1]]) - if level == configV1.LogLevels_UNDEFINED { - fmt.Printf("Unknown level %v\n", args[1]) - cmd.Help() - os.Exit(1) - } - - client, conn := connection.NewConfigClient() - defer conn.Close() - - r, err := client.GetLogConfig(context.Background(), &empty.Empty{}) - if err != nil { - log.Fatalf("could not get log config: %v", err) - } - - var loggers map[string]configV1.LogLevels_Level - loggers = make(map[string]configV1.LogLevels_Level) - for _, l := range r.LogLevel { - if l.Logger != "" { - loggers[l.Logger] = l.Level - } - } - - loggers[logger] = level - n := &configV1.LogLevels{} - for k, v := range loggers { - n.LogLevel = append(n.LogLevel, &configV1.LogLevels_LogLevel{Logger: k, Level: v}) - } - - _, err = client.SaveLogConfig(context.Background(), n) - if err != nil { - log.Fatalf("could not get log config: %v", err) - } - }, -} - -func init() { - LogconfigCmd.AddCommand(SetLoggerCmd) -} diff --git a/src/cmd/login.go b/src/cmd/login.go deleted file mode 100644 index 8cb4e33..0000000 --- a/src/cmd/login.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - log "github.com/sirupsen/logrus" - - "github.com/nlnwa/veidemannctl/src/configutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/spf13/cobra" -) - -var manualLogin bool - -// loginCmd represents the login command -var loginCmd = &cobra.Command{ - Use: "login", - Short: "Initiate browser session for logging in to Veidemann", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - idp, _ := connection.GetIdp() - if idp == "" { - return - } - - log.Debugf("login using idp at %v", idp) - a := connection.NewAuth(idp) - - a.Login(manualLogin) - claims := a.Claims() - configutil.WriteConfig() - fmt.Printf("Hello %s\n", claims.Name) - }, -} - -func init() { - RootCmd.AddCommand(loginCmd) - loginCmd.PersistentFlags().BoolVarP(&manualLogin, "manual", "m", false, - "Manually copy and paste login url and code. Use this to log in from a remote terminal.") -} diff --git a/src/cmd/reports/crawlexecution.go b/src/cmd/reports/crawlexecution.go deleted file mode 100644 index 34e2530..0000000 --- a/src/cmd/reports/crawlexecution.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reports - -import ( - "context" - "fmt" - commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" - frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" - reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" - "github.com/nlnwa/veidemannctl/src/apiutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "google.golang.org/protobuf/types/known/timestamppb" - "io" - "time" -) - -var crawlExecFlags struct { - filters []string - pageSize int32 - page int32 - goTemplate string - format string - file string - orderByPath string - orderDesc bool - to *time.Time - from *time.Time - watch bool - states []string -} - -func newCrawlExecutionCmd() *cobra.Command { - var crawlexecutionCmd = &cobra.Command{ - Use: "crawlexecution", - Short: "Get current status for crawl executions", - Long: `Get current status for crawl executions.`, - PreRunE: func(cmd *cobra.Command, args []string) error { - v := viper.New() - - if err := v.BindPFlag("to", cmd.Flag("to")); err != nil { - return fmt.Errorf("failed to bind flag: %w", err) - } else if v.IsSet("to") { - to := v.GetTime("to") - crawlExecFlags.to = &to - } - if err := v.BindPFlag("from", cmd.Flag("from")); err != nil { - return fmt.Errorf("failed to bind flag: %w", err) - } else if v.IsSet("from") { - from := v.GetTime("from") - crawlExecFlags.from = &from - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - client, conn := connection.NewReportClient() - defer conn.Close() - - var ids []string - - if len(args) > 0 { - ids = args - } - - request, err := createCrawlExecutionsListRequest(ids) - if err != nil { - return fmt.Errorf("failed creating request: %w", err) - } - - cmd.SilenceUsage = true - - r, err := client.ListExecutions(context.Background(), request) - if err != nil { - return fmt.Errorf("error from controller: %w", err) - } - out, err := format.ResolveWriter(crawlExecFlags.file) - if err != nil { - return fmt.Errorf("could not resolve output '%s': %w", crawlExecFlags.file, err) - } - s, err := format.NewFormatter("CrawlExecutionStatus", out, crawlExecFlags.format, crawlExecFlags.goTemplate) - if err != nil { - return err - } - defer s.Close() - - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - return err - } - if err := s.WriteRecord(msg); err != nil { - return err - } - } - return nil - }, - } - crawlexecutionCmd.Flags().Int32VarP(&crawlExecFlags.pageSize, "pagesize", "s", 10, "Number of objects to get") - crawlexecutionCmd.Flags().Int32VarP(&crawlExecFlags.page, "page", "p", 0, "The page number") - crawlexecutionCmd.Flags().StringVarP(&crawlExecFlags.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") - crawlexecutionCmd.Flags().StringVarP(&crawlExecFlags.goTemplate, "template", "t", "", "A Go template used to format the output") - crawlexecutionCmd.Flags().StringSliceVarP(&crawlExecFlags.filters, "filter", "q", nil, "Filter objects by field (i.e. meta.description=foo)") - crawlexecutionCmd.Flags().StringSliceVar(&crawlExecFlags.states, "state", nil, "Filter objects by state. Valid states are UNDEFINED, FETCHING, SLEEPING, FINISHED or FAILED") - crawlexecutionCmd.Flags().StringVarP(&crawlExecFlags.file, "filename", "f", "", "File name to write to") - crawlexecutionCmd.Flags().StringVar(&crawlExecFlags.orderByPath, "order-by", "", "Order by path") - crawlexecutionCmd.Flags().String("to", "", "To start time") - crawlexecutionCmd.Flags().String("from", "", "From start time") - crawlexecutionCmd.Flags().BoolVar(&crawlExecFlags.orderDesc, "desc", false, "Order descending") - crawlexecutionCmd.Flags().BoolVarP(&crawlExecFlags.watch, "watch", "w", false, "Get a continous stream of changes") - - return crawlexecutionCmd -} - -func createCrawlExecutionsListRequest(ids []string) (*reportV1.CrawlExecutionsListRequest, error) { - request := &reportV1.CrawlExecutionsListRequest{ - Id: ids, - Watch: crawlExecFlags.watch, - PageSize: crawlExecFlags.pageSize, - Offset: crawlExecFlags.page, - OrderByPath: crawlExecFlags.orderByPath, - OrderDescending: crawlExecFlags.orderDesc, - } - - if crawlExecFlags.watch { - request.PageSize = 0 - } - - if crawlExecFlags.from != nil { - fmt.Println(crawlExecFlags.from) - request.StartTimeFrom = timestamppb.New(*crawlExecFlags.from) - } - - if crawlExecFlags.to != nil { - request.StartTimeTo = timestamppb.New(*crawlExecFlags.to) - } - - if len(crawlExecFlags.states) > 0 { - for _, state := range crawlExecFlags.states { - if s, ok := frontierV1.CrawlExecutionStatus_State_value[state]; !ok { - return nil, fmt.Errorf("not a crawlexecution state: %s", state) - } else { - request.State = append(request.State, frontierV1.CrawlExecutionStatus_State(s)) - } - } - } - - if len(crawlExecFlags.filters) > 0 { - queryTemplate := new(frontierV1.CrawlExecutionStatus) - queryMask := new(commonsV1.FieldMask) - - for _, filter := range crawlExecFlags.filters { - err := apiutil.CreateTemplateFilter(filter, queryTemplate, queryMask) - if err != nil { - return nil, err - } - } - - request.QueryMask = queryMask - request.QueryTemplate = queryTemplate - } - - return request, nil -} diff --git a/src/cmd/reports/crawllog.go b/src/cmd/reports/crawllog.go deleted file mode 100644 index 49b1899..0000000 --- a/src/cmd/reports/crawllog.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reports - -import ( - "context" - "fmt" - commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" - logV1 "github.com/nlnwa/veidemann-api/go/log/v1" - "github.com/nlnwa/veidemannctl/src/apiutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "io" -) - -var crawllogFlags struct { - executionId string - filter string - pageSize int32 - page int32 - goTemplate string - format string - file string - watch bool -} - -func init() { - crawllogCmd.Flags().Int32VarP(&crawllogFlags.pageSize, "pagesize", "s", 10, "Number of objects to get") - crawllogCmd.Flags().Int32VarP(&crawllogFlags.page, "page", "p", 0, "The page number") - crawllogCmd.Flags().StringVarP(&crawllogFlags.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") - crawllogCmd.Flags().StringVarP(&crawllogFlags.goTemplate, "template", "t", "", "A Go template used to format the output") - crawllogCmd.Flags().StringVarP(&crawllogFlags.filter, "filter", "q", "", "Filter objects by field (i.e. meta.description=foo") - crawllogCmd.Flags().StringVarP(&crawllogFlags.file, "filename", "f", "", "File name to write to") - crawllogCmd.Flags().BoolVarP(&crawllogFlags.watch, "watch", "w", false, "Get a continous stream of changes") -} - -// crawllogCmd represents the crawllog command -var crawllogCmd = &cobra.Command{ - Use: "crawllog", - Short: "View crawl log", - Long: `View crawl log.`, - RunE: func(cmd *cobra.Command, args []string) error { - client, conn := connection.NewLogClient() - defer conn.Close() - - var ids []string - - if len(args) > 0 { - ids = args - } - - request, err := createCrawlLogListRequest(ids) - if err != nil { - return fmt.Errorf("error creating request: %w", err) - } - - r, err := client.ListCrawlLogs(context.Background(), request) - if err != nil { - return fmt.Errorf("error from controller: %w", err) - } - - out, err := format.ResolveWriter(crawllogFlags.file) - if err != nil { - return fmt.Errorf("could not resolve output '%s': %w", crawllogFlags.file, err) - } - s, err := format.NewFormatter("CrawlLog", out, crawllogFlags.format, crawllogFlags.goTemplate) - if err != nil { - return err - } - defer s.Close() - - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - return err - } - log.Debugf("Outputting crawl log record with WARC id '%s'", msg.WarcId) - if err := s.WriteRecord(msg); err != nil { - return err - } - } - return nil - }, -} - -func createCrawlLogListRequest(ids []string) (*logV1.CrawlLogListRequest, error) { - request := &logV1.CrawlLogListRequest{} - request.WarcId = ids - request.Watch = crawllogFlags.watch - if crawllogFlags.watch { - crawllogFlags.pageSize = 0 - } - - request.Offset = crawllogFlags.page - request.PageSize = crawllogFlags.pageSize - - if crawllogFlags.filter != "" { - queryMask := new(commonsV1.FieldMask) - queryTemplate := new(logV1.CrawlLog) - err := apiutil.CreateTemplateFilter(crawllogFlags.filter, queryTemplate, queryMask) - if err != nil { - return nil, err - } - request.QueryMask = queryMask - request.QueryTemplate = queryTemplate - } - - return request, nil -} diff --git a/src/cmd/reports/jobexecution.go b/src/cmd/reports/jobexecution.go deleted file mode 100644 index b61e301..0000000 --- a/src/cmd/reports/jobexecution.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reports - -import ( - "context" - "fmt" - commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" - frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1" - reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" - "github.com/nlnwa/veidemannctl/src/apiutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "io" - "time" -) - -type jobExecFlags struct { - filters []string - states []string - pageSize int32 - page int32 - orderByPath string - orderDesc bool - to *time.Time - from *time.Time - goTemplate string - format string - file string - watch bool -} - -var jobExecConf = jobExecFlags{} - -func newJobExecutionCmd() *cobra.Command { - // jobexecutionCmd represents the jobexecution command - var cmd = &cobra.Command{ - Use: "jobexecution", - Short: "Get current status for job executions", - Long: `Get current status for job executions.`, - PreRunE: func(cmd *cobra.Command, args []string) error { - v := viper.New() - - if err := v.BindPFlag("to", cmd.Flag("to")); err != nil { - return fmt.Errorf("failed to bind flag: %w", err) - } - if v.IsSet("to") { - to := v.GetTime("to") - crawlExecFlags.to = &to - } - if err := v.BindPFlag("from", cmd.Flag("from")); err != nil { - return fmt.Errorf("failed to bind flag: %w", err) - } - if v.IsSet("from") { - from := v.GetTime("from") - crawlExecFlags.from = &from - } - return nil - }, - RunE: func(cmd *cobra.Command, args []string) error { - client, conn := connection.NewReportClient() - defer conn.Close() - - var ids []string - - if len(args) > 0 { - ids = args - } - - request, err := createJobExecutionsListRequest(ids) - if err != nil { - return fmt.Errorf("error creating request: %w", err) - } - - cmd.SilenceUsage = true - - r, err := client.ListJobExecutions(context.Background(), request) - if err != nil { - return fmt.Errorf("error from controller: %v", err) - } - - out, err := format.ResolveWriter(jobExecConf.file) - if err != nil { - return fmt.Errorf("could not resolve output '%v': %v", jobExecConf.file, err) - } - s, err := format.NewFormatter("JobExecutionStatus", out, jobExecConf.format, jobExecConf.goTemplate) - if err != nil { - return err - } - defer s.Close() - - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - return err - } - if err := s.WriteRecord(msg); err != nil { - return err - } - } - return nil - }, - } - - cmd.Flags().Int32VarP(&jobExecConf.pageSize, "pagesize", "s", 10, "Number of objects to get") - cmd.Flags().Int32VarP(&jobExecConf.page, "page", "p", 0, "The page number") - cmd.Flags().StringVarP(&jobExecConf.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") - cmd.Flags().StringVarP(&jobExecConf.goTemplate, "template", "t", "", "A Go template used to format the output") - cmd.Flags().StringSliceVarP(&jobExecConf.filters, "filter", "q", nil, "Filter objects by field (i.e. meta.description=foo") - cmd.Flags().StringSliceVar(&jobExecConf.states, "state", nil, "Filter objects by state(s)") - cmd.Flags().StringVarP(&jobExecConf.file, "filename", "f", "", "File name to write to") - cmd.Flags().StringVar(&jobExecConf.orderByPath, "order-by", "", "Order by path") - cmd.Flags().BoolVar(&jobExecConf.orderDesc, "desc", false, "Order descending") - cmd.Flags().String("to", "", "To start time") - cmd.Flags().String("from", "", "From start time") - cmd.Flags().BoolVarP(&jobExecConf.watch, "watch", "w", false, "Get a continous stream of changes") - - return cmd -} - -func createJobExecutionsListRequest(ids []string) (*reportV1.JobExecutionsListRequest, error) { - request := &reportV1.JobExecutionsListRequest{ - Id: ids, - Watch: jobExecConf.watch, - PageSize: jobExecConf.pageSize, - Offset: jobExecConf.page, - OrderByPath: jobExecConf.orderByPath, - OrderDescending: jobExecConf.orderDesc, - } - if jobExecConf.watch { - request.PageSize = 0 - } - - if len(jobExecConf.states) > 0 { - for _, state := range jobExecConf.states { - if s, ok := frontierV1.JobExecutionStatus_State_value[state]; !ok { - return nil, fmt.Errorf("not a jobexecution state: %s", state) - } else { - request.State = append(request.State, frontierV1.JobExecutionStatus_State(s)) - } - } - } - - if len(jobExecConf.filters) > 0 { - queryMask := new(commonsV1.FieldMask) - queryTemplate := new(frontierV1.JobExecutionStatus) - - for _, filter := range jobExecConf.filters { - err := apiutil.CreateTemplateFilter(filter, queryTemplate, queryMask) - if err != nil { - return nil, err - } - } - request.QueryMask = queryMask - request.QueryTemplate = queryTemplate - } - - return request, nil -} diff --git a/src/cmd/reports/pagelog.go b/src/cmd/reports/pagelog.go deleted file mode 100644 index 39576b9..0000000 --- a/src/cmd/reports/pagelog.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reports - -import ( - "context" - "fmt" - commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" - logV1 "github.com/nlnwa/veidemann-api/go/log/v1" - "github.com/nlnwa/veidemannctl/src/apiutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "io" -) - -var pagelogFlags struct { - filter string - pageSize int32 - page int32 - goTemplate string - format string - file string - watch bool -} - -// pagelogCmd represents the pagelog command -var pagelogCmd = &cobra.Command{ - Use: "pagelog", - Short: "View page log", - Long: `View page log.`, - RunE: func(cmd *cobra.Command, args []string) error { - client, conn := connection.NewLogClient() - defer conn.Close() - - var ids []string - - if len(args) > 0 { - ids = args - } - - request, err := createPageLogListRequest(ids) - if err != nil { - return fmt.Errorf("Error creating request: %v", err) - } - - r, err := client.ListPageLogs(context.Background(), request) - if err != nil { - return fmt.Errorf("Error from controller: %v", err) - } - - out, err := format.ResolveWriter(pagelogFlags.file) - if err != nil { - return fmt.Errorf("Could not resolve output '%v': %v", pagelogFlags.file, err) - } - s, err := format.NewFormatter("PageLog", out, pagelogFlags.format, pagelogFlags.goTemplate) - if err != nil { - return err - } - defer s.Close() - - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("Error getting object: %v", err) - } - log.Debugf("Outputting page log record with WARC id '%s'", msg.WarcId) - if err := s.WriteRecord(msg); err != nil { - return err - } - } - return nil - }, -} - -func init() { - pagelogCmd.Flags().Int32VarP(&pagelogFlags.pageSize, "pagesize", "s", 10, "Number of objects to get") - pagelogCmd.Flags().Int32VarP(&pagelogFlags.page, "page", "p", 0, "The page number") - pagelogCmd.Flags().StringVarP(&pagelogFlags.format, "output", "o", "table", "Output format (table|wide|json|yaml|template|template-file)") - pagelogCmd.Flags().StringVarP(&pagelogFlags.goTemplate, "template", "t", "", "A Go template used to format the output") - pagelogCmd.Flags().StringVarP(&pagelogFlags.filter, "filter", "q", "", "Filter objects by field (i.e. meta.description=foo") - pagelogCmd.Flags().StringVarP(&pagelogFlags.file, "filename", "f", "", "File name to write to") - pagelogCmd.Flags().BoolVarP(&pagelogFlags.watch, "watch", "w", false, "Get a continous stream of changes") -} - -func createPageLogListRequest(ids []string) (*logV1.PageLogListRequest, error) { - request := &logV1.PageLogListRequest{} - request.WarcId = ids - request.Watch = pagelogFlags.watch - if pagelogFlags.watch { - pagelogFlags.pageSize = 0 - } - - request.Offset = pagelogFlags.page - request.PageSize = pagelogFlags.pageSize - - if pagelogFlags.filter != "" { - queryMask := new(commonsV1.FieldMask) - queryTemplate := new(logV1.PageLog) - err := apiutil.CreateTemplateFilter(pagelogFlags.filter, queryTemplate, queryMask) - if err != nil { - return nil, err - } - request.QueryMask = queryMask - request.QueryTemplate = queryTemplate - } - - return request, nil -} diff --git a/src/cmd/reports/query.go b/src/cmd/reports/query.go deleted file mode 100644 index 4ec4323..0000000 --- a/src/cmd/reports/query.go +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright © 2018 NAME HERE -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reports - -import ( - "context" - "fmt" - "github.com/ghodss/yaml" - reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" - "github.com/nlnwa/veidemannctl/src/configutil" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" -) - -type queryConf struct { - pageSize int32 - page int32 - goTemplate string - file string - format string -} - -var queryFlags = &queryConf{} - -// queryCmd represents the query command -var queryCmd = &cobra.Command{ - Use: "query [queryString|file] args...", - Short: "Run a database query", - Long: `Run a database query. The query should be a java script string like the ones used by RethinkDb javascript driver.`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 0 { - client, conn := connection.NewReportClient() - defer conn.Close() - - request := reportV1.ExecuteDbQueryRequest{} - - queryDef, err := getQueryDef(args[0]) - if err != nil { - return err - } - defer queryDef.marshalSpec.Close() - - var params []interface{} - - for _, v := range args[1:] { - params = append(params, v) - } - - request.Query = fmt.Sprintf(queryDef.Query, params...) - - request.Limit = queryFlags.pageSize - log.Debugf("Executing query: %s", request.GetQuery()) - - // from now on we don't want usage when error occurs - cmd.SilenceUsage = true - - stream, err := client.ExecuteDbQuery(context.Background(), &request) - if err != nil { - return fmt.Errorf("Failed executing query: %v", err) - } - - for { - value, err := stream.Recv() - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("Query error: %v", err) - } - if err := queryDef.marshalSpec.WriteRecord(value.GetRecord()); err != nil { - return err - } - } - } else { - d := configutil.GetConfigDir("query") - if configutil.GlobalFlags.IsShellCompletion { - q := listStoredQueries(d) - for _, s := range q { - fmt.Println(s.Name) - } - } else { - fmt.Println("Missing query.\nSee 'veidemannctl report query -h' for help") - q := listStoredQueries(d) - if len(q) > 0 { - fmt.Printf("\nStored queries in '%s':\n", d) - for _, s := range q { - fmt.Printf(" * %-20s - %s", s.Name, s.Description) - } - } - } - } - return nil - }, -} - -func init() { - queryCmd.PersistentFlags().Int32VarP(&queryFlags.pageSize, "pagesize", "s", 10, "Number of objects to get") - queryCmd.PersistentFlags().Int32VarP(&queryFlags.page, "page", "p", 0, "The page number") - queryCmd.PersistentFlags().StringVarP(&queryFlags.format, "output", "o", "json", "Output format (json|yaml|template|template-file)") - queryCmd.PersistentFlags().StringVarP(&queryFlags.goTemplate, "template", "t", "", "A Go template used to format the output") - queryCmd.Flags().StringVarP(&queryFlags.file, "filename", "f", "", "File name to write to") -} - -type queryDef struct { - Name string - Description string - Query string - Template string - marshalSpec format.Formatter -} - -func getQueryDef(queryArg string) (queryDef, error) { - var queryDef queryDef - - if strings.HasPrefix(queryArg, "r.") { - queryDef.Query = queryArg - } else { - filename, err := findFile(queryArg) - if err != nil { - return queryDef, err - } - log.Debugf("Using query definition from file '%s'", filename) - err = readFile(filename, &queryDef) - if err != nil { - return queryDef, err - } - } - - out, err := format.ResolveWriter(queryFlags.file) - if err != nil { - return queryDef, fmt.Errorf("Could not resolve output '%v': %v", queryFlags.file, err) - } - if queryDef.Template == "" { - queryDef.marshalSpec, err = format.NewFormatter("", out, queryFlags.format, queryFlags.goTemplate) - } else { - queryDef.marshalSpec, err = format.NewFormatter("", out, "template", queryDef.Template) - } - if err != nil { - return queryDef, err - } - return queryDef, nil -} - -func findFile(name string) (string, error) { - filename := name - if _, err := os.Stat(filename); !os.IsNotExist(err) { - return filename, nil - } - - queryDir := configutil.GetConfigDir("query") - - filename = filepath.Join(queryDir, name) - if _, err := os.Stat(filename); !os.IsNotExist(err) { - return filename, nil - } - filename = filepath.Join(queryDir, name) + ".yml" - if _, err := os.Stat(filename); !os.IsNotExist(err) { - return filename, nil - } - filename = filepath.Join(queryDir, name) + ".yaml" - if _, err := os.Stat(filename); !os.IsNotExist(err) { - return filename, nil - } - return "", fmt.Errorf("query not found: %s", name) -} - -func readFile(name string, queryDef *queryDef) error { - data, err := ioutil.ReadFile(name) - if err != nil { - return fmt.Errorf("failed to read file: %s: %w", name, err) - } - // Found file - if strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml") { - yaml.Unmarshal(data, &queryDef) - } else { - queryDef.Query = string(data) - } - queryDef.Name = strings.TrimSuffix(filepath.Base(name), filepath.Ext(name)) - return nil -} - -func listStoredQueries(path string) []queryDef { - var r []queryDef - - if files, err := ioutil.ReadDir(path); err == nil { - for _, f := range files { - if !f.IsDir() { - var q queryDef - readFile(filepath.Join(path, f.Name()), &q) - r = append(r, q) - } - } - } - return r -} diff --git a/src/cmd/root.go b/src/cmd/root.go deleted file mode 100644 index 8709916..0000000 --- a/src/cmd/root.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "github.com/nlnwa/veidemannctl/bindata" - "github.com/nlnwa/veidemannctl/src/cmd/config" - "github.com/nlnwa/veidemannctl/src/cmd/importcmd" - "github.com/nlnwa/veidemannctl/src/cmd/logconfig" - "github.com/nlnwa/veidemannctl/src/cmd/reports" - "github.com/nlnwa/veidemannctl/src/configutil" - "github.com/nlnwa/veidemannctl/src/version" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "os" - "path/filepath" -) - -var ( - cfgFile string - debug bool -) - -// RootCmd represents the base command when called without any subcommands -var RootCmd = &cobra.Command{ - Use: "veidemannctl", - Short: "Veidemann command line client", - Long: `A command line client for Veidemann which can manipulate configs and request status of the crawler.`, - Run: func(cmd *cobra.Command, args []string) { - _ = cmd.Help() - }, - DisableAutoGenTag: true, -} - -// Execute adds all child commands to the root command and sets flags appropriately. -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - RootCmd.Version = version.Version.String() - - data, err := bindata.Asset("completion.sh") - if err != nil { - panic(err) - } - RootCmd.BashCompletionFunction = string(data) - - if err := RootCmd.Execute(); err != nil { - os.Exit(1) - } -} - -func init() { - cobra.OnInitialize(initConfig) - - RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.veidemannctl.yaml)") - - RootCmd.PersistentFlags().StringVar(&configutil.GlobalFlags.Context, "context", "", "The name of the veidemannconfig context to use.") - - RootCmd.PersistentFlags().StringVarP(&configutil.GlobalFlags.ControllerAddress, "controllerAddress", "c", "localhost:50051", "Address to the Controller service") - - RootCmd.PersistentFlags().StringVar(&configutil.GlobalFlags.ServerNameOverride, "serverNameOverride", "", - "If set, it will override the virtual host name of authority (e.g. :authority header field) in requests.") - - RootCmd.PersistentFlags().StringVar(&configutil.GlobalFlags.ApiKey, "apiKey", "", - "Api-key used for authentication instead of interactive logon trough IDP.") - - RootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Turn on debugging") - - RootCmd.PersistentFlags().BoolVar(&configutil.GlobalFlags.IsShellCompletion, "comp", false, "Clean output used for shell completion") - _ = RootCmd.PersistentFlags().MarkHidden("comp") - - RootCmd.SetVersionTemplate("{{.Version}}") - - RootCmd.AddCommand(reports.NewReportCmd()) - RootCmd.AddCommand(logconfig.LogconfigCmd) - RootCmd.AddCommand(config.ConfigCmd) - RootCmd.AddCommand(importcmd.ImportCmd) -} - -// initConfig reads in config file and ENV variables if set. -var contextDir string - -func initConfig() { - if debug { - log.SetLevel(log.DebugLevel) - } - - if cfgFile != "" { - // Use config file from the flag. - viper.SetConfigFile(cfgFile) - } else { - var err error - configutil.GlobalFlags.Context, err = configutil.GetCurrentContext() - if err != nil { - log.Fatalf("Could not get current context: %v", err) - } - contextDir = configutil.GetConfigDir("contexts") - - // Search config in home directory with name ".veidemannctl" (without extension). - viper.AddConfigPath(contextDir) - viper.SetConfigName(configutil.GlobalFlags.Context) - - log.Debug("Using context: ", configutil.GlobalFlags.Context) - } - - viper.AutomaticEnv() // read in environment variables that match - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - log.Debugf("Using config file: %s", viper.ConfigFileUsed()) - } else { - log.Debugf("Setting config file to the non-existing file: %s", filepath.Join(contextDir, configutil.GlobalFlags.Context+".yaml")) - viper.SetConfigFile(filepath.Join(contextDir, configutil.GlobalFlags.Context+".yaml")) - } - - if !RootCmd.PersistentFlags().Changed("controllerAddress") { - configutil.GlobalFlags.ControllerAddress = viper.GetString("controllerAddress") - } - - if !RootCmd.PersistentFlags().Changed("serverNameOverride") { - configutil.GlobalFlags.ServerNameOverride = viper.GetString("serverNameOverride") - } - - if !RootCmd.PersistentFlags().Changed("apiKey") { - configutil.GlobalFlags.ApiKey = viper.GetString("apiKey") - } - log.Debug("Using controller address: ", configutil.GlobalFlags.ControllerAddress) - log.Debug("Using server name override: ", configutil.GlobalFlags.ServerNameOverride) - log.Debug("Using api-key: ", configutil.GlobalFlags.ApiKey) -} diff --git a/src/cmd/status.go b/src/cmd/status.go deleted file mode 100644 index a772c1d..0000000 --- a/src/cmd/status.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "fmt" - "log" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/nlnwa/veidemann-api/go/controller/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/spf13/cobra" -) - -// statusCmd represents the status command -var statusCmd = &cobra.Command{ - Use: "status", - Short: "Get crawler status", - Long: `Get crawler status.`, - Run: func(cmd *cobra.Command, args []string) { - crawlerStatus, err := getCrawlerStatus() - if err != nil { - log.Fatalf("could not get crawler status: %v", err) - } - fmt.Printf("Status: %v, Url queue size: %v, Busy crawl host groups: %v\n", - crawlerStatus.RunStatus, crawlerStatus.QueueSize, crawlerStatus.BusyCrawlHostGroupCount) - }, -} - -func getCrawlerStatus() (*controller.CrawlerStatus, error) { - client, conn := connection.NewControllerClient() - defer func() { - _ = conn.Close() - }() - - request := empty.Empty{} - return client.Status(context.Background(), &request) -} - -func init() { - RootCmd.AddCommand(statusCmd) -} diff --git a/src/cmd/update.go b/src/cmd/update.go deleted file mode 100644 index b85c812..0000000 --- a/src/cmd/update.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - commonsV1 "github.com/nlnwa/veidemann-api/go/commons/v1" - "github.com/nlnwa/veidemannctl/src/apiutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/connection" - "github.com/nlnwa/veidemannctl/src/format" - "golang.org/x/net/context" -) - -var updateflags struct { - label string - name string - filter string - updateField string - pageSize int32 -} - -// updateCmd represents the get command -var updateCmd = &cobra.Command{ - Use: "update [object_type]", - Short: "Update the value(s) for an object type", - Long: `Update one or many objects with new values.`, - Example: `# Add CrawlJob for a seed. -veidemannctl update seed -n "http://www.gwpda.org/" -u seed.jobRef+=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709 - -# Replace all configured CrawlJobs for a seed with a new one. -veidemannctl update seed -n "http://www.gwpda.org/" -u seed.jobRef=crawlJob:e46863ae-d076-46ca-8be3-8a8ef72e709`, - - Args: func(cmd *cobra.Command, args []string) error { - if err := cobra.ExactArgs(1)(cmd, args); err != nil { - msg := "An object type must be specified. %sSee 'veidemannctl update -h' for help" - return fmt.Errorf(msg, printValidObjectTypes()) - } - if err := cobra.OnlyValidArgs(cmd, args); err != nil { - msg := "%s is not a valid object type. %vSee 'veidemannctl update -h' for help" - return fmt.Errorf(msg, args[0], printValidObjectTypes()) - } - return nil - }, - Run: func(cmd *cobra.Command, args []string) { - configClient, conn := connection.NewConfigClient() - defer conn.Close() - - k := format.GetKind(args[0]) - - var ids []string - - if len(args) == 2 { - ids = args[1:] - } - - if k == configV1.Kind_undefined { - fmt.Printf("Unknown object type\n") - _ = cmd.Usage() - return - } - - selector, err := apiutil.CreateListRequest(k, ids, updateflags.name, updateflags.label, updateflags.filter, updateflags.pageSize, 0) - if err != nil { - log.Fatalf("Error creating request: %v", err) - } - - updateMask := new(commonsV1.FieldMask) - updateTemplate := new(configV1.ConfigObject) - if err := apiutil.CreateTemplateFilter(updateflags.updateField, updateTemplate, updateMask); err != nil { - log.Fatalf("Error creating request: %v", err) - } - - updateRequest := &configV1.UpdateRequest{ - ListRequest: selector, - UpdateMask: updateMask, - UpdateTemplate: updateTemplate, - } - - u, err := configClient.UpdateConfigObjects(context.Background(), updateRequest) - if err != nil { - log.Fatalf("Error from controller: %v", err) - } - fmt.Printf("Objects updated: %v\n", u.Updated) - }, - ValidArgs: format.GetObjectNames(), - SilenceUsage: true, - SilenceErrors: true, -} - -func init() { - RootCmd.AddCommand(updateCmd) - - updateCmd.PersistentFlags().StringVarP(&updateflags.label, "label", "l", "", "List objects by label (: | )") - - updateCmd.PersistentFlags().StringVarP(&updateflags.name, "name", "n", "", "List objects by name (accepts regular expressions)") - annotation := make(map[string][]string) - annotation[cobra.BashCompCustom] = []string{"__veidemannctl_get_name"} - updateCmd.PersistentFlags().Lookup("name").Annotations = annotation - - updateCmd.PersistentFlags().StringVarP(&updateflags.filter, "filter", "q", "", "Filter objects by field (i.e. meta.description=foo)") - updateCmd.PersistentFlags().StringVarP(&updateflags.updateField, "updateField", "u", "", "Filter objects by field (i.e. meta.description=foo)") - _ = updateCmd.MarkPersistentFlagRequired("updateField") - updateCmd.PersistentFlags().Int32VarP(&updateflags.pageSize, "limit", "s", 0, "Limit the number of objects to update. 0 = no limit") -} diff --git a/src/configutil/config.go b/src/configutil/config.go deleted file mode 100644 index dbe0f32..0000000 --- a/src/configutil/config.go +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package configutil - -import ( - "fmt" - "github.com/ghodss/yaml" - "github.com/mitchellh/go-homedir" - log "github.com/sirupsen/logrus" - "github.com/spf13/viper" - yaml2 "gopkg.in/yaml.v2" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" -) - -type config struct { - ControllerAddress string `json:"controllerAddress"` - AccessToken string `json:"accessToken"` - Nonce string `json:"nonce"` - RootCAs string `json:"rootCAs"` - ServerNameOverride string `json:"serverNameOverride"` - ApiKey string `json:"apiKey"` -} - -var GlobalFlags struct { - Context string - ControllerAddress string - ServerNameOverride string - ApiKey string - IsShellCompletion bool -} - -func WriteConfig() { - log.Debug("Writing config") - - c := config{ - viper.GetString("controllerAddress"), - viper.GetString("accessToken"), - viper.GetString("nonce"), - viper.GetString("rootCAs"), - viper.GetString("serverNameOverride"), - viper.GetString("apiKey"), - } - - y, err := yaml.Marshal(c) - if err != nil { - log.Fatalf("err: %v\n", err) - } - - f, err := os.Create(viper.ConfigFileUsed()) - if err != nil { - log.Fatalf("Could not create config file '%s': %v", viper.ConfigFileUsed(), err) - } - if err = f.Chmod(0600); err != nil { - log.Fatalf("Could not set access mode on config file '%s': %v", viper.ConfigFileUsed(), err) - } - defer f.Close() - - if _, err = f.Write(y); err != nil { - log.Fatalf("Could write to config file '%s': %v", viper.ConfigFileUsed(), err) - } -} - -func GetConfigDir(subdir string) string { - // Find home directory. - home, err := homedir.Dir() - if err != nil { - log.Fatal(err) - } - - return filepath.Join(home, ".veidemann", subdir) -} - -type context struct { - Context string -} - -func GetCurrentContext() (string, error) { - switch GlobalFlags.Context { - case "": - if GlobalFlags.Context == "kubectl" { - } - contextDir := GetConfigDir("contexts") - log.Debugf("Creating context directory: %s", contextDir) - if err := os.MkdirAll(contextDir, 0777); err != nil { - fmt.Println(err) - os.Exit(1) - } - - contextFile := GetConfigDir("context.yaml") - f, err := os.Open(contextFile) - if err != nil { - err = SetCurrentContext("default") - if err != nil { - return "", err - } - - f, err = os.Open(contextFile) - if err != nil { - log.Fatalf("Could not read file '%s': %v", contextFile, err) - return "", err - } - } - defer f.Close() - - var c context - dec := yaml2.NewDecoder(f) - err = dec.Decode(&c) - if err != nil { - log.Fatalf("Could not read file '%s': %v", contextFile, err) - return "", err - } - - return c.Context, err - case "kubectl": - output, err := exec.Command("kubectl", "config", "current-context").CombinedOutput() - if err != nil { - _, _ = os.Stderr.WriteString(err.Error()) - return "", err - } - return strings.TrimSpace(string(output)), nil - default: - return GlobalFlags.Context, nil - } -} - -func SetCurrentContext(ctx string) error { - contextFile := GetConfigDir("context.yaml") - w, err := os.Create(contextFile) - if err != nil { - log.Fatalf("Could not open or create file '%s': %v", contextFile, err) - return err - } - enc := yaml2.NewEncoder(w) - err = enc.Encode(context{ctx}) - if err != nil { - log.Fatalf("Could not write file '%s': %v", contextFile, err) - return err - } - _ = enc.Close() - _ = w.Close() - - return nil -} - -func ListContexts() ([]string, error) { - var files []string - contextDir := GetConfigDir("contexts") - fileInfo, err := ioutil.ReadDir(contextDir) - if err != nil { - return files, err - } - - for _, file := range fileInfo { - if !file.IsDir() { - sufIdx := strings.LastIndex(file.Name(), ".") - if sufIdx > 0 { - files = append(files, file.Name()[:sufIdx]) - } - } - } - return files, nil -} - -func ContextExists(ctx string) (bool, error) { - cs, err := ListContexts() - if err != nil { - return false, err - } - - for _, c := range cs { - if ctx == c { - return true, nil - } - } - return false, nil -} diff --git a/src/connection/auth.go b/src/connection/auth.go deleted file mode 100644 index 5773068..0000000 --- a/src/connection/auth.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright © 2017 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package connection - -import ( - "context" - "crypto/tls" - "crypto/x509" - "encoding/base64" - "fmt" - "github.com/coreos/go-oidc" - "github.com/ghodss/yaml" - log "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "golang.org/x/oauth2" - "math/rand" - "net" - "net/http" - "os/exec" - "runtime" - "time" -) - -const ( - manualRedirectURI = "urn:ietf:wg:oauth:2.0:oob" - autoRedirectURI = "http://localhost:9876" -) - -// return an HTTP client which trusts the provided root CAs. -func httpClientForRootCAs() *http.Client { - // Create a pool with systems CAs - certPool, err := x509.SystemCertPool() - if err != nil { - log.Warn("Could not read systems trusted certificates, uses only the configured ones") - certPool = x509.NewCertPool() - } - tlsConfig := tls.Config{RootCAs: certPool} - - // Add CAs from config - if viper.GetString("rootCAs") != "" { - rootCABytes := []byte(viper.GetString("rootCAs")) - if !tlsConfig.RootCAs.AppendCertsFromPEM(rootCABytes) { - log.Warn("no certs found in root CA file") - } - } - - return &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tlsConfig, - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - } -} - -type auth struct { - clientID string - clientSecret string - redirectURI string - rawIdToken string - idToken *oidc.IDToken - oauth2Token *oauth2.Token - - idTokenVerifier *oidc.IDTokenVerifier - provider *oidc.Provider - - // Does the provider use "offline_access" scope to request a refresh token - // or does it use "access_type=offline" (e.g. Google)? - offlineAsScope bool - - client *http.Client - ctx context.Context - state string - enabled bool -} - -func NewAuth(idp string) *auth { - a := auth{} - a.enabled = true - a.offlineAsScope = true - a.clientID = "veidemann-cli" - a.clientSecret = "cli-app-secret" - - a.client = httpClientForRootCAs() - - if a.client == nil { - a.client = http.DefaultClient - } - - a.ctx = oidc.ClientContext(context.Background(), a.client) - - if idp == "" { - a.enabled = false - return &a - } - - // Initialize a provider by specifying dex's issuer URL. - p, err := oidc.NewProvider(a.ctx, idp) - if err != nil { - log.Warn(fmt.Errorf("Could not connect to authentication server '%s': %v. Proceeding without authentication", idp, err)) - a.enabled = false - } else { - a.provider = p - oc := oidc.Config{ClientID: a.clientID} - a.idTokenVerifier = a.provider.Verifier(&oc) - } - return &a -} - -func (a *auth) Login(manual bool) { - authCodeURL := a.CreateAuthCodeURL(manual) - var code string - var state string - - if manual { - fmt.Println("Paste this uri in a browser window. Follow the login steps and paste the code here.") - fmt.Println(authCodeURL) - - fmt.Print("Code: ") - fmt.Scan(&code) - state = a.state - } else { - a.Openbrowser(authCodeURL) - code, state = a.listen() - } - a.VerifyCode(code, state) -} - -func (a *auth) oauth2Config() *oauth2.Config { - return &oauth2.Config{ - ClientID: a.clientID, - ClientSecret: a.clientSecret, - Endpoint: a.provider.Endpoint(), - Scopes: []string{oidc.ScopeOpenID, "profile", "email", "groups", "offline_access", "audience:server:client_id:veidemann-api"}, - RedirectURL: a.redirectURI, - } -} - -func (a *auth) CreateAuthCodeURL(manual bool) string { - if !a.enabled { - log.Fatal("Cannot create authentication URL since authentication is not enabled") - } - - a.state = RandStringBytesMaskImprSrc(16) - viper.Set("nonce", a.state) - - nonce := oidc.Nonce(a.state) - if manual { - a.redirectURI = manualRedirectURI - } else { - a.redirectURI = autoRedirectURI - } - return a.oauth2Config().AuthCodeURL(a.state, nonce) -} - -func (a *auth) Openbrowser(authCodeURL string) { - var err error - - switch runtime.GOOS { - case "linux": - err = exec.Command("xdg-open", authCodeURL).Start() - case "windows": - err = exec.Command("rundll32", "url.dll,FileProtocolHandler", authCodeURL).Start() - case "darwin": - err = exec.Command("open", authCodeURL).Start() - default: - err = fmt.Errorf("unsupported platform") - } - if err != nil { - log.Fatal(err) - } -} - -func (a *auth) VerifyCode(code string, state string) { - if a.state != state { - log.Fatal("State is not equal") - } - - oauth2Token, err := a.oauth2Config().Exchange(a.ctx, code) - if err != nil { - log.Fatal(err) - } - a.oauth2Token = oauth2Token - - // Extract the ID Token from OAuth2 token. - rawIDToken, ok := oauth2Token.Extra("id_token").(string) - if !ok { - log.Fatal("No token found") - } - a.rawIdToken = rawIDToken - - a.verifyIdToken() - - viper.Set("accessToken", marshlAccessToken(oauth2Token)) -} - -func (a *auth) verifyIdToken() { - // Parse and verify ID Token payload. - idToken, err := a.idTokenVerifier.Verify(a.ctx, a.rawIdToken) - if err != nil { - log.Fatal(err) - } - log.Debugf("IdToken: %s", idToken) - - if idToken.Nonce != viper.GetString("nonce") { - log.Fatal("Nonce did not match") - } - a.idToken = idToken -} - -func (a *auth) CheckStoredAccessToken() { - accessToken := viper.GetString("accessToken") - if accessToken == "" { - log.Debugf("No accessToken") - return - } - - a.oauth2Token = unmarshalAccessToken(accessToken) - log.Debugf("AccessToken: %v", a.oauth2Token) - - // Extract the ID Token from OAuth2 token. - rawIdToken, ok := a.oauth2Token.Extra("id_token").(string) - if !ok { - log.Fatal("No token found") - } - a.rawIdToken = rawIdToken - - a.verifyIdToken() -} - -// Extract custom claims. -type Claims struct { - Email string `json:"email"` - Verified bool `json:"email_verified"` - Groups []string `json:"groups"` - Name string `json:"name"` -} - -func (a *auth) Claims() (claims Claims) { - if err := a.idToken.Claims(&claims); err != nil { - log.Fatal(err) - } - return claims -} - -type yamlToken struct { - // AccessToken is the token that authorizes and authenticates - // the requests. - AccessToken string - - // TokenType is the type of token. - // The Type method returns either this or "Bearer", the default. - TokenType string - - // RefreshToken is a token that's used by the application - // (as opposed to the user) to refresh the access token - // if it expires. - RefreshToken string - - // Expiry is the optional expiration time of the access token. - // - // If zero, TokenSource implementations will reuse the same - // token forever and RefreshToken or equivalent - // mechanisms for that TokenSource will not be used. - Expiry time.Time - - // Raw optionally contains extra metadata from the server - // when updating a token. - IdToken interface{} -} - -func marshlAccessToken(token *oauth2.Token) string { - tmp := yamlToken{ - AccessToken: token.AccessToken, - TokenType: token.Type(), - RefreshToken: token.RefreshToken, - Expiry: token.Expiry, - IdToken: token.Extra("id_token"), - } - r, err := yaml.Marshal(tmp) - if err != nil { - log.Fatalf("Could not marshal Access Token: %v", err) - } - s := base64.StdEncoding.EncodeToString(r) - return s -} - -func unmarshalAccessToken(accessToken string) *oauth2.Token { - s, err := base64.StdEncoding.DecodeString(accessToken) - if err != nil { - log.Fatalf("Could not unmarshal Access Token: %v", err) - } - var tmp yamlToken - if err := yaml.Unmarshal(s, &tmp); err != nil { - log.Fatalf("Could not unmarshal Access Token: %v", err) - } - - at := &oauth2.Token{ - AccessToken: tmp.AccessToken, - TokenType: tmp.TokenType, - RefreshToken: tmp.RefreshToken, - Expiry: tmp.Expiry, - } - extra := map[string]interface{}{"id_token": tmp.IdToken} - at = at.WithExtra(extra) - - return at -} - -const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1<= 0; { - if remain == 0 { - cache, remain = src.Int63(), letterIdxMax - } - if idx := int(cache & letterIdxMask); idx < len(letterBytes) { - b[i] = letterBytes[idx] - i-- - } - cache >>= letterIdxBits - remain-- - } - - return string(b) -} diff --git a/src/connection/connection.go b/src/connection/connection.go deleted file mode 100644 index f0c103c..0000000 --- a/src/connection/connection.go +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package connection - -import ( - "crypto/x509" - "github.com/golang/protobuf/ptypes/empty" - logV1 "github.com/nlnwa/veidemann-api/go/log/v1" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - controllerV1 "github.com/nlnwa/veidemann-api/go/controller/v1" - reportV1 "github.com/nlnwa/veidemann-api/go/report/v1" - "github.com/nlnwa/veidemannctl/src/configutil" - log "github.com/sirupsen/logrus" - "github.com/spf13/viper" - "golang.org/x/net/context" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "net" - "strings" - "time" -) - -func NewControllerClient() (controllerV1.ControllerClient, *grpc.ClientConn) { - conn := newConnection() - c := controllerV1.NewControllerClient(conn) - return c, conn -} - -func NewReportClient() (reportV1.ReportClient, *grpc.ClientConn) { - conn := newConnection() - c := reportV1.NewReportClient(conn) - return c, conn -} - -func NewLogClient() (logV1.LogClient, *grpc.ClientConn) { - conn := newConnection() - c := logV1.NewLogClient(conn) - return c, conn -} - -func NewConfigClient() (configV1.ConfigClient, *grpc.ClientConn) { - conn := newConnection() - c := configV1.NewConfigClient(conn) - return c, conn -} - -func newConnection() *grpc.ClientConn { - idp, tls := GetIdp() - - // Set up a connection to the server. - conn, _ := connect(idp, tls) - return conn -} - -func connect(idp string, tls bool) (*grpc.ClientConn, bool) { - address := configutil.GlobalFlags.ControllerAddress - apiKey := configutil.GlobalFlags.ApiKey - - dialOptions := []grpc.DialOption{ - grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), - } - - if idp != "" { - dialOptions = addCredentials(idp, apiKey, dialOptions) - } - - // Set up a connection to the server. - creds := clientTransportCredentials(tls) - log.Debugf("connecting to %v", address) - conn, err := blockingDial(context.Background(), address, creds, dialOptions...) - if err != nil { - if strings.Contains(err.Error(), "first record does not look like a TLS handshake") { - log.Debug("Could not connect with TLS, retrying without credentials") - tls = false - conn, err = blockingDial(context.Background(), address, nil, dialOptions...) - if err != nil { - log.Fatalf("Could not connect: %v", err) - } - log.Warn("Connected with insecure transport. Server does not support TLS") - } else { - log.Fatalf("Could not connect: %v", err) - } - } - return conn, tls -} - -func clientTransportCredentials(tls bool) credentials.TransportCredentials { - var creds credentials.TransportCredentials - - if tls { - certPool, err := x509.SystemCertPool() - if err != nil { - log.Fatalf("Failed to create TLS credentials %v", err) - } - - // Add CAs from config - if viper.GetString("rootCAs") != "" { - rootCABytes := []byte(viper.GetString("rootCAs")) - if !certPool.AppendCertsFromPEM(rootCABytes) { - log.Warn("no certs found in root CA file") - } - } - - serverNameOverride := configutil.GlobalFlags.ServerNameOverride - log.Debugf("Using server name override: %s", serverNameOverride) - creds = credentials.NewClientTLSFromCert(certPool, serverNameOverride) - } - - return creds -} - -func GetIdp() (string, bool) { - conn, tls := connect("", true) - defer conn.Close() - - c := controllerV1.NewControllerClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - log.Debug("requesting OpenIdConnectIssuer") - reply, err := c.GetOpenIdConnectIssuer(ctx, &empty.Empty{}) - if err != nil { - log.Fatalf("Could not get idp address: %v", err) - } - - idp := reply.GetOpenIdConnectIssuer() - log.Debugf("using OpenIdConnectIssuer %v", idp) - if idp == "" { - log.Warn("Server was not configured with an idp. Proceeding without authentication") - } - - return idp, tls -} - -type bearerTokenCred struct { - tokenType string - token string -} - -func (b bearerTokenCred) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - return map[string]string{ - "authorization": b.tokenType + " " + b.token, - }, nil -} - -func (b bearerTokenCred) RequireTransportSecurity() bool { - return false -} - -func addCredentials(idp, apiKey string, opts []grpc.DialOption) []grpc.DialOption { - if apiKey != "" { - at := &bearerTokenCred{"ApiKey", apiKey} - opts = append(opts, grpc.WithPerRPCCredentials(at)) - return opts - } - - a := NewAuth(idp) - if !a.enabled { - return opts - } - - a.CheckStoredAccessToken() - if a.oauth2Token != nil { - log.Debugf("Raw IdToken: %s", a.oauth2Token.TokenType) - } - if a.rawIdToken == "" { - return opts - } - - bt := &bearerTokenCred{a.oauth2Token.TokenType, a.rawIdToken} - opts = append(opts, grpc.WithPerRPCCredentials(bt)) - return opts -} - -// BlockingDial is a helper method to dial the given address, using optional TLS credentials, -// and blocking until the returned connection is ready. If the given credentials are nil, the -// connection will be insecure (plain-text). -// This function is borrowed from https://github.com/fullstorydev/grpcurl/blob/master/grpcurl.go -func blockingDial(ctx context.Context, address string, creds credentials.TransportCredentials, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - // grpc.Dial doesn't provide any information on permanent connection errors (like - // TLS handshake failures). So in order to provide good error messages, we need a - // custom dialer that can provide that info. That means we manage the TLS handshake. - result := make(chan interface{}, 1) - - writeResult := func(res interface{}) { - // non-blocking write: we only need the first result - select { - case result <- res: - default: - } - } - - dialer := func(address string, timeout time.Duration) (net.Conn, error) { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - conn, err := (&net.Dialer{Cancel: ctx.Done()}).Dial("tcp", address) - if err != nil { - writeResult(err) - return nil, err - } - if creds != nil { - conn, _, err = creds.ClientHandshake(ctx, address, conn) - if err != nil { - writeResult(err) - return nil, err - } - } - return conn, nil - } - - // Even with grpc.FailOnNonTempDialError, this call will usually timeout in - // the face of TLS handshake errors. So we can't rely on grpc.WithBlock() to - // know when we're done. So we run it in a goroutine and then use result - // channel to either get the channel or fail-fast. - go func() { - opts = append(opts, - grpc.WithBlock(), - grpc.FailOnNonTempDialError(true), - grpc.WithDialer(dialer), - grpc.WithInsecure(), // we are handling TLS, so tell grpc not to - ) - conn, err := grpc.DialContext(ctx, address, opts...) - var res interface{} - if err != nil { - res = err - } else { - res = conn - } - writeResult(res) - }() - - select { - case res := <-result: - if conn, ok := res.(*grpc.ClientConn); ok { - return conn, nil - } else { - return nil, res.(error) - } - case <-ctx.Done(): - return nil, ctx.Err() - } -} diff --git a/src/connection/loginserver.go b/src/connection/loginserver.go deleted file mode 100644 index 6f1e665..0000000 --- a/src/connection/loginserver.go +++ /dev/null @@ -1,41 +0,0 @@ -package connection - -import ( - "fmt" - "golang.org/x/net/context" - "net/http" -) - -type LoginResponseServer struct { - server *http.Server - channel chan string -} - -func (s *LoginResponseServer) handleLoginResponse() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query() - fmt.Fprintf(w, "

%s

", "Window can safely be closed") - go s.setResponse(query.Get("code"), query.Get("state")) - } -} - -func (s *LoginResponseServer) setResponse(code string, state string) { - s.channel <- code - s.channel <- state -} - -func (a *auth) listen() (string, string) { - s := &LoginResponseServer{ - channel: make(chan string), - server: &http.Server{ - Addr: ":9876", - }, - } - - http.HandleFunc("/", s.handleLoginResponse()) - go s.server.ListenAndServe() - //go log.Fatal(s.server.ListenAndServe()) - code, state := <- s.channel, <- s.channel - go s.server.Shutdown(context.Background()) - return code, state -} diff --git a/src/format/formatter.go b/src/format/formatter.go deleted file mode 100644 index 9724a7f..0000000 --- a/src/format/formatter.go +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package format - -import ( - "bufio" - "bytes" - "encoding/json" - "errors" - "fmt" - "github.com/ghodss/yaml" - "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/bindata" - "google.golang.org/protobuf/encoding/protojson" - "io" - "io/ioutil" - "os" - "strings" -) - -var jsonMarshaler = &protojson.MarshalOptions{EmitUnpopulated: true, Indent: " "} -var jsonUnMarshaler = &protojson.UnmarshalOptions{DiscardUnknown: true} - -type Formatter interface { - WriteRecord(record interface{}) error - Close() error -} - -type MarshalSpec struct { - ObjectType string - Format string - Template string - defaultTemplateName string - - rFormat string - rTemplate string - rWriter io.Writer - resolved bool -} - -func NewFormatter(objectType string, out io.Writer, format string, template string) (formatter Formatter, err error) { - if out == nil { - return nil, errors.New("missing writer") - } - - s := &MarshalSpec{ - ObjectType: objectType, - Format: format, - Template: template, - rWriter: out, - } - - if err := s.resolve(); err != nil { - return nil, err - } - - switch s.rFormat { - case "json": - formatter = newJsonFormatter(s) - case "yaml": - formatter = newYamlFormatter(s) - case "template": - formatter, err = newTemplateFormatter(s) - case "table": - formatter, err = newTemplateFormatter(s) - case "wide": - formatter, err = newTemplateFormatter(s) - default: - return nil, errors.New(fmt.Sprintf("Illegal or missing format '%s'", s.rFormat)) - } - return -} - -// ResolveWriter creates a file and returns a io.Writer for the file. -// If filename is empty, os.StdOut is returned -func ResolveWriter(filename string) (io.Writer, error) { - if filename == "" { - return os.Stdout, nil - } else { - f, err := os.Create(filename) - if err != nil { - return nil, errors.New(fmt.Sprintf("Could not create file '%s': %v", filename, err)) - } - return f, nil - } -} - -func (s *MarshalSpec) resolve() (err error) { - if !s.resolved { - switch s.Format { - case "template": - if s.Template == "" { - if s.defaultTemplateName != "" { - data, err := bindata.Asset(s.defaultTemplateName) - if err != nil { - return err - } - s.rTemplate = string(data) - } else { - return errors.New("Format is 'template', but template is missing") - } - } else { - s.rTemplate = s.Template - } - s.rFormat = s.Format - case "template-file": - if s.Template == "" { - return errors.New("format is 'template-file', but template is missing") - } - data, err := ioutil.ReadFile(s.Template) - if err != nil { - return fmt.Errorf("template not found: %v", err) - } - s.rTemplate = string(data) - s.rFormat = s.Format - case "json": - s.rTemplate = "" - s.rFormat = s.Format - case "yaml": - s.rTemplate = "" - s.rFormat = s.Format - case "table": - if s.ObjectType == "" { - return fmt.Errorf("format is table, but object type is missing") - } - - templateName := s.ObjectType + "_table.template" - data, err := bindata.Asset(templateName) - if err != nil { - return err - } - s.rTemplate = string(data) - s.rFormat = s.Format - case "wide": - if s.ObjectType == "" { - return fmt.Errorf("format is wide, but object type is missing") - } - - templateName := s.ObjectType + "_wide.template" - data, err := bindata.Asset(templateName) - if err != nil { - return err - } - s.rTemplate = string(data) - s.rFormat = s.Format - default: - s.rTemplate = s.Template - s.rFormat = s.Format - } - } - s.resolved = true - return nil -} - -func (s *MarshalSpec) Close() error { - if nil != s { - if c, ok := s.rWriter.(io.Closer); ok && c != os.Stdout { - if err := c.Close(); err != nil { - return err - } - } - } - return nil -} - -func Unmarshal(filename string) ([]*config.ConfigObject, error) { - result := make([]*config.ConfigObject, 0, 16) - if filename == "" { - return UnmarshalYaml(os.Stdin, result) - } else { - f, err := os.Open(filename) - if err != nil { - return nil, errors.New(fmt.Sprintf("Could not open file '%s': %v", filename, err)) - } - defer f.Close() - if fi, _ := f.Stat(); fi.IsDir() { - fis, _ := f.Readdir(0) - for _, fi = range fis { - if !fi.IsDir() && HasSuffix(fi.Name(), ".yaml", ".yml", ".json") { - fmt.Println("Reading file: ", fi.Name()) - f, err = os.Open(fi.Name()) - if err != nil { - return nil, errors.New(fmt.Sprintf("Could not open file '%s': %v", filename, err)) - } - defer f.Close() - - if HasSuffix(f.Name(), ".yaml", ".yml") { - result, err = UnmarshalYaml(f, result) - } else { - result, err = UnmarshalJson(f, result) - } - if err != nil { - return nil, err - } - } - } - return result, nil - } else { - if HasSuffix(f.Name(), ".yaml", ".yml") { - return UnmarshalYaml(f, result) - } else { - return UnmarshalJson(f, result) - } - } - } - return result, nil -} - -func ReadYamlDocument(r *bufio.Reader) ([]byte, error) { - delim := []byte{'-', '-', '-'} - var ( - inDoc bool = true - err error = nil - l, doc []byte - ) - for inDoc && err == nil { - isPrefix := true - ln := []byte{} - for isPrefix && err == nil { - l, isPrefix, err = r.ReadLine() - ln = append(ln, l...) - } - - if len(ln) >= 3 && bytes.Equal(delim, ln[:3]) { - inDoc = false - } else { - doc = append(doc, ln...) - doc = append(doc, '\n') - } - } - return doc, err -} - -func UnmarshalYaml(r io.Reader, result []*config.ConfigObject) ([]*config.ConfigObject, error) { - br := bufio.NewReader(r) - - var ( - data []byte - readErr error = nil - ) - for readErr == nil { - data, readErr = ReadYamlDocument(br) - if readErr != nil && readErr != io.EOF { - return nil, readErr - } - - data = bytes.TrimSpace(data) - if len(data) == 0 { - continue - } - - b, err := yaml.YAMLToJSON(data) - if err != nil { - return nil, errors.New(fmt.Sprintf("Error decoding: %v, %v", err, data)) - } - - target := &config.ConfigObject{} - err = jsonUnMarshaler.Unmarshal(b, target) - if err != nil { - return nil, err - } - result = append(result, target) - } - return result, nil -} - -func UnmarshalJson(r io.Reader, result []*config.ConfigObject) ([]*config.ConfigObject, error) { - dec := json.NewDecoder(r) - var err error - for err == nil { - var msg json.RawMessage - err = dec.Decode(&msg) - target := &config.ConfigObject{} - jsonUnMarshaler.Unmarshal(msg, target) - if err == nil { - result = append(result, target) - } - } - - return result, nil -} - -func HasSuffix(s string, suffix ...string) bool { - for _, suf := range suffix { - if strings.HasSuffix(s, suf) { - return true - } - } - return false -} diff --git a/src/importutil/db.go b/src/importutil/db.go deleted file mode 100644 index 64aeefa..0000000 --- a/src/importutil/db.go +++ /dev/null @@ -1,451 +0,0 @@ -// Copyright © 2019 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package importutil - -import ( - "bytes" - "context" - "encoding/gob" - "encoding/json" - "fmt" - "github.com/dgraph-io/badger/v2" - "github.com/dgraph-io/badger/v2/pb" - configV1 "github.com/nlnwa/veidemann-api/go/config/v1" - "github.com/nlnwa/veidemannctl/src/configutil" - log "github.com/sirupsen/logrus" - "io" - "os" - "path" - "sync" - "sync/atomic" - "time" -) - -type ExistsCode int - -const ( - ERROR ExistsCode = iota - NEW - DUPLICATE_NEW - EXISTS_VEIDEMANN - DUPLICATE_VEIDEMANN -) - -func (e ExistsCode) ExistsInVeidemann() bool { - return e == EXISTS_VEIDEMANN || e == DUPLICATE_VEIDEMANN -} - -func (e ExistsCode) String() string { - names := [...]string{ - "ERROR", - "NEW", - "DUPLICATE_NEW", - "EXISTS_VEIDEMANN", - "DUPLICATE_VEIDEMANN"} - if e < ERROR || e > DUPLICATE_VEIDEMANN { - return "UNKNOWN" - } - return names[e] -} - -type ExistsResponse struct { - NormalizedKey string - Code ExistsCode - KnownIds []string -} - -type ImportDb struct { - db *badger.DB - gc *time.Ticker - client configV1.ConfigClient - kind configV1.Kind - keyNormalizer KeyNormalizer - vmMux sync.Mutex -} - -type KeyNormalizer interface { - Normalize(key string) (string, error) -} - -type NoopKeyNormalizer struct { -} - -func (u *NoopKeyNormalizer) Normalize(s string) (key string, err error) { - return s, nil -} - -func NewImportDb(client configV1.ConfigClient, dbDir string, kind configV1.Kind, keyNormalizer KeyNormalizer, resetDb bool) *ImportDb { - kindName := kind.String() - dbDir = path.Join(dbDir, configutil.GlobalFlags.Context, kindName) - if err := os.MkdirAll(dbDir, 0777); err != nil { - log.Fatal(err) - } - opts := badger.DefaultOptions(dbDir) - opts.Logger = log.StandardLogger() - if resetDb { - if err := os.RemoveAll(dbDir); err != nil { - log.Fatal(err) - } - } - db, err := badger.Open(opts) - if err != nil { - log.Fatal(err) - } - - d := &ImportDb{ - db: db, - client: client, - kind: kind, - keyNormalizer: keyNormalizer, - } - - d.gc = time.NewTicker(5 * time.Minute) - go func() { - for range d.gc.C { - again: - err := db.RunValueLogGC(0.7) - if err == nil { - goto again - } - } - }() - - return d -} - -func (d *ImportDb) ImportExisting() { - req := &configV1.ListRequest{ - Kind: d.kind, - } - r, err := d.client.ListConfigObjects(context.Background(), req) - if err != nil { - log.Fatalf("Error reading %s from Veidemann: %v", d.kind.String(), err) - } - - var stat struct { - processed int - imported int - err int - } - - start := time.Now() - for { - msg, err := r.Recv() - if err == io.EOF { - break - } - if err != nil { - log.Fatalf("Error reading %s from Veidemann: %v", d.kind.String(), err) - } - - metaName := msg.GetMeta().GetName() - - exists := d.contains(metaName, msg.Id, true) - switch exists.Code { - case ERROR: - stat.err++ - log.Infof("Failed handling: %v", metaName) - case NEW: - stat.imported++ - log.Debugf("New key '%v' added", exists.NormalizedKey) - case EXISTS_VEIDEMANN: - log.Debugf("Already exists in Veidemann: %v", exists.NormalizedKey) - case DUPLICATE_NEW: - log.Infof("Found new duplicate: %v", exists.NormalizedKey) - case DUPLICATE_VEIDEMANN: - log.Infof("Found duplicate already existing in Veidemann: %v", exists.NormalizedKey) - } - stat.processed++ - } - elapsed := time.Since(start) - fmt.Printf("Processed %v %s from Veidemann in %s. %v imported, %v errors\n", stat.processed, d.kind.String(), elapsed, stat.imported, stat.err) -} - -type SeedDuplicateReportRecord struct { - Host string - Seeds []SeedRecord -} - -type SeedRecord struct { - SeedId string - Uri string - SeedDescription string - EntityId string - EntityName string - EntityDescription string -} - -func (d *ImportDb) SeedDuplicateReport(w io.Writer) error { - stream := d.db.NewStream() - - // -- Optional settings - stream.NumGo = 16 // Set number of goroutines to use for iteration. - stream.LogPrefix = "Badger.Streaming" // For identifying stream logs. Outputs to Logger. - // -- End of optional settings. - - var uniqueDuplicatedNames int32 - var duplicatedNames int32 - - // Send is called serially, while Stream.Orchestrate is running. - stream.Send = func(list *pb.KVList) error { - for _, kv := range list.GetKv() { - val := d.bytesToStringArray(kv.Value) - if len(val) > 1 { - rec := &SeedDuplicateReportRecord{Host: string(kv.Key)} - for _, id := range val { - ref := &configV1.ConfigRef{Id: id, Kind: configV1.Kind_seed} - seed, err := d.client.GetConfigObject(context.Background(), ref) - if err == nil { - sr := SeedRecord{ - SeedId: seed.GetId(), - Uri: seed.GetMeta().GetName(), - SeedDescription: seed.GetMeta().GetDescription(), - EntityId: seed.GetSeed().GetEntityRef().GetId(), - } - rec.Seeds = append(rec.Seeds, sr) - entity, err := d.client.GetConfigObject(context.Background(), seed.GetSeed().GetEntityRef()) - if err == nil { - sr.EntityName = entity.GetMeta().GetName() - sr.EntityDescription = entity.GetMeta().GetDescription() - } else { - log.Warnf("error getting entity from Veidemann: %v", err) - } - } else { - log.Warnf("error getting seed from Veidemann: %v", err) - } - } - atomic.AddInt32(&uniqueDuplicatedNames, 1) - atomic.AddInt32(&duplicatedNames, int32(len(val))) - b, err := json.Marshal(rec) - if err == nil { - d.vmMux.Lock() - if _, err := w.Write(b); err != nil { - log.Warnf("error wirting record: %v", err) - } - if _, err := w.Write([]byte{'\n'}); err != nil { - log.Warnf("error wirting record: %v", err) - } - d.vmMux.Unlock() - } else { - log.Warnf("error formatting json: %v", err) - } - } - } - return nil - } - - // Run the stream - if err := stream.Orchestrate(context.Background()); err != nil { - return err - } - fmt.Printf("%v seed urls exist more than once. Total duplicates: %v.\n", uniqueDuplicatedNames, duplicatedNames) - return nil -} - -type CrawlEntityDuplicateReportRecord struct { - Name string - CrawlEntities []CrawlEntityRecord -} - -type CrawlEntityRecord struct { - Id string -} - -func (d *ImportDb) CrawlEntityDuplicateReport(w io.Writer) error { - stream := d.db.NewStream() - - // -- Optional settings - stream.NumGo = 16 // Set number of goroutines to use for iteration. - stream.LogPrefix = "Badger.Streaming" // For identifying stream logs. Outputs to Logger. - // -- End of optional settings. - - var uniqueDuplicatedNames int32 - var duplicatedNames int32 - - // Send is called serially, while Stream.Orchestrate is running. - stream.Send = func(list *pb.KVList) error { - for _, kv := range list.GetKv() { - val := d.bytesToStringArray(kv.Value) - if len(val) > 1 { - rec := &CrawlEntityDuplicateReportRecord{Name: string(kv.Key)} - for _, id := range val { - sr := CrawlEntityRecord{ - Id: id, - } - rec.CrawlEntities = append(rec.CrawlEntities, sr) - } - atomic.AddInt32(&uniqueDuplicatedNames, 1) - atomic.AddInt32(&duplicatedNames, int32(len(val))) - b, err := json.Marshal(rec) - if err == nil { - d.vmMux.Lock() - if _, err := w.Write(b); err != nil { - log.Warnf("error wirting record: %v", err) - } - if _, err := w.Write([]byte{'\n'}); err != nil { - log.Warnf("error wirting record: %v", err) - } - d.vmMux.Unlock() - } else { - log.Warnf("error formatting json: %v", err) - } - } - } - return nil - } - - // Run the stream - if err := stream.Orchestrate(context.Background()); err != nil { - return err - } - fmt.Printf("%v crawl entities names exist more than once. Total duplicates: %v.\n", uniqueDuplicatedNames, duplicatedNames) - return nil -} - -func (d *ImportDb) Check(metaName string) (*ExistsResponse, error) { - return d.contains(metaName, "", false), nil -} - -func (d *ImportDb) CheckAndUpdate(metaName string) (*ExistsResponse, error) { - return d.contains(metaName, "", true), nil -} - -func (d *ImportDb) CheckAndUpdateVeidemann(metaName string, data interface{}, - createFunc func(client configV1.ConfigClient, data interface{}) (id string, err error)) (*ExistsResponse, error) { - - d.vmMux.Lock() - defer d.vmMux.Unlock() - - exists := d.contains(metaName, "", true) - if !exists.Code.ExistsInVeidemann() { - id, err := createFunc(d.client, data) - if err != nil { - return exists, err - } - d.contains(metaName, id, true) - } - return exists, nil -} - -func (d *ImportDb) contains(metaName, id string, update bool) (response *ExistsResponse) { - response = &ExistsResponse{} - - var err error - response.NormalizedKey, err = d.keyNormalizer.Normalize(metaName) - if err != nil { - response.Code = ERROR - return - } - - for { - err = d.db.Update(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(response.NormalizedKey)) - if err == badger.ErrKeyNotFound { - response.Code = NEW - if id != "" { - response.KnownIds = []string{id} - } - if update { - v := d.stringArrayToBytes(response.KnownIds) - _ = txn.Set([]byte(response.NormalizedKey), v) - } - return nil - } - var val []string - if err == nil { - err := item.Value(func(v []byte) error { - val = d.bytesToStringArray(v) - if len(val) == 0 { - val = nil - } - return nil - }) - if err != nil { - return nil - } - - if id != "" && !d.stringArrayContains(val, id) { - val = append(val, id) - if update { - v := d.stringArrayToBytes(val) - _ = txn.Set([]byte(response.NormalizedKey), v) - } - } - - switch len(val) { - case 0: - response.Code = DUPLICATE_NEW - case 1: - if id == "" || d.stringArrayContains(val, id) { - response.Code = EXISTS_VEIDEMANN - } else { - response.Code = DUPLICATE_VEIDEMANN - } - default: - response.Code = DUPLICATE_VEIDEMANN - } - - response.KnownIds = val - return nil - } - return nil - }) - if err != badger.ErrConflict { - break - } - } - if err != nil { - log.Fatalf("DB err %v", err) - } - - return response -} - -func (d *ImportDb) Close() { - for { - err := d.db.RunValueLogGC(0.7) - if err != nil { - break - } - } - - d.gc.Stop() - _ = d.db.Close() -} - -func (d *ImportDb) stringArrayToBytes(v []string) []byte { - buf := &bytes.Buffer{} - if err := gob.NewEncoder(buf).Encode(v); err != nil { - log.Fatal(err) - } - return buf.Bytes() -} - -func (d *ImportDb) bytesToStringArray(v []byte) []string { - buf := bytes.NewBuffer(v) - strs := []string{} - if err := gob.NewDecoder(buf).Decode(&strs); err != nil && err != io.EOF { - log.Fatal(err) - } - return strs -} - -func (d *ImportDb) stringArrayContains(s []string, e string) bool { - for _, a := range s { - if a == e { - return true - } - } - return false -} diff --git a/src/importutil/executor.go b/src/importutil/executor.go deleted file mode 100644 index 3ac8a07..0000000 --- a/src/importutil/executor.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright © 2019 National Library of Norway -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package importutil - -import ( - "fmt" - "os" - "time" -) - -type executor struct { - Count int - Success int - Failed int - Processor func(value interface{}, state *State) error - ErrHandler func(state *StateVal) - LogHandler func(state *StateVal) - - threadCount int - dataChan chan *StateVal - completeChan chan *processorResponse - errorChan chan *StateVal - errsHandled int -} - -type StateVal struct { - *State - Val interface{} -} - -type processorResponse struct { - count int - success int - failed int -} - -func NewExecutor(threadCount int, proc func(value interface{}, state *State) error, errorHandler func(state *StateVal)) *executor { - e := &executor{ - threadCount: threadCount, - Processor: proc, - ErrHandler: errorHandler, - dataChan: make(chan *StateVal), - completeChan: make(chan *processorResponse), - errorChan: make(chan *StateVal), - } - - go e.handleError() - - for i := 0; i < e.threadCount; i++ { - go e.execute() - } - - return e -} - -func (e *executor) Do(state *State, val interface{}) { - if val != nil { - e.dataChan <- &StateVal{state, val} - e.Count++ - e.printProgress() - } -} - -func (e *executor) Finish() { - close(e.dataChan) - subCompleted := 0 - for i := range e.completeChan { - e.Success += i.success - e.Failed += i.failed - subCompleted++ - if subCompleted >= e.threadCount { - break - } - } - for e.errsHandled < e.Failed { - time.Sleep(time.Second) - } -} - -func (e *executor) execute() { - res := &processorResponse{} - - for request := range e.dataChan { - res.count++ - err := e.Processor(request.Val, request.State) - if err != nil { - request.State.err = err - e.errorChan <- request - res.failed++ - } else { - res.success++ - } - } - e.completeChan <- res -} - -func (e *executor) handleError() { - for err := range e.errorChan { - e.ErrHandler(err) - e.errsHandled++ - } -} - -func (e *executor) printProgress() { - // Print progress - _, _ = fmt.Fprint(os.Stderr, ".") - if e.Count%100 == 0 { - _, _ = fmt.Fprintln(os.Stderr, e.Count) - } -} diff --git a/src/version/version.go b/src/version/version.go deleted file mode 100644 index 9f09cbc..0000000 --- a/src/version/version.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package version - -import ( - "fmt" - "runtime" -) - -type VeidemannVersion struct { - gitVersion string -} - -func (v *VeidemannVersion) SetGitVersion(version string) { - v.gitVersion = version -} - -func (v *VeidemannVersion) String() string { - return fmt.Sprintf("Client version: %s, Go version: %s, Platform: %s/%s\n", - v.gitVersion, - runtime.Version(), - runtime.GOOS, - runtime.GOARCH) -} - -var Version = &VeidemannVersion{} diff --git a/tools.go b/tools.go deleted file mode 100644 index 1e55463..0000000 --- a/tools.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright © 2017 National Library of Norway. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build tools - -// This file ensures the import required for go generate is retained when running go mod tidy. -// See https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module -// and https://github.com/go-modules-by-example/index/blob/master/010_tools/README.md - -package main - -import _ "github.com/shurcooL/vfsgen" diff --git a/version/version.go b/version/version.go new file mode 100644 index 0000000..3cbfe3f --- /dev/null +++ b/version/version.go @@ -0,0 +1,46 @@ +// Copyright © 2017 National Library of Norway. +// Licensed under the Apache License, GitVersion 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "fmt" + "runtime" +) + +var versionInfo Version + +// Version contains version information +type Version struct { + ClientVersion string + GitCommit string + BuildDate string +} + +// Set sets the version information +func Set(gitVersion, gitCommit, buildDate string) { + versionInfo.ClientVersion = gitVersion + versionInfo.GitCommit = gitCommit + versionInfo.BuildDate = buildDate +} + +// String returns a string representation of the version information +func String() string { + return fmt.Sprintf("Client version: %s, Git commit: %s, Build date: %s, Go version: %s, Platform: %s/%s", + versionInfo.ClientVersion, + versionInfo.GitCommit, + versionInfo.BuildDate, + runtime.Version(), + runtime.GOOS, + runtime.GOARCH) +}