Skip to content

Commit

Permalink
[patch] add storage backup option to agent
Browse files Browse the repository at this point in the history
Signed-off-by: kpango <i.can.feel.gravity@gmail.com>
  • Loading branch information
kpango committed May 11, 2020
1 parent 54fba1f commit 5d3fd4f
Show file tree
Hide file tree
Showing 13 changed files with 520 additions and 145 deletions.
1 change: 1 addition & 0 deletions Makefile.d/functions.mk
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ define gen-test
gotests -w -template_dir $${path} -all $${file}; \
done; \
done
rm $(ROOTDIR)/internal/core/ngt/*test.go
endef

define fix-test
Expand Down
14 changes: 14 additions & 0 deletions charts/vald/templates/agent/statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,19 @@ spec:
priorityClassName: {{ .Values.agent.name }}-priority
{{- end }}
{{- end }}
{{- if .Values.agent.volumeStore }}
{{- if .Values.agent.volumeStore.enabled }}
volumeClaimTemplates:
- metadata:
name: {{ .Values.agent.name }}-pvc
spec:
accessModes:
- {{ .Values.agent.volumeStore.accessMode }}
storageClassName: {{ .Values.agent.volumeStore.storageClass }}
resources:
requests:
storage: {{ .Values.agent.volumeStore.size }}
{{- end }}
{{- end }}
status:
{{- end }}
9 changes: 9 additions & 0 deletions charts/vald/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,15 @@ agent:
terminationGracePeriodSeconds: 30
# agent.podManagementPolicy -- pod management policy: OrderedReady or Parallel
podManagementPolicy: OrderedReady
volumeStore:
# agent.volumeStore.enabled -- enables agent pod's file store functionality when non in-memory mode
enabled: false
# agent.volumeStore.accessMode -- agent pod storage accessMode
accessMode: ReadWriteOnce
# agent.volumeStore.storageClass -- storageClass name for agent pod volume
storageClass: vald-sc
# agent.volumeStore.size -- size of agent pod volume
size: 100Gi
podPriority:
# agent.podPriority.enabled -- agent pod PriorityClass enabled
enabled: true
Expand Down
15 changes: 8 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ go 1.14
replace (
github.com/boltdb/bolt => github.com/boltdb/bolt v1.3.1
github.com/envoyproxy/protoc-gen-validate => github.com/envoyproxy/protoc-gen-validate v0.3.0-java
github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae
github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20200410100145-b454769479c6
github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.5.1-0.20200509130255-343c8030d2cb
github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20200505093417-effcbd8bcf0e
github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.1
github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.10.0
github.com/gorilla/mux => github.com/gorilla/mux v1.7.4
golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0+incompatible
golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37
k8s.io/api => k8s.io/api v0.18.2
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.2
k8s.io/apimachinery => k8s.io/apimachinery v0.18.2
Expand Down Expand Up @@ -46,18 +47,18 @@ require (
github.com/lucasb-eyer/go-colorful v1.0.3
github.com/pierrec/lz4/v3 v3.3.2
github.com/scylladb/gocqlx v1.5.0
github.com/tensorflow/tensorflow v2.1.0+incompatible
github.com/tensorflow/tensorflow v0.0.0-00010101000000-000000000000
github.com/yahoojapan/gongt v0.0.0-20190517050727-966dcc7aa5e8
github.com/yahoojapan/ngtd v0.0.0-20200424071638-9872bbae3700
go.opencensus.io v0.22.3
go.uber.org/automaxprocs v1.3.0
go.uber.org/goleak v1.0.0
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d // indirect
gonum.org/v1/hdf5 v0.0.0-20191105085658-fe04b73f3b53
golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5 // indirect
gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614
gonum.org/v1/netlib v0.0.0-20200317120129-c5a04cffd98a // indirect
gonum.org/v1/plot v0.7.0
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31
google.golang.org/grpc v1.29.1
gopkg.in/yaml.v2 v2.2.8
k8s.io/api v0.18.2
Expand Down
26 changes: 12 additions & 14 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,11 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae h1:L6V0ANsMIMdLgXly241UXhXNFWYgXbgjHupTAAURrV0=
github.com/go-sql-driver/mysql v1.5.1-0.20200311113236-681ffa848bae/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.5.1-0.20200509130255-343c8030d2cb h1:CgPA5m8ozn8gM7yL2Ohz/HLfRCtJ6mbOpA5KC+nZ4vQ=
github.com/go-sql-driver/mysql v1.5.1-0.20200509130255-343c8030d2cb/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gocql/gocql v0.0.0-20200410100145-b454769479c6 h1:esX8kOWgz5dyrcn/QQQgMmPqPWzX/hNO65/nmfcEOYw=
github.com/gocql/gocql v0.0.0-20200410100145-b454769479c6/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
github.com/gocql/gocql v0.0.0-20200505093417-effcbd8bcf0e h1:k4NKEJg2i2J7l6y44DHiq+U2ua0vvLUKbpktGA4UhQM=
github.com/gocql/gocql v0.0.0-20200505093417-effcbd8bcf0e/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
github.com/gocraft/dbr/v2 v2.7.0 h1:x+UnhSBYPFBBdtikLSMLQ9KPuquSUj4yBijsQAhhNZo=
github.com/gocraft/dbr/v2 v2.7.0/go.mod h1:wQdbxPBSloo2OlSedMxfNW0mgk0GXys9O1VFmQiwcx4=
github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
Expand Down Expand Up @@ -485,8 +485,8 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.14.0 h1:/pduUoebOeeJzTDFuoMgC6nRkiasr1sBCIEorly7m4o=
go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw=
golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -596,10 +596,8 @@ golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200501005904-d351ea090f9b h1:2hSR2MyOaYEy6yJYg/CpErymr/m7xJEJpm9kfT7ZMg4=
golang.org/x/tools v0.0.0-20200501005904-d351ea090f9b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY=
golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5 h1:MeC2gMlMdkd67dn17MEby3rGXRxZtWeiRXOnISfTQ74=
golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
Expand All @@ -611,8 +609,8 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ
gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
gonum.org/v1/hdf5 v0.0.0-20190227001252-83207889d689/go.mod h1:g+PDU5ogjIKcc3Cg4ALAK7X4c8bBQvPzPKWNW5NB7I0=
gonum.org/v1/hdf5 v0.0.0-20191105085658-fe04b73f3b53 h1:y/s3CWVbrGN3fN1yT9VD5+5iox1bYrXh3sXnXQk+N5k=
gonum.org/v1/hdf5 v0.0.0-20191105085658-fe04b73f3b53/go.mod h1:g+PDU5ogjIKcc3Cg4ALAK7X4c8bBQvPzPKWNW5NB7I0=
gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614 h1:6opGC6JlNr8Efmo4HPh6P7JFrNL/8VH2/prV5esilyk=
gonum.org/v1/hdf5 v0.0.0-20200504100616-496fefe91614/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/netlib v0.0.0-20200317120129-c5a04cffd98a h1:y158/g9tKwBGw9gnNENlUIi9NTJCoiQg2RFB1gr9atQ=
gonum.org/v1/netlib v0.0.0-20200317120129-c5a04cffd98a/go.mod h1:6EVtvAMWMjOBOsTVX0xrjO4A6ULtEgWtAWHzqxDWdJs=
Expand All @@ -630,8 +628,8 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84 h1:pSLkPbrjnPyLDYUO2VM9mDLqo2V6CFBY84lFSZAfoi4=
google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU=
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand Down
1 change: 1 addition & 0 deletions hack/go.mod.default
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ replace (
github.com/gocql/gocql => github.com/gocql/gocql master
github.com/gogo/protobuf => github.com/gogo/protobuf master
github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.10.0
github.com/tensorflow/tensorflow => github.com/tensorflow/tensorflow v2.1.0
github.com/gorilla/mux => github.com/gorilla/mux master
golang.org/x/crypto => golang.org/x/crypto master
k8s.io/api => k8s.io/api v0.18.2
Expand Down
14 changes: 8 additions & 6 deletions internal/core/ngt/ngt.go
Original file line number Diff line number Diff line change
Expand Up @@ -435,14 +435,16 @@ func (n *ngt) CreateIndex(poolSize uint32) error {

// SaveIndex stores NGT index to storage.
func (n *ngt) SaveIndex() error {
n.mu.RLock()
ret := C.ngt_save_index(n.index, C.CString(n.idxPath), n.ebuf)
if ret == ErrorCode {
ne := n.ebuf
if !n.inMemory {
n.mu.Lock()
ret := C.ngt_save_index(n.index, C.CString(n.idxPath), n.ebuf)
if ret == ErrorCode {
ne := n.ebuf
n.mu.Unlock()
return n.newGoError(ne)
}
n.mu.Unlock()
return n.newGoError(ne)
}
n.mu.RUnlock()

return nil
}
Expand Down
57 changes: 57 additions & 0 deletions internal/file/file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
//
// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package file provides file I/O functionality
package file

import (
"os"
"path/filepath"
)

func Open(path string, flg int, perm os.FileMode) *os.File {
if path == "" {
return nil
}

var err error
var file *os.File
if _, err = os.Stat(path); err != nil {
if _, err = os.Stat(filepath.Dir(path)); err != nil {
err = os.MkdirAll(filepath.Dir(path), perm)
if err != nil {
return nil
}
}
file, err = os.Create(path)
if err != nil {
return nil
}

err = file.Close()
if err != nil {
return nil
}
}

file, err = os.OpenFile(path, flg, perm)

if err != nil {
return nil
}

return file
}
104 changes: 104 additions & 0 deletions internal/file/file_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
//
// Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// Package file provides file I/O functionality
package file

import (
"os"
"reflect"
"testing"

"github.com/vdaas/vald/internal/errors"
"go.uber.org/goleak"
)

func TestOpen(t *testing.T) {
type args struct {
path string
flg int
perm os.FileMode
}
type want struct {
want *os.File
}
type test struct {
name string
args args
want want
checkFunc func(want, *os.File) error
beforeFunc func(args)
afterFunc func(args)
}
defaultCheckFunc := func(w want, got *os.File) error {
if !reflect.DeepEqual(got, w.want) {
return errors.Errorf("got = %v, want %v", got, w.want)
}
return nil
}
tests := []test{
// TODO test cases
/*
{
name: "test_case_1",
args: args {
path: "",
flg: 0,
perm: nil,
},
want: want{},
checkFunc: defaultCheckFunc,
},
*/

// TODO test cases
/*
func() test {
return test {
name: "test_case_2",
args: args {
path: "",
flg: 0,
perm: nil,
},
want: want{},
checkFunc: defaultCheckFunc,
}
}(),
*/
}

for _, test := range tests {
t.Run(test.name, func(tt *testing.T) {
defer goleak.VerifyNone(t)
if test.beforeFunc != nil {
test.beforeFunc(test.args)
}
if test.afterFunc != nil {
defer test.afterFunc(test.args)
}
if test.checkFunc == nil {
test.checkFunc = defaultCheckFunc
}

got := Open(test.args.path, test.args.flg, test.args.perm)
if err := test.checkFunc(test.want, got); err != nil {
tt.Errorf("error = %v", err)
}

})
}
}
4 changes: 2 additions & 2 deletions pkg/agent/ngt/handler/grpc/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ func (s *server) SaveIndex(ctx context.Context, _ *payload.Empty) (res *payload.
}
}()
res = new(payload.Empty)
err = s.ngt.SaveIndex()
err = s.ngt.SaveIndex(ctx)
if err != nil {
log.Errorf("[SaveIndex]\tUnknown error\t%+v", err)
if span != nil {
Expand All @@ -394,7 +394,7 @@ func (s *server) CreateAndSaveIndex(ctx context.Context, c *payload.Control_Crea
}
}()
res = new(payload.Empty)
err = s.ngt.CreateAndSaveIndex(c.GetPoolSize())
err = s.ngt.CreateAndSaveIndex(ctx, c.GetPoolSize())
if err != nil {
log.Errorf("[CreateAndSaveIndex]\tUnknown error\t%+v", err)
if span != nil {
Expand Down
Loading

0 comments on commit 5d3fd4f

Please sign in to comment.