From 14962fe6a94f13d9fc0c561af2f602e9ce135cd6 Mon Sep 17 00:00:00 2001 From: Balachandar Mani Date: Tue, 12 Sep 2023 14:00:31 -0700 Subject: [PATCH] CVL database access layer changes (#100) * cvl database access layer changes * cvl database access layer changes to fix test code * cvl database access layer changes to fix benchmark test code * cvl database access layer changes to fix CvlEditConfigData test cases * cvl database access layer - removed the cvl_db_test.go, will be added it after integerating the changes --- cvl/common/cvl_types.go | 57 ++++ cvl/common/db_access.go | 92 ++++++ cvl/custom_validation/common.go | 121 ++++---- translib/db/cvl_db_access.go | 535 ++++++++++++++++++++++++++++++++ translib/tlerr/tlerr.go | 17 +- 5 files changed, 760 insertions(+), 62 deletions(-) create mode 100644 cvl/common/cvl_types.go create mode 100644 cvl/common/db_access.go create mode 100644 translib/db/cvl_db_access.go diff --git a/cvl/common/cvl_types.go b/cvl/common/cvl_types.go new file mode 100644 index 000000000000..f7c5ce7a345c --- /dev/null +++ b/cvl/common/cvl_types.go @@ -0,0 +1,57 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2022 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package common + +import ( + "github.com/antchfx/xmlquery" +) + +// CVLEditConfigData Strcture for key and data in API +type CVLEditConfigData struct { + VType CVLValidateType //Validation type + VOp CVLOperation //Operation type + Key string //Key format : "PORT|Ethernet4" + Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down} + ReplaceOp bool +} + +type CVLValidateType uint + +const ( + VALIDATE_NONE CVLValidateType = iota //Data is used as dependent data + VALIDATE_SYNTAX //Syntax is checked and data is used as dependent data + VALIDATE_SEMANTICS //Semantics is checked + VALIDATE_ALL //Syntax and Semantics are checked +) + +type CVLOperation uint + +const ( + OP_NONE CVLOperation = 0 //Used to just validate the config without any operation + OP_CREATE = 1 << 0 //For Create operation + OP_UPDATE = 1 << 1 //For Update operation + OP_DELETE = 1 << 2 //For Delete operation +) + +// RequestCacheType Struct for request data and YANG data +type RequestCacheType struct { + ReqData CVLEditConfigData + YangData *xmlquery.Node +} diff --git a/cvl/common/db_access.go b/cvl/common/db_access.go new file mode 100644 index 000000000000..431ec61f03e8 --- /dev/null +++ b/cvl/common/db_access.go @@ -0,0 +1,92 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2023 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package common + +// DBAccess is used by cvl and custom validation functions to access the ConfigDB. +// This allows cvl clients to plugin additional data source, like transaction cache, +// into cvl. Most of the interface methods mimic the go-redis APIs. It also defines +// Lookup and Count methods to perform advanced search by matching hash field values. +type DBAccess interface { + Exists(key string) IntResult + Keys(pattern string) StrSliceResult + HGet(key, field string) StrResult + HMGet(key string, fields ...string) SliceResult + HGetAll(key string) StrMapResult + Pipeline() PipeResult + + // Lookup entries using a Search criteria and return them in sonic db json format. + // E.g, {"INTERFACE": {"Ethernet0": {"vrf", "Vrf1"}, "Ethernet0|1.2.3.4": {"NULL": "NULL"}}} + // TODO fix the return value for not found case + Lookup(s Search) JsonResult + // Count entries using a Search criteria. Returns 0 if there are no matches. + Count(s Search) IntResult +} + +type IntResult interface { + Result() (int64, error) +} + +type StrResult interface { + Result() (string, error) +} + +type StrSliceResult interface { + Result() ([]string, error) +} + +type SliceResult interface { + Result() ([]interface{}, error) +} + +type StrMapResult interface { + Result() (map[string]string, error) +} + +type JsonResult interface { + Result() (string, error) //TODO have it as map instead of string +} + +type PipeResult interface { + Keys(pattern string) StrSliceResult + HGet(key, field string) StrResult + HMGet(key string, fields ...string) SliceResult + HGetAll(key string) StrMapResult + Exec() error + Close() +} + +// Search criteria for advanced lookup. Initial filtering is done by matching the key Pattern. +// Results are further refined by applying Predicate, WithField and Limit constraints (optional) +type Search struct { + // Pattern to match the keys from a redis table. Must contain a table name prefix. + // E.g, `INTERFACE|Ethernet0` `INTERFACE|*` "INTERFACE|*|*" + Pattern string + // Predicate is a lua condition statement to inspect an entry's key and hash attributes. + // It can use map variables 'k' and 'h' to access key & hash attributes. + // E.g, `k['type'] == 'L3' and h['enabled'] == true` + Predicate string + // KeyNames must contain the key component names in order. Required only if Predicate uses 'k'. + // E.g, if ["name","type"], a key "ACL|TEST|L3" will expand to lua map {name="TEST", type="L3"} + KeyNames []string + // WithField selects a entry only if it contains this hash field + WithField string + // Limit the results to maximum these number of entries + Limit int +} diff --git a/cvl/custom_validation/common.go b/cvl/custom_validation/common.go index 9625113c73c8..9a61ab2da48a 100644 --- a/cvl/custom_validation/common.go +++ b/cvl/custom_validation/common.go @@ -21,98 +21,106 @@ package custom_validation import ( "reflect" - "github.com/antchfx/xmlquery" - "github.com/go-redis/redis/v7" + + "github.com/Azure/sonic-mgmt-common/cvl/common" "github.com/Azure/sonic-mgmt-common/cvl/internal/util" "github.com/Azure/sonic-mgmt-common/cvl/internal/yparser" - ) + "github.com/antchfx/xmlquery" + "github.com/go-redis/redis/v7" +) -type CustomValidation struct {} +type CustomValidation struct{} type CVLValidateType uint + const ( - VALIDATE_NONE CVLValidateType = iota //Data is used as dependent data - VALIDATE_SYNTAX //Syntax is checked and data is used as dependent data - VALIDATE_SEMANTICS //Semantics is checked - VALIDATE_ALL //Syntax and Semantics are checked + VALIDATE_NONE CVLValidateType = iota //Data is used as dependent data + VALIDATE_SYNTAX //Syntax is checked and data is used as dependent data + VALIDATE_SEMANTICS //Semantics is checked + VALIDATE_ALL //Syntax and Semantics are checked ) type CVLOperation uint + const ( - OP_NONE CVLOperation = 0 //Used to just validate the config without any operation - OP_CREATE = 1 << 0//For Create operation - OP_UPDATE = 1 << 1//For Update operation - OP_DELETE = 1 << 2//For Delete operation + OP_NONE CVLOperation = 0 //Used to just validate the config without any operation + OP_CREATE = 1 << 0 //For Create operation + OP_UPDATE = 1 << 1 //For Update operation + OP_DELETE = 1 << 2 //For Delete operation ) -//CVLRetCode CVL Error codes +// CVLRetCode CVL Error codes type CVLRetCode int + const ( CVL_SUCCESS CVLRetCode = iota CVL_ERROR CVL_NOT_IMPLEMENTED CVL_INTERNAL_UNKNOWN CVL_FAILURE - CVL_SYNTAX_ERROR = CVLRetCode(yparser.YP_SYNTAX_ERROR) - CVL_SEMANTIC_ERROR = CVLRetCode(yparser.YP_SEMANTIC_ERROR) - CVL_SYNTAX_MISSING_FIELD = CVLRetCode(yparser.YP_SYNTAX_MISSING_FIELD) - CVL_SYNTAX_INVALID_FIELD = CVLRetCode(yparser.YP_SYNTAX_INVALID_FIELD) /* Invalid Field */ - CVL_SYNTAX_INVALID_INPUT_DATA = CVLRetCode(yparser.YP_SYNTAX_INVALID_INPUT_DATA) /*Invalid Input Data */ - CVL_SYNTAX_MULTIPLE_INSTANCE = CVLRetCode(yparser.YP_SYNTAX_MULTIPLE_INSTANCE) /* Multiple Field Instances */ - CVL_SYNTAX_DUPLICATE = CVLRetCode(yparser.YP_SYNTAX_DUPLICATE) /* Duplicate Fields */ - CVL_SYNTAX_ENUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID) /* Invalid enum value */ - CVL_SYNTAX_ENUM_INVALID_NAME = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID_NAME) /* Invalid enum name */ - CVL_SYNTAX_ENUM_WHITESPACE = CVLRetCode(yparser.YP_SYNTAX_ENUM_WHITESPACE) /* Enum name with leading/trailing whitespaces */ - CVL_SYNTAX_OUT_OF_RANGE = CVLRetCode(yparser.YP_SYNTAX_OUT_OF_RANGE) /* Value out of range/length/pattern (data) */ - CVL_SYNTAX_MINIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MINIMUM_INVALID) /* min-elements constraint not honored */ - CVL_SYNTAX_MAXIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MAXIMUM_INVALID) /* max-elements constraint not honored */ - CVL_SEMANTIC_DEPENDENT_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_DEPENDENT_DATA_MISSING) /* Dependent Data is missing */ + CVL_SYNTAX_ERROR = CVLRetCode(yparser.YP_SYNTAX_ERROR) + CVL_SEMANTIC_ERROR = CVLRetCode(yparser.YP_SEMANTIC_ERROR) + CVL_SYNTAX_MISSING_FIELD = CVLRetCode(yparser.YP_SYNTAX_MISSING_FIELD) + CVL_SYNTAX_INVALID_FIELD = CVLRetCode(yparser.YP_SYNTAX_INVALID_FIELD) /* Invalid Field */ + CVL_SYNTAX_INVALID_INPUT_DATA = CVLRetCode(yparser.YP_SYNTAX_INVALID_INPUT_DATA) /*Invalid Input Data */ + CVL_SYNTAX_MULTIPLE_INSTANCE = CVLRetCode(yparser.YP_SYNTAX_MULTIPLE_INSTANCE) /* Multiple Field Instances */ + CVL_SYNTAX_DUPLICATE = CVLRetCode(yparser.YP_SYNTAX_DUPLICATE) /* Duplicate Fields */ + CVL_SYNTAX_ENUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID) /* Invalid enum value */ + CVL_SYNTAX_ENUM_INVALID_NAME = CVLRetCode(yparser.YP_SYNTAX_ENUM_INVALID_NAME) /* Invalid enum name */ + CVL_SYNTAX_ENUM_WHITESPACE = CVLRetCode(yparser.YP_SYNTAX_ENUM_WHITESPACE) /* Enum name with leading/trailing whitespaces */ + CVL_SYNTAX_OUT_OF_RANGE = CVLRetCode(yparser.YP_SYNTAX_OUT_OF_RANGE) /* Value out of range/length/pattern (data) */ + CVL_SYNTAX_MINIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MINIMUM_INVALID) /* min-elements constraint not honored */ + CVL_SYNTAX_MAXIMUM_INVALID = CVLRetCode(yparser.YP_SYNTAX_MAXIMUM_INVALID) /* max-elements constraint not honored */ + CVL_SEMANTIC_DEPENDENT_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_DEPENDENT_DATA_MISSING) /* Dependent Data is missing */ CVL_SEMANTIC_MANDATORY_DATA_MISSING = CVLRetCode(yparser.YP_SEMANTIC_MANDATORY_DATA_MISSING) /* Mandatory Data is missing */ - CVL_SEMANTIC_KEY_ALREADY_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_ALREADY_EXIST) /* Key already existing. */ - CVL_SEMANTIC_KEY_NOT_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_NOT_EXIST) /* Key is missing. */ - CVL_SEMANTIC_KEY_DUPLICATE = CVLRetCode(yparser.YP_SEMANTIC_KEY_DUPLICATE) /* Duplicate key. */ - CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID) + CVL_SEMANTIC_KEY_ALREADY_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_ALREADY_EXIST) /* Key already existing. */ + CVL_SEMANTIC_KEY_NOT_EXIST = CVLRetCode(yparser.YP_SEMANTIC_KEY_NOT_EXIST) /* Key is missing. */ + CVL_SEMANTIC_KEY_DUPLICATE = CVLRetCode(yparser.YP_SEMANTIC_KEY_DUPLICATE) /* Duplicate key. */ + CVL_SEMANTIC_KEY_INVALID = CVLRetCode(yparser.YP_SEMANTIC_KEY_INVALID) ) -//CVLEditConfigData Strcture for key and data in API +// CVLEditConfigData Strcture for key and data in API type CVLEditConfigData struct { - VType CVLValidateType //Validation type - VOp CVLOperation //Operation type - Key string //Key format : "PORT|Ethernet4" - Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down} + VType CVLValidateType //Validation type + VOp CVLOperation //Operation type + Key string //Key format : "PORT|Ethernet4" + Data map[string]string //Value : {"alias": "40GE0/28", "mtu" : 9100, "admin_status": down} } -//CVLErrorInfo CVL Error Structure +// CVLErrorInfo CVL Error Structure type CVLErrorInfo struct { - TableName string /* Table having error */ - ErrCode CVLRetCode /* CVL Error return Code. */ - CVLErrDetails string /* CVL Error Message details. */ - Keys []string /* Keys of the Table having error. */ - Value string /* Field Value throwing error */ - Field string /* Field Name throwing error . */ - Msg string /* Detailed error message. */ - ConstraintErrMsg string /* Constraint error message. */ - ErrAppTag string + TableName string /* Table having error */ + ErrCode CVLRetCode /* CVL Error return Code. */ + CVLErrDetails string /* CVL Error Message details. */ + Keys []string /* Keys of the Table having error. */ + Value string /* Field Value throwing error */ + Field string /* Field Name throwing error . */ + Msg string /* Detailed error message. */ + ConstraintErrMsg string /* Constraint error message. */ + ErrAppTag string } type CustValidationCache struct { Data interface{} } -//CustValidationCtxt Custom validation context passed to custom validation function +// CustValidationCtxt Custom validation context passed to custom validation function type CustValidationCtxt struct { - ReqData []CVLEditConfigData //All request data - CurCfg *CVLEditConfigData //Current request data for which validation should be done - YNodeName string //YANG node name - YNodeVal string //YANG node value, leaf-list will have "," separated value - YCur *xmlquery.Node //YANG data tree + ReqData []CVLEditConfigData //All request data + CurCfg *CVLEditConfigData //Current request data for which validation should be done + YNodeName string //YANG node name + YNodeVal string //YANG node value, leaf-list will have "," separated value + YCur *xmlquery.Node //YANG data tree SessCache *CustValidationCache //Session cache, can be used for storing data, persistent in session - RClient *redis.Client //Redis client + RClient *redis.Client //Redis client } -//InvokeCustomValidation Common function to invoke custom validation -//TBD should we do this using GO plugin feature ? -func InvokeCustomValidation(cv *CustomValidation, name string, args... interface{}) CVLErrorInfo { +// Search criteria for advanced lookup through DBAccess APIs +type Search = common.Search + +// InvokeCustomValidation Common function to invoke custom validation +// TBD should we do this using GO plugin feature ? +func InvokeCustomValidation(cv *CustomValidation, name string, args ...interface{}) CVLErrorInfo { inputs := make([]reflect.Value, len(args)) for i := range args { inputs[i] = reflect.ValueOf(args[i]) @@ -129,4 +137,3 @@ func InvokeCustomValidation(cv *CustomValidation, name string, args... interface return CVLErrorInfo{ErrCode: CVL_SUCCESS} } - diff --git a/translib/db/cvl_db_access.go b/translib/db/cvl_db_access.go new file mode 100644 index 000000000000..6d8209cb12a7 --- /dev/null +++ b/translib/db/cvl_db_access.go @@ -0,0 +1,535 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2023 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package db + +import ( + "encoding/json" + "strings" + + "github.com/Azure/sonic-mgmt-common/cvl" + ctypes "github.com/Azure/sonic-mgmt-common/cvl/common" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" + "github.com/go-redis/redis/v7" + log "github.com/golang/glog" +) + +type cvlDBAccess struct { + Db *DB +} + +func (d *DB) NewValidationSession() (*cvl.CVL, error) { + if d == nil || d.Opts.DBNo != ConfigDB { + return nil, tlerr.TranslibDBNotSupported{} + } + + //TODO: INTG_CHANGES: uncomment the below commented line + //if c, status := cvl.ValidationSessOpen(&cvlDBAccess{d}); status != cvl.CVL_SUCCESS { + if c, status := cvl.ValidationSessOpen(); status != cvl.CVL_SUCCESS { + return nil, tlerr.TranslibCVLFailure{Code: int(status)} + } else { + return c, nil + } +} + +func NewValidationSession() (*cvl.CVL, error) { + //TODO: INTG_CHANGES: uncomment the below commented line + //if c, status := cvl.ValidationSessOpen(nil); status != cvl.CVL_SUCCESS { + if c, status := cvl.ValidationSessOpen(); status != cvl.CVL_SUCCESS { + return nil, tlerr.TranslibCVLFailure{Code: int(status)} + } else { + return c, nil + } +} + +func (c *cvlDBAccess) Exists(key string) ctypes.IntResult { + keys, err := c.Keys(key).Result() + switch err.(type) { + case tlerr.TranslibRedisClientEntryNotExist: + err = redis.Nil + } + if len(keys) > 1 { + //TODO have an optimized implementation in DBAL for Exists + return intResult{int64(0), err} + } + return intResult{int64(len(keys)), err} +} + +func (c *cvlDBAccess) Keys(pattern string) ctypes.StrSliceResult { + //TODO: INTG_CHANGES: uncomment the below commented line + //ts, pat := c.Db.redis2ts_key(pattern) + ts, pat := TableSpec{}, Key{} + keys, err := c.Db.GetKeysPattern(&ts, pat) + switch err.(type) { + case tlerr.TranslibRedisClientEntryNotExist: + err = redis.Nil + } + if err != nil { + return strSliceResult{nil, err} + } + keyArr := make([]string, len(keys)) + for i, k := range keys { + keyArr[i] = c.Db.key2redis(&ts, k) + } + return strSliceResult{keyArr, nil} +} + +func (c *cvlDBAccess) HGet(key, field string) ctypes.StrResult { + //TODO have an optimized implementation in DBAL + data, err := c.HGetAll(key).Result() + if err != nil { + return strResult{"", err} + } + if v, ok := data[field]; ok { + return strResult{v, nil} + } else { + return strResult{"", redis.Nil} + } +} + +func (c *cvlDBAccess) HMGet(key string, fields ...string) ctypes.SliceResult { + //TODO have an optimized implementation in DBAL + data, err := c.HGetAll(key).Result() + if err != nil { + return sliceResult{nil, err} + } + + vals := make([]interface{}, len(fields)) + for i, field := range fields { + if v, ok := data[field]; ok { + vals[i] = v + } + } + return sliceResult{vals, nil} +} + +func (c *cvlDBAccess) HGetAll(key string) ctypes.StrMapResult { + //TODO: INTG_CHANGES: uncomment the below commented line + //ts, k := c.Db.redis2ts_key(key) + ts, k := TableSpec{}, Key{} + v, err := c.Db.GetEntry(&ts, k) + switch err.(type) { + case tlerr.TranslibRedisClientEntryNotExist: + err = nil + } + if v.Field == nil { + v.Field = map[string]string{} + } + return mapResult{v.Field, err} +} + +func (c *cvlDBAccess) getTxData(pattern string, incRow bool) ([]byte, error) { + //TODO: INTG_CHANGES: uncomment the below commented line + //ts, dbKey := c.Db.redis2ts_key(pattern) + ts, dbKey := TableSpec{}, Key{} + if log.V(5) { + log.Infof("cvlDBAccess: getTxData: TableSpec: %v, Key: %v", ts, dbKey) + } + + keyVals := make(map[string]map[string]string) + + //TODO: INTG_CHANGES: uncomment the below commented line + //for k := range c.Db.txTsEntryMap[ts.Name] { + // if patternMatch(k, 0, pattern, 0) { + // if len(c.Db.txTsEntryMap[ts.Name][k].Field) > 0 { + // if incRow { + // keyVals[k] = c.Db.txTsEntryMap[ts.Name][k].Field + // } else { + // keyVals[k] = map[string]string{} + // } + // } else { + // keyVals[k] = nil + // } + // } + //} + + return json.Marshal(keyVals) +} + +func (c *cvlDBAccess) Lookup(s ctypes.Search) ctypes.JsonResult { + //TODO: INTG_CHANGES: uncomment the below commented lines + //var count string + //if s.Limit > 0 { + // count = strconv.Itoa(s.Limit) + //} + // + //txEntries, err := c.getTxData(s.Pattern, true) + //if err != nil { + // log.Warningf("cvlDBAccess: Lookup: error in getTxData: %v", err) + // return strResult{"", err} + //} + // + //v, err := cvl.RunLua( + // "filter_entries", + // s.Pattern, + // strings.Join(s.KeyNames, "|"), + // predicateToReturnStmt(s.Predicate), + // "", // Select fields -- not used by the lua script + // count, + // txEntries, + //) + //if err != nil { + // return strResult{"", err} + //} + //return strResult{v.(string), nil} + return nil +} + +func (c *cvlDBAccess) Count(s ctypes.Search) ctypes.IntResult { + //TODO: INTG_CHANGES: uncomment the below commented lines + //incRow := len(s.Predicate) > 0 || len(s.WithField) > 0 + //txEntries, err := c.getTxData(s.Pattern, incRow) + //if err != nil { + // log.Warningf("cvlDBAccess: Count: error in getTxData: %v", err) + // return intResult{0, err} + //} + //// Advanced key search, with match criteria on has values + //v, err := cvl.RunLua( + // "count_entries", + // s.Pattern, + // strings.Join(s.KeyNames, "|"), + // predicateToReturnStmt(s.Predicate), + // s.WithField, + // txEntries, + //) + //if err != nil { + // return intResult{0, err} + //} + //return intResult{v.(int64), nil} + return nil +} + +func (c *cvlDBAccess) Pipeline() ctypes.PipeResult { + pipe := c.Db.client.Pipeline() + if log.V(5) { + log.Infof("cvlDBAccess: Pipeline: redis pipeline: %v", pipe) + } + return &dbAccessPipe{dbAccess: c, rp: pipe} +} + +func predicateToReturnStmt(p string) string { + if len(p) == 0 || strings.HasPrefix(p, "return") { + return p + } + return "return (" + p + ")" +} + +//================================== + +type strResult struct { + val string + err error +} + +func (r strResult) Result() (string, error) { + return r.val, r.err +} + +//================================== + +type strSliceResult struct { + val []string + err error +} + +func (r strSliceResult) Result() ([]string, error) { + return r.val, r.err +} + +//================================== + +type sliceResult struct { + val []interface{} + err error +} + +func (r sliceResult) Result() ([]interface{}, error) { + return r.val, r.err +} + +//================================== + +type mapResult struct { + val map[string]string + err error +} + +func (r mapResult) Result() (map[string]string, error) { + return r.val, r.err +} + +//================================== + +type intResult struct { + val int64 + err error +} + +func (ir intResult) Result() (int64, error) { + return ir.val, ir.err +} + +//================================== + +type dbAccessPipe struct { + rp redis.Pipeliner + qryResList []pipeQueryResult + dbAccess *cvlDBAccess +} + +type pipeQueryResult interface { + update(c *cvlDBAccess) // to update the db results with cache +} + +type pipeKeysResult struct { + pattern string + sRes strSliceResult + rsRes *redis.StringSliceCmd +} + +func (p *dbAccessPipe) Keys(pattern string) ctypes.StrSliceResult { + if log.V(5) { + log.Infof("dbAccessPipe: Keys: for the given pattern: %v", pattern) + } + + pr := &pipeKeysResult{pattern: pattern, rsRes: p.rp.Keys(pattern)} + p.qryResList = append(p.qryResList, pr) + return &pr.sRes +} + +func (pr *pipeKeysResult) update(c *cvlDBAccess) { + if log.V(5) { + log.Infof("pipeQuery: update: key pattern: %v; redis pipe result: %v", pr.pattern, pr.rsRes) + } + + keys, err := pr.rsRes.Result() + if err != nil { + log.Warningf("pipeKeysResult: update: error in Keys pipe query: keys: %v; err: %v", keys, err) + pr.sRes.err = err + return + } + + keyMap := make(map[string]bool) + for i := 0; i < len(keys); i++ { + keyMap[keys[i]] = true + } + + //TODO: INTG_CHANGES: uncomment the below commented line + //ts, dbKey := c.Db.redis2ts_key(pr.pattern) + //if log.V(5) { + // log.Infof("dbAccessPipe: TableSpec: %v, Key: %v", ts, dbKey) + //} + // + //for k := range c.Db.txTsEntryMap[ts.Name] { + // if patternMatch(k, 0, pr.pattern, 0) { + // if keyMap[k] { + // if len(c.Db.txTsEntryMap[ts.Name][k].Field) == 0 { + // keyMap[k] = false + // } + // } else if len(c.Db.txTsEntryMap[ts.Name][k].Field) > 0 { + // keyMap[k] = true + // } + // } + //} + + for k, v := range keyMap { + if v { + pr.sRes.val = append(pr.sRes.val, k) + } + } + + if len(pr.sRes.val) == 0 { + pr.sRes.val = make([]string, 0) + } +} + +type pipeHMGetResult struct { + key string + fields []string + sRes sliceResult + rsRes *redis.SliceCmd + vals []interface{} +} + +func (p *dbAccessPipe) HMGet(key string, fields ...string) ctypes.SliceResult { + if log.V(5) { + log.Infof("dbAccessPipe: HMGet for the given key: %v, and the fields: %v", key, fields) + } + + //TODO: INTG_CHANGES: uncomment the below commented line + //ts, dbKey := p.dbAccess.Db.redis2ts_key(key) + //if log.V(5) { + // log.Infof("dbAccessPipe: HMGet: TableSpec: %v, Key: %v", ts, dbKey) + //} + // + pr := &pipeHMGetResult{key: key, fields: fields} + // + //if txEntry, ok := p.dbAccess.Db.txTsEntryMap[ts.Name][key]; ok { + // for _, fn := range fields { + // if fv, ok := txEntry.Field[fn]; ok { + // pr.vals = append(pr.vals, fv) + // } else { + // pr.vals = append(pr.vals, nil) + // } + // } + //} else { + // pr.rsRes = p.rp.HMGet(key, fields...) + //} + + p.qryResList = append(p.qryResList, pr) + return &pr.sRes +} + +func (pr *pipeHMGetResult) update(c *cvlDBAccess) { + if log.V(5) { + log.Infof("pipeHMGetResult: update: key: %v; fields: %v; "+ + "redis result: %v; cache result: %v", pr.key, pr.fields, pr.rsRes, pr.vals) + } + if pr.rsRes != nil { + pr.sRes.val = pr.rsRes.Val() + pr.sRes.err = pr.rsRes.Err() + } else { + pr.sRes.val = pr.vals + pr.sRes.err = nil + } +} + +type pipeHGetResult struct { + key string + field string + sRes strResult + rsRes *redis.StringCmd + val string + fldExist bool +} + +func (p *dbAccessPipe) HGet(key, field string) ctypes.StrResult { + if log.V(5) { + log.Infof("dbAccessPipe: HGet for the given key: %v, and the field: %v", key, field) + } + + pr := &pipeHGetResult{key: key, field: field} + + //TODO: INTG_CHANGES: uncomment the below commented lines + //ts, dbKey := p.dbAccess.Db.redis2ts_key(key) + //if log.V(5) { + // log.Infof("dbAccessPipe: HGet: TableSpec: %v, Key: %v", ts, dbKey) + //} + + //if txEntry, ok := p.dbAccess.Db.txTsEntryMap[ts.Name][key]; ok { + // pr.val, pr.fldExist = txEntry.Field[field] + //} else { + // pr.rsRes = p.rp.HGet(key, field) + //} + + p.qryResList = append(p.qryResList, pr) + return &pr.sRes +} + +func (pr *pipeHGetResult) update(c *cvlDBAccess) { + if log.V(5) { + log.Infof("pipeHGetResult: update: key: %v; field: %v; "+ + "redis result: %v; cache result: %v", pr.key, pr.field, pr.rsRes, pr.val) + } + if pr.rsRes != nil { + pr.sRes.val = pr.rsRes.Val() + pr.sRes.err = pr.rsRes.Err() + } else if !pr.fldExist { + pr.sRes.err = redis.Nil + } else { + pr.sRes.val = pr.val + } +} + +type pipeHGetAllResult struct { + key string + sRes mapResult + rsRes *redis.StringStringMapCmd + fnvMap map[string]string +} + +func (p *dbAccessPipe) HGetAll(key string) ctypes.StrMapResult { + if log.V(5) { + log.Infof("dbAccessPipe: HGetAll for the given key: %v", key) + } + + pr := &pipeHGetAllResult{key: key, fnvMap: make(map[string]string)} + + //TODO: INTG_CHANGES: uncomment the below commented lines + //ts, dbKey := p.dbAccess.Db.redis2ts_key(key) + //if log.V(5) { + // log.Infof("dbAccessPipe: HGetAll: TableSpec: %v, Key: %v", ts, dbKey) + //} + + //if txEntry, ok := p.dbAccess.Db.txTsEntryMap[ts.Name][key]; ok { + // for k, v := range txEntry.Field { + // pr.fnvMap[k] = v + // } + //} else { + // pr.rsRes = p.rp.HGetAll(key) + //} + + p.qryResList = append(p.qryResList, pr) + return &pr.sRes +} + +func (pr *pipeHGetAllResult) update(c *cvlDBAccess) { + if log.V(5) { + log.Infof("pipeHGetAllResult: update: key: %v; "+ + "redis result: %v; cache result: %v", pr.key, pr.rsRes, pr.fnvMap) + } + if pr.rsRes != nil { + pr.sRes.val = pr.rsRes.Val() + pr.sRes.err = pr.rsRes.Err() + } else { + pr.sRes.val = pr.fnvMap + pr.sRes.err = nil + } +} + +func (p *dbAccessPipe) Exec() error { + if log.V(5) { + log.Infof("dbAccessPipe: Exec: query list: %v", p.qryResList) + } + + cmder, err := p.rp.Exec() + if err != nil && err != redis.Nil { + log.Warningf("dbAccessPipe: Exec: error in pipeline.Exec; error: %v; "+ + "cmder: %v; pw.qryMap: %v", err, cmder, p.qryResList) + } + + // update the pipe query results with db cache + for _, pr := range p.qryResList { + pr.update(p.dbAccess) + } + + if log.V(5) { + log.Infof("dbAccessPipe: updated: pipe query list: %v; cmder: %v; error: %v", p.qryResList, cmder, err) + } + return err +} + +func (p *dbAccessPipe) Close() { + if log.V(5) { + log.Infof("dbAccessPipe: Close: redis pipeliner: %v", p.rp) + } + p.rp.Close() +} + +//================================== diff --git a/translib/tlerr/tlerr.go b/translib/tlerr/tlerr.go index 6bad578eb7f5..ea49a32854f9 100644 --- a/translib/tlerr/tlerr.go +++ b/translib/tlerr/tlerr.go @@ -26,7 +26,6 @@ returns the English version, and are only meant for log files. For message strings that are returned to the users, the localization will happen at when the GNMI/REST client's locale is known. Hence, it cannot occur here. - */ package tlerr @@ -109,8 +108,8 @@ func (e TranslibSyntaxValidationError) Error() string { } type TranslibUnsupportedClientVersion struct { - ClientVersion string - ServerVersion string + ClientVersion string + ServerVersion string ServerBaseVersion string } @@ -119,11 +118,11 @@ func (e TranslibUnsupportedClientVersion) Error() string { } type TranslibXfmrRetError struct { - XlateFailDelReq bool + XlateFailDelReq bool } func (e TranslibXfmrRetError) Error() string { - return p.Sprintf("Translib transformer return %s", e.XlateFailDelReq) + return p.Sprintf("Translib transformer return %s", e.XlateFailDelReq) } type TranslibDBConnectionReset struct { @@ -132,3 +131,11 @@ type TranslibDBConnectionReset struct { func (e TranslibDBConnectionReset) Error() string { return p.Sprintf("Translib Redis Error: DB Connection Reset") } + +type TranslibDBNotSupported struct { + Description string +} + +func (e TranslibDBNotSupported) Error() string { + return p.Sprintf("Translib Redis Error: Not Supported: %s", e.Description) +}