From 4acb2c3d87cfe3a34e399c810a7328353293c7fb Mon Sep 17 00:00:00 2001 From: Kwan Date: Wed, 11 Nov 2020 11:34:39 -0800 Subject: [PATCH] fix bugs and enhance Transformer (#35) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimize transformer for GET performance For any request URI and its children, table-name & db-key are queried for the first time and cached internally by the transformer infra. This cache block will be subsequently referred by the infra to avoid redundant callback invocations while the yang model tree is being traversed for GET operation. The cache block is local & valid for a single transaction. Data from Redis DB is queried only once for a table & cached in the infra for further reference. This cache is local & valid for a single transaction On-change, sample based subscription support for gNMI On receipt of subscription request uri, the transformer infra translates it to corresponding Redis Db-table and key, using YANG extensions and provides this info along with other subscription parameters to the translib, which takes care of sending notifications on-change of table/key. YANG extensions introduced : "subscribe-preference", "subscribe-on-change", "subscribe-min-interval". rpc callback support RPC statements defined in both OpenConfig and SONiC yang can be annotated with YANG extension “rpc-callback” to invoke the callback to take an action for the given rpc node YANG default value support Infra has been enhanced to fill the oc yang defaults by referring internal yang-metadata for terminal nodes, if it is not present in the request payload. These defaults will be pushed along with other payload data when a resource is created. RFC compliance support - resource check etc. Parent resource is checked for CRUD and GET requests for the incoming URI, before processing the request. If the parent resource is not available in DB, the transaction is rejected with 404 / "Resource not found" error. Cascade delete to de-configure all related nodes Cascade delete enables dependent config clean up for a DELETE request, based on SONiC yang table dependency. YANG extension "cascade-delete" can be annotated to a node that can be deleted along with all dependent nodes. value xfmr callback The value-xfmr callback supports data conversion before writing into the DB for CRUD request and after reading from DB for GET request. Value-xfmr annotated for a leaf will also be invoked, for all other leaves having a leaf-ref to it. The value-xfmr callback is valid only for the SONIC yang. Operation and tables ordering, dependent tables watch list For a set of DB table/instances, grouped by operation, resulting from processing a request URI, infra supports the following : CRUD Operation ordering, sequencing tables before operating based on sonic-yang/Redis dependencies providing translib, with a list of Redis-tables to watch for changes, when performing operations on dependent tables. Yang model capability Provide yang module name, organization and version information to translib in order to serve model capability request. Miscellaneous bug fixes --- translib/common_app.go | 688 ++++++-- translib/tlerr/tlerr.go | 15 +- translib/transformer/transformer.go | 63 +- translib/transformer/xconst.go | 39 + translib/transformer/xfmr_acl.go | 976 ----------- .../transformer/xfmr_dbtbl_cbk_interface.go | 53 + translib/transformer/xfmr_interface.go | 190 ++- translib/transformer/xfmr_path_utils.go | 66 +- translib/transformer/xlate.go | 646 ++++++-- translib/transformer/xlate_cascade_del.go | 151 ++ translib/transformer/xlate_datastructs.go | 109 ++ translib/transformer/xlate_del_to_db.go | 888 ++++++++++ translib/transformer/xlate_from_db.go | 1107 +++++++++---- translib/transformer/xlate_to_db.go | 1469 +++++++++++++---- translib/transformer/xlate_utils.go | 1118 ++++++++++--- translib/transformer/xlate_xfmr_handler.go | 316 ++++ translib/transformer/xspec.go | 723 +++++++- translib/utils/utils.go | 71 + 18 files changed, 6455 insertions(+), 2233 deletions(-) delete mode 100644 translib/transformer/xfmr_acl.go create mode 100644 translib/transformer/xfmr_dbtbl_cbk_interface.go create mode 100644 translib/transformer/xlate_cascade_del.go create mode 100644 translib/transformer/xlate_datastructs.go create mode 100644 translib/transformer/xlate_del_to_db.go create mode 100644 translib/transformer/xlate_xfmr_handler.go create mode 100644 translib/utils/utils.go diff --git a/translib/common_app.go b/translib/common_app.go index 6f57ddabf..725ab7707 100644 --- a/translib/common_app.go +++ b/translib/common_app.go @@ -20,26 +20,32 @@ package translib import ( "errors" - "fmt" "strings" log "github.com/golang/glog" "github.com/openconfig/ygot/ygot" + "github.com/openconfig/ygot/ytypes" + "github.com/openconfig/ygot/util" "reflect" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" "github.com/Azure/sonic-mgmt-common/translib/tlerr" "github.com/Azure/sonic-mgmt-common/translib/transformer" - "encoding/json" + "github.com/Azure/sonic-mgmt-common/translib/utils" + "sync" ) var () type CommonApp struct { pathInfo *PathInfo + body []byte ygotRoot *ygot.GoStruct ygotTarget *interface{} - cmnAppTableMap map[string]map[string]db.Value - cmnAppOrdTbllist []string + skipOrdTableChk bool + cmnAppTableMap map[int]map[db.DBNum]map[string]map[string]db.Value + cmnAppYangDefValMap map[string]map[string]db.Value + cmnAppYangAuxMap map[string]map[string]db.Value + appOptions } var cmnAppInfo = appInfo{appType: reflect.TypeOf(CommonApp{}), @@ -57,13 +63,24 @@ func init() { log.Fatal("Register Common app module with App Interface failed with error=", err, "for path=", mdl_pth) } } - + mdlCpblt := transformer.AddModelCpbltInfo() + if mdlCpblt == nil { + log.Warning("Failure in fetching model capabilities data.") + } else { + for yngMdlNm, mdlDt := range(mdlCpblt) { + err := addModel(&ModelData{Name: yngMdlNm, Org: mdlDt.Org, Ver: mdlDt.Ver}) + if err != nil { + log.Warningf("Adding model data for module %v to appinterface failed with error=%v", yngMdlNm, err) + } + } + } } func (app *CommonApp) initialize(data appData) { log.Info("initialize:path =", data.path) pathInfo := NewPathInfo(data.path) - *app = CommonApp{pathInfo: pathInfo, ygotRoot: data.ygotRoot, ygotTarget: data.ygotTarget} + *app = CommonApp{pathInfo: pathInfo, body: data.payload, ygotRoot: data.ygotRoot, ygotTarget: data.ygotTarget, skipOrdTableChk: false} + app.appOptions = data.appOptions } @@ -112,15 +129,65 @@ func (app *CommonApp) translateGet(dbs [db.MaxDB]*db.DB) error { return err } -func (app *CommonApp) translateAction(dbs [db.MaxDB]*db.DB) error { - err := errors.New("Not supported") - return err +func (app *CommonApp) translateSubscribe(dbs [db.MaxDB]*db.DB, path string) (*notificationOpts, *notificationInfo, error) { + var err error + var subscDt transformer.XfmrTranslateSubscribeInfo + var notifInfo notificationInfo + var notifOpts notificationOpts + txCache := new(sync.Map) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + + log.Info("tranlateSubscribe:path", path) + subscDt, err = transformer.XlateTranslateSubscribe(path, dbs, txCache) + if subscDt.PType == transformer.OnChange { + notifOpts.pType = OnChange + } else { + notifOpts.pType = Sample + } + notifOpts.mInterval = subscDt.MinInterval + notifOpts.isOnChangeSupported = subscDt.OnChange + if err != nil { + log.Infof("returning: notificationOpts - %v, nil, error - %v", notifOpts, err) + return ¬ifOpts, nil, err + } + if subscDt.DbDataMap == nil { + log.Infof("DB data is nil so returning: notificationOpts - %v, nil, error - %v", notifOpts, err) + return ¬ifOpts, nil, err + } else { + for dbNo, dbDt := range(subscDt.DbDataMap) { + if (len(dbDt) == 0) { //ideally all tables for a given uri should be from same DB + continue + } + log.Infof("Adding to notifInfo, Db Data - %v for DB No - %v", dbDt, dbNo) + notifInfo.dbno = dbNo + // in future there will be, multi-table in a DB, support from translib, for now its just single table + for tblNm, tblDt := range(dbDt) { + notifInfo.table = db.TableSpec{Name:tblNm} + if (len(tblDt) == 1) { + for tblKy := range(tblDt) { + notifInfo.key = asKey(tblKy) + notifInfo.needCache = subscDt.NeedCache + } + } else { + if (len(tblDt) > 1) { + log.Warningf("More than one DB key found for subscription path - %v", path) + } else { + log.Warningf("No DB key found for subscription path - %v", path) + } + return ¬ifOpts, nil, err + } + + } + } + } + log.Infof("For path - %v, returning: notifOpts - %v, notifInfo - %v, error - nil", path, notifOpts, notifInfo) + return ¬ifOpts, ¬ifInfo, nil } -func (app *CommonApp) translateSubscribe(dbs [db.MaxDB]*db.DB, path string) (*notificationOpts, *notificationInfo, error) { - err := errors.New("Not supported") - notifInfo := notificationInfo{dbno: db.ConfigDB} - return nil, ¬ifInfo, err +func (app *CommonApp) translateAction(dbs [db.MaxDB]*db.DB) error { + var err error + log.Info("translateAction:path =", app.pathInfo.Path, app.body) + return err } func (app *CommonApp) processCreate(d *db.DB) (SetResponse, error) { @@ -131,7 +198,7 @@ func (app *CommonApp) processCreate(d *db.DB) (SetResponse, error) { targetType := reflect.TypeOf(*app.ygotTarget) log.Infof("processCreate: Target object is a <%s> of Type: %s", targetType.Kind().String(), targetType.Elem().Name()) if err = app.processCommon(d, CREATE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } @@ -143,7 +210,7 @@ func (app *CommonApp) processUpdate(d *db.DB) (SetResponse, error) { var resp SetResponse log.Info("processUpdate:path =", app.pathInfo.Path) if err = app.processCommon(d, UPDATE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } @@ -155,7 +222,7 @@ func (app *CommonApp) processReplace(d *db.DB) (SetResponse, error) { var resp SetResponse log.Info("processReplace:path =", app.pathInfo.Path) if err = app.processCommon(d, REPLACE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } return resp, err @@ -165,10 +232,10 @@ func (app *CommonApp) processDelete(d *db.DB) (SetResponse, error) { var err error var resp SetResponse - log.Info("processDelete:path =", app.pathInfo.Path) + log.Infof("processDelete:path = %s, deleteEmptyEntry = %v", app.pathInfo.Path, app.deleteEmptyEntry) if err = app.processCommon(d, DELETE); err != nil { - log.Error(err) + log.Warning(err) resp = SetResponse{ErrSrc: AppErr} } @@ -180,145 +247,292 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB) (GetResponse, error) { var payload []byte var resPayload []byte log.Info("processGet:path =", app.pathInfo.Path) + txCache := new(sync.Map) - payload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, app.ygotRoot, dbs) - if err != nil { - log.Error("transformer.transformer.GetAndXlateFromDB failure. error:", err) - return GetResponse{Payload: payload, ErrSrc: AppErr}, err - } + for { + origXfmrYgotRoot, _ := ygot.DeepCopy((*app.ygotRoot).(ygot.GoStruct)) - targetObj, _ := (*app.ygotTarget).(ygot.GoStruct) - if targetObj != nil { - err = ocbinds.Unmarshal(payload, targetObj) + isEmptyPayload := false + appYgotStruct := (*app.ygotRoot).(ygot.GoStruct) + payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache) if err != nil { - log.Error("ocbinds.Unmarshal() failed. error:", err) - return GetResponse{Payload: payload, ErrSrc: AppErr}, err + log.Warning("transformer.GetAndXlateFromDB() returned : ", err) + resPayload = payload + break + } + if strings.HasPrefix(app.pathInfo.Path, "/sonic") && isEmptyPayload { + log.Info("transformer.GetAndXlateFromDB() returned EmptyPayload") + resPayload = payload + break } - resPayload, err = generateGetResponsePayload(app.pathInfo.Path, (*app.ygotRoot).(*ocbinds.Device), app.ygotTarget) - if err != nil { - log.Error("generateGetResponsePayload() failed") - return GetResponse{Payload: payload, ErrSrc: AppErr}, err + targetObj, tgtObjCastOk := (*app.ygotTarget).(ygot.GoStruct) + if !tgtObjCastOk { + /*For ygotTarget populated by tranlib, for query on leaf level and list(without instance) level, + casting to GoStruct fails so use the parent node of ygotTarget to Unmarshall the payload into*/ + log.Infof("Use GetParentNode() instead of casting ygotTarget to GoStruct, uri - %v", app.pathInfo.Path) + targetUri := app.pathInfo.Path + parentTargetObj, _, getParentNodeErr := getParentNode(&targetUri, (*app.ygotRoot).(*ocbinds.Device)) + if getParentNodeErr != nil { + log.Warningf("getParentNode() failure for uri %v", app.pathInfo.Path) + resPayload = payload + break + } + if parentTargetObj != nil { + targetObj, tgtObjCastOk = (*parentTargetObj).(ygot.GoStruct) + if !tgtObjCastOk { + log.Warningf("Casting of parent object returned from getParentNode() to GoStruct failed(uri - %v)", app.pathInfo.Path) + resPayload = payload + break + } + } else { + log.Warningf("getParentNode() returned a nil Object for uri %v", app.pathInfo.Path) + resPayload = payload + break + } + } + if targetObj != nil { + updateListEntriesOpt := ytypes.AllowUpdateInListMap{} + err = ocbinds.Unmarshal(payload, targetObj, &updateListEntriesOpt) + if err != nil { + log.Warning("ocbinds.Unmarshal() returned : ", err) + resPayload = payload + break + } + + resYgot := (*app.ygotRoot) + if !strings.HasPrefix(app.pathInfo.Path, "/sonic") { + if isEmptyPayload { + if areEqual(appYgotStruct, origXfmrYgotRoot) { + log.Info("origXfmrYgotRoot and appYgotStruct are equal.") + // No data available in appYgotStruct. + if transformer.IsLeafNode(app.pathInfo.Path) { + //if leaf not exist in DB subtree won't fill ygotRoot, as per RFC return err + resPayload = payload + log.Info("No data found for leaf.") + err = tlerr.NotFound("Resource not found") + break + } + resPayload = payload + log.Info("No data available") + //TODO: Return not found error + //err = tlerr.NotFound("Resource not found") + break + + } + resYgot = appYgotStruct + } + } + if resYgot != nil { + resPayload, err = generateGetResponsePayload(app.pathInfo.Path, resYgot.(*ocbinds.Device), app.ygotTarget) + if err != nil { + log.Warning("generateGetResponsePayload() couldn't generate payload.") + resPayload = payload + } + } else { + resPayload = payload + } + + break + } else { + log.Warning("processGet. targetObj is null. Unable to Unmarshal payload") + resPayload = payload + break } - var dat map[string]interface{} - err = json.Unmarshal(resPayload, &dat) - } else { - log.Warning("processGet. targetObj is null. Unable to Unmarshal payload") - resPayload = payload } return GetResponse{Payload: resPayload}, err } func (app *CommonApp) processAction(dbs [db.MaxDB]*db.DB) (ActionResponse, error) { - var resp ActionResponse - err := errors.New("Not implemented") + var resp ActionResponse + var err error + + resp.Payload, err = transformer.CallRpcMethod(app.pathInfo.Path, app.body, dbs) + log.Info("transformer.CallRpcMethod() returned") - return resp, err + return resp, err } func (app *CommonApp) translateCRUDCommon(d *db.DB, opcode int) ([]db.WatchKeys, error) { var err error var keys []db.WatchKeys var tblsToWatch []*db.TableSpec - var OrdTblList []string - var moduleNm string + txCache := new(sync.Map) log.Info("translateCRUDCommon:path =", app.pathInfo.Path) - /* retrieve schema table order for incoming module name request */ - moduleNm, err = transformer.GetModuleNmFromPath(app.pathInfo.Path) - if (err != nil) || (len(moduleNm) == 0) { - log.Error("GetModuleNmFromPath() failed") - return keys, err - } - log.Info("getModuleNmFromPath() returned module name = ", moduleNm) - OrdTblList, err = transformer.GetOrdDBTblList(moduleNm) - if (err != nil) || (len(OrdTblList) == 0) { - log.Error("GetOrdDBTblList() failed") - return keys, err - } - - log.Info("GetOrdDBTblList() returned ordered table list = ", OrdTblList) - app.cmnAppOrdTbllist = OrdTblList - - /* enhance this to handle dependent tables - need CVL to provide list of such tables for a given request */ - for _, tblnm := range OrdTblList { // OrdTblList already has has all tables corresponding to a module - tblsToWatch = append(tblsToWatch, &db.TableSpec{Name: tblnm}) - } - log.Info("Tables to watch", tblsToWatch) - - cmnAppInfo.tablesToWatch = tblsToWatch - // translate yang to db - result, err := transformer.XlateToDb(app.pathInfo.Path, opcode, d, (*app).ygotRoot, (*app).ygotTarget) - fmt.Println(result) - log.Info("transformer.XlateToDb() returned", result) + result, defValMap, auxMap, err := transformer.XlateToDb(app.pathInfo.Path, opcode, d, (*app).ygotRoot, (*app).ygotTarget, (*app).body, txCache, &app.skipOrdTableChk) + log.Info("transformer.XlateToDb() returned result DB map - ", result, "\nDefault value Db Map - ", defValMap, "\nAux Db Map - ", auxMap) + if err != nil { - log.Error(err) + log.Warning(err) return keys, err } + app.cmnAppTableMap = result + app.cmnAppYangDefValMap = defValMap + app.cmnAppYangAuxMap = auxMap //used for Replace case if len(result) == 0 { - log.Error("XlatetoDB() returned empty map") - err = errors.New("transformer.XlatetoDB() returned empty map") + log.Info("XlatetoDB() returned empty map") + //Note: Get around for no redis ABNF Schema for set(temporary) + //`err = errors.New("transformer.XlatetoDB() returned empty map") return keys, err } - app.cmnAppTableMap = result - keys, err = app.generateDbWatchKeys(d, false) + moduleNm, err := transformer.GetModuleNmFromPath(app.pathInfo.Path) + if (err != nil) || (len(moduleNm) == 0) { + log.Warning("GetModuleNmFromPath() couldn't fetch module name.") + return keys, err + } + + var resultTblList []string + for _, dbMap := range result { //Get dependency list for all tables in result + for _, resMap := range dbMap { //Get dependency list for all tables in result + for tblnm := range resMap { //Get dependency list for all tables in result + resultTblList = append(resultTblList, tblnm) + } + } + } + log.Info("Result Tables List", resultTblList) + + // Get list of tables to watch + if len(resultTblList) > 0 { + depTbls := transformer.GetTablesToWatch(resultTblList, moduleNm) + if len(depTbls) == 0 { + log.Warningf("Couldn't get Tables to watch for module %v", moduleNm) + err = errors.New("GetTablesToWatch returned empty slice") + return keys, err + } + for _, tbl := range depTbls { + tblsToWatch = append(tblsToWatch, &db.TableSpec{Name: tbl}) + } + } + log.Info("Tables to watch", tblsToWatch) + cmnAppInfo.tablesToWatch = tblsToWatch + keys, err = app.generateDbWatchKeys(d, false) return keys, err } func (app *CommonApp) processCommon(d *db.DB, opcode int) error { var err error + if len(app.cmnAppTableMap) == 0 { + return err + } log.Info("Processing DB operation for ", app.cmnAppTableMap) switch opcode { case CREATE: log.Info("CREATE case") - err = app.cmnAppCRUCommonDbOpn(d, opcode) case UPDATE: log.Info("UPDATE case") - err = app.cmnAppCRUCommonDbOpn(d, opcode) case REPLACE: log.Info("REPLACE case") - err = app.cmnAppCRUCommonDbOpn(d, opcode) case DELETE: log.Info("DELETE case") - err = app.cmnAppDelDbOpn(d, opcode) } - if err != nil { - log.Info("Returning from processCommon() - fail") - } else { - log.Info("Returning from processCommon() - success") + + // Handle delete first if any available + if _, ok := app.cmnAppTableMap[DELETE][db.ConfigDB]; ok { + err = app.cmnAppDelDbOpn(d, DELETE, app.cmnAppTableMap[DELETE][db.ConfigDB]) + if err != nil { + log.Info("Process delete fail. cmnAppDelDbOpn error:", err) + return err + } + } + // Handle create operation next + if _, ok := app.cmnAppTableMap[CREATE][db.ConfigDB]; ok { + err = app.cmnAppCRUCommonDbOpn(d, CREATE, app.cmnAppTableMap[CREATE][db.ConfigDB]) + if err != nil { + log.Info("Process create fail. cmnAppCRUCommonDbOpn error:", err) + return err + } + } + // Handle update and replace operation next + if _, ok := app.cmnAppTableMap[UPDATE][db.ConfigDB]; ok { + err = app.cmnAppCRUCommonDbOpn(d, UPDATE, app.cmnAppTableMap[UPDATE][db.ConfigDB]) + if err != nil { + log.Info("Process update fail. cmnAppCRUCommonDbOpn error:", err) + return err + } + } + if _, ok := app.cmnAppTableMap[REPLACE][db.ConfigDB]; ok { + err = app.cmnAppCRUCommonDbOpn(d, REPLACE, app.cmnAppTableMap[REPLACE][db.ConfigDB]) + if err != nil { + log.Info("Process replace fail. cmnAppCRUCommonDbOpn error:", err) + return err + } } + log.Info("Returning from processCommon() - success") return err } -func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { +func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int, dbMap map[string]map[string]db.Value) error { var err error var cmnAppTs *db.TableSpec + var xfmrTblLst []string + var resultTblLst []string - /* currently ordered by schema table order needs to be discussed */ - for _, tblNm := range app.cmnAppOrdTbllist { + for tblNm := range(dbMap) { + xfmrTblLst = append(xfmrTblLst, tblNm) + } + resultTblLst, err = utils.SortAsPerTblDeps(xfmrTblLst) + if err != nil { + return err + } + + /* CVL sorted order is in child first, parent later order. CRU ops from parent first order */ + for idx := len(resultTblLst)-1; idx >= 0; idx-- { + tblNm := resultTblLst[idx] log.Info("In Yang to DB map returned from transformer looking for table = ", tblNm) - if tblVal, ok := app.cmnAppTableMap[tblNm]; ok { + if tblVal, ok := dbMap[tblNm]; ok { cmnAppTs = &db.TableSpec{Name: tblNm} log.Info("Found table entry in yang to DB map") + if ((tblVal == nil) || (len(tblVal) == 0)) { + log.Info("No table instances/rows found.") + continue + } for tblKey, tblRw := range tblVal { - log.Info("Processing Table key and row ", tblKey, tblRw) + log.Info("Processing Table key ", tblKey) + // REDIS doesn't allow to create a table instance without any fields + if tblRw.Field == nil { + tblRw.Field = map[string]string{"NULL": "NULL"} + } + if len(tblRw.Field) == 0 { + tblRw.Field["NULL"] = "NULL" + } + if len(tblRw.Field) > 1 { + delete(tblRw.Field, "NULL") + } existingEntry, _ := d.GetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) switch opcode { case CREATE: if existingEntry.IsPopulated() { - log.Info("Entry already exists hence return.") - return tlerr.AlreadyExists("Entry %s already exists", tblKey) + log.Info("Create case - Entry ", tblKey, " already exists hence modifying it.") + /* Handle leaf-list merge if any leaf-list exists + A leaf-list field in redis has "@" suffix as per swsssdk convention. + */ + resTblRw := db.Value{Field: map[string]string{}} + resTblRw = checkAndProcessLeafList(existingEntry, tblRw, UPDATE, d, tblNm, tblKey) + log.Info("Processing Table row ", resTblRw) + err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) + if err != nil { + log.Warning("CREATE case - d.ModEntry() failure") + return err + } } else { + if tblRwDefaults, defaultOk := app.cmnAppYangDefValMap[tblNm][tblKey]; defaultOk { + log.Info("Entry ", tblKey, " doesn't exist so fill defaults - ", tblRwDefaults) + for fld, val := range tblRwDefaults.Field { + tblRw.Field[fld] = val + } + } + log.Info("Processing Table row ", tblRw) err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) if err != nil { - log.Error("CREATE case - d.CreateEntry() failure") + log.Warning("CREATE case - d.CreateEntry() failure") return err } } @@ -326,37 +540,82 @@ func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { if existingEntry.IsPopulated() { log.Info("Entry already exists hence modifying it.") /* Handle leaf-list merge if any leaf-list exists - A leaf-list field in redis has "@" suffix as per swsssdk convention. - */ + A leaf-list field in redis has "@" suffix as per swsssdk convention. + */ resTblRw := db.Value{Field: map[string]string{}} resTblRw = checkAndProcessLeafList(existingEntry, tblRw, UPDATE, d, tblNm, tblKey) err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) if err != nil { - log.Error("UPDATE case - d.ModEntry() failure") + log.Warning("UPDATE case - d.ModEntry() failure") return err } } else { - // workaround to patch operation from CLI - log.Info("Create(pathc) an entry.") - err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) + // workaround to patch operation from CLI + log.Info("Create(pathc) an entry.") + if tblRwDefaults, defaultOk := app.cmnAppYangDefValMap[tblNm][tblKey]; defaultOk { + log.Info("Entry ", tblKey, " doesn't exist so fill defaults - ", tblRwDefaults) + for fld, val := range tblRwDefaults.Field { + tblRw.Field[fld] = val + } + } + log.Info("Processing Table row ", tblRw) + err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) if err != nil { - log.Error("UPDATE case - d.CreateEntry() failure") + log.Warning("UPDATE case - d.CreateEntry() failure") return err } } case REPLACE: + if tblRwDefaults, defaultOk := app.cmnAppYangDefValMap[tblNm][tblKey]; defaultOk { + log.Info("For entry ", tblKey, ", being replaced, fill defaults - ", tblRwDefaults) + for fld, val := range tblRwDefaults.Field { + tblRw.Field[fld] = val + } + } + log.Info("Processing Table row ", tblRw) if existingEntry.IsPopulated() { - log.Info("Entry already exists hence execute db.SetEntry") - err := d.SetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) - if err != nil { - log.Error("REPLACE case - d.SetEntry() failure") - return err + log.Info("Entry already exists.") + auxRwOk := false + auxRw := db.Value{Field: map[string]string{}} + auxRw, auxRwOk = app.cmnAppYangAuxMap[tblNm][tblKey] + log.Info("Process Aux row ", auxRw) + isTlNd := false + if !strings.HasPrefix(app.pathInfo.Path, "/sonic") { + isTlNd, err = transformer.IsTerminalNode(app.pathInfo.Path) + log.Info("transformer.IsTerminalNode() returned - ", isTlNd, " error ", err) + if err != nil { + return err + } + } + if isTlNd && isPartialReplace(existingEntry, tblRw, auxRw) { + log.Info("Since its partial replace modifying fields - ", tblRw) + err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) + if err != nil { + log.Warning("REPLACE case - d.ModEntry() failure") + return err + } + if auxRwOk { + if len(auxRw.Field) > 0 { + log.Info("Since its partial replace delete aux fields - ", auxRw) + err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, auxRw) + if err != nil { + log.Warning("REPLACE case - d.DeleteEntryFields() failure") + return err + } + } + } + } else { + err := d.SetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) + if err != nil { + log.Warning("REPLACE case - d.SetEntry() failure") + return err + } } } else { log.Info("Entry doesn't exist hence create it.") err = d.CreateEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, tblRw) if err != nil { - log.Error("REPLACE case - d.CreateEntry() failure") + log.Warning("REPLACE case - d.CreateEntry() failure") return err } } @@ -367,26 +626,64 @@ func (app *CommonApp) cmnAppCRUCommonDbOpn(d *db.DB, opcode int) error { return err } -func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { +func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int, dbMap map[string]map[string]db.Value) error { var err error var cmnAppTs, dbTblSpec *db.TableSpec + var moduleNm string + var xfmrTblLst []string + var resultTblLst []string + var ordTblList []string - /* needs enhancements from CVL to give table dependencies, and grouping of related tables only - if such a case where the sonic yang has unrelated tables */ - for tblidx, tblNm := range app.cmnAppOrdTbllist { + for tblNm := range(dbMap) { + xfmrTblLst = append(xfmrTblLst, tblNm) + } + resultTblLst, err = utils.SortAsPerTblDeps(xfmrTblLst) + if err != nil { + return err + } + + + /* Retrieve module Name */ + moduleNm, err = transformer.GetModuleNmFromPath(app.pathInfo.Path) + if (err != nil) || (len(moduleNm) == 0) { + log.Warning("GetModuleNmFromPath() failed") + return err + } + log.Info("getModuleNmFromPath() returned module name = ", moduleNm) + + /* resultTblLst has child first, parent later order */ + for _, tblNm := range resultTblLst { log.Info("In Yang to DB map returned from transformer looking for table = ", tblNm) - if tblVal, ok := app.cmnAppTableMap[tblNm]; ok { + if tblVal, ok := dbMap[tblNm]; ok { cmnAppTs = &db.TableSpec{Name: tblNm} log.Info("Found table entry in yang to DB map") + if !app.skipOrdTableChk { + ordTblList = transformer.GetXfmrOrdTblList(tblNm) + if len(ordTblList) == 0 { + ordTblList = transformer.GetOrdTblList(tblNm, moduleNm) + } + if len(ordTblList) == 0 { + log.Warning("GetOrdTblList returned empty slice") + err = errors.New("GetOrdTblList returned empty slice. Insufficient information to process request") + return err + } + log.Infof("GetOrdTblList for table - %v, module %v returns %v", tblNm, moduleNm, ordTblList) + } if len(tblVal) == 0 { log.Info("DELETE case - No table instances/rows found hence delete entire table = ", tblNm) - for idx := len(app.cmnAppOrdTbllist)-1; idx >= tblidx+1; idx-- { - log.Info("Since parent table is to be deleted, first deleting child table = ", app.cmnAppOrdTbllist[idx]) - dbTblSpec = &db.TableSpec{Name: app.cmnAppOrdTbllist[idx]} - err = d.DeleteTable(dbTblSpec) - if err != nil { - log.Warning("DELETE case - d.DeleteTable() failure for Table = ", app.cmnAppOrdTbllist[idx]) - return err + if !app.skipOrdTableChk { + for _, ordtbl := range ordTblList { + if ordtbl == tblNm { + // Handle the child tables only till you reach the parent table entry + break + } + log.Info("Since parent table is to be deleted, first deleting child table = ", ordtbl) + dbTblSpec = &db.TableSpec{Name: ordtbl} + err = d.DeleteTable(dbTblSpec) + if err != nil { + log.Warning("DELETE case - d.DeleteTable() failure for Table = ", ordtbl) + return err + } } } err = d.DeleteTable(cmnAppTs) @@ -395,8 +692,8 @@ func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { return err } log.Info("DELETE case - Deleted entire table = ", tblNm) - log.Info("Done processing all tables.") - break + // Continue to repeat ordered deletion for all tables + continue } @@ -404,40 +701,67 @@ func (app *CommonApp) cmnAppDelDbOpn(d *db.DB, opcode int) error { if len(tblRw.Field) == 0 { log.Info("DELETE case - no fields/cols to delete hence delete the entire row.") log.Info("First, delete child table instances that correspond to parent table instance to be deleted = ", tblKey) - for idx := len(app.cmnAppOrdTbllist)-1; idx >= tblidx+1; idx-- { - dbTblSpec = &db.TableSpec{Name: app.cmnAppOrdTbllist[idx]} - keyPattern := tblKey + "|*" - log.Info("Key pattern to be matched for deletion = ", keyPattern) - err = d.DeleteKeys(dbTblSpec, db.Key{Comp: []string{keyPattern}}) - if err != nil { - log.Warning("DELETE case - d.DeleteTable() failure for Table = ", app.cmnAppOrdTbllist[idx]) - return err + if !app.skipOrdTableChk { + for _, ordtbl := range ordTblList { + if ordtbl == tblNm { + // Handle the child tables only till you reach the parent table entry + break; + } + dbTblSpec = &db.TableSpec{Name: ordtbl} + keyPattern := tblKey + "|*" + log.Info("Key pattern to be matched for deletion = ", keyPattern) + err = d.DeleteKeys(dbTblSpec, db.Key{Comp: []string{keyPattern}}) + if err != nil { + log.Warning("DELETE case - d.DeleteTable() failure for Table = ", ordtbl) + return err + } + log.Info("Deleted keys matching parent table key pattern for child table = ", ordtbl) } - log.Info("Deleted keys matching parent table key pattern for child table = ", app.cmnAppOrdTbllist[idx]) - } err = d.DeleteEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) - if err != nil { - log.Warning("DELETE case - d.DeleteEntry() failure") - return err - } + if err != nil { + log.Warning("DELETE case - d.DeleteEntry() failure") + return err + } log.Info("Finally deleted the parent table row with key = ", tblKey) } else { log.Info("DELETE case - fields/cols to delete hence delete only those fields.") - existingEntry, _ := d.GetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) - if !existingEntry.IsPopulated() { + existingEntry, exstErr := d.GetEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}) + if exstErr != nil { log.Info("Table Entry from which the fields are to be deleted does not exist") + err = exstErr return err } /* handle leaf-list merge if any leaf-list exists */ resTblRw := checkAndProcessLeafList(existingEntry, tblRw, DELETE, d, tblNm, tblKey) - err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) - if err != nil { - log.Error("DELETE case - d.DeleteEntryFields() failure") - return err + log.Info("DELETE case - checkAndProcessLeafList() returned table row ", resTblRw) + if len(resTblRw.Field) > 0 { + if !app.deleteEmptyEntry { + /* add the NULL field if the last field gets deleted && deleteEmpyEntry is false */ + deleteCount := 0 + for field := range existingEntry.Field { + if resTblRw.Has(field) { + deleteCount++ + } + } + if deleteCount == len(existingEntry.Field) { + nullTblRw := db.Value{Field: map[string]string{"NULL": "NULL"}} + log.Info("Last field gets deleted, add NULL field to keep an db entry") + err = d.ModEntry(cmnAppTs, db.Key{Comp: []string{tblKey}}, nullTblRw) + if err != nil { + log.Warning("UPDATE case - d.ModEntry() failure") + return err + } + } + } + /* deleted fields */ + err := d.DeleteEntryFields(cmnAppTs, db.Key{Comp: []string{tblKey}}, resTblRw) + if err != nil { + log.Warning("DELETE case - d.DeleteEntryFields() failure") + return err + } } } - } } } /* end of ordered table list for loop */ @@ -458,8 +782,14 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, for field, value := range tblRw.Field { if strings.HasSuffix(field, "@") { exstLst := existingEntry.GetList(field) + log.Infof("Existing DB value for field %v - %v", field, exstLst) + var valueLst []string + if value != "" { //zero len string as leaf-list value is treated as delete entire leaf-list + valueLst = strings.Split(value, ",") + } + log.Infof("Incoming value for field %v - %v", field, valueLst) if len(exstLst) != 0 { - valueLst := strings.Split(value, ",") + log.Infof("Existing list is not empty for field %v", field) for _, item := range valueLst { if !contains(exstLst, item) { if opcode == UPDATE { @@ -467,23 +797,48 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, } } else { if opcode == DELETE { - exstLst = removeElement(exstLst, item) + exstLst = utils.RemoveElement(exstLst, item) } } } - log.Infof("For field %v value after merge %v", field, exstLst) + log.Infof("For field %v value after merging incoming with existing %v", field, exstLst) if opcode == DELETE { - mergeTblRw.SetList(field, exstLst) - delete(tblRw.Field, field) + if len(valueLst) > 0 { + mergeTblRw.SetList(field, exstLst) + if len(exstLst) == 0 { + tblRw.Field[field] = "" + } else { + delete(tblRw.Field, field) + } + } + } else if opcode == UPDATE { + tblRw.SetList(field, exstLst) } - } - tblRw.SetList(field, exstLst) + } else { //when existing list is empty(either empty string val in field or no field at all n entry) + log.Infof("Existing list is empty for field %v", field) + if opcode == UPDATE { + if len(valueLst) > 0 { + exstLst = valueLst + tblRw.SetList(field, exstLst) + } else { + tblRw.Field[field] = "" + } + } else if opcode == DELETE { + _, fldExistsOk := existingEntry.Field[field] + if (fldExistsOk && (len(valueLst) == 0)) { + tblRw.Field[field] = "" + } else { + delete(tblRw.Field, field) + } + } + } } } /* delete specific item from leaf-list */ if opcode == DELETE { if len(mergeTblRw.Field) == 0 { + log.Infof("mergeTblRow is empty - Returning Table Row %v", tblRw) return tblRw } err := d.ModEntry(dbTblSpec, db.Key{Comp: []string{tblKey}}, mergeTblRw) @@ -495,3 +850,46 @@ func checkAndProcessLeafList(existingEntry db.Value, tblRw db.Value, opcode int, return tblRw } +// This function is a copy of the function areEqual in ygot.util package. +// areEqual compares a and b. If a and b are both pointers, it compares the +// values they are pointing to. +func areEqual(a, b interface{}) bool { + if util.IsValueNil(a) && util.IsValueNil(b) { + return true + } + va, vb := reflect.ValueOf(a), reflect.ValueOf(b) + if va.Kind() == reflect.Ptr && vb.Kind() == reflect.Ptr { + return reflect.DeepEqual(va.Elem().Interface(), vb.Elem().Interface()) + } + + return reflect.DeepEqual(a, b) +} + +func isPartialReplace(exstRw db.Value, replTblRw db.Value, auxRw db.Value) bool { + /* if existing entry contains field thats not present in result, + default and auxillary map then its a partial replace + */ + partialReplace := false + for exstFld := range exstRw.Field { + if exstFld == "NULL" { + continue + } + isIncomingFld := false + if replTblRw.Has(exstFld) { + continue + } + if auxRw.Has(exstFld) { + continue + } + if !isIncomingFld { + log.Info("Entry contains field ", exstFld, " not found in result, default and aux fields hence its partial replace.") + partialReplace = true + break + } + } + log.Info("returning partialReplace - ", partialReplace) + return partialReplace +} + + + diff --git a/translib/tlerr/tlerr.go b/translib/tlerr/tlerr.go index 350a86021..936825765 100644 --- a/translib/tlerr/tlerr.go +++ b/translib/tlerr/tlerr.go @@ -103,12 +103,19 @@ func (e TranslibSyntaxValidationError) Error() string { } type TranslibUnsupportedClientVersion struct { - ClientVersion string - ServerVersion string - ServerBaseVersion string + ClientVersion string + ServerVersion string + ServerBaseVersion string } func (e TranslibUnsupportedClientVersion) Error() string { - return p.Sprintf("Unsupported client version %s", e.ClientVersion) + return p.Sprintf("Unsupported client version %s", e.ClientVersion) } +type TranslibXfmrRetError struct { + XlateFailDelReq bool +} + +func (e TranslibXfmrRetError) Error() string { + return p.Sprintf("Translib transformer return %s", e.XlateFailDelReq) +} diff --git a/translib/transformer/transformer.go b/translib/transformer/transformer.go index 2a06c2dde..cca5dc525 100644 --- a/translib/transformer/transformer.go +++ b/translib/transformer/transformer.go @@ -21,7 +21,6 @@ package transformer import ( "fmt" "github.com/openconfig/goyang/pkg/yang" - "github.com/openconfig/ygot/ygot" "os" "strings" "bufio" @@ -30,15 +29,8 @@ import ( ) var YangPath = "/usr/models/yang/" // OpenConfig-*.yang and sonic yang models path - -var entries = map[string]*yang.Entry{} - -//Interface for xfmr methods -type xfmrInterface interface { - tableXfmr(s *ygot.GoStruct, t *interface{}) (string, error) - keyXfmr(s *ygot.GoStruct, t *interface{}) (string, error) - fieldXfmr(s *ygot.GoStruct, t *interface{}) (string, error) -} +var ModelsListFile = "models_list" +var TblInfoJsonFile = "sonic_table_info.json" func reportIfError(errs []error) { if len(errs) > 0 { @@ -50,7 +42,7 @@ func reportIfError(errs []error) { func getOcModelsList () ([]string) { var fileList []string - file, err := os.Open(YangPath + "models_list") + file, err := os.Open(YangPath + ModelsListFile) if err != nil { return fileList } @@ -58,7 +50,7 @@ func getOcModelsList () ([]string) { scanner := bufio.NewScanner(file) for scanner.Scan() { fileEntry := scanner.Text() - if strings.HasPrefix(fileEntry, "#") != true { + if !strings.HasPrefix(fileEntry, "#") { _, err := os.Stat(YangPath + fileEntry) if err != nil { continue @@ -86,9 +78,9 @@ func getDefaultModelsList () ([]string) { func init() { initYangModelsPath() - yangFiles := []string{} + initRegex() ocList := getOcModelsList() - yangFiles = getDefaultModelsList() + yangFiles := getDefaultModelsList() yangFiles = append(yangFiles, ocList...) fmt.Println("Yang model List:", yangFiles) err := loadYangModules(yangFiles...) @@ -105,7 +97,7 @@ func initYangModelsPath() { YangPath = path } - fmt.Println("Yang models path:", YangPath) + fmt.Println("Yang modles path:", YangPath) } func loadYangModules(files ...string) error { @@ -147,15 +139,19 @@ func loadYangModules(files ...string) error { } } - sonic_entries := make([]*yang.Entry, len(names)) + sonic_entries := make([]*yang.Entry, 0) oc_entries := make(map[string]*yang.Entry) - oc_annot_entries := make([]*yang.Entry, len(names)) - sonic_annot_entries := make([]*yang.Entry, len(names)) + oc_annot_entries := make([]*yang.Entry, 0) + sonic_annot_entries := make([]*yang.Entry, 0) for _, n := range names { if strings.Contains(n, "annot") && strings.Contains(n, "sonic") { sonic_annot_entries = append(sonic_annot_entries, yang.ToEntry(mods[n])) } else if strings.Contains(n, "annot") { + yangMdlNmDt := strings.Split(n, "-annot") + if len(yangMdlNmDt) > 0 { + addMdlCpbltEntry(yangMdlNmDt[0]) + } oc_annot_entries = append(oc_annot_entries, yang.ToEntry(mods[n])) } else if strings.Contains(n, "sonic") { sonic_entries = append(sonic_entries, yang.ToEntry(mods[n])) @@ -164,6 +160,37 @@ func loadYangModules(files ...string) error { } } + // populate model capabilities data + for yngMdlNm := range(xMdlCpbltMap) { + org := "" + ver := "" + ocVerSet := false + yngEntry := oc_entries[yngMdlNm] + if (yngEntry != nil) { + // OC yang has version in standard extension oc-ext:openconfig-version + if strings.HasPrefix(yngMdlNm, "openconfig-") { + for _, ext := range yngEntry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + if tagType == "openconfig-version" { + ver = ext.NName() + fmt.Printf("Found version %v for yang module %v", ver, yngMdlNm) + if len(strings.TrimSpace(ver)) > 0 { + ocVerSet = true + } + break + } + + } + } + } + if ((strings.HasPrefix(yngMdlNm, "ietf-")) || (!ocVerSet)) { + // as per RFC7895 revision date to be used as version + ver = mods[yngMdlNm].Current() //gives the most recent revision date for yang module + } + org = mods[yngMdlNm].Organization.Name + addMdlCpbltData(yngMdlNm, ver, org) + } dbMapBuild(sonic_entries) annotDbSpecMap(sonic_annot_entries) annotToDbMapBuild(oc_annot_entries) diff --git a/translib/transformer/xconst.go b/translib/transformer/xconst.go index 8fcdabdc7..a29c8e907 100644 --- a/translib/transformer/xconst.go +++ b/translib/transformer/xconst.go @@ -24,6 +24,8 @@ const ( YANG_CONTAINER = "container" YANG_LEAF = "leaf" YANG_LEAF_LIST = "leaf-list" + YANG_CHOICE = "choice" + YANG_CASE = "case" YANG_ANNOT_DB_NAME = "db-name" YANG_ANNOT_TABLE_NAME = "table-name" @@ -46,6 +48,43 @@ const ( XPATH_SEP_FWD_SLASH = "/" XFMR_EMPTY_STRING = "" + XFMR_NONE_STRING = "NONE" SONIC_TABLE_INDEX = 2 + SONIC_LIST_INDEX = 3 + SONIC_FIELD_INDEX = 4 + SONIC_MDL_PFX = "sonic" + OC_MDL_PFX = "openconfig-" + IETF_MDL_PFX = "ietf-" + IANA_MDL_PFX = "iana-" + YTDB_KEY_XFMR_RET_ARGS = 2 + YTDB_KEY_XFMR_RET_VAL_INDX = 0 + YTDB_KEY_XFMR_RET_ERR_INDX = 1 + YTDB_SBT_XFMR_RET_ARGS = 2 + YTDB_SBT_XFMR_RET_VAL_INDX = 0 + YTDB_SBT_XFMR_RET_ERR_INDX = 1 + YTDB_FLD_XFMR_RET_ARGS = 2 + YTDB_FLD_XFMR_RET_VAL_INDX = 0 + YTDB_FLD_XFMR_RET_ERR_INDX = 1 + DBTY_KEY_XFMR_RET_ARGS = 2 + DBTY_KEY_XFMR_RET_VAL_INDX = 0 + DBTY_KEY_XFMR_RET_ERR_INDX = 1 + DBTY_FLD_XFMR_RET_ARGS = 2 + DBTY_FLD_XFMR_RET_VAL_INDX = 0 + DBTY_FLD_XFMR_RET_ERR_INDX = 1 + SUBSC_SBT_XFMR_RET_ARGS = 2 + SUBSC_SBT_XFMR_RET_VAL_INDX = 0 + SUBSC_SBT_XFMR_RET_ERR_INDX = 1 + DBTY_SBT_XFMR_RET_ERR_INDX = 0 + TBL_XFMR_RET_ARGS = 2 + TBL_XFMR_RET_VAL_INDX = 0 + TBL_XFMR_RET_ERR_INDX = 1 + POST_XFMR_RET_ARGS = 2 + POST_XFMR_RET_VAL_INDX = 0 + POST_XFMR_RET_ERR_INDX = 1 + PRE_XFMR_RET_ARGS = 1 + PRE_XFMR_RET_ERR_INDX = 0 + XFMR_INVALID = -1 + XFMR_DISABLE = 0 + XFMR_ENABLE = 1 ) diff --git a/translib/transformer/xfmr_acl.go b/translib/transformer/xfmr_acl.go deleted file mode 100644 index 7db975153..000000000 --- a/translib/transformer/xfmr_acl.go +++ /dev/null @@ -1,976 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// // -// Copyright 2019 Dell, Inc. // -// // -// Licensed under the Apache License, Version 2.0 (the "License"); // -// you may not use this file except in compliance with the License. // -// You may obtain a copy of the License at // -// // -// http://www.apache.org/licenses/LICENSE-2.0 // -// // -// Unless required by applicable law or agreed to in writing, software // -// distributed under the License is distributed on an "AS IS" BASIS, // -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // -// See the License for the specific language governing permissions and // -// limitations under the License. // -// // -//////////////////////////////////////////////////////////////////////////////// - -package transformer - -import ( - "bytes" - "errors" - "fmt" - log "github.com/golang/glog" - "github.com/openconfig/ygot/ygot" - "reflect" - "strconv" - "strings" - "github.com/Azure/sonic-mgmt-common/translib/db" - "github.com/Azure/sonic-mgmt-common/translib/ocbinds" - "github.com/Azure/sonic-mgmt-common/translib/tlerr" -) - -func init() { - XlateFuncBind("DbToYang_acl_set_name_xfmr", DbToYang_acl_set_name_xfmr) - XlateFuncBind("YangToDb_acl_type_field_xfmr", YangToDb_acl_type_field_xfmr) - XlateFuncBind("DbToYang_acl_type_field_xfmr", DbToYang_acl_type_field_xfmr) - XlateFuncBind("YangToDb_acl_set_key_xfmr", YangToDb_acl_set_key_xfmr) - XlateFuncBind("DbToYang_acl_set_key_xfmr", DbToYang_acl_set_key_xfmr) - XlateFuncBind("YangToDb_acl_entry_key_xfmr", YangToDb_acl_entry_key_xfmr) - XlateFuncBind("DbToYang_acl_entry_key_xfmr", DbToYang_acl_entry_key_xfmr) - XlateFuncBind("DbToYang_acl_entry_sequenceid_xfmr", DbToYang_acl_entry_sequenceid_xfmr) - XlateFuncBind("YangToDb_acl_l2_ethertype_xfmr", YangToDb_acl_l2_ethertype_xfmr) - XlateFuncBind("DbToYang_acl_l2_ethertype_xfmr", DbToYang_acl_l2_ethertype_xfmr) - XlateFuncBind("YangToDb_acl_ip_protocol_xfmr", YangToDb_acl_ip_protocol_xfmr) - XlateFuncBind("DbToYang_acl_ip_protocol_xfmr", DbToYang_acl_ip_protocol_xfmr) - XlateFuncBind("YangToDb_acl_source_port_xfmr", YangToDb_acl_source_port_xfmr) - XlateFuncBind("DbToYang_acl_source_port_xfmr", DbToYang_acl_source_port_xfmr) - XlateFuncBind("YangToDb_acl_destination_port_xfmr", YangToDb_acl_destination_port_xfmr) - XlateFuncBind("DbToYang_acl_destination_port_xfmr", DbToYang_acl_destination_port_xfmr) - XlateFuncBind("YangToDb_acl_tcp_flags_xfmr", YangToDb_acl_tcp_flags_xfmr) - XlateFuncBind("DbToYang_acl_tcp_flags_xfmr", DbToYang_acl_tcp_flags_xfmr) - XlateFuncBind("YangToDb_acl_port_bindings_xfmr", YangToDb_acl_port_bindings_xfmr) - XlateFuncBind("DbToYang_acl_port_bindings_xfmr", DbToYang_acl_port_bindings_xfmr) - XlateFuncBind("YangToDb_acl_forwarding_action_xfmr", YangToDb_acl_forwarding_action_xfmr) - XlateFuncBind("DbToYang_acl_forwarding_action_xfmr", DbToYang_acl_forwarding_action_xfmr) - XlateFuncBind("validate_ipv4", validate_ipv4) - XlateFuncBind("validate_ipv6", validate_ipv6) - XlateFuncBind("acl_post_xfmr", acl_post_xfmr) -} - -const ( - ACL_TABLE = "ACL_TABLE" - RULE_TABLE = "ACL_RULE" - SONIC_ACL_TYPE_IPV4 = "L3" - SONIC_ACL_TYPE_L2 = "L2" - SONIC_ACL_TYPE_IPV6 = "L3V6" - OPENCONFIG_ACL_TYPE_IPV4 = "ACL_IPV4" - OPENCONFIG_ACL_TYPE_IPV6 = "ACL_IPV6" - OPENCONFIG_ACL_TYPE_L2 = "ACL_L2" - ACL_TYPE = "type" - MIN_PRIORITY = 1 - MAX_PRIORITY = 65535 -) - -/* E_OpenconfigAcl_ACL_TYPE */ -var ACL_TYPE_MAP = map[string]string{ - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4), 10): SONIC_ACL_TYPE_IPV4, - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6), 10): SONIC_ACL_TYPE_IPV6, - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2), 10): SONIC_ACL_TYPE_L2, -} - -/* E_OpenconfigAcl_FORWARDING_ACTION */ -var ACL_FORWARDING_ACTION_MAP = map[string]string{ - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_FORWARDING_ACTION_ACCEPT), 10): "FORWARD", - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_FORWARDING_ACTION_DROP), 10): "DROP", - strconv.FormatInt(int64(ocbinds.OpenconfigAcl_FORWARDING_ACTION_REJECT), 10): "REDIRECT", -} - -/* E_OpenconfigPacketMatchTypes_IP_PROTOCOL */ -var IP_PROTOCOL_MAP = map[string]string{ - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_ICMP), 10): "1", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_IGMP), 10): "2", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_TCP), 10): "6", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_UDP), 10): "17", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_RSVP), 10): "46", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_GRE), 10): "47", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_AUTH), 10): "51", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_PIM), 10): "103", - strconv.FormatInt(int64(ocbinds.OpenconfigPacketMatchTypes_IP_PROTOCOL_IP_L2TP), 10): "115", -} - -var ETHERTYPE_MAP = map[ocbinds.E_OpenconfigPacketMatchTypes_ETHERTYPE]uint32{ - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_LLDP: 0x88CC, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_VLAN: 0x8100, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_ROCE: 0x8915, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_ARP: 0x0806, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_IPV4: 0x0800, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_IPV6: 0x86DD, - ocbinds.OpenconfigPacketMatchTypes_ETHERTYPE_ETHERTYPE_MPLS: 0x8847, -} - -func getAclRoot(s *ygot.GoStruct) *ocbinds.OpenconfigAcl_Acl { - deviceObj := (*s).(*ocbinds.Device) - return deviceObj.Acl -} - -func getAclTypeOCEnumFromName(val string) (ocbinds.E_OpenconfigAcl_ACL_TYPE, error) { - switch val { - case "ACL_IPV4", "openconfig-acl:ACL_IPV4": - return ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4, nil - case "ACL_IPV6", "openconfig-acl:ACL_IPV6": - return ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6, nil - case "ACL_L2", "openconfig-acl:ACL_L2": - return ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2, nil - default: - return ocbinds.OpenconfigAcl_ACL_TYPE_UNSET, - tlerr.NotSupported("ACL Type '%s' not supported", val) - } -} -func getAclKeyStrFromOCKey(aclname string, acltype ocbinds.E_OpenconfigAcl_ACL_TYPE) string { - aclN := strings.Replace(strings.Replace(aclname, " ", "_", -1), "-", "_", -1) - aclT := acltype.ΛMap()["E_OpenconfigAcl_ACL_TYPE"][int64(acltype)].Name - return aclN + "_" + aclT -} - -func getOCAclKeysFromStrDBKey(aclKey string) (string, ocbinds.E_OpenconfigAcl_ACL_TYPE) { - var aclOrigName string - var aclOrigType ocbinds.E_OpenconfigAcl_ACL_TYPE - - if strings.Contains(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV4) { - aclOrigName = strings.Replace(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV4, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4 - } else if strings.Contains(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV6) { - aclOrigName = strings.Replace(aclKey, "_"+OPENCONFIG_ACL_TYPE_IPV6, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6 - } else if strings.Contains(aclKey, "_"+OPENCONFIG_ACL_TYPE_L2) { - aclOrigName = strings.Replace(aclKey, "_"+OPENCONFIG_ACL_TYPE_L2, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2 - } - - return aclOrigName, aclOrigType -} - -func getTransportConfigTcpFlags(tcpFlags string) []ocbinds.E_OpenconfigPacketMatchTypes_TCP_FLAGS { - var flags []ocbinds.E_OpenconfigPacketMatchTypes_TCP_FLAGS - if len(tcpFlags) > 0 { - flagStr := strings.Split(tcpFlags, "/")[0] - flagNumber, _ := strconv.ParseUint(strings.Replace(flagStr, "0x", "", -1), 16, 32) - for i := 0; i < 8; i++ { - mask := 1 << uint(i) - if (int(flagNumber) & mask) > 0 { - switch int(flagNumber) & mask { - case 0x01: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_FIN) - case 0x02: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_SYN) - case 0x04: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_RST) - case 0x08: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_PSH) - case 0x10: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ACK) - case 0x20: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_URG) - case 0x40: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ECE) - case 0x80: - flags = append(flags, ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_CWR) - default: - } - } - } - } - return flags -} - -func getL2EtherType(etherType uint64) interface{} { - for k, v := range ETHERTYPE_MAP { - if uint32(etherType) == v { - return k - } - } - return uint16(etherType) -} - -//////////////////////////////////////////// -// Validate callpoints -//////////////////////////////////////////// -var validate_ipv4 ValidateCallpoint = func(inParams XfmrParams) (bool) { - if strings.Contains(inParams.key, "ACL_IPV4") { - return true - } - return false -} -var validate_ipv6 ValidateCallpoint = func(inParams XfmrParams) (bool) { - if strings.Contains(inParams.key, "ACL_IPV6") { - return true - } - return false -} - -//////////////////////////////////////////// -// Post Transformer -//////////////////////////////////////////// -var acl_post_xfmr PostXfmrFunc = func(inParams XfmrParams) (map[string]map[string]db.Value, error) { - log.Info("In Post transformer") - //TODO: check if a default ACL Rule exists, else create one and update the resultMap with default rule - // Return will be the updated result map - return (*inParams.dbDataMap)[inParams.curDb], nil -} - -//////////////////////////////////////////// -// Bi-directoonal overloaded methods -//////////////////////////////////////////// -var YangToDb_acl_forwarding_action_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map["PACKET_ACTION"] = "" - return res_map, err - } - action, _ := inParams.param.(ocbinds.E_OpenconfigAcl_FORWARDING_ACTION) - log.Info("YangToDb_acl_forwarding_action_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " forwarding_action: ", action) - res_map["PACKET_ACTION"] = findInMap(ACL_FORWARDING_ACTION_MAP, strconv.FormatInt(int64(action), 10)) - return res_map, err -} -var DbToYang_acl_forwarding_action_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_forwarding_action_xfmr", data, inParams.ygRoot) - oc_action := findInMap(ACL_FORWARDING_ACTION_MAP, data[RULE_TABLE][inParams.key].Field["PACKET_ACTION"]) - n, err := strconv.ParseInt(oc_action, 10, 64) - result["forwarding-action"] = ocbinds.E_OpenconfigAcl_FORWARDING_ACTION(n).ΛMap()["E_OpenconfigAcl_FORWARDING_ACTION"][n].Name - return result, err -} - -var YangToDb_acl_type_field_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map[ACL_TYPE] = "" - return res_map, err - } - - acltype, _ := inParams.param.(ocbinds.E_OpenconfigAcl_ACL_TYPE) - log.Info("YangToDb_acl_type_field_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " acltype: ", acltype) - res_map[ACL_TYPE] = findInMap(ACL_TYPE_MAP, strconv.FormatInt(int64(acltype), 10)) - return res_map, err -} -var DbToYang_acl_type_field_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_type_field_xfmr", data, inParams.ygRoot) - oc_acltype := findInMap(ACL_TYPE_MAP, data[ACL_TABLE][inParams.key].Field[ACL_TYPE]) - n, err := strconv.ParseInt(oc_acltype, 10, 64) - result[ACL_TYPE] = ocbinds.E_OpenconfigAcl_ACL_TYPE(n).ΛMap()["E_OpenconfigAcl_ACL_TYPE"][n].Name - return result, err -} - -var YangToDb_acl_set_key_xfmr KeyXfmrYangToDb = func(inParams XfmrParams) (string, error) { - var aclkey string - var err error - var oc_aclType ocbinds.E_OpenconfigAcl_ACL_TYPE - log.Info("YangToDb_acl_set_key_xfmr: ", inParams.ygRoot, inParams.uri) - pathInfo := NewPathInfo(inParams.uri) - - if len(pathInfo.Vars) < 2 { - err = errors.New("Invalid xpath, key attributes not found") - return aclkey, err - } - - oc_aclType, err = getAclTypeOCEnumFromName(pathInfo.Var("type")) - if err != nil { - err = errors.New("OC Acl type name to OC Acl Enum failed") - return aclkey, err - } - - aclkey = getAclKeyStrFromOCKey(pathInfo.Var("name"), oc_aclType) - log.Info("YangToDb_acl_set_key_xfmr - acl_set_key : ", aclkey) - - return aclkey, err -} - -var DbToYang_acl_set_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { - rmap := make(map[string]interface{}) - var err error - var aclNameStr string - var aclTypeStr string - aclkey := inParams.key - log.Info("DbToYang_acl_set_key_xfmr: ", aclkey) - if strings.Contains(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV4) { - aclNameStr = strings.Replace(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV4, "", 1) - aclTypeStr = "ACL_IPV4" - } else if strings.Contains(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV6) { - aclNameStr = strings.Replace(aclkey, "_"+OPENCONFIG_ACL_TYPE_IPV6, "", 1) - aclTypeStr = "ACL_IPV6" - } else if strings.Contains(aclkey, "_"+OPENCONFIG_ACL_TYPE_L2) { - aclNameStr = strings.Replace(aclkey, "_"+OPENCONFIG_ACL_TYPE_L2, "", 1) - aclTypeStr = "ACL_L2" - } else { - err = errors.New("Invalid key for acl set.") - log.Info("Invalid Keys for acl acl set", aclkey) - } - rmap["name"] = aclNameStr - rmap["type"] = aclTypeStr - return rmap, err -} - -var DbToYang_acl_set_name_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - res_map := make(map[string]interface{}) - var err error - log.Info("DbToYang_acl_set_name_xfmr: ", inParams.key) - /*name attribute corresponds to key in redis table*/ - aclName, _ := getOCAclKeysFromStrDBKey(inParams.key) - res_map["name"] = aclName - log.Info("acl-set/config/name ", res_map) - return res_map, err -} - -var YangToDb_acl_entry_key_xfmr KeyXfmrYangToDb = func(inParams XfmrParams) (string, error) { - var entry_key string - var err error - var oc_aclType ocbinds.E_OpenconfigAcl_ACL_TYPE - log.Info("YangToDb_acl_entry_key_xfmr: ", inParams.ygRoot, inParams.uri) - pathInfo := NewPathInfo(inParams.uri) - - if len(pathInfo.Vars) < 3 { - err = errors.New("Invalid xpath, key attributes not found") - return entry_key, err - } - - oc_aclType, err = getAclTypeOCEnumFromName(pathInfo.Var("type")) - if err != nil { - err = errors.New("OC Acl type name to OC Acl Enum failed") - return entry_key, err - } - - aclkey := getAclKeyStrFromOCKey(pathInfo.Var("name"), oc_aclType) - var rulekey string - if strings.Contains(pathInfo.Template, "/acl-entry{sequence-id}") { - rulekey = "RULE_" + pathInfo.Var("sequence-id") - } - entry_key = aclkey + "|" + rulekey - - log.Info("YangToDb_acl_entry_key_xfmr - entry_key : ", entry_key) - - return entry_key, err -} - -var DbToYang_acl_entry_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[string]interface{}, error) { - rmap := make(map[string]interface{}) - var err error - entry_key := inParams.key - log.Info("DbToYang_acl_entry_key_xfmr: ", entry_key) - - key := strings.Split(entry_key, "|") - if len(key) < 2 { - err = errors.New("Invalid key for acl entries.") - log.Info("Invalid Keys for acl enmtries", entry_key) - return rmap, err - } - - dbAclRule := key[1] - seqId := strings.Replace(dbAclRule, "RULE_", "", 1) - rmap["sequence-id"], _ = strconv.ParseFloat(seqId, 64) - return rmap, err -} - -var DbToYang_acl_entry_sequenceid_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - res_map := make(map[string]interface{}) - var err error - log.Info("DbToYang_acl_entry_sequenceid_xfmr: ", inParams.key) - /*sequenec-id attribute corresponds to key in redis table*/ - res, err := DbToYang_acl_entry_key_xfmr(inParams) - log.Info("acl-entry/config/sequence-id ", res) - if err != nil { - return res_map, err - } - if seqId, ok := res["sequence-id"]; !ok { - log.Error("sequence-id not found in acl entry") - return res_map, err - } else { - res_map["sequence-id"] = seqId - } - return res_map, err -} - -var YangToDb_acl_l2_ethertype_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - - if inParams.param == nil { - res_map["ETHER_TYPE"] = "" - return res_map, err - } - ethertypeType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " ethertypeType: ", ethertypeType) - var b bytes.Buffer - switch ethertypeType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_E_OpenconfigPacketMatchTypes_ETHERTYPE{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_E_OpenconfigPacketMatchTypes_ETHERTYPE) - fmt.Fprintf(&b, "0x%0.4x", ETHERTYPE_MAP[v.E_OpenconfigPacketMatchTypes_ETHERTYPE]) - res_map["ETHER_TYPE"] = b.String() - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_Uint16{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_L2_Config_Ethertype_Union_Uint16) - fmt.Fprintf(&b, "0x%0.4x", v.Uint16) - res_map["ETHER_TYPE"] = b.String() - break - } - return res_map, err -} - -var DbToYang_acl_l2_ethertype_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_l2_ethertype_xfmr", data, inParams.ygRoot) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - etype, ok := ruleInst.Field["ETHER_TYPE"] - - if ok { - etypeVal, _ := strconv.ParseUint(strings.Replace(etype, "0x", "", -1), 16, 32) - result["protocol"] = getL2EtherType(etypeVal) - } else { - err = errors.New("ETHER_TYPE field not found in DB") - } - return result, nil -} - -var YangToDb_acl_ip_protocol_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - - if inParams.param == nil { - res_map["IP_PROTOCOL"] = "" - return res_map, err - } - protocolType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " protocolType: ", protocolType) - switch protocolType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_E_OpenconfigPacketMatchTypes_IP_PROTOCOL{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_E_OpenconfigPacketMatchTypes_IP_PROTOCOL) - res_map["IP_PROTOCOL"] = findInMap(IP_PROTOCOL_MAP, strconv.FormatInt(int64(v.E_OpenconfigPacketMatchTypes_IP_PROTOCOL), 10)) - v = nil - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_Uint8{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Ipv4_Config_Protocol_Union_Uint8) - res_map["IP_PROTOCOL"] = strconv.FormatInt(int64(v.Uint8), 10) - break - } - return res_map, err -} - -var DbToYang_acl_ip_protocol_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_ip_protocol_xfmr", data, inParams.ygRoot) - oc_protocol := findByValue(IP_PROTOCOL_MAP, data[RULE_TABLE][inParams.key].Field["IP_PROTOCOL"]) - n, err := strconv.ParseInt(oc_protocol, 10, 64) - result["protocol"] = ocbinds.E_OpenconfigPacketMatchTypes_IP_PROTOCOL(n).ΛMap()["E_OpenconfigPacketMatchTypes_IP_PROTOCOL"][n].Name - return result, err -} - -var YangToDb_acl_source_port_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map["L4_SRC_PORT"] = "" - return res_map, err - } - sourceportType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " sourceportType: ", sourceportType) - switch sourceportType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort) - res_map["L4_SRC_PORT"] = v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort.ΛMap()["E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort"][int64(v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort)].Name - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_String{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_String) - res_map["L4_SRC_PORT_RANGE"] = strings.Replace(v.String, "..", "-", 1) - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_Uint16{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_SourcePort_Union_Uint16) - res_map["L4_SRC_PORT"] = strconv.FormatInt(int64(v.Uint16), 10) - break - } - return res_map, err -} - -var DbToYang_acl_source_port_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_source_port_xfmr: ", data, inParams.ygRoot) - result := make(map[string]interface{}) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - port, ok := ruleInst.Field["L4_SRC_PORT"] - if ok { - result["source-port"] = port - return result, nil - } - - portRange, ok := ruleInst.Field["L4_SRC_PORT_RANGE"] - if ok { - result["source-port"] = portRange - return result, nil - } else { - err = errors.New("PORT/PORT_RANGE field not found in DB") - } - return result, err -} - -var YangToDb_acl_destination_port_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - if inParams.param == nil { - res_map["L4_DST_PORT_RANGE"] = "" - return res_map, err - } - destportType := reflect.TypeOf(inParams.param).Elem() - log.Info("YangToDb_acl_ip_protocol_xfmr: ", inParams.ygRoot, " Xpath: ", inParams.uri, " destportType: ", destportType) - switch destportType { - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort) - res_map["L4_DST_PORT"] = v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort.ΛMap()["E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort"][int64(v.E_OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort)].Name - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_String{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_String) - res_map["L4_DST_PORT_RANGE"] = strings.Replace(v.String, "..", "-", 1) - break - case reflect.TypeOf(ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_Uint16{}): - v := (inParams.param).(*ocbinds.OpenconfigAcl_Acl_AclSets_AclSet_AclEntries_AclEntry_Transport_Config_DestinationPort_Union_Uint16) - res_map["L4_DST_PORT"] = strconv.FormatInt(int64(v.Uint16), 10) - break - } - return res_map, err -} - -var DbToYang_acl_destination_port_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_destination_port_xfmr: ", data, inParams.ygRoot) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - port, ok := ruleInst.Field["L4_DST_PORT"] - if ok { - result["destination-port"] = port - return result, nil - } - - portRange, ok := ruleInst.Field["L4_DST_PORT_RANGE"] - if ok { - result["destination-port"] = portRange - return result, nil - } else { - err = errors.New("DST PORT/PORT_RANGE field not found in DB") - } - return result, err -} - -var YangToDb_acl_tcp_flags_xfmr FieldXfmrYangToDb = func(inParams XfmrParams) (map[string]string, error) { - res_map := make(map[string]string) - var err error - log.Info("YangToDb_acl_tcp_flags_xfmr: ") - var tcpFlags uint32 = 0x00 - var b bytes.Buffer - if inParams.param == nil { - res_map["TCP_FLAGS"] = b.String() - return res_map, err - } - log.Info("YangToDb_acl_tcp_flags_xfmr: ", inParams.ygRoot, inParams.uri) - v := reflect.ValueOf(inParams.param) - - flags := v.Interface().([]ocbinds.E_OpenconfigPacketMatchTypes_TCP_FLAGS) - for _, flag := range flags { - fmt.Println("TCP Flag name: " + flag.ΛMap()["E_OpenconfigPacketMatchTypes_TCP_FLAGS"][int64(flag)].Name) - switch flag { - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_FIN: - tcpFlags |= 0x01 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_SYN: - tcpFlags |= 0x02 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_RST: - tcpFlags |= 0x04 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_PSH: - tcpFlags |= 0x08 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ACK: - tcpFlags |= 0x10 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_URG: - tcpFlags |= 0x20 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_ECE: - tcpFlags |= 0x40 - break - case ocbinds.OpenconfigPacketMatchTypes_TCP_FLAGS_TCP_CWR: - tcpFlags |= 0x80 - break - } - } - fmt.Fprintf(&b, "0x%0.2x/0x%0.2x", tcpFlags, tcpFlags) - res_map["TCP_FLAGS"] = b.String() - return res_map, err -} - -var DbToYang_acl_tcp_flags_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_tcp_flags_xfmr: ", data, inParams.ygRoot) - result := make(map[string]interface{}) - if _, ok := data[RULE_TABLE]; !ok { - err = errors.New("RULE_TABLE entry not found in the input param") - return result, err - } - ruleTbl := data[RULE_TABLE] - ruleInst := ruleTbl[inParams.key] - tcpFlag, ok := ruleInst.Field["TCP_FLAGS"] - if ok { - result["tcp-flags"] = getTransportConfigTcpFlags(tcpFlag) - return result, nil - } - return result, nil -} - -var YangToDb_acl_port_bindings_xfmr SubTreeXfmrYangToDb = func(inParams XfmrParams) (map[string]map[string]db.Value, error) { - var err error - res_map := make(map[string]map[string]db.Value) - aclTableMap := make(map[string]db.Value) - log.Info("YangToDb_acl_port_bindings_xfmr: ", inParams.ygRoot, inParams.uri) - - aclObj := getAclRoot(inParams.ygRoot) - if aclObj.Interfaces == nil { - return res_map, err - } - aclInterfacesMap := make(map[string][]string) - for intfId, _ := range aclObj.Interfaces.Interface { - intf := aclObj.Interfaces.Interface[intfId] - if intf != nil { - if intf.IngressAclSets != nil && len(intf.IngressAclSets.IngressAclSet) > 0 { - for inAclKey, _ := range intf.IngressAclSets.IngressAclSet { - aclName := getAclKeyStrFromOCKey(inAclKey.SetName, inAclKey.Type) - aclInterfacesMap[aclName] = append(aclInterfacesMap[aclName], *intf.Id) - _, ok := aclTableMap[aclName] - if !ok { - aclTableMap[aclName] = db.Value{Field: make(map[string]string)} - } - aclTableMap[aclName].Field["stage"] = "INGRESS" - } - } - if intf.EgressAclSets != nil && len(intf.EgressAclSets.EgressAclSet) > 0 { - for outAclKey, _ := range intf.EgressAclSets.EgressAclSet { - aclName := getAclKeyStrFromOCKey(outAclKey.SetName, outAclKey.Type) - aclInterfacesMap[aclName] = append(aclInterfacesMap[aclName], *intf.Id) - _, ok := aclTableMap[aclName] - if !ok { - aclTableMap[aclName] = db.Value{Field: make(map[string]string)} - } - aclTableMap[aclName].Field["stage"] = "EGRESS" - } - } - } - } - for k, _ := range aclInterfacesMap { - val := aclTableMap[k] - (&val).SetList("ports", aclInterfacesMap[k]) - } - res_map[ACL_TABLE] = aclTableMap - return res_map, err -} - -var DbToYang_acl_port_bindings_xfmr SubTreeXfmrDbToYang = func(inParams XfmrParams) error { - var err error - data := (*inParams.dbDataMap)[inParams.curDb] - log.Info("DbToYang_acl_port_bindings_xfmr: ", data, inParams.ygRoot) - - aclTbl := data["ACL_TABLE"] - var ruleTbl map[string]map[string]db.Value - - // repopulate to use existing code - ruleTbl = make(map[string]map[string]db.Value) - for key, element := range data["ACL_RULE"] { - // split into aclKey and ruleKey - tokens := strings.Split(key, "|") - if ruleTbl[tokens[0]] == nil { - ruleTbl[tokens[0]] = make(map[string]db.Value) - } - ruleTbl[tokens[0]][tokens[1]] = db.Value{Field: make(map[string]string)} - ruleTbl[tokens[0]][tokens[1]] = element - } - - pathInfo := NewPathInfo(inParams.uri) - - acl := getAclRoot(inParams.ygRoot) - targetUriPath, _ := getYangPathFromUri(pathInfo.Path) - if isSubtreeRequest(pathInfo.Template, "/openconfig-acl:acl/interfaces/interface{}") { - for intfId := range acl.Interfaces.Interface { - intfData := acl.Interfaces.Interface[intfId] - ygot.BuildEmptyTree(intfData) - if isSubtreeRequest(targetUriPath, "/openconfig-acl:acl/interfaces/interface/ingress-acl-sets") { - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "INGRESS") - } else if isSubtreeRequest(targetUriPath, "/openconfig-acl:acl/interfaces/interface/egress-acl-sets") { - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "EGRESS") - } else { - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "INGRESS") - if err != nil { - return err - } - err = getAclBindingInfoForInterfaceData(aclTbl, ruleTbl, intfData, intfId, "EGRESS") - } - } - } else { - err = getAllBindingsInfo(aclTbl, ruleTbl, inParams.ygRoot) - } - - return err -} - -func convertInternalToOCAclRuleBinding(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, priority uint32, seqId int64, direction string, aclSet ygot.GoStruct, entrySet ygot.GoStruct) { - if seqId == -1 { - seqId = int64(MAX_PRIORITY - priority) - } - - var num uint64 - num = 0 - var ruleId uint32 = uint32(seqId) - - if direction == "INGRESS" { - var ingressEntrySet *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_AclEntries_AclEntry - var ok bool - if entrySet == nil { - ingressAclSet := aclSet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet) - if ingressEntrySet, ok = ingressAclSet.AclEntries.AclEntry[ruleId]; !ok { - ingressEntrySet, _ = ingressAclSet.AclEntries.NewAclEntry(ruleId) - } - } else { - ingressEntrySet = entrySet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_AclEntries_AclEntry) - } - if ingressEntrySet != nil { - ygot.BuildEmptyTree(ingressEntrySet) - ingressEntrySet.State.SequenceId = &ruleId - ingressEntrySet.State.MatchedPackets = &num - ingressEntrySet.State.MatchedOctets = &num - } - } else if direction == "EGRESS" { - var egressEntrySet *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_AclEntries_AclEntry - var ok bool - if entrySet == nil { - egressAclSet := aclSet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet) - if egressEntrySet, ok = egressAclSet.AclEntries.AclEntry[ruleId]; !ok { - egressEntrySet, _ = egressAclSet.AclEntries.NewAclEntry(ruleId) - } - } else { - egressEntrySet = entrySet.(*ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_AclEntries_AclEntry) - } - if egressEntrySet != nil { - ygot.BuildEmptyTree(egressEntrySet) - egressEntrySet.State.SequenceId = &ruleId - egressEntrySet.State.MatchedPackets = &num - egressEntrySet.State.MatchedOctets = &num - } - } -} - -func convertInternalToOCAclBinding(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, aclName string, intfId string, direction string, intfAclSet ygot.GoStruct) error { - var err error - if _, ok := aclTableMap[aclName]; !ok { - err = errors.New("Acl entry not found, convertInternalToOCAclBinding") - return err - } else { - aclEntry := aclTableMap[aclName] - if !contains(aclEntry.GetList("ports"), intfId) { - return tlerr.InvalidArgs("Acl %s not binded with %s", aclName, intfId) - } - } - - for ruleName := range ruleTableMap[aclName] { - if ruleName != "DEFAULT_RULE" { - seqId, _ := strconv.Atoi(strings.Replace(ruleName, "RULE_", "", 1)) - convertInternalToOCAclRuleBinding(aclTableMap, ruleTableMap, 0, int64(seqId), direction, intfAclSet, nil) - } - } - - return err -} - -func getAllBindingsInfo(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, ygRoot *ygot.GoStruct) error { - var err error - acl := getAclRoot(ygRoot) - - var interfaces []string - for aclName := range aclTableMap { - aclData := aclTableMap[aclName] - if len(aclData.Get("ports@")) > 0 { - aclIntfs := aclData.GetList("ports") - for i, _ := range aclIntfs { - if !contains(interfaces, aclIntfs[i]) && aclIntfs[i] != "" { - interfaces = append(interfaces, aclIntfs[i]) - } - } - } - } - ygot.BuildEmptyTree(acl) - for _, intfId := range interfaces { - var intfData *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface - intfData, ok := acl.Interfaces.Interface[intfId] - if !ok { - intfData, _ = acl.Interfaces.NewInterface(intfId) - } - ygot.BuildEmptyTree(intfData) - err = getAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfData, intfId, "INGRESS") - err = getAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfData, intfId, "EGRESS") - } - return err -} - -func getAclBindingInfoForInterfaceData(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, intfData *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface, intfId string, direction string) error { - var err error - if intfData != nil { - intfData.Config.Id = intfData.Id - intfData.State.Id = intfData.Id - } - if direction == "INGRESS" { - if intfData.IngressAclSets != nil && len(intfData.IngressAclSets.IngressAclSet) > 0 { - for ingressAclSetKey, _ := range intfData.IngressAclSets.IngressAclSet { - aclName := strings.Replace(strings.Replace(ingressAclSetKey.SetName, " ", "_", -1), "-", "_", -1) - aclType := ingressAclSetKey.Type.ΛMap()["E_OpenconfigAcl_ACL_TYPE"][int64(ingressAclSetKey.Type)].Name - aclKey := aclName + "_" + aclType - - ingressAclSet := intfData.IngressAclSets.IngressAclSet[ingressAclSetKey] - if ingressAclSet != nil && ingressAclSet.AclEntries != nil && len(ingressAclSet.AclEntries.AclEntry) > 0 { - for seqId, _ := range ingressAclSet.AclEntries.AclEntry { - rulekey := "RULE_" + strconv.Itoa(int(seqId)) - entrySet := ingressAclSet.AclEntries.AclEntry[seqId] - _, ok := ruleTableMap[aclKey+"|"+rulekey] - if !ok { - log.Info("Acl Rule not found ", aclKey, rulekey) - err = errors.New("Acl Rule not found ingress, getAclBindingInfoForInterfaceData") - return err - } - convertInternalToOCAclRuleBinding(aclTableMap, ruleTableMap, 0, int64(seqId), direction, nil, entrySet) - } - } else { - ygot.BuildEmptyTree(ingressAclSet) - ingressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_Config{SetName: &aclName, Type: ingressAclSetKey.Type} - ingressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_State{SetName: &aclName, Type: ingressAclSetKey.Type} - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclKey, intfId, direction, ingressAclSet) - } - } - } else { - err = findAndGetAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfId, direction, intfData) - } - } else if direction == "EGRESS" { - if intfData.EgressAclSets != nil && len(intfData.EgressAclSets.EgressAclSet) > 0 { - for egressAclSetKey, _ := range intfData.EgressAclSets.EgressAclSet { - aclName := strings.Replace(strings.Replace(egressAclSetKey.SetName, " ", "_", -1), "-", "_", -1) - aclType := egressAclSetKey.Type.ΛMap()["E_OpenconfigAcl_ACL_TYPE"][int64(egressAclSetKey.Type)].Name - aclKey := aclName + "_" + aclType - - egressAclSet := intfData.EgressAclSets.EgressAclSet[egressAclSetKey] - if egressAclSet != nil && egressAclSet.AclEntries != nil && len(egressAclSet.AclEntries.AclEntry) > 0 { - for seqId, _ := range egressAclSet.AclEntries.AclEntry { - rulekey := "RULE_" + strconv.Itoa(int(seqId)) - entrySet := egressAclSet.AclEntries.AclEntry[seqId] - _, ok := ruleTableMap[aclKey+"|"+rulekey] - if !ok { - log.Info("Acl Rule not found ", aclKey, rulekey) - err = errors.New("Acl Rule not found egress, getAclBindingInfoForInterfaceData") - return err - } - convertInternalToOCAclRuleBinding(aclTableMap, ruleTableMap, 0, int64(seqId), direction, nil, entrySet) - } - } else { - ygot.BuildEmptyTree(egressAclSet) - egressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_Config{SetName: &aclName, Type: egressAclSetKey.Type} - egressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_State{SetName: &aclName, Type: egressAclSetKey.Type} - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclKey, intfId, direction, egressAclSet) - } - } - } else { - err = findAndGetAclBindingInfoForInterfaceData(aclTableMap, ruleTableMap, intfId, direction, intfData) - } - } else { - log.Error("Unknown direction") - } - return err -} - -func findAndGetAclBindingInfoForInterfaceData(aclTableMap map[string]db.Value, ruleTableMap map[string]map[string]db.Value, intfId string, direction string, intfData *ocbinds.OpenconfigAcl_Acl_Interfaces_Interface) error { - var err error - for aclName, _ := range aclTableMap { - aclData := aclTableMap[aclName] - aclIntfs := aclData.GetList("ports") - aclType := aclData.Get(ACL_TYPE) - var aclOrigName string - var aclOrigType ocbinds.E_OpenconfigAcl_ACL_TYPE - if SONIC_ACL_TYPE_IPV4 == aclType { - aclOrigName = strings.Replace(aclName, "_"+OPENCONFIG_ACL_TYPE_IPV4, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV4 - } else if SONIC_ACL_TYPE_IPV6 == aclType { - aclOrigName = strings.Replace(aclName, "_"+OPENCONFIG_ACL_TYPE_IPV6, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_IPV6 - } else if SONIC_ACL_TYPE_L2 == aclType { - aclOrigName = strings.Replace(aclName, "_"+OPENCONFIG_ACL_TYPE_L2, "", 1) - aclOrigType = ocbinds.OpenconfigAcl_ACL_TYPE_ACL_L2 - } - - if contains(aclIntfs, intfId) && direction == aclData.Get("stage") { - if direction == "INGRESS" { - if intfData.IngressAclSets != nil { - aclSetKey := ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_Key{SetName: aclOrigName, Type: aclOrigType} - ingressAclSet, ok := intfData.IngressAclSets.IngressAclSet[aclSetKey] - if !ok { - ingressAclSet, _ = intfData.IngressAclSets.NewIngressAclSet(aclOrigName, aclOrigType) - ygot.BuildEmptyTree(ingressAclSet) - ingressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_Config{SetName: &aclOrigName, Type: aclOrigType} - ingressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_IngressAclSets_IngressAclSet_State{SetName: &aclOrigName, Type: aclOrigType} - } - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclName, intfId, direction, ingressAclSet) - if err != nil { - return err - } - } - } else if direction == "EGRESS" { - if intfData.EgressAclSets != nil { - aclSetKey := ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_Key{SetName: aclOrigName, Type: aclOrigType} - egressAclSet, ok := intfData.EgressAclSets.EgressAclSet[aclSetKey] - if !ok { - egressAclSet, _ = intfData.EgressAclSets.NewEgressAclSet(aclOrigName, aclOrigType) - ygot.BuildEmptyTree(egressAclSet) - egressAclSet.Config = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_Config{SetName: &aclOrigName, Type: aclOrigType} - egressAclSet.State = &ocbinds.OpenconfigAcl_Acl_Interfaces_Interface_EgressAclSets_EgressAclSet_State{SetName: &aclOrigName, Type: aclOrigType} - } - err = convertInternalToOCAclBinding(aclTableMap, ruleTableMap, aclName, intfId, direction, egressAclSet) - if err != nil { - return err - } - } - } - } - } - return err -} diff --git a/translib/transformer/xfmr_dbtbl_cbk_interface.go b/translib/transformer/xfmr_dbtbl_cbk_interface.go new file mode 100644 index 000000000..8b2ffdea4 --- /dev/null +++ b/translib/transformer/xfmr_dbtbl_cbk_interface.go @@ -0,0 +1,53 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2019 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + "github.com/Azure/sonic-mgmt-common/translib/db" +) + +type XfmrDbTblCbkParams struct { + d *db.DB //Config DB handler + oper int + delDepRefKey string + tblName string + dbKey string + delDepEntry map[string]string + dbDataMap map[db.DBNum]map[string]map[string]db.Value + delDepDataMap map[int]*RedisDbMap // Call back methods can add the data +} + +func formXfmrDbTblCbkParams (d *db.DB, oper int, delDepRefKey string, tblName string, dbKey string, delDepEntry map[string]string, dbDataMap RedisDbMap) XfmrDbTblCbkParams { + + var inParams XfmrDbTblCbkParams + + inParams.d = d + inParams.oper = oper + inParams.delDepRefKey = delDepRefKey + inParams.tblName = tblName + inParams.dbKey = dbKey + inParams.delDepEntry = delDepEntry + inParams.dbDataMap = dbDataMap + inParams.delDepDataMap = make(map[int]*RedisDbMap) + + return inParams +} + +type XfmrDbTblCbkMethod func (inParams XfmrDbTblCbkParams) error + diff --git a/translib/transformer/xfmr_interface.go b/translib/transformer/xfmr_interface.go index 57bc650c8..f705d0dcb 100644 --- a/translib/transformer/xfmr_interface.go +++ b/translib/transformer/xfmr_interface.go @@ -21,115 +21,175 @@ package transformer import ( "github.com/openconfig/ygot/ygot" "github.com/Azure/sonic-mgmt-common/translib/db" - log "github.com/golang/glog" + "sync" ) +type RedisDbMap = map[db.DBNum]map[string]map[string]db.Value + +// XfmrParams represents input parameters for table-transformer, key-transformer, field-transformer & subtree-transformer type XfmrParams struct { d *db.DB dbs [db.MaxDB]*db.DB curDb db.DBNum ygRoot *ygot.GoStruct uri string + requestUri string //original uri using which a curl/NBI request is made oper int + table string key string dbDataMap *map[db.DBNum]map[string]map[string]db.Value + subOpDataMap map[int]*RedisDbMap // used to add an in-flight data with a sub-op param interface{} + txCache *sync.Map + skipOrdTblChk *bool + isVirtualTbl *bool + pCascadeDelTbl *[] string //used to populate list of tables needed cascade delete by subtree overloaded methods + yangDefValMap map[string]map[string]db.Value +} + +// SubscProcType represents subcription process type identifying the type of subscription request made from translib. +type SubscProcType int +const ( + TRANSLATE_SUBSCRIBE SubscProcType = iota + PROCESS_SUBSCRIBE +) + +/*susbcription sampling interval and subscription preference type*/ +type notificationOpts struct { + mInterval int + pType NotificationType +} + +// XfmrSubscInParams represents input to subscribe subtree callbacks - request uri, DBs info access-pointers, DB info for request uri and subscription process type from translib. +type XfmrSubscInParams struct { + uri string + dbs [db.MaxDB]*db.DB + dbDataMap RedisDbMap + subscProc SubscProcType +} + +// XfmrSubscOutParams represents output from subscribe subtree callback - DB data for request uri, Need cache, OnChange, subscription preference and interval. +type XfmrSubscOutParams struct { + dbDataMap RedisDbMap + needCache bool + onChange bool + nOpts *notificationOpts //these can be set regardless of error + isVirtualTbl bool //used for RFC parent table check, set to true when no Redis Mapping +} + +// XfmrDbParams represents input paraDeters for value-transformer +type XfmrDbParams struct { + oper int + dbNum db.DBNum + tableName string + key string + fieldName string + value string } -/** - * KeyXfmrYangToDb type is defined to use for conversion of Yang key to DB Key - * Transformer function definition. - * Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath - * Return: Database keys to access db entry, error - **/ + +// KeyXfmrYangToDb type is defined to use for conversion of Yang key to DB Key, +// Transformer function definition. +// Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath +// Return: Database keys to access db entry, error type KeyXfmrYangToDb func (inParams XfmrParams) (string, error) -/** - * KeyXfmrDbToYang type is defined to use for conversion of DB key to Yang key - * Transformer function definition. - * Param: XfmrParams structure having Database info, operation, Database keys to access db entry - * Return: multi dimensional map to hold the yang key attributes of complete xpath, error - **/ + +// KeyXfmrDbToYang type is defined to use for conversion of DB key to Yang key, +// Transformer function definition. +// Param: XfmrParams structure having Database info, operation, Database keys to access db entry +// Return: multi dimensional map to hold the yang key attributes of complete xpath, error */ type KeyXfmrDbToYang func (inParams XfmrParams) (map[string]interface{}, error) -/** - * FieldXfmrYangToDb type is defined to use for conversion of yang Field to DB field - * Transformer function definition. - * Param: Database info, YgotRoot, operation, Xpath - * Return: multi dimensional map to hold the DB data, error - **/ +// FieldXfmrYangToDb type is defined to use for conversion of yang Field to DB field +// Transformer function definition. +// Param: Database info, YgotRoot, operation, Xpath +// Return: multi dimensional map to hold the DB data, error type FieldXfmrYangToDb func (inParams XfmrParams) (map[string]string, error) -/** - * FieldXfmrDbtoYang type is defined to use for conversion of DB field to Yang field - * Transformer function definition. - * Param: XfmrParams structure having Database info, operation, DB data in multidimensional map, output param YgotRoot - * Return: error - **/ + +// FieldXfmrDbtoYang type is defined to use for conversion of DB field to Yang field +// Transformer function definition. +// Param: XfmrParams structure having Database info, operation, DB data in multidimensional map, output param YgotRoot +// Return: error type FieldXfmrDbtoYang func (inParams XfmrParams) (map[string]interface{}, error) -/** - * SubTreeXfmrYangToDb type is defined to use for handling the yang subtree to DB - * Transformer function definition. - * Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath - * Return: multi dimensional map to hold the DB data, error - **/ +// SubTreeXfmrYangToDb type is defined to use for handling the yang subtree to DB +// Transformer function definition. +// Param: XfmrParams structure having Database info, YgotRoot, operation, Xpath +// Return: multi dimensional map to hold the DB data, error type SubTreeXfmrYangToDb func (inParams XfmrParams) (map[string]map[string]db.Value, error) -/** - * SubTreeXfmrDbToYang type is defined to use for handling the DB to Yang subtree - * Transformer function definition. - * Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri - * Return : error - **/ + +// SubTreeXfmrDbToYang type is defined to use for handling the DB to Yang subtree +// Transformer function definition. +// Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri +// Return : error type SubTreeXfmrDbToYang func (inParams XfmrParams) (error) -/** - * ValidateCallpoint is used to validate a YANG node during data translation back to YANG as a response to GET - * Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri - * Return : bool - **/ + +// SubTreeXfmrSubscribe type is defined to use for handling subscribe(translateSubscribe & processSubscribe) subtree +// Transformer function definition. +// Param : XfmrSubscInParams structure having uri, database pointers, subcribe process(translate/processSusbscribe), DB data in multidimensional map +// Return : XfmrSubscOutParams structure (db data in multiD map, needCache, pType, onChange, minInterval), error +type SubTreeXfmrSubscribe func (inParams XfmrSubscInParams) (XfmrSubscOutParams, error) + +// ValidateCallpoint is used to validate a YANG node during data translation back to YANG as a response to GET +// Param : XfmrParams structure having Database pointers, current db, operation, DB data in multidimensional map, output param YgotRoot, uri +// Return : bool type ValidateCallpoint func (inParams XfmrParams) (bool) -/** - * PostXfmrFunc type is defined to use for handling any default handling operations required as part of the CREATE - * Transformer function definition. - * Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri - * Return: multi dimensional map to hold the DB data, error - **/ -type PostXfmrFunc func (inParams XfmrParams) (map[string]map[string]db.Value, error) +// RpcCallpoint is used to invoke a callback for action +// Param : []byte input payload, dbi indices +// Return : []byte output payload, error +type RpcCallpoint func (body []byte, dbs [db.MaxDB]*db.DB) ([]byte, error) +// PostXfmrFunc type is defined to use for handling any default handling operations required as part of the CREATE +// Transformer function definition. +// Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri +// Return: Multi dimensional map to hold the DB data Map (tblName, key and Fields), error +type PostXfmrFunc func (inParams XfmrParams) (map[string]map[string]db.Value, error) -/** - * TableXfmrFunc type is defined to use for table transformer function for dynamic derviation of redis table. - * Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri - * Return: List of table names, error - **/ +// TableXfmrFunc type is defined to use for table transformer function for dynamic derviation of redis table. +// Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri +// Return: List of table names, error type TableXfmrFunc func (inParams XfmrParams) ([]string, error) +// ValueXfmrFunc type is defined to use for conversion of DB field value from one forma to another +// Transformer function definition. +// Param: XfmrDbParams structure having Database info, operation, db-number, table, key, field, value +// Return: value string, error +type ValueXfmrFunc func (inParams XfmrDbParams) (string, error) -/** - * Xfmr validation interface for validating the callback registration of app modules - * transformer methods. - **/ + // PreXfmrFunc type is defined to use for handling any default handling operations required as part of the CREATE, UPDATE, REPLACE, DELETE & GET + // Transformer function definition. + // Param: XfmrParams structure having database pointers, current db, operation, DB data in multidimensional map, YgotRoot, uri + // Return: error +type PreXfmrFunc func (inParams XfmrParams) (error) + +// XfmrInterface is a validation interface for validating the callback registration of app modules +// transformer methods. type XfmrInterface interface { xfmrInterfaceValiidate() } func (KeyXfmrYangToDb) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for KeyXfmrYangToDb") + xfmrLogInfo("xfmrInterfaceValiidate for KeyXfmrYangToDb") } func (KeyXfmrDbToYang) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for KeyXfmrDbToYang") + xfmrLogInfo("xfmrInterfaceValiidate for KeyXfmrDbToYang") } func (FieldXfmrYangToDb) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for FieldXfmrYangToDb") + xfmrLogInfo("xfmrInterfaceValiidate for FieldXfmrYangToDb") } func (FieldXfmrDbtoYang) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for FieldXfmrDbtoYang") + xfmrLogInfo("xfmrInterfaceValiidate for FieldXfmrDbtoYang") } func (SubTreeXfmrYangToDb) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for SubTreeXfmrYangToDb") + xfmrLogInfo("xfmrInterfaceValiidate for SubTreeXfmrYangToDb") } func (SubTreeXfmrDbToYang) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for SubTreeXfmrDbToYang") + xfmrLogInfo("xfmrInterfaceValiidate for SubTreeXfmrDbToYang") +} +func (SubTreeXfmrSubscribe) xfmrInterfaceValiidate () { + xfmrLogInfo("xfmrInterfaceValiidate for SubTreeXfmrSubscribe") } func (TableXfmrFunc) xfmrInterfaceValiidate () { - log.Info("xfmrInterfaceValiidate for TableXfmrFunc") + xfmrLogInfo("xfmrInterfaceValiidate for TableXfmrFunc") } diff --git a/translib/transformer/xfmr_path_utils.go b/translib/transformer/xfmr_path_utils.go index 8a052cd24..6fbd8830e 100644 --- a/translib/transformer/xfmr_path_utils.go +++ b/translib/transformer/xfmr_path_utils.go @@ -20,6 +20,12 @@ type PathInfo struct { Vars map[string]string } +// HasVar checks if the PathInfo contains given variable. +func (p *PathInfo) HasVar(name string) bool { + _, exists := p.Vars[name] + return exists +} + // Var returns the string value for a path variable. Returns // empty string if no such variable exists. func (p *PathInfo) Var(name string) string { @@ -45,8 +51,17 @@ func NewPathInfo(path string) *PathInfo { name := readUntil(r, '=') value := readUntil(r, ']') + // Handle duplicate parameter names by suffixing "#N" to it. + // N is the number of occurance of that parameter name. + if info.HasVar(name) { + namePrefix := name + for k := 2; info.HasVar(name); k++ { + name = fmt.Sprintf("%s#%d", namePrefix, k) + } + } + if len(name) != 0 { - fmt.Fprintf(&template, "{%s}", name) + fmt.Fprintf(&template, "{}") info.Vars[name] = value } } @@ -57,17 +72,50 @@ func NewPathInfo(path string) *PathInfo { } func readUntil(r *strings.Reader, delim byte) string { - var buff strings.Builder - for { - c, err := r.ReadByte() - if err == nil && c != delim { - buff.WriteByte(c) - } else { - break + var buff strings.Builder + var escaped bool + + for { + c, err := r.ReadByte() + if err != nil || (c == delim && !escaped) { + break + } else if c == '\\' && !escaped { + escaped = true + } else { + escaped = false + buff.WriteByte(c) + } + } + + return buff.String() +} + +// SplitPath splits the ygot path into parts. +func SplitPath(path string) []string { + var parts []string + var start int + var inEscape, inKey bool + + path = strings.TrimPrefix(path, "/") + + for i, c := range path { + switch { + case inEscape: + inEscape = false + case c == '\'': + inEscape = true + case c == '[': + inKey = true + case c == ']': + inKey = false + case c == '/' && !inEscape && !inKey: + parts = append(parts, path[start:i]) + start = i + 1 } } - return buff.String() + parts = append(parts, path[start:]) + return parts } func RemoveXPATHPredicates(s string) (string, error) { diff --git a/translib/transformer/xlate.go b/translib/transformer/xlate.go index 20d80a162..3317eaaeb 100644 --- a/translib/transformer/xlate.go +++ b/translib/transformer/xlate.go @@ -28,6 +28,7 @@ import ( "strings" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" ) const ( @@ -36,15 +37,10 @@ const ( REPLACE UPDATE DELETE + SUBSCRIBE + MAXOPER ) -type KeySpec struct { - dbNum db.DBNum - Ts db.TableSpec - Key db.Key - Child []KeySpec -} - var XlateFuncs = make(map[string]reflect.Value) var ( @@ -63,18 +59,24 @@ func XlateFuncBind(name string, fn interface{}) (err error) { v.Type().NumIn() XlateFuncs[name] = v } else { - log.Info("Duplicate entry found in the XlateFunc map " + name) + xfmrLogInfo("Duplicate entry found in the XlateFunc map " + name) } return } - +func IsXlateFuncBinded(name string) bool { + if _, ok := XlateFuncs[name]; !ok { + return false + } else { + return true + } +} func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, err error) { if _, ok := XlateFuncs[name]; !ok { - err = errors.New(name + " Xfmr function does not exist.") - return nil, err + log.Warning(name + " Xfmr function does not exist.") + return nil, nil } if len(params) != XlateFuncs[name].Type().NumIn() { - err = ErrParamsNotAdapted + log.Warning("Error parameters not adapted") return nil, nil } in := make([]reflect.Value, len(params)) @@ -85,53 +87,103 @@ func XlateFuncCall(name string, params ...interface{}) (result []reflect.Value, return result, nil } -func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key) error { +func TraverseDb(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) error { + var dataMap = make(RedisDbMap) + + for i := db.ApplDB; i < db.MaxDB; i++ { + dataMap[i] = make(map[string]map[string]db.Value) + } + + err := traverseDbHelper(dbs, spec, &dataMap, parentKey, dbTblKeyGetCache) + if err != nil { + log.Warning("Couldn't get data from traverseDbHelper") + return err + } + /* db data processing */ + curMap := make(map[int]map[db.DBNum]map[string]map[string]db.Value) + curMap[GET] = dataMap + err = dbDataXfmrHandler(curMap) + if err != nil { + log.Warning("No conversion in dbdata-xfmr") + return err + } + + for oper, dbData := range curMap { + if oper == GET { + for dbNum, tblData := range dbData { + mapCopy((*result)[dbNum], tblData) + } + } + } + return nil +} + +func traverseDbHelper(dbs [db.MaxDB]*db.DB, spec KeySpec, result *map[db.DBNum]map[string]map[string]db.Value, parentKey *db.Key, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) error { var err error - var dbOpts db.Options + var dbOpts db.Options = getDBOptions(spec.DbNum) - dbOpts = getDBOptions(spec.dbNum) separator := dbOpts.KeySeparator - log.Infof("key separator for table %v in Db %v is %v", spec.Ts.Name, spec.dbNum, separator) if spec.Key.Len() > 0 { // get an entry with a specific key - data, err := dbs[spec.dbNum].GetEntry(&spec.Ts, spec.Key) - if err != nil { - return err - } + if spec.Ts.Name != XFMR_NONE_STRING { // Do not traverse for NONE table + data, err := dbs[spec.DbNum].GetEntry(&spec.Ts, spec.Key) + queriedDbInfo := make(map[string]map[string]bool) + queriedDbTblInfo := make(map[string]bool) + queriedDbTblInfo[strings.Join(spec.Key.Comp, separator)] = true + queriedDbInfo[spec.Ts.Name] = queriedDbTblInfo + if dbTblKeyGetCache == nil { + dbTblKeyGetCache = make(map[db.DBNum]map[string]map[string]bool) + } + dbTblKeyGetCache[spec.DbNum] = queriedDbInfo + if err != nil { + log.Warningf("Couldn't get data for tbl(%v), key(%v) in traverseDbHelper", spec.Ts.Name, spec.Key) + return err + } - if (*result)[spec.dbNum][spec.Ts.Name] == nil { - (*result)[spec.dbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, separator): data} - } else { - (*result)[spec.dbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, separator)] = data + if (*result)[spec.DbNum][spec.Ts.Name] == nil { + (*result)[spec.DbNum][spec.Ts.Name] = map[string]db.Value{strings.Join(spec.Key.Comp, separator): data} + } else { + (*result)[spec.DbNum][spec.Ts.Name][strings.Join(spec.Key.Comp, separator)] = data + } } - if len(spec.Child) > 0 { for _, ch := range spec.Child { - err = TraverseDb(dbs, ch, result, &spec.Key) + err = traverseDbHelper(dbs, ch, result, &spec.Key, dbTblKeyGetCache) } } } else { // TODO - GetEntry support with regex patten, 'abc*' for optimization - keys, err := dbs[spec.dbNum].GetKeys(&spec.Ts) - if err != nil { - return err - } - for i, _ := range keys { - if parentKey != nil { - // TODO - multi-depth with a custom delimiter - if strings.Index(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, separator)) == -1 { - continue + if spec.Ts.Name != XFMR_NONE_STRING { //Do not traverse for NONE table + keys, err := dbs[spec.DbNum].GetKeys(&spec.Ts) + if err != nil { + log.Warningf("Couldn't get keys for tbl(%v) in traverseDbHelper", spec.Ts.Name) + return err + } + xfmrLogInfoAll("keys for table %v in Db %v are %v", spec.Ts.Name, spec.DbNum, keys) + for i := range keys { + if parentKey != nil && !spec.IgnoreParentKey { + // TODO - multi-depth with a custom delimiter + if !strings.Contains(strings.Join(keys[i].Comp, separator), strings.Join((*parentKey).Comp, separator)) { + continue + } } + spec.Key = keys[i] + err = traverseDbHelper(dbs, spec, result, parentKey, dbTblKeyGetCache) + if err != nil { + log.Warningf("Traversal didn't fetch for : %v", err) + } } - spec.Key = keys[i] - err = TraverseDb(dbs, spec, result, parentKey) - } + } else if len(spec.Child) > 0 { + for _, ch := range spec.Child { + err = traverseDbHelper(dbs, ch, result, &spec.Key, dbTblKeyGetCache) + } + } } return err } -func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]KeySpec, error) { +func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}) (*[]KeySpec, error) { var err error var retdbFormat = make([]KeySpec, 0) @@ -140,17 +192,27 @@ func XlateUriToKeySpec(uri string, ygRoot *ygot.GoStruct, t *interface{}) (*[]Ke if isSonicYang(uri) { /* Extract the xpath and key from input xpath */ xpath, keyStr, tableName := sonicXpathKeyExtract(uri) + if tblSpecInfo, ok := xDbSpecMap[tableName]; ok && tblSpecInfo.hasXfmrFn { + /* key from uri should be converted into redis-db key, to read data */ + keyStr, err = dbKeyValueXfmrHandler(CREATE, tblSpecInfo.dbIndex, tableName, keyStr) + if err != nil { + log.Warningf("Value-xfmr for table(%v) & key(%v) didn't do conversion.", tableName, keyStr) + return &retdbFormat, err + } + } + retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr) } else { /* Extract the xpath and key from input xpath */ - xpath, keyStr, _ := xpathKeyExtract(nil, ygRoot, 0, uri) - retdbFormat = FillKeySpecs(xpath, keyStr, &retdbFormat) + retData, _ := xpathKeyExtract(nil, ygRoot, GET, uri, requestUri, nil, nil, txCache, nil) + retdbFormat = FillKeySpecs(retData.xpath, retData.dbKey, &retdbFormat) } return &retdbFormat, err } func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]KeySpec){ + var err error if xYangSpecMap == nil { return *retdbFormat } @@ -160,11 +222,26 @@ func FillKeySpecs(yangXpath string , keyStr string, retdbFormat *[]KeySpec) ([]K if xpathInfo.tableName != nil { dbFormat := KeySpec{} dbFormat.Ts.Name = *xpathInfo.tableName - dbFormat.dbNum = xpathInfo.dbIndex + dbFormat.DbNum = xpathInfo.dbIndex + if len(xYangSpecMap[yangXpath].xfmrKey) > 0 || xYangSpecMap[yangXpath].keyName != nil { + dbFormat.IgnoreParentKey = true + } else { + dbFormat.IgnoreParentKey = false + } if keyStr != "" { + if tblSpecInfo, ok := xDbSpecMap[dbFormat.Ts.Name]; ok && tblSpecInfo.hasXfmrFn { + /* key from uri should be converted into redis-db key, to read data */ + keyStr, err = dbKeyValueXfmrHandler(CREATE, dbFormat.DbNum, dbFormat.Ts.Name, keyStr) + if err != nil { + log.Warningf("Value-xfmr for table(%v) & key(%v) didn't do conversion.", dbFormat.Ts.Name, keyStr) + } + } dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) } for _, child := range xpathInfo.childTable { + if child == dbFormat.Ts.Name { + continue + } if xDbSpecMap != nil { if _, ok := xDbSpecMap[child]; ok { chlen := len(xDbSpecMap[child].yangXpath) @@ -208,7 +285,7 @@ func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpe if _, ok := xDbSpecMap[tableName]; ok { cdb = xDbSpecMap[tableName].dbIndex } - dbFormat.dbNum = cdb + dbFormat.DbNum = cdb if keyStr != "" { dbFormat.Key.Comp = append(dbFormat.Key.Comp, keyStr) } @@ -220,13 +297,13 @@ func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpe if _, ok := xDbSpecMap[container]; ok { dbInfo := xDbSpecMap[container] if dbInfo.fieldType == "container" { - for dir, _ := range dbInfo.dbEntry.Dir { + for dir := range dbInfo.dbEntry.Dir { _, ok := xDbSpecMap[dir] if ok && xDbSpecMap[dir].dbEntry.Node.Statement().Keyword == "container" { cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir - dbFormat.dbNum = cdb + dbFormat.DbNum = cdb retdbFormat = append(retdbFormat, dbFormat) } } @@ -237,12 +314,14 @@ func fillSonicKeySpec(xpath string , tableName string, keyStr string) ( []KeySpe return retdbFormat } -func XlateToDb(path string, opcode int, d *db.DB, yg *ygot.GoStruct, yt *interface{}) (map[string]map[string]db.Value, error) { +func XlateToDb(path string, opcode int, d *db.DB, yg *ygot.GoStruct, yt *interface{}, jsonPayload []byte, txCache interface{}, skipOrdTbl *bool) (map[int]RedisDbMap, map[string]map[string]db.Value, map[string]map[string]db.Value, error) { var err error + requestUri := path + jsonData := make(map[string]interface{}) device := (*yg).(*ocbinds.Device) - jsonStr, err := ygot.EmitJSON(device, &ygot.EmitJSONConfig{ + jsonStr, _ := ygot.EmitJSON(device, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, Indent: " ", SkipValidation: true, @@ -251,85 +330,106 @@ func XlateToDb(path string, opcode int, d *db.DB, yg *ygot.GoStruct, yt *interfa }, }) - jsonData := make(map[string]interface{}) err = json.Unmarshal([]byte(jsonStr), &jsonData) if err != nil { - log.Errorf("Error: failed to unmarshal json.") - return nil, err + errStr := "Error: failed to unmarshal json." + err = tlerr.InternalError{Format: errStr} + return nil, nil, nil, err } // Map contains table.key.fields - var result = make(map[string]map[string]db.Value) + var result = make(map[int]RedisDbMap) + var yangDefValMap = make(map[string]map[string]db.Value) + var yangAuxValMap = make(map[string]map[string]db.Value) switch opcode { case CREATE: - log.Info("CREATE case") - err = dbMapCreate(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("CREATE case") + err = dbMapCreate(d, yg, opcode, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for create request.") + log.Warning("Data translation from yang to db failed for create request.") } case UPDATE: - log.Info("UPDATE case") - err = dbMapUpdate(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("UPDATE case") + err = dbMapUpdate(d, yg, opcode, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for update request.") + log.Warning("Data translation from yang to db failed for update request.") } case REPLACE: - log.Info("REPLACE case") - err = dbMapUpdate(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("REPLACE case") + err = dbMapUpdate(d, yg, opcode, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for replace request.") + log.Warning("Data translation from yang to db failed for replace request.") } case DELETE: - log.Info("DELETE case") - err = dbMapDelete(d, yg, opcode, path, jsonData, result) + xfmrLogInfo("DELETE case") + err = dbMapDelete(d, yg, opcode, path, requestUri, jsonData, result, txCache, skipOrdTbl) if err != nil { - log.Errorf("Error: Data translation from yang to db failed for delete request.") + log.Warning("Data translation from yang to db failed for delete request.") } } - return result, err + return result, yangDefValMap, yangAuxValMap, err } -func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB) ([]byte, error) { +func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, txCache interface{}) ([]byte, bool, error) { var err error var payload []byte - log.Info("received xpath =", uri) - - keySpec, err := XlateUriToKeySpec(uri, ygRoot, nil) - var dbresult = make(map[db.DBNum]map[string]map[string]db.Value) + var inParamsForGet xlateFromDbParams + xfmrLogInfo("received xpath = " + uri) + requestUri := uri + keySpec, _ := XlateUriToKeySpec(uri, requestUri, ygRoot, nil, txCache) + var dbresult = make(RedisDbMap) for i := db.ApplDB; i < db.MaxDB; i++ { dbresult[i] = make(map[string]map[string]db.Value) } + inParamsForGet.dbTblKeyGetCache = make(map[db.DBNum]map[string]map[string]bool) + for _, spec := range *keySpec { - err := TraverseDb(dbs, spec, &dbresult, nil) + err := TraverseDb(dbs, spec, &dbresult, nil, inParamsForGet.dbTblKeyGetCache) if err != nil { - log.Error("TraverseDb() failure") - return payload, err + log.Warning("TraverseDb() didn't fetch data.") } } - payload, err = XlateFromDb(uri, ygRoot, dbs, dbresult) + isEmptyPayload := false + payload, isEmptyPayload, err = XlateFromDb(uri, ygRoot, dbs, dbresult, txCache, inParamsForGet) if err != nil { - log.Error("XlateFromDb() failure.") - return payload, err + return payload, true, err } - return payload, err + return payload, isEmptyPayload, err } -func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data map[db.DBNum]map[string]map[string]db.Value) ([]byte, error) { +func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data RedisDbMap, txCache interface{}, inParamsForGet xlateFromDbParams) ([]byte, bool, error) { var err error var result []byte - var dbData = make(map[db.DBNum]map[string]map[string]db.Value) + var dbData = make(RedisDbMap) var cdb db.DBNum = db.ConfigDB + var xpath string dbData = data + requestUri := uri + /* Check if the parent table exists for RFC compliance */ + var exists bool + subOpMapDiscard := make(map[int]*RedisDbMap) + exists, err = verifyParentTable(nil, dbs, ygRoot, GET, uri, dbData, txCache, subOpMapDiscard) + xfmrLogInfoAll("verifyParentTable() returned - exists - %v, err - %v", exists, err) + if err != nil { + log.Warningf("Cannot perform GET Operation on uri %v due to - %v", uri, err) + return []byte(""), true, err + } + if !exists { + err = tlerr.NotFoundError{Format:"Resource Not found"} + return []byte(""), true, err + } + if isSonicYang(uri) { - xpath, keyStr, tableName := sonicXpathKeyExtract(uri) + lxpath, keyStr, tableName := sonicXpathKeyExtract(uri) + xpath = lxpath if (tableName != "") { dbInfo, ok := xDbSpecMap[tableName] if !ok { @@ -340,51 +440,104 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data m tokens:= strings.Split(xpath, "/") // Format /module:container/tableName/listname[key]/fieldName if tokens[SONIC_TABLE_INDEX] == tableName { - fieldName := tokens[len(tokens)-1] - dbSpecField := tableName + "/" + fieldName - _, ok := xDbSpecMap[dbSpecField] - if ok && xDbSpecMap[dbSpecField].fieldType == "leaf" { - dbData[cdb] = extractFieldFromDb(tableName, keyStr, fieldName, data[cdb]) + fieldName := "" + if len(tokens) > SONIC_FIELD_INDEX { + fieldName = tokens[SONIC_FIELD_INDEX] + dbSpecField := tableName + "/" + fieldName + dbSpecFieldInfo, ok := xDbSpecMap[dbSpecField] + if ok && fieldName != "" { + yangNodeType := yangTypeGet(xDbSpecMap[dbSpecField].dbEntry) + if yangNodeType == YANG_LEAF_LIST { + fieldName = fieldName + "@" + } + if ((yangNodeType == YANG_LEAF_LIST) || (yangNodeType == YANG_LEAF)) { + dbData[cdb], err = extractFieldFromDb(tableName, keyStr, fieldName, data[cdb]) + // return resource not found when the leaf/leaf-list instance(not entire leaf-list GET) not found + if ((err != nil) && ((yangNodeType == YANG_LEAF) || ((yangNodeType == YANG_LEAF_LIST) && (strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/"))))) { + return []byte(""), true, err + } + if ((yangNodeType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/")))) { + leafListInstVal, valErr := extractLeafListInstFromUri(uri) + if valErr != nil { + return []byte(""), true, valErr + } + if dbSpecFieldInfo.xfmrValue != nil { + inParams := formXfmrDbInputRequest(CREATE, cdb, tableName, keyStr, fieldName, leafListInstVal) + retVal, err := valueXfmrHandler(inParams, *dbSpecFieldInfo.xfmrValue) + if err != nil { + log.Warningf("value-xfmr:fldpath(\"%v\") val(\"%v\"):err(\"%v\").", dbSpecField, leafListInstVal, err) + return []byte(""), true, err + } + log.Info("valueXfmrHandler() retuned ", retVal) + leafListInstVal = retVal + } + if leafListInstExists(dbData[cdb][tableName][keyStr].Field[fieldName], leafListInstVal) { + /* Since translib already fills in ygRoot with queried leaf-list instance, do not + fill in resFldValMap or else Unmarshall of payload(resFldValMap) into ygotTgt in + app layer will create duplicate instances in result. + */ + log.Info("Queried leaf-list instance exists.") + return []byte("{}"), false, nil + } else { + xfmrLogInfoAll("Queried leaf-list instance does not exist - %v", uri) + return []byte(""), true, tlerr.NotFoundError{Format:"Resource not found"} + } + } + } + } } } } } else { - xpath, _ := XfmrRemoveXPATHPredicates(uri) + lxpath, _, _ := XfmrRemoveXPATHPredicates(uri) + xpath = lxpath if _, ok := xYangSpecMap[xpath]; ok { cdb = xYangSpecMap[xpath].dbIndex } } - payload, err := dbDataToYangJsonCreate(uri, ygRoot, dbs, &dbData, cdb) - log.Info("Payload generated:", payload) + dbTblKeyGetCache := inParamsForGet.dbTblKeyGetCache + inParamsForGet = formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, xpath, GET, "", "", &dbData, txCache, nil, false) + inParamsForGet.xfmrDbTblKeyCache = make(map[string]tblKeyCache) + inParamsForGet.dbTblKeyGetCache = dbTblKeyGetCache + payload, isEmptyPayload, err := dbDataToYangJsonCreate(inParamsForGet) + xfmrLogInfoAll("Payload generated : " + payload) if err != nil { - log.Errorf("Error: failed to create json response from DB data.") - return nil, err + log.Warning("Couldn't create json response from DB data.") + return nil, isEmptyPayload, err } + xfmrLogInfo("Created json response from DB data.") result = []byte(payload) - return result, err + return result, isEmptyPayload, err } -func extractFieldFromDb(tableName string, keyStr string, fieldName string, data map[string]map[string]db.Value) (map[string]map[string]db.Value) { +func extractFieldFromDb(tableName string, keyStr string, fieldName string, data map[string]map[string]db.Value) (map[string]map[string]db.Value, error) { var dbVal db.Value var dbData = make(map[string]map[string]db.Value) + var err error if tableName != "" && keyStr != "" && fieldName != "" { if data[tableName][keyStr].Field != nil { - dbData[tableName] = make(map[string]db.Value) - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = data[tableName][keyStr].Field[fieldName] - dbData[tableName][keyStr] = dbVal + fldVal, fldValExists := data[tableName][keyStr].Field[fieldName] + if fldValExists { + dbData[tableName] = make(map[string]db.Value) + dbVal.Field = make(map[string]string) + dbVal.Field[fieldName] = fldVal + dbData[tableName][keyStr] = dbVal + } else { + log.Warningf("Field %v doesn't exist in table - %v, instance - %v", fieldName, tableName, keyStr) + err = tlerr.NotFoundError{Format: "Resource not found"} + } } } - return dbData + return dbData, err } func GetModuleNmFromPath(uri string) (string, error) { - log.Infof("received uri %s to extract module name from ", uri) + xfmrLogInfo("received uri %s to extract module name from ", uri) moduleNm, err := uriModuleNameGet(uri) return moduleNm, err } @@ -395,13 +548,308 @@ func GetOrdDBTblList(ygModuleNm string) ([]string, error) { if dbTblList, ok := xDbSpecOrdTblMap[ygModuleNm]; ok { result = dbTblList if len(dbTblList) == 0 { - log.Error("Ordered DB Table list is empty for module name = ", ygModuleNm) + log.Warning("Ordered DB Table list is empty for module name = ", ygModuleNm) err = fmt.Errorf("Ordered DB Table list is empty for module name %v", ygModuleNm) } } else { - log.Error("No entry found in the map of module names to ordered list of DB Tables for module = ", ygModuleNm) + log.Warning("No entry found in the map of module names to ordered list of DB Tables for module = ", ygModuleNm) err = fmt.Errorf("No entry found in the map of module names to ordered list of DB Tables for module = %v", ygModuleNm) } return result, err } + +func GetOrdTblList(xfmrTbl string, uriModuleNm string) []string { + var ordTblList []string + processedTbl := false + var sncMdlList []string = getYangMdlToSonicMdlList(uriModuleNm) + + for _, sonicMdlNm := range(sncMdlList) { + sonicMdlTblInfo := xDbSpecTblSeqnMap[sonicMdlNm] + for _, ordTblNm := range(sonicMdlTblInfo.OrdTbl) { + if xfmrTbl == ordTblNm { + xfmrLogInfo("Found sonic module(%v) whose ordered table list contains table %v", sonicMdlNm, xfmrTbl) + ordTblList = sonicMdlTblInfo.OrdTbl + processedTbl = true + break + } + } + if processedTbl { + break + } + } + return ordTblList + } + +func GetXfmrOrdTblList(xfmrTbl string) []string { + /* get the table hierarchy read from json file */ + var ordTblList []string + if _, ok := sonicOrdTblListMap[xfmrTbl]; ok { + ordTblList = sonicOrdTblListMap[xfmrTbl] + } + return ordTblList +} + +func GetTablesToWatch(xfmrTblList []string, uriModuleNm string) []string { + var depTblList []string + depTblMap := make(map[string]bool) //create to avoid duplicates in depTblList, serves as a Set + processedTbl := false + var sncMdlList []string + var lXfmrTblList []string + + sncMdlList = getYangMdlToSonicMdlList(uriModuleNm) + + // remove duplicates from incoming list of tables + xfmrTblMap := make(map[string]bool) //create to avoid duplicates in xfmrTblList + for _, xfmrTblNm :=range(xfmrTblList) { + xfmrTblMap[xfmrTblNm] = true + } + for xfmrTblNm := range(xfmrTblMap) { + lXfmrTblList = append(lXfmrTblList, xfmrTblNm) + } + + for _, xfmrTbl := range(lXfmrTblList) { + processedTbl = false + //can be optimized if there is a way to know all sonic modules, a given OC-Yang spans over + for _, sonicMdlNm := range(sncMdlList) { + sonicMdlTblInfo := xDbSpecTblSeqnMap[sonicMdlNm] + for _, ordTblNm := range(sonicMdlTblInfo.OrdTbl) { + if xfmrTbl == ordTblNm { + xfmrLogInfo("Found sonic module(%v) whose ordered table list contains table %v", sonicMdlNm, xfmrTbl) + ldepTblList := sonicMdlTblInfo.DepTbl[xfmrTbl] + for _, depTblNm := range(ldepTblList) { + depTblMap[depTblNm] = true + } + //assumption that a table belongs to only one sonic module + processedTbl = true + break + } + } + if processedTbl { + break + } + } + if !processedTbl { + depTblMap[xfmrTbl] = false + } + } + for depTbl := range(depTblMap) { + depTblList = append(depTblList, depTbl) + } + return depTblList +} + +func CallRpcMethod(path string, body []byte, dbs [db.MaxDB]*db.DB) ([]byte, error) { + var err error + var ret []byte + var data []reflect.Value + var rpcFunc = "" + + // TODO - check module name + if isSonicYang(path) { + rpcName := strings.Split(path, ":") + if dbXpathData, ok := xDbSpecMap[rpcName[1]]; ok { + rpcFunc = dbXpathData.rpcFunc + } + } else { + if xpathData, ok := xYangSpecMap[path]; ok { + rpcFunc = xpathData.rpcFunc + } + } + + if rpcFunc != "" { + xfmrLogInfo("RPC callback invoked (%v) \r\n", rpcFunc) + data, err = XlateFuncCall(rpcFunc, body, dbs) + if err != nil { + return nil, err + } + ret = data[0].Interface().([]byte) + if !data[1].IsNil() { + err = data[1].Interface().(error) + } + } else { + log.Warning("Not supported RPC", path) + err = tlerr.NotSupported("Not supported RPC") + } + return ret, err +} + +func AddModelCpbltInfo() map[string]*mdlInfo { + return xMdlCpbltMap +} + +func xfmrSubscSubtreeHandler(inParams XfmrSubscInParams, xfmrFuncNm string) (XfmrSubscOutParams, error) { + var retVal XfmrSubscOutParams + retVal.dbDataMap = nil + retVal.needCache = false + retVal.onChange = false + retVal.nOpts = nil + retVal.isVirtualTbl = false + + xfmrLogInfo("Received inParams %v Subscribe Subtree function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall("Subscribe_" + xfmrFuncNm, inParams) + if err != nil { + return retVal, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == SUBSC_SBT_XFMR_RET_ARGS { + // subtree xfmr returns err as second value in return data list from .Call() + if ret[SUBSC_SBT_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[SUBSC_SBT_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Subscribe Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return retVal, err + } + } + } + if ret[SUBSC_SBT_XFMR_RET_VAL_INDX].Interface() != nil { + retVal = ret[SUBSC_SBT_XFMR_RET_VAL_INDX].Interface().(XfmrSubscOutParams) + } + } + return retVal, err +} + +func XlateTranslateSubscribe(path string, dbs [db.MaxDB]*db.DB, txCache interface{}) (XfmrTranslateSubscribeInfo, error) { + xfmrLogInfo("Received subcription path : %v", path) + var err error + var subscribe_result XfmrTranslateSubscribeInfo + subscribe_result.DbDataMap = make(RedisDbMap) + subscribe_result.PType = Sample + subscribe_result.MinInterval = 0 + subscribe_result.OnChange = false + subscribe_result.NeedCache = true + + for { + done := true + xpath, _, predc_err := XfmrRemoveXPATHPredicates(path) + if predc_err != nil { + log.Warningf("cannot convert request Uri to yang xpath - %v, %v", path, predc_err) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + xpathData, ok := xYangSpecMap[xpath] + if ((!ok) || (xpathData == nil)) { + log.Warningf("xYangSpecMap data not found for xpath : %v", xpath) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + if (xpathData.subscribePref == nil || ((xpathData.subscribePref != nil) &&(len(strings.TrimSpace(*xpathData.subscribePref)) == 0))) { + subscribe_result.PType = Sample + } else { + if *xpathData.subscribePref == "onchange" { + subscribe_result.PType = OnChange + } else { + subscribe_result.PType = Sample + } + } + subscribe_result.MinInterval = xpathData.subscribeMinIntvl + + if xpathData.subscribeOnChg == XFMR_DISABLE { + xfmrLogInfo("Susbcribe OnChange disabled for request Uri - %v", path) + subscribe_result.PType = Sample + subscribe_result.DbDataMap = nil + //err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + //request uri should be terminal yang object for onChange to be supported + if xpathData.hasNonTerminalNode { + xfmrLogInfo("Susbcribe request Uri is not a terminal yang object - %v", path) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + /*request uri is a key-leaf directly under the list + eg. /openconfig-xyz:xyz/listA[key=value]/key + /openconfig-xyz:xyz/listA[key_1=value][key_2=value]/key_1 + */ + if xpathData.isKey { + xfmrLogInfo("Susbcribe request Uri is not a terminal yang object - %v", path) + err = tlerr.NotSupportedError{Format: "Subscribe not supported", Path: path} + break + } + + xpath_dbno := xpathData.dbIndex + retData, xPathKeyExtractErr := xpathKeyExtract(dbs[xpath_dbno], nil, SUBSCRIBE, path, path, nil, nil, txCache, nil) + if ((len(xpathData.xfmrFunc) == 0) && ((xPathKeyExtractErr != nil) || ((len(strings.TrimSpace(retData.dbKey)) == 0) || (len(strings.TrimSpace(retData.tableName)) == 0)))) { + log.Warning("Error while extracting DB table/key for uri", path, "error - ", xPathKeyExtractErr) + err = xPathKeyExtractErr + break + } + if (len(xpathData.xfmrFunc) > 0) { //subtree + var inParams XfmrSubscInParams + inParams.uri = path + inParams.dbDataMap = subscribe_result.DbDataMap + inParams.dbs = dbs + inParams.subscProc = TRANSLATE_SUBSCRIBE + st_result, st_err := xfmrSubscSubtreeHandler(inParams, xpathData.xfmrFunc) + if st_err != nil { + err = st_err + break + } + subscribe_result.OnChange = st_result.onChange + xfmrLogInfo("Subtree subcribe on change %v", subscribe_result.OnChange) + if subscribe_result.OnChange { + if st_result.dbDataMap != nil { + subscribe_result.DbDataMap = st_result.dbDataMap + xfmrLogInfo("Subtree subcribe dbData %v", subscribe_result.DbDataMap) + } + subscribe_result.NeedCache = st_result.needCache + xfmrLogInfo("Subtree subcribe need Cache %v", subscribe_result.NeedCache) + } else { + subscribe_result.DbDataMap = nil + } + if st_result.nOpts != nil { + subscribe_result.PType = st_result.nOpts.pType + xfmrLogInfo("Subtree subcribe pType %v", subscribe_result.PType) + subscribe_result.MinInterval = st_result.nOpts.mInterval + xfmrLogInfo("Subtree subcribe min interval %v", subscribe_result.MinInterval) + } + } else { + subscribe_result.OnChange = true + subscribe_result.DbDataMap[xpath_dbno] = map[string]map[string]db.Value{retData.tableName: {retData.dbKey: {}}} + } + if done { + break + } + } // end of infinite for + + return subscribe_result, err + +} + +func IsTerminalNode(uri string) (bool, error) { + xpath, _, err := XfmrRemoveXPATHPredicates(uri) + if xpathData, ok := xYangSpecMap[xpath]; ok { + if !xpathData.hasNonTerminalNode { + return true, nil + } + } else { + log.Warningf("xYangSpecMap data not found for xpath : %v", xpath) + errStr := "xYangSpecMap data not found for xpath." + err = tlerr.InternalError{Format: errStr} + } + + return false, err +} + +func IsLeafNode(uri string) bool { + result := false + xpath, _, err := XfmrRemoveXPATHPredicates(uri) + if err != nil { + log.Warningf("For uri - %v, couldn't convert to xpath - %v", uri, err) + return result + } + xfmrLogInfoAll("received xpath - %v", xpath) + if xpathData, ok := xYangSpecMap[xpath]; ok { + if yangTypeGet(xpathData.yangEntry) == YANG_LEAF { + result = true + } + } else { + errStr := "xYangSpecMap data not found for xpath - " + xpath + log.Warning(errStr) + } + return result +} diff --git a/translib/transformer/xlate_cascade_del.go b/translib/transformer/xlate_cascade_del.go new file mode 100644 index 000000000..1a5be4a3a --- /dev/null +++ b/translib/transformer/xlate_cascade_del.go @@ -0,0 +1,151 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2019 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + log "github.com/golang/glog" + "strings" + "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/Azure/sonic-mgmt-common/cvl" + "errors" + "fmt" +) + +func getDbTblCbkName(tableName string) string { + return tableName + "_cascade_cfg_hdl" +} + +func xfmrDbTblCbkHandler (inParams XfmrDbTblCbkParams, tableName string) error { + xfmrLogInfoAll("Received inParams %v xfmrDbTblCbkHandler function name %v", inParams, tableName) + + ret, err := XlateFuncCall(getDbTblCbkName(tableName), inParams) + if err != nil { + xfmrLogInfo("xfmrDbTblCbkHandler: %v failed.", getDbTblCbkName(tableName)) + return err + } + if ((ret != nil) && (len(ret)>0)) { + if ret[0].Interface() != nil { + err = ret[0].Interface().(error) + if err != nil { + log.Warningf("Table callback method i(%v) returned error - %v.", getDbTblCbkName(tableName), err) + } + } + } + + return err +} + + + +func handleCascadeDelete(d *db.DB, dbDataMap map[int]map[db.DBNum]map[string]map[string]db.Value, cascadeDelTbl [] string) error { + xfmrLogInfo("handleCascadeDelete : %v, cascadeDelTbl : %v.", dbDataMap, cascadeDelTbl) + + var err error + cvlSess, cvlRetSess := cvl.ValidationSessOpen() + if cvlRetSess != cvl.CVL_SUCCESS { + xfmrLogInfo("handleCascadeDelete : cvl.ValidationSessOpen failed.") + err = fmt.Errorf("%v", "cvl.ValidationSessOpen failed") + return err + } + defer cvl.ValidationSessClose(cvlSess) + + for operIndex, redisMap := range dbDataMap { + if operIndex != DELETE { + continue + } + for dbIndex, dbMap := range redisMap { + if dbIndex != db.ConfigDB { + continue + } + for tblIndex, tblMap := range dbMap { + if !contains(cascadeDelTbl, tblIndex) { + continue + } else { + for key, entry := range tblMap { + // need to generate key based on the db type as of now just considering configdb + // and using "|" as tablename and key seperator + depKey := tblIndex + "|" + key + depList := cvlSess.GetDepDataForDelete(depKey) + xfmrLogInfo("handleCascadeDelete : depKey : %v, depList- %v, entry : %v", depKey, depList, entry) + for depIndex, depEntry := range depList { + for depEntkey, depEntkeyInst := range depEntry.Entry { + depEntkeyList := strings.SplitN(depEntkey, "|", 2) + cbkHdlName := depEntkeyList[0] + "_cascade_cfg_hdl" + if IsXlateFuncBinded(cbkHdlName) { + //handle callback for table call Table Call back method and consolidate the data + inParams := formXfmrDbTblCbkParams(d, DELETE, depEntry.RefKey, depEntkeyList[0], depEntkeyList[1], depEntkeyInst, dbDataMap[DELETE]) + xfmrLogInfo("handleCascadeDelete CBKHDL present depIndex %v, inParams : %v ", depIndex, inParams) + err = xfmrDbTblCbkHandler(inParams, depEntkeyList[0]) + if err == nil { + for operIdx, operMap := range inParams.delDepDataMap { + if _, ok := dbDataMap[operIdx]; !ok { + dbDataMap[operIdx] = make(map[db.DBNum]map[string]map[string]db.Value) + } + for dbIndx, dbMap := range *operMap { + if _, ok := dbDataMap[operIdx][dbIndx]; !ok { + dbDataMap[operIdx][dbIndx] = make(map[string]map[string]db.Value) + } + mapMerge(dbDataMap[operIdx][dbIndx], dbMap, operIdx) + } + } + } else { + xfmrLogInfo("handleCascadeDelete - xfmrDbTblCbkHandler failed.") + return errors.New("xfmrDbTblCbkHandler failed for table: " + depEntkeyList[0] + ", Key: " + depEntkeyList[1]) + } + } else { + if _, ok := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]]; !ok { + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]] = make(map[string]db.Value) + } + if _, ok := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]]; !ok { + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]] = db.Value{Field: make(map[string]string)} + } + + if len(depEntkeyInst) > 0 { + for depEntAttr, depEntAttrInst := range depEntkeyInst { + if _, ok := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]].Field[depEntAttr]; !ok { + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]].Field[depEntAttr] = "" + } + + if len(depEntAttrInst) > 0 { + val := dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]] + if strings.HasSuffix(depEntAttr, "@") { + valList := val.GetList(depEntAttr) + if !contains(valList, depEntAttrInst) { + valList = append(valList, depEntAttrInst) + val.SetList(depEntAttr, valList) + } + } else { + val.Set(depEntAttr, depEntAttrInst) + } + + dbDataMap[DELETE][db.ConfigDB][depEntkeyList[0]][depEntkeyList[1]] = val + } + } + } + } + } + } + } + } + } + } + } + return nil +} + diff --git a/translib/transformer/xlate_datastructs.go b/translib/transformer/xlate_datastructs.go new file mode 100644 index 000000000..052efd115 --- /dev/null +++ b/translib/transformer/xlate_datastructs.go @@ -0,0 +1,109 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/openconfig/ygot/ygot" + "regexp" +) + +var rgpIpv6, rgpMac, rgpIsMac, rgpKeyExtract, rgpSncKeyExtract *regexp.Regexp + +type tblKeyCache struct { + dbKey string + dbTblList []string +} + +type KeySpec struct { + DbNum db.DBNum + Ts db.TableSpec + Key db.Key + Child []KeySpec + IgnoreParentKey bool +} + +type NotificationType int +const ( + Sample NotificationType = iota + OnChange +) + +type XfmrTranslateSubscribeInfo struct { + DbDataMap RedisDbMap + MinInterval int + NeedCache bool + PType NotificationType + OnChange bool +} + +type xpathTblKeyExtractRet struct { + xpath string + tableName string + dbKey string + isVirtualTbl bool +} + +type xlateFromDbParams struct { + d *db.DB //current db + dbs [db.MaxDB]*db.DB + curDb db.DBNum + ygRoot *ygot.GoStruct + uri string + requestUri string //original uri using which a curl/NBI request is made + oper int + dbDataMap *map[db.DBNum]map[string]map[string]db.Value + // subOpDataMap map[int]*RedisDbMap // used to add an in-flight data with a sub-op + // param interface{} + txCache interface{} + // skipOrdTblChk *bool + // pCascadeDelTbl *[] string //used to populate list of tables needed cascade delete by subtree overloaded methods + xpath string //curr uri xpath + tbl string + tblKey string + resultMap map[string]interface{} + validate bool + xfmrDbTblKeyCache map[string]tblKeyCache + dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool +} + +type xlateToParams struct { + d *db.DB + ygRoot *ygot.GoStruct + oper int + uri string + requestUri string + xpath string + keyName string + jsonData interface{} + resultMap map[int]RedisDbMap + result map[string]map[string]db.Value + txCache interface{} + tblXpathMap map[string]map[string]map[string]bool + subOpDataMap map[int]*RedisDbMap + pCascadeDelTbl *[]string + xfmrErr *error + name string + value interface{} + tableName string + yangDefValMap map[string]map[string]db.Value + yangAuxValMap map[string]map[string]db.Value + xfmrDbTblKeyCache map[string]tblKeyCache + dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool +} diff --git a/translib/transformer/xlate_del_to_db.go b/translib/transformer/xlate_del_to_db.go new file mode 100644 index 000000000..8da6b79bb --- /dev/null +++ b/translib/transformer/xlate_del_to_db.go @@ -0,0 +1,888 @@ + //////////////////////////////////////////////////////////////////////////////// + // // + // Copyright 2019 Dell, Inc. // + // // + // Licensed under the Apache License, Version 2.0 (the "License"); // + // you may not use this file except in compliance with the License. // + // You may obtain a copy of the License at // + // // + // http://www.apache.org/licenses/LICENSE-2.0 // + // // + // Unless required by applicable law or agreed to in writing, software // + // distributed under the License is distributed on an "AS IS" BASIS, // + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // + // See the License for the specific language governing permissions and // + // limitations under the License. // + // // + //////////////////////////////////////////////////////////////////////////////// + +package transformer +import ( + "errors" + "strings" + "fmt" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" + "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/openconfig/goyang/pkg/yang" + "github.com/openconfig/ygot/ygot" + log "github.com/golang/glog" +) + +func tblKeyDataGet(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum) ([]string, bool, error) { + var err error + var dbs [db.MaxDB]*db.DB + var tblList []string + dbs[cdb] = xlateParams.d + isVirtualTbl := false + + xfmrLogInfoAll("Get table data for (\"%v\")", xlateParams.uri) + if (xYangSpecMap[xlateParams.xpath].tableName != nil) && (len(*xYangSpecMap[xlateParams.xpath].tableName) > 0) { + tblList = append(tblList, *xYangSpecMap[xlateParams.xpath].tableName) + } else if xYangSpecMap[xlateParams.xpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[xlateParams.xpath].xfmrTbl + if len(xfmrTblFunc) > 0 { + inParams := formXfmrInputRequest(xlateParams.d, dbs, cdb, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, xlateParams.keyName, dbDataMap, nil, nil, xlateParams.txCache) + tblList, err = xfmrTblHandlerFunc(xfmrTblFunc, inParams, xlateParams.xfmrDbTblKeyCache) + if err != nil { + return tblList, isVirtualTbl, err + } + if inParams.isVirtualTbl != nil { + isVirtualTbl = *(inParams.isVirtualTbl) + } + } + } + tbl := xlateParams.tableName + if tbl != "" { + if !contains(tblList, tbl) { + tblList = append(tblList, tbl) + } + } + return tblList, isVirtualTbl, err +} + +func subTreeXfmrDelDataGet(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum, spec *yangXpathInfo, chldSpec *yangXpathInfo, subTreeResMap *map[string]map[string]db.Value) error { + var dbs [db.MaxDB]*db.DB + dbs[cdb] = xlateParams.d + + xfmrLogInfoAll("Handle subtree for (\"%v\")", xlateParams.uri) + if (len(chldSpec.xfmrFunc) > 0) { + if ((len(spec.xfmrFunc) == 0) || ((len(spec.xfmrFunc) > 0) && + (spec.xfmrFunc != chldSpec.xfmrFunc))) { + inParams := formXfmrInputRequest(xlateParams.d, dbs, cdb, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, "", + dbDataMap, xlateParams.subOpDataMap, nil, xlateParams.txCache) + retMap, err := xfmrHandler(inParams, chldSpec.xfmrFunc) + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", chldSpec.xfmrFunc, err) + return err + } + mapCopy(*subTreeResMap, retMap) + if xlateParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(*xlateParams.pCascadeDelTbl, tblNm) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, tblNm) + } + } + } + } + } + return nil +} + +func yangListDelData(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, subTreeResMap *map[string]map[string]db.Value, isFirstCall bool) error { + var err, perr error + var dbs [db.MaxDB]*db.DB + var tblList []string + xfmrLogInfoAll("yangListDelData Received xlateParams - %v \n dbDataMap - %v\n subTreeResMap - %v\n isFirstCall - %v", xlateParams, dbDataMap, subTreeResMap, isFirstCall) + fillFields := false + removedFillFields := false + virtualTbl := false + tblOwner := true + keyName := xlateParams.keyName + parentTbl := "" + parentKey := "" + + spec, xpathOk := xYangSpecMap[xlateParams.xpath] + if !xpathOk { + return err + } + if ((spec.yangEntry != nil) && (spec.yangEntry.ReadOnly())) { + xfmrLogInfoAll("For Uri - %v skip delete processing since its a Read Only node", xlateParams.uri) + return err + } + cdb := spec.dbIndex + dbs[cdb] = xlateParams.d + dbOpts := getDBOptions(cdb) + separator := dbOpts.KeySeparator + + if !isFirstCall { + xpathKeyExtRet, err := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + if err != nil { + xfmrLogInfoAll("Received error from xpathKeyExtract for uri : %v, error: %v", xlateParams.uri, err) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + log.Warningf("Error received (\"%v\"), ecode :%v", err, ecode) + if ecode { + return err + } + } + } + if xpathKeyExtRet.isVirtualTbl { + virtualTbl = true + } + keyName = xpathKeyExtRet.dbKey + xlateParams.tableName = xpathKeyExtRet.tableName + xlateParams.keyName = keyName + } + + tblList, virtualTbl, err = tblKeyDataGet(xlateParams, dbDataMap, cdb) + if err != nil { + return err + } + + xfmrLogInfoAll("tblList(%v), tbl(%v), key(%v) for uri (\"%v\")", tblList, xlateParams.tableName, xlateParams.keyName, xlateParams.uri) + for _, tbl := range(tblList) { + curDbDataMap, ferr := fillDbDataMapForTbl(xlateParams.uri, xlateParams.xpath, tbl, keyName, cdb, dbs, xlateParams.dbTblKeyGetCache) + if ((ferr == nil) && len(curDbDataMap) > 0) { + mapCopy((*dbDataMap)[cdb], curDbDataMap[cdb]) + } + } + + // Not required to process parent and current table as subtree is already invoked before we get here + // We only need to traverse nested subtrees here + if isFirstCall && len(spec.xfmrFunc) == 0 { + parentUri := parentUriGet(xlateParams.uri) + xpathKeyExtRet , xerr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, parentUri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + parentTbl = xpathKeyExtRet.tableName + parentKey = xpathKeyExtRet.dbKey + perr = xerr + xfmrLogInfoAll("Parent Uri - %v, ParentTbl - %v, parentKey - %v", parentUri, parentTbl, parentKey) + } + + for _, tbl := range(tblList) { + tblData, ok := (*dbDataMap)[cdb][tbl] + xfmrLogInfoAll("Process Tbl - %v", tbl) + var curTbl, curKey string + var cerr error + if ok { + for dbKey := range tblData { + xfmrLogInfoAll("Process Tbl - %v with dbKey - %v", tbl, dbKey) + _, curUri, kerr := dbKeyToYangDataConvert(xlateParams.uri, xlateParams.requestUri, xlateParams.xpath, tbl, dbDataMap, dbKey, separator, xlateParams.txCache) + if kerr != nil { + continue + } + if spec.virtualTbl != nil && *spec.virtualTbl { + virtualTbl = true + } + + // Not required to check for table inheritence case here as we have a subtree and subtree is already processed before we get here + // We only need to traverse nested subtrees here + if len(spec.xfmrFunc) == 0 { + + xpathKeyExtRet, xerr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + curKey = xpathKeyExtRet.dbKey + curTbl = xpathKeyExtRet.tableName + cerr = xerr + xfmrLogInfoAll("Current Uri - %v, CurrentTbl - %v, CurrentKey - %v", curUri, curTbl, curKey) + + if dbKey != curKey { + continue + } + if isFirstCall { + if perr == nil && cerr == nil { + if len(curTbl) > 0 && parentTbl != curTbl { + /* Non-inhertited table case */ + xfmrLogInfoAll("Non-inhertaed table case, uri - %v", curUri) + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false + /* Fill only fields */ + fillFields = true + } + } else if len(curTbl) > 0 { + /* Inhertited table case */ + xfmrLogInfoAll("Inherited table case, uri - %v", curUri) + if len(parentKey) > 0 { + if parentKey == curKey { // List within list or List within container, where container map to entire table + xfmrLogInfoAll("Parent key is same as current key") + xfmrLogInfoAll("Request from NBI is at same level as that of current list - %v", curUri) + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false // since query is at this level, this will make sure to add instance to result + } + /* Fill only fields */ + fillFields = true + } else { /*same table but different keys */ + xfmrLogInfoAll("Inherited table but parent key is NOT same as current key") + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false + /* Fill only fields */ + fillFields = true + } + } + } else { + /*same table but no parent-key exists, parent must be a container wth just tableNm annot with no keyXfmr/Nm */ + xfmrLogInfoAll("Inherited table but no parent key available") + if spec.tblOwner != nil && !*spec.tblOwner { + xfmrLogInfoAll("For uri - %v, table owner - %v", xlateParams.uri, *spec.tblOwner) + tblOwner = false + /* Fill only fields */ + fillFields = true + + } + } + } else { // len(curTbl) = 0 + log.Warning("No table found for Uri - %v ", curUri) + } + } + } else { + /* if table instance already filled and there are no feilds present then it's instance level delete. + If fields present then its fields delet and not instance delete + */ + if tblData, tblDataOk := xlateParams.result[curTbl]; tblDataOk { + if fieldMap, fieldMapOk := tblData[curKey]; fieldMapOk { + xfmrLogInfoAll("Found table instance same as that of requestUri") + if len(fieldMap.Field) > 0 { + /* Fill only fields */ + xfmrLogInfoAll("Found table instance same as that of requestUri with fields.") + fillFields = true + } + } + } + + }// end of if isFirstCall + } // end if !subtree case + xfmrLogInfoAll("For uri - %v , table-owner - %v, fillFields - %v", curUri, tblOwner, fillFields) + if fillFields || spec.hasChildSubTree || isFirstCall { + for yangChldName := range spec.yangEntry.Dir { + chldXpath := xlateParams.xpath+"/"+yangChldName + chldUri := curUri+"/"+yangChldName + chldSpec, ok := xYangSpecMap[chldXpath] + if (ok && ((chldSpec.yangEntry != nil) && (!chldSpec.yangEntry.ReadOnly()))) { + chldYangType := chldSpec.yangDataType + curXlateParams := xlateParams + curXlateParams.uri = chldUri + curXlateParams.xpath = chldXpath + curXlateParams.tableName = "" + curXlateParams.keyName = "" + + if (chldSpec.dbIndex == db.ConfigDB) && (len(chldSpec.xfmrFunc) > 0) { + err = subTreeXfmrDelDataGet(curXlateParams, dbDataMap, cdb, spec, chldSpec, subTreeResMap) + if err != nil { + return err + } + } + if chldYangType == YANG_CONTAINER { + err = yangContainerDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if chldYangType == YANG_LIST { + err = yangListDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if (chldSpec.dbIndex == db.ConfigDB) && ((chldYangType == YANG_LEAF) || (chldYangType == YANG_LEAF_LIST)) && !virtualTbl { + if len(curTbl) == 0 { + continue + } + if len(curKey) == 0 { //at list level key should always be there + xfmrLogInfoAll("No key avaialble for uri - %v", curUri) + continue + } + if chldYangType == YANG_LEAF && chldSpec.isKey { + if isFirstCall { + if !tblOwner { //add dummy field to identify when to fill fields only at children traversal + dataToDBMapAdd(curTbl, curKey, curXlateParams.result, "FillFields", "true") + } else { + dataToDBMapAdd(curTbl, curKey, curXlateParams.result, "", "") + } + } + } else if fillFields { + //strip off the leaf/leaf-list for mapFillDataUtil takes uri without it + curXlateParams.uri = xlateParams.uri + curXlateParams.name = chldSpec.yangEntry.Name + curXlateParams.tableName = curTbl + curXlateParams.keyName = curKey + err = mapFillDataUtil(curXlateParams) + if err != nil { + xfmrLogInfoAll("Error received (\"%v\")", err) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + log.Warningf("Error received (\"%v\"), ecode :%v", err, ecode) + if ecode { + return err + } + } + } + if !removedFillFields { + if fieldMap, ok := curXlateParams.result[curTbl][curKey]; ok { + if len(fieldMap.Field) > 1 { + delete(curXlateParams.result[curTbl][curKey].Field, "FillFields") + removedFillFields = true + } else if len(fieldMap.Field) == 1 { + if _, ok := curXlateParams.result[curTbl][curKey].Field["FillFields"]; !ok { + removedFillFields = true + } + } + } + } + } + } // end of Leaf case + } // if rw + } // end of curUri children traversal loop + } // Child Subtree or fill fields + } // end of for dbKey loop + } // end of tbl in dbDataMap +} // end of for tbl loop +return err +} + +func yangContainerDelData(xlateParams xlateToParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, subTreeResMap *map[string]map[string]db.Value, isFirstCall bool) error { + var err error + var dbs [db.MaxDB]*db.DB + spec, ok := xYangSpecMap[xlateParams.xpath] + cdb := spec.dbIndex + dbs[cdb] = xlateParams.d + removedFillFields := false + var curTbl, curKey string + + if !ok { + return err + } + + if (ok && (spec.yangEntry != nil) && (spec.yangEntry.ReadOnly())) { + return err + } + + xfmrLogInfoAll("Traverse container for DELETE at uri (\"%v\")", xlateParams.uri) + + fillFields := false + + // Not required to process parent and current table as subtree is already invoked before we get here + // We only need to traverse nested subtrees here + if len(spec.xfmrFunc) == 0 { + xpathKeyExtRet, cerr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + curKey = xpathKeyExtRet.dbKey + curTbl = xpathKeyExtRet.tableName + if cerr != nil { + log.Warningf("Received xpathKeyExtract error for uri: %v : err %v", xlateParams.uri, cerr) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + xfmrLogInfoAll("Error received (\"%v\"), ecode :%v", cerr, ecode) + if ecode { + return cerr + } + } + } + + if isFirstCall { + parentUri := parentUriGet(xlateParams.uri) + parentXpathKeyExtRet, perr := xpathKeyExtract(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, parentUri, xlateParams.requestUri, nil, xlateParams.subOpDataMap, xlateParams.txCache, xlateParams.xfmrDbTblKeyCache) + parentTbl := parentXpathKeyExtRet.tableName + parentKey := parentXpathKeyExtRet.dbKey + if perr == nil && cerr == nil && len(curTbl) > 0 { + if len(curKey) > 0 { + xfmrLogInfoAll("DELETE handling at Container parentTbl %v, curTbl %v, curKey %v", parentTbl, curTbl, curKey) + if (parentTbl != curTbl) { + // Non inhertited table + if (spec.tblOwner != nil) && !(*spec.tblOwner) { + // Fill fields only + xfmrLogInfoAll("DELETE handling at Container Non inhertited table and not table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "FillFields", "true") + fillFields = true + } else if (spec.keyName != nil && len(*spec.keyName) > 0) || len(spec.xfmrKey) > 0 { + // Table owner && Key transformer present. Fill table instance + xfmrLogInfoAll("DELETE handling at Container Non inhertited table & table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "","") + } else { + // Fallback case. Ideally should not enter here + fillFields = true + } + } else { + if curKey != parentKey { + if (spec.tblOwner != nil) && !(*spec.tblOwner) { + xfmrLogInfoAll("DELETE handling at Container inhertited table and not table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "FillFields", "true") + fillFields = true + } else { + // Instance delete + xfmrLogInfoAll("DELETE handling at Container Non inhertited table & table Owner") + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "","") + } + } else { + // if Instance already filled do not fill fields + xfmrLogInfoAll("DELETE handling at Container Inherited table") + //Fill fields only + if len(curTbl) > 0 && len(curKey) > 0 { + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "FillFields", "true") + fillFields = true + } + } + } + } else { + if (spec.tblOwner != nil) && !(*spec.tblOwner) { + // Fill fields only + xfmrLogInfoAll("DELETE handling at Container Non inhertited table and not table Owner No Key available. table: %v, key: %v", curTbl, curKey) + } else { + // Table owner && Key transformer present. Fill table instance + xfmrLogInfoAll("DELETE handling at Container Non inhertited table & table Owner. No Key Delete complete TABLE : %v", curTbl) + dataToDBMapAdd(curTbl, curKey, xlateParams.result, "","") + } + } + } else { + xfmrLogInfoAll("perr: %v cerr: %v curTbl: %v, curKey: %v", perr, cerr, curTbl, curKey) + } + } else { + // Inherited Table. We always expect the curTbl entry in xlateParams.result + // if Instance already filled do not fill fields + xfmrLogInfoAll("DELETE handling at Container Inherited table curTbl: %v, curKey %v", curTbl, curKey) + if tblMap, ok := xlateParams.result[curTbl]; ok { + if fieldMap, ok := tblMap[curKey]; ok { + if len(fieldMap.Field) == 0 { + xfmrLogInfoAll("Inhertited table & Instance delete case. Skip fields fill") + } else { + xfmrLogInfoAll("Inhertited table & fields fill for table :%v", curTbl) + fillFields = true + } + } + } + } + + } + xfmrLogInfoAll("uri %v fillFields %v, hasChildSubtree %v, isFirstCall %v", xlateParams.uri, fillFields, spec.hasChildSubTree, isFirstCall) + + if (fillFields || spec.hasChildSubTree || isFirstCall) { + for yangChldName := range spec.yangEntry.Dir { + chldXpath := xlateParams.xpath+"/"+yangChldName + chldUri := xlateParams.uri+"/"+yangChldName + chldSpec, ok := xYangSpecMap[chldXpath] + if (ok && ((chldSpec.yangEntry != nil) && (!chldSpec.yangEntry.ReadOnly()))) { + chldYangType := chldSpec.yangDataType + curXlateParams := xlateParams + curXlateParams.uri = chldUri + curXlateParams.xpath = chldXpath + curXlateParams.tableName = curTbl + curXlateParams.keyName = curKey + + if (chldSpec.dbIndex == db.ConfigDB) && (len(chldSpec.xfmrFunc) > 0) { + err = subTreeXfmrDelDataGet(curXlateParams, dbDataMap, cdb, spec, chldSpec, subTreeResMap) + if err != nil { + return err + } + } + if chldYangType == YANG_CONTAINER { + err = yangContainerDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if chldYangType == YANG_LIST { + err = yangListDelData(curXlateParams, dbDataMap, subTreeResMap, false) + if err != nil { + return err + } + } else if (chldSpec.dbIndex == db.ConfigDB) && (chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST) && fillFields { + //strip off the leaf/leaf-list for mapFillDataUtil takes uri without it + curXlateParams.uri = xlateParams.uri + curXlateParams.name = chldSpec.yangEntry.Name + err = mapFillDataUtil(curXlateParams) + if err != nil { + xfmrLogInfoAll("Error received during leaf fill (\"%v\")", err) + switch e := err.(type) { + case tlerr.TranslibXfmrRetError: + ecode := e.XlateFailDelReq + log.Warningf("Error received (\"%v\"), ecode :%v", err, ecode) + if ecode { + return err + } + } + } + if !removedFillFields { + if fieldMap, ok := curXlateParams.result[curTbl][curKey]; ok { + if len(fieldMap.Field) > 1 { + delete(curXlateParams.result[curTbl][curKey].Field, "FillFields") + removedFillFields = true + } else if len(fieldMap.Field) == 1 { + if _, ok := curXlateParams.result[curTbl][curKey].Field["FillFields"]; !ok { + removedFillFields = true + } + } + } + } + } else { + xfmrLogInfoAll("%v", "Instance Fill case. Have filled the result table with table and key") + } + } + } + } + return err +} + +func allChildTblGetToDelete(xlateParams xlateToParams) (map[string]map[string]db.Value, error) { + var err error + subTreeResMap := make(map[string]map[string]db.Value) + xpath, _, _ := XfmrRemoveXPATHPredicates(xlateParams.requestUri) + spec, ok := xYangSpecMap[xpath] + isFirstCall := true + if !ok { + errStr := "Xpath not found in spec-map:" + xpath + return subTreeResMap, errors.New(errStr) + } + + dbDataMap := make(RedisDbMap) + for i := db.ApplDB; i < db.MaxDB; i++ { + dbDataMap[i] = make(map[string]map[string]db.Value) + } + + if len(spec.xfmrFunc) > 0 { + // Subtree is already invoked before we get here + // Not required to process parent and current tables + isFirstCall = false + } + + xfmrLogInfoAll("Req-uri (\"%v\") to traverse for delete", xlateParams.requestUri) + if ok && spec.yangEntry != nil { + xlateParams.uri = xlateParams.requestUri + xlateParams.xpath = xpath + if (spec.yangDataType == YANG_LIST) { + err = yangListDelData(xlateParams, &dbDataMap, &subTreeResMap, isFirstCall) + return subTreeResMap, err + } else if (spec.yangDataType == YANG_CONTAINER) { + err = yangContainerDelData(xlateParams, &dbDataMap, &subTreeResMap, isFirstCall) + } + } + return subTreeResMap, err +} + +/* Get the db table, key and field name for the incoming delete request */ +func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, jsonData interface{}, resultMap map[int]map[db.DBNum]map[string]map[string]db.Value, txCache interface{}, skipOrdTbl *bool) error { + var err error + var result = make(map[string]map[string]db.Value) + subOpDataMap := make(map[int]*RedisDbMap) + var xfmrErr error + *skipOrdTbl = false + var cascadeDelTbl []string + + /* Check if the parent table exists for RFC compliance */ + var exists bool + var dbs [db.MaxDB]*db.DB + subOpMapDiscard := make(map[int]*RedisDbMap) + exists, err = verifyParentTable(d, dbs, ygRoot, oper, uri, nil, txCache, subOpMapDiscard) + xfmrLogInfoAll("verifyParentTable() returned - exists - %v, err - %v", exists, err) + if err != nil { + if exists { + // Special case when we delete at container that does exist. Not required to do translation. Do not return error either. + return nil + } + log.Warningf("Cannot perform Operation %v on uri %v due to - %v", oper, uri, err) + return err + } + if !exists { + errStr := fmt.Sprintf("Parent table does not exist for uri(%v)", uri) + return tlerr.InternalError{Format: errStr} + } + for i := 0; i < MAXOPER; i++ { + resultMap[i] = make(map[db.DBNum]map[string]map[string]db.Value) + } + + if isSonicYang(uri) { + xpathPrefix, keyName, tableName := sonicXpathKeyExtract(uri) + xfmrLogInfo("Delete req: uri(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", uri, keyName, xpathPrefix, tableName) + resultMap[oper][db.ConfigDB] = result + xlateToData := formXlateToDbParam(d, ygRoot, oper, uri, requestUri, xpathPrefix, keyName, jsonData, resultMap, result, txCache, nil, subOpDataMap, &cascadeDelTbl, &xfmrErr, "","",tableName) + err = sonicYangReqToDbMapDelete(xlateToData) + if err != nil { + return err + } + } else { + xpathKeyExtRet, err := xpathKeyExtract(d, ygRoot, oper, uri, requestUri, nil, subOpDataMap, txCache, nil) + if err != nil { + return err + } + xfmrLogInfo("Delete req: uri(\"%v\"), key(\"%v\"), xpath(\"%v\"), tableName(\"%v\").", uri, xpathKeyExtRet.dbKey, xpathKeyExtRet.xpath, xpathKeyExtRet.tableName) + spec, ok := xYangSpecMap[xpathKeyExtRet.xpath] + if ok { + specYangType := yangTypeGet(spec.yangEntry) + moduleNm := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleNm) + if modSpecInfo, specOk := xYangSpecMap[moduleNm]; specOk && (len(modSpecInfo.xfmrPre) > 0) { + var dbs [db.MaxDB]*db.DB + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) + xfmrLogInfo("Invoked pre-transformer: %v, oper: %v, subOpDataMap: %v ", + modSpecInfo.xfmrPre, oper, subOpDataMap) + if err != nil { + log.Warningf("Pre-transformer: %v failed.(err:%v)", modSpecInfo.xfmrPre, err) + return err + } + } + + if spec.cascadeDel == XFMR_ENABLE && xpathKeyExtRet.tableName != "" && xpathKeyExtRet.tableName != XFMR_NONE_STRING { + if !contains(cascadeDelTbl, xpathKeyExtRet.tableName) { + cascadeDelTbl = append(cascadeDelTbl, xpathKeyExtRet.tableName) + } + } + curXlateParams := formXlateToDbParam(d, ygRoot, oper, uri, requestUri, xpathKeyExtRet.xpath, xpathKeyExtRet.dbKey, jsonData, resultMap, result, txCache, nil, subOpDataMap, &cascadeDelTbl, &xfmrErr, "", "", xpathKeyExtRet.tableName) + curXlateParams.xfmrDbTblKeyCache = make(map[string]tblKeyCache) + if len(spec.xfmrFunc) > 0 { + var dbs [db.MaxDB]*db.DB + cdb := spec.dbIndex + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, uri, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + stRetData, err := xfmrHandler(inParams, spec.xfmrFunc) + if err == nil { + mapCopy(result, stRetData) + } else { + return err + } + // TODO: Nested subtree invoke + curResult, cerr := allChildTblGetToDelete(curXlateParams) + if cerr != nil { + return cerr + } else { + mapCopy(result, curResult) + } + + if inParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(cascadeDelTbl, tblNm) { + cascadeDelTbl = append(cascadeDelTbl, tblNm) + } + } + } + } else if specYangType == YANG_LEAF || specYangType == YANG_LEAF_LIST { + if len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) > 0 { + dataToDBMapAdd(xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, result, "", "") + xpath := xpathKeyExtRet.xpath + uriItemList := splitUri(strings.TrimSuffix(uri, "/")) + uriItemListLen := len(uriItemList) + var luri string + if uriItemListLen > 0 { + luri = strings.Join(uriItemList[:uriItemListLen-1], "/") //strip off the leaf/leaf-list for mapFillDataUtil takes uri without it + + } + if specYangType == YANG_LEAF { + _, ok := xYangSpecMap[xpath] + if ok && len(xYangSpecMap[xpath].defVal) > 0 { + // Do not fill def value if leaf does not map to any redis field + dbSpecXpath := xpathKeyExtRet.tableName + "/" + xYangSpecMap[xpath].fieldName + _, mapped := xDbSpecMap[dbSpecXpath] + if mapped || len(xYangSpecMap[xpath].xfmrField) > 0 { + curXlateParams.uri = luri + curXlateParams.name = spec.yangEntry.Name + curXlateParams.value = xYangSpecMap[xpath].defVal + err = mapFillDataUtil(curXlateParams) + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } + if len(subOpDataMap) > 0 && subOpDataMap[UPDATE] != nil { + subOperMap := subOpDataMap[UPDATE] + mapCopy((*subOperMap)[db.ConfigDB], result) + } else { + var redisMap = new(RedisDbMap) + var dbresult = make(RedisDbMap) + for i := db.ApplDB; i < db.MaxDB; i++ { + dbresult[i] = make(map[string]map[string]db.Value) + } + redisMap = &dbresult + (*redisMap)[db.ConfigDB] = result + subOpDataMap[UPDATE] = redisMap + } + } + result = make(map[string]map[string]db.Value) + } else { + curXlateParams.uri = luri + curXlateParams.name = spec.yangEntry.Name + err = mapFillDataUtil(curXlateParams) + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } + } + } else if specYangType == YANG_LEAF_LIST { + var fieldVal []interface{} + leafListInstVal, valErr := extractLeafListInstFromUri(uri) + if valErr == nil && leafListInstVal != "" { + fieldVal = append(fieldVal, leafListInstVal) + } + curXlateParams.uri = luri + curXlateParams.name = spec.yangEntry.Name + curXlateParams.value = fieldVal + err = mapFillDataUtil(curXlateParams) + + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } + } + } else { + log.Warningf("No proper table and key information to fill result map for uri %v, table: %v, key %v", uri, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + } + } else { + xfmrLogInfoAll("Before calling allChildTblGetToDelete result: %v", curXlateParams.result) + curResult, cerr := allChildTblGetToDelete(curXlateParams) + if cerr != nil { + err = cerr + return err + } else { + mapCopy(result, curResult) + } + xfmrLogInfoAll("allChildTblGetToDelete result: %v subtree curResult: %v", result, curResult) + // Add the child tables to delete when table at request URI is not available or its complete table delete request (not specific instance) + chResult := make(map[string]map[string]db.Value) + if (len(xpathKeyExtRet.tableName) == 0 || (len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) == 0)) && len(spec.childTable) > 0 { + for _, child := range spec.childTable { + chResult[child] = make(map[string]db.Value) + } + xfmrLogInfoAll("Before adding children result: %v result with child tables: %v", result, chResult) + } + mapCopy(result, chResult) + } + + _, ok = xYangSpecMap[moduleNm] + if ok && len(xYangSpecMap[moduleNm].xfmrPost) > 0 { + xfmrLogInfo("Invoke post transformer: %v", xYangSpecMap[moduleNm].xfmrPost) + var dbs [db.MaxDB]*db.DB + var dbresult = make(RedisDbMap) + dbresult[db.ConfigDB] = result + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", &dbresult, subOpDataMap, nil, txCache) + result, err = postXfmrHandlerFunc(xYangSpecMap[moduleNm].xfmrPost, inParams) + if err != nil { + return err + } + if inParams.skipOrdTblChk != nil { + *skipOrdTbl = *(inParams.skipOrdTblChk) + xfmrLogInfo("skipOrdTbl flag: %v", *skipOrdTbl) + } + if inParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(cascadeDelTbl, tblNm) { + cascadeDelTbl = append(cascadeDelTbl, tblNm) + } + } + } + } + + if len(result) > 0 { + resultMap[oper][db.ConfigDB] = result + } + + if len(subOpDataMap) > 0 { + for op, data := range subOpDataMap { + if len(*data) > 0 { + for dbType, dbData := range (*data) { + if len(dbData) > 0 { + if _, ok := resultMap[op][dbType]; !ok { + resultMap[op][dbType] = make(map[string]map[string]db.Value) + } + mapCopy(resultMap[op][dbType], (*subOpDataMap[op])[dbType]) + } + } + } + } + + } + /* for container/list delete req , it should go through, even if there are any leaf default-yang-values */ + } + } // End OC yang handling + + err = dbDataXfmrHandler(resultMap) + if err != nil { + log.Warningf("Failed in dbdata-xfmr for %v", resultMap) + return err + } + if (len(cascadeDelTbl) > 0) { + cdErr := handleCascadeDelete(d, resultMap, cascadeDelTbl) + if cdErr != nil { + xfmrLogInfo("Cascade Delete Failed for cascadeDelTbl (%v), Error: (%v)", cascadeDelTbl, cdErr) + return cdErr + } + } + + printDbData(resultMap, nil, "/tmp/yangToDbDataDel.txt") + xfmrLogInfo("Delete req: uri(\"%v\") resultMap(\"%v\").", uri, resultMap) + return err +} + +func sonicYangReqToDbMapDelete(xlateParams xlateToParams) error { + var err error + if (xlateParams.tableName != "") { + // Specific table entry case + xlateParams.result[xlateParams.tableName] = make(map[string]db.Value) + isFieldReq := false + if (xlateParams.keyName != "") { + // Specific key case + var dbVal db.Value + tokens:= strings.Split(xlateParams.xpath, "/") + if tokens[SONIC_TABLE_INDEX] == xlateParams.tableName { + fieldName := "" + if len(tokens) > SONIC_FIELD_INDEX { + fieldName = tokens[SONIC_FIELD_INDEX] + } + + if fieldName != "" { + isFieldReq = true + dbSpecField := xlateParams.tableName + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok { + yangType := xDbSpecMap[dbSpecField].fieldType + // terminal node case + if yangType == YANG_LEAF_LIST { + dbVal.Field = make(map[string]string) + dbFldVal := "" + //check if it is a specific item in leaf-list delete + leafListInstVal, valErr := extractLeafListInstFromUri(xlateParams.requestUri) + if valErr == nil { + dbFldVal, err = unmarshalJsonToDbData(xDbSpecMap[dbSpecField].dbEntry, dbSpecField, fieldName, leafListInstVal) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", dbSpecField, err) + return err + } + } + fieldName = fieldName + "@" + dbVal.Field[fieldName] = dbFldVal + } + if yangType == YANG_LEAF { + dbVal.Field = make(map[string]string) + dbVal.Field[fieldName] = "" + } + } + } + } + xlateParams.result[xlateParams.tableName][xlateParams.keyName] = dbVal + } + if !isFieldReq { + if tblSpecInfo, ok := xDbSpecMap[xlateParams.tableName]; ok && (tblSpecInfo.cascadeDel == XFMR_ENABLE) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, xlateParams.tableName) + } + } + } else { + // Get all table entries + // If table name not available in xpath get top container name + _, ok := xDbSpecMap[xlateParams.xpath] + if ok && xDbSpecMap[xlateParams.xpath] != nil { + dbInfo := xDbSpecMap[xlateParams.xpath] + if dbInfo.fieldType == "container" { + for dir := range dbInfo.dbEntry.Dir { + if tblSpecInfo, ok := xDbSpecMap[dir]; ok && tblSpecInfo.cascadeDel == XFMR_ENABLE { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, dir) + } + if dbInfo.dbEntry.Dir[dir].Config != yang.TSFalse { + xlateParams.result[dir] = make(map[string]db.Value) + } + } + } + } + } + return nil +} + diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index 949f6c27c..8d26c75d2 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -23,91 +23,18 @@ import ( "github.com/Azure/sonic-mgmt-common/translib/db" "strings" "encoding/json" - "os" "strconv" "errors" - "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "sync" "github.com/openconfig/goyang/pkg/yang" - "github.com/openconfig/ygot/ygot" - "github.com/openconfig/ygot/ytypes" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" log "github.com/golang/glog" ) type typeMapOfInterface map[string]interface{} -func xfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { - result := make(map[string]interface{}) - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - log.Infof("Subtree transformer function(\"%v\") invoked for yang path(\"%v\").", xYangSpecMap[xpath].xfmrFunc, xpath) - _, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - log.Infof("Failed to retrieve data for xpath(\"%v\") err(%v).", inParams.uri, err) - return result, err - } - - ocbSch, _ := ocbinds.Schema() - schRoot := ocbSch.RootSchema() - device := (*inParams.ygRoot).(*ocbinds.Device) - - path, _ := ygot.StringToPath(inParams.uri, ygot.StructuredPath, ygot.StringSlicePath) - for _, p := range path.Elem { - pathSlice := strings.Split(p.Name, ":") - p.Name = pathSlice[len(pathSlice)-1] - if len(p.Key) > 0 { - for ekey, ent := range p.Key { - eslice := strings.Split(ent, ":") - p.Key[ekey] = eslice[len(eslice)-1] - } - } - } - - nodeList, nodeErr := ytypes.GetNode(schRoot, device, path) - if nodeErr != nil { - log.Infof("Failed to get node for xpath(\"%v\") err(%v).", inParams.uri, err) - return result, err - } - node := nodeList[0].Data - nodeYgot, _ := (node).(ygot.ValidatedGoStruct) - payload, err := ygot.EmitJSON(nodeYgot, &ygot.EmitJSONConfig{ Format: ygot.RFC7951, - Indent: " ", SkipValidation: true, - RFC7951Config: &ygot.RFC7951JSONConfig{ AppendModuleName: false, }, - }) - err = json.Unmarshal([]byte(payload), &result) - return result, err -} - -func leafXfmrHandlerFunc(inParams XfmrParams) (map[string]interface{}, error) { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - return nil, err - } - if ret != nil { - fldValMap := ret[0].Interface().(map[string]interface{}) - return fldValMap, nil - } else { - return nil, nil - } -} - -func validateHandlerFunc(inParams XfmrParams) (bool) { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xYangSpecMap[xpath].validateFunc, inParams) - if err != nil { - return false - } - return ret[0].Interface().(bool) -} - -func xfmrTblHandlerFunc(xfmrTblFunc string, inParams XfmrParams) []string { - ret, err := XlateFuncCall(xfmrTblFunc, inParams) - if err != nil { - return []string{} - } - return ret[0].Interface().([]string) -} - +var mapCopyMutex = &sync.Mutex{} func DbValToInt(dbFldVal string, base int, size int, isUint bool) (interface{}, error) { var res interface{} @@ -124,11 +51,88 @@ func DbValToInt(dbFldVal string, base int, size int, isUint bool) (interface{}, return res, err } -func DbToYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string, dbFldVal string) (interface{}, error) { - log.Infof("Received FieldXpath %v, yngTerminalNdDtType %v and Db field value %v to be converted to yang data-type.", fldXpath, yngTerminalNdDtType, dbFldVal) +func getLeafrefRefdYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string) (yang.TypeKind) { + if yngTerminalNdDtType == yang.Yleafref { + var entry *yang.Entry + var path string + if _, ok := xDbSpecMap[fldXpath]; ok { + path = xDbSpecMap[fldXpath].dbEntry.Type.Path + entry = xDbSpecMap[fldXpath].dbEntry + } else if _, ok := xYangSpecMap[fldXpath]; ok { + path = xYangSpecMap[fldXpath].yangEntry.Type.Path + entry = xYangSpecMap[fldXpath].yangEntry + } + path = stripAugmentedModuleNames(path) + path = path[1:] + xfmrLogInfoAll("Received path %v for FieldXpath %v", path, fldXpath) + if strings.Contains(path, "..") { + if entry != nil && len(path) > 0 { + // Referenced path within same yang file + xpath, _, err := XfmrRemoveXPATHPredicates(path) + if err != nil { + log.Warningf("error in XfmrRemoveXPATHPredicates %v", path) + return yngTerminalNdDtType + } + xpath = xpath[1:] + pathList := strings.Split(xpath, "/") + for _, x := range pathList { + if x == ".." { + entry = entry.Parent + } else { + if _,ok := entry.Dir[x]; ok { + entry = entry.Dir[x] + } + } + } + if entry != nil && entry.Type != nil { + yngTerminalNdDtType = entry.Type.Kind + xfmrLogInfoAll("yangLeaf datatype %v", yngTerminalNdDtType) + if yngTerminalNdDtType == yang.Yleafref { + leafPath := getXpathFromYangEntry(entry) + xfmrLogInfoAll("getLeafrefRefdYangType: xpath for leafref type:%v",leafPath) + return getLeafrefRefdYangType(yngTerminalNdDtType, leafPath) + } + } + } + } else if len(path) > 0 { + // Referenced path in a different yang file + xpath, _, err := XfmrRemoveXPATHPredicates(path) + if err != nil { + log.Warningf("error in XfmrRemoveXPATHPredicates %v", xpath) + return yngTerminalNdDtType + } + // Form xpath based on sonic or non sonic yang path + if strings.Contains(xpath, "sonic") { + pathList := strings.Split(xpath, "/") + xpath = pathList[SONIC_TABLE_INDEX]+ "/" + pathList[SONIC_FIELD_INDEX] + if _, ok := xDbSpecMap[xpath]; ok { + yngTerminalNdDtType = xDbSpecMap[xpath].dbEntry.Type.Kind + } + + } else { + xpath = replacePrefixWithModuleName(xpath) + if _, ok := xYangSpecMap[xpath]; ok { + yngTerminalNdDtType = xYangSpecMap[xpath].dbEntry.Type.Kind + } + } + + } + xfmrLogInfoAll("yangLeaf datatype %v", yngTerminalNdDtType) + } + return yngTerminalNdDtType +} + +func DbToYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string, dbFldVal string) (interface{}, interface{}, error) { + xfmrLogInfoAll("Received FieldXpath %v, yngTerminalNdDtType %v and Db field value %v to be converted to yang data-type.", fldXpath, yngTerminalNdDtType, dbFldVal) var res interface{} + var resPtr interface{} var err error const INTBASE = 10 + + if yngTerminalNdDtType == yang.Yleafref { + yngTerminalNdDtType = getLeafrefRefdYangType(yngTerminalNdDtType, fldXpath) + } + switch yngTerminalNdDtType { case yang.Ynone: log.Warning("Yang node data-type is non base yang type") @@ -136,53 +140,70 @@ func DbToYangType(yngTerminalNdDtType yang.TypeKind, fldXpath string, dbFldVal s err = errors.New("Yang node data-type is non base yang type") case yang.Yint8: res, err = DbValToInt(dbFldVal, INTBASE, 8, false) + var resInt8 int8 = int8(res.(int64)) + resPtr = &resInt8 case yang.Yint16: res, err = DbValToInt(dbFldVal, INTBASE, 16, false) + var resInt16 int16 = int16(res.(int64)) + resPtr = &resInt16 case yang.Yint32: res, err = DbValToInt(dbFldVal, INTBASE, 32, false) + var resInt32 int32 = int32(res.(int64)) + resPtr = &resInt32 case yang.Yuint8: res, err = DbValToInt(dbFldVal, INTBASE, 8, true) + var resUint8 uint8 = uint8(res.(uint64)) + resPtr = &resUint8 case yang.Yuint16: res, err = DbValToInt(dbFldVal, INTBASE, 16, true) + var resUint16 uint16 = uint16(res.(uint64)) + resPtr = &resUint16 case yang.Yuint32: res, err = DbValToInt(dbFldVal, INTBASE, 32, true) + var resUint32 uint32 = uint32(res.(uint64)) + resPtr = &resUint32 case yang.Ybool: if res, err = strconv.ParseBool(dbFldVal); err != nil { log.Warningf("Non Bool type for yang leaf-list item %v", dbFldVal) } - case yang.Ybinary, yang.Ydecimal64, yang.Yenum, yang.Yidentityref, yang.Yint64, yang.Yuint64, yang.Ystring, yang.Yunion,yang.Yleafref: + var resBool bool = res.(bool) + resPtr = &resBool + case yang.Ybinary, yang.Ydecimal64, yang.Yenum, yang.Yidentityref, yang.Yint64, yang.Yuint64, yang.Ystring, yang.Yunion, yang.Yleafref: // TODO - handle the union type // Make sure to encode as string, expected by util_types.go: ytypes.yangToJSONType - log.Info("Yenum/Ystring/Yunion(having all members as strings) type for yangXpath ", fldXpath) + xfmrLogInfoAll("Yenum/Ystring/Yunion(having all members as strings) type for yangXpath %v", fldXpath) res = dbFldVal + var resString string = res.(string) + resPtr = &resString case yang.Yempty: logStr := fmt.Sprintf("Yang data type for xpath %v is Yempty.", fldXpath) - log.Error(logStr) + log.Warning(logStr) err = errors.New(logStr) default: logStr := fmt.Sprintf("Unrecognized/Unhandled yang-data type(%v) for xpath %v.", fldXpath, yang.TypeKindToName[yngTerminalNdDtType]) - log.Error(logStr) + log.Warning(logStr) err = errors.New(logStr) } - return res, err + return res, resPtr, err } /*convert leaf-list in Db to leaf-list in yang*/ -func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { +func processLfLstDbToYang(fieldXpath string, dbFldVal string, yngTerminalNdDtType yang.TypeKind) []interface{} { valLst := strings.Split(dbFldVal, ",") var resLst []interface{} - const INTBASE = 10 - yngTerminalNdDtType := xDbSpecMap[fieldXpath].dbEntry.Type.Kind + + xfmrLogInfoAll("xpath: %v, dbFldVal: %v", fieldXpath, dbFldVal) switch yngTerminalNdDtType { - case yang.Yenum, yang.Ystring, yang.Yunion, yang.Yleafref: - // TODO handle leaf-ref base type - log.Info("DB leaf-list and Yang leaf-list are of same data-type") + case yang.Ybinary, yang.Ydecimal64, yang.Yenum, yang.Yidentityref, yang.Yint64, yang.Yuint64, yang.Ystring, yang.Yunion: + // TODO - handle the union type.OC yang should have field xfmr.sonic-yang? + // Make sure to encode as string, expected by util_types.go: ytypes.yangToJSONType: + xfmrLogInfoAll("DB leaf-list and Yang leaf-list are of same data-type") for _, fldVal := range valLst { resLst = append(resLst, fldVal) } default: for _, fldVal := range valLst { - resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, fldVal) + resVal, _, err := DbToYangType(yngTerminalNdDtType, fieldXpath, fldVal) if err == nil { resLst = append(resLst, resVal) } @@ -191,16 +212,28 @@ func processLfLstDbToYang(fieldXpath string, dbFldVal string) []interface{} { return resLst } -func sonicDbToYangTerminalNodeFill(tblName string, field string, value string, resultMap map[string]interface{}) { +func sonicDbToYangTerminalNodeFill(field string, inParamsForGet xlateFromDbParams) { resField := field - if len(value) == 0 { - return + value := "" + + if inParamsForGet.dbDataMap != nil { + tblInstFields, dbDataExists := (*inParamsForGet.dbDataMap)[inParamsForGet.curDb][inParamsForGet.tbl][inParamsForGet.tblKey] + if dbDataExists { + fieldVal, valueExists := tblInstFields.Field[field] + if !valueExists { + return + } + value = fieldVal + } else { + return + } } + if strings.HasSuffix(field, "@") { fldVals := strings.Split(field, "@") resField = fldVals[0] } - fieldXpath := tblName + "/" + resField + fieldXpath := inParamsForGet.tbl + "/" + resField xDbSpecMapEntry, ok := xDbSpecMap[fieldXpath] if !ok { log.Warningf("No entry found in xDbSpecMap for xpath %v", fieldXpath) @@ -212,46 +245,60 @@ func sonicDbToYangTerminalNodeFill(tblName string, field string, value string, r } yangType := yangTypeGet(xDbSpecMapEntry.dbEntry) + yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind if yangType == YANG_LEAF_LIST { /* this should never happen but just adding for safetty */ if !strings.HasSuffix(field, "@") { log.Warningf("Leaf-list in Sonic yang should also be a leaf-list in DB, its not for xpath %v", fieldXpath) return } - resLst := processLfLstDbToYang(fieldXpath, value) - resultMap[resField] = resLst + resLst := processLfLstDbToYang(fieldXpath, value, yngTerminalNdDtType) + inParamsForGet.resultMap[resField] = resLst } else { /* yangType is leaf - there are only 2 types of yang terminal node leaf and leaf-list */ - yngTerminalNdDtType := xDbSpecMapEntry.dbEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) + resVal, _, err := DbToYangType(yngTerminalNdDtType, fieldXpath, value) if err != nil { log.Warningf("Failure in converting Db value type to yang type for xpath", fieldXpath) } else { - resultMap[resField] = resVal + inParamsForGet.resultMap[resField] = resVal } } - return } -func sonicDbToYangListFill(uri string, xpath string, dbIdx db.DBNum, table string, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) []typeMapOfInterface { +func sonicDbToYangListFill(inParamsForGet xlateFromDbParams) []typeMapOfInterface { var mapSlice []typeMapOfInterface + dbDataMap := inParamsForGet.dbDataMap + table := inParamsForGet.tbl + dbIdx := inParamsForGet.curDb + xpath := inParamsForGet.xpath dbTblData := (*dbDataMap)[dbIdx][table] - for keyStr, _ := range dbTblData { + for keyStr := range dbTblData { curMap := make(map[string]interface{}) - sonicDbToYangDataFill(uri, xpath, dbIdx, table, keyStr, dbDataMap, curMap) + linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + sonicDbToYangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + inParamsForGet.dbDataMap = dbDataMap dbSpecData, ok := xDbSpecMap[table] if ok && dbSpecData.keyName == nil { yangKeys := yangKeyFromEntryGet(xDbSpecMap[xpath].dbEntry) - sonicKeyDataAdd(dbIdx, yangKeys, keyStr, curMap) + sonicKeyDataAdd(dbIdx, yangKeys, table, keyStr, curMap) } - if curMap != nil { + if len(curMap) > 0 { mapSlice = append(mapSlice, curMap) } } return mapSlice } -func sonicDbToYangDataFill(uri string, xpath string, dbIdx db.DBNum, table string, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}) { +func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { + xpath := inParamsForGet.xpath + uri := inParamsForGet.uri + table := inParamsForGet.tbl + key := inParamsForGet.tblKey + resultMap := inParamsForGet.resultMap + dbDataMap := inParamsForGet.dbDataMap + dbIdx := inParamsForGet.curDb yangNode, ok := xDbSpecMap[xpath] if ok && yangNode.dbEntry != nil { @@ -264,48 +311,86 @@ func sonicDbToYangDataFill(uri string, xpath string, dbIdx db.DBNum, table strin chldYangType := yangTypeGet(xDbSpecMap[chldXpath].dbEntry) if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { - log.Infof("tbl(%v), k(%v), yc(%v)", table, key, yangChldName) + xfmrLogInfoAll("tbl(%v), k(%v), yc(%v)", table, key, yangChldName) fldName := yangChldName if chldYangType == YANG_LEAF_LIST { fldName = fldName + "@" } - sonicDbToYangTerminalNodeFill(table, fldName, (*dbDataMap)[dbIdx][table][key].Field[fldName], resultMap) + curUri := inParamsForGet.uri + "/" + yangChldName + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, curUri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + sonicDbToYangTerminalNodeFill(fldName, linParamsForGet) + resultMap = linParamsForGet.resultMap + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CONTAINER { curMap := make(map[string]interface{}) curUri := xpath + "/" + yangChldName // container can have a static key, so extract key for current container _, curKey, curTable := sonicXpathKeyExtract(curUri) // use table-name as xpath from now on - sonicDbToYangDataFill(curUri, curTable, xDbSpecMap[curTable].dbIndex, curTable, curKey, dbDataMap, curMap) + d := inParamsForGet.dbs[xDbSpecMap[curTable].dbIndex] + linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, curTable, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + sonicDbToYangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap if len(curMap) > 0 { resultMap[yangChldName] = curMap } else { - log.Infof("Empty container for xpath(%v)", curUri) + xfmrLogInfoAll("Empty container for xpath(%v)", curUri) } + inParamsForGet.dbDataMap = linParamsForGet.dbDataMap + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_LIST { - var mapSlice []typeMapOfInterface - curUri := xpath + "/" + yangChldName - mapSlice = sonicDbToYangListFill(curUri, curUri, dbIdx, table, key, dbDataMap) - if len(key) > 0 && len(mapSlice) == 1 {// Single instance query. Don't return array of maps - for k, val := range mapSlice[0] { - resultMap[k] = val - } - - } else if len(mapSlice) > 0 { - resultMap[yangChldName] = mapSlice + pathList := strings.Split(uri, "/") + // Skip the list entries if the uri has specific list query + if len(pathList) > SONIC_TABLE_INDEX+1 && !strings.Contains(uri,yangChldName) { + xfmrLogInfoAll("Skipping yangChldName: %v, pathList:%v, len:%v", yangChldName, pathList, len(pathList)) } else { - log.Infof("Empty list for xpath(%v)", curUri) + var mapSlice []typeMapOfInterface + curUri := xpath + "/" + yangChldName + inParamsForGet.uri = curUri + inParamsForGet.xpath = curUri + mapSlice = sonicDbToYangListFill(inParamsForGet) + dbDataMap = inParamsForGet.dbDataMap + if len(key) > 0 && len(mapSlice) == 1 {// Single instance query. Don't return array of maps + for k, val := range mapSlice[0] { + resultMap[k] = val + } + + } else if len(mapSlice) > 0 { + resultMap[yangChldName] = mapSlice + } else { + xfmrLogInfoAll("Empty list for xpath(%v)", curUri) + } + inParamsForGet.resultMap = resultMap } + } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { + curUri := table + "/" + yangChldName + inParamsForGet.uri = curUri + inParamsForGet.xpath = curUri + inParamsForGet.curDb = xDbSpecMap[table].dbIndex + sonicDbToYangDataFill(inParamsForGet) + dbDataMap = inParamsForGet.dbDataMap + resultMap = inParamsForGet.resultMap + } else { + xfmrLogInfoAll("Not handled case %v", chldXpath) } + } else { + xfmrLogInfoAll("Yang entry not found for %v", chldXpath) } } } - return } /* Traverse db map and create json for cvl yang */ -func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}) (string, error) { +func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, error) { + var err error + uri := inParamsForGet.uri + dbDataMap := inParamsForGet.dbDataMap + resultMap := inParamsForGet.resultMap xpath, key, table := sonicXpathKeyExtract(uri) + inParamsForGet.xpath = xpath + inParamsForGet.tbl = table + inParamsForGet.tblKey = key if len(xpath) > 0 { var dbNode *dbInfo @@ -319,12 +404,13 @@ func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map if ok && (xDbSpecMap[dbSpecField].fieldType == YANG_LEAF || xDbSpecMap[dbSpecField].fieldType == YANG_LEAF_LIST) { dbNode = xDbSpecMap[dbSpecField] xpath = dbSpecField + inParamsForGet.xpath = xpath } else { dbNode = xDbSpecMap[table] } } } else { - dbNode, _ = xDbSpecMap[xpath] + dbNode = xDbSpecMap[xpath] } if dbNode != nil && dbNode.dbEntry != nil { @@ -333,20 +419,25 @@ func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map if len(table) > 0 { cdb = xDbSpecMap[table].dbIndex } + inParamsForGet.curDb = cdb if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { fldName := xDbSpecMap[xpath].dbEntry.Name if yangType == YANG_LEAF_LIST { fldName = fldName + "@" } - sonicDbToYangTerminalNodeFill(table, fldName, (*dbDataMap)[cdb][table][key].Field[fldName], resultMap) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, cdb, inParamsForGet.ygRoot, xpath, inParamsForGet.requestUri, uri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + sonicDbToYangTerminalNodeFill(fldName, linParamsForGet) + resultMap = linParamsForGet.resultMap } else if yangType == YANG_CONTAINER { if len(table) > 0 { xpath = table + inParamsForGet.xpath = xpath } - sonicDbToYangDataFill(uri, xpath, cdb, table, key, dbDataMap, resultMap) + sonicDbToYangDataFill(inParamsForGet) + resultMap = inParamsForGet.resultMap } else if yangType == YANG_LIST { - mapSlice := sonicDbToYangListFill(uri, xpath, cdb, table, key, dbDataMap) + mapSlice := sonicDbToYangListFill(inParamsForGet) if len(key) > 0 && len(mapSlice) == 1 {// Single instance query. Don't return array of maps for k, val := range mapSlice[0] { resultMap[k] = val @@ -362,9 +453,12 @@ func directDbToYangJsonCreate(uri string, dbDataMap *map[db.DBNum]map[string]map } jsonMapData, _ := json.Marshal(resultMap) + isEmptyPayload := isJsonDataEmpty(string(jsonMapData)) jsonData := fmt.Sprintf("%v", string(jsonMapData)) - jsonDataPrint(jsonData) - return jsonData, nil + if isEmptyPayload { + log.Warning("No data available") + } + return jsonData, isEmptyPayload, err } func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (string, string, error) { @@ -372,26 +466,35 @@ func tableNameAndKeyFromDbMapGet(dbDataMap map[string]map[string]db.Value) (stri tableKey := "" for tn, tblData := range dbDataMap { tableName = tn - for kname, _ := range tblData { + for kname := range tblData { tableKey = kname } } return tableName, tableKey, nil } -func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB) (map[db.DBNum]map[string]map[string]db.Value, error) { +func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string, cdb db.DBNum, dbs [db.MaxDB]*db.DB, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool) (map[db.DBNum]map[string]map[string]db.Value, error) { var err error - dbresult := make(map[db.DBNum]map[string]map[string]db.Value) + dbresult := make(RedisDbMap) dbresult[cdb] = make(map[string]map[string]db.Value) dbFormat := KeySpec{} dbFormat.Ts.Name = tblName - dbFormat.dbNum = cdb + dbFormat.DbNum = cdb if tblKey != "" { + if tblSpecInfo, ok := xDbSpecMap[tblName]; ok && tblSpecInfo.hasXfmrFn { + /* key from uri should be converted into redis-db key, to read data */ + tblKey, err = dbKeyValueXfmrHandler(CREATE, cdb, tblName, tblKey) + if err != nil { + log.Warningf("Value-xfmr for table(%v) & key(%v) didn't do conversion.", tblName, tblKey) + return nil, err + } + } + dbFormat.Key.Comp = append(dbFormat.Key.Comp, tblKey) } - err = TraverseDb(dbs, dbFormat, &dbresult, nil) + err = TraverseDb(dbs, dbFormat, &dbresult, nil, dbTblKeyGetCache) if err != nil { - log.Errorf("TraverseDb() failure for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) + log.Warningf("TraverseDb() didn't fetch data for tbl(DB num) %v(%v) for xpath %v", tblName, cdb, xpath) return nil, err } if _, ok := dbresult[cdb]; !ok { @@ -404,126 +507,348 @@ func fillDbDataMapForTbl(uri string, xpath string, tblName string, tblKey string } // Assumption: All tables are from the same DB -func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value) error { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - curDbDataMap, err := fillDbDataMapForTbl(inParams.uri, xpath, tbl, inParams.key, inParams.curDb, inParams.dbs) - if err == nil { - mapCopy((*dbDataMap)[inParams.curDb], curDbDataMap[inParams.curDb]) +func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool, xpath string) error { + // skip the query if the table is already visited + if _,ok := (*dbDataMap)[inParams.curDb][tbl]; ok { + if len(inParams.key) > 0 { + if _,ok = (*dbDataMap)[inParams.curDb][tbl][inParams.key]; ok { + return nil + } + } else { + return nil + } } + + terminalNodeGet := false + qdbMapHasTblData := false + qdbMapHasTblKeyData := false + if !xYangSpecMap[xpath].hasNonTerminalNode && len(inParams.key) > 0 { + terminalNodeGet = true + } + if qdbMap, getOk := dbTblKeyGetCache[inParams.curDb]; getOk { + if dbTblData, tblPresent := qdbMap[tbl]; tblPresent { + qdbMapHasTblData = true + if _, keyPresent := dbTblData[inParams.key]; keyPresent { + qdbMapHasTblKeyData = true; + } + } + } + + if !qdbMapHasTblData || (terminalNodeGet && qdbMapHasTblData && !qdbMapHasTblKeyData) { + curDbDataMap, err := fillDbDataMapForTbl(inParams.uri, xpath, tbl, inParams.key, inParams.curDb, inParams.dbs, dbTblKeyGetCache) + if err == nil { + mapCopy((*dbDataMap)[inParams.curDb], curDbDataMap[inParams.curDb]) + } + } return nil } -func yangListDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { +func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error { var tblList []string + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + cdb := inParamsForGet.curDb + resultMap := inParamsForGet.resultMap + xpath := inParamsForGet.xpath + tbl := inParamsForGet.tbl + tblKey := inParamsForGet.tblKey + - if tbl == "" && xYangSpecMap[xpath].xfmrTbl != nil { + _, ok := xYangSpecMap[xpath] + if ok { + if xYangSpecMap[xpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[xpath].xfmrTbl if len(xfmrTblFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) - tblList = xfmrTblHandlerFunc(xfmrTblFunc, inParams) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + tblList, _ = xfmrTblHandlerFunc(xfmrTblFunc, inParams, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if len(tblList) != 0 { for _, curTbl := range tblList { - dbDataFromTblXfmrGet(curTbl, inParams, dbDataMap) + dbDataFromTblXfmrGet(curTbl, inParams, dbDataMap, inParamsForGet.dbTblKeyGetCache, xpath) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } } } + if tbl != "" { + if !contains(tblList, tbl) { + tblList = append(tblList, tbl) + } + } } else if tbl != "" && xYangSpecMap[xpath].xfmrTbl == nil { tblList = append(tblList, tbl) - } else if tbl != "" && xYangSpecMap[xpath].xfmrTbl != nil { - /*key instance level GET, table name and table key filled from xpathKeyExtract which internally calls table transformer*/ - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) - dbDataFromTblXfmrGet(tbl, inParams, dbDataMap) - tblList = append(tblList, tbl) + } else if tbl == "" && xYangSpecMap[xpath].xfmrTbl == nil { + // Handling for case: Parent list is not associated with a tableName but has children containers/lists having tableNames. + if tblKey != "" { + var mapSlice []typeMapOfInterface + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + dbDataMap = inParamsForGet.dbDataMap + if err != nil { + log.Infof("Error(%v) returned for %v", err, uri) + } else if ((instMap != nil) && (len(instMap) > 0)) { + mapSlice = append(mapSlice, instMap) + } + if len(mapSlice) > 0 { + listInstanceGet := false + // Check if it is a list instance level Get + if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) { + listInstanceGet = true + for k, v := range mapSlice[0] { + resultMap[k] = v + } + } + if !listInstanceGet { + resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice + } + inParamsForGet.resultMap = resultMap + } + } + } } for _, tbl = range(tblList) { + inParamsForGet.tbl = tbl + tblData, ok := (*dbDataMap)[cdb][tbl] if ok { var mapSlice []typeMapOfInterface - for dbKey, _ := range tblData { - curMap := make(map[string]interface{}) - curKeyMap, curUri, _ := dbKeyToYangDataConvert(uri, xpath, dbKey, dbs[cdb].Opts.KeySeparator) - if len(xYangSpecMap[xpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - mapSlice = append(mapSlice, curMap) - } else { - log.Infof("Empty container returned from overloaded transformer for(\"%v\")", curUri) + for dbKey := range tblData { + inParamsForGet.tblKey = dbKey + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + dbDataMap = inParamsForGet.dbDataMap + if err != nil { + log.Infof("Error(%v) returned for %v", err, uri) + } else if ((instMap != nil) && (len(instMap) > 0)) { + mapSlice = append(mapSlice, instMap) + } + } + + if len(mapSlice) > 0 { + listInstanceGet := false + /*Check if it is a list instance level Get*/ + if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) { + listInstanceGet = true + for k, v := range mapSlice[0] { + resultMap[k] = v } - } else { - _, keyFromCurUri, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri) - if dbKey == keyFromCurUri { - for k, kv := range curKeyMap { - curMap[k] = kv + } + if !listInstanceGet { + if _, specOk := xYangSpecMap[xpath]; specOk { + if _, ok := resultMap[xYangSpecMap[xpath].yangEntry.Name]; ok { + mlen := len(resultMap[xYangSpecMap[xpath].yangEntry.Name].([]typeMapOfInterface)) + for i := 0; i < mlen; i++ { + mapSlice = append(mapSlice, resultMap[xYangSpecMap[xpath].yangEntry.Name].([]typeMapOfInterface)[i]) } - curXpath, _ := XfmrRemoveXPATHPredicates(curUri) - yangDataFill(dbs, ygRoot, curUri, curXpath, dbDataMap, curMap, tbl, dbKey, cdb, validate) - mapSlice = append(mapSlice, curMap) + } + resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice + inParamsForGet.resultMap = resultMap } } - } - if len(mapSlice) > 0 { - resultMap[xYangSpecMap[xpath].yangEntry.Name] = mapSlice } else { - log.Infof("Empty slice for (\"%v\").\r\n", uri) + xfmrLogInfoAll("Empty slice for (\"%v\").\r\n", uri) } } }// end of tblList for + return nil } -func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, tbl string, tblKey string) (map[string]interface{}, error) { - log.Infof("Received xpath - %v, uri - %v, table - %v, table key - %v", xpath, uri, tbl, tblKey) +func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) (typeMapOfInterface,error) { + + var err error + curMap := make(map[string]interface{}) + err = nil + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + cdb := inParamsForGet.curDb + xpath := inParamsForGet.xpath + tbl := inParamsForGet.tbl + dbKey := inParamsForGet.tblKey + + curKeyMap, curUri, err := dbKeyToYangDataConvert(uri, requestUri, xpath, tbl, dbDataMap, dbKey, dbs[cdb].Opts.KeySeparator, txCache) + if ((err != nil) || (curKeyMap == nil) || (len(curKeyMap) == 0)) { + xfmrLogInfoAll("Skip filling list instance for uri %v since no yang key found corresponding to db-key %v", uri, dbKey) + return curMap, err + } + parentXpath := parentXpathGet(xpath) + _, ok := xYangSpecMap[xpath] + if ok && len(xYangSpecMap[xpath].xfmrFunc) > 0 { + if isFirstCall || (!isFirstCall && (uri != requestUri) && ((len(xYangSpecMap[parentXpath].xfmrFunc) == 0) || + (len(xYangSpecMap[parentXpath].xfmrFunc) > 0 && (xYangSpecMap[parentXpath].xfmrFunc != xYangSpecMap[xpath].xfmrFunc)))) { + xfmrLogInfoAll("Parent subtree already handled cur uri: %v", xpath) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, GET, dbKey, dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrFunc) + inParamsForGet.ygRoot = ygRoot + inParamsForGet.dbDataMap = dbDataMap + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + } + } + if xYangSpecMap[xpath].hasChildSubTree { + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + yangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + } else { + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + keyFromCurUri := xpathKeyExtRet.dbKey + inParamsForGet.ygRoot = ygRoot + if dbKey == keyFromCurUri || keyFromCurUri == "" { + if dbKey == keyFromCurUri { + for k, kv := range curKeyMap { + curMap[k] = kv + } + } + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpathKeyExtRet.xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + yangDataFill(linParamsForGet) + curMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + } + return curMap, err +} + +func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery bool) (map[string]interface{}, error) { + xfmrLogInfoAll("Received xpath - %v, uri - %v, table - %v, table key - %v", inParamsForGet.xpath, inParamsForGet.uri, inParamsForGet.tbl, inParamsForGet.tblKey) var err error resFldValMap := make(map[string]interface{}) - if xYangSpecMap[xpath].yangEntry == nil { + xpath := inParamsForGet.xpath + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + tbl := inParamsForGet.tbl + tblKey := inParamsForGet.tblKey + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + + _, ok := xYangSpecMap[xpath] + if !ok || xYangSpecMap[xpath].yangEntry == nil { logStr := fmt.Sprintf("No yang entry found for xpath %v.", xpath) err = fmt.Errorf("%v", logStr) return resFldValMap, err } cdb := xYangSpecMap[xpath].dbIndex - if len(xYangSpecMap[xpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, tblKey, dbDataMap, nil) - fldValMap, err := leafXfmrHandlerFunc(inParams) + if len(xYangSpecMap[xpath].xfmrField) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + fldValMap, err := leafXfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrField) + inParamsForGet.ygRoot = ygRoot + inParamsForGet.dbDataMap = dbDataMap if err != nil { - logStr := fmt.Sprintf("%Failed to get data from overloaded function for %v -v.", uri, err) - err = fmt.Errorf("%v", logStr) + xfmrLogInfoAll("No data from field transformer for %v: %v.", uri, err) return resFldValMap, err } - if fldValMap != nil { - for lf, val := range fldValMap { + if (uri == requestUri) { + yangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) + if len(fldValMap) == 0 { + // field transformer returns empty map when no data in DB + if ((yangType == YANG_LEAF) || ((yangType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))))) { + log.Warningf("Field transformer returned empty data , uri - %v", requestUri) + err = tlerr.NotFoundError{Format:"Resource not found"} + return resFldValMap, err + } + } else { + if ((yangType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/")))) { + return resFldValMap, nil + } + } + } + for lf, val := range fldValMap { resFldValMap[lf] = val - } - } + } } else { dbFldName := xYangSpecMap[xpath].fieldName - if dbFldName == "NONE" { + if dbFldName == XFMR_NONE_STRING { return resFldValMap, err } /* if there is no transformer extension/annotation then it means leaf-list in yang is also leaflist in db */ if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { yangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) + yngTerminalNdDtType := xYangSpecMap[xpath].yangEntry.Type.Kind if yangType == YANG_LEAF_LIST { dbFldName += "@" val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] + leafLstInstGetReq := false + + if terminalNodeQuery && ((strings.HasSuffix(requestUri, "]")) || (strings.HasSuffix(requestUri, "]/"))) { + xfmrLogInfoAll("Request URI is leaf-list instance GET - %v", requestUri) + leafLstInstGetReq = true + } if ok { - resLst := processLfLstDbToYang(xpath, val) - resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resLst + if leafLstInstGetReq { + leafListInstVal, valErr := extractLeafListInstFromUri(requestUri) + if valErr != nil { + return resFldValMap, valErr + } + dbSpecField := tbl + "/" + strings.TrimSuffix(dbFldName, "@") + dbSpecFieldInfo, dbSpecOk := xDbSpecMap[dbSpecField] + if dbSpecOk && dbSpecFieldInfo.xfmrValue != nil { + inParams := formXfmrDbInputRequest(CREATE, cdb, tbl, tblKey, dbFldName, leafListInstVal) + retVal, valXfmrErr := valueXfmrHandler(inParams, *dbSpecFieldInfo.xfmrValue) + if valXfmrErr != nil { + log.Warningf("value-xfmr:fldpath(\"%v\") val(\"%v\"):err(\"%v\").", dbSpecField, leafListInstVal, valXfmrErr) + return resFldValMap, valXfmrErr + } + log.Info("valueXfmrHandler() retuned ", retVal) + leafListInstVal = retVal + } + if !leafListInstExists((*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName], leafListInstVal) { + log.Warningf("Queried leaf-list instance does not exists, uri - %v, dbData - %v", requestUri, (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName]) + err = tlerr.NotFoundError{Format:"Resource not found"} + } + if err == nil { + /* Since translib already fills in ygRoot with queried leaf-list instance, do not + fill in resFldValMap or else Unmarshall of payload(resFldValMap) into ygotTgt in + app layer will create duplicate instances in result. + */ + log.Info("Queried leaf-list instance exists but Since translib already fills in ygRoot with queried leaf-list instance do not populate payload.") + } + return resFldValMap, err + } else { + resLst := processLfLstDbToYang(xpath, val, yngTerminalNdDtType) + resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resLst + } + } else { + if leafLstInstGetReq { + log.Warningf("Queried leaf-list does not exist in DB, uri - %v", requestUri) + err = tlerr.NotFoundError{Format:"Resource not found"} + } } } else { val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] if ok { - yngTerminalNdDtType := xYangSpecMap[xpath].yangEntry.Type.Kind - resVal, err := DbToYangType(yngTerminalNdDtType, xpath, val) + resVal, _, err := DbToYangType(yngTerminalNdDtType, xpath, val) if err != nil { - log.Error("Failure in converting Db value type to yang type for field", xpath) + log.Warning("Conversion of Db value type to yang type for field didn't happen.", xpath) } else { resFldValMap[xYangSpecMap[xpath].yangEntry.Name] = resVal } + } else { + xfmrLogInfoAll("Field value does not exist in DB for - %v" , uri) + err = tlerr.NotFoundError{Format:"Resource not found"} } } } @@ -531,90 +856,199 @@ func terminalNodeProcess(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string return resFldValMap, err } -func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, resultMap map[string]interface{}, tbl string, tblKey string, cdb db.DBNum, validate bool) error { +func yangDataFill(inParamsForGet xlateFromDbParams) error { var err error + validate := inParamsForGet.validate isValid := validate + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + resultMap := inParamsForGet.resultMap + xpath := inParamsForGet.xpath + var chldUri string + yangNode, ok := xYangSpecMap[xpath] if ok && yangNode.yangEntry != nil { for yangChldName := range yangNode.yangEntry.Dir { chldXpath := xpath+"/"+yangChldName - chldUri := uri+"/"+yangChldName + if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].nameWithMod != nil { + chldUri = uri+"/"+ *(xYangSpecMap[chldXpath].nameWithMod) + } else { + chldUri = uri+"/"+yangChldName + } + inParamsForGet.xpath = chldXpath + inParamsForGet.uri = chldUri if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].yangEntry != nil { - cdb = xYangSpecMap[chldXpath].dbIndex + cdb := xYangSpecMap[chldXpath].dbIndex + inParamsForGet.curDb = cdb if len(xYangSpecMap[chldXpath].validateFunc) > 0 && !validate { - _, key, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri) + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.ygRoot = ygRoot // TODO - handle non CONFIG-DB - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, key, dbDataMap, nil) - res := validateHandlerFunc(inParams) - if res != true { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + res := validateHandlerFunc(inParams, xYangSpecMap[chldXpath].validateFunc) + if !res { continue } else { isValid = res } + inParamsForGet.validate = isValid + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } - chldYangType := yangTypeGet(xYangSpecMap[chldXpath].yangEntry) + chldYangType := xYangSpecMap[chldXpath].yangDataType if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { - fldValMap, err := terminalNodeProcess(dbs, ygRoot, chldUri, chldXpath, dbDataMap, tbl, tblKey) + if len(xYangSpecMap[xpath].xfmrFunc) > 0 { + continue + } + fldValMap, err := terminalNodeProcess(inParamsForGet, false) + dbDataMap = inParamsForGet.dbDataMap + ygRoot = inParamsForGet.ygRoot if err != nil { - log.Infof("Failed to get data(\"%v\").", chldUri) + xfmrLogInfoAll("Failed to get data(\"%v\").", chldUri) } for lf, val := range fldValMap { resultMap[lf] = val } + inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CONTAINER { + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + tblKey := xpathKeyExtRet.dbKey + chtbl := xpathKeyExtRet.tableName + inParamsForGet.ygRoot = ygRoot + + if _, ok := (*dbDataMap)[cdb][chtbl]; !ok && len(chtbl) > 0 { + childDBKey := "" + terminalNodeGet := false + qdbMapHasTblData := false + qdbMapHasTblKeyData := false + if !xYangSpecMap[chldXpath].hasNonTerminalNode { + childDBKey = tblKey + terminalNodeGet = true + } + if qdbMap, getOk := inParamsForGet.dbTblKeyGetCache[cdb]; getOk { + if dbTblData, tblPresent := qdbMap[chtbl]; tblPresent { + qdbMapHasTblData = true + if _, keyPresent := dbTblData[tblKey]; keyPresent { + qdbMapHasTblKeyData = true; + } + } + } + + if !qdbMapHasTblData || (terminalNodeGet && qdbMapHasTblData && !qdbMapHasTblKeyData) { + curDbDataMap, err := fillDbDataMapForTbl(chldUri, chldXpath, chtbl, childDBKey, cdb, dbs, inParamsForGet.dbTblKeyGetCache) + if err == nil { + mapCopy((*dbDataMap)[cdb], curDbDataMap[cdb]) + inParamsForGet.dbDataMap = dbDataMap + } + } + } cname := xYangSpecMap[chldXpath].yangEntry.Name if xYangSpecMap[chldXpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[chldXpath].xfmrTbl if len(xfmrTblFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, tblKey, dbDataMap, nil) - tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + tblList, _ := xfmrTblHandlerFunc(xfmrTblFunc, inParams, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if len(tblList) > 1 { log.Warningf("Table transformer returned more than one table for container %v", chldXpath) } if len(tblList) == 0 { continue } - dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) - tbl = tblList[0] + dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap, inParamsForGet.dbTblKeyGetCache, chldXpath) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + chtbl = tblList[0] } } if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) + if (len(xYangSpecMap[xpath].xfmrFunc) == 0) || + (len(xYangSpecMap[xpath].xfmrFunc) > 0 && + (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + } } - continue + if !xYangSpecMap[chldXpath].hasChildSubTree { + continue + } + } + cmap2 := make(map[string]interface{}) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, chtbl, tblKey, dbDataMap, inParamsForGet.txCache, cmap2, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + err = yangDataFill(linParamsForGet) + cmap2 = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + if err != nil && len(cmap2) == 0 { + xfmrLogInfoAll("Empty container.(\"%v\").\r\n", chldUri) } else { - cmap := make(map[string]interface{}) - err = yangDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, cmap, tbl, tblKey, cdb, isValid) - if len(cmap) > 0 { - resultMap[cname] = cmap - } else { - log.Infof("Empty container(\"%v\").\r\n", chldUri) + if len(cmap2) > 0 { + resultMap[cname] = cmap2 } + inParamsForGet.resultMap = resultMap } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } else if chldYangType == YANG_LIST { + xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, chldUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.ygRoot = ygRoot cdb = xYangSpecMap[chldXpath].dbIndex + inParamsForGet.curDb = cdb if len(xYangSpecMap[chldXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap = cmap - } else { - log.Infof("Empty list(\"%v\").\r\n", chldUri) + if (len(xYangSpecMap[xpath].xfmrFunc) == 0) || + (len(xYangSpecMap[xpath].xfmrFunc) > 0 && + (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) + if err != nil { + xfmrLogInfoAll("Error returned by %v: %v", xYangSpecMap[chldXpath].xfmrFunc, err) + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot } - } else { - ynode, ok := xYangSpecMap[chldXpath] - lTblName := "" - if ok && ynode.tableName != nil { - lTblName = *ynode.tableName + if !xYangSpecMap[chldXpath].hasChildSubTree { + continue } - yangListDataFill(dbs, ygRoot, chldUri, chldXpath, dbDataMap, resultMap, lTblName, "", cdb, isValid) } + ynode, ok := xYangSpecMap[chldXpath] + lTblName := "" + if ok && ynode.tableName != nil { + lTblName = *ynode.tableName + } + if _, ok := (*dbDataMap)[cdb][lTblName]; !ok && len(lTblName) > 0 { + curDbDataMap, err := fillDbDataMapForTbl(chldUri, chldXpath, lTblName, "", cdb, dbs, inParamsForGet.dbTblKeyGetCache) + if err == nil { + mapCopy((*dbDataMap)[cdb], curDbDataMap[cdb]) + inParamsForGet.dbDataMap = dbDataMap + } + } + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, lTblName, xpathKeyExtRet.dbKey, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache + linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache + yangListDataFill(linParamsForGet, false) + resultMap = linParamsForGet.resultMap + dbDataMap = linParamsForGet.dbDataMap + ygRoot = linParamsForGet.ygRoot + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.resultMap = resultMap + inParamsForGet.ygRoot = ygRoot + + } else if chldYangType == "choice" || chldYangType == "case" { + yangDataFill(inParamsForGet) + resultMap = inParamsForGet.resultMap + dbDataMap = inParamsForGet.dbDataMap } else { return err } @@ -625,24 +1059,57 @@ func yangDataFill(dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, uri string, xpath } /* Traverse linear db-map data and add to nested json data */ -func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, cdb db.DBNum) (string, error) { +func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, error) { var err error - jsonData := "" + var fldSbtErr error // used only when direct query on leaf/leaf-list having subtree + var fldErr error //used only when direct query on leaf/leaf-list having field transformer + jsonData := "{}" resultMap := make(map[string]interface{}) + d := inParamsForGet.d + dbs := inParamsForGet.dbs + ygRoot := inParamsForGet.ygRoot + uri := inParamsForGet.uri + requestUri := inParamsForGet.requestUri + dbDataMap := inParamsForGet.dbDataMap + txCache := inParamsForGet.txCache + cdb := inParamsForGet.curDb + inParamsForGet.resultMap = resultMap + if isSonicYang(uri) { - return directDbToYangJsonCreate(uri, dbDataMap, resultMap) + return directDbToYangJsonCreate(inParamsForGet) } else { - var d *db.DB - reqXpath, keyName, tableName := xpathKeyExtract(d, ygRoot, GET, uri) - yangNode, ok := xYangSpecMap[reqXpath] + xpathKeyExtRet, _ := xpathKeyExtract(d, ygRoot, GET, uri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) + + inParamsForGet.xpath = xpathKeyExtRet.xpath + inParamsForGet.tbl = xpathKeyExtRet.tableName + inParamsForGet.tblKey = xpathKeyExtRet.dbKey + inParamsForGet.ygRoot = ygRoot + yangNode, ok := xYangSpecMap[xpathKeyExtRet.xpath] if ok { + /* Invoke pre-xfmr is present for the yang module */ + moduleName := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleName) + if modSpecInfo, specOk := xYangSpecMap[moduleName]; specOk && (len(modSpecInfo.xfmrPre) > 0) { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) + xfmrLogInfo("Invoked pre transformer: %v, dbDataMap: %v ", modSpecInfo.xfmrPre, dbDataMap) + if err != nil { + log.Warningf("Pre-transformer: %v failed.(err:%v)", modSpecInfo.xfmrPre, err) + return jsonData, true, err + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + yangType := yangTypeGet(yangNode.yangEntry) validateHandlerFlag := false tableXfmrFlag := false IsValidate := false - if len(xYangSpecMap[reqXpath].validateFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, keyName, dbDataMap, nil) - res := validateHandlerFunc(inParams) + if len(xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + res := validateHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if !res { validateHandlerFlag = true /* cannot immediately return from here since reXpath yangtype decides the return type */ @@ -650,6 +1117,7 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db IsValidate = res } } + inParamsForGet.validate = IsValidate isList := false switch yangType { case YANG_LIST: @@ -657,89 +1125,147 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db case YANG_LEAF, YANG_LEAF_LIST, YANG_CONTAINER: isList = false default: - log.Infof("Unknown yang object type for path %v", reqXpath) + xfmrLogInfo("Unknown yang object type for path %v", xpathKeyExtRet.xpath) isList = true //do not want non-list processing to happen } /*If yangtype is a list separate code path is to be taken in case of table transformer since that code path already handles the calling of table transformer and subsequent processing */ if (!validateHandlerFlag) && (!isList) { - if xYangSpecMap[reqXpath].xfmrTbl != nil { - xfmrTblFunc := *xYangSpecMap[reqXpath].xfmrTbl + if xYangSpecMap[xpathKeyExtRet.xpath].xfmrTbl != nil { + xfmrTblFunc := *xYangSpecMap[xpathKeyExtRet.xpath].xfmrTbl if len(xfmrTblFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, keyName, dbDataMap, nil) - tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + tblList, _ := xfmrTblHandlerFunc(xfmrTblFunc, inParams, inParamsForGet.xfmrDbTblKeyCache) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot if len(tblList) > 1 { - log.Warningf("Table transformer returned more than one table for container %v", reqXpath) - tableXfmrFlag = true + log.Warningf("Table transformer returned more than one table for container %v", xpathKeyExtRet.xpath) } if len(tblList) == 0 { - log.Warningf("Table transformer returned no table for conatiner %v", reqXpath) + log.Warningf("Table transformer returned no table for conatiner %v", xpathKeyExtRet.xpath) tableXfmrFlag = true } if !tableXfmrFlag { - dbDataFromTblXfmrGet(tblList[0], inParams, dbDataMap) + for _, tbl := range tblList { + dbDataFromTblXfmrGet(tbl, inParams, dbDataMap, inParamsForGet.dbTblKeyGetCache, xpathKeyExtRet.xpath) + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } + } } else { - log.Warningf("empty table transformer function name for xpath - %v", reqXpath) + log.Warningf("empty table transformer function name for xpath - %v", xpathKeyExtRet.xpath) tableXfmrFlag = true } } } for { + done := true if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { - yangName := xYangSpecMap[reqXpath].yangEntry.Name + yangName := xYangSpecMap[xpathKeyExtRet.xpath].yangEntry.Name if validateHandlerFlag || tableXfmrFlag { resultMap[yangName] = "" break } - tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) - fldValMap, err := terminalNodeProcess(dbs, ygRoot, uri, reqXpath, dbDataMap, tbl, key) - if err != nil { - log.Infof("Empty terminal node (\"%v\").", uri) + if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + fldSbtErr = xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) + if fldSbtErr != nil { + /*For request Uri pointing to leaf/leaf-list having subtree, error will be propagated + to handle check of leaf/leaf-list-instance existence in Db , which will be performed + by subtree + */ + xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + inParamsForGet.ygRoot = ygRoot + break + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + } else { + tbl, key, _ := tableNameAndKeyFromDbMapGet((*dbDataMap)[cdb]) + inParamsForGet.tbl = tbl + inParamsForGet.tblKey = key + var fldValMap map[string]interface{} + fldValMap, fldErr = terminalNodeProcess(inParamsForGet, true) + if ((fldErr != nil) || (len(fldValMap) == 0)) { + if fldErr == nil { + if yangType == YANG_LEAF { + xfmrLogInfo("Empty terminal node (\"%v\").", uri) + fldErr = tlerr.NotFoundError{Format:"Resource Not found"} + } else if ((yangType == YANG_LEAF_LIST) && ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/")))) { + jsonMapData, _ := json.Marshal(resultMap) + jsonData = fmt.Sprintf("%v", string(jsonMapData)) + return jsonData, false, nil + } + } + } + resultMap = fldValMap } - resultMap = fldValMap break } else if yangType == YANG_CONTAINER { - cname := xYangSpecMap[reqXpath].yangEntry.Name cmap := make(map[string]interface{}) - resultMap[cname] = cmap + resultMap = cmap if validateHandlerFlag || tableXfmrFlag { break } - if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) - cmap, _ = xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap[cname] = cmap + if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) + if err != nil { + xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + return jsonData, true, err + } + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + if !xYangSpecMap[xpathKeyExtRet.xpath].hasChildSubTree { + break } - break } - err = yangDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, IsValidate) + inParamsForGet.resultMap = make(map[string]interface{}) + err = yangDataFill(inParamsForGet) if err != nil { - log.Infof("Empty container(\"%v\").\r\n", uri) + xfmrLogInfo("Empty container(\"%v\").\r\n", uri) } + resultMap = inParamsForGet.resultMap break } else if yangType == YANG_LIST { - if len(xYangSpecMap[reqXpath].xfmrFunc) > 0 { - inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, GET, "", dbDataMap, nil) - cmap, _ := xfmrHandlerFunc(inParams) - if cmap != nil && len(cmap) > 0 { - resultMap = cmap - } else { - log.Infof("Empty list(\"%v\").\r\n", uri) - } - } else { - err = yangListDataFill(dbs, ygRoot, uri, reqXpath, dbDataMap, resultMap, tableName, keyName, cdb, IsValidate) + isFirstCall := true + if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { + inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) if err != nil { - log.Infof("yangListDataFill failed for list case(\"%v\").\r\n", uri) + if (((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) && (uri == requestUri)) { + // The error handling here is for the deferred resource check error being handled by the subtree for virtual table cases. + log.Warningf("Subtree at list instance level returns error %v for uri - %v", err, uri) + return jsonData, true, err + + } else { + + xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + } } + isFirstCall = false + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.ygRoot = ygRoot + if !xYangSpecMap[xpathKeyExtRet.xpath].hasChildSubTree { + break + } + } + inParamsForGet.resultMap = make(map[string]interface{}) + err = yangListDataFill(inParamsForGet, isFirstCall) + if err != nil { + xfmrLogInfo("yangListDataFill failed for list case(\"%v\").\r\n", uri) } + resultMap = inParamsForGet.resultMap break } else { - log.Warningf("Unknown yang object type for path %v", reqXpath) + log.Warningf("Unknown yang object type for path %v", xpathKeyExtRet.xpath) + break + } + if done { break } } //end of for @@ -747,20 +1273,21 @@ func dbDataToYangJsonCreate(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db } jsonMapData, _ := json.Marshal(resultMap) + isEmptyPayload := isJsonDataEmpty(string(jsonMapData)) jsonData = fmt.Sprintf("%v", string(jsonMapData)) - jsonDataPrint(jsonData) - return jsonData, nil -} - -func jsonDataPrint(data string) { - fp, err := os.Create("/tmp/dbToYangJson.txt") - if err != nil { - return - } - defer fp.Close() + if fldSbtErr != nil { + /*error should be propagated only when request Uri points to leaf/leaf-list-instance having subtree, + This is to handle check of leaf/leaf-list-instance existence in Db , which will be performed + by subtree, and depending whether queried node exists or not subtree should return error + */ + return jsonData, isEmptyPayload, fldSbtErr + } + if fldErr != nil { + /* error should be propagated only when request Uri points to leaf/leaf-list-instance and the data + is not available(via field-xfmr or field name) + */ + return jsonData, isEmptyPayload, fldErr + } - fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") - fmt.Fprintf (fp, "%v \r\n", data) - fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + return jsonData, isEmptyPayload, nil } - diff --git a/translib/transformer/xlate_to_db.go b/translib/transformer/xlate_to_db.go index 6bcccb4a6..eda61b9c8 100644 --- a/translib/transformer/xlate_to_db.go +++ b/translib/transformer/xlate_to_db.go @@ -19,7 +19,6 @@ package transformer import ( - "errors" "fmt" "github.com/openconfig/ygot/ygot" "os" @@ -27,181 +26,244 @@ import ( "strings" "github.com/Azure/sonic-mgmt-common/translib/db" "github.com/Azure/sonic-mgmt-common/translib/ocbinds" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" "github.com/openconfig/ygot/ytypes" - "github.com/openconfig/goyang/pkg/yang" - + "github.com/openconfig/goyang/pkg/yang" log "github.com/golang/glog" ) -/* Invoke the post tansformer */ -func postXfmrHandlerFunc(inParams XfmrParams) (map[string]map[string]db.Value, error) { - xpath, _ := XfmrRemoveXPATHPredicates(inParams.uri) - ret, err := XlateFuncCall(xYangSpecMap[xpath].xfmrPost, inParams) - if err != nil { - return nil, err - } - retData := ret[0].Interface().(map[string]map[string]db.Value) - log.Info("Post Transformer function :", xYangSpecMap[xpath].xfmrPost, " Xpath: ", xpath, " retData: ", retData) - return retData, err -} +var ocbSch, _ = ocbinds.Schema() /* Fill redis-db map with field & value info */ func dataToDBMapAdd(tableName string, dbKey string, result map[string]map[string]db.Value, field string, value string) { - _, ok := result[tableName] - if !ok { - result[tableName] = make(map[string]db.Value) - } + if len(tableName) > 0 { + _, ok := result[tableName] + if !ok { + result[tableName] = make(map[string]db.Value) + } - _, ok = result[tableName][dbKey] - if !ok { - result[tableName][dbKey] = db.Value{Field: make(map[string]string)} - } + if len(dbKey) > 0 { + _, ok = result[tableName][dbKey] + if !ok { + result[tableName][dbKey] = db.Value{Field: make(map[string]string)} + } - if field == "NONE" { - result[tableName][dbKey].Field["NULL"] = "NULL" - return - } + if field == XFMR_NONE_STRING { + if len(result[tableName][dbKey].Field) == 0 { + result[tableName][dbKey].Field["NULL"] = "NULL" + } + return + } - result[tableName][dbKey].Field[field] = value - return + if len(field) > 0 { + result[tableName][dbKey].Field[field] = value + } + } + } } -func tblNameFromTblXfmrGet(xfmrTblFunc string, inParams XfmrParams) (string, error){ - tblList := xfmrTblHandlerFunc(xfmrTblFunc, inParams) +/*use when single table name is expected*/ +func tblNameFromTblXfmrGet(xfmrTblFunc string, inParams XfmrParams, xfmrDbTblKeyCache map[string]tblKeyCache) (string, error){ + var err error + var tblList []string + tblList, err = xfmrTblHandlerFunc(xfmrTblFunc, inParams, xfmrDbTblKeyCache) + if err != nil { + return "", err + } if len(tblList) != 1 { - logStr := fmt.Sprintf("Invalid return value(%v) from table transformer for (%v)", tblList, inParams.uri) - log.Error(logStr) - err := errors.New(logStr) + xfmrLogInfoAll("Uri (\"%v\") translates to 0 or multiple tables instead of single table - %v", inParams.uri, tblList) return "", err } - return tblList[0], nil + return tblList[0], err } /* Fill the redis-db map with data */ -func mapFillData(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbKey string, result map[string]map[string]db.Value, xpathPrefix string, name string, value interface{}) error { +func mapFillData(xlateParams xlateToParams) error { var dbs [db.MaxDB]*db.DB var err error - xpath := xpathPrefix + "/" + name + xpath := xlateParams.xpath + "/" + xlateParams.name xpathInfo, ok := xYangSpecMap[xpath] - log.Infof("name: \"%v\", xpathPrefix(\"%v\").", name, xpathPrefix) + xfmrLogInfoAll("name: \"%v\", xpathPrefix(\"%v\").", xlateParams.name, xlateParams.xpath) if !ok || xpathInfo == nil { - log.Errorf("Yang path(\"%v\") not found.", xpath) - return errors.New("Invalid URI") + log.Warningf("Yang path(\"%v\") not found.", xpath) + return nil } if xpathInfo.tableName == nil && xpathInfo.xfmrTbl == nil{ - log.Errorf("Table for yang-path(\"%v\") not found.", xpath) - return errors.New("Invalid table name") + log.Warningf("Table for yang-path(\"%v\") not found.", xpath) + return nil } - if len(dbKey) == 0 { - log.Errorf("Table key for yang path(\"%v\") not found.", xpath) - return errors.New("Invalid table key") + if xpathInfo.tableName != nil && *xpathInfo.tableName == XFMR_NONE_STRING { + log.Warningf("Table for yang-path(\"%v\") NONE.", xpath) + return nil } - if xpathInfo.isKey { - return nil + if len(xlateParams.keyName) == 0 { + log.Warningf("Table key for yang path(\"%v\") not found.", xpath) + return nil } tableName := "" if xpathInfo.xfmrTbl != nil { - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, "") + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, "", xlateParams.txCache) // expecting only one table name from tbl-xfmr - tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams) + tableName, err = tblNameFromTblXfmrGet(*xYangSpecMap[xpath].xfmrTbl, inParams, xlateParams.xfmrDbTblKeyCache) if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return err + } + if tableName == "" { + log.Warningf("No table name found for uri (\"%v\")", xlateParams.uri) return err } } else { tableName = *xpathInfo.tableName } - if len(xpathInfo.xfmrFunc) > 0 { - uri = uri + "/" + name - - /* field transformer present */ - log.Infof("Transformer function(\"%v\") invoked for yang path(\"%v\").", xpathInfo.xfmrFunc, xpath) - path, _ := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) - for _, p := range path.Elem { - pathSlice := strings.Split(p.Name, ":") - p.Name = pathSlice[len(pathSlice)-1] - if len(p.Key) > 0 { - for ekey, ent := range p.Key { - eslice := strings.Split(ent, ":") - p.Key[ekey] = eslice[len(eslice)-1] - } - } - } - ocbSch, _ := ocbinds.Schema() - schRoot := ocbSch.RootSchema() - node, nErr := ytypes.GetNode(schRoot, (*ygRoot).(*ocbinds.Device), path) - log.Info("GetNode data: ", node[0].Data, " nErr :", nErr) - if nErr != nil { - return nErr - } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, uri, oper, "", nil, node[0].Data) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - return err - } - if ret != nil { - retData := ret[0].Interface().(map[string]string) - log.Info("Transformer function :", xpathInfo.xfmrFunc, " Xpath: ", xpath, " retData: ", retData) - for f, v := range retData { - dataToDBMapAdd(tableName, dbKey, result, f, v) - } - } - return nil + // tblXpathMap used for default value processing for a given request + if tblUriMapVal, tblUriMapOk := xlateParams.tblXpathMap[tableName][xlateParams.keyName]; !tblUriMapOk { + if _, tblOk := xlateParams.tblXpathMap[tableName]; !tblOk { + xlateParams.tblXpathMap[tableName] = make(map[string]map[string]bool) + } + tblUriMapVal = map[string]bool{xlateParams.uri: true} + xlateParams.tblXpathMap[tableName][xlateParams.keyName] = tblUriMapVal + } else { + if tblUriMapVal == nil { + tblUriMapVal = map[string]bool{xlateParams.uri: true} + } else { + tblUriMapVal[xlateParams.uri] = true + } + xlateParams.tblXpathMap[tableName][xlateParams.keyName] = tblUriMapVal } - if len(xpathInfo.fieldName) == 0 { - log.Infof("Field for yang-path(\"%v\") not found in DB.", xpath) - return errors.New("Invalid field name") - } - fieldName := xpathInfo.fieldName - valueStr := "" - if xpathInfo.yangEntry.IsLeafList() { - /* Both yang side and Db side('@' suffix field) the data type is leaf-list */ - log.Info("Yang type and Db type is Leaflist for field = ", xpath) - fieldName += "@" - if reflect.ValueOf(value).Kind() != reflect.Slice { - logStr := fmt.Sprintf("Value for yang xpath %v which is a leaf-list should be a slice", xpath) - log.Error(logStr) - err := errors.New(logStr) - return err + curXlateParams := xlateParams + curXlateParams.tableName = tableName + curXlateParams.xpath = xpath + err = mapFillDataUtil(curXlateParams) + return err +} + +func mapFillDataUtil(xlateParams xlateToParams) error { + var dbs [db.MaxDB]*db.DB + + xpathInfo, ok := xYangSpecMap[xlateParams.xpath] + if !ok { + errStr := fmt.Sprintf("Invalid yang-path(\"%v\").", xlateParams.xpath) + return tlerr.InternalError{Format: errStr} } - valData := reflect.ValueOf(value) - for fidx := 0; fidx < valData.Len(); fidx++ { - if fidx > 0 { - valueStr += "," - } - fVal := fmt.Sprintf("%v", valData.Index(fidx).Interface()) - valueStr = valueStr + fVal + + if len(xpathInfo.xfmrField) > 0 { + xlateParams.uri = xlateParams.uri + "/" + xlateParams.name + + /* field transformer present */ + xfmrLogInfoAll("Transformer function(\"%v\") invoked for yang path(\"%v\"). uri: %v", xpathInfo.xfmrField, xlateParams.xpath, xlateParams.uri) + curYgotNodeData, nodeErr := yangNodeForUriGet(xlateParams.uri, xlateParams.ygRoot) + if nodeErr != nil && xlateParams.oper != DELETE { + return nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, xlateParams.keyName, nil, xlateParams.subOpDataMap, curYgotNodeData, xlateParams.txCache) + retData, err := leafXfmrHandler(inParams, xpathInfo.xfmrField) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return err + } + if retData != nil { + xfmrLogInfoAll("Transformer function : %v Xpath : %v retData: %v", xpathInfo.xfmrField, xlateParams.xpath, retData) + for f, v := range retData { + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, f, v) + } + } + return nil + } + + if len(xpathInfo.fieldName) == 0 { + xfmrLogInfoAll("Field for yang-path(\"%v\") not found in DB.", xlateParams.xpath) + errStr := fmt.Sprintf("Field for yang-path(\"%v\") not found in DB.", xlateParams.xpath) + return tlerr.InternalError{Format: errStr} } - log.Infof("leaf-list value after conversion to DB format %v : %v", fieldName, valueStr) + fieldName := xpathInfo.fieldName + valueStr := "" - } else { // xpath is a leaf - valueStr = fmt.Sprintf("%v", value) - if strings.Contains(valueStr, ":") { - valueStr = strings.Split(valueStr, ":")[1] + fieldXpath := xlateParams.tableName + "/" + fieldName + _, ok = xDbSpecMap[fieldXpath] + if !ok { + logStr := fmt.Sprintf("Failed to find the xDbSpecMap: xpath(\"%v\").", fieldXpath) + log.Warning(logStr) + return nil + } + + if xpathInfo.yangEntry.IsLeafList() { + /* Both yang side and Db side('@' suffix field) the data type is leaf-list */ + xfmrLogInfoAll("Yang type and Db type is Leaflist for field = %v", xlateParams.xpath) + fieldName += "@" + if reflect.ValueOf(xlateParams.value).Kind() != reflect.Slice { + logStr := fmt.Sprintf("Value for yang xpath %v which is a leaf-list should be a slice", xlateParams.xpath) + log.Warning(logStr) + return nil } - } + valData := reflect.ValueOf(xlateParams.value) + for fidx := 0; fidx < valData.Len(); fidx++ { + if fidx > 0 { + valueStr += "," + } - dataToDBMapAdd(tableName, dbKey, result, fieldName, valueStr) - log.Infof("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", tableName, dbKey, - fieldName, valueStr) - return nil + // SNC-3626 - string conversion based on the primitive type + fVal, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, fieldName, valData.Index(fidx).Interface()) + if err == nil { + if ((strings.Contains(fVal, ":")) && (strings.HasPrefix(fVal, OC_MDL_PFX) || strings.HasPrefix(fVal, IETF_MDL_PFX) || strings.HasPrefix(fVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + fVal = strings.SplitN(fVal, ":", 2)[1] + } + valueStr = valueStr + fVal + } else { + logStr := fmt.Sprintf("Couldn't unmarshal Json to DbData: table(\"%v\") field(\"%v\") value(\"%v\").", xlateParams.tableName, fieldName, valData.Index(fidx).Interface()) + log.Warning(logStr) + return nil + } + } + xfmrLogInfoAll("leaf-list value after conversion to DB format %v : %v", fieldName, valueStr) + + } else { // xpath is a leaf + + // SNC-3626 - string conversion based on the primitive type + fVal, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, fieldName, xlateParams.value) + if err == nil { + valueStr = fVal + } else { + logStr := fmt.Sprintf("Couldn't unmarshal Json to DbData: table(\"%v\") field(\"%v\") value(\"%v\").", xlateParams.tableName, fieldName, xlateParams.value) + log.Warning(logStr) + return nil + } + + if ((strings.Contains(valueStr, ":")) && (strings.HasPrefix(valueStr, OC_MDL_PFX) || strings.HasPrefix(valueStr, IETF_MDL_PFX) || strings.HasPrefix(valueStr, IANA_MDL_PFX))) { + // identity-ref/enum might has module prefix + valueStr = strings.SplitN(valueStr, ":", 2)[1] + } + } + + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, fieldName, valueStr) + xfmrLogInfoAll("TblName: \"%v\", key: \"%v\", field: \"%v\", valueStr: \"%v\".", xlateParams.tableName, xlateParams.keyName, + fieldName, valueStr) + return nil } -func sonicYangReqToDbMapCreate(jsonData interface{}, result map[string]map[string]db.Value) error { - if reflect.ValueOf(jsonData).Kind() == reflect.Map { - data := reflect.ValueOf(jsonData) +func sonicYangReqToDbMapCreate(xlateParams xlateToParams) error { + if reflect.ValueOf(xlateParams.jsonData).Kind() == reflect.Map { + data := reflect.ValueOf(xlateParams.jsonData) for _, key := range data.MapKeys() { _, ok := xDbSpecMap[key.String()] if ok { - directDbMapData("", key.String(), data.MapIndex(key).Interface(), result) + directDbMapData("", key.String(), data.MapIndex(key).Interface(), xlateParams.result) } else { - sonicYangReqToDbMapCreate(data.MapIndex(key).Interface(), result) + curXlateParams := xlateParams + curXlateParams.jsonData = data.MapIndex(key).Interface() + sonicYangReqToDbMapCreate(curXlateParams) } } } @@ -213,9 +275,9 @@ func dbMapDataFill(uri string, tableName string, keyName string, d map[string]in for field, value := range d { fieldXpath := tableName + "/" + field if _, fieldOk := xDbSpecMap[fieldXpath]; (fieldOk && (xDbSpecMap[fieldXpath].dbEntry != nil)) { - log.Info("Found non-nil yang entry in xDbSpecMap for field xpath = ", fieldXpath) + xfmrLogInfoAll("Found non-nil yang entry in xDbSpecMap for field xpath = %v", fieldXpath) if xDbSpecMap[fieldXpath].dbEntry.IsLeafList() { - log.Info("Yang type is Leaflist for field = ", field) + xfmrLogInfoAll("Yang type is Leaflist for field = %v", field) field += "@" fieldDt := reflect.ValueOf(value) fieldValue := "" @@ -223,19 +285,27 @@ func dbMapDataFill(uri string, tableName string, keyName string, d map[string]in if fidx > 0 { fieldValue += "," } - fVal := fmt.Sprintf("%v", fieldDt.Index(fidx).Interface()) - fieldValue = fieldValue + fVal + fVal, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, field, fieldDt.Index(fidx).Interface()) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } else { + fieldValue = fieldValue + fVal + } } result[tableName][keyName].Field[field] = fieldValue continue } + dbval, err := unmarshalJsonToDbData(xDbSpecMap[fieldXpath].dbEntry, fieldXpath, field, value) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } else { + result[tableName][keyName].Field[field] = dbval + } } else { // should ideally never happen , just adding for safety - log.Info("Did not find entry in xDbSpecMap for field xpath = ", fieldXpath) + xfmrLogInfoAll("Did not find entry in xDbSpecMap for field xpath = %v", fieldXpath) } - result[tableName][keyName].Field[field] = fmt.Sprintf("%v", value) } - return } func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonData interface{}, result map[string]map[string]db.Value) { @@ -248,12 +318,17 @@ func dbMapListDataFill(uri string, tableName string, dbEntry *yang.Entry, jsonDa if i > 0 { keyName += "|" } - keyName += fmt.Sprintf("%v", d[k]) + fieldXpath := tableName + "/" + k + val, err := unmarshalJsonToDbData(dbEntry.Dir[k], fieldXpath, k, d[k]) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } else { + keyName += val + } delete(d, k) } dbMapDataFill(uri, tableName, keyName, d, result) } - return } func directDbMapData(uri string, tableName string, jsonData interface{}, result map[string]map[string]db.Value) bool { @@ -266,7 +341,7 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result if dbSpecData.keyName != nil { key = *dbSpecData.keyName - log.Infof("Fill data for container uri(%v), key(%v)", uri, key) + xfmrLogInfoAll("Fill data for container uri(%v), key(%v)", uri, key) dbMapDataFill(uri, tableName, key, data, result) return true } @@ -278,10 +353,10 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result eType := yangTypeGet(curDbSpecData.dbEntry) switch eType { case "list": - log.Infof("Fill data for list uri(%v)", uri) + xfmrLogInfoAll("Fill data for list uri(%v)", uri) dbMapListDataFill(uri, tableName, curDbSpecData.dbEntry, v, result) default: - log.Infof("Invalid node type for uri(%v)", uri) + xfmrLogInfoAll("Invalid node type for uri(%v)", uri) } } } @@ -289,277 +364,979 @@ func directDbMapData(uri string, tableName string, jsonData interface{}, result return true } -/* Get the db table, key and field name for the incoming delete request */ -func dbMapDelete(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { - var err error - if isSonicYang(path) { - xpathPrefix, keyName, tableName := sonicXpathKeyExtract(path) - log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) - err = sonicYangReqToDbMapDelete(xpathPrefix, tableName, keyName, result) - } else { - xpathPrefix, keyName, tableName := xpathKeyExtract(d, ygRoot, oper, path) - log.Infof("Delete req: path(\"%v\"), key(\"%v\"), xpathPrefix(\"%v\"), tableName(\"%v\").", path, keyName, xpathPrefix, tableName) - spec, ok := xYangSpecMap[xpathPrefix] +/* Get the data from incoming update/replace request, create map and fill with dbValue(ie. field:value to write into redis-db */ +func dbMapUpdate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, requestUri string, jsonData interface{}, result map[int]map[db.DBNum]map[string]map[string]db.Value, yangDefValMap map[string]map[string]db.Value, yangAuxValMap map[string]map[string]db.Value, txCache interface{}) error { + xfmrLogInfo("Update/replace req: path(\"%v\").", path) + err := dbMapCreate(d, ygRoot, oper, path, requestUri, jsonData, result, yangDefValMap, yangAuxValMap, txCache) + xfmrLogInfo("Update/replace req: path(\"%v\") result(\"%v\").", path, result) + printDbData(result, nil, "/tmp/yangToDbDataUpRe.txt") + return err +} + +func dbMapDefaultFieldValFill(xlateParams xlateToParams, tblUriList []string) error { + tblData := xlateParams.result[xlateParams.tableName] + var dbs [db.MaxDB]*db.DB + tblName := xlateParams.tableName + dbKey := xlateParams.keyName + for _, tblUri := range tblUriList { + xfmrLogInfoAll("Processing uri %v for default value filling(Table - %v, dbKey - %v)", tblUri, tblName, dbKey) + yangXpath, _, prdErr := XfmrRemoveXPATHPredicates(tblUri) + if prdErr != nil { + continue + } + yangNode, ok := xYangSpecMap[yangXpath] if ok { - if spec.tableName != nil { - result[*spec.tableName] = make(map[string]db.Value) - if len(keyName) > 0 { - result[*spec.tableName][keyName] = db.Value{Field: make(map[string]string)} - if spec.yangEntry != nil && spec.yangEntry.Node.Statement().Keyword == "leaf" { - result[*spec.tableName][keyName].Field[spec.fieldName] = "" + for childName := range yangNode.yangEntry.Dir { + childXpath := yangXpath + "/" + childName + childUri := tblUri + "/" + childName + childNode, ok := xYangSpecMap[childXpath] + if ok { + if (len(childNode.xfmrFunc) > 0) { + xfmrLogInfoAll("Skip default filling since a subtree Xfmr found for path - %v", childXpath) + continue + } + + if (childNode.yangDataType == YANG_LIST || childNode.yangDataType == YANG_CONTAINER) && !childNode.yangEntry.ReadOnly() { + var tblList []string + tblList = append(tblList, childUri) + err := dbMapDefaultFieldValFill(xlateParams, tblList) + if err != nil { + return err + } + } + if (childNode.tableName != nil && *childNode.tableName == tblName) || (childNode.xfmrTbl != nil) { + tblXfmrPresent := false + var inParamsTblXfmr XfmrParams + if childNode.xfmrTbl != nil { + if len(*childNode.xfmrTbl) > 0 { + inParamsTblXfmr = formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, childUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, "", xlateParams.txCache) + //performance optimization - call table transformer only for default val leaf and avoid for other leaves unless its REPLACE operi(aux-map filling) + tblXfmrPresent = true + + } + } + _, ok := tblData[dbKey].Field[childName] + if !ok { + if len(childNode.xfmrField) > 0 { + childYangType := childNode.yangEntry.Type.Kind + var param interface{} + oper := xlateParams.oper + if len(childNode.defVal) > 0 { + if tblXfmrPresent { + chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) + xfmrLogInfoAll("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) + if ctErr != nil || chldTblNm != tblName { + continue + } + } + xfmrLogInfoAll("Update(\"%v\") default: tbl[\"%v\"]key[\"%v\"]fld[\"%v\"] = val(\"%v\").", + childXpath, tblName, dbKey, childNode.fieldName, childNode.defVal) + _, defValPtr, err := DbToYangType(childYangType, childXpath, childNode.defVal) + if err == nil && defValPtr != nil { + param = defValPtr + } else { + xfmrLogInfoAll("Failed to update(\"%v\") default: tbl[\"%v\"]key[\"%v\"]fld[\"%v\"] = val(\"%v\").", + childXpath, tblName, dbKey, childNode.fieldName, childNode.defVal) + } + } else { + if xlateParams.oper != REPLACE { + continue + } + oper = DELETE + if tblXfmrPresent { + chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) + xfmrLogInfoAll("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) + if ctErr != nil || chldTblNm != tblName { + continue + } + } + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, tblUri+"/"+childName, xlateParams.requestUri, oper, "", nil, xlateParams.subOpDataMap, param, xlateParams.txCache) + retData, err := leafXfmrHandler(inParams, childNode.xfmrField) + if err != nil { + log.Warningf("Default/AuxMap Value filling. Received error %v from %v", err, childNode.xfmrField) + } + if retData != nil { + xfmrLogInfoAll("Transformer function : %v Xpath: %v retData: %v", childNode.xfmrField, childXpath, retData) + for f, v := range retData { + // Fill default value only if value is not available in result Map + // else we overwrite the value filled in resultMap with default value + _, ok := xlateParams.result[tblName][dbKey].Field[f] + if !ok { + if len(childNode.defVal) > 0 { + dataToDBMapAdd(tblName, dbKey, xlateParams.yangDefValMap, f, v) + } else { + // Fill the yangAuxValMap with all fields that are not in either resultMap or defaultValue Map + dataToDBMapAdd(tblName, dbKey, xlateParams.yangAuxValMap, f, "") + } + } + } + } + } else if len(childNode.fieldName) > 0 { + var xfmrErr error + if _, ok := xDbSpecMap[tblName+"/"+childNode.fieldName]; ok { + if tblXfmrPresent { + chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) + xfmrLogInfoAll("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) + if ctErr != nil || chldTblNm != tblName { + continue + } + } + // Fill default value only if value is not available in result Map + // else we overwrite the value filled in resultMap with default value + _, ok = xlateParams.result[tblName][dbKey].Field[childNode.fieldName] + if !ok { + if len(childNode.defVal) > 0 { + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, childXpath, dbKey, xlateParams.jsonData, xlateParams.resultMap, xlateParams.yangDefValMap, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, &xfmrErr, childName, childNode.defVal, tblName) + err := mapFillDataUtil(curXlateParams) + if err != nil { + log.Warningf("Default/AuxMap Value filling. Received error %v from %v", err, childNode.fieldName) + } + } else { + if xlateParams.oper != REPLACE { + continue + } + dataToDBMapAdd(tblName, dbKey, xlateParams.yangAuxValMap, childNode.fieldName, "") + } + } + } + } + } } - } - } else if len(spec.childTable) > 0 { - for _, child := range spec.childTable { - result[child] = make(map[string]db.Value) } } } } - log.Infof("Delete req: path(\"%v\") result(\"%v\").", path, result) - return err + return nil } -func sonicYangReqToDbMapDelete(xpathPrefix string, tableName string, keyName string, result map[string]map[string]db.Value) error { - if (tableName != "") { - // Specific table entry case - result[tableName] = make(map[string]db.Value) - if (keyName != "") { - // Specific key case - var dbVal db.Value - tokens:= strings.Split(xpathPrefix, "/") - if tokens[SONIC_TABLE_INDEX] == tableName { - fieldName := tokens[len(tokens)-1] - dbSpecField := tableName + "/" + fieldName - _, ok := xDbSpecMap[dbSpecField] - if ok { - yangType := xDbSpecMap[dbSpecField].fieldType - // terminal node case - if yangType == YANG_LEAF_LIST { - fieldName = fieldName + "@" - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = "" - } - if yangType == YANG_LEAF { - dbVal.Field = make(map[string]string) - dbVal.Field[fieldName] = "" - } - } - } - result[tableName][keyName] = dbVal - } else { - // Get all keys - } - } else { - // Get all table entries - // If table name not available in xpath get top container name - _, ok := xDbSpecMap[xpathPrefix] - if ok && xDbSpecMap[xpathPrefix] != nil { - dbInfo := xDbSpecMap[xpathPrefix] - if dbInfo.fieldType == "container" { - for dir, _ := range dbInfo.dbEntry.Dir { - result[dir] = make(map[string]db.Value) - } - } - } - } - return nil +func dbMapDefaultValFill(xlateParams xlateToParams) error { + for tbl, tblData := range xlateParams.result { + for dbKey := range tblData { + var yxpathList []string //contains all uris(with keys) that were traversed for a table while processing the incoming request + if tblUriMapVal, ok := xlateParams.tblXpathMap[tbl][dbKey]; ok { + for tblUri := range tblUriMapVal { + yxpathList = append(yxpathList, tblUri) + } + } + curXlateParams := xlateParams + curXlateParams.tableName = tbl + curXlateParams.keyName = dbKey + if len(yxpathList) > 0 { + err := dbMapDefaultFieldValFill(curXlateParams, yxpathList) + if err != nil { + return err + } + } + } + } + return nil } -/* Get the data from incoming update/replace request, create map and fill with dbValue(ie. field:value to write into redis-db */ -func dbMapUpdate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { - log.Infof("Update/replace req: path(\"%v\").", path) - dbMapCreate(d, ygRoot, oper, path, jsonData, result) - log.Infof("Update/replace req: path(\"%v\") result(\"%v\").", path, result) - printDbData(result, "/tmp/yangToDbDataUpRe.txt") - return nil -} /* Get the data from incoming create request, create map and fill with dbValue(ie. field:value to write into redis-db */ -func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, jsonData interface{}, result map[string]map[string]db.Value) error { - var err error - root := xpathRootNameGet(path) - if isSonicYang(path) { - err = sonicYangReqToDbMapCreate(jsonData, result) +func dbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, jsonData interface{}, resultMap map[int]RedisDbMap, yangDefValMap map[string]map[string]db.Value, yangAuxValMap map[string]map[string]db.Value, txCache interface{}) error { + var err, xfmrErr error + var cascadeDelTbl []string + var result = make(map[string]map[string]db.Value) + tblXpathMap := make(map[string]map[string]map[string]bool) + subOpDataMap := make(map[int]*RedisDbMap) + root := xpathRootNameGet(uri) + + /* Check if the parent table exists for RFC compliance */ + var exists bool + var dbs [db.MaxDB]*db.DB + subOpMapDiscard := make(map[int]*RedisDbMap) + exists, err = verifyParentTable(d, dbs, ygRoot, oper, uri, nil, txCache, subOpMapDiscard) + xfmrLogInfoAll("verifyParentTable() returned - exists - %v, err - %v", exists, err) + if err != nil { + log.Warningf("Cannot perform Operation %v on uri %v due to - %v", oper, uri, err) + return err + } + if !exists { + errStr := fmt.Sprintf("Parent table does not exist for uri(%v)", uri) + return tlerr.NotFoundError{Format: errStr} + } + + xlateToData := formXlateToDbParam(d, ygRoot, oper, root, uri, "", "", jsonData, resultMap, result, txCache, tblXpathMap, subOpDataMap, &cascadeDelTbl, &xfmrErr, "", "", "") + + moduleNm := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleNm) + + if isSonicYang(uri) { + err = sonicYangReqToDbMapCreate(xlateToData) + xpathPrefix, keyName, tableName := sonicXpathKeyExtract(uri) + xfmrLogInfoAll("xpath - %v, keyName - %v, tableName - %v , for uri - %v", xpathPrefix, keyName, tableName, uri) + fldPth := strings.Split(xpathPrefix, "/") + if len(fldPth) > SONIC_FIELD_INDEX { + fldNm := fldPth[SONIC_FIELD_INDEX] + xfmrLogInfoAll("Field Name : %v", fldNm) + if fldNm != "" { + _, ok := xDbSpecMap[tableName] + if ok { + dbSpecField := tableName + "/" + fldNm + _, dbFldok := xDbSpecMap[dbSpecField] + if dbFldok { + /* RFC compliance - REPLACE on leaf/leaf-list becomes UPDATE/merge */ + resultMap[UPDATE] = make(RedisDbMap) + resultMap[UPDATE][db.ConfigDB] = result + } else { + log.Warningf("For uri - %v, no entry found in xDbSpecMap for table(%v)/field(%v)", uri, tableName, fldNm) + } + } else { + log.Warningf("For uri - %v, no entry found in xDbSpecMap with tableName - %v", uri, tableName) + } + } + } else { + resultMap[oper] = make(RedisDbMap) + resultMap[oper][db.ConfigDB] = result + } } else { - err = yangReqToDbMapCreate(d, ygRoot, oper, root, "", "", jsonData, result) + /* Invoke pre-xfmr is present for the yang module */ + if modSpecInfo, specOk := xYangSpecMap[moduleNm]; specOk && (len(modSpecInfo.xfmrPre) > 0) { + var dbs [db.MaxDB]*db.DB + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", nil, xlateToData.subOpDataMap, nil, txCache) + err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) + xfmrLogInfo("Invoked pre-transformer: %v, oper: %v, subOpDataMap: %v ", + modSpecInfo.xfmrPre, oper, subOpDataMap) + if err != nil { + log.Warningf("Pre-transformer: %v failed.(err:%v)", modSpecInfo.xfmrPre, err) + return err + } + } + + err = yangReqToDbMapCreate(xlateToData) + if xfmrErr != nil { + return xfmrErr + } + if err != nil { + return err + } } if err == nil { - if oper == CREATE { - moduleNm := "/" + strings.Split(path, "/")[1] - log.Infof("Module name for path %s is %s", path, moduleNm) + if !isSonicYang(uri) { + xpath, _, _ := XfmrRemoveXPATHPredicates(uri) + yangNode, ok := xYangSpecMap[xpath] + defSubOpDataMap := make(map[int]*RedisDbMap) + if ok { + xfmrLogInfo("Fill default value for %v, oper(%v)\r\n", uri, oper) + curXlateToParams := formXlateToDbParam(d, ygRoot, oper, uri, requestUri, xpath, "", jsonData, resultMap, result, txCache, tblXpathMap, defSubOpDataMap, &cascadeDelTbl, &xfmrErr, "", "", "") + curXlateToParams.yangDefValMap = yangDefValMap + curXlateToParams.yangAuxValMap = yangAuxValMap + err = dbMapDefaultValFill(curXlateToParams) + if err != nil { + return err + } + } + + if ok && oper == REPLACE { + if yangNode.yangDataType == YANG_LEAF { + xfmrLogInfo("Change leaf oper to UPDATE for %v, oper(%v)\r\n", uri, oper) + resultMap[UPDATE] = make(RedisDbMap) + resultMap[UPDATE][db.ConfigDB] = result + result = make(map[string]map[string]db.Value) + } else if yangNode.yangDataType == YANG_LEAF_LIST { + /* RFC compliance - REPLACE on leaf-list becomes UPDATE/merge */ + xfmrLogInfo("Change leaflist oper to UPDATE for %v, oper(%v)\r\n", uri, oper) + resultMap[UPDATE] = make(RedisDbMap) + resultMap[UPDATE][db.ConfigDB] = result + result = make(map[string]map[string]db.Value) + } + } + + moduleNm := "/" + strings.Split(uri, "/")[1] + xfmrLogInfo("Module name for uri %s is %s", uri, moduleNm) if _, ok := xYangSpecMap[moduleNm]; ok { if xYangSpecMap[moduleNm].yangDataType == "container" && len(xYangSpecMap[moduleNm].xfmrPost) > 0 { - log.Info("Invoke post transformer: ", xYangSpecMap[moduleNm].xfmrPost) - dbDataMap := make(map[db.DBNum]map[string]map[string]db.Value) + xfmrLogInfo("Invoke post transformer: %v, result map: %v ", xYangSpecMap[moduleNm].xfmrPost, result) + var dbDataMap = make(RedisDbMap) dbDataMap[db.ConfigDB] = result var dbs [db.MaxDB]*db.DB - inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, path, oper, "", &dbDataMap, nil) - result, err = postXfmrHandlerFunc(inParams) + inParams := formXfmrInputRequest(d, dbs, db.ConfigDB, ygRoot, uri, requestUri, oper, "", &dbDataMap, subOpDataMap, nil, txCache) + inParams.yangDefValMap = yangDefValMap + result, err = postXfmrHandlerFunc(xYangSpecMap[moduleNm].xfmrPost, inParams) + if err != nil { + return err + } + if inParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(cascadeDelTbl, tblNm) { + cascadeDelTbl = append(cascadeDelTbl, tblNm) + } + } + } } } else { - log.Errorf("No Entry exists for module %s in xYangSpecMap. Unable to process post xfmr (\"%v\") path(\"%v\") error (\"%v\").", oper, path, err) + log.Warningf("No Entry exists for module %s in xYangSpecMap. Unable to process post xfmr (\"%v\") uri(\"%v\") error (\"%v\").", oper, uri, err) } + if len(result) > 0 || len(subOpDataMap) > 0 { + resultMap[oper] = make(RedisDbMap) + resultMap[oper][db.ConfigDB] = result + for op, redisMapPtr := range subOpDataMap { + if redisMapPtr != nil { + if _,ok := resultMap[op]; !ok { + resultMap[op] = make(RedisDbMap) + } + for dbNum, dbMap := range *redisMapPtr { + if _,ok := resultMap[op][dbNum]; !ok { + resultMap[op][dbNum] = make(map[string]map[string]db.Value) + } + mapCopy(resultMap[op][dbNum],dbMap) + } + } + } + } + } + + err = dbDataXfmrHandler(resultMap) + if err != nil { + log.Warningf("Failed in dbdata-xfmr for %v", resultMap) + return err } - printDbData(result, "/tmp/yangToDbDataCreate.txt") + + if (len(cascadeDelTbl) > 0) { + cdErr := handleCascadeDelete(d, resultMap, cascadeDelTbl) + if cdErr != nil { + xfmrLogInfo("Cascade Delete Failed for cascadeDelTbl (%v), Error (%v).", cascadeDelTbl, cdErr) + return cdErr + } + } + + printDbData(resultMap, yangDefValMap, "/tmp/yangToDbDataCreate.txt") } else { - log.Errorf("DBMapCreate req failed for oper (\"%v\") path(\"%v\") error (\"%v\").", oper, path, err) + log.Warningf("DBMapCreate req failed for oper (\"%v\") uri(\"%v\") error (\"%v\").", oper, uri, err) } return err } func yangNodeForUriGet(uri string, ygRoot *ygot.GoStruct) (interface{}, error) { - path, _ := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + path, err := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + if path == nil || err != nil { + log.Warningf("For uri %v - StringToPath failure", uri) + errStr := fmt.Sprintf("Ygot stringTopath failed for uri(%v)", uri) + return nil, tlerr.InternalError{Format: errStr} + } + for _, p := range path.Elem { pathSlice := strings.Split(p.Name, ":") p.Name = pathSlice[len(pathSlice)-1] if len(p.Key) > 0 { for ekey, ent := range p.Key { - eslice := strings.Split(ent, ":") - p.Key[ekey] = eslice[len(eslice)-1] + // SNC-2126: check the occurrence of ":" + if ((strings.Contains(ent, ":")) && (strings.HasPrefix(ent, OC_MDL_PFX) || strings.HasPrefix(ent, IETF_MDL_PFX) || strings.HasPrefix(ent, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + eslice := strings.SplitN(ent, ":", 2) + // TODO - exclude the prexix by checking enum type + p.Key[ekey] = eslice[len(eslice)-1] + } else { + p.Key[ekey] = ent + } } } } - ocbSch, _ := ocbinds.Schema() schRoot := ocbSch.RootSchema() node, nErr := ytypes.GetNode(schRoot, (*ygRoot).(*ocbinds.Device), path) if nErr != nil { - return nil, nErr + log.Warningf("For uri %v - GetNode failure - %v", uri, nErr) + errStr := fmt.Sprintf("%v", nErr) + return nil, tlerr.InternalError{Format: errStr} + } + if ((node == nil) || (len(node) == 0) || (node[0].Data == nil)) { + log.Warningf("GetNode returned nil for uri %v", uri) + errStr := "GetNode returned nil for the given uri." + return nil, tlerr.InternalError{Format: errStr} } + xfmrLogInfoAll("GetNode data: %v", node[0].Data) return node[0].Data, nil } -func yangReqToDbMapCreate(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, xpathPrefix string, keyName string, jsonData interface{}, result map[string]map[string]db.Value) error { - log.Infof("key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) - var dbs [db.MaxDB]*db.DB +func yangReqToDbMapCreate(xlateParams xlateToParams) error { + xfmrLogInfoAll("key(\"%v\"), xpathPrefix(\"%v\").", xlateParams.keyName, xlateParams.xpath) + var dbs [db.MaxDB]*db.DB + var retErr error - if reflect.ValueOf(jsonData).Kind() == reflect.Slice { - log.Infof("slice data: key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) - jData := reflect.ValueOf(jsonData) - dataMap := make([]interface{}, jData.Len()) - for idx := 0; idx < jData.Len(); idx++ { - dataMap[idx] = jData.Index(idx).Interface() - } - for _, data := range dataMap { - curKey := "" - curUri, _ := uriWithKeyCreate(uri, xpathPrefix, data) - _, ok := xYangSpecMap[xpathPrefix] - if ok && len(xYangSpecMap[xpathPrefix].xfmrKey) > 0 { - /* key transformer present */ - curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) - if nodeErr != nil { - curYgotNode = nil - } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpathPrefix].xfmrKey), inParams) - if err != nil { - return err - } - if ret != nil { - curKey = ret[0].Interface().(string) - } - } else if xYangSpecMap[xpathPrefix].keyName != nil { - curKey = *xYangSpecMap[xpathPrefix].keyName - } else { - curKey = keyCreate(keyName, xpathPrefix, data, d.Opts.KeySeparator) - } - yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpathPrefix, curKey, data, result) - } - } else { - if reflect.ValueOf(jsonData).Kind() == reflect.Map { - jData := reflect.ValueOf(jsonData) - for _, key := range jData.MapKeys() { - typeOfValue := reflect.TypeOf(jData.MapIndex(key).Interface()).Kind() - - log.Infof("slice/map data: key(\"%v\"), xpathPrefix(\"%v\").", keyName, xpathPrefix) - xpath := uri - curUri := uri - curKey := keyName - pathAttr := key.String() - if len(xpathPrefix) > 0 { - if strings.Contains(pathAttr, ":") { - pathAttr = strings.Split(pathAttr, ":")[1] - } - xpath = xpathPrefix + "/" + pathAttr - curUri = uri + "/" + pathAttr - } - _, ok := xYangSpecMap[xpath] - log.Infof("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", - curKey, xpath, curUri) - if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrKey) > 0 { - /* key transformer present */ - curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) + if reflect.ValueOf(xlateParams.jsonData).Kind() == reflect.Slice { + xfmrLogInfoAll("slice data: key(\"%v\"), xpathPrefix(\"%v\").", xlateParams.keyName, xlateParams.xpath) + jData := reflect.ValueOf(xlateParams.jsonData) + dataMap := make([]interface{}, jData.Len()) + for idx := 0; idx < jData.Len(); idx++ { + dataMap[idx] = jData.Index(idx).Interface() + } + for _, data := range dataMap { + curKey := "" + curUri, _ := uriWithKeyCreate(xlateParams.uri, xlateParams.xpath, data) + _, ok := xYangSpecMap[xlateParams.xpath] + if ok && len(xYangSpecMap[xlateParams.xpath].xfmrKey) > 0 { + /* key transformer present */ + curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + + ktRetData, err := keyXfmrHandler(inParams, xYangSpecMap[xlateParams.xpath].xfmrKey) + //if key transformer is called without key values in curUri ignore the error + if err != nil && strings.HasSuffix(curUri, "]") { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil + } + curKey = ktRetData + } else if ok && xYangSpecMap[xlateParams.xpath].keyName != nil { + curKey = *xYangSpecMap[xlateParams.xpath].keyName + } else { + curKey = keyCreate(xlateParams.keyName, xlateParams.xpath, data, xlateParams.d.Opts.KeySeparator) + } + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, xlateParams.xpath, curKey, data, xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, "", "", "") + retErr = yangReqToDbMapCreate(curXlateParams) + } + } else { + if reflect.ValueOf(xlateParams.jsonData).Kind() == reflect.Map { + jData := reflect.ValueOf(xlateParams.jsonData) + for _, key := range jData.MapKeys() { + typeOfValue := reflect.TypeOf(jData.MapIndex(key).Interface()).Kind() + + xfmrLogInfoAll("slice/map data: key(\"%v\"), xpathPrefix(\"%v\").", xlateParams.keyName, xlateParams.xpath) + xpath := xlateParams.uri + curUri := xlateParams.uri + curKey := xlateParams.keyName + pathAttr := key.String() + if len(xlateParams.xpath) > 0 { + curUri = xlateParams.uri + "/" + pathAttr + if strings.Contains(pathAttr, ":") { + pathAttr = strings.Split(pathAttr, ":")[1] + } + xpath = xlateParams.xpath + "/" + pathAttr + } + _, ok := xYangSpecMap[xpath] + xfmrLogInfoAll("slice/map data: curKey(\"%v\"), xpath(\"%v\"), curUri(\"%v\").", + curKey, xpath, curUri) + if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrKey) > 0 { + specYangType := yangTypeGet(xYangSpecMap[xpath].yangEntry) + curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) if nodeErr != nil { curYgotNode = nil } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) - if err != nil { - return err - } - if ret != nil { - curKey = ret[0].Interface().(string) + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + ktRetData, err := keyXfmrHandler(inParams, xYangSpecMap[xpath].xfmrKey) + if ((err != nil) && (specYangType != YANG_LIST || strings.HasSuffix(curUri, "]"))) { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil } - } else if xYangSpecMap[xpath].keyName != nil { + curKey = ktRetData + } else if ok && xYangSpecMap[xpath].keyName != nil { curKey = *xYangSpecMap[xpath].keyName } - if (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { - if ok && xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 { - /* subtree transformer present */ - curYgotNode, nodeErr := yangNodeForUriGet(curUri, ygRoot) - if nodeErr != nil { - curYgotNode = nil + if ok && (typeOfValue == reflect.Map || typeOfValue == reflect.Slice) && xYangSpecMap[xpath].yangDataType != "leaf-list" { + // Call subtree only if start processing for the requestUri. Skip for parent uri traversal + curXpath, _, _ := XfmrRemoveXPATHPredicates(curUri) + reqXpath, _, _ := XfmrRemoveXPATHPredicates(xlateParams.requestUri) + xfmrLogInfoAll("CurUri: %v, requestUri: %v\r\n", curUri, xlateParams.requestUri) + xfmrLogInfoAll("curxpath: %v, requestxpath: %v\r\n", curXpath, reqXpath) + if strings.HasPrefix(curXpath, reqXpath) { + if xYangSpecMap[xpath] != nil && len(xYangSpecMap[xpath].xfmrFunc) > 0 && + (xYangSpecMap[xlateParams.xpath] != xYangSpecMap[xpath]) { + /* subtree transformer present */ + curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + stRetData, err := xfmrHandler(inParams, xYangSpecMap[xpath].xfmrFunc) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil + } + if stRetData != nil { + mapCopy(xlateParams.result, stRetData) + } + if xlateParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(*xlateParams.pCascadeDelTbl, tblNm) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, tblNm) + } + } + } } - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curUri, oper, "", nil, curYgotNode) - ret, err := XlateFuncCall(yangToDbXfmrFunc(xYangSpecMap[xpath].xfmrFunc), inParams) - if err != nil { - return nil - } - if ret != nil { - mapCopy(result, ret[0].Interface().(map[string]map[string]db.Value)) + } + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, xpath, curKey, jData.MapIndex(key).Interface(), xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, "", "", "") + retErr = yangReqToDbMapCreate(curXlateParams) + } else { + pathAttr := key.String() + if strings.Contains(pathAttr, ":") { + pathAttr = strings.Split(pathAttr, ":")[1] + } + xpath := xlateParams.xpath + "/" + pathAttr + xfmrLogInfoAll("LEAF Case: xpath: %v, xpathPrefix: %v, pathAttr: %v", xpath, xlateParams.xpath, pathAttr) + /* skip processing for list key-leaf outside of config container(OC yang) directly under the list. + Inside full-spec isKey is set to true for list key-leaf dierctly under the list(outside of config container) + For ietf yang(eg.ietf-ptp) list key-leaf might have a field transformer. + */ + _, ok := xYangSpecMap[xpath] + if ok && ((!xYangSpecMap[xpath].isKey) || (len(xYangSpecMap[xpath].xfmrField) > 0)) { + if len(xYangSpecMap[xpath].xfmrFunc) == 0 { + value := jData.MapIndex(key).Interface() + xfmrLogInfoAll("data field: key(\"%v\"), value(\"%v\").", key, value) + curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, xlateParams.uri, xlateParams.requestUri, xlateParams.xpath, curKey, xlateParams.jsonData, xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, pathAttr, value, "") + retErr = mapFillData(curXlateParams) + if retErr != nil { + log.Warningf("Failed constructing data for db write: key(\"%v\"), value(\"%v\"), path(\"%v\").", + pathAttr, value, xlateParams.xpath) + return retErr + } + } else { + xfmrLogInfoAll("write: key(\"%v\"), xpath(\"%v\"), uri(%v).",key, xpath, xlateParams.uri) + curYgotNode, nodeErr := yangNodeForUriGet(xlateParams.uri, xlateParams.ygRoot) + if nodeErr != nil { + curYgotNode = nil + } + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, xlateParams.uri, xlateParams.requestUri, xlateParams.oper, curKey, nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + stRetData, err := xfmrHandler(inParams, xYangSpecMap[xpath].xfmrFunc) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + return nil + } + if stRetData != nil { + mapCopy(xlateParams.result, stRetData) + } + if xlateParams.pCascadeDelTbl != nil && len(*inParams.pCascadeDelTbl) > 0 { + for _, tblNm := range *inParams.pCascadeDelTbl { + if !contains(*xlateParams.pCascadeDelTbl, tblNm) { + *xlateParams.pCascadeDelTbl = append(*xlateParams.pCascadeDelTbl, tblNm) + } + } + } + } + } + } + } + } + } + + return retErr +} + +func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper int, uri string, dbData RedisDbMap) (bool, error) { + var err error + pathList := splitUri(uri) + + xpath, dbKey, table := sonicXpathKeyExtract(uri) + xfmrLogInfoAll("uri: %v xpath: %v table: %v, key: %v", uri, xpath, table, dbKey) + + if (len(table) > 0) && (len(dbKey) > 0) { + tableExists := false + var derr error + if oper == GET { + var cdb db.DBNum = db.ConfigDB + dbInfo, ok := xDbSpecMap[table] + if !ok { + log.Warningf("No entry in xDbSpecMap for xpath %v, for uri - %v", table, uri) + } else { + cdb = dbInfo.dbIndex + } + tableExists = dbTableExistsInDbData(cdb, table, dbKey, dbData) + derr = tlerr.NotFoundError{Format:"Resource not found"} + } else { + // Valid table mapping exists. Read the table entry from DB + tableExists, derr = dbTableExists(d, table, dbKey, oper) + if derr != nil { + return false, derr } - } else { - yangReqToDbMapCreate(d, ygRoot, oper, curUri, xpath, curKey, jData.MapIndex(key).Interface(), result) - } + } + if len(pathList) == SONIC_LIST_INDEX && (oper == UPDATE || oper == CREATE || oper == DELETE || oper == GET) && !tableExists { + // Uri is at /sonic-module:sonic-module/container-table/list + // PATCH opertion permitted only if table exists in DB. + // POST case since the uri is the parent, the parent needs to exist + // PUT case allow operation(Irrespective of table existence update the DB either through CREATE or REPLACE) + // DELETE case Table instance should be available to perform delete else, CVL may throw error + log.Warningf("Parent table %v with key %v does not exist for oper %v in DB", table, dbKey, oper) + err = tlerr.NotFound("Resource not found") + return false, err + } else if len(pathList) > SONIC_LIST_INDEX && !tableExists { + // Uri is at /sonic-module/container-table/list or /sonic-module/container-table/list/leaf + // Parent table should exist for all CRUD cases + log.Warningf("Parent table %v with key %v does not exist in DB", table, dbKey) + err = tlerr.NotFound("Resource not found") + return false, err } else { - pathAttr := key.String() - if strings.Contains(pathAttr, ":") { - pathAttr = strings.Split(pathAttr, ":")[1] - } - value := jData.MapIndex(key).Interface() - log.Infof("data field: key(\"%v\"), value(\"%v\").", key, value) - err := mapFillData(d, ygRoot, oper, uri, curKey, result, xpathPrefix, - pathAttr, value) - if err != nil { - log.Errorf("Failed constructing data for db write: key(\"%v\"), value(\"%v\"), path(\"%v\").", - pathAttr, value, xpathPrefix) - } + // Allow all other operations + return true, err } - } + } else { + // Request is at module level. No need to check for parent table. Hence return true always or + // Request at /sonic-module:sonic-module/container-table level + return true, err } - } - return nil } -/* Debug function to print the map data into file */ -func printDbData(db map[string]map[string]db.Value, fileName string) { - fp, err := os.Create(fileName) - if err != nil { - return - } - defer fp.Close() - - for k, v := range db { - fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") - fmt.Fprintf(fp, "table name : %v\r\n", k) - for ik, iv := range v { - fmt.Fprintf(fp, " key : %v\r\n", ik) - for k, d := range iv.Field { - fmt.Fprintf(fp, " %v :%v\r\n", k, d) - } +/* This function checks the existence of Parent tables in DB for the given URI request + and returns a boolean indicating if the operation is permitted based on the operation type*/ +func verifyParentTable(d *db.DB, dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbData RedisDbMap, txCache interface{}, subOpDataMap map[int]*RedisDbMap) (bool, error) { + xfmrLogInfoAll("Checking for Parent table existence for uri: %v", uri) + if isSonicYang(uri) { + return verifyParentTableSonic(d, dbs, oper, uri, dbData) + } else { + return verifyParentTableOc(d, dbs, ygRoot, oper, uri, dbData, txCache, subOpDataMap) } - } - fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") - return +} + +func verifyParentTblSubtree(dbs [db.MaxDB]*db.DB, uri string, xfmrFuncNm string, oper int, dbData RedisDbMap) (bool, error) { + var inParams XfmrSubscInParams + inParams.uri = uri + inParams.dbDataMap = make(RedisDbMap) + inParams.dbs = dbs + inParams.subscProc = TRANSLATE_SUBSCRIBE + parentTblExists := true + var err error + + st_result, st_err := xfmrSubscSubtreeHandler(inParams, xfmrFuncNm) + if st_result.isVirtualTbl { + xfmrLogInfoAll("Subtree returned Virtual table true.") + goto Exit + } + if st_err != nil { + err = st_err + parentTblExists = false + goto Exit + } else if st_result.dbDataMap != nil && len(st_result.dbDataMap) > 0 { + xfmrLogInfoAll("Subtree subcribe dbData %v", st_result.dbDataMap) + for dbNo, dbMap := range st_result.dbDataMap { + xfmrLogInfoAll("processing Db no - %v", dbNo) + for table, keyInstance := range dbMap { + xfmrLogInfoAll("processing Db table - %v", table) + for dbKey := range keyInstance { + xfmrLogInfoAll("processing Db key - %v", dbKey) + exists := false + if oper != GET { + dptr, derr := db.NewDB(getDBOptions(dbNo)) + if derr != nil { + log.Warningf("Couldn't allocate NewDb/DbOptions for db - %v, while processing uri - %v", dbNo, uri) + err = derr + parentTblExists = false + goto Exit + } + defer dptr.DeleteDB() + exists, err = dbTableExists(dptr, table, dbKey, oper) + } else { + d := dbs[dbNo] + if dbKey == "*" { //dbKey is "*" for GET on entire list + xfmrLogInfoAll("Found table instance in dbData") + goto Exit + } + // GET case - attempt to find in dbData before doing a dbGet in dbTableExists() + exists = dbTableExistsInDbData(dbNo, table, dbKey, dbData) + if exists { + xfmrLogInfoAll("Found table instance in dbData") + goto Exit + } + exists, err = dbTableExists(d, table, dbKey, oper) + } + if !exists || err != nil { + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", table, dbKey, uri) + err = tlerr.NotFound("Resource not found") + parentTblExists = false + goto Exit + } + } + } + + } + } else { + log.Warningf("No Table information retrieved from subtree for uri %v", uri) + err = tlerr.NotFound("Resource not found") + parentTblExists = false + goto Exit + } + Exit: + xfmrLogInfoAll("For subtree at uri - %v, returning ,parentTblExists - %v, err - %v", parentTblExists, err) + return parentTblExists, err +} + +func verifyParentTableOc(d *db.DB, dbs [db.MaxDB]*db.DB, ygRoot *ygot.GoStruct, oper int, uri string, dbData RedisDbMap, txCache interface{}, subOpDataMap map[int]*RedisDbMap) (bool, error) { + var err error + var cdb db.DBNum + uriList := splitUri(uri) + parentTblExists := true + curUri := "/" + yangType := "" + xpath, _, _ := XfmrRemoveXPATHPredicates(uri) + xpathInfo, ok := xYangSpecMap[xpath] + if !ok { + errStr := fmt.Sprintf("No entry found in xYangSpecMap for uri - %v", uri) + err = tlerr.InternalError{Format:errStr} + return false, err + } + yangType = yangTypeGet(xpathInfo.yangEntry) + if yangType == YANG_LEAF_LIST { + /*query is for leaf-list instance, hence remove that from uriList to avoid list-key like processing*/ + if ((strings.HasSuffix(uriList[len(uriList)-1], "]")) || (strings.HasSuffix(uriList[len(uriList)-1], "]/"))) { //splitUri chops off the leaf-list value having square brackets + uriList[len(uriList)-1] = strings.SplitN(uriList[len(uriList)-1], "[", 2)[0] + xfmrLogInfoAll("Uri list after removing leaf-list instance portion - %v", uriList) + } + } + + parentUriList := uriList[:len(uriList)-1] + xfmrLogInfoAll("Parent uri list - %v", parentUriList) + + // Loop for the parent uri to check parent table existence + for idx, path := range parentUriList { + curUri += uriList[idx] + + /* Check for parent table for oc- yang lists*/ + keyList := rgpKeyExtract.FindAllString(path, -1) + if len(keyList) > 0 { + + //Check for subtree existence + curXpath, _, _ := XfmrRemoveXPATHPredicates(curUri) + curXpathInfo, ok := xYangSpecMap[curXpath] + if !ok { + errStr := fmt.Sprintf("No entry found in xYangSpecMap for uri - %v", curUri) + err = tlerr.InternalError{Format:errStr} + parentTblExists = false + break + } + if oper == GET { + cdb = curXpathInfo.dbIndex + xfmrLogInfoAll("db index for curXpath - %v is %v", curXpath, cdb) + d = dbs[cdb] + } + if curXpathInfo.virtualTbl != nil && *curXpathInfo.virtualTbl { + curUri += "/" + continue + } + // Check for subtree case and invoke subscribe xfmr + if len(curXpathInfo.xfmrFunc) > 0 { + xfmrLogInfoAll("Found subtree for uri - %v", curUri) + stParentTblExists := false + stParentTblExists, err = verifyParentTblSubtree(dbs, curUri, curXpathInfo.xfmrFunc, oper, dbData) + if err != nil { + parentTblExists = false + break + } + if !stParentTblExists { + log.Warningf("Parent Table does not exist for uri %v", uri) + err = tlerr.NotFound("Resource not found") + parentTblExists = false + break + } + } else { + + xfmrLogInfoAll("Check parent table for uri: %v", curUri) + // Get Table and Key only for yang list instances + xpathKeyExtRet, xerr := xpathKeyExtract(d, ygRoot, oper, curUri, uri, nil, subOpDataMap, txCache, nil) + if xerr != nil { + log.Warningf("Failed to get table and key for uri: %v err: %v", curUri, xerr) + err = xerr + parentTblExists = false + break + } + if (xpathKeyExtRet.isVirtualTbl) { + curUri += "/" + continue + } + + if len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) > 0 { + // Check for Table existence + xfmrLogInfoAll("DB Entry Check for uri: %v table: %v, key: %v", uri, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + existsInDbData := false + if oper == GET { + // GET case - attempt to find in dbData before doing a dbGet in dbTableExists() + existsInDbData = dbTableExistsInDbData(cdb, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, dbData) + } + // Read the table entry from DB + if !existsInDbData { + exists, derr := dbTableExists(d, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, oper) + if derr != nil { + return false, derr + } + if !exists { + parentTblExists = false + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, uri) + err = tlerr.NotFound("Resource not found") + break + } + } + } else { + // We always expect a valid table and key to be returned. Else we cannot validate parent check + parentTblExists = false + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, curUri) + err = tlerr.NotFound("Resource not found") + break + } + } + } + curUri += "/" + } + if !parentTblExists { + // For all operations Parent Table has to exist + return false, err + } + + if yangType == YANG_LIST && (oper == UPDATE || oper == CREATE || oper == DELETE || oper == GET) { + // For PATCH request the current table instance should exist for the operation to go through + // For POST since the target URI is the parent URI, it should exist. + // For DELETE we handle the table verification here to avoid any CVL error thrown for delete on non existent table + xfmrLogInfoAll("Check last parent table for uri: %v", uri) + xpath, _, xpathErr := XfmrRemoveXPATHPredicates(uri) + if xpathErr != nil { + log.Warningf("Xpath conversion didn't happen for Uri - %v, due to - %v", uri, xpathErr) + return false, xpathErr + } + xpathInfo, ok := xYangSpecMap[xpath] + if !ok { + err = fmt.Errorf("xYangSpecMap does not contain xpath - %v", xpath) + return false, err + } + virtualTbl := false + if xpathInfo.virtualTbl != nil { + virtualTbl = *xpathInfo.virtualTbl + } + if virtualTbl { + xfmrLogInfoAll("Virtual table at uri - %v", uri) + return true, nil + } + // Check for subtree case and invoke subscribe xfmr + if len(xpathInfo.xfmrFunc) > 0 { + if !((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) {//uri points to entire list + xfmrLogInfoAll("subtree , whole list case for uri - %v", uri) + return true, nil + } + xfmrLogInfoAll("Found subtree for uri - %v", uri) + parentTblExists, err = verifyParentTblSubtree(dbs, uri, xpathInfo.xfmrFunc, oper, dbData) + if err != nil { + return false, err + } + if !parentTblExists { + log.Warningf("Parent Table does not exist for uri %v", uri) + err = tlerr.NotFound("Resource not found") + return false, err + } + return true, nil + } + + xpathKeyExtRet, xerr := xpathKeyExtract(d, ygRoot, oper, uri, uri, nil, subOpDataMap, txCache, nil) + if xerr != nil { + log.Warningf("xpathKeyExtract failed err: %v, table %v, key %v", xerr, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + return false, xerr + } + if xpathKeyExtRet.isVirtualTbl { + return true, nil + } + if len(xpathKeyExtRet.tableName) > 0 && len(xpathKeyExtRet.dbKey) > 0 { + // Read the table entry from DB + exists := false + var derr error + if oper == GET { + // GET case - find in dbData instead of doing a dbGet in dbTableExists() + cdb = xpathInfo.dbIndex + xfmrLogInfoAll("db index for xpath - %v is %v", xpath, cdb) + exists = dbTableExistsInDbData(cdb, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, dbData) + if !exists { + exists, derr = dbTableExists(dbs[cdb], xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, oper) + if derr != nil { + return false, derr + } + if !exists { + parentTblExists = false + log.Warningf("Parent Tbl :%v, dbKey: %v does not exist for uri %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, uri) + err = tlerr.NotFound("Resource not found") + } + } + } else { + exists, derr = dbTableExists(d, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, oper) + } + if derr != nil { + log.Warningf("GetEntry failed for table: %v, key: %v err: %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, derr) + return false, derr + } + if !exists { + log.Warningf("GetEntry failed for table: %v, key: %v err: %v", xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey, derr) + err = tlerr.NotFound("Resource not found") + return false, err + } else { + return true, nil + } + } else if !((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) {//uri points to entire list + return true, nil + } else { + log.Warningf("Unable to get valid table and key err: %v, table %v, key %v", xerr, xpathKeyExtRet.tableName, xpathKeyExtRet.dbKey) + return false, xerr + } + } else if (yangType == YANG_CONTAINER && oper == DELETE && ((xpathInfo.keyName != nil && len(*xpathInfo.keyName) > 0) || len(xpathInfo.xfmrKey) > 0)) { + // If the delete is at container level and the container is mapped to a unique table, then check for table existence to avoid CVL throwing error + parentUri := "" + if len(parentUriList) > 0 { + parentUri = strings.Join(parentUriList, "/") + parentUri = "/" + parentUri + } + // Get table for parent xpath + parentTable, perr := dbTableFromUriGet(d, ygRoot, oper, parentUri, uri, nil, txCache, nil) + // Get table for current xpath + xpathKeyExtRet, cerr := xpathKeyExtract(d, ygRoot, oper, uri, uri, nil, subOpDataMap, txCache, nil) + curKey := xpathKeyExtRet.dbKey + curTable := xpathKeyExtRet.tableName + if len(curTable) > 0 { + if perr == nil && cerr == nil && (curTable != parentTable) && len(curKey) > 0 { + exists, derr := dbTableExists(d, curTable, curKey, oper) + if !exists { + return true, derr + } else { + return true, nil + } + } else { + return true, nil + } + } else { + return true, nil + } + } else { + // PUT at list is allowed to do a create if table does not exist else replace OR + // This is a container or leaf at the end of the URI. Parent check already done and hence all operations are allowed + return true, err + } +} + +/* Debug function to print the map data into file */ +func printDbData(resMap map[int]map[db.DBNum]map[string]map[string]db.Value, yangDefValMap map[string]map[string]db.Value, fileName string) { + fp, err := os.Create(fileName) + if err != nil { + return + } + defer fp.Close() + for oper, dbRes := range resMap { + fmt.Fprintf(fp, "-------------------------- REQ DATA -----------------------------\r\n") + fmt.Fprintf(fp, "Oper Type : %v\r\n", oper) + for d, dbMap := range dbRes { + fmt.Fprintf(fp, "DB num : %v\r\n", d) + for k, v := range dbMap { + fmt.Fprintf(fp, "table name : %v\r\n", k) + for ik, iv := range v { + fmt.Fprintf(fp, " key : %v\r\n", ik) + for k, d := range iv.Field { + fmt.Fprintf(fp, " %v :%v\r\n", k, d) + } + } + } + } + } + fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") + fmt.Fprintf(fp, "-------------------------- YANG DEFAULT DATA --------------------\r\n") + for k, v := range yangDefValMap { + fmt.Fprintf(fp, "table name : %v\r\n", k) + for ik, iv := range v { + fmt.Fprintf(fp, " key : %v\r\n", ik) + for k, d := range iv.Field { + fmt.Fprintf(fp, " %v :%v\r\n", k, d) + } + } + } + fmt.Fprintf(fp, "-----------------------------------------------------------------\r\n") } diff --git a/translib/transformer/xlate_utils.go b/translib/transformer/xlate_utils.go index 216041609..650ff9d7f 100644 --- a/translib/transformer/xlate_utils.go +++ b/translib/transformer/xlate_utils.go @@ -20,16 +20,30 @@ package transformer import ( "fmt" + "errors" "strings" - "reflect" - "regexp" + "regexp" + "runtime" "github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/Azure/sonic-mgmt-common/translib/tlerr" "github.com/openconfig/goyang/pkg/yang" "github.com/openconfig/gnmi/proto/gnmi" "github.com/openconfig/ygot/ygot" + "github.com/openconfig/ygot/ytypes" + "github.com/Azure/sonic-mgmt-common/translib/ocbinds" log "github.com/golang/glog" + "sync" ) +func initRegex() { + rgpKeyExtract = regexp.MustCompile(`\[([^\[\]]*)\]`) + rgpIpv6 = regexp.MustCompile(`(([^:]+:){6}(([^:]+:[^:]+)|(.*\..*)))|((([^:]+:)*[^:]+)?::(([^:]+:)*[^:]+)?)(%.+)?`) + rgpMac = regexp.MustCompile(`([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$`) + rgpIsMac = regexp.MustCompile(`^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$`) + rgpSncKeyExtract = regexp.MustCompile(`\[([^\[\]]*)\]`) + +} + /* Create db key from data xpath(request) */ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string) string { _, ok := xYangSpecMap[xpath] @@ -39,18 +53,25 @@ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string delim := dbKeySep if len(xYangSpecMap[xpath].delim) > 0 { delim = xYangSpecMap[xpath].delim - log.Infof("key concatenater(\"%v\") found for xpath %v ", delim, xpath) + xfmrLogInfoAll("key concatenater(\"%v\") found for xpath %v ", delim, xpath) } if len(keyPrefix) > 0 { keyPrefix += delim } keyVal := "" for i, k := range (strings.Split(yangEntry.Key, " ")) { if i > 0 { keyVal = keyVal + delim } - val := fmt.Sprint(data.(map[string]interface{})[k]) - if strings.Contains(val, ":") { - val = strings.Split(val, ":")[1] + fieldXpath := xpath + "/" + k + fVal, err := unmarshalJsonToDbData(yangEntry.Dir[k], fieldXpath, k, data.(map[string]interface{})[k]) + if err != nil { + log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) + } + + if ((strings.Contains(fVal, ":")) && + (strings.HasPrefix(fVal, OC_MDL_PFX) || strings.HasPrefix(fVal, IETF_MDL_PFX) || strings.HasPrefix(fVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + fVal = strings.SplitN(fVal, ":", 2)[1] } - keyVal += val + keyVal += fVal } keyPrefix += string(keyVal) } @@ -60,6 +81,7 @@ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string /* Copy redis-db source to destn map */ func mapCopy(destnMap map[string]map[string]db.Value, srcMap map[string]map[string]db.Value) { + mapCopyMutex.Lock() for table, tableData := range srcMap { _, ok := destnMap[table] if !ok { @@ -75,6 +97,44 @@ func mapCopy(destnMap map[string]map[string]db.Value, srcMap map[string]map[stri } } } + mapCopyMutex.Unlock() +} +var mapMergeMutex = &sync.Mutex{} +/* Merge redis-db source to destn map */ +func mapMerge(destnMap map[string]map[string]db.Value, srcMap map[string]map[string]db.Value, oper int) { + mapMergeMutex.Lock() + for table, tableData := range srcMap { + _, ok := destnMap[table] + if !ok { + destnMap[table] = make(map[string]db.Value) + } + for rule, ruleData := range tableData { + _, ok = destnMap[table][rule] + if !ok { + destnMap[table][rule] = db.Value{Field: make(map[string]string)} + } else { + if (oper == DELETE) { + if ((len(destnMap[table][rule].Field) == 0) && (len(ruleData.Field) > 0)) { + continue; + } + if ((len(destnMap[table][rule].Field) > 0) && (len(ruleData.Field) == 0)) { + destnMap[table][rule] = db.Value{Field: make(map[string]string)}; + } + } + } + for field, value := range ruleData.Field { + dval := destnMap[table][rule] + if dval.IsPopulated() && dval.Has(field) && strings.HasSuffix(field, "@") { + attrList := dval.GetList(field) + attrList = append(attrList, value) + dval.SetList(field, attrList) + } else { + destnMap[table][rule].Field[field] = value + } + } + } + } + mapMergeMutex.Unlock() } func parentXpathGet(xpath string) string { @@ -86,11 +146,17 @@ func parentXpathGet(xpath string) string { return path } -func isYangResType(ytype string) bool { - if ytype == "choose" || ytype == "case" { - return true - } - return false +func parentUriGet(uri string) string { + parentUri := "" + if len(uri) > 0 { + uriList := splitUri(uri) + if len(uriList) > 2 { + parentUriList := uriList[:len(uriList)-1] + parentUri = strings.Join(parentUriList, "/") + parentUri = "/" + parentUri + } + } + return parentUri } func yangTypeGet(entry *yang.Entry) string { @@ -100,7 +166,7 @@ func yangTypeGet(entry *yang.Entry) string { return "" } -func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep string) (map[string]interface{}, string, error) { +func dbKeyToYangDataConvert(uri string, requestUri string, xpath string, tableName string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, dbKey string, dbKeySep string, txCache interface{}) (map[string]interface{}, string, error) { var err error if len(uri) == 0 && len(xpath) == 0 && len(dbKey) == 0 { err = fmt.Errorf("Insufficient input") @@ -109,7 +175,7 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep str if _, ok := xYangSpecMap[xpath]; ok { if xYangSpecMap[xpath].yangEntry == nil { - err = fmt.Errorf("Yang Entry not available for xpath ", xpath) + log.Warningf("Yang Entry not available for xpath %v", xpath) return nil, "", nil } } @@ -125,21 +191,23 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep str /* if uri contins key, use it else use xpath */ if strings.Contains(uri, "[") { - uriXpath, _ := XfmrRemoveXPATHPredicates(uri) - if (uriXpath == xpath && (strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/"))) { - uriWithKeyCreate = false - } + if strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/") { + uriXpath, _, _ := XfmrRemoveXPATHPredicates(uri) + if uriXpath == xpath { + uriWithKeyCreate = false + } + } uriWithKey = fmt.Sprintf("%v", uri) } if len(xYangSpecMap[xpath].xfmrKey) > 0 { var dbs [db.MaxDB]*db.DB - inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, GET, dbKey, nil, nil) - ret, err := XlateFuncCall(dbToYangXfmrFunc(xYangSpecMap[xpath].xfmrKey), inParams) + inParams := formXfmrInputRequest(nil, dbs, db.MaxDB, nil, uri, requestUri, GET, dbKey, dbDataMap, nil, nil, txCache) + inParams.table = tableName + rmap, err := keyXfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrKey) if err != nil { return nil, "", err } - rmap := ret[0].Interface().(map[string]interface{}) if uriWithKeyCreate { for k, v := range rmap { uriWithKey += fmt.Sprintf("[%v=%v]", k, v) @@ -155,12 +223,28 @@ func dbKeyToYangDataConvert(uri string, xpath string, dbKey string, dbKeySep str rmap := make(map[string]interface{}) if len(keyNameList) > 1 { - log.Infof("No key transformer found for multi element yang key mapping to a single redis key string.") - return rmap, uriWithKey, nil + log.Warningf("No key transformer found for multi element yang key mapping to a single redis key string, for uri %v", uri) + errStr := fmt.Sprintf("Error processing key for list %v", uri) + err = fmt.Errorf("%v", errStr) + return rmap, uriWithKey, err + } + keyXpath := xpath + "/" + keyNameList[0] + xyangSpecInfo, ok := xYangSpecMap[keyXpath] + if !ok || xyangSpecInfo == nil { + errStr := fmt.Sprintf("Failed to find key xpath %v in xYangSpecMap or is nil, needed to fetch the yangEntry data-type", keyXpath) + err = fmt.Errorf("%v", errStr) + return rmap, uriWithKey, err + } + yngTerminalNdDtType := xyangSpecInfo.yangEntry.Type.Kind + resVal, _, err := DbToYangType(yngTerminalNdDtType, keyXpath, keyDataList[0]) + if err != nil { + err = fmt.Errorf("Failure in converting Db value type to yang type for field %v", keyXpath) + return rmap, uriWithKey, err + } else { + rmap[keyNameList[0]] = resVal } - rmap[keyNameList[0]] = keyDataList[0] if uriWithKeyCreate { - uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[0], keyDataList[0]) + uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[0], resVal) } return rmap, uriWithKey, nil @@ -188,13 +272,13 @@ func getYangPathFromUri(uri string) (string, error) { path, err = ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) if err != nil { - log.Errorf("Error in uri to path conversion: %v", err) + log.Warningf("Error in uri to path conversion: %v", err) return "", err } yangPath, yperr := ygot.PathToSchemaPath(path) if yperr != nil { - log.Errorf("Error in Gnmi path to Yang path conversion: %v", yperr) + log.Warningf("Error in Gnmi path to Yang path conversion: %v", yperr) return "", yperr } @@ -203,31 +287,113 @@ func getYangPathFromUri(uri string) (string, error) { func yangKeyFromEntryGet(entry *yang.Entry) []string { var keyList []string - for _, key := range strings.Split(entry.Key, " ") { - keyList = append(keyList, key) - } + keyList = append(keyList, strings.Split(entry.Key, " ")...) return keyList } func isSonicYang(path string) bool { - if strings.HasPrefix(path, "/sonic") { - return true - } - return false + return strings.HasPrefix(path, "/sonic") } -func sonicKeyDataAdd(dbIndex db.DBNum, keyNameList []string, keyStr string, resultMap map[string]interface{}) { - var dbOpts db.Options - dbOpts = getDBOptions(dbIndex) - keySeparator := dbOpts.KeySeparator - keyValList := strings.Split(keyStr, keySeparator) +func hasIpv6AddString(val string) bool { + return rgpIpv6.MatchString(val) +} - if len(keyNameList) != len(keyValList) { - return - } +func hasMacAddString(val string) bool { + return rgpMac.MatchString(val) +} + +func isMacAddString(val string) bool { + return rgpIsMac.MatchString(val) +} + +func getYangTerminalNodeTypeName(xpathPrefix string, keyName string) string { + keyXpath := xpathPrefix + "/" + keyName + xfmrLogInfoAll("getYangTerminalNodeTypeName keyXpath: %v ", keyXpath) + dbInfo, ok := xDbSpecMap[keyXpath] + if ok { + yngTerminalNdTyName := dbInfo.dbEntry.Type.Name + xfmrLogInfoAll("yngTerminalNdTyName: %v", yngTerminalNdTyName) + return yngTerminalNdTyName + } + return "" +} + + +func sonicKeyDataAdd(dbIndex db.DBNum, keyNameList []string, xpathPrefix string, keyStr string, resultMap map[string]interface{}) { + var dbOpts db.Options + var keyValList []string + xfmrLogInfoAll("sonicKeyDataAdd keyNameList:%v, keyStr:%v", keyNameList, keyStr) + + dbOpts = getDBOptions(dbIndex) + keySeparator := dbOpts.KeySeparator + /* num of key separators will be less than number of keys */ + if len(keyNameList) == 1 && keySeparator == ":" { + yngTerminalNdTyName := getYangTerminalNodeTypeName(xpathPrefix, keyNameList[0]) + if yngTerminalNdTyName == "mac-address" && isMacAddString(keyStr) { + keyValList = strings.SplitN(keyStr, keySeparator, len(keyNameList)) + } else if (yngTerminalNdTyName == "ip-address" || yngTerminalNdTyName == "ip-prefix" || yngTerminalNdTyName == "ipv6-prefix" || yngTerminalNdTyName == "ipv6-address") && hasIpv6AddString(keyStr) { + keyValList = strings.SplitN(keyStr, keySeparator, len(keyNameList)) + } else { + keyValList = strings.SplitN(keyStr, keySeparator, -1) + xfmrLogInfoAll("Single key non ipv6/mac address for : separator") + } + } else if (strings.Count(keyStr, keySeparator) == len(keyNameList)-1) { + /* number of keys will match number of key values */ + keyValList = strings.SplitN(keyStr, keySeparator, len(keyNameList)) + } else { + /* number of dbKey values more than number of keys */ + if keySeparator == ":" && (hasIpv6AddString(keyStr)|| hasMacAddString(keyStr)) { + xfmrLogInfoAll("Key Str has ipv6/mac address with : separator") + if len(keyNameList) == 2 { + valList := strings.SplitN(keyStr, keySeparator, -1) + /* IPV6 address is first entry */ + yngTerminalNdTyName := getYangTerminalNodeTypeName(xpathPrefix, keyNameList[0]) + if yngTerminalNdTyName == "ip-address" || yngTerminalNdTyName == "ip-prefix" || yngTerminalNdTyName == "ipv6-prefix" || yngTerminalNdTyName == "ipv6-address" || yngTerminalNdTyName == "mac-address"{ + keyValList = append(keyValList, strings.Join(valList[:len(valList)-2], keySeparator)) + keyValList = append(keyValList, valList[len(valList)-1]) + } else { + yngTerminalNdTyName := getYangTerminalNodeTypeName(xpathPrefix, keyNameList[1]) + if yngTerminalNdTyName == "ip-address" || yngTerminalNdTyName == "ip-prefix" || yngTerminalNdTyName == "ipv6-prefix" || yngTerminalNdTyName == "ipv6-address" || yngTerminalNdTyName == "mac-address" { + keyValList = append(keyValList, valList[0]) + keyValList = append(keyValList, strings.Join(valList[1:], keySeparator)) + } else { + xfmrLogInfoAll("No ipv6 or mac address found in value. Cannot split value ") + } + } + xfmrLogInfoAll("KeyValList has %v", keyValList) + } else { + xfmrLogInfoAll("Number of keys : %v", len(keyNameList)) + } + } else { + keyValList = strings.SplitN(keyStr, keySeparator, -1) + xfmrLogInfoAll("Split all keys KeyValList has %v", keyValList) + } + } + xfmrLogInfoAll("yang keys list - %v, xpathprefix - %v, DB-key string - %v, DB-key list after db key separator split - %v, dbIndex - %v", keyNameList, xpathPrefix, keyStr, keyValList, dbIndex) + + if len(keyNameList) != len(keyValList) { + return + } for i, keyName := range keyNameList { - resultMap[keyName] = keyValList[i] + keyXpath := xpathPrefix + "/" + keyName + dbInfo, ok := xDbSpecMap[keyXpath] + var resVal interface{} + resVal = keyValList[i] + if !ok || dbInfo == nil { + log.Warningf("xDbSpecMap entry not found or is nil for xpath %v, hence data-type conversion cannot happen", keyXpath) + } else { + yngTerminalNdDtType := dbInfo.dbEntry.Type.Kind + var err error + resVal, _, err = DbToYangType(yngTerminalNdDtType, keyXpath, keyValList[i]) + if err != nil { + log.Warningf("Data-type conversion unsuccessfull for xpath %v", keyXpath) + resVal = keyValList[i] + } + } + + resultMap[keyName] = resVal } } @@ -241,13 +407,35 @@ func uriWithKeyCreate (uri string, xpathTmplt string, data interface{}) (string, yangEntry := xYangSpecMap[xpathTmplt].yangEntry if yangEntry != nil { for _, k := range (strings.Split(yangEntry.Key, " ")) { - uri += fmt.Sprintf("[%v=%v]", k, data.(map[string]interface{})[k]) + keyXpath := xpathTmplt + "/" + k + if _, keyXpathEntryOk := xYangSpecMap[keyXpath]; !keyXpathEntryOk { + log.Warningf("No entry found in xYangSpec map for xapth %v", keyXpath) + err = fmt.Errorf("No entry found in xYangSpec map for xapth %v", keyXpath) + break + } + keyYangEntry := xYangSpecMap[keyXpath].yangEntry + if keyYangEntry == nil { + log.Warningf("Yang Entry not available for xpath %v", keyXpath) + err = fmt.Errorf("Yang Entry not available for xpath %v", keyXpath) + break + } + keyVal, keyValErr := unmarshalJsonToDbData(keyYangEntry, keyXpath, k, data.(map[string]interface{})[k]) + if keyValErr != nil { + log.Warningf("unmarshalJsonToDbData() didn't unmarshal for key %v with xpath %v", k, keyXpath) + err = keyValErr + break + } + if ((strings.Contains(keyVal, ":")) && (strings.HasPrefix(keyVal, OC_MDL_PFX) || strings.HasPrefix(keyVal, IETF_MDL_PFX) || strings.HasPrefix(keyVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + keyVal = strings.SplitN(keyVal, ":", 2)[1] + } + uri += fmt.Sprintf("[%v=%v]", k, keyVal) } } else { - err = fmt.Errorf("Yang Entry not available for xpath ", xpathTmplt) + err = fmt.Errorf("Yang Entry not available for xpath %v", xpathTmplt) } } else { - err = fmt.Errorf("No entry in xYangSpecMap for xpath ", xpathTmplt) + err = fmt.Errorf("No entry in xYangSpecMap for xpath %v", xpathTmplt) } return uri, err } @@ -260,16 +448,6 @@ func xpathRootNameGet(path string) string { return "" } -func getDbNum(xpath string ) db.DBNum { - _, ok := xYangSpecMap[xpath] - if ok { - xpathInfo := xYangSpecMap[xpath] - return xpathInfo.dbIndex - } - // Default is ConfigDB - return db.ConfigDB -} - func dbToYangXfmrFunc(funcName string) string { return ("DbToYang_" + funcName) } @@ -278,96 +456,43 @@ func uriModuleNameGet(uri string) (string, error) { var err error result := "" if len(uri) == 0 { - log.Error("Empty uri string supplied") + log.Warning("Empty uri string supplied") err = fmt.Errorf("Empty uri string supplied") return result, err } urislice := strings.Split(uri, ":") if len(urislice) == 1 { - log.Errorf("uri string %s does not have module name", uri) - err = fmt.Errorf("uri string does not have module name: ", uri) + log.Warningf("uri string %s does not have module name", uri) + err = fmt.Errorf("uri string does not have module name: %v", uri) return result, err } moduleNm := strings.Split(urislice[0], "/") result = moduleNm[1] if len(strings.Trim(result, " ")) == 0 { - log.Error("Empty module name") + log.Warning("Empty module name") err = fmt.Errorf("No module name found in uri %s", uri) } - log.Info("module name = ", result) + xfmrLogInfo("module name = %v", result) return result, err } -func recMap(rMap interface{}, name []string, id int, max int) { - if id == max || id < 0 { - return - } - val := name[id] - if reflect.ValueOf(rMap).Kind() == reflect.Map { - data := reflect.ValueOf(rMap) - dMap := data.Interface().(map[string]interface{}) - _, ok := dMap[val] - if ok { - recMap(dMap[val], name, id+1, max) - } else { - dMap[val] = make(map[string]interface{}) - recMap(dMap[val], name, id+1, max) - } - } - return -} - -func mapCreate(xpath string) map[string]interface{} { - retMap := make(map[string]interface{}) - if len(xpath) > 0 { - attrList := strings.Split(xpath, "/") - alLen := len(attrList) - recMap(retMap, attrList, 1, alLen) - } - return retMap -} - -func mapInstGet(name []string, id int, max int, inMap interface{}) map[string]interface{} { - if inMap == nil { - return nil - } - result := reflect.ValueOf(inMap).Interface().(map[string]interface{}) - if id == max { - return result - } - val := name[id] - if reflect.ValueOf(inMap).Kind() == reflect.Map { - data := reflect.ValueOf(inMap) - dMap := data.Interface().(map[string]interface{}) - _, ok := dMap[val] - if ok { - result = mapInstGet(name, id+1, max, dMap[val]) - } else { - return result - } - } - return result -} - -func mapGet(xpath string, inMap map[string]interface{}) map[string]interface{} { - attrList := strings.Split(xpath, "/") - alLen := len(attrList) - recMap(inMap, attrList, 1, alLen) - retMap := mapInstGet(attrList, 1, alLen, inMap) - return retMap -} - -func formXfmrInputRequest(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, oper int, key string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, param interface{}) XfmrParams { +func formXfmrInputRequest(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, oper int, key string, dbDataMap *RedisDbMap, subOpDataMap map[int]*RedisDbMap, param interface{}, txCache interface{}) XfmrParams { var inParams XfmrParams inParams.d = d inParams.dbs = dbs inParams.curDb = cdb inParams.ygRoot = ygRoot inParams.uri = uri + inParams.requestUri = requestUri inParams.oper = oper inParams.key = key inParams.dbDataMap = dbDataMap + inParams.subOpDataMap = subOpDataMap inParams.param = param // generic param + inParams.txCache = txCache.(*sync.Map) + inParams.skipOrdTblChk = new(bool) + inParams.isVirtualTbl = new(bool) + inParams.pCascadeDelTbl = new([]string) return inParams } @@ -407,10 +532,8 @@ func getDBOptions(dbNo db.DBNum) db.Options { switch dbNo { case db.ApplDB, db.CountersDB: opt = getDBOptionsWithSeparator(dbNo, "", ":", ":") - break - case db.FlexCounterDB, db.AsicDB, db.LogLevelDB, db.ConfigDB, db.StateDB: + case db.FlexCounterDB, db.AsicDB, db.LogLevelDB, db.ConfigDB, db.StateDB, db.ErrorDB, db.UserDB: opt = getDBOptionsWithSeparator(dbNo, "", "|", "|") - break } return opt @@ -425,7 +548,29 @@ func getDBOptionsWithSeparator(dbNo db.DBNum, initIndicator string, tableSeparat }) } +func getXpathFromYangEntry(entry *yang.Entry) string { + xpath := "" + if entry != nil { + xpath = entry.Name + entry = entry.Parent + for { + if entry.Parent != nil { + xpath = entry.Name + "/" + xpath + entry = entry.Parent + } else { + // This is the module entry case + xpath = "/" + entry.Name + ":" + xpath + break + } + } + } + return xpath +} + func stripAugmentedModuleNames(xpath string) string { + if !strings.HasPrefix(xpath, "/") { + xpath = "/" + xpath + } pathList := strings.Split(xpath, "/") pathList = pathList[1:] for i, pvar := range pathList { @@ -438,49 +583,74 @@ func stripAugmentedModuleNames(xpath string) string { return path } -func XfmrRemoveXPATHPredicates(xpath string) (string, error) { - pathList := strings.Split(xpath, "/") - pathList = pathList[1:] - for i, pvar := range pathList { - if strings.Contains(pvar, "[") && strings.Contains(pvar, "]") { - si, ei := strings.Index(pvar, "["), strings.Index(pvar, "]") - // substring contains [] entries - if (si < ei) { - pvar = strings.Split(pvar, "[")[0] - pathList[i] = pvar +func XfmrRemoveXPATHPredicates(uri string) (string, []string, error) { + var uriList []string + var pathList []string + uriList = SplitPath(uri) - } else { - // This substring contained a ] before a [. - return "", fmt.Errorf("Incorrect ordering of [] within substring %s of %s, [ pos: %d, ] pos: %d", pvar, xpath, si, ei) - } - } else if strings.Contains(pvar, "[") || strings.Contains(pvar, "]") { - // This substring contained a mismatched pair of []s. - return "", fmt.Errorf("Mismatched brackets within substring %s of %s", pvar, xpath) - } - if (i > 0) && strings.Contains(pvar, ":") { - pvar = strings.Split(pvar,":")[1] - pathList[i] = pvar - } - } - path := "/" + strings.Join(pathList, "/") - return path,nil + // Strip keys for xpath creation + for _, path := range uriList { + si := strings.Index(path, "[") + if si != -1 { + pathList = append(pathList, path[:si]) + } else { + pathList = append(pathList, path) + } + } + + inPath := strings.Join(pathList, "/") + if !strings.HasPrefix(uri, "..") { + inPath = "/" + inPath + } + + xpath := stripAugmentedModuleNames(inPath) + return xpath, uriList, nil } - /* Extract key vars, create db key and xpath */ - func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string) (string, string, string) { +func replacePrefixWithModuleName(xpath string) (string) { + //Input xpath is after removing the xpath Predicates + var moduleNm string + if _, ok := xYangSpecMap[xpath]; ok { + moduleNm = xYangSpecMap[xpath].dbEntry.Prefix.Parent.NName() + pathList := strings.Split(xpath, ":") + if len(moduleNm) > 0 && len(pathList) == 2 { + xpath = "/" + moduleNm + ":" + pathList[1] + } + } + return xpath +} + + +/* Extract key vars, create db key and xpath */ +func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper int, path string, requestUri string, dbDataMap *map[db.DBNum]map[string]map[string]db.Value, subOpDataMap map[int]*RedisDbMap, txCache interface{}, xfmrTblKeyCache map[string]tblKeyCache) (xpathTblKeyExtractRet, error) { + xfmrLogInfoAll("In uri(%v), reqUri(%v), oper(%v)", path, requestUri, oper) + var retData xpathTblKeyExtractRet keyStr := "" - tableName := "" - pfxPath := "" - rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) curPathWithKey := "" cdb := db.ConfigDB var dbs [db.MaxDB]*db.DB + var err error + var isUriForListInstance bool + var pathList []string - pfxPath, _ = XfmrRemoveXPATHPredicates(path) - xpathInfo, ok := xYangSpecMap[pfxPath] + retData.xpath = "" + retData.tableName = "" + retData.dbKey = "" + retData.isVirtualTbl = false + + isUriForListInstance = false + retData.xpath, pathList, _ = XfmrRemoveXPATHPredicates(path) + xpathInfo, ok := xYangSpecMap[retData.xpath] if !ok { - log.Errorf("No entry found in xYangSpecMap for xpath %v.", pfxPath) - return pfxPath, keyStr, tableName + log.Warningf("No entry found in xYangSpecMap for xpath %v.", retData.xpath) + return retData, err + } + // for SUBSCRIBE reuestUri = path + requestUriYangType := yangTypeGet(xpathInfo.yangEntry) + if requestUriYangType == YANG_LIST { + if strings.HasSuffix(path, "]") { //uri is for list instance + isUriForListInstance = true + } } cdb = xpathInfo.dbIndex dbOpts := getDBOptions(cdb) @@ -488,26 +658,67 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { if len(xpathInfo.delim) > 0 { keySeparator = xpathInfo.delim } - - for _, k := range strings.Split(path, "/") { - curPathWithKey += k - yangXpath, _ := XfmrRemoveXPATHPredicates(curPathWithKey) - _, ok := xYangSpecMap[yangXpath] + xpathList := strings.Split(retData.xpath, "/") + xpathList = xpathList[1:] + yangXpath := "" + xfmrLogInfoAll("path elements are : %v", pathList) + for i, k := range pathList { + curPathWithKey += "/" + k + callKeyXfmr := true + yangXpath += "/" + xpathList[i] + xpathInfo, ok := xYangSpecMap[yangXpath] if ok { + yangType := yangTypeGet(xpathInfo.yangEntry) + /* when deleting a specific element from leaf-list query uri is of the form + /prefix-path/leafList-field-name[leafList-field-name=value]. + Here the syntax is like a list-key instance enclosed in square + brackets .So avoid list key instance like processing for such a case + */ + if yangType == YANG_LEAF_LIST { + break + } if strings.Contains(k, "[") { if len(keyStr) > 0 { keyStr += keySeparator } if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + if xfmrTblKeyCache != nil { + if tkCache, _ok := xfmrTblKeyCache[curPathWithKey]; _ok { + if len(tkCache.dbKey) != 0 { + keyStr = tkCache.dbKey + callKeyXfmr = false + } + } + } + if callKeyXfmr { xfmrFuncName := yangToDbXfmrFunc(xYangSpecMap[yangXpath].xfmrKey) - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) - ret, err := XlateFuncCall(xfmrFuncName, inParams) - if err != nil { - return "", "", "" + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + ret, err := XlateFuncCall(xfmrFuncName, inParams) + if err != nil { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + if ret != nil { + keyStr = ret[0].Interface().(string) + } + } else { + ret, err := keyXfmrHandler(inParams, xYangSpecMap[yangXpath].xfmrKey) + if err != nil { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + keyStr = ret } - if ret != nil { - keyStr = ret[0].Interface().(string) + if xfmrTblKeyCache != nil { + if _, _ok := xfmrTblKeyCache[curPathWithKey]; !_ok { + xfmrTblKeyCache[curPathWithKey] = tblKeyCache{} + } + tkCache := xfmrTblKeyCache[curPathWithKey] + tkCache.dbKey = keyStr + xfmrTblKeyCache[curPathWithKey] = tkCache } + } } else if xYangSpecMap[yangXpath].keyName != nil { keyStr += *xYangSpecMap[yangXpath].keyName } else { @@ -515,50 +726,124 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { There should be key-transformer, if not then the yang key leaves will be concatenated with respective default DB type key-delimiter */ - for idx, kname := range rgp.FindAllString(k, -1) { + for idx, kname := range rgpKeyExtract.FindAllString(k, -1) { if idx > 0 { keyStr += keySeparator } keyl := strings.TrimRight(strings.TrimLeft(kname, "["), "]") - if strings.Contains(keyl, ":") { - keyl = strings.Split(keyl, ":")[1] - } - keyStr += strings.Split(keyl, "=")[1] + keys := strings.Split(keyl, "=") + keyStr += keys[1] } } - } else if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + } else if len(xYangSpecMap[yangXpath].xfmrKey) > 0 { + if xfmrTblKeyCache != nil { + if tkCache, _ok := xfmrTblKeyCache[curPathWithKey]; _ok { + if len(tkCache.dbKey) != 0 { + keyStr = tkCache.dbKey + callKeyXfmr = false + } + } + } + if callKeyXfmr { xfmrFuncName := yangToDbXfmrFunc(xYangSpecMap[yangXpath].xfmrKey) - inParams := formXfmrInputRequest(d, dbs, db.MaxDB, ygRoot, curPathWithKey, oper, "", nil, nil) - ret, err := XlateFuncCall(xfmrFuncName, inParams) - if err != nil { - return "", "", "" + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + ret, err := XlateFuncCall(xfmrFuncName, inParams) + if err != nil { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + if ret != nil { + keyStr = ret[0].Interface().(string) + } + } else { + ret, err := keyXfmrHandler(inParams, xYangSpecMap[yangXpath].xfmrKey) + if ((yangType != YANG_LIST) && (err != nil)) { + retData.dbKey,retData.tableName,retData.xpath = "","","" + return retData, err + } + keyStr = ret } - if ret != nil { - keyStr = ret[0].Interface().(string) + if xfmrTblKeyCache != nil { + if _, _ok := xfmrTblKeyCache[curPathWithKey]; !_ok { + xfmrTblKeyCache[curPathWithKey] = tblKeyCache{} + } + tkCache := xfmrTblKeyCache[curPathWithKey] + tkCache.dbKey = keyStr + xfmrTblKeyCache[curPathWithKey] = tkCache } + } } else if xYangSpecMap[yangXpath].keyName != nil { keyStr += *xYangSpecMap[yangXpath].keyName } } - curPathWithKey += "/" } curPathWithKey = strings.TrimSuffix(curPathWithKey, "/") + if !strings.HasPrefix(curPathWithKey, "/") { + curPathWithKey = "/" + curPathWithKey + } + retData.dbKey = keyStr tblPtr := xpathInfo.tableName - if tblPtr != nil { + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { + retData.tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + inParams.dbDataMap = dbDataMap + } + retData.tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, xfmrTblKeyCache) + if inParams.isVirtualTbl != nil { + retData.isVirtualTbl = *(inParams.isVirtualTbl) + } + if err != nil && oper != GET { + return retData, err + } + } + if ((oper == SUBSCRIBE) && (strings.TrimSpace(keyStr) == "") && (requestUriYangType == YANG_LIST) && (!isUriForListInstance)) { + keyStr="*" + } + retData.dbKey = keyStr + xfmrLogInfoAll("Return uri(%v), xpath(%v), key(%v), tableName(%v), isVirtualTbl:%v", path, retData.xpath, keyStr, retData.tableName, retData.isVirtualTbl) + return retData, err +} + +func dbTableFromUriGet(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, subOpDataMap map[int]*RedisDbMap, txCache interface{}, xfmrTblKeyCache map[string]tblKeyCache) (string, error) { + tableName := "" + var err error + cdb := db.ConfigDB + var dbs [db.MaxDB]*db.DB + + xPath, _, _ := XfmrRemoveXPATHPredicates(uri) + xpathInfo, ok := xYangSpecMap[xPath] + if !ok { + log.Warningf("No entry found in xYangSpecMap for xpath %v.", xPath) + return tableName, err + } + + tblPtr := xpathInfo.tableName + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { tableName = *tblPtr } else if xpathInfo.xfmrTbl != nil { - inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, oper, "", nil, nil) - tableName, _ = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams) + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, uri, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, xfmrTblKeyCache) } - return pfxPath, keyStr, tableName - } + return tableName, err +} - func sonicXpathKeyExtract(path string) (string, string, string) { - xpath, keyStr, tableName := "", "", "" +func sonicXpathKeyExtract(path string) (string, string, string) { + xfmrLogInfoAll("In uri(%v)", path) + xpath, keyStr, tableName, fldNm := "", "", "", "" var err error - xpath, err = XfmrRemoveXPATHPredicates(path) + lpath := path + xpath, _, err = XfmrRemoveXPATHPredicates(path) if err != nil { return xpath, keyStr, tableName } - rgp := regexp.MustCompile(`\[([^\[\]]*)\]`) + if xpath != "" { + fldPth := strings.Split(xpath, "/") + if len(fldPth) > SONIC_FIELD_INDEX { + fldNm = fldPth[SONIC_FIELD_INDEX] + xfmrLogInfoAll("Field Name : %v", fldNm) + } + } pathsubStr := strings.Split(path , "/") if len(pathsubStr) > SONIC_TABLE_INDEX { if strings.Contains(pathsubStr[2], "[") { @@ -569,7 +854,7 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { dbInfo, ok := xDbSpecMap[tableName] cdb := db.ConfigDB if !ok { - log.Infof("No entry in xDbSpecMap for xpath %v in order to fetch DB index.", tableName) + xfmrLogInfoAll("No entry in xDbSpecMap for xpath %v in order to fetch DB index", tableName) return xpath, keyStr, tableName } cdb = dbInfo.dbIndex @@ -577,7 +862,16 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { if dbInfo.keyName != nil { keyStr = *dbInfo.keyName } else { - for i, kname := range rgp.FindAllString(path, -1) { + /* chomp off the field portion to avoid processing specific item delete in leaf-list + eg. /sonic-acl:sonic-acl/ACL_TABLE/ACL_TABLE_LIST[aclname=MyACL2_ACL_IPV4]/ports[ports=Ethernet12] + */ + if fldNm != "" { + pathLst := splitUri(path) + xfmrLogInfoAll("pathList after uri split %v", pathLst) + lpath = "/" + strings.Join(pathLst[:SONIC_FIELD_INDEX-1], "/") + xfmrLogInfoAll("path after removing the field portion %v", lpath) + } + for i, kname := range rgpSncKeyExtract.FindAllString(lpath, -1) { if i > 0 { keyStr += dbOpts.KeySeparator } @@ -586,6 +880,444 @@ func XfmrRemoveXPATHPredicates(xpath string) (string, error) { } } } + xfmrLogInfoAll("Return uri(%v), xpath(%v), key(%v), tableName(%v)", path, xpath, keyStr, tableName) return xpath, keyStr, tableName } +func getYangMdlToSonicMdlList(moduleNm string) []string { + var sncMdlList []string + if len(xDbSpecTblSeqnMap) == 0 { + xfmrLogInfo("xDbSpecTblSeqnMap is empty.") + return sncMdlList + } + if strings.HasPrefix(moduleNm, SONIC_MDL_PFX) { + sncMdlList = append(sncMdlList, moduleNm) + } else { + //can be optimized if there is a way to know sonic modules, a given OC-Yang spans over + for sncMdl := range(xDbSpecTblSeqnMap) { + sncMdlList = append(sncMdlList, sncMdl) + } + } + return sncMdlList +} + +func yangFloatIntToGoType(t yang.TypeKind, v float64) (interface{}, error) { + switch t { + case yang.Yint8: + return int8(v), nil + case yang.Yint16: + return int16(v), nil + case yang.Yint32: + return int32(v), nil + case yang.Yuint8: + return uint8(v), nil + case yang.Yuint16: + return uint16(v), nil + case yang.Yuint32: + return uint32(v), nil + } + return nil, fmt.Errorf("unexpected YANG type %v", t) +} + +func unmarshalJsonToDbData(schema *yang.Entry, fieldXpath string, fieldName string, value interface{}) (string, error) { + var data string + + switch v := value.(type) { + case string: + return fmt.Sprintf("%v", v), nil + } + + ykind := schema.Type.Kind + if ykind == yang.Yleafref { + ykind = getLeafrefRefdYangType(ykind, fieldXpath) + } + + switch ykind { + case yang.Ystring, yang.Ydecimal64, yang.Yint64, yang.Yuint64, + yang.Yenum, yang.Ybool, yang.Ybinary, yang.Yidentityref, yang.Yunion: + data = fmt.Sprintf("%v", value) + + case yang.Yint8, yang.Yint16, yang.Yint32, + yang.Yuint8, yang.Yuint16, yang.Yuint32: + pv, err := yangFloatIntToGoType(ykind, value.(float64)) + if err != nil { + errStr := fmt.Sprintf("error parsing %v for schema %s: %v", value, schema.Name, err) + return "", tlerr.InternalError{Format: errStr} + } + data = fmt.Sprintf("%v", pv) + default: + // TODO - bitset, empty + data = fmt.Sprintf("%v", value) + } + + return data, nil +} + +func copyYangXpathSpecData(dstNode *yangXpathInfo, srcNode *yangXpathInfo) { + if dstNode != nil && srcNode != nil { + *dstNode = *srcNode + } +} + +func isJsonDataEmpty(jsonData string) bool { + return string(jsonData) == "{}" +} + +func getFileNmLineNumStr() string { + _, AbsfileName, lineNum, _ := runtime.Caller(2) + fileNmElems := strings.Split(AbsfileName, "/") + fileNm := fileNmElems[len(fileNmElems)-1] + fNmLnoStr := fmt.Sprintf("[%v:%v]", fileNm, lineNum) + return fNmLnoStr +} + +func xfmrLogInfo(format string, args ...interface{}) { + fNmLnoStr := getFileNmLineNumStr() + log.Infof(fNmLnoStr + format, args...) +} + +func xfmrLogInfoAll(format string, args ...interface{}) { + if log.V(5) { + fNmLnoStr := getFileNmLineNumStr() + log.Infof(fNmLnoStr + format, args...) + } +} + +func formXfmrDbInputRequest(oper int, d db.DBNum, tableName string, key string, field string, value string) XfmrDbParams { + var inParams XfmrDbParams + inParams.oper = oper + inParams.dbNum = d + inParams.tableName = tableName + inParams.key = key + inParams.fieldName = field + inParams.value = value + return inParams +} + +func hasKeyValueXfmr(tblName string) bool { + if specTblInfo, ok := xDbSpecMap[tblName]; ok { + for _, lname := range specTblInfo.listName { + listXpath := tblName + "/" + lname + if specListInfo, ok := xDbSpecMap[listXpath]; ok { + for _, key := range specListInfo.keyList { + keyXpath := tblName + "/" + key + if specKeyInfo, ok := xDbSpecMap[keyXpath]; ok { + if specKeyInfo.xfmrValue != nil { + return true + } + } + } + } + } + } + return false +} + +func dbKeyValueXfmrHandler(oper int, dbNum db.DBNum, tblName string, dbKey string) (string, error) { + var err error + var keyValList []string + + xfmrLogInfoAll("dbKeyValueXfmrHandler: oper(%v), db(%v), tbl(%v), dbKey(%v)", + oper, dbNum, tblName, dbKey) + if specTblInfo, ok := xDbSpecMap[tblName]; ok { + for _, lname := range specTblInfo.listName { + listXpath := tblName + "/" + lname + keyMap := make(map[string]interface{}) + + if specListInfo, ok := xDbSpecMap[listXpath]; ok && len(specListInfo.keyList) > 0 { + sonicKeyDataAdd(dbNum, specListInfo.keyList, tblName, dbKey, keyMap) + + if len(keyMap) == len(specListInfo.keyList) { + for _, kname := range specListInfo.keyList { + keyXpath := tblName + "/" + kname + curKeyVal := fmt.Sprintf("%v", keyMap[kname]) + if kInfo, ok := xDbSpecMap[keyXpath]; ok && xDbSpecMap[keyXpath].xfmrValue != nil { + inParams := formXfmrDbInputRequest(oper, dbNum, tblName, dbKey, kname, curKeyVal) + curKeyVal, err = valueXfmrHandler(inParams, *kInfo.xfmrValue) + if err != nil { + log.Warningf("value-xfmr: keypath(\"%v\") value (\"%v\"):err(%v).", + keyXpath, curKeyVal, err) + return "", err + } + } + keyValList = append(keyValList, curKeyVal) + } + } + } + } + } + + dbOpts := getDBOptions(dbNum) + retKey := strings.Join(keyValList, dbOpts.KeySeparator) + xfmrLogInfoAll("dbKeyValueXfmrHandler: tbl(%v), dbKey(%v), retKey(%v), keyValList(%v)", + tblName, dbKey, retKey, keyValList) + + return retKey, nil +} + +func dbDataXfmrHandler(resultMap map[int]map[db.DBNum]map[string]map[string]db.Value) error { + xfmrLogInfoAll("Received resultMap(%v)", resultMap) + for oper, dbDataMap := range resultMap { + for dbNum, tblData := range dbDataMap { + for tblName, data := range tblData { + if specTblInfo, ok := xDbSpecMap[tblName]; ok && specTblInfo.hasXfmrFn { + skipKeySet := make(map[string]bool) + for dbKey, fldData := range data { + if _, ok := skipKeySet[dbKey]; !ok { + for fld, val := range fldData.Field { + fldName := fld + if strings.HasSuffix(fld, "@") { + fldName = strings.Split(fld, "@")[0] + } + /* check & invoke value-xfmr */ + fldXpath := tblName + "/" + fldName + if fInfo, ok := xDbSpecMap[fldXpath]; ok && fInfo.xfmrValue != nil { + inParams := formXfmrDbInputRequest(oper, dbNum, tblName, dbKey, fld, val) + retVal, err := valueXfmrHandler(inParams, *fInfo.xfmrValue) + if err != nil { + log.Warningf("value-xfmr:fldpath(\"%v\") val(\"%v\"):err(\"%v\").", + fldXpath, val, err) + return err + } + resultMap[oper][dbNum][tblName][dbKey].Field[fld] = retVal + } + } + + /* split tblkey and invoke value-xfmr if present */ + if hasKeyValueXfmr(tblName) { + retKey, err := dbKeyValueXfmrHandler(oper, dbNum, tblName, dbKey) + if err != nil { + return err + } + /* cache processed keys */ + skipKeySet[retKey] = true + if dbKey != retKey { + resultMap[oper][dbNum][tblName][retKey] = resultMap[oper][dbNum][tblName][dbKey] + delete(resultMap[oper][dbNum][tblName], dbKey) + } + } + } + } + } + } + } + } + xfmrLogInfoAll("Transformed resultMap(%v)", resultMap) + return nil +} + +func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, xpath string, oper int, tbl string, tblKey string, dbDataMap *RedisDbMap, txCache interface{}, resultMap map[string]interface{}, validate bool) xlateFromDbParams { + var inParamsForGet xlateFromDbParams + inParamsForGet.d = d + inParamsForGet.dbs = dbs + inParamsForGet.curDb = cdb + inParamsForGet.ygRoot = ygRoot + inParamsForGet.uri = uri + inParamsForGet.requestUri = requestUri + inParamsForGet.xpath = xpath + inParamsForGet.oper = oper + inParamsForGet.tbl = tbl + inParamsForGet.tblKey = tblKey + inParamsForGet.dbDataMap = dbDataMap + inParamsForGet.txCache = txCache + inParamsForGet.resultMap = resultMap + inParamsForGet.validate = validate + + return inParamsForGet +} + +func formXlateToDbParam(d *db.DB, ygRoot *ygot.GoStruct, oper int, uri string, requestUri string, xpathPrefix string, keyName string, jsonData interface{}, resultMap map[int]RedisDbMap, result map[string]map[string]db.Value, txCache interface{}, tblXpathMap map[string]map[string]map[string]bool, subOpDataMap map[int]*RedisDbMap, pCascadeDelTbl *[]string, xfmrErr *error, name string, value interface{}, tableName string) xlateToParams { + var inParamsForSet xlateToParams + inParamsForSet.d = d + inParamsForSet.ygRoot = ygRoot + inParamsForSet.oper = oper + inParamsForSet.uri = uri + inParamsForSet.requestUri = requestUri + inParamsForSet.xpath = xpathPrefix + inParamsForSet.keyName = keyName + inParamsForSet.jsonData = jsonData + inParamsForSet.resultMap = resultMap + inParamsForSet.result = result + inParamsForSet.txCache = txCache.(*sync.Map) + inParamsForSet.tblXpathMap = tblXpathMap + inParamsForSet.subOpDataMap = subOpDataMap + inParamsForSet.pCascadeDelTbl = pCascadeDelTbl + inParamsForSet.xfmrErr = xfmrErr + inParamsForSet.name = name + inParamsForSet.value = value + inParamsForSet.tableName = tableName + + return inParamsForSet +} + +func xlateUnMarshallUri(ygRoot *ygot.GoStruct, uri string) (*interface{}, error) { + if len(uri) == 0 { + errMsg := errors.New("Error: URI is empty") + log.Warning(errMsg) + return nil, errMsg + } + + path, err := ygot.StringToPath(uri, ygot.StructuredPath, ygot.StringSlicePath) + if err != nil { + return nil, err + } + + for _, p := range path.Elem { + if strings.Contains(p.Name, ":") { + pathSlice := strings.Split(p.Name, ":") + p.Name = pathSlice[len(pathSlice)-1] + } + } + + deviceObj := (*ygRoot).(*ocbinds.Device) + ygNode, _, errYg := ytypes.GetOrCreateNode(ocbSch.RootSchema(), deviceObj, path) + + if errYg != nil { + log.Warning("Error in creating the target object: ", errYg) + return nil, errYg + } + + return &ygNode, nil +} + +func splitUri(uri string) []string { + pathList := SplitPath(uri) + xfmrLogInfoAll("uri: %v ", uri) + xfmrLogInfoAll("uri path elems: %v", pathList) + return pathList +} + +func dbTableExists(d *db.DB, tableName string, dbKey string, oper int) (bool, error) { + var err error + // Read the table entry from DB + if len(tableName) > 0 { + if hasKeyValueXfmr(tableName) { + if oper == GET { //value tranformer callback decides based on oper type + oper = CREATE + } + retKey, err := dbKeyValueXfmrHandler(oper, d.Opts.DBNo, tableName, dbKey) + if err != nil { + return false, err + } + xfmrLogInfoAll("dbKeyValueXfmrHandler() returned db key %v", retKey) + dbKey = retKey + } + + dbTblSpec := &db.TableSpec{Name: tableName} + + if strings.Contains(dbKey, "*") { + keys, derr := d.GetKeysByPattern(dbTblSpec, dbKey) + if derr != nil { + log.Warningf("Failed to get keys for tbl(%v) dbKey pattern %v error: %v", tableName, dbKey, derr) + err = tlerr.NotFound("Resource not found") + return false, err + } + xfmrLogInfoAll("keys for table %v are %v", tableName, keys) + if len(keys) > 0 { + return true, nil + } else { + log.Warningf("dbKey %v does not exist in DB for table %v", dbKey, tableName) + err = tlerr.NotFound("Resource not found") + return false, err + } + } else { + + existingEntry, derr := d.GetEntry(dbTblSpec, db.Key{Comp: []string{dbKey}}) + if derr != nil { + log.Warningf("GetEntry failed for table: %v, key: %v err: %v", tableName, dbKey, derr) + err = tlerr.NotFound("Resource not found") + return false, err + } + return existingEntry.IsPopulated(), err + } + } else { + log.Warning("Empty table name received") + return false, nil + } +} + +func dbTableExistsInDbData(dbNo db.DBNum, table string, dbKey string, dbData RedisDbMap) bool { + xfmrLogInfoAll("received Db no - %v, table - %v, dbkey - %v", dbNo, table, dbKey) + if _, exists := dbData[dbNo][table][dbKey]; exists { + return true + } else { + return false + } +} + +func leafListInstExists(leafListInDbVal string, checkLeafListInstVal string) bool { + /*function to check if leaf-list DB value contains the given instance*/ + exists := false + xfmrLogInfoAll("received value of leaf-list in DB - %v, Value to be checked if exists in leaf-list - %v", leafListInDbVal, checkLeafListInstVal) + leafListItemLst := strings.Split(leafListInDbVal, ",") + for idx := range(leafListItemLst) { + if leafListItemLst[idx] == checkLeafListInstVal { + exists = true + xfmrLogInfoAll("Leaf-list instance exists") + break + } + } + return exists +} + +func extractLeafListInstFromUri(uri string) (string, error) { + /*function to extract leaf-list instance value coming as part of uri + Handling [ ] in value*/ + xfmrLogInfoAll("received uri - %v", uri) + var leafListInstVal string + yangType := "" + err := fmt.Errorf("Unable to extract leaf-list instance value for uri - %v", uri) + + xpath, _, xerr := XfmrRemoveXPATHPredicates(uri) + if !isSonicYang(uri) { + specInfo, ok := xYangSpecMap[xpath] + if !ok { + return leafListInstVal, xerr + } + yangType = yangTypeGet(specInfo.yangEntry) + if !(yangType == YANG_LEAF_LIST) { + return leafListInstVal, err + } + } else { + tokens:= strings.Split(xpath, "/") + fieldName := "" + tableName := "" + if len(tokens) > SONIC_FIELD_INDEX { + fieldName = tokens[SONIC_FIELD_INDEX] + tableName = tokens[SONIC_TABLE_INDEX] + } + dbSpecField := tableName + "/" + fieldName + _, ok := xDbSpecMap[dbSpecField] + if ok { + yangType := xDbSpecMap[dbSpecField].fieldType + // terminal node case + if !(yangType == YANG_LEAF_LIST) { + return leafListInstVal, err + } + } + } + + //Check if uri has Leaf-list value + if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) { + xpathList := strings.Split(xpath, "/") + ll_name := xpathList[len(xpathList)-1] + ll_inx := strings.LastIndex(uri,ll_name) + if ll_inx != -1 { + ll_value := uri[ll_inx:] + ll_value = strings.TrimSuffix(ll_value, "]") + valueLst := strings.SplitN(ll_value, "=", 2) + leafListInstVal = valueLst[1] + + if ((strings.Contains(leafListInstVal, ":")) && (strings.HasPrefix(leafListInstVal, OC_MDL_PFX) || strings.HasPrefix(leafListInstVal, IETF_MDL_PFX) || strings.HasPrefix(leafListInstVal, IANA_MDL_PFX))) { + // identity-ref/enum has module prefix + leafListInstVal = strings.SplitN(leafListInstVal, ":", 2)[1] + xfmrLogInfoAll("Leaf-list instance value after removing identityref prefix - %v", leafListInstVal) + } + xfmrLogInfoAll("Leaf-list instance value to be returned - %v", leafListInstVal) + + return leafListInstVal, nil + } + } + return leafListInstVal, err +} + diff --git a/translib/transformer/xlate_xfmr_handler.go b/translib/transformer/xlate_xfmr_handler.go new file mode 100644 index 000000000..40feaeb52 --- /dev/null +++ b/translib/transformer/xlate_xfmr_handler.go @@ -0,0 +1,316 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2020 Dell, Inc. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + +package transformer + +import ( + "github.com/Azure/sonic-mgmt-common/translib/db" + log "github.com/golang/glog" +) + +func xfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) (error) { + xfmrLogInfoAll("Received inParams %v Subtree function name %v", inParams, xfmrFuncNm) + if inParams.uri != inParams.requestUri { + _, yerr := xlateUnMarshallUri(inParams.ygRoot, inParams.uri) + if yerr != nil { + xfmrLogInfoAll("Failed to generate the ygot Node for uri(\"%v\") err(%v).", inParams.uri, yerr) + } + } + ret, err := XlateFuncCall(dbToYangXfmrFunc(xfmrFuncNm), inParams) + if err != nil { + xfmrLogInfoAll("Failed to retrieve data for xpath(\"%v\") err(%v).", inParams.uri, err) + return err + } + if ((ret != nil) && (len(ret)>0)) { + // db to yang subtree xfmr returns err as the only value in return data list from .Call() + if ret[DBTY_SBT_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[DBTY_SBT_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + } + } + } + return err +} + +func leafXfmrHandlerFunc(inParams XfmrParams, xfmrFieldFuncNm string) (map[string]interface{}, error) { + var err error + var fldValMap map[string]interface{} + + xfmrLogInfoAll("Received inParams %v to invoke Field transformer %v", inParams, xfmrFieldFuncNm) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xfmrFieldFuncNm), inParams) + if err != nil { + return fldValMap, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == DBTY_FLD_XFMR_RET_ARGS { + // field xfmr returns err as second value in return data list from .Call() + if ret[DBTY_FLD_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[DBTY_FLD_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFieldFuncNm, err) + } + } + } + + if ret[DBTY_FLD_XFMR_RET_VAL_INDX].Interface() != nil { + fldValMap = ret[DBTY_FLD_XFMR_RET_VAL_INDX].Interface().(map[string]interface{}) + } + } + return fldValMap, err +} + +func keyXfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) (map[string]interface{}, error) { + xfmrLogInfoAll("Received inParams %v key transformer function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall(dbToYangXfmrFunc(xfmrFuncNm), inParams) + retVal := make(map[string]interface{}) + if err != nil { + return retVal, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == DBTY_KEY_XFMR_RET_ARGS { + // key xfmr returns err as second value in return data list from .Call() + if ret[DBTY_KEY_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[DBTY_KEY_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return retVal, err + } + } + } + if ret[DBTY_KEY_XFMR_RET_VAL_INDX].Interface() != nil { + retVal = ret[DBTY_KEY_XFMR_RET_VAL_INDX].Interface().(map[string]interface{}) + return retVal, nil + } + } + return retVal, nil +} + +func validateHandlerFunc(inParams XfmrParams, validateFuncNm string) (bool) { + xfmrLogInfoAll("Received inParams %v, validate transformer function name %v", inParams, validateFuncNm) + ret, err := XlateFuncCall(validateFuncNm, inParams) + if err != nil { + return false + } + return ret[0].Interface().(bool) +} + +func xfmrTblHandlerFunc(xfmrTblFunc string, inParams XfmrParams, xfmrTblKeyCache map[string]tblKeyCache) ([]string, error) { + + xfmrLogInfoAll("Received inParams %v, table transformer function name %v", inParams, xfmrTblFunc) + if (inParams.oper == GET && xfmrTblKeyCache != nil) { + if tkCache, _ok := xfmrTblKeyCache[inParams.uri]; _ok { + if len(tkCache.dbTblList) > 0 { + return tkCache.dbTblList, nil + } + } + } + + var retTblLst []string + ret, err := XlateFuncCall(xfmrTblFunc, inParams) + if err != nil { + return retTblLst, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == TBL_XFMR_RET_ARGS { + // table xfmr returns err as second value in return data list from .Call() + if ret[TBL_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[TBL_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrTblFunc, err) + return retTblLst, err + } + } + } + + if ret[TBL_XFMR_RET_VAL_INDX].Interface() != nil { + retTblLst = ret[TBL_XFMR_RET_VAL_INDX].Interface().([]string) + } + } + if (inParams.oper == GET && xfmrTblKeyCache != nil) { + if _, _ok := xfmrTblKeyCache[inParams.uri]; !_ok { + xfmrTblKeyCache[inParams.uri] = tblKeyCache{} + } + tkCache := xfmrTblKeyCache[inParams.uri] + tkCache.dbTblList = retTblLst + xfmrTblKeyCache[inParams.uri] = tkCache + } + + return retTblLst, err +} + +func valueXfmrHandler(inParams XfmrDbParams, xfmrValueFuncNm string) (string, error) { + xfmrLogInfoAll("Received inParams %v Value transformer name %v", inParams, xfmrValueFuncNm) + + ret, err := XlateFuncCall(xfmrValueFuncNm, inParams) + if err != nil { + return "", err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_FLD_XFMR_RET_ARGS { + // value xfmr returns err as second value in return data list from .Call() + if ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrValueFuncNm, err) + return "", err + } + } + } + + if ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface() != nil { + retVal := ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface().(string) + return retVal, nil + } + } + + return "", err +} + +func leafXfmrHandler(inParams XfmrParams, xfmrFieldFuncNm string) (map[string]string, error) { + xfmrLogInfoAll("Received inParams %v Field transformer name %v", inParams, xfmrFieldFuncNm) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xfmrFieldFuncNm), inParams) + if err != nil { + return nil, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_FLD_XFMR_RET_ARGS { + // field xfmr returns err as second value in return data list from .Call() + if ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_FLD_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFieldFuncNm, err) + return nil, err + } + } + } + + if ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface() != nil { + fldValMap := ret[YTDB_FLD_XFMR_RET_VAL_INDX].Interface().(map[string]string) + return fldValMap, nil + } + } else { + retFldValMap := map[string]string{"NULL":"NULL"} + return retFldValMap, nil + } + + return nil, nil +} + +func xfmrHandler(inParams XfmrParams, xfmrFuncNm string) (map[string]map[string]db.Value, error) { + xfmrLogInfoAll("Received inParams %v Subtree function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xfmrFuncNm), inParams) + if err != nil { + return nil, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_SBT_XFMR_RET_ARGS { + // subtree xfmr returns err as second value in return data list from .Call() + if ret[YTDB_SBT_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_SBT_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return nil, err + } + } + } + if ret[YTDB_SBT_XFMR_RET_VAL_INDX].Interface() != nil { + retMap := ret[YTDB_SBT_XFMR_RET_VAL_INDX].Interface().(map[string]map[string]db.Value) + return retMap, nil + } + } + return nil, nil +} + +func keyXfmrHandler(inParams XfmrParams, xfmrFuncNm string) (string, error) { + xfmrLogInfoAll("Received inParams %v key transformer function name %v", inParams, xfmrFuncNm) + ret, err := XlateFuncCall(yangToDbXfmrFunc(xfmrFuncNm), inParams) + retVal := "" + if err != nil { + return retVal, err + } + + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == YTDB_KEY_XFMR_RET_ARGS { + // key xfmr returns err as second value in return data list from .Call() + if ret[YTDB_KEY_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[YTDB_KEY_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrFuncNm, err) + return retVal, err + } + } + } + if ret[YTDB_KEY_XFMR_RET_VAL_INDX].Interface() != nil { + retVal = ret[YTDB_KEY_XFMR_RET_VAL_INDX].Interface().(string) + return retVal, nil + } + } + return retVal, nil +} + +/* Invoke the post tansformer */ +func postXfmrHandlerFunc(xfmrPost string, inParams XfmrParams) (map[string]map[string]db.Value, error) { + retData := make(map[string]map[string]db.Value) + xfmrLogInfoAll("Received inParams %v, post transformer function name %v", inParams, xfmrPost) + ret, err := XlateFuncCall(xfmrPost, inParams) + if err != nil { + return nil, err + } + if ((ret != nil) && (len(ret)>0)) { + if len(ret) == POST_XFMR_RET_ARGS { + // post xfmr returns err as second value in return data list from .Call() + if ret[POST_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[POST_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrPost, err) + return retData, err + } + } + } + if ret[POST_XFMR_RET_VAL_INDX].Interface() != nil { + retData = ret[POST_XFMR_RET_VAL_INDX].Interface().(map[string]map[string]db.Value) + xfmrLogInfoAll("Post Transformer function : %v retData : %v", xfmrPost, retData) + } + } + return retData, err +} + +/* Invoke the pre tansformer */ +func preXfmrHandlerFunc(xfmrPre string, inParams XfmrParams) error { + xfmrLogInfoAll("Received inParams %v, pre transformer function name %v", inParams, xfmrPre) + ret, err := XlateFuncCall(xfmrPre, inParams) + if err != nil { + log.Warningf("Pre-transformer function(\"%v\") returned error - %v.", xfmrPre, err) + return err + } + if ((ret != nil) && (len(ret)>0)) { + if ret[PRE_XFMR_RET_ERR_INDX].Interface() != nil { + err = ret[PRE_XFMR_RET_ERR_INDX].Interface().(error) + if err != nil { + log.Warningf("Transformer function(\"%v\") returned error - %v.", xfmrPre, err) + return err + } + } + } + return err +} + diff --git a/translib/transformer/xspec.go b/translib/transformer/xspec.go index 9a6951fee..ae61ad297 100644 --- a/translib/transformer/xspec.go +++ b/translib/transformer/xspec.go @@ -23,8 +23,9 @@ import ( "os" "strings" log "github.com/golang/glog" + "github.com/Azure/sonic-mgmt-common/cvl" "github.com/Azure/sonic-mgmt-common/translib/db" - + "strconv" "github.com/openconfig/goyang/pkg/yang" ) @@ -40,30 +41,101 @@ type yangXpathInfo struct { delim string fieldName string xfmrFunc string + xfmrField string xfmrPost string validateFunc string + rpcFunc string xfmrKey string keyName *string dbIndex db.DBNum keyLevel int isKey bool + defVal string + tblOwner *bool + hasChildSubTree bool + hasNonTerminalNode bool + subscribePref *string + subscribeOnChg int + subscribeMinIntvl int + cascadeDel int + virtualTbl *bool + nameWithMod *string + xfmrPre string } type dbInfo struct { dbIndex db.DBNum keyName *string fieldType string + rpcFunc string dbEntry *yang.Entry yangXpath []string module string + delim string + leafRefPath []string + listName []string + keyList []string + xfmrValue *string + hasXfmrFn bool + cascadeDel int +} + +type sonicTblSeqnInfo struct { + OrdTbl []string + DepTbl map[string][]string +} + +type mdlInfo struct { + Org string + Ver string } var xYangSpecMap map[string]*yangXpathInfo var xDbSpecMap map[string]*dbInfo var xDbSpecOrdTblMap map[string][]string //map of module-name to ordered list of db tables { "sonic-acl" : ["ACL_TABLE", "ACL_RULE"] } +var xDbSpecTblSeqnMap map[string]*sonicTblSeqnInfo +var xMdlCpbltMap map[string]*mdlInfo +var sonicOrdTblListMap map[string][]string +var sonicLeafRefMap map[string][]string + +/* Add module name to map storing model info for model capabilities */ +func addMdlCpbltEntry(yangMdlNm string) { + if xMdlCpbltMap == nil { + xMdlCpbltMap = make(map[string]*mdlInfo) + } + mdlInfoEntry := new(mdlInfo) + if mdlInfoEntry == nil { + log.Warningf("Memory allocation failure for storing model info for gnmi - module %v", yangMdlNm) + return + } + mdlInfoEntry.Org = "" + mdlInfoEntry.Ver = "" + xMdlCpbltMap[yangMdlNm] = mdlInfoEntry +} + +/* Add version and organization info for model capabilities into map */ +func addMdlCpbltData(yangMdlNm string, version string, organization string) { + if xMdlCpbltMap == nil { + xMdlCpbltMap = make(map[string]*mdlInfo) + } + mdlInfoEntry, ok := xMdlCpbltMap[yangMdlNm] + if ((!ok) || (mdlInfoEntry == nil)) { + mdlInfoEntry = new(mdlInfo) + if mdlInfoEntry == nil { + log.Warningf("Memory allocation failure for storing model info for gnmi - module %v", yangMdlNm) + return + } + } + mdlInfoEntry.Ver = version + mdlInfoEntry.Org = organization + xMdlCpbltMap[yangMdlNm] = mdlInfoEntry +} /* update transformer spec with db-node */ func updateDbTableData (xpath string, xpathData *yangXpathInfo, tableName string) { + if len(tableName) == 0 { + return + } _, ok := xDbSpecMap[tableName] if ok { xDbSpecMap[tableName].yangXpath = append(xDbSpecMap[tableName].yangXpath, xpath) @@ -71,9 +143,59 @@ func updateDbTableData (xpath string, xpathData *yangXpathInfo, tableName string } } +func childContainerListPresenceFlagSet(xpath string) { + parXpath := parentXpathGet(xpath) + for { + if parXpath == "" { + break + } + if parXpathData, ok := xYangSpecMap[parXpath]; ok { + parXpathData.hasNonTerminalNode = true + } + parXpath = parentXpathGet(parXpath) + } +} + +func childSubTreePresenceFlagSet(xpath string) { + parXpath := parentXpathGet(xpath) + for { + if parXpath == "" { + break + } + if parXpathData, ok := xYangSpecMap[parXpath]; ok { + parXpathData.hasChildSubTree = true + } + parXpath = parentXpathGet(parXpath) + } +} + /* Recursive api to fill the map with yang details */ -func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string) { +func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string, xpathFull string) { xpath := "" + curKeyLevel := 0 + curXpathFull := "" + + if entry != nil && (entry.Kind == yang.CaseEntry || entry.Kind == yang.ChoiceEntry) { + curXpathFull = xpathFull + "/" + entry.Name + if _, ok := xYangSpecMap[xpathPrefix]; ok { + curKeyLevel = xYangSpecMap[xpathPrefix].keyLevel + } + curXpathData, ok := xYangSpecMap[curXpathFull] + if !ok { + curXpathData = new(yangXpathInfo) + curXpathData.dbIndex = db.ConfigDB // default value + xYangSpecMap[curXpathFull] = curXpathData + } + curXpathData.yangDataType = strings.ToLower(yang.EntryKindToName[entry.Kind]) + curXpathData.yangEntry = entry + if xYangSpecMap[xpathPrefix].subscribePref != nil { + curXpathData.subscribePref = xYangSpecMap[xpathPrefix].subscribePref + } + curXpathData.subscribeOnChg = xYangSpecMap[xpathPrefix].subscribeOnChg + curXpathData.subscribeMinIntvl = xYangSpecMap[xpathPrefix].subscribeMinIntvl + curXpathData.cascadeDel = xYangSpecMap[xpathPrefix].cascadeDel + xpath = xpathPrefix + } else { /* create the yang xpath */ if xYangSpecMap[xpathPrefix] != nil && xYangSpecMap[xpathPrefix].yangDataType == "module" { /* module name is separated from the rest of xpath with ":" */ @@ -82,23 +204,41 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr xpath = xpathPrefix + "/" + entry.Name } + updateChoiceCaseXpath := false + curXpathFull = xpath + if xpathPrefix != xpathFull { + curXpathFull = xpathFull + "/" + entry.Name + if annotNode, ok := xYangSpecMap[curXpathFull]; ok { + xpathData := new(yangXpathInfo) + xpathData.dbIndex = db.ConfigDB // default value + xYangSpecMap[xpath] = xpathData + copyYangXpathSpecData(xYangSpecMap[xpath], annotNode) + updateChoiceCaseXpath = true + } + } + xpathData, ok := xYangSpecMap[xpath] if !ok { xpathData = new(yangXpathInfo) xYangSpecMap[xpath] = xpathData xpathData.dbIndex = db.ConfigDB // default value + xpathData.subscribeOnChg = XFMR_INVALID + xpathData.subscribeMinIntvl = XFMR_INVALID + xpathData.cascadeDel = XFMR_INVALID } else { - xpathData = xYangSpecMap[xpath] + if len(xpathData.xfmrFunc) > 0 { + childSubTreePresenceFlagSet(xpath) + } } xpathData.yangDataType = entry.Node.Statement().Keyword - if entry.Node.Statement().Keyword == "list" && xpathData.tableName != nil { + if (xpathData.tableName != nil && *xpathData.tableName != "") { childToUpdateParent(xpath, *xpathData.tableName) } parentXpathData, ok := xYangSpecMap[xpathPrefix] /* init current xpath table data with its parent data, change only if needed. */ - if ok { + if ok && xpathData.tableName == nil { if xpathData.tableName == nil && parentXpathData.tableName != nil && xpathData.xfmrTbl == nil { xpathData.tableName = parentXpathData.tableName } else if xpathData.xfmrTbl == nil && parentXpathData.xfmrTbl != nil { @@ -119,20 +259,79 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr xpathData.xfmrFunc = parentXpathData.xfmrFunc } - if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) == 0 { + if ok && (parentXpathData.subscribeMinIntvl == XFMR_INVALID || + parentXpathData.subscribeOnChg == XFMR_INVALID) { + log.Warningf("Susbscribe MinInterval/OnChange flag is set to invalid for(%v) \r\n", xpathPrefix) + return + } + + if ok { + if xpathData.subscribeOnChg == XFMR_INVALID { + xpathData.subscribeOnChg = parentXpathData.subscribeOnChg + } + + if xpathData.subscribeMinIntvl == XFMR_INVALID { + xpathData.subscribeMinIntvl = parentXpathData.subscribeMinIntvl + } + + if xpathData.subscribePref == nil && parentXpathData.subscribePref != nil { + xpathData.subscribePref = parentXpathData.subscribePref + } + + if xpathData.subscribePref != nil && *xpathData.subscribePref == "NONE" { + xpathData.subscribePref = nil + } + + if parentXpathData.cascadeDel == XFMR_INVALID { + /* should not hit this case */ + log.Warningf("Cascade-delete flag is set to invalid for(%v) \r\n", xpathPrefix) + return + } + + if xpathData.cascadeDel == XFMR_INVALID && xpathData.dbIndex == db.ConfigDB{ + xpathData.cascadeDel = parentXpathData.cascadeDel + } + + if entry.Prefix != nil && entry.Prefix.Parent != nil && + entry.Prefix.Parent.Statement().Keyword == "module" && + parentXpathData.yangEntry.Prefix != nil && parentXpathData.yangEntry.Prefix.Parent != nil { + + if (len(parentXpathData.yangEntry.Prefix.Parent.NName()) > 0) && + (len(parentXpathData.yangEntry.Prefix.Parent.Statement().Keyword) > 0) && + (parentXpathData.yangEntry.Prefix.Parent.NName() != entry.Prefix.Parent.NName()) { + xpathData.nameWithMod = new(string) + *xpathData.nameWithMod = entry.Prefix.Parent.NName() + ":" + entry.Name + } + + } + } + + if ((xpathData.yangDataType == YANG_LEAF || xpathData.yangDataType == YANG_LEAF_LIST) && (len(xpathData.fieldName) == 0)) { + + if len(xpathData.xfmrField) != 0 { + xpathData.xfmrFunc = "" + } if xpathData.tableName != nil && xDbSpecMap[*xpathData.tableName] != nil { - if xDbSpecMap[*xpathData.tableName].dbEntry.Dir[entry.Name] != nil { + if _, ok := xDbSpecMap[*xpathData.tableName + "/" + entry.Name]; ok { xpathData.fieldName = entry.Name - } else if xDbSpecMap[*xpathData.tableName].dbEntry.Dir[strings.ToUpper(entry.Name)] != nil { - xpathData.fieldName = strings.ToUpper(entry.Name) + } else { + if _, ok := xDbSpecMap[*xpathData.tableName + "/" + strings.ToUpper(entry.Name)]; ok { + xpathData.fieldName = strings.ToUpper(entry.Name) + } + if _, ok := xDbSpecMap[*xpathData.tableName + "/" + entry.Name]; ok { + xpathData.fieldName = entry.Name + } } } else if xpathData.xfmrTbl != nil { /* table transformer present */ xpathData.fieldName = entry.Name } } + if xpathData.yangDataType == YANG_LEAF && len(entry.Default) > 0 { + xpathData.defVal = entry.Default + } - if xpathData.yangDataType == "leaf" && len(xpathData.fieldName) > 0 && xpathData.tableName != nil { + if (xpathData.yangDataType == YANG_LEAF || xpathData.yangDataType == YANG_LEAF_LIST) && len(xpathData.fieldName) > 0 && xpathData.tableName != nil { dbPath := *xpathData.tableName + "/" + xpathData.fieldName if xDbSpecMap[dbPath] != nil { xDbSpecMap[dbPath].yangXpath = append(xDbSpecMap[dbPath].yangXpath, xpath) @@ -148,8 +347,11 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr keyXpath := make([]string, len(strings.Split(entry.Key, " "))) for id, keyName := range(strings.Split(entry.Key, " ")) { keyXpath[id] = xpath + "/" + keyName - keyXpathData := new(yangXpathInfo) - xYangSpecMap[xpath + "/" + keyName] = keyXpathData + if _, ok := xYangSpecMap[xpath + "/" + keyName]; !ok { + keyXpathData := new(yangXpathInfo) + keyXpathData.dbIndex = db.ConfigDB // default value + xYangSpecMap[xpath + "/" + keyName] = keyXpathData + } xYangSpecMap[xpath + "/" + keyName].isKey = true } @@ -165,6 +367,33 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr } else if parentXpathData != nil && parentXpathData.keyXpath != nil { xpathData.keyXpath = parentXpathData.keyXpath } + xpathData.yangEntry = entry + + if xpathData.subscribeMinIntvl == XFMR_INVALID { + xpathData.subscribeMinIntvl = 0 + } + + if xpathData.subscribeOnChg == XFMR_INVALID { + xpathData.subscribeOnChg = XFMR_ENABLE + } + if ((xpathData.subscribePref != nil) && (*xpathData.subscribePref == "onchange") && (xpathData.subscribeOnChg == XFMR_DISABLE)) { + log.Infof("subscribe OnChange is disabled so setting subscribe preference to default/sample from onchange for xpath - %v", xpath) + xpathData.subscribePref = nil + } + if xpathData.cascadeDel == XFMR_INVALID { + /* set to default value */ + xpathData.cascadeDel = XFMR_DISABLE + } + + if updateChoiceCaseXpath { + copyYangXpathSpecData(xYangSpecMap[curXpathFull], xYangSpecMap[xpath]) + } + + if xpathData.yangDataType == YANG_CONTAINER || xpathData.yangDataType == YANG_LIST { + childContainerListPresenceFlagSet(xpath) + } + + } /* get current obj's children */ var childList []string @@ -172,10 +401,9 @@ func yangToDbMapFill (keyLevel int, xYangSpecMap map[string]*yangXpathInfo, entr childList = append(childList, k) } - xpathData.yangEntry = entry /* now recurse, filling the map with current node's children info */ for _, child := range childList { - yangToDbMapFill(curKeyLevel, xYangSpecMap, entry.Dir[child], xpath) + yangToDbMapFill(curKeyLevel, xYangSpecMap, entry.Dir[child], xpath, curXpathFull) } } @@ -196,27 +424,78 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { /* Start to fill xpath based map with yang data */ keyLevel := 0 - yangToDbMapFill(keyLevel, xYangSpecMap, e, "") + yangToDbMapFill(keyLevel, xYangSpecMap, e, "", "") // Fill the ordered map of child tables list for oc yangs updateSchemaOrderedMap(module, e) } - mapPrint(xYangSpecMap, "/tmp/fullSpec.txt") - dbMapPrint("/tmp/dbSpecMap.txt") + + //sonicOrdTblListMap = make(map[string][]string) + //jsonfile := YangPath + TblInfoJsonFile + //xlateJsonTblInfoLoad(sonicOrdTblListMap, jsonfile) + + mapPrint(xYangSpecMap, "/tmp/fullSpec.txt") + dbMapPrint("/tmp/dbSpecMap.txt") + xDbSpecTblSeqnMapPrint("/tmp/dbSpecTblSeqnMap.txt") + sonicLeafRefDataPrint("/tmp/sonicLeafRef.log") + sonicLeafRefMap = nil +} + +func dbSpecXpathGet(inPath string) (string, error){ + /* This api currently handles only cointainer inside a list for sonic-yang. + Should be enhanced to support nested list in future. */ + var err error + specPath := "" + + xpath, _, err := XfmrRemoveXPATHPredicates(inPath) + if err != nil { + log.Warningf("xpath conversion failed for(%v) \r\n", inPath) + return specPath, err + } + + pathList := strings.Split(xpath, "/") + if len(pathList) < 3 { + log.Warningf("Leaf-ref path not valid(%v) \r\n", inPath) + return specPath, err + } + + tableName := pathList[2] + fieldName := pathList[len(pathList)-1] + specPath = tableName+"/"+fieldName + return specPath, err +} + +/* Convert a relative path to an absolute path */ +func relativeToActualPathGet(relPath string, entry *yang.Entry) string { + actDbSpecPath := "" + xpath, _, err := XfmrRemoveXPATHPredicates(relPath) + if err != nil { + return actDbSpecPath + } + + pathList := strings.Split(xpath[1:], "/") + if len(pathList) > 3 && (pathList[len(pathList)-3] != "..") { + tableName := pathList[len(pathList)-3] + fieldName := pathList[len(pathList)-1] + actDbSpecPath = tableName+"/"+fieldName + } + return actDbSpecPath } /* Fill the map with db details */ -func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { +func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map[string]*dbInfo, entry *yang.Entry) { + if entry == nil { + return + } + entryType := entry.Node.Statement().Keyword if entry.Name != moduleNm { - if entryType == "container" { + if entryType == "container" || entryType == "rpc" { tableName = entry.Name } - - if !isYangResType(entryType) { dbXpath := tableName - if entryType != "container" { + if entryType != "container" && entryType != "rpc" { dbXpath = tableName + "/" + entry.Name } xDbSpecMap[dbXpath] = new(dbInfo) @@ -224,6 +503,7 @@ func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap[dbXpath].dbEntry = entry xDbSpecMap[dbXpath].fieldType = entryType xDbSpecMap[dbXpath].module = moduleNm + xDbSpecMap[dbXpath].cascadeDel = XFMR_INVALID if entryType == "container" { xDbSpecMap[dbXpath].dbIndex = db.ConfigDB if entry.Exts != nil && len(entry.Exts) > 0 { @@ -236,35 +516,112 @@ func dbMapFill(tableName string, curPath string, moduleNm string, trkTpCnt bool, xDbSpecMap[dbXpath].keyName = new(string) } *xDbSpecMap[dbXpath].keyName = ext.NName() + case "rpc-callback" : + xDbSpecMap[dbXpath].rpcFunc = ext.NName() + case "db-name" : + xDbSpecMap[dbXpath].dbIndex = dbNameToIndex(ext.NName()) + case "key-delim" : + xDbSpecMap[dbXpath].delim = ext.NName() default : log.Infof("Unsupported ext type(%v) for xpath(%v).", tagType, dbXpath) } } } + } else if tblSpecInfo, ok := xDbSpecMap[tableName]; ok && (entryType == YANG_LIST && len(entry.Key) != 0) { + tblSpecInfo.listName = append(tblSpecInfo.listName, entry.Name) + xDbSpecMap[dbXpath].keyList = append(xDbSpecMap[dbXpath].keyList, strings.Split(entry.Key, " ")...) + } else if entryType == YANG_LEAF || entryType == YANG_LEAF_LIST { + if entry.Type.Kind == yang.Yleafref { + var lerr error + lrefpath := entry.Type.Path + if (strings.Contains(lrefpath, "..")) { + lrefpath = relativeToActualPathGet(lrefpath, entry) + } else { + lrefpath, lerr = dbSpecXpathGet(lrefpath) + if lerr != nil { + log.Warningf("Failed to add leaf-ref for(%v) \r\n", entry.Type.Path) + return + } + + } + xDbSpecMap[dbXpath].leafRefPath = append(xDbSpecMap[dbXpath].leafRefPath, lrefpath) + sonicLeafRefMap[lrefpath] = append(sonicLeafRefMap[lrefpath], dbXpath) + } else if entry.Type.Kind == yang.Yunion && len(entry.Type.Type) > 0 { + for _, ltype := range entry.Type.Type { + if ltype.Kind == yang.Yleafref { + var lerr error + lrefpath := ltype.Path + if (strings.Contains(lrefpath, "..")) { + lrefpath = relativeToActualPathGet(lrefpath, entry) + } else { + lrefpath, lerr = dbSpecXpathGet(lrefpath) + if lerr != nil { + log.Warningf("Failed to add leaf-ref for(%v) \r\n", ltype.Path) + return + } + + } + xDbSpecMap[dbXpath].leafRefPath = append(xDbSpecMap[dbXpath].leafRefPath, lrefpath) + sonicLeafRefMap[lrefpath] = append(sonicLeafRefMap[lrefpath], dbXpath) + } + } + } } - } } else { moduleXpath := "/" + moduleNm + ":" + entry.Name xDbSpecMap[moduleXpath] = new(dbInfo) xDbSpecMap[moduleXpath].dbEntry = entry xDbSpecMap[moduleXpath].fieldType = entryType xDbSpecMap[moduleXpath].module = moduleNm - } + xDbSpecMap[moduleXpath].cascadeDel = XFMR_INVALID + for { + done := true + sncTblInfo := new(sonicTblSeqnInfo) + if sncTblInfo == nil { + log.Warningf("Memory allocation failure for storing Tbl order and dependency info for sonic module %v", moduleNm) + break + } + cvlSess, cvlRetSess := cvl.ValidationSessOpen() + if cvlRetSess != cvl.CVL_SUCCESS { + log.Warningf("Failure in creating CVL validation session object required to use CVl API to get Tbl info for module %v - %v", moduleNm, cvlRetSess) + break + } + var cvlRetOrdTbl cvl.CVLRetCode + sncTblInfo.OrdTbl, cvlRetOrdTbl = cvlSess.GetOrderedTables(moduleNm) + if cvlRetOrdTbl != cvl.CVL_SUCCESS { + log.Warningf("Failure in cvlSess.GetOrderedTables(%v) - %v", moduleNm, cvlRetOrdTbl) + + } + sncTblInfo.DepTbl = make(map[string][]string) + if sncTblInfo.DepTbl == nil { + log.Warningf("sncTblInfo.DepTbl is nill , no space to store dependency table list for sonic module %v", moduleNm) + cvl.ValidationSessClose(cvlSess) + break + } + for _, tbl := range(sncTblInfo.OrdTbl) { + var cvlRetDepTbl cvl.CVLRetCode + sncTblInfo.DepTbl[tbl], cvlRetDepTbl = cvlSess.GetDepTables(moduleNm, tbl) + if cvlRetDepTbl != cvl.CVL_SUCCESS { + log.Warningf("Failure in cvlSess.GetDepTables(%v, %v) - %v", moduleNm, tbl, cvlRetDepTbl) + } - var childList []string - for _, k := range entry.DirOKeys { - childList = append(childList, k) - } - if entryType == "container" && trkTpCnt { - xDbSpecOrdTblMap[moduleNm] = childList - log.Info("xDbSpecOrdTblMap after appending ", xDbSpecOrdTblMap) - trkTpCnt = false + } + xDbSpecTblSeqnMap[moduleNm] = sncTblInfo + cvl.ValidationSessClose(cvlSess) + if done { + break + } + } + } + var childList []string + childList = append(childList, entry.DirOKeys...) + for _, child := range childList { childPath := tableName + "/" + entry.Dir[child].Name - dbMapFill(tableName, childPath, moduleNm, trkTpCnt, xDbSpecMap, entry.Dir[child]) + dbMapFill(tableName, childPath, moduleNm, xDbSpecMap, entry.Dir[child]) } } @@ -275,15 +632,15 @@ func dbMapBuild(entries []*yang.Entry) { } xDbSpecMap = make(map[string]*dbInfo) xDbSpecOrdTblMap = make(map[string][]string) + xDbSpecTblSeqnMap = make(map[string]*sonicTblSeqnInfo) + sonicLeafRefMap = make(map[string][]string) for _, e := range entries { if e == nil || len(e.Dir) == 0 { continue } moduleNm := e.Name - log.Infof("Module name(%v)", moduleNm) - trkTpCnt := true - dbMapFill("", "", moduleNm, trkTpCnt, xDbSpecMap, e) + dbMapFill("", "", moduleNm, xDbSpecMap, e) } } @@ -297,25 +654,55 @@ func childToUpdateParent( xpath string, tableName string) { _, ok := xYangSpecMap[parent] if !ok { xpathData = new(yangXpathInfo) + xpathData.dbIndex = db.ConfigDB // default value xYangSpecMap[parent] = xpathData } - xYangSpecMap[parent].childTable = append(xYangSpecMap[parent].childTable, tableName) - if xYangSpecMap[parent].yangEntry != nil && - xYangSpecMap[parent].yangEntry.Node.Statement().Keyword == "list" { + + parentXpathData := xYangSpecMap[parent] + if !contains(parentXpathData.childTable, tableName) { + parentXpathData.childTable = append(parentXpathData.childTable, tableName) + } + + if parentXpathData.yangEntry != nil && parentXpathData.yangEntry.Node.Statement().Keyword == "list" && + (parentXpathData.tableName != nil || parentXpathData.xfmrTbl != nil) { return } childToUpdateParent(parent, tableName) } +func dbNameToIndex(dbName string) db.DBNum { + dbIndex := db.ConfigDB + switch dbName { + case "APPL_DB" : + dbIndex = db.ApplDB + case "ASIC_DB" : + dbIndex = db.AsicDB + case "COUNTERS_DB" : + dbIndex = db.CountersDB + case "LOGLEVEL_DB" : + dbIndex = db.LogLevelDB + case "CONFIG_DB" : + dbIndex = db.ConfigDB + case "FLEX_COUNTER_DB" : + dbIndex = db.FlexCounterDB + case "STATE_DB" : + dbIndex = db.StateDB + case "ERROR_DB" : + dbIndex = db.ErrorDB + case "USER_DB" : + dbIndex = db.UserDB + } + return dbIndex +} + /* Build lookup map based on yang xpath */ func annotEntryFill(xYangSpecMap map[string]*yangXpathInfo, xpath string, entry *yang.Entry) { xpathData := new(yangXpathInfo) - _, ok := xYangSpecMap[xpath] - if !ok { - fmt.Printf("Xpath not found(%v) \r\n", xpath) - } xpathData.dbIndex = db.ConfigDB // default value + xpathData.subscribeOnChg = XFMR_INVALID + xpathData.subscribeMinIntvl = XFMR_INVALID + xpathData.cascadeDel = XFMR_INVALID /* fill table with yang extension data. */ if entry != nil && len(entry.Exts) > 0 { for _, ext := range entry.Exts { @@ -347,30 +734,62 @@ func annotEntryFill(xYangSpecMap map[string]*yangXpathInfo, xpath string, entry case "key-delimiter" : xpathData.delim = ext.NName() case "field-transformer" : - xpathData.xfmrFunc = ext.NName() + xpathData.xfmrField = ext.NName() case "post-transformer" : xpathData.xfmrPost = ext.NName() + case "pre-transformer" : + xpathData.xfmrPre = ext.NName() case "get-validate" : xpathData.validateFunc = ext.NName() + case "rpc-callback" : + xpathData.rpcFunc = ext.NName() case "use-self-key" : xpathData.keyXpath = nil case "db-name" : - if ext.NName() == "APPL_DB" { - xpathData.dbIndex = db.ApplDB - } else if ext.NName() == "ASIC_DB" { - xpathData.dbIndex = db.AsicDB - } else if ext.NName() == "COUNTERS_DB" { - xpathData.dbIndex = db.CountersDB - } else if ext.NName() == "LOGLEVEL_DB" { - xpathData.dbIndex = db.LogLevelDB - } else if ext.NName() == "CONFIG_DB" { - xpathData.dbIndex = db.ConfigDB - } else if ext.NName() == "FLEX_COUNTER_DB" { - xpathData.dbIndex = db.FlexCounterDB - } else if ext.NName() == "STATE_DB" { - xpathData.dbIndex = db.StateDB + xpathData.dbIndex = dbNameToIndex(ext.NName()) + case "table-owner" : + if xpathData.tblOwner == nil { + xpathData.tblOwner = new(bool) + *xpathData.tblOwner = true + } + if strings.EqualFold(ext.NName(), "False") { + *xpathData.tblOwner = false + } + case "subscribe-preference" : + if xpathData.subscribePref == nil { + xpathData.subscribePref = new(string) + } + *xpathData.subscribePref = ext.NName() + case "subscribe-on-change" : + if ext.NName() == "enable" || ext.NName() == "ENABLE" { + xpathData.subscribeOnChg = XFMR_ENABLE + } else { + xpathData.subscribeOnChg = XFMR_DISABLE + } + case "subscribe-min-interval" : + if ext.NName() == "NONE" { + xpathData.subscribeMinIntvl = 0 } else { - xpathData.dbIndex = db.ConfigDB + minIntvl, err := strconv.Atoi(ext.NName()) + if err != nil { + log.Warningf("Invalid subscribe min interval time(%v).\r\n", ext.NName()) + return + } + xpathData.subscribeMinIntvl = minIntvl + } + case "cascade-delete" : + if ext.NName() == "ENABLE" || ext.NName() == "enable" { + xpathData.cascadeDel = XFMR_ENABLE + } else { + xpathData.cascadeDel = XFMR_DISABLE + } + case "virtual-table" : + if xpathData.virtualTbl == nil { + xpathData.virtualTbl = new(bool) + *xpathData.virtualTbl = false + } + if strings.EqualFold(ext.NName(), "True") { + *xpathData.virtualTbl = true } } } @@ -419,19 +838,45 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya var dbXpathData *dbInfo var ok bool - //Currently sonic-yang annotation is supported for "list" type only. - listName := strings.Split(dbXpath, "/") - if len(listName) < 3 { - log.Errorf("Invalid list xpath length(%v) \r\n", dbXpath) - return err + pname := strings.Split(dbXpath, "/") + if len(pname) < 3 { + // check rpc? + rpcName := strings.Split(pname[1], ":") + if len(rpcName) < 2 { + log.Warningf("DB spec-map data not found(%v) \r\n", rpcName) + return err + } + dbXpathData, ok = xDbSpecMap[rpcName[1]] + if ok && dbXpathData.fieldType == "rpc" { + if entry != nil && len(entry.Exts) > 0 { + for _, ext := range entry.Exts { + dataTagArr := strings.Split(ext.Keyword, ":") + tagType := dataTagArr[len(dataTagArr)-1] + switch tagType { + case "rpc-callback" : + dbXpathData.rpcFunc = ext.NName() + default : + } + } + } + return err + } else { + log.Warningf("DB spec-map data not found(%v) \r\n", dbXpath) + return err + } } - dbXpathData, ok = xDbSpecMap[listName[2]] + + tableName := pname[2] + // container(redis tablename) + dbXpathData, ok = xDbSpecMap[tableName] if !ok { - log.Errorf("DB spec-map data not found(%v) \r\n", dbXpath) + log.Warningf("DB spec-map data not found(%v) \r\n", dbXpath) return err } - log.Infof("Annotate dbSpecMap for (%v)(listName:%v)\r\n", dbXpath, listName[2]) - dbXpathData.dbIndex = db.ConfigDB // default value + + if dbXpathData.dbIndex >= db.MaxDB { + dbXpathData.dbIndex = db.ConfigDB // default value + } /* fill table with cvl yang extension data. */ if entry != nil && len(entry.Exts) > 0 { @@ -445,29 +890,37 @@ func annotDbSpecMapFill(xDbSpecMap map[string]*dbInfo, dbXpath string, entry *ya } *dbXpathData.keyName = ext.NName() case "db-name" : - if ext.NName() == "APPL_DB" { - dbXpathData.dbIndex = db.ApplDB - } else if ext.NName() == "ASIC_DB" { - dbXpathData.dbIndex = db.AsicDB - } else if ext.NName() == "COUNTERS_DB" { - dbXpathData.dbIndex = db.CountersDB - } else if ext.NName() == "LOGLEVEL_DB" { - dbXpathData.dbIndex = db.LogLevelDB - } else if ext.NName() == "CONFIG_DB" { - dbXpathData.dbIndex = db.ConfigDB - } else if ext.NName() == "FLEX_COUNTER_DB" { - dbXpathData.dbIndex = db.FlexCounterDB - } else if ext.NName() == "STATE_DB" { - dbXpathData.dbIndex = db.StateDB + dbXpathData.dbIndex = dbNameToIndex(ext.NName()) + case "value-transformer" : + fieldName := pname[len(pname) - 1] + fieldXpath := tableName+"/"+fieldName + if fldXpathData, ok := xDbSpecMap[fieldXpath]; ok { + fldXpathData.xfmrValue = new(string) + *fldXpathData.xfmrValue = ext.NName() + dbXpathData.hasXfmrFn = true + if xpathList, ok := sonicLeafRefMap[fieldXpath]; ok { + for _, curpath := range(xpathList) { + if curSpecData, ok := xDbSpecMap[curpath]; ok && curSpecData.xfmrValue == nil { + curSpecData.xfmrValue = fldXpathData.xfmrValue + curTableName := strings.Split(curpath, "/")[0] + if curTblSpecInfo, ok := xDbSpecMap[curTableName]; ok { + curTblSpecInfo.hasXfmrFn = true + } + } + } + } + } + case "cascade-delete" : + if ext.NName() == "ENABLE" || ext.NName() == "enable" { + dbXpathData.cascadeDel = XFMR_ENABLE } else { - dbXpathData.dbIndex = db.ConfigDB + dbXpathData.cascadeDel = XFMR_DISABLE } default : } } } - dbMapPrint("/tmp/dbSpecMapFull.txt") return err } @@ -490,6 +943,7 @@ func annotDbSpecMap(annotEntries []*yang.Entry) { } } } + dbMapPrint("/tmp/dbSpecMapFull.txt") } /* Debug function to print the yang xpath lookup map */ @@ -504,10 +958,32 @@ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") fmt.Fprintf(fp, "%v:\r\n", k) fmt.Fprintf(fp, " yangDataType: %v\r\n", d.yangDataType) + if d.nameWithMod != nil { + fmt.Fprintf(fp, " nameWithMod : %v\r\n", *d.nameWithMod) + } + fmt.Fprintf(fp, " cascadeDel : %v\r\n", d.cascadeDel) + fmt.Fprintf(fp, " hasChildSubTree : %v\r\n", d.hasChildSubTree) + fmt.Fprintf(fp, " hasNonTerminalNode : %v\r\n", d.hasNonTerminalNode) + fmt.Fprintf(fp, " subscribeOnChg : %v\r\n", d.subscribeOnChg) + fmt.Fprintf(fp, " subscribeMinIntvl : %v\r\n", d.subscribeMinIntvl) + if d.subscribePref != nil { + fmt.Fprintf(fp, " subscribePref : %v\r\n", *d.subscribePref) + } + fmt.Fprintf(fp, " hasChildSubTree: %v\r\n", d.hasChildSubTree) fmt.Fprintf(fp, " tableName: ") if d.tableName != nil { fmt.Fprintf(fp, "%v", *d.tableName) } + fmt.Fprintf(fp, "\r\n postXfmr : %v", d.xfmrPost) + fmt.Fprintf(fp, "\r\n tblOwner: ") + if d.tblOwner != nil { + fmt.Fprintf(fp, "%v", *d.tblOwner) + } + fmt.Fprintf(fp, "\r\n virtualTbl: ") + if d.virtualTbl != nil { + fmt.Fprintf(fp, "%v", *d.virtualTbl) + } + fmt.Fprintf(fp, "\r\n preXfmr : %v", d.xfmrPre) fmt.Fprintf(fp, "\r\n xfmrTbl : ") if d.xfmrTbl != nil { fmt.Fprintf(fp, "%v", *d.xfmrTbl) @@ -518,11 +994,14 @@ func mapPrint(inMap map[string]*yangXpathInfo, fileName string) { } fmt.Fprintf(fp, "\r\n childTbl : %v", d.childTable) fmt.Fprintf(fp, "\r\n FieldName: %v", d.fieldName) + fmt.Fprintf(fp, "\r\n defVal : %v", d.defVal) fmt.Fprintf(fp, "\r\n keyLevel : %v", d.keyLevel) fmt.Fprintf(fp, "\r\n xfmrKeyFn: %v", d.xfmrKey) fmt.Fprintf(fp, "\r\n xfmrFunc : %v", d.xfmrFunc) + fmt.Fprintf(fp, "\r\n xfmrField :%v", d.xfmrField) fmt.Fprintf(fp, "\r\n dbIndex : %v", d.dbIndex) fmt.Fprintf(fp, "\r\n validateFunc : %v", d.validateFunc) + fmt.Fprintf(fp, "\r\n rpcFunc : %v", d.rpcFunc) fmt.Fprintf(fp, "\r\n yangEntry: ") if d.yangEntry != nil { fmt.Fprintf(fp, "%v", *d.yangEntry) @@ -550,21 +1029,89 @@ func dbMapPrint( fname string) { defer fp.Close() fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") for k, v := range xDbSpecMap { - fmt.Fprintf(fp, " field:%v \r\n", k) + fmt.Fprintf(fp, " field:%v: \r\n", k) fmt.Fprintf(fp, " type :%v \r\n", v.fieldType) fmt.Fprintf(fp, " db-type :%v \r\n", v.dbIndex) + fmt.Fprintf(fp, " rpcFunc :%v \r\n", v.rpcFunc) + fmt.Fprintf(fp, " hasXfmrFn:%v \r\n", v.hasXfmrFn) fmt.Fprintf(fp, " module :%v \r\n", v.module) + fmt.Fprintf(fp, " listName :%v \r\n", v.listName) + fmt.Fprintf(fp, " keyList :%v \r\n", v.keyList) + fmt.Fprintf(fp, " cascadeDel :%v \r\n", v.cascadeDel) + if v.xfmrValue != nil { + fmt.Fprintf(fp, " xfmrValue:%v \r\n", *v.xfmrValue) + } + fmt.Fprintf(fp, " leafRefPath:%v \r\n", v.leafRefPath) fmt.Fprintf(fp, " KeyName: ") if v.keyName != nil { - fmt.Fprintf(fp, "%v", *v.keyName) + fmt.Fprintf(fp, "%v\r\n", *v.keyName) } - fmt.Fprintf(fp, "\r\n oc-yang :%v \r\n", v.yangXpath) - fmt.Fprintf(fp, " cvl-yang :%v \r\n", v.dbEntry) + for _, yxpath := range v.yangXpath { + fmt.Fprintf(fp, "\r\n oc-yang :%v ", yxpath) + } + fmt.Fprintf(fp, "\r\n cvl-yang :%v \r\n", v.dbEntry) fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") } } +func xDbSpecTblSeqnMapPrint(fname string) { + fp, err := os.Create(fname) + if err != nil { + return + } + defer fp.Close() + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + if xDbSpecTblSeqnMap == nil { + return + } + for mdlNm, mdlTblSeqnDt := range xDbSpecTblSeqnMap { + fmt.Fprintf(fp, "%v : { \r\n", mdlNm) + if mdlTblSeqnDt == nil { + continue + } + fmt.Fprintf(fp, " OrderedTableList : %v\r\n", mdlTblSeqnDt.OrdTbl) + fmt.Fprintf(fp, " Dependent table list : {\r\n") + if mdlTblSeqnDt.DepTbl == nil { + fmt.Fprintf(fp, " }\r\n") + fmt.Fprintf(fp, "}\r\n") + continue + } + for tblNm, DepTblLst := range mdlTblSeqnDt.DepTbl { + fmt.Fprintf(fp, " %v : %v\r\n", tblNm, DepTblLst) + } + fmt.Fprintf(fp, " }\r\n") + fmt.Fprintf(fp, "}\r\n") + } + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + fmt.Fprintf (fp, "OrderedTableList from json: \r\n") + + for tbl, tlist := range sonicOrdTblListMap { + fmt.Fprintf(fp, " %v : %v\r\n", tbl, tlist) + } + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + +} + +func sonicLeafRefDataPrint(fname string) { + fp, err := os.Create(fname) + if err != nil { + return + } + defer fp.Close() + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + if xDbSpecTblSeqnMap == nil { + return + } + for lref, data := range sonicLeafRefMap { + fmt.Fprintf (fp, "leafref: %v\r\n", lref) + for i, d := range data { + fmt.Fprintf (fp, " (%v) %v\r\n", i, d) + } + fmt.Fprintf (fp, "-----------------------------------------------------------------\r\n") + } +} + func updateSchemaOrderedMap(module string, entry *yang.Entry) { var children []string if entry.Node.Statement().Keyword == "module" { diff --git a/translib/utils/utils.go b/translib/utils/utils.go new file mode 100644 index 000000000..466c618bf --- /dev/null +++ b/translib/utils/utils.go @@ -0,0 +1,71 @@ +//////////////////////////////////////////////////////////////////////////////// +// // +// Copyright 2019 Broadcom. The term Broadcom refers to Broadcom Inc. and/or // +// its subsidiaries. // +// // +// Licensed under the Apache License, Version 2.0 (the "License"); // +// you may not use this file except in compliance with the License. // +// You may obtain a copy of the License at // +// // +// http://www.apache.org/licenses/LICENSE-2.0 // +// // +// Unless required by applicable law or agreed to in writing, software // +// distributed under the License is distributed on an "AS IS" BASIS, // +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // +// See the License for the specific language governing permissions and // +// limitations under the License. // +// // +//////////////////////////////////////////////////////////////////////////////// + + +package utils + +import ( + //"github.com/Azure/sonic-mgmt-common/translib/db" + "github.com/Azure/sonic-mgmt-common/cvl" + "fmt" + log "github.com/golang/glog" +) + +// SortAsPerTblDeps - sort transformer result table list based on dependencies (using CVL API) tables to be used for CRUD operations +func SortAsPerTblDeps(tblLst []string) ([]string, error) { + var resultTblLst []string + var err error + logStr := "Failure in CVL API to sort table list as per dependencies." + + cvSess, cvlRetSess := cvl.ValidationSessOpen() + if cvlRetSess != cvl.CVL_SUCCESS { + + log.Errorf("Failure in creating CVL validation session object required to use CVl API(sort table list as per dependencies) - %v", cvlRetSess) + err = fmt.Errorf("%v", logStr) + return resultTblLst, err + } + cvlSortDepTblList, cvlRetDepTbl := cvSess.SortDepTables(tblLst) + if cvlRetDepTbl != cvl.CVL_SUCCESS { + log.Warningf("Failure in cvlSess.SortDepTables: %v", cvlRetDepTbl) + cvl.ValidationSessClose(cvSess) + err = fmt.Errorf("%v", logStr) + return resultTblLst, err + } + log.Info("cvlSortDepTblList = ", cvlSortDepTblList) + resultTblLst = cvlSortDepTblList + + cvl.ValidationSessClose(cvSess) + return resultTblLst, err + +} + +// RemoveElement - Remove a specific string from a list of strings +func RemoveElement(sl []string, str string) []string { + for i := 0; i < len(sl); i++ { + if sl[i] == str { + sl = append(sl[:i], sl[i+1:]...) + i-- + break + } + } + return sl +} + + +