set up cache multiple ads exp (prebid#310)

* set up cache multiple ads exp

* configure max cached ads count

* update

pkg/modules/ad_cache/module/ad_cache.go

@@ -59,6 +59,10 @@ func GetAdCachePrimaryKey(userId string, placementId string) string {
 	return fmt.Sprintf("%s_%s", userId, placementId)
 }
 
+func GetMultiAdCachePrimaryKey(userId string, placementId string) string {
+	return fmt.Sprintf("mca_%s_%s", userId, placementId)
+}
+
 func GetUnifiedAdCacheKey(userId string, format string) string {
 	return fmt.Sprintf("%s_%s", userId, format)
 }

pkg/modules/ad_cache/module/config.go

@@ -27,6 +27,7 @@ type AdCacheBiddersConfig struct {
 	RedisConfig            db.RedisConfig          `json:"redis_config"`
 	DamDFEndPoint          string                  `json:"dam_dynamic_floor_endpoint"`
 	WriteToCacheAdDynamoDB bool                    `json:"write_to_cached_ad_dynamodb"`
+	MaxCachedAdsCount      int                     `json:"max_cached_ads_count"`
 }
 
 type BidderConfig struct {

pkg/modules/ad_cache/module/hook_auction_response.go

@@ -36,6 +36,8 @@ func handleAuctionResponsesHook(
 	if config.WriteToCacheAdDynamoDB {
 		if formatCachedData, found := miCtx.ModuleContext[MSP_FORMAT_AD_CACHE_DATA]; found {
 			winningSeat = processFormatCachedData(formatCachedData.(map[openrtb_ext.BidType]FormatCachedAds), payload, seatBidMap, config, placementId.(string), userId.(string), fetcher)
+		} else if cachedAdsList, found := miCtx.ModuleContext[MSP_MULTI_AD_CACHE_DATA]; found {
+			winningSeat = processCachedDataList(cachedAdsList.(BiddersAdDataList), payload, seatBidMap, config, placementId.(string), userId.(string), fetcher)
 		} else {
 			adCacheKey, cacheKeyFound := miCtx.ModuleContext[MSP_AD_CACHE_KEY]
 			if cacheKeyFound {
@@ -281,6 +283,107 @@ func processFormatCachedData(formatCachedAds map[openrtb_ext.BidType]FormatCache
 	return winningSeat
 }
 
+func processCachedDataList(cachedAdsList BiddersAdDataList, payload hookstage.AuctionResponsePayload, seatBidMap map[string]*openrtb2.SeatBid, config *AdCacheBiddersConfig, placementId string, userId string, fetcher *db.DyNamoDbFetcher) *string {
+	// insert filled bids from current auction to cached list
+	for seat, bid := range seatBidMap {
+		bidderCfg, bidderEnabled := GetBidderConfig(config, seat, placementId)
+		if !bidderEnabled {
+			continue
+		}
+		responseBid := bid.Bid[0]
+		newAdData := AdData{
+			Bid: responseBid,
+			TTL: int(time.Now().Unix()) + bidderCfg.TTL,
+		}
+		bidList, seatExist := cachedAdsList[seat]
+		if seatExist {
+			bidList = insertBidToList(bidList, newAdData)
+		} else {
+			bidList = []AdData{newAdData}
+		}
+		cachedAdsList[seat] = bidList
+	}
+	// glog.Infof("after fill response bid formatCachedAds: %+v", toString(cachedAdsList))
+	// keep all cached bid that are not stale yet
+	for seat, bidList := range cachedAdsList {
+		// biddersAdsList := BiddersAdDataList{}
+		_, bidderEnabled := GetBidderConfig(config, seat, placementId)
+		if !bidderEnabled {
+			continue
+		}
+		updatedBidList := []AdData{}
+		for _, bid := range bidList {
+			if bid.TTL >= int(time.Now().Unix()) {
+				updatedBidList = append(updatedBidList, bid)
+			}
+		}
+		cachedAdsList[seat] = updatedBidList
+	}
+
+	// glog.Infof("after TTL check formatCachedAds: %+v", toString(cachedAdsList))
+	// put best bid for each bidder from cache to the final payload to decide the winner bid
+	// we can safely do it since all latest filled bids from the current auction have been added to cache already
+	for seat, bidList := range cachedAdsList {
+		responseSeatBid, foundInResponse := seatBidMap[seat]
+		// the first bid in cached list is always the best for this bidder
+		if len(bidList) > 0 {
+			bestBid := bidList[0].Bid
+			if !foundInResponse {
+				newSeatBid := openrtb2.SeatBid{
+					Seat: seat,
+					Bid:  []openrtb2.Bid{bestBid},
+				}
+				seatBidMap[seat] = &newSeatBid
+			} else if bestBid.Price > responseSeatBid.Bid[0].Price {
+				newSeatBid := openrtb2.SeatBid{
+					Seat: seat,
+					Bid:  []openrtb2.Bid{bestBid},
+				}
+				seatBidMap[seat] = &newSeatBid
+			}
+		}
+	}
+
+	payload.BidResponse.SeatBid = seatbidMap2Array(seatBidMap)
+	winningBid, winningSeat := updateWinningBid(payload.BidResponse)
+
+	// remove bid that wins the current auction from the cached data since it will be used
+	if winningBid != nil && winningSeat != nil {
+		bidList, seatExist := cachedAdsList[*winningSeat]
+		if seatExist {
+			deleteIdx := -1
+			for idx := range bidList {
+				if bidList[idx].Bid.ID == winningBid.ID {
+					deleteIdx = idx
+					break
+				}
+			}
+			if deleteIdx >= 0 {
+				bidList = append(bidList[:deleteIdx], bidList[deleteIdx+1:]...)
+			}
+			cachedAdsList[*winningSeat] = bidList
+		}
+	}
+	// glog.Infof("after remove winning bid formatCachedAds: %+v", toString(cachedAdsList))
+	// limit the bidder level cached ads size to 2
+	for seat, bidList := range cachedAdsList {
+		if len(bidList) > config.MaxCachedAdsCount {
+			bidList = bidList[:config.MaxCachedAdsCount]
+			cachedAdsList[seat] = bidList
+		}
+	}
+
+	// glog.Infof("after size limit formatCachedAds: %+v", toString(cachedAdsList))
+	// update cache in DynamoDB
+	cacheKey := GetMultiAdCachePrimaryKey(userId, placementId)
+	updatedCache := cachedAdsList
+	go func(key string, data BiddersAdDataList) {
+		SaveUserCachedAdsList(key, data, fetcher)
+	}(cacheKey, updatedCache)
+
+	return winningSeat
+}
+
 // this is for debugging purpose, leave it here for troubleshooting later
 // func toString(ads map[openrtb_ext.BidType]FormatCachedAds) string {
 // 	str := ""
@@ -296,6 +399,18 @@ func processFormatCachedData(formatCachedAds map[openrtb_ext.BidType]FormatCache
 // 	return str
 // }
 
+// this is for debugging purpose, leave it here for troubleshooting later
+// func toString(ads BiddersAdDataList) string {
+// 	str := ""
+// 	for bidder, ad := range ads {
+// 		str += bidder + "\n"
+// 		for _, bid := range ad {
+// 			str += bid.Bid.ID + "|" + string(strconv.FormatFloat(bid.Bid.Price, 'f', -1, 64)) + "\n"
+// 		}
+// 	}
+// 	return str
+// }
+
 func extractBidType(bid openrtb2.Bid) *openrtb_ext.BidType {
 	var ext map[string]json.RawMessage
 	err := json.Unmarshal(bid.Ext, &ext)

pkg/modules/ad_cache/module/hook_processed_auction.go

@@ -75,6 +75,23 @@ func handleProcessedAuctionHook(
 						}
 					}
 					result.ModuleContext[MSP_FORMAT_AD_CACHE_DATA] = allCachedAds
+				} else if u.Contains(multi_cached_ads_placements, placementId) {
+					cacheKey := GetMultiAdCachePrimaryKey(userId, placementId)
+					cachedAdsList, exist := GetUserCachedAdsList(cacheKey, fetcher)
+					if exist {
+						for bidder, adsList := range cachedAdsList {
+							// the bidder level cached ads list is expected to be sorted by bid price in reverse order
+							// so the first bid in the list (if present) must be the best bid
+							ad, exist := cachedAd[bidder]
+							if len(adsList) > 0 && (!exist || (adsList[0].Bid.Price > ad.Bid.Price)) {
+								cachedAd[bidder] = adsList[0]
+							}
+						}
+						result.ModuleContext[MSP_MULTI_AD_CACHE_DATA] = cachedAdsList
+					} else {
+						result.ModuleContext[MSP_MULTI_AD_CACHE_DATA] = BiddersAdDataList{}
+					}
+
 				} else {
 					cacheKey := GetAdCachePrimaryKey(userId, placementId)
 					result.ModuleContext[MSP_AD_CACHE_KEY] = cacheKey

pkg/modules/ad_cache/module/module.go

@@ -44,6 +44,7 @@ const (
 	MSP_AD_CACHE_KEY         = "msp-ad-cahe-key"
 	MSP_AD_CACHE_DATA        = "msp-ad-cahe-data"
 	MSP_FORMAT_AD_CACHE_DATA = "msp-format-ad-cache-data"
+	MSP_MULTI_AD_CACHE_DATA  = "msp-multi-ad-cache-data"
 	MSP_PLACEMENT_ID         = "msp-ad-cache-placement"
 	MSP_USER_ID              = "msp-ad-cache-userid"
 	MSP_BIDDER_DYNAMIC_FLOOR = "msp-ad-cache-dynamic-floor"
@@ -54,6 +55,10 @@ var format_cache_placement = map[string][]openrtb_ext.BidType{
 	"msp-android-article-inside-display-exp": {openrtb_ext.BidTypeBanner, openrtb_ext.BidTypeNative},
 }
 
+var multi_cached_ads_placements = []string{
+	"msp-android-foryou-large-display-exp4",
+}
+
 // old version
 var dynamic_floor_experiment_placements = []string{
 	"msp-android-foryou-large-display-exp2",