Merge pull request #1874 from wowsims/fixes

A bit of organizing in db generator

tools/generate_items/database.go

@@ -24,28 +24,6 @@ func EnchantToDBKey(enchant *proto.UIEnchant) EnchantDBKey {
  }
 }
 
-func mergeItemProtos(dst, src *proto.UIItem) {
- // googleproto.Merge concatenates lists but we want replacement, so do them manually.
- if src.Stats != nil {
- dst.Stats = src.Stats
- src.Stats = nil
- }
- if src.SocketBonus != nil {
- dst.SocketBonus = src.SocketBonus
- src.SocketBonus = nil
- }
- googleProto.Merge(dst, src)
-}
-
-func mergeGemProtos(dst, src *proto.UIGem) {
- // googleproto.Merge concatenates lists but we want replacement, so do them manually.
- if src.Stats != nil {
- dst.Stats = src.Stats
- src.Stats = nil
- }
- googleProto.Merge(dst, src)
-}
-
 type WowDatabase struct {
  items map[int32]*proto.UIItem
  enchants map[EnchantDBKey]*proto.UIEnchant
@@ -75,11 +53,6 @@ func NewWowDatabase(itemOverrides []*proto.UIItem, gemOverrides []*proto.UIGem,
  db.items[itemProto.Id] = itemProto
  }
  }
- for _, itemOverride := range itemOverrides {
- if _, ok := db.items[itemOverride.Id]; ok {
- mergeItemProtos(db.items[itemOverride.Id], itemOverride)
- }
- }
 
  for id, response := range itemTooltipsDB {
  if response.IsGem() {
@@ -88,15 +61,11 @@ func NewWowDatabase(itemOverrides []*proto.UIItem, gemOverrides []*proto.UIGem,
  db.gems[gemProto.Id] = gemProto
  }
  }
- for _, gemOverride := range gemOverrides {
- if _, ok := db.gems[gemOverride.Id]; ok {
- mergeGemProtos(db.gems[gemOverride.Id], gemOverride)
- }
- }
 
- for _, enchant := range enchantOverrides {
- db.enchants[EnchantToDBKey(enchant)] = enchant
- }
+ db.MergeItems(itemOverrides)
+ db.MergeGems(gemOverrides)
+ db.MergeEnchants(enchantOverrides)
+
  for _, enchant := range db.enchants {
  if enchant.ItemId != 0 {
  if tooltip, ok := itemTooltipsDB[enchant.ItemId]; ok {
@@ -111,18 +80,85 @@ func NewWowDatabase(itemOverrides []*proto.UIItem, gemOverrides []*proto.UIGem,
  }
 
  for _, itemID := range extraItemIcons {
- if itemID != 0 {
- if tooltip, ok := itemTooltipsDB[itemID]; ok {
- db.itemIcons[itemID] = &proto.IconData{Id: itemID, Name: tooltip.GetName(), Icon: tooltip.GetIcon()}
- }
+ if tooltip, ok := itemTooltipsDB[itemID]; ok {
+ db.itemIcons[itemID] = &proto.IconData{Id: itemID, Name: tooltip.GetName(), Icon: tooltip.GetIcon()}
  }
+ //if item, ok := db.items[itemID]; ok {
+ // db.itemIcons[itemID] = &proto.IconData{Id: itemID, Name: item.Name, Icon: item.Icon}
+ //}
  }
 
- db.applyGlobalFilters()
-
  return db
 }
 
+func (db *WowDatabase) MergeItems(arr []*proto.UIItem) {
+ for _, item := range arr {
+ db.MergeItem(item)
+ }
+}
+func (db *WowDatabase) MergeItem(newItem *proto.UIItem) {
+ if curItem, ok := db.items[newItem.Id]; ok {
+ mergeItemProtos(curItem, newItem)
+ } else {
+ db.items[newItem.Id] = newItem
+ }
+}
+func mergeItemProtos(dst, src *proto.UIItem) {
+ // googleproto.Merge concatenates lists but we want replacement, so do them manually.
+ if src.Stats != nil {
+ dst.Stats = src.Stats
+ src.Stats = nil
+ }
+ if src.SocketBonus != nil {
+ dst.SocketBonus = src.SocketBonus
+ src.SocketBonus = nil
+ }
+ googleProto.Merge(dst, src)
+}
+
+func (db *WowDatabase) MergeEnchants(arr []*proto.UIEnchant) {
+ for _, enchant := range arr {
+ db.MergeEnchant(enchant)
+ }
+}
+func (db *WowDatabase) MergeEnchant(newEnchant *proto.UIEnchant) {
+ key := EnchantToDBKey(newEnchant)
+ if curEnchant, ok := db.enchants[key]; ok {
+ mergeEnchantProtos(curEnchant, newEnchant)
+ } else {
+ db.enchants[key] = newEnchant
+ }
+}
+func mergeEnchantProtos(dst, src *proto.UIEnchant) {
+ // googleproto.Merge concatenates lists but we want replacement, so do them manually.
+ if src.Stats != nil {
+ dst.Stats = src.Stats
+ src.Stats = nil
+ }
+ googleProto.Merge(dst, src)
+}
+
+func (db *WowDatabase) MergeGems(arr []*proto.UIGem) {
+ for _, gem := range arr {
+ db.MergeGem(gem)
+ }
+}
+func (db *WowDatabase) MergeGem(newGem *proto.UIGem) {
+ if curGem, ok := db.gems[newGem.Id]; ok {
+ mergeGemProtos(curGem, newGem)
+ } else {
+ db.gems[newGem.Id] = newGem
+ }
+}
+func mergeGemProtos(dst, src *proto.UIGem) {
+ // googleproto.Merge concatenates lists but we want replacement, so do them manually.
+ if src.Stats != nil {
+ dst.Stats = src.Stats
+ src.Stats = nil
+ }
+ googleProto.Merge(dst, src)
+}
+
 // Filters out entities which shouldn't be included anywhere.
 func (db *WowDatabase) applyGlobalFilters() {
  db.items = core.FilterMap(db.items, func(_ int32, item *proto.UIItem) bool {

tools/generate_items/main.go

@@ -19,5 +19,6 @@ func main() {
  getWowheadTooltipsDB("./assets/item_data/all_item_tooltips.csv"),
  getWowheadTooltipsDB("./assets/spell_data/all_spell_tooltips.csv"))
 
+ db.applyGlobalFilters()
  writeDatabaseFile(db)
 }

tools/generate_items/readers.go

@@ -15,17 +15,9 @@ import (
 // Returns the prefetched list of all wowhead tooltips.
 // Maps item IDs to tooltip strings.
 func getWowheadTooltipsDB(filepath string) map[int32]WowheadItemResponse {
- file, err := os.Open(filepath)
- if err != nil {
- log.Fatalf("Failed to open %s: %s", filepath, err)
- }
- defer file.Close()
-
+ lines := readFileLines(filepath)
  db := make(map[int32]WowheadItemResponse)
- scanner := bufio.NewScanner(file)
- for scanner.Scan() {
- line := scanner.Text()
-
+ for _, line := range lines {
  itemIDStr := line[:strings.Index(line, ",")]
  itemID, err := strconv.Atoi(itemIDStr)
  if err != nil {
@@ -40,6 +32,22 @@ func getWowheadTooltipsDB(filepath string) map[int32]WowheadItemResponse {
  return db
 }
 
+func readFileLines(filePath string) []string {
+ file, err := os.Open(filePath)
+ if err != nil {
+ log.Fatalf("Failed to open %s: %s", filePath, err)
+ }
+ defer file.Close()
+
+ lines := []string{}
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ lines = append(lines, scanner.Text())
+ }
+
+ return lines
+}
+
 func readCsvFile(filePath string) [][]string {
  f, err := os.Open(filePath)
  if err != nil {

tools/generate_items/wotlk_api.go

@@ -160,35 +160,25 @@ func (item WotlkItemResponse) GetClassAllowlist() []proto.Class {
  return allowlist
 }
 
-func (item WotlkItemResponse) IsEquippable() bool {
- found := false
- for _, pattern := range requiredEquippableRegexes {
- if pattern.MatchString(item.Tooltip) {
- found = true
- }
- }
- if !found {
- return false
- }
-
- for _, pattern := range nonEquippableRegexes {
- if pattern.MatchString(item.Tooltip) {
- return false
- }
- }
-
- return true
-}
-
 func (item WotlkItemResponse) IsPattern() bool {
- for _, pattern := range nonEquippableRegexes {
+ for _, pattern := range patternRegexes {
  if pattern.MatchString(item.Tooltip) {
  return true
  }
  }
  return false
 }
 
+func (item WotlkItemResponse) IsRandomEnchant() bool {
+ return randomEnchantRegex.MatchString(item.Tooltip)
+}
+
+func (item WotlkItemResponse) IsEquippable() bool {
+ return item.GetItemType() != proto.ItemType_ItemTypeUnknown &&
+ !item.IsPattern() &&
+ !item.IsRandomEnchant()
+}
+
 var wotlkItemLevelRegex = regexp.MustCompile("Item Level ([0-9]+)<")
 
 func (item WotlkItemResponse) GetItemLevel() int {
@@ -225,7 +215,7 @@ func (item WotlkItemResponse) GetItemType() proto.ItemType {
  return itemType
  }
  }
- panic("Could not find item type from tooltip: " + item.Tooltip)
+ return proto.ItemType_ItemTypeUnknown
 }
 
 var wotlkArmorTypePatterns = map[proto.ArmorType]*regexp.Regexp{

tools/generate_items/wowhead_api.go

@@ -242,68 +242,33 @@ func (item WowheadItemResponse) GetClassAllowlist() []proto.Class {
  return allowlist
 }
 
-// At least one of these regexes must be present for the item to be equippable.
-var requiredEquippableRegexes = []*regexp.Regexp{
- regexp.MustCompile(`<td>Head</td>`),
- regexp.MustCompile(`<td>Neck</td>`),
- regexp.MustCompile(`<td>Shoulder</td>`),
- regexp.MustCompile(`<td>Back</td>`),
- regexp.MustCompile(`<td>Chest</td>`),
- regexp.MustCompile(`<td>Wrist</td>`),
- regexp.MustCompile(`<td>Hands</td>`),
- regexp.MustCompile(`<td>Waist</td>`),
- regexp.MustCompile(`<td>Legs</td>`),
- regexp.MustCompile(`<td>Feet</td>`),
- regexp.MustCompile(`<td>Finger</td>`),
- regexp.MustCompile(`<td>Trinket</td>`),
- regexp.MustCompile(`<td>Ranged</td>`),
- regexp.MustCompile(`<td>Thrown</td>`),
- regexp.MustCompile(`<td>Relic</td>`),
- regexp.MustCompile(`<td>Main Hand</td>`),
- regexp.MustCompile(`<td>Two-Hand</td>`),
- regexp.MustCompile(`<td>One-Hand</td>`),
- regexp.MustCompile(`<td>Off Hand</td>`),
- regexp.MustCompile(`<td>Held In Off-hand</td>`),
- regexp.MustCompile(`<td>Held In Off-Hand</td>`),
-}
-
-// If any of these regexes are present, the item is not equippable.
-var nonEquippableRegexes = []*regexp.Regexp{
+var patternRegexes = []*regexp.Regexp{
  regexp.MustCompile(`Design:`),
  regexp.MustCompile(`Recipe:`),
  regexp.MustCompile(`Pattern:`),
  regexp.MustCompile(`Plans:`),
  regexp.MustCompile(`Schematic:`),
- regexp.MustCompile(`Random enchantment`),
 }
 
-func (item WowheadItemResponse) IsEquippable() bool {
- found := false
- for _, pattern := range requiredEquippableRegexes {
+func (item WowheadItemResponse) IsPattern() bool {
+ for _, pattern := range patternRegexes {
  if pattern.MatchString(item.Tooltip) {
- found = true
+ return true
  }
  }
- if !found {
- return false
- }
+ return false
+}
 
- for _, pattern := range nonEquippableRegexes {
- if pattern.MatchString(item.Tooltip) {
- return false
- }
- }
+var randomEnchantRegex = regexp.MustCompile(`Random enchantment`)
 
- return true
+func (item WowheadItemResponse) IsRandomEnchant() bool {
+ return randomEnchantRegex.MatchString(item.Tooltip)
 }
 
-func (item WowheadItemResponse) IsPattern() bool {
- for _, pattern := range nonEquippableRegexes {
- if pattern.MatchString(item.Tooltip) {
- return true
- }
- }
- return false
+func (item WowheadItemResponse) IsEquippable() bool {
+ return item.GetItemType() != proto.ItemType_ItemTypeUnknown &&
+ !item.IsPattern() &&
+ !item.IsRandomEnchant()
 }
 
 var itemLevelRegex = regexp.MustCompile(`Item Level <!--ilvl-->([0-9]+)<`)
@@ -371,7 +336,7 @@ func (item WowheadItemResponse) GetItemType() proto.ItemType {
  return itemType
  }
  }
- panic("Could not find item type from tooltip: " + item.Tooltip)
+ return proto.ItemType_ItemTypeUnknown
 }
 
 var armorTypePatterns = map[proto.ArmorType]*regexp.Regexp{