Merge dev to master (#4889)

* Expand simple logging options (#4832)

* Fix bot crash at start on permaban

* Expanded logging options

Added "logging" section to config, with options "color",
"show_datetime", "show_process_name" and "show_log_level"

* Added warning about deprecated logging_color arg

* Display log message moved

No point trying to use the logger before it's been initialised. Moved to
init_config.

* Remove milliseconds from datetime

Because really, do we need that?

* Reversed condition order for clarity

First check: "if not in config", OR
Second check: "is in config AND set to true"

If either condition matches, the logging detail will be displayed.

* Documented new log options

* Modified conditions again

Removed unnecessary second check for config values and slightly modified
parentheses as per suggestion from @mjmadsen

* Changed ) to } (#4845)

Fixed an faulty character

* fix incubator logic (#4848)

* corrected logic to respect snipe = true

* Update configuration_files.md (#4854)

* corrected logic to respect snipe = true (#4855)

* Revert "corrected logic to respect snipe = true" (#4857)

* dont forget to update the docs when adding config changes... (#4856)

* dont forget to update the docs when adding config changes...

* reflect config changes....

* please keep this as is (#4859)

Add stuff in the right order.

* Clarify Max_distance for Sniping (#4858)

* Clarify Max_distance

* Added distance unit and updated configuration_files.md

* - debug improvements for MoveToMap (#4860)

- fix for Telegram to accept "@username" as "master", too, along with numeric  IDs

* Fixes catch rates. (#4863)

* Implemented more granularity in the "alert_catch" parameter for Telegram
alerts.

* Add exceptions to json file read/writes (#4877)

* Fix bot crash at start on permaban

* Expanded logging options

Added "logging" section to config, with options "color",
"show_datetime", "show_process_name" and "show_log_level"

* Added warning about deprecated logging_color arg

* Display log message moved

No point trying to use the logger before it's been initialised. Moved to
init_config.

* Remove milliseconds from datetime

Because really, do we need that?

* Reversed condition order for clarity

First check: "if not in config", OR
Second check: "is in config AND set to true"

If either condition matches, the logging detail will be displayed.

* Documented new log options

* Modified conditions again

Removed unnecessary second check for config values and slightly modified
parentheses as per suggestion from @mjmadsen

* Add exception handling to json file read/write ops

* Removed API call in update live stats

Instead of making a new api call, utilise stats already contained in
metrics.

* Incubate eggs fix (#4881)

* Fixed incubator_eggs wrong print

* Fixed pokemon hatched from eggs not added to cached inventory

* Fix

* Fixed not using breakable incubators

* Fixed error adding pokemon to cached inventory

* Moved remove egg and add Pokemon to _hatch_eggs

configs/config.json.example

@@ -12,7 +12,12 @@
         "config": {
           "enabled": false,
           "master": null,
-          "alert_catch": ["all"]
+          "// old syntax, still supported: alert_catch": ["all"],
+	  "// new syntax:": {},
+	  "alert_catch": {
+		  "all": {"operator": "and", "cp": 1300, "iv": 0.95},
+		  "Snorlax": {"operator": "or", "cp": 900, "iv": 0.9}
+	  }
         }
       },
       {

pokemongo_bot/__init__.py

@@ -41,6 +41,9 @@
 from pgoapi.protos.POGOProtos.Enums import BadgeType_pb2
 import struct
 
+class FileIOException(Exception):
+    pass
+
 class PokemonGoBot(Datastore):
     @property
     def position(self):
@@ -1095,9 +1098,19 @@ def _set_starting_position(self):
                     level='debug',
                     formatted='Loading cached location...'
                 )
-                with open(os.path.join(_base_dir, 'data', 'last-location-%s.json' %
-                    self.config.username)) as f:
-                    location_json = json.load(f)
+
+                json_file = os.path.join(_base_dir, 'data', 'last-location-%s.json' % self.config.username)
+
+                try:
+                    with open(json_file, "r") as infile:
+                        location_json = json.load(infile)
+                except (IOError, ValueError):
+                    # Unable to read json file.
+                    # File may be corrupt. Create a new one.            
+                    location_json = []
+                except:
+                    raise FileIOException("Unexpected error reading from {}".web_inventory)
+
                 location = (
                     location_json['lat'],
                     location_json['lng'],

pokemongo_bot/cell_workers/incubate_eggs.py

@@ -1,5 +1,6 @@
 from datetime import datetime, timedelta
 
+from pokemongo_bot import inventory
 from pokemongo_bot.human_behaviour import sleep
 from pokemongo_bot.base_task import BaseTask
 
@@ -35,15 +36,18 @@ def work(self):
         except:
             return
 
+        should_print = self._should_print()
+
         if self.used_incubators and IncubateEggs.last_km_walked != self.km_walked:
             self.used_incubators.sort(key=lambda x: x.get("km"))
             km_left = self.used_incubators[0]['km']-self.km_walked
             if km_left <= 0:
                 self._hatch_eggs()
+                should_print = False
             else:
                 self.bot.metrics.next_hatching_km(km_left)
 
-        if self._should_print():
+        if should_print:
             self._print_eggs()
             self._compute_next_update()
 
@@ -58,9 +62,9 @@ def work(self):
 
         if self.ready_breakable_incubators:
             # get available eggs
-            eggs = self._filter_sort_eggs(self.infinite_incubator,
-                    self.infinite_longer_eggs_first)
-            self._apply_incubators(eggs, self.ready_infinite_incubators)
+            eggs = self._filter_sort_eggs(self.breakable_incubator,
+                    self.breakable_longer_eggs_first)
+            self._apply_incubators(eggs, self.ready_breakable_incubators)
 
 
     def _filter_sort_eggs(self, allowed, sorting):
@@ -203,13 +207,15 @@ def _hatch_eggs(self):
         candy = result.get('candy_awarded', "error")
         xp = result.get('experience_awarded', "error")
         sleep(self.hatching_animation_delay)
-        self.bot.latest_inventory = None
         try:
             pokemon_data = self._check_inventory(pokemon_ids)
             for pokemon in pokemon_data:
                 # pokemon ids seem to be offset by one
                 if pokemon['pokemon_id']!=-1:
                     pokemon['name'] = self.bot.pokemon_list[(pokemon.get('pokemon_id')-1)]['Name']
+                    #remove as egg and add as pokemon
+                    inventory.pokemons().remove(pokemon['id'])
+                    inventory.pokemons().add(inventory.Pokemon(pokemon))
                 else:
                     pokemon['name'] = "error"
         except:

pokemongo_bot/cell_workers/pokemon_optimizer.py

@@ -96,30 +96,7 @@ def open_inventory(self):
             self.family_by_family_id.setdefault(family_id, []).append(pokemon)
 
     def save_web_inventory(self):
-        web_inventory = os.path.join(_base_dir, "web", "inventory-%s.json" % self.bot.config.username)
-
-        with open(web_inventory, "r") as infile:
-            ii = json.load(infile)
-
-        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("pokedex_entry", None)]
-        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("candy", None)]
-        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("item", None)]
-        ii = [x for x in ii if not x.get("inventory_item_data", {}).get("pokemon_data", None)]
-
-        for pokedex in inventory.pokedex().all():
-            ii.append({"inventory_item_data": {"pokedex_entry": pokedex}})
-
-        for family_id, candy in inventory.candies()._data.items():
-            ii.append({"inventory_item_data": {"candy": {"family_id": family_id, "candy": candy.quantity}}})
-
-        for item_id, item in inventory.items()._data.items():
-            ii.append({"inventory_item_data": {"item": {"item_id": item_id, "count": item.count}}})
-
-        for pokemon in inventory.pokemons().all():
-            ii.append({"inventory_item_data": {"pokemon_data": pokemon._data}})
-
-        with open(web_inventory, "w") as outfile:
-            json.dump(ii, outfile)
+        inventory.update_web_inventory()
 
     def get_family_optimized(self, family_id, family):
         evolve_best = []

pokemongo_bot/cell_workers/telegram_task.py

@@ -10,6 +10,9 @@
 from pprint import pprint
 import re
 
+class FileIOException(Exception):
+    pass
+
 class TelegramTask(BaseTask):
     SUPPORTED_TASK_API_VERSION = 1
     update_id = None
@@ -18,7 +21,6 @@ class TelegramTask(BaseTask):
     def initialize(self):
         if not self.enabled:
             return
-        self.logger = logging.getLogger(type(self).__name__)
         api_key = self.bot.config.telegram_token
         if api_key == None:
             self.emit_event(
@@ -40,7 +42,7 @@ def work(self):
         for update in self.tbot.getUpdates(offset=self.update_id, timeout=10):
             self.update_id = update.update_id+1
             if update.message:
-                self.logger.info("message from {} ({}): {}".format(update.message.from_user.username, update.message.from_user.id, update.message.text))
+                self.bot.logger.info("message from {} ({}): {}".format(update.message.from_user.username, update.message.from_user.id, update.message.text))
                 if self.config.get('master',None) and self.config.get('master',None) not in [update.message.from_user.id, "@{}".format(update.message.from_user.username)]:
                     self.emit_event( 
                             'debug', 
@@ -89,9 +91,18 @@ def _get_player_stats(self):
         :rtype: dict
         """
         web_inventory = os.path.join(_base_dir, "web", "inventory-%s.json" % self.bot.config.username)
-        with open(web_inventory, "r") as infile:
-            json_inventory = json.load(infile)
-            infile.close()
+
+        try:
+            with open(web_inventory, "r") as infile:
+                json_inventory = json.load(infile)
+        except ValueError:
+            # Unable to read json from web inventory
+            # File may be corrupt. Create a new one.  
+            self.bot.logger.info('[x] Error while opening inventory file for read: %s' % e)
+            json_inventory = []
+        except:
+            raise FileIOException("Unexpected error reading from {}".web_inventory)
+
         return next((x["inventory_item_data"]["player_stats"]
                      for x in json_inventory
                      if x.get("inventory_item_data", {}).get("player_stats", {})),

pokemongo_bot/cell_workers/update_live_stats.py

@@ -1,6 +1,8 @@
 import ctypes
 import json
 import os
+import logging
+
 from sys import stdout, platform as _platform
 from datetime import datetime, timedelta
 
@@ -12,6 +14,9 @@
 # XP file
 import json
 
+class FileIOException(Exception):
+    pass
+
 class UpdateLiveStats(BaseTask):
     """
     Periodically displays stats about the bot in the terminal and/or in its title.
@@ -420,25 +425,31 @@ def _get_player_stats(self):
         :rtype: dict
         """
         # TODO : find a better solution than calling the api
-        inventory_items = self.bot.api.get_inventory() \
-            .get('responses', {}) \
-            .get('GET_INVENTORY', {}) \
-            .get('inventory_delta', {}) \
-            .get('inventory_items', {})
-        return next((x["inventory_item_data"]["player_stats"]
-                     for x in inventory_items
-                     if x.get("inventory_item_data", {}).get("player_stats", {})),
-                    None)
-
+        return self.bot.metrics.player_stats
+
     def update_web_stats(self,player_data):
         web_inventory = os.path.join(_base_dir, "web", "inventory-%s.json" % self.bot.config.username)
 
-        with open(web_inventory, "r") as infile:
-            json_stats = json.load(infile)
+        try:
+            with open(web_inventory, "r") as infile:
+                json_stats = json.load(infile)
+        except (IOError, ValueError):
+            # Unable to read json from web inventory
+            # File may be corrupt. Create a new one.
+            self.bot.logger.info('[x] Error while opening inventory file for read: %s' % e, 'red')
+            json_stats = []
+        except:
+            raise FileIOException("Unexpected error loading information from json.")
 
         json_stats = [x for x in json_stats if not x.get("inventory_item_data", {}).get("player_stats", None)]
 
         json_stats.append({"inventory_item_data": {"player_stats": player_data}})
 
-        with open(web_inventory, "w") as outfile:
-            json.dump(json_stats, outfile)
+        try:
+            with open(web_inventory, "w") as outfile:
+                json.dump(json_stats, outfile)
+        except (IOError, ValueError):
+            self.bot.logger.info('[x] Error while opening inventory file for write: %s' % e, 'red')
+            pass
+        except:
+            raise FileIOException("Unexpected error writing to {}".web_inventory)

pokemongo_bot/event_handlers/telegram_handler.py

@@ -21,10 +21,22 @@ def handle_event(self, event, sender, level, formatted_msg, data):
             if event == 'level_up':
                 msg = "level up ({})".format(data["current_level"])
             elif event == 'pokemon_caught':
-                if data["pokemon"] in self.pokemons or self.pokemons[0]=="all":
-                    msg = "Caught {} CP: {}, IV: {}".format(data["pokemon"],data["cp"],data["iv"])
+                if isinstance(self.pokemons, list):
+                    if data["pokemon"] in self.pokemons or "all" in self.pokemons:
+                        msg = "Caught {} CP: {}, IV: {}".format(data["pokemon"],data["cp"],data["iv"])
+                    else:
+                        return
                 else:
-                    return
+                    if data["pokemon"] in self.pokemons:
+                        trigger = self.pokemons[data["pokemon"]]
+                    elif "all" in self.pokemons:
+                        trigger = self.pokemons["all"]
+                    else:
+                        return
+                    if (not "operator" in trigger or trigger["operator"] == "and") and data["cp"] >= trigger["cp"] and data["iv"] >= trigger["iv"] or ("operator" in trigger and trigger["operator"] == "or" and (data["cp"] >= trigger["cp"] or data["iv"] >= trigger["iv"])):
+                        msg = "Caught {} CP: {}, IV: {}".format(data["pokemon"],data["cp"],data["iv"])
+                    else:
+                        return
             else:
                 return
             self.tbot.sendMessage(chat_id=master, parse_mode='Markdown', text=msg)

pokemongo_bot/inventory.py

@@ -14,6 +14,9 @@
 https://www.reddit.com/r/pokemongodev/comments/4w7mdg/combat_damage_calculation_formula_exactly/
 '''
 
+class FileIOException(Exception):
+    pass
+
 
 #
 # Abstraction
@@ -1122,6 +1125,8 @@ def init_inventory_outfile(self):
         web_inventory = os.path.join(_base_dir, "web", "inventory-%s.json" % self.bot.config.username)
 
         if not os.path.exists(web_inventory):
+            self.bot.logger.info('No inventory file %s found. Creating a new one' % web_inventory)
+
             json_inventory = []
 
             with open(web_inventory, "w") as outfile:
@@ -1134,9 +1139,16 @@ def update_web_inventory(self):
         if not os.path.exists(web_inventory):
             self.init_inventory_outfile()
 
-        with open(web_inventory, "r") as infile:
-            json_inventory = json.load(infile)
-            infile.close()
+        try:
+            with open(web_inventory, "r") as infile:
+                json_inventory = json.load(infile)
+        except (IOError, ValueError):
+            # Unable to read json from web inventory
+            # File may be corrupt. Create a new one.            
+            self.bot.logger.info('[x] Error while opening inventory file for read: %s' % e, 'red')
+            json_inventory = []
+        except:
+            raise FileIOException("Unexpected error reading from {}".web_inventory)
 
         json_inventory = [x for x in json_inventory if not x.get("inventory_item_data", {}).get("pokedex_entry", None)]
         json_inventory = [x for x in json_inventory if not x.get("inventory_item_data", {}).get("candy", None)]
@@ -1145,9 +1157,14 @@ def update_web_inventory(self):
 
         json_inventory = json_inventory + self.jsonify_inventory()
 
-        with open(web_inventory, "w") as outfile:
-            json.dump(json_inventory, outfile)
-            outfile.close()
+        try:
+            with open(web_inventory, "w") as outfile:
+                json.dump(json_inventory, outfile)
+        except (IOError, ValueError):
+            self.bot.logger.info('[x] Error while opening inventory file for write: %s' % e, 'red')
+            pass
+        except:
+            raise FileIOException("Unexpected error writing to {}".web_inventory)
 
     def jsonify_inventory(self):
         json_inventory = []

pokemongo_bot/metrics.py

@@ -24,6 +24,9 @@ def __init__(self, bot):
 
         self.uniq_pokemons_caught = None
         self.uniq_pokemons_list = None
+
+        self.player_stats = []
+        self.inventory_data = []
 
     def runtime(self):
         return timedelta(seconds=round(time.time() - self.start_time))
@@ -114,6 +117,7 @@ def capture_stats(self):
                 if 'inventory_item_data' in item:
                     if 'player_stats' in item['inventory_item_data']:
                         playerdata = item['inventory_item_data']['player_stats']
+                        self.player_stats = playerdata
 
                         self.xp['latest'] = playerdata.get('experience', 0)
                         if self.xp['start'] < 0: self.xp['start'] = self.xp['latest']