Fix issues in file system notifications. Fixes #710

.github/workflows/tests-robocode-python-ls-core.yml

@@ -21,19 +21,19 @@ jobs:
  matrix:
  name: [
  "windows-py37",
- "ubuntu-py38",
- "macos-py37"
+ "ubuntu-py39",
+ "macos-py38"
  ]
 
  include:
  - name: "windows-py37"
  python: "3.7"
  os: windows-latest
- - name: "ubuntu-py38"
- python: "3.8"
+ - name: "ubuntu-py39"
+ python: "3.9"
  os: ubuntu-latest
- - name: "macos-py37"
- python: "3.7"
+ - name: "macos-py38"
+ python: "3.8"
  os: macos-latest
 
  steps:

robocorp-python-ls-core/.settings/org.eclipse.core.resources.prefs

@@ -2,5 +2,3 @@ eclipse.preferences.version=1
 encoding//.settings/org.python.pydev.analysis.yaml=UTF-8
 encoding//.settings/org.python.pydev.yaml=UTF-8
 encoding//src/robocorp_ls_core/debug_adapter_core/dap/dap_schema.py=utf-8
-encoding//src/robocorp_ls_core/libs/watchdog_lib/watchdog/events.py=utf-8
-encoding//src/robocorp_ls_core/libs/watchdog_lib/watchdog/observers/api.py=utf-8

robocorp-python-ls-core/src/robocorp_ls_core/libs/fsnotify_lib/fsnotify/__init__.py

@@ -1,4 +1,4 @@
-'''
+"""
 Sample usage to track changes in a thread.
 
  import threading
@@ -36,10 +36,10 @@ def start_watching(): # Called from thread
 
 
 Note: changes are only reported for files (added/modified/deleted), not directories.
-'''
+"""
 import threading
 import sys
-from typing import Tuple
+from typing import Tuple, List
 
 try:
  from os import scandir
@@ -58,13 +58,14 @@ def start_watching(): # Called from thread
  class IntEnum(object):
  pass
 
+
 from collections import deque
 
 import time
 
-__author__ = 'Fabio Zadrozny'
-__email__ = 'fabiofz@gmail.com'
-__version__ = '0.2.1' # Version here and in setup.py
+__author__ = "Fabio Zadrozny"
+__email__ = "fabiofz@gmail.com"
+__version__ = "0.2.1" # Version here and in setup.py
 
 PRINT_SINGLE_POLL_TIME = False
 
@@ -76,7 +77,6 @@ class Change(IntEnum):
 
 
 class _SingleVisitInfo(object):
-
  def __init__(self):
  self.count = 0
  self.visited_dirs = set()
@@ -86,26 +86,35 @@ def __init__(self):
 
 class TrackedPath(object):
 
- __slots__ = ['path', 'recursive']
+ __slots__ = ["path", "recursive"]
 
  def __init__(self, path, recursive):
  self.path = path
  self.recursive = recursive
 
 
 class _PathWatcher(object):
- '''
+ """
  Helper to watch a single path.
- '''
-
- def __init__(self, root_path, accept_directory, accept_file, single_visit_info, max_recursion_level, sleep_time=.0, recursive=True):
- '''
+ """
+
+ def __init__(
+ self,
+ root_path,
+ accept_directory,
+ accept_file,
+ single_visit_info,
+ max_recursion_level,
+ sleep_time=0.0,
+ recursive=True,
+ ):
+ """
  :type root_path: str
  :type accept_directory: Callback[str, bool]
  :type accept_file: Callback[str, bool]
  :type max_recursion_level: int
  :type sleep_time: float
- '''
+ """
  self.accept_directory = accept_directory
  self.accept_file = accept_file
  self._max_recursion_level = max_recursion_level
@@ -117,7 +126,7 @@ def __init__(self, root_path, accept_directory, accept_file, single_visit_info,
  # Watcher.target_time_for_single_scan.
  self.sleep_time = sleep_time
 
- self.sleep_at_elapsed = 1. / 30.
+ self.sleep_at_elapsed = 1.0 / 30.0
 
  # When created, do the initial snapshot right away!
  old_file_to_mtime = {}
@@ -135,9 +144,14 @@ def __ne__(self, o):
  def __hash__(self):
  return hash(self._root_path)
 
- def _check_dir(self, dir_path, single_visit_info, append_change, old_file_to_mtime, level):
+ def _check_dir(
+ self, dir_path, single_visit_info, append_change, old_file_to_mtime, level
+ ):
  # This is the actual poll loop
- if dir_path in single_visit_info.visited_dirs or level > self._max_recursion_level:
+ if (
+ dir_path in single_visit_info.visited_dirs
+ or level > self._max_recursion_level
+ ):
  return
  single_visit_info.visited_dirs.add(dir_path)
  try:
@@ -146,7 +160,7 @@ def _check_dir(self, dir_path, single_visit_info, append_change, old_file_to_mti
  dir_path = dir_path.decode(sys.getfilesystemencoding())
  except UnicodeDecodeError:
  try:
- dir_path = dir_path.decode('utf-8')
+ dir_path = dir_path.decode("utf-8")
  except UnicodeDecodeError:
  return # Ignore if we can't deal with the path.
 
@@ -168,7 +182,13 @@ def _check_dir(self, dir_path, single_visit_info, append_change, old_file_to_mti
  if entry.is_dir():
  if self.accept_directory(entry.path):
  if self._recursive:
- self._check_dir(entry.path, single_visit_info, append_change, old_file_to_mtime, level + 1)
+ self._check_dir(
+ entry.path,
+ single_visit_info,
+ append_change,
+ old_file_to_mtime,
+ level + 1,
+ )
 
  elif self.accept_file(entry.path):
  stat = entry.stat()
@@ -186,14 +206,16 @@ def _check_dir(self, dir_path, single_visit_info, append_change, old_file_to_mti
  pass # Directory was removed in the meanwhile.
 
  def _check(self, single_visit_info, append_change, old_file_to_mtime):
- self._check_dir(self._root_path, single_visit_info, append_change, old_file_to_mtime, 0)
+ self._check_dir(
+ self._root_path, single_visit_info, append_change, old_file_to_mtime, 0
+ )
 
 
 class Watcher(object):
 
  # By default (if accept_directory is not specified), these will be the
  # ignored directories.
- ignored_dirs = {u'.git', u'__pycache__', u'.idea', u'node_modules', u'.metadata'}
+ ignored_dirs = {u".git", u"__pycache__", u".idea", u"node_modules", u".metadata"}
 
  # By default (if accept_file is not specified), these will be the
  # accepted files.
@@ -218,26 +240,31 @@ class Watcher(object):
  max_recursion_level = 10
 
  def __init__(self, accept_directory=None, accept_file=None):
- '''
+ """
  :param Callable[str, bool] accept_directory:
  Callable that returns whether a directory should be watched.
  Note: if passed it'll override the `ignored_dirs`
 
  :param Callable[str, bool] accept_file:
  Callable that returns whether a file should be watched.
  Note: if passed it'll override the `accepted_file_extensions`.
- '''
+ """
  self._lock = threading.Lock()
 
  self._path_watchers = set()
  self._disposed = threading.Event()
 
  if accept_directory is None:
  from os.path import basename
- accept_directory = lambda dir_path: basename(dir_path) not in self.ignored_dirs
+
+ accept_directory = (
+ lambda dir_path: basename(dir_path) not in self.ignored_dirs
+ )
  if accept_file is None:
- accept_file = lambda path_name: \
- not self.accepted_file_extensions or path_name.endswith(self.accepted_file_extensions)
+ accept_file = (
+ lambda path_name: not self.accepted_file_extensions
+ or path_name.endswith(self.accepted_file_extensions)
+ )
  self.accept_file = accept_file
  self.accept_directory = accept_directory
  self._single_visit_info = _SingleVisitInfo()
@@ -277,29 +304,48 @@ def set_tracked_paths(self, paths):
  if not isinstance(paths, (list, tuple, set)):
  paths = (paths,)
 
- def key(path_or_str):
- if isinstance(path_or_str, TrackedPath):
- return -len(path_or_str.path)
- return -len(path_or_str)
+ new_paths = []
+ path_to_recursive = {}
+ for p in paths:
+ if isinstance(p, str):
+ p = TrackedPath(p, True)
+ new_paths.append(p)
+ curr = path_to_recursive.get(p.path)
+
+ if curr is None:
+ path_to_recursive[p.path] = p.recursive
+ else:
+ if p.recursive:
+ path_to_recursive[p.path] = True
+ else:
+ if curr:
+ pass # Already recursive, don't override with non-recursive.
+ else:
+ # Set to not recursive.
+ path_to_recursive[p.path] = False
+
+ def key(path):
+ return -len(path)
 
  # Sort by the path len so that the bigger paths come first (so,
  # if there's any nesting we want the nested paths to be visited
  # before the parent paths so that the max_recursion_level is correct).
- paths = sorted(set(paths), key=key)
+ paths: List[str] = sorted(set(p.path for p in paths), key=key)
  path_watchers = set()
 
  single_visit_info = _SingleVisitInfo()
 
+ path: str
  for path in paths:
- sleep_time = 0. # When collecting the first time, sleep_time should be 0!
+ sleep_time = 0.0 # When collecting the first time, sleep_time should be 0!
  path_watcher = _PathWatcher(
- path.path if isinstance(path, TrackedPath) else path,
+ path,
  self.accept_directory,
  self.accept_file,
  single_visit_info,
  max_recursion_level=self.max_recursion_level,
  sleep_time=sleep_time,
- recursive=path.recursive if isinstance(path, TrackedPath) else path
+ recursive=path_to_recursive.get(path, True),
  )
 
  path_watchers.add(path_watcher)
@@ -309,13 +355,13 @@ def key(path_or_str):
  self._path_watchers = path_watchers
 
  def iter_changes(self):
- '''
+ """
  Continuously provides changes (until dispose() is called).
 
  Changes provided are tuples with the Change enum and filesystem path.
 
  :rtype: Iterable[Tuple[Change, str]]
- '''
+ """
  while not self._disposed.is_set():
 
  with self._lock:
@@ -338,9 +384,9 @@ def iter_changes(self):
  for change in changes:
  yield change
 
- actual_time = (time.time() - initial_time)
+ actual_time = time.time() - initial_time
  if self.print_poll_time:
- print('--- Total poll time: %.3fs' % actual_time)
+ print("--- Total poll time: %.3fs" % actual_time)
 
  if actual_time > 0:
  if self.target_time_for_single_scan <= 0.0:
@@ -353,8 +399,8 @@ def iter_changes(self):
  # direction).
  # (to prevent from cases where the user puts the machine on sleep and
  # values become too skewed).
- if perc > 2.:
- perc = 2.
+ if perc > 2.0:
+ perc = 2.0
  elif perc < 0.5:
  perc = 0.5
 
@@ -368,7 +414,13 @@ def iter_changes(self):
  # (to prevent from cases where the user puts the machine on sleep and
  # values become too skewed).
  diff_sleep_time = new_sleep_time - path_watcher.sleep_time
- path_watcher.sleep_time += (diff_sleep_time / (3.0 * len(self._path_watchers)))
+ path_watchers_len = len(self._path_watchers)
+ if not path_watchers_len:
+ continue
+
+ path_watcher.sleep_time += diff_sleep_time / (
+ 3.0 * path_watchers_len
+ )
 
  if actual_time > 0:
  self._disposed.wait(actual_time)
@@ -379,6 +431,5 @@ def iter_changes(self):
  # print('new sleep time: %s' % path_watcher.sleep_time)
 
  diff = self.target_time_for_notification - actual_time
- if diff > 0.:
+ if diff > 0.0:
  self._disposed.wait(diff)
-

robocorp-python-ls-core/src/robocorp_ls_core/libs/watchdog_lib/README.md

@@ -1,13 +1,14 @@
 The files in this directory contain watchdog and the needed deps.
 
-To update, erase it and run (note: using 0.10.3 due to https://github.com/gorakhargosh/watchdog/issues/706 in 0.10.4):
+To update, erase it and run:
 
-pip install watchdog==0.10.3 --target .
+pip install watchdog --target .
 
 Then:
 - Remove bin
 - commit the code
 - Verify if the license brought in from the libraries downloaded are OK
+- Make sure that there's a `watchdog_lib/__init__.py` (so that it's added during setup).
 
 Then, go on to:
 
@@ -16,8 +17,8 @@ the `_watchdog_fsevents.xxx.so` to the watchdog_lib.
 
 Then update the permissions of the file:
 
-git update-index --add --chmod=+x _watchdog_fsevents.cpython-37m-darwin.so
 git update-index --add --chmod=+x _watchdog_fsevents.cpython-38-darwin.so
 git update-index --add --chmod=+x _watchdog_fsevents.cpython-39-darwin.so
+git update-index --add --chmod=+x _watchdog_fsevents.cpython-310-darwin.so
 
 This folder should be automatically added to the PYTHONPATH when needed.