general: prefer proper PIP packages to cloned repos (impacts reddit/hypothesis/pocket/instapaper/github)

doc/MODULES.org

@@ -150,10 +150,6 @@ for cls, p in modules:
 
  # path[s]/glob to the exported JSON data
  export_path: Paths
-
- # path to a local clone of rexport
- # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/rexport
- rexport : Optional[PathIsh] = None
  #+end_src
 ** [[file:../my/pocket.py][my.pocket]]
 
@@ -167,10 +163,6 @@ for cls, p in modules:
 
  # paths[s]/glob to the exported JSON data
  export_path: Paths
-
- # path to a local clone of pockexport
- # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/pockexport
- pockexport : Optional[PathIsh] = None
  #+end_src
 ** [[file:../my/twitter/twint.py][my.twitter.twint]]
 
@@ -247,10 +239,6 @@ for cls, p in modules:
  # path[s]/glob to the exported JSON data
  export_path: Paths
 
- # path to a local clone of ghexport
- # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/ghexport
- ghexport : Optional[PathIsh] = None
-
  # path to a cache directory
  # if omitted, will use /tmp
  cache_dir: Optional[PathIsh] = None

doc/example_config/my/config/__init__.py

@@ -11,3 +11,12 @@ class hypothesis:
 
 class instapaper:
  export_path: Paths = ''
+
+class pocket:
+ export_path: Paths = ''
+
+class github:
+ export_path: Paths = ''
+
+class reddit:
+ export_path: Paths = ''

my/github/ghexport.py

@@ -20,23 +20,9 @@ class github(user_config):
  # path[s]/glob to the exported JSON data
  export_path: Paths
 
- # path to a local clone of ghexport
- # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/ghexport
- ghexport : Optional[PathIsh] = None
-
  # path to a cache directory
  # if omitted, will use /tmp
  cache_dir: Optional[PathIsh] = None
-
- @property
- def dal_module(self):
- rpath = self.ghexport
- if rpath is not None:
- from ..core.common import import_dir
- return import_dir(rpath, '.dal')
- else:
- import my.config.repos.ghexport.dal as dal
- return dal
 ###
 
 # TODO perhaps using /tmp in case of None isn't ideal... maybe it should be treated as if cache is off
@@ -52,11 +38,11 @@ def migration(attrs: Attrs) -> Attrs:
 config = make_config(github, migration=migration)
 
 
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
- import my.config.repos.ghexport.dal as dal
-else:
- dal = config.dal_module
+try:
+ from ghexport import dal
+except ModuleNotFoundError as e:
+ from ..core.compat import pre_pip_dal_handler
+ dal = pre_pip_dal_handler('ghexport', e, config, requires=REQUIRES)
 
 ############################
 

my/hypothesis.py

@@ -1,6 +1,9 @@
 """
 [[https://hypothes.is][Hypothes.is]] highlights and annotations
 """
+REQUIRES = [
+ 'git+https://github.com/karlicoss/hypexport',
+]
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Optional, Callable

my/pocket.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from typing import Optional
 
-from .core import Paths, PathIsh
+from .core import Paths
 
 from my.config import pocket as user_config
 
@@ -21,30 +21,16 @@ class pocket(user_config):
  # paths[s]/glob to the exported JSON data
  export_path: Paths
 
- # path to a local clone of pockexport
- # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/pockexport
- pockexport : Optional[PathIsh] = None
-
- @property
- def dal_module(self):
- rpath = self.pockexport
- if rpath is not None:
- from .core.common import import_dir
- return import_dir(rpath, '.dal')
- else:
- import my.config.repos.pockexport.dal as dal
- return dal
-
 
 from .core.cfg import make_config
 config = make_config(pocket)
 
 
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
- import my.config.repos.pockexport.dal as dal
-else:
- dal = config.dal_module
+try:
+ from pockexport import dal
+except ModuleNotFoundError as e:
+ from .core.compat import pre_pip_dal_handler
+ dal = pre_pip_dal_handler('pockexport', e, config, requires=REQUIRES)
 
 ############################
 

my/reddit.py

@@ -6,9 +6,8 @@
 ]
 
 from typing import Optional
-from .core.common import Paths, PathIsh
+from .core.common import Paths
 
-from types import ModuleType
 from my.config import reddit as uconfig
 from dataclasses import dataclass
 
@@ -21,20 +20,6 @@ class reddit(uconfig):
  # path[s]/glob to the exported JSON data
  export_path: Paths
 
- # path to a local clone of rexport
- # alternatively, you can put the repository (or a symlink) in $MY_CONFIG/my/config/repos/rexport
- rexport : Optional[PathIsh] = None
-
- @property
- def dal_module(self) -> ModuleType:
- rpath = self.rexport
- if rpath is not None:
- from .core.common import import_dir
- return import_dir(rpath, '.dal')
- else:
- import my.config.repos.rexport.dal as dal
- return dal
-
 
 from .core.cfg import make_config, Attrs
 # hmm, also nice thing about this is that migration is possible to test without the rest of the config?
@@ -50,16 +35,15 @@ def migration(attrs: Attrs) -> Attrs:
 ###
 # TODO not sure about the laziness...
 
-from typing import TYPE_CHECKING
-if TYPE_CHECKING:
- # TODO not sure what is the right way to handle this..
- import my.config.repos.rexport.dal as dal
-else:
- # TODO ugh. this would import too early
- # but on the other hand we do want to bring the objects into the scope for easier imports, etc. ugh!
- # ok, fair enough I suppose. It makes sense to configure something before using it. can always figure it out later..
- # maybe, the config could dynamically detect change and reimport itself? dunno.
- dal = config.dal_module
+try:
+ from rexport import dal
+except ModuleNotFoundError as e:
+ from .core.compat import pre_pip_dal_handler
+ dal = pre_pip_dal_handler('rexport', e, config, requires=REQUIRES)
+# TODO ugh. this would import too early
+# but on the other hand we do want to bring the objects into the scope for easier imports, etc. ugh!
+# ok, fair enough I suppose. It makes sense to configure something before using it. can always figure it out later..
+# maybe, the config could dynamically detect change and reimport itself? dunno.
 ###
 
 ############################
@@ -225,8 +209,9 @@ def events(*args, **kwargs) -> List[Event]:
  inp = inputs()
  # 2.2s for 300 files without cachew
  # 0.2s for 300 files with cachew
- evit = _get_events(inp, *args, **kwargs)
- return list(sorted(evit, key=lambda e: e.cmp_key))
+ evit = _get_events(inp, *args, **kwargs) # type: ignore[call-arg]
+ # todo mypy is confused here and thinks it's iterable of Path? perhaps something to do with mcachew?
+ return list(sorted(evit, key=lambda e: e.cmp_key)) # type: ignore[attr-defined,arg-type]
 
 
 def stats():

tox.ini

@@ -31,11 +31,17 @@ setenv = MYPYPATH=doc/example_config
 commands =
  # TODO ugh. somehow, user installed (editable??) packages leak into this environment??
  pip install -e .[testing] .[optional]
+ pip install git+https://github.com/karlicoss/ghexport
  pip install git+https://github.com/karlicoss/hypexport
  pip install git+https://github.com/karlicoss/instapexport
+ pip install git+https://github.com/karlicoss/pockexport
+ pip install git+https://github.com/karlicoss/rexport
  # todo I guess use a script...
- python3 -m mypy -p my.hypothesis \
- -p my.instapaper \
+ python3 -m mypy -p my.github.ghexport \
+ -p my.hypothesis \
+ -p my.instapaper \
+ -p my.pocket \
+ -p my.reddit \
  --txt-report .mypy-coverage \
  --html-report .mypy-coverage \
  {posargs}