Skip to content

Commit

Permalink
Add (and set) UpdatedFeed.{unmodified,total}.
Browse files Browse the repository at this point in the history
For #96.
  • Loading branch information
lemon24 committed Sep 9, 2022
1 parent 27aca7e commit 229392f
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 8 deletions.
7 changes: 7 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,13 @@ Version 3.2

Unreleased

* :class:`UpdatedFeed` changes:
added field :attr:`~UpdatedFeed.unmodified`
and property :attr:`~UpdatedFeed.total`;
fields :attr:`~UpdatedFeed.new` and :attr:`~UpdatedFeed.modified`
became optional.
(:issue:`96`)


Version 3.1
-----------
Expand Down
8 changes: 7 additions & 1 deletion src/reader/_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -298,10 +298,16 @@ def iter_update_status(it, length):

if result.not_modified:
status = 'not modified'
if result.updated_feed:
status += f", {result.value.total} total"
elif result.error:
status = red(result.error)
else:
status = green(f"{result.value.new} new, {result.value.modified} modified")
status = green(
f"{result.value.new} new, "
f"{result.value.modified} modified, "
f"{result.value.total} total"
)

click.echo(f"{elapsed}\t{pos}\t{result.url}\t{status}")

Expand Down
9 changes: 8 additions & 1 deletion src/reader/_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from ._types import FeedForUpdate
from ._types import FeedUpdateIntent
from ._types import ParsedFeed
from ._utils import count_consumed
from ._utils import PrefixLogger
from .exceptions import FeedNotFoundError
from .exceptions import ParseError
Expand Down Expand Up @@ -411,20 +412,26 @@ def process_parse_result(
try:
# assemble pipeline
entry_pairs = self.get_entry_pairs(result)

if result and not isinstance(result, Exception):
entry_pairs = self.parser.process_entry_pairs(
feed.url, result.mime_type, entry_pairs
)
entry_pairs, get_total_count = count_consumed(entry_pairs)
else:
get_total_count = lambda: 0 # noqa: E731

intents = make_intents(entry_pairs)
counts = self.update_feed(feed.url, *intents)
total = get_total_count()

except Exception as e:
return feed.url, e

if not result or isinstance(result, Exception):
return feed.url, result

return feed.url, UpdatedFeed(feed.url, *counts)
return feed.url, UpdatedFeed(feed.url, *counts, total - sum(counts))

def get_entry_pairs(
self, result: Union[Optional[ParsedFeed], ParseError]
Expand Down
15 changes: 15 additions & 0 deletions src/reader/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,21 @@ def chunks(n: int, iterable: Iterable[_T]) -> Iterable[Iterable[_T]]:
yield itertools.chain([first], chunk)


def count_consumed(it: Iterable[_T]) -> Tuple[Iterable[_T], Callable[[], int]]:
consumed = 0

def wrapper() -> Iterable[_T]:
nonlocal consumed
for e in it:
yield e
consumed += 1

def get_count() -> int:
return consumed

return wrapper(), get_count


@contextmanager
def make_pool_map(workers: int) -> Iterator[F]:
pool = multiprocessing.dummy.Pool(workers)
Expand Down
28 changes: 25 additions & 3 deletions src/reader/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ class Enclosure(_namedtuple_compat):

"""Data type representing an external file."""

# WARNING: When changing attributes, keep content_from_obj in sync.
# WARNING: When changing attributes, keep enclosure_from_obj in sync.

#: The file URL.
href: str
Expand Down Expand Up @@ -873,12 +873,34 @@ class UpdatedFeed:

#: The number of new entries
#: (entries that did not previously exist in storage).
new: int
#:
#: .. versionchanged:: 3.2
#: This field is now optional, and defaults to 0.
new: int = 0

#: The number of modified entries
#: (entries that existed in storage,
#: but had different data than the corresponding feed file entry.)
modified: int
#:
#: .. versionchanged:: 3.2
#: This field is now optional, and defaults to 0.
modified: int = 0

#: The number of unmodified entries
#: (entries that existed in storage,
#: but had the same data in the corresponding feed file entry.)
#:
#: .. versionadded:: 3.2
unmodified: int = 0

@property
def total(self) -> int:
"""The total number of entries in the retrieved feed.
.. versionadded:: 3.2
"""
return self.new + self.modified + self.unmodified


class UpdateResult(NamedTuple):
Expand Down
6 changes: 3 additions & 3 deletions tests/test_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -1016,7 +1016,7 @@ def test_update_feeds_iter(reader, call_update_iter_method):
}

assert next(call_update_iter_method(reader)) == UpdateResult(
'1', UpdatedFeed(url='1', new=0, modified=0)
'1', UpdatedFeed(url='1', new=0, modified=0, unmodified=2)
)

one_two = parser.entry(1, 2, datetime(2010, 2, 2), title='new title')
Expand All @@ -1029,8 +1029,8 @@ def test_update_feeds_iter(reader, call_update_iter_method):
rv = dict(call_update_iter_method(reader))
assert set(rv) == set('123')

assert rv['1'] == UpdatedFeed(url='1', new=2, modified=1)
assert rv['2'] == UpdatedFeed(url='2', new=0, modified=0)
assert rv['1'] == UpdatedFeed(url='1', new=2, modified=1, unmodified=1)
assert rv['2'] == UpdatedFeed(url='2', new=0, modified=0, unmodified=1)

assert isinstance(rv['3'], ParseError)
assert rv['3'].url == '3'
Expand Down
5 changes: 5 additions & 0 deletions tests/test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,11 @@ def test_content_is_html(content, expected):
assert content.is_html == expected


def test_updated_feed_properties():
feed = UpdatedFeed('url', new=1, modified=2, unmodified=3)
assert feed.total == 6


def test_update_result_properties():
feed = UpdatedFeed('url', 0, 1)
result = UpdateResult('url', feed)
Expand Down

0 comments on commit 229392f

Please sign in to comment.