Budzich Maxim

.gitignore

@@ -23,6 +23,7 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+.idea
 MANIFEST
 
 # PyInstaller

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.8
+
+RUN mkdir /code
+
+WORKDIR /code
+
+ADD app/support_files code/app/support_files
+ADD requirements.txt code/requirements.txt
+ADD app/__init__.py code/app/__init__.py
+ADD app/core.py code/app/core.py
+ADD README.md code/README.md
+ADD setup.py code/setup.py
+RUN cd code; python3.8 setup.py install
+

README.md

@@ -0,0 +1,67 @@
+## It is a one-shot command-line RSS reader by Zviger.
+### Installation
+Clone this repository and run setup.py file with parameters "install --user"
+or
+Download docker [https://docs.docker.com/] and docker-compose [https://docs.docker.com/compose/install/]
+after this run command:
+```text
+docker-compose up -d
+```
+and
+```text
+docker exec -it rss_reader bash
+```
+Fine!
+
+Now you can write in the docker console "rss_reader" with some parameters
+### User interface
+```text
+usage: rss_reader [-h] [--version] [-l LIMIT] [--verbose] [--json] [--length LENGTH] [--date DATE] source
+
+It is a python command-line rss reader
+
+positional arguments:
+  source                RSS URL
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --version             Print version info
+  -l LIMIT, --limit LIMIT
+                        Limit news topics if this parameter provided
+  --verbose             Print result as JSON in stdout
+  --json                Outputs verbose status messages
+  --length LENGTH       Sets the length of each line of news output
+  --date DATE           Search past news by date in format yeardaymonth (19991311)
+
+```
+
+### Json structure
+```json
+[
+  {
+    "title": "Yahoo News - Latest News & Headlines",
+    "link": "https://www.yahoo.com/news",
+    "items":
+      [
+        {
+          "title": "Sorry, Hillary: Democrats don't need a savior",
+          "link": "https://news.yahoo.com/sorry-hillary-democrats-dont-need-a-savior-194253123.html",
+          "author": "no author",
+          "published_parsed": [2019, 11, 13, 19, 42, 53, 2, 317, 0],
+          "description": "With the Iowa caucuses fast approaching, Hillary Clinton is just the latest in the colorful cast of characters who seem to have surveyed the sprawling Democratic field, sensed something lacking and decided that \u201csomething\u201d might be them.",
+          "img_links":
+            [
+              "http://l.yimg.com/uu/api/res/1.2/xq3Ser6KXPfV6aeoxbq9Uw--/YXBwaWQ9eXRhY2h5b247aD04Njt3PTEzMDs-/https://media-mbst-pub-ue1.s3.amazonaws.com/creatr-uploaded-images/2019-11/14586fd0-064d-11ea-b7df-7288f8d8c1a7"
+            ]
+        }
+      ]
+  }
+]
+```
+### Cashing
+The news is saved to the database when news output commands are executed. MongoDB is used as a database management system.
+When the --date parameter is used, news is downloaded from the database by the entered date and the entered RSS link.
+
+Features:
+* The --limit parameter affects the amount of data loaded into the database.
+* Date must be written in the yearmonthday (example - 19991113) format. 

app/__init__.py

@@ -0,0 +1 @@
+__version__ = "3.1"

app/core.py

@@ -0,0 +1,9 @@
+from app.support_files.rss_reader import Reader
+
+
+def main() -> None:
+    Reader.exec_console_args()
+
+
+if __name__ == "__main__":
+    main()

app/support_files/app_logger.py

@@ -0,0 +1,22 @@
+"""
+This module provides functions to work with logging.
+"""
+import logging
+from logging import Logger
+import sys
+
+
+def init_logger(name: str) -> Logger:
+    """
+    Initialize and return logger object.
+    :param name: Name of the logger object.
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    # create the logging file handler
+    stream_handler = logging.StreamHandler(sys.stdout)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s')
+    stream_handler.setFormatter(formatter)
+    # add handler to logger object
+    logger.addHandler(stream_handler)
+    return logger

app/support_files/args_parser.py

@@ -0,0 +1,25 @@
+"""
+This module is a parser of console arguments for this project.
+"""
+import argparse
+from argparse import Namespace
+
+import app
+
+
+def get_args() -> Namespace:
+    """
+    Function, that parse console args.
+    :return: An object that provides the values ​​of parsed arguments.
+    """
+    parser = argparse.ArgumentParser(description="It is a python command-line rss reader")
+    parser.add_argument("source", help="RSS URL")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {app.__version__}", help="Print version info")
+    parser.add_argument("-l", "--limit", type=int, help="Limit news topics if this parameter provided", default=-1)
+    parser.add_argument("--verbose", action="store_true", help="Print result as JSON in stdout", default=False)
+    parser.add_argument("--json", action="store_true", help="Outputs verbose status messages", default=False)
+    parser.add_argument("--length", type=int, help="Sets the length of each line of news output", default=120)
+    parser.add_argument("--date", help="Search past news by date in format yeardaymonth (19991311)")
+    parser.parse_args()
+    args = parser.parse_args()
+    return args

app/support_files/database.py

@@ -0,0 +1,95 @@
+"""
+This module contains class to work with database.
+"""
+from dataclasses import asdict
+from typing import Optional
+from time import strptime, mktime, altzone, localtime, struct_time
+
+from pymongo import MongoClient
+
+from app.support_files.dtos import Feed, Item
+from app.support_files.exeptions import FindFeedError, DateError
+
+
+class DB:
+    """
+    Class to work with database.
+    """
+
+    def __init__(self) -> None:
+        client = MongoClient("mongodb://mongo:27017/")
+        self._db = client["feed_db"]
+        self._collection = self._db["feed_collection"]
+
+    def insert_feed(self, feed: Feed) -> None:
+        """
+        Insert feed in database.
+        If this feed exists in the database, then news is added that was not there.
+        :param feed: Feed, which should be inserted.
+        """
+        cashed_feed = self.find_feed_by_link(feed.rss_link)
+
+        if cashed_feed is not None:
+            items = set(feed.items)
+            cashed_items = set(cashed_feed.items)
+            result_items = list(set(items).union(set(cashed_items)))
+            result_items = list(map(asdict, result_items))
+            self._collection.update_one({"rss_link": feed.rss_link}, {"$set": {"items": result_items}})
+        else:
+            self._collection.insert_one(asdict(feed))
+
+    def find_feed_by_link(self, link: str) -> Optional[Feed]:
+        """
+        Looks for feed in the database by rss link and returns it.
+        :param link: Rss link.
+        :return: Feed, if it exist, otherwise None.
+        """
+        dict_feed = self._collection.find_one({"rss_link": link})
+        if dict_feed is None:
+            return None
+        del dict_feed["_id"]
+        feed = Feed(**dict_feed)
+        feed.items = [Item(**item) for item in dict_feed["items"]]
+        return feed
+
+    def find_feed_by_link_and_date(self, link: str, date: str, limit: int = -1) -> Feed:
+        """
+        Looks for feed in the database by rss link and date and returns it.
+        Raise DateError, in it not exist.
+        :param link: Rss link.
+        :param date: Need date.
+        :param limit: Limit count of returned items.
+        :return: Feed, if it exist.
+        """
+        try:
+            date = strptime(date, "%Y%m%d")
+        except ValueError as err:
+            raise DateError(err.__str__())
+        feed = self.find_feed_by_link(link)
+        if feed is None:
+            raise FindFeedError("This feed is not cashed")
+        result_items = []
+        count = limit
+        for item in feed.items:
+            i_date = struct_time(item.published_parsed)
+            l_i_date = localtime(mktime(tuple(i_date)) - altzone)
+            if (l_i_date.tm_year, l_i_date.tm_mon, l_i_date.tm_mday) == (date.tm_year, date.tm_mon, date.tm_mday):
+                result_items.append(item)
+                count -= 1
+            if count == 0:
+                break
+        feed.items = result_items
+        return feed
+
+    def truncate_collection(self) -> None:
+        """
+        Truncate database.
+        """
+        self._collection.delete_many({})
+
+
+if __name__ == "__main__":
+    db = DB()
+    db.find_feed_by_link_and_date("", "201")
+    print([len(feed["items"]) for feed in db._collection.find({})])
+    print([feed["rss_link"] for feed in db._collection.find({})])

app/support_files/dtos.py

@@ -0,0 +1,33 @@
+"""
+This module contains data classes to work with feeds.
+"""
+from dataclasses import dataclass, field
+from time import struct_time, localtime, time
+from typing import List
+
+
+@dataclass
+class Item:
+    """
+    This class represents each item in feed.
+    """
+    title: str = "no title"
+    link: str = "no link"
+    author: str = "no author"
+    published_parsed: struct_time = localtime(time())
+    description: str = "description"
+    img_links: List[str] = field(default_factory=list)
+
+    def __hash__(self) -> int:
+        return hash(str(self.__dict__))
+
+
+@dataclass
+class Feed:
+    """
+    This class represents feed.
+    """
+    rss_link: str
+    title: str = "no title"
+    link: str = "no link"
+    items: List[Item] = field(default_factory=list)

app/support_files/exeptions.py

@@ -0,0 +1,17 @@
+"""
+This module provides exception classes.
+"""
+
+
+class FindFeedError(Exception):
+    """
+    This class should be raised, if received some problems with getting feed.
+    """
+    pass
+
+
+class DateError(ValueError):
+    """
+    This class should be raised, if received some problems with converting date.
+    """
+    pass

app/support_files/format_converter.py

@@ -0,0 +1,66 @@
+"""
+This module contains class for converting parsed data from RSS.
+"""
+import textwrap
+import functools
+import json
+import dataclasses
+from typing import List
+from time import strftime, altzone, mktime, localtime
+
+from app.support_files.dtos import Feed
+
+
+class Converter:
+    """
+    This class represents format converter for parsed data from RSS.
+    """
+
+    def __init__(self, feeds: List[Feed]) -> None:
+        """
+        :param feeds: Parsed data from RSS.
+        """
+        self.__feeds = feeds
+
+    def to_console_format(self, str_len: int = 80) -> str:
+        """
+        Convert data to console format.
+        :param str_len: Length of output strings.
+        :return: Converted data.
+        """
+        strings = []
+        out_separator = "*" * str_len
+        in_separator = "-" * str_len
+        for feed in self.__feeds:
+            strings.append(out_separator)
+            strings.append(f"Feed: {feed.title}")
+            for item in feed.items:
+                strings.append(in_separator)
+                strings.append(f"Author: {item.author}")
+                published = localtime(mktime(tuple(item.published_parsed)) - altzone)
+                strings.append(f"Published: {strftime('%a, %d %b %Y %X', published)} {-altzone / 3600}")
+                strings.append("\n")
+                strings.append(f"Title: {item.title}")
+                strings.append(f"Description: {item.description}")
+                strings.append("\n")
+                strings.append(f"Link: {item.link}")
+                strings.append("Image links:")
+                for img_link in item.img_links:
+                    strings.append(f"{img_link}")
+                strings.append(in_separator)
+            strings.append(out_separator)
+
+        strings = map(lambda s: textwrap.fill(s, width=str_len) + "\n", strings)
+
+        result_string = functools.reduce(lambda a, b: a + b, strings)
+
+        return result_string
+
+    def to_json_format(self, str_len: int = 80) -> str:
+        """
+        Convert data to json format.
+        :param str_len: Length of output strings.
+        :return: Converted data.
+        """
+        dicts_of_feeds = list(map(dataclasses.asdict, self.__feeds))
+        return textwrap.fill(json.dumps(dicts_of_feeds), width=str_len)