Remove six library and upgrade Python syntax

docs/source/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # spidermon documentation build configuration file, created by
 # sphinx-quickstart on Tue Apr 21 21:43:18 2015.
@@ -47,9 +46,9 @@
 master_doc = "index"
 
 # General information about the project.
-project = u"Spidermon"
-copyright = u"2021, Zyte"
-author = u"Zyte"
+project = "Spidermon"
+copyright = "2021, Zyte"
+author = "Zyte"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -220,7 +219,7 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "spidermon.tex", u"Spidermon Documentation", u"Zyte", "manual")
+    (master_doc, "spidermon.tex", "Spidermon Documentation", "Zyte", "manual")
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -248,7 +247,7 @@
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [(master_doc, "spidermon", u"Spidermon Documentation", [author], 1)]
+man_pages = [(master_doc, "spidermon", "Spidermon Documentation", [author], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -263,7 +262,7 @@
     (
         master_doc,
         "spidermon",
-        u"Spidermon Documentation",
+        "Spidermon Documentation",
         author,
         "spidermon",
         "One line description of project.",

docs/source/installation.rst

@@ -4,7 +4,7 @@ Installation
 ============
 
 Spidermon's core functionality provides some useful features that allow you to
-build your monitors on top of it. The library depends on six_, jsonschema_ and
+build your monitors on top of it. The library depends on jsonschema_ and
 `python-slugify`_.
 
 If you want to set up any notifications, additional `monitoring` dependencies will help with that.
@@ -18,7 +18,6 @@ So the recommended way to install the library is by adding both:
     pip install "spidermon[monitoring,validation]"
 
 
-.. _`six`: https://pypi.org/project/six/
 .. _`jsonschema`: https://pypi.org/project/jsonschema/
 .. _`python-slugify`: https://pypi.org/project/python-slugify/
 .. _`schematics`: https://pypi.org/project/schematics/

examples/tutorial/tutorial/middlewares.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 # Define here the models for your spider middleware
 #
 # See documentation in:
@@ -8,7 +6,7 @@
 from scrapy import signals
 
 
-class TutorialSpiderMiddleware(object):
+class TutorialSpiderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the spider middleware does not modify the
     # passed objects.
@@ -32,8 +30,7 @@ def process_spider_output(self, response, result, spider):
         # it has processed the response.
 
         # Must return an iterable of Request, dict or Item objects.
-        for i in result:
-            yield i
+        yield from result
 
     def process_spider_exception(self, response, exception, spider):
         # Called when a spider or process_spider_input() method
@@ -49,14 +46,13 @@ def process_start_requests(self, start_requests, spider):
         # that it doesn’t have a response associated.
 
         # Must return only requests (not items).
-        for r in start_requests:
-            yield r
+        yield from start_requests
 
     def spider_opened(self, spider):
         spider.logger.info("Spider opened: %s" % spider.name)
 
 
-class TutorialDownloaderMiddleware(object):
+class TutorialDownloaderMiddleware:
     # Not all methods need to be defined. If a method is not defined,
     # scrapy acts as if the downloader middleware does not modify the
     # passed objects.

examples/tutorial/tutorial/monitors.py

@@ -14,7 +14,7 @@ def test_minimum_number_of_items_extracted(self):
         item_extracted = getattr(self.data.stats, "item_scraped_count", 0)
         self.assertFalse(
             item_extracted < minimum_threshold,
-            msg="Extracted less than {} items".format(minimum_threshold),
+            msg=f"Extracted less than {minimum_threshold} items",
         )
 
 
@@ -28,7 +28,7 @@ def test_no_item_validation_errors(self):
         self.assertEqual(
             validation_errors,
             0,
-            msg="Found validation errors in {} fields".format(validation_errors),
+            msg=f"Found validation errors in {validation_errors} fields",
         )
 
         self.data.stats

examples/tutorial/tutorial/pipelines.py

@@ -1,11 +1,9 @@
-# -*- coding: utf-8 -*-
-
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 
 
-class TutorialPipeline(object):
+class TutorialPipeline:
     def process_item(self, item, spider):
         return item

examples/tutorial/tutorial/settings.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 BOT_NAME = "tutorial"
 
 SPIDER_MODULES = ["tutorial.spiders"]

examples/tutorial/tutorial/spiders/quotes.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import scrapy
 
 from tutorial.items import QuoteItem

requirements.txt

@@ -1,4 +1,3 @@
-six
 Jinja2
 slackclient
 boto

setup.py

@@ -25,7 +25,7 @@
     package_data={"spidermon": ["VERSION"]},
     zip_safe=False,
     include_package_data=True,
-    install_requires=["jsonschema[format]", "python-slugify", "six>=1.12.2"],
+    install_requires=["jsonschema[format]", "python-slugify"],
     tests_require=test_requirements,
     extras_require={
         # Specific monitors and tools to support notifications and reports

spidermon/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, division, print_function
-
 __version__ = "1.15.0"
 
 from .core.monitors import Monitor

spidermon/contrib/__init__.py

@@ -1 +0,0 @@
-

spidermon/contrib/actions/__init__.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from spidermon.templates import template_loader
 
 template_loader.auto_discover("reports")

spidermon/contrib/actions/email/__init__.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 import logging
@@ -42,7 +41,7 @@ def __init__(
         *args,
         **kwargs
     ):
-        super(SendEmail, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.sender = sender or self.sender
         self.subject = subject or self.subject
         self.subject_template = subject_template or self.subject_template

spidermon/contrib/actions/email/ses.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import boto3
 
 from spidermon.exceptions import NotConfigured
@@ -21,7 +19,7 @@ def __init__(
         *args,
         **kwargs
     ):
-        super(SendSESEmail, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.aws_access_key = aws_access_key or self.aws_access_key
         self.aws_secret_key = aws_secret_key or self.aws_secret_key
         self.aws_region_name = aws_region_name or self.aws_region_name
@@ -32,7 +30,7 @@ def __init__(
 
     @classmethod
     def from_crawler_kwargs(cls, crawler):
-        kwargs = super(SendSESEmail, cls).from_crawler_kwargs(crawler)
+        kwargs = super().from_crawler_kwargs(crawler)
         (aws_access_key_id, aws_secret_access_key) = get_aws_credentials(
             crawler.settings
         )

spidermon/contrib/actions/jobs/__init__.py

@@ -1 +0,0 @@
-

spidermon/contrib/actions/jobs/tags.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from spidermon.core.actions import Action
 from spidermon.exceptions import NotConfigured
 
@@ -8,13 +7,13 @@ class JobTagsAction(Action):
     tag_settings = None
 
     def __init__(self, tags=None):
-        super(JobTagsAction, self).__init__()
+        super().__init__()
         tags = tags or self.tags
         self.tags = tags if isinstance(tags, list) else [tags]
 
     @classmethod
     def from_crawler_kwargs(cls, crawler):
-        kwargs = super(JobTagsAction, cls).from_crawler_kwargs(crawler)
+        kwargs = super().from_crawler_kwargs(crawler)
         if cls.tag_settings:
             kwargs.update({"tags": crawler.settings.get(cls.tag_settings)})
         return kwargs

spidermon/contrib/actions/reports/__init__.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from spidermon.contrib.actions.templates import ActionWithTemplates
 from spidermon.exceptions import NotConfigured
 
@@ -7,15 +6,15 @@ class CreateReport(ActionWithTemplates):
     template = None
 
     def __init__(self, template=None, *args, **kwargs):
-        super(CreateReport, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.template = template or self.template
         self.report = ""
         if not self.template:
             raise NotConfigured("You must define one template file.")
 
     @classmethod
     def from_crawler_kwargs(cls, crawler):
-        kwargs = super(CreateReport, cls).from_crawler_kwargs(crawler)
+        kwargs = super().from_crawler_kwargs(crawler)
         kwargs.update(
             {
                 "template": crawler.settings.get("SPIDERMON_REPORT_TEMPLATE"),

spidermon/contrib/actions/reports/files.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from spidermon.exceptions import NotConfigured
 
 from . import CreateReport
@@ -8,14 +7,14 @@ class CreateFileReport(CreateReport):
     filename = None
 
     def __init__(self, filename, *args, **kwargs):
-        super(CreateFileReport, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.filename = filename or self.filename
         if not self.filename:
             raise NotConfigured("You must define a template output file.")
 
     @classmethod
     def from_crawler_kwargs(cls, crawler):
-        kwargs = super(CreateFileReport, cls).from_crawler_kwargs(crawler)
+        kwargs = super().from_crawler_kwargs(crawler)
         kwargs.update({"filename": crawler.settings.get("SPIDERMON_REPORT_FILENAME")})
         return kwargs
 

spidermon/contrib/actions/reports/s3.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import hashlib
 
 from boto.s3.connection import S3Connection
@@ -14,7 +13,7 @@
 URL_SECRET_KEY = "The secret to life the universe and everything"
 
 
-class S3Uploader(object):
+class S3Uploader:
     def __init__(self, aws_key, aws_secret):
         self.connection = S3Connection(
             aws_access_key_id=aws_key, aws_secret_access_key=aws_secret
@@ -23,7 +22,7 @@ def __init__(self, aws_key, aws_secret):
     def upload_from_file(
         self, source_filename, s3_bucket, s3_filename, headers=None, make_public=False
     ):
-        with open(source_filename, "r") as f:
+        with open(source_filename) as f:
             self._upload_with_method(
                 bucket=s3_bucket,
                 method_name="set_contents_from_file",
@@ -78,7 +77,7 @@ def __init__(
         *args,
         **kwargs
     ):
-        super(CreateS3Report, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.aws_access_key = aws_access_key or self.aws_access_key
         self.aws_secret_key = aws_secret_key or self.aws_secret_key
@@ -98,7 +97,7 @@ def __init__(
 
     @classmethod
     def from_crawler_kwargs(cls, crawler):
-        kwargs = super(CreateS3Report, cls).from_crawler_kwargs(crawler)
+        kwargs = super().from_crawler_kwargs(crawler)
         (aws_access_key_id, aws_secret_access_key) = get_aws_credentials(
             crawler.settings
         )

spidermon/contrib/actions/sentry/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import
-
 import logging
 
 from slugify import slugify
@@ -28,7 +26,7 @@ def __init__(
         project_name="",
         environment="",
     ):
-        super(SendSentryMessage, self).__init__()
+        super().__init__()
         self.fake = fake or self.fake
         self.sentry_log_level = sentry_log_level or self.sentry_log_level
         self.sentry_dsn = sentry_dsn or self.sentry_dsn

spidermon/contrib/actions/slack/__init__.py

@@ -1,10 +1,7 @@
-from __future__ import absolute_import
-
 import ast
 import json
 import logging
 
-import six
 from slack import WebClient
 
 from spidermon.contrib.actions.templates import ActionWithTemplates
@@ -151,7 +148,7 @@ def __init__(
         include_attachments=None,
         fake=None,
     ):
-        super(SendSlackMessage, self).__init__()
+        super().__init__()
 
         self.fake = fake or self.fake
         self.manager = SlackMessageManager(

spidermon/contrib/actions/slack/notifiers.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from . import SendSlackMessage
 
 
@@ -24,7 +23,7 @@ def __init__(
         *args,
         **kwargs
     ):
-        super(SendSlackMessageSpiderFinished, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.include_ok_attachments = (
             include_ok_attachments or self.include_ok_attachments
         )
@@ -36,7 +35,7 @@ def __init__(
 
     @classmethod
     def from_crawler_kwargs(cls, crawler):
-        kwargs = super(SendSlackMessageSpiderFinished, cls).from_crawler_kwargs(crawler)
+        kwargs = super().from_crawler_kwargs(crawler)
         kwargs.update(
             {
                 "include_ok_attachments": crawler.settings.get(
@@ -59,12 +58,12 @@ def get_attachments(self):
         if (self.monitors_failed and self.include_error_attachments) or (
             self.monitors_passed and self.include_ok_attachments
         ):
-            return super(SendSlackMessageSpiderFinished, self).get_attachments()
+            return super().get_attachments()
         else:
             return None
 
     def get_template_context(self):
-        context = super(SendSlackMessageSpiderFinished, self).get_template_context()
+        context = super().get_template_context()
         context.update(
             {
                 "include_ok_attachments": self.include_ok_attachments,