updates for quick endpoint changes and riot trust_level (#498)

.bumpversion.cfg

@@ -1,7 +1,7 @@
 [bumpversion]
-current_version = 1.0.0
-tag = True
-commit = True
+current_version = 1.1.0
+tag = False
+commit = False
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

CHANGELOG.rst

@@ -2,6 +2,24 @@
 Changelog
 =========
 
+Version `1.1.0`_
+================
+**Date**: June 23, 2021
+
+* API client:
+
+  * Update filter command to use new quick endpoint with noise/riot updates
+  * Update analyze command to use new quick endpoint with noise/riot updates
+
+* CLI:
+
+  * Update quick command to handle quick endpoint noise/riot updates
+  * Add support for RIOT trust_level output
+
+* Dependencies:
+
+  * Updated Click to 8.0.1
+
 Version `1.0.0`_
 ================
 **Date**: June 02, 2021
@@ -23,7 +41,6 @@ Version `1.0.0`_
   * Updated cachetools to 4.2.2
   * Updated six to 1.16.0
   * Updated jinja2 to 3.0.1 for py36 and py37
-  * Updated Click to 8.0.1
   * Updated click-repl to 0.2.0
   * Updated more-itertools to 8.8.0
 
@@ -220,3 +237,4 @@ Version `0.2.0`_
 .. _`0.9.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v0.8.0...0.9.0
 .. _`0.9.1`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v0.9.0...0.9.1
 .. _`1.0.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v0.9.1...1.0.0
+.. _`1.1.0`: https://github.com/GreyNoise-Intelligence/pygreynoise/compare/v1.0.0...1.1.0

docs/source/conf.py

@@ -25,7 +25,7 @@
 author = "GreyNoise Intelligence"
 
 # The full version, including alpha/beta/rc tags
-release = "1.0.0"
+release = "1.1.0"
 
 
 # -- General configuration ---------------------------------------------------

requirements/common.txt

@@ -1,4 +1,4 @@
-Click==7.1.2
+Click==8.0.1
 ansimarkup==1.4.0
 cachetools==4.2.2;python_version>='3'
 click-default-group==1.2.2

requirements/dev.txt

@@ -1,6 +1,6 @@
 # Requirements needed to develop the application
 -r test.txt
 advbumpversion==1.2.0
-ipython==7.24.0;python_version>='3'
+ipython==7.25.0;python_version>='3'
 pre-commit==2.13.0
 tox==3.23.1

requirements/docs.txt

@@ -1,5 +1,5 @@
 # Requirements needed to build the documentation
 -r common.txt
 Sphinx==4.0.2
-sphinx-click==3.0.0
+sphinx-click==3.0.1
 sphinx-rtd-theme==0.5.2

requirements/test.txt

@@ -1,12 +1,12 @@
-black==21.5b2;python_version>='3.6'
+black==21.6b0;python_version>='3.6'
 flake8==3.9.2
 isort==4.3.21;python_version=='3.5' # pyup: ignore
-isort==5.8.0;python_version>='3.6'
+isort==5.9.1;python_version>='3.6'
 mock==3.0.5;python_version=='3.5' # pyup: ignore
 mock==4.0.3;python_version>='3.6'
 pylint==2.6.2;python_version=='3.5' # pyup: ignore
-pylint==2.8.2;python_version>='3.6'
-pytest-cov==2.12.0
+pylint==2.9.0;python_version>='3.6'
+pytest-cov==2.12.1
 pytest==6.1.2;python_version=='3.5' # pyup: ignore
 pytest==6.2.4;python_version>='3.6'
 restructuredtext-lint==1.3.2

setup.py

@@ -27,7 +27,7 @@ def read(fname):
 
 setup(
     name="greynoise",
-    version="1.0.0",
+    version="1.1.0",
     description="Abstraction to interact with GreyNoise API.",
     url="https://greynoise.io/",
     author="GreyNoise Intelligence",

src/greynoise/__version__.py

@@ -5,4 +5,4 @@
 __maintainer__ = "GreyNoise Intelligence"
 __email__ = "hello@greynoise.io"
 __status__ = "BETA"
-__version__ = "1.0.0"
+__version__ = "1.1.0"

src/greynoise/api/__init__.py

@@ -73,6 +73,8 @@ class GreyNoise(object):
             "IP was classified as noise, but has not been observed "
             "engaging in Internet-wide scans or attacks in over 60 days"
         ),
+        "0x09": "IP was found in RIOT",
+        "0x10": "IP has been observed by the GreyNoise sensor network and is in RIOT",
         "404": "IP is Invalid",
     }
 
@@ -227,7 +229,7 @@ def analyze(self, text):
             analyzer = Analyzer(self)
             return analyzer.analyze(text)
 
-    def filter(self, text, noise_only=False):
+    def filter(self, text, noise_only=False, riot_only=False):
         """Filter lines that contain IP addresses from a given text.
 
         :param text: Text input
@@ -236,12 +238,18 @@ def filter(self, text, noise_only=False):
             If set, return only lines that contain IP addresses classified as noise,
             otherwise, return lines that contain IP addresses not classified as noise.
         :type noise_only: bool
+        :param riot_only:
+            If set, return only lines that contain IP addresses in RIOT,
+            otherwise, return lines that contain IP addresses not in RIOT.
+        :type riot_only: bool
         :return: Iterator that yields lines in chunks
         :rtype: iterable
 
         """
         filter = Filter(self)
-        for filtered_chunk in filter.filter(text, noise_only=noise_only):
+        for filtered_chunk in filter.filter(
+            text, noise_only=noise_only, riot_only=riot_only
+        ):
             yield filtered_chunk
 
     def interesting(self, ip_address):

src/greynoise/api/analyzer.py

@@ -66,21 +66,15 @@ def analyze(self, text):
             )
 
         if text_ip_addresses:
-            noise_ip_addresses = {
-                result["ip"]
-                for result in self.api.quick(text_ip_addresses)
-                if result["noise"]
-            }
+            noise_ip_addresses = []
             riot_ip_addresses = []
-            for ip_address in text_ip_addresses:
-                result = self.api.riot(ip_address)
+
+            for result in self.api.quick(text_ip_addresses):
+                if result["noise"]:
+                    noise_ip_addresses.append(result["ip"])
                 if result["riot"]:
                     riot_ip_addresses.append(result["ip"])
 
-            noise_ip_addresses = [
-                ip for ip in noise_ip_addresses if ip not in riot_ip_addresses
-            ]
-
         else:
             noise_ip_addresses = set()
             riot_ip_addresses = set()

src/greynoise/api/filter.py

@@ -16,7 +16,7 @@ class Filter(object):
     def __init__(self, api):
         self.api = api
 
-    def filter(self, text, noise_only):
+    def filter(self, text, noise_only, riot_only):
         """Filter lines that contain IP addresses from a given text.
 
         :param text: Text input
@@ -25,6 +25,10 @@ def filter(self, text, noise_only):
             If set, return only lines that contain IP addresses classified as noise,
             otherwise, return lines that contain IP addresses not classified as noise.
         :type noise_only: bool
+        :param riot_only:
+            If set, return only lines that contain IP addresses in RIOT,
+            otherwise, return lines that contain IP addresses not in RIOT.
+        :type riot_only: bool
         :return: Iterator that yields lines in chunks
         :rtype: iterable
 
@@ -33,9 +37,9 @@ def filter(self, text, noise_only):
             text = text.splitlines(True)
         chunks = more_itertools.chunked(text, self.FILTER_TEXT_CHUNK_SIZE)
         for chunk in chunks:
-            yield self._filter_chunk(chunk, noise_only)
+            yield self._filter_chunk(chunk, noise_only, riot_only)
 
-    def _filter_chunk(self, text, noise_only):
+    def _filter_chunk(self, text, noise_only, riot_only):  # noqa: C901
         """Filter chunk of lines that contain IP addresses from a given text.
 
         :param text: Text input
@@ -44,18 +48,25 @@ def _filter_chunk(self, text, noise_only):
             If set, return only lines that contain IP addresses classified as noise,
             otherwise, return lines that contain IP addresses not classified as noise.
         :type noise_only: bool
+        :param riot_only:
+            If set, return only lines that contain IP addresses in RIOT,
+            otherwise, return lines that contain IP addresses not in RIOT.
+        :type riot_only: bool
         :return: Filtered line
 
         """
         text_ip_addresses = set()
         for input_line in text:
             text_ip_addresses.update(self.api.IPV4_REGEX.findall(input_line))
 
-        noise_ip_addresses = {
-            result["ip"]
-            for result in self.api.quick(text_ip_addresses)
-            if result["noise"]
-        }
+        noise_ip_addresses = []
+        riot_ip_addresses = []
+
+        for result in self.api.quick(text_ip_addresses):
+            if result["noise"]:
+                noise_ip_addresses.append(result["ip"])
+            if result["riot"]:
+                riot_ip_addresses.append(result["ip"])
 
         def all_ip_addresses_noisy(line):
             """Select lines that contain IP addresses and all of them are noisy.
@@ -72,6 +83,21 @@ def all_ip_addresses_noisy(line):
                 for line_ip_address in line_ip_addresses
             )
 
+        def all_ip_addresses_riot(line):
+            """Select lines that contain IP addresses and all of them are in RIOT.
+
+            :param line: Line being processed.
+            :type line: str
+            :return: True if line contains IP addresses and all of them are noisy.
+            :rtype: bool
+
+            """
+            line_ip_addresses = self.api.IPV4_REGEX.findall(line)
+            return line_ip_addresses and all(
+                line_ip_address in riot_ip_addresses
+                for line_ip_address in line_ip_addresses
+            )
+
         def add_markup(match):
             """Add markup to surround IP address value with proper tag.
 
@@ -84,13 +110,17 @@ def add_markup(match):
             ip_address = match.group(0)
             if ip_address in noise_ip_addresses:
                 tag = "noise"
+            elif ip_address in riot_ip_addresses:
+                tag = "riot"
             else:
                 tag = "not-noise"
 
             return "<{tag}>{ip_address}</{tag}>".format(ip_address=ip_address, tag=tag)
 
         if noise_only:
             line_matches = all_ip_addresses_noisy
+        elif riot_only:
+            line_matches = all_ip_addresses_riot
         else:
 
             def line_matches(line):
@@ -102,7 +132,9 @@ def line_matches(line):
                 :rtype: bool
 
                 """
-                return not all_ip_addresses_noisy(line)
+                return not all_ip_addresses_noisy(line) and not all_ip_addresses_riot(
+                    line
+                )
 
         filtered_lines = [
             self.api.IPV4_REGEX.subn(add_markup, input_line)[0]

src/greynoise/cli/formatter.py

@@ -5,9 +5,9 @@
 
 import functools
 import json
+import shutil
 
 import ansimarkup
-import click
 import colorama
 from dict2xml import dict2xml
 from jinja2 import Environment, PackageLoader, select_autoescape
@@ -26,7 +26,7 @@
         "value": ansimarkup.parse("<green>"),
         "noise": ansimarkup.parse("<light-yellow>"),
         "not-noise": ansimarkup.parse(DIM),
-        "riot": ansimarkup.parse("<light-yellow>"),
+        "riot": ansimarkup.parse("<magenta>"),
         "not-riot": ansimarkup.parse(DIM),
         "malicious": ansimarkup.parse("<light-red>"),
         "unknown": ansimarkup.parse(DIM),
@@ -135,31 +135,31 @@ def gnql_query_formatter(results, verbose):
 def gnql_stats_formatter(results, verbose):
     """Convert GNQL stats result into human-readable text."""
     template = JINJA2_ENV.get_template("gnql_stats.txt.j2")
-    max_width, _ = click.get_terminal_size()
+    max_width, _ = shutil.get_terminal_size()
     return template.render(results=results, verbose=verbose, max_width=max_width)
 
 
 @colored_output
 def analyze_formatter(result, verbose):
     """Conver analyze result into human-readable text."""
     template = JINJA2_ENV.get_template("analyze.txt.j2")
-    max_width, _ = click.get_terminal_size()
+    max_width, _ = shutil.get_terminal_size()
     return template.render(result=result, verbose=verbose, max_width=max_width)
 
 
 @colored_output
 def riot_formatter(results, verbose):
     """Convert RIOT to human-readable text."""
     template = JINJA2_ENV.get_template("riot.txt.j2")
-    max_width, _ = click.get_terminal_size()
+    max_width, _ = shutil.get_terminal_size()
     return template.render(results=results, verbose=verbose, max_width=max_width)
 
 
 @colored_output
 def interesting_formatter(results, verbose):
     """Convert RIOT to human-readable text."""
     template = JINJA2_ENV.get_template("interesting.txt.j2")
-    max_width, _ = click.get_terminal_size()
+    max_width, _ = shutil.get_terminal_size()
     return template.render(results=results, verbose=verbose, max_width=max_width)
 
 

src/greynoise/cli/subcommand.py

@@ -87,10 +87,15 @@ def feedback():
 @click.option(
     "--noise-only", is_flag=True, help="Select lines containing noisy addresses"
 )
+@click.option(
+    "--riot-only", is_flag=True, help="Select lines containing RIOT addresses"
+)
 @pass_api_client
 @click.pass_context
 @handle_exceptions
-def filter(context, api_client, api_key, input_file, output_file, noise_only):
+def filter(
+    context, api_client, api_key, input_file, output_file, noise_only, riot_only
+):
     """Filter the noise from a log file, stdin, etc."""
     if input_file is None:
         if sys.stdin.isatty():
@@ -108,7 +113,9 @@ def filter(context, api_client, api_key, input_file, output_file, noise_only):
     if output_file is None:
         output_file = click.open_file("-", mode="w")
 
-    for chunk in api_client.filter(input_file, noise_only=noise_only):
+    for chunk in api_client.filter(
+        input_file, noise_only=noise_only, riot_only=riot_only
+    ):
         output_file.write(ANSI_MARKUP(chunk))
 
 
@@ -136,12 +143,6 @@ def interesting(context, api_client, api_key, input_file, ip_address):
 
 
 @ip_lookup_command
-@click.option(
-    "-O",
-    "--offering",
-    help="Which API offering to use, enterprise or community, "
-    "defaults to enterprise",
-)
 @click.option("-v", "--verbose", count=True, help="Verbose output")
 def ip(
     context,
@@ -203,12 +204,6 @@ def query(
 
 
 @ip_lookup_command
-@click.option(
-    "-O",
-    "--offering",
-    help="Which API offering to use, enterprise or community, "
-    "defaults to enterprise",
-)
 def quick(
     context,
     api_client,