Merge pull request #28 from Yelp/fuzz_only_input_tags

Allow developers to whitelist tags for fuzzing

README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.com/Yelp/fuzz-lightyear.svg?branch=master)](https://travis-ci.com/Yelp/fuzz-lightyear)
+
 # fuzz-lightyear
 
 fuzz-lightyear is a [pytest-inspired](https://docs.pytest.org/en/latest/),

fuzz_lightyear/__init__.py

@@ -1,4 +1,5 @@
-from .supplements import exclusions                     # noqa: F401
+from .supplements import exclude                        # noqa: F401
+from .supplements import include                        # noqa: F401
 from .supplements.auth import attacker_account          # noqa: F401
 from .supplements.auth import victim_account            # noqa: F401
 from .supplements.factory import register_factory       # noqa: F401

fuzz_lightyear/datastore.py

@@ -6,6 +6,7 @@
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Set
 from typing import Tuple
 
 
@@ -20,6 +21,15 @@ def get_user_defined_mapping() -> Dict:
     return {}
 
 
+@lru_cache(maxsize=1)
+def get_included_tags() -> Set[str]:
+    """This is a global set containing tags which should
+    be fuzzed. Each element is a string for the tag which
+    should be included.
+    """
+    return set()
+
+
 @lru_cache(maxsize=1)
 def get_excluded_operations() -> Dict[str, Optional[str]]:
     """

fuzz_lightyear/generator.py

@@ -3,13 +3,44 @@
 from typing import Optional
 from typing import Tuple
 
+from bravado.client import SwaggerClient
+
 from .datastore import get_excluded_operations
+from .datastore import get_included_tags
 from .output.util import print_warning
 from .request import FuzzingRequest
 from .result import FuzzingResult
 from .supplements.abstraction import get_abstraction
 
 
+def get_fuzzable_tags(client: Optional[SwaggerClient]) -> List[str]:
+    """Given a Swagger client, returns a list of tags that should
+    actually be fuzzed. This respects the user-defined allowlist for
+    tags.
+
+    :param client: The Swagger client being fuzzed.
+    :returns: A list of tags to fuzz.
+    """
+    if not client:
+        return []
+
+    allowlisted_tags = get_included_tags()
+    if not allowlisted_tags:
+        return dir(client)
+
+    fuzzable_tags = []
+    for tag in allowlisted_tags:
+        if tag not in dir(client):
+            print_warning(
+                f'The tag "{tag}" is not in the Swagger schema, will not fuzz it.',
+            )
+            continue
+
+        fuzzable_tags.append(tag)
+
+    return fuzzable_tags
+
+
 def generate_sequences(
     n: int,
     tests: Optional[Tuple[str, ...]] = None,
@@ -24,7 +55,8 @@ def generate_sequences(
     #       (rather than starting operation), so that it's clearer for
     #       output.
     client = get_abstraction().client
-    for tag_group in dir(client):
+
+    for tag_group in get_fuzzable_tags(client):
         last_results = []   # type: List
         for _ in range(n):
             good_sequences = []

fuzz_lightyear/supplements/exclusions.py → fuzz_lightyear/supplements/exclude.py

@@ -33,12 +33,12 @@ def non_vulnerable_operations(func: Callable) -> Callable:
 
     Examples:
         Ignoring operations specified by operation ids in lists
-            >>> @fuzz_lightyear.exclusions.non_vulnerable_operations
+            >>> @fuzz_lightyear.exclude.non_vulnerable_operations
             ... def b():
             ...     return ['get_pets', 'get_store_inventory']
 
         Ignoring operations specified by "tag.operation_id" in lists
-            >>> @fuzz_lightyear.exclusions.non_vulnerable_operations
+            >>> @fuzz_lightyear.exclude.non_vulnerable_operations
             ... def c():
                     return ['pets.get_pets', 'store.get_store_inventory']
     """
@@ -54,12 +54,12 @@ def operations(func: Callable) -> Callable:
 
     Examples:
         Ignoring operations specified by operation ids in lists
-            >>> @fuzz_lightyear.exclusions.operations
+            >>> @fuzz_lightyear.exclude.operations
             ... def b():
             ...     return ['get_pets', 'get_store_inventory']
 
         Ignoring operations specified by "tag.operation_id" in lists
-            >>> @fuzz_lightyear.exclusions.operations
+            >>> @fuzz_lightyear.exclude.operations
             ... def c():
                     return ['pets.get_pets', 'store.get_store_inventory']
     """

fuzz_lightyear/supplements/include.py

@@ -0,0 +1,21 @@
+from typing import Callable
+from typing import Iterable
+
+from fuzz_lightyear.datastore import get_included_tags
+
+
+def tags(func: Callable[[], Iterable[str]]) -> Callable:
+    """Allows developers to specify Swagger tags which
+    should be fuzzed.
+
+    Example:
+        Only fuzz operations with the 'user_account' tag.
+            >>> @fuzz_lightyear.include.tags
+            ... def a():
+            ...     return ['user_account']
+    """
+    tags_to_include = func()
+    if tags_to_include:
+        get_included_tags().update(tags_to_include)
+
+    return func

tests/conftest.py

@@ -1,6 +1,7 @@
 import pytest
 
 from fuzz_lightyear.datastore import get_excluded_operations
+from fuzz_lightyear.datastore import get_included_tags
 from fuzz_lightyear.datastore import get_non_vulnerable_operations
 from fuzz_lightyear.datastore import get_user_defined_mapping
 from fuzz_lightyear.plugins import get_enabled_plugins
@@ -16,3 +17,4 @@ def clear_caches():
     get_victim_session_factory.cache_clear()
     get_excluded_operations.cache_clear()
     get_non_vulnerable_operations.cache_clear()
+    get_included_tags.cache_clear()

tests/integration/generator_test.py

@@ -17,10 +17,22 @@ def excluded_operations(request):
     def get_exclusions():
         return request.param
 
-    fuzz_lightyear.exclusions.operations(get_exclusions)
+    fuzz_lightyear.exclude.operations(get_exclusions)
     yield
 
 
+@pytest.fixture
+def included_tags(request):
+    if not isinstance(request.param, list):
+        raise ValueError
+
+    def get_included_tags():
+        return request.param
+
+    fuzz_lightyear.include.tags(get_included_tags)
+    yield request.param
+
+
 def test_length_one(mock_client):
     results = list(generate_sequences(1))
     for result in results:
@@ -60,6 +72,22 @@ def test_exclude_operations(mock_client, excluded_operations):
     )
 
 
+@pytest.mark.parametrize(
+    'included_tags',
+    [
+        (['user']),
+    ],
+    indirect=['included_tags'],
+)
+def test_included_tags(mock_client, included_tags):
+    results = list(generate_sequences(1))
+
+    assert len(results) > 0
+    for sequence in [result.requests for result in results]:
+        for request in sequence:
+            assert request.tag in included_tags
+
+
 def test_supply_single_test(mock_client):
     results = list(generate_sequences(1, ['basic.get_public_listing']))
     assert len(results) == 1

tests/integration/runner_test.py

@@ -15,7 +15,7 @@ def non_vulnerable_operations(request):
     def get_exclusions():
         return request.param
 
-    fuzz_lightyear.exclusions.non_vulnerable_operations(get_exclusions)
+    fuzz_lightyear.exclude.non_vulnerable_operations(get_exclusions)
     yield
 
 

tests/unit/supplements/exclusions_test.py → tests/unit/supplements/exclude_test.py

@@ -38,11 +38,11 @@
     'exclusions_decorator, get_exclusions_function',
     [
         (
-            fuzz_lightyear.exclusions.operations,
+            fuzz_lightyear.exclude.operations,
             get_excluded_operations,
         ),
         (
-            fuzz_lightyear.exclusions.non_vulnerable_operations,
+            fuzz_lightyear.exclude.non_vulnerable_operations,
             get_non_vulnerable_operations,
         ),
     ],

tests/unit/supplements/include_test.py

@@ -0,0 +1,29 @@
+import pytest
+
+import fuzz_lightyear
+from fuzz_lightyear.datastore import get_included_tags
+
+
+@pytest.mark.parametrize(
+    'included_tags, expected_result',
+    [
+        (
+            ['tag1', 'tag2'],
+            {'tag1', 'tag2'},
+        ),
+        (
+            set(['tag1', 'tag2']),
+            {'tag1', 'tag2'},
+        ),
+        (
+            [],
+            set(),
+        ),
+    ],
+)
+def test_include_tags(included_tags, expected_result):
+    def foobar():
+        return included_tags
+
+    fuzz_lightyear.include.tags(foobar)
+    assert get_included_tags() == expected_result