Merge branch 'mitre:master' into master

.github/CODEOWNERS

@@ -3,3 +3,4 @@
 /app/service/interfaces/i_planning_svc.py   @mitre/squad-x
 /app/utility/base_planning_svc.py   @mitre/squad-x
 /tests/services/test_planning_svc.py  @mitre/squad-x
+/requirements.txt @wbooth

app/api/v2/handlers/ability_api.py

@@ -36,22 +36,25 @@ async def get_ability_by_id(self, request: web.Request):
         ability = await self.get_object(request)
         return web.json_response(ability)
 
-    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.docs(tags=['abilities'], summary='"name", "tactic", and "executors" are all required fields.')
     @aiohttp_apispec.request_schema(AbilitySchema)
     @aiohttp_apispec.response_schema(AbilitySchema)
     async def create_ability(self, request: web.Request):
         ability = await self.create_on_disk_object(request)
         return web.json_response(ability.display)
 
-    @aiohttp_apispec.docs(tags=['abilities'])
+    @aiohttp_apispec.docs(tags=['abilities'], summary='"name", "tactic", and "executors" are all required fields.')
     @aiohttp_apispec.request_schema(AbilitySchema(partial=True))
     @aiohttp_apispec.response_schema(AbilitySchema)
     async def create_or_update_ability(self, request: web.Request):
         ability = await self.create_or_update_on_disk_object(request)
         return web.json_response(ability.display)
 
     @aiohttp_apispec.docs(tags=['abilities'])
-    @aiohttp_apispec.request_schema(AbilitySchema(partial=True))
+    @aiohttp_apispec.request_schema(AbilitySchema(partial=True, exclude=['ability_id',
+                                                                         'requirements',
+                                                                         'additional_info',
+                                                                         'access']))
     @aiohttp_apispec.response_schema(AbilitySchema)
     async def update_ability(self, request: web.Request):
         ability = await self.update_on_disk_object(request)

app/api/v2/handlers/adversary_api.py

@@ -46,7 +46,7 @@ async def create_adversary(self, request: web.Request):
         return web.json_response(adversary.display)
 
     @aiohttp_apispec.docs(tags=['adversaries'])
-    @aiohttp_apispec.request_schema(AdversarySchema(partial=True))
+    @aiohttp_apispec.request_schema(AdversarySchema(partial=True, exclude=['adversary_id']))
     @aiohttp_apispec.response_schema(AdversarySchema)
     async def update_adversary(self, request: web.Request):
         adversary = await self.update_on_disk_object(request)

app/api/v2/handlers/agent_api.py

@@ -48,7 +48,12 @@ async def create_agent(self, request: web.Request):
         return web.json_response(agent.display)
 
     @aiohttp_apispec.docs(tags=['agents'])
-    @aiohttp_apispec.request_schema(AgentSchema(partial=True))
+    @aiohttp_apispec.request_schema(AgentSchema(partial=True, only=['group',
+                                                                    'trusted',
+                                                                    'sleep_min',
+                                                                    'sleep_max',
+                                                                    'watchdog',
+                                                                    'pending_contact']))
     @aiohttp_apispec.response_schema(AgentSchema)
     async def update_agent(self, request: web.Request):
         agent = await self.update_object(request)

app/api/v2/handlers/fact_source_api.py

@@ -43,7 +43,8 @@ async def create_fact_source(self, request: web.Request):
         source = await self.create_on_disk_object(request)
         return web.json_response(source.display)
 
-    @aiohttp_apispec.docs(tags=['sources'])
+    @aiohttp_apispec.docs(tags=['sources'], summary='All fields can be updated excluding "id" '
+                                                    'and "adjustments".')
     @aiohttp_apispec.request_schema(SourceSchema(partial=True))
     @aiohttp_apispec.response_schema(SourceSchema)
     async def update_fact_source(self, request: web.Request):

app/api/v2/handlers/objective_api.py

@@ -43,7 +43,7 @@ async def create_objective(self, request: web.Request):
         return web.json_response(objective.display)
 
     @aiohttp_apispec.docs(tags=['objectives'])
-    @aiohttp_apispec.request_schema(ObjectiveSchema(partial=True))
+    @aiohttp_apispec.request_schema(ObjectiveSchema(partial=True, exclude=['id', 'percentage']))
     @aiohttp_apispec.response_schema(ObjectiveSchema)
     async def update_objective(self, request: web.Request):
         objective = await self.update_on_disk_object(request)

app/api/v2/handlers/operation_api.py

@@ -48,15 +48,19 @@ async def get_operation_by_id(self, request: web.Request):
         operation = await self.get_object(request)
         return web.json_response(operation)
 
-    @aiohttp_apispec.docs(tags=['operations'])
-    @aiohttp_apispec.request_schema(OperationSchema)
+    @aiohttp_apispec.docs(tags=['operations'],
+                          summary='Required nested schema fields are as follows: "adversary.adversary_id", '
+                                  '"planner.planner_id", and "source.id".')
+    @aiohttp_apispec.request_schema(OperationSchema())
     @aiohttp_apispec.response_schema(OperationSchema)
     async def create_operation(self, request: web.Request):
         operation = await self.create_object(request)
         return web.json_response(operation.display)
 
     @aiohttp_apispec.docs(tags=['operations'])
-    @aiohttp_apispec.request_schema(OperationSchema(partial=True))
+    @aiohttp_apispec.request_schema(OperationSchema(partial=True, only=['state',
+                                                                        'autonomous',
+                                                                        'obfuscator']))
     @aiohttp_apispec.response_schema(OperationSchema(partial=True))
     async def update_operation(self, request: web.Request):
         operation = await self.update_object(request)
@@ -118,7 +122,7 @@ async def get_operation_link_result(self, request: web.Request):
         return web.json_response(result)
 
     @aiohttp_apispec.docs(tags=['operations'])
-    @aiohttp_apispec.request_schema(LinkSchema(partial=True))
+    @aiohttp_apispec.request_schema(LinkSchema(partial=True, only=['command', 'status']))
     @aiohttp_apispec.response_schema(LinkSchema)
     async def update_operation_link(self, request: web.Request):
         operation_id = request.match_info.get('id')
@@ -128,7 +132,9 @@ async def update_operation_link(self, request: web.Request):
         link = await self._api_manager.update_operation_link(operation_id, link_id, data, access)
         return web.json_response(link)
 
-    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.docs(tags=['operations'], summary='The only required fields for this endpoint are "paw", '
+                                                       '"executor.name", "executor.command", and "executor.platform". '
+                                                       '"executor.command" is expected to be unencoded.')
     @aiohttp_apispec.request_schema(LinkSchema)
     @aiohttp_apispec.response_schema(LinkSchema)
     async def create_potential_link(self, request: web.Request):

app/objects/c_agent.py

@@ -37,7 +37,7 @@ class AgentFieldsSchema(ma.Schema):
     proxy_receivers = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.List(ma.fields.String()),
                                      allow_none=True)
     proxy_chain = ma.fields.List(ma.fields.List(ma.fields.String()), allow_none=True)
-    origin_link_id = ma.fields.Integer()
+    origin_link_id = ma.fields.String()
     deadman_enabled = ma.fields.Boolean(allow_none=True)
     available_contacts = ma.fields.List(ma.fields.String(), allow_none=True)
     host_ip_addrs = ma.fields.List(ma.fields.String(), allow_none=True)
@@ -98,7 +98,7 @@ def is_global_variable(cls, variable):
     def __init__(self, sleep_min=30, sleep_max=60, watchdog=0, platform='unknown', server='unknown', host='unknown',
                  username='unknown', architecture='unknown', group='red', location='unknown', pid=0, ppid=0,
                  trusted=True, executors=(), privilege='User', exe_name='unknown', contact='unknown', paw=None,
-                 proxy_receivers=None, proxy_chain=None, origin_link_id=0, deadman_enabled=False,
+                 proxy_receivers=None, proxy_chain=None, origin_link_id='', deadman_enabled=False,
                  available_contacts=None, host_ip_addrs=None, upstream_dest=None, pending_contact=None):
         super().__init__()
         self.paw = paw if paw else self.generate_name(size=6)

app/objects/secondclass/c_link.py

@@ -268,6 +268,10 @@ async def _save_fact(self, operation, fact, score, relationship):
         source = operation.id if operation else self.id
         rl = [relationship] if relationship else []
         if all([fact.trait, fact.value]):
+            if operation and operation.source:
+                if any([(fact.trait, fact.value) == (x.trait, x.value) for x in
+                        await knowledge_svc_handle.get_facts(criteria=dict(source=operation.source.id))]):
+                    source = operation.source.id
             fact.source = source  # Manual addition to ensure the check works correctly
             if not await knowledge_svc_handle.check_fact_exists(fact, all_facts):
                 f_gen = Fact(trait=fact.trait, value=fact.value, source=source, score=score, collected_by=self.paw,

app/service/learning_svc.py

@@ -73,8 +73,6 @@ async def _store_results(self, link, facts, operation=None):
                 if fact.trait in relationship:
                     matches.append(fact)
                     facts_covered.append(fact)
-                else:
-                    await link._save_fact(operation=operation, fact=fact, score=link.score, relationship=None)
             for pair in itertools.combinations(matches, r=2):
                 if pair[0].trait != pair[1].trait:
                     await link._create_relationships([Relationship(source=pair[0], edge='has', target=pair[1])],

requirements.txt

@@ -25,3 +25,4 @@ dnspython==2.1.0
 asyncssh==2.5.0
 aioftp~=0.18.1; python_version >= '3.7'
 aioftp==0.16.1; python_version < '3.7'
+pyminizip==0.2.4

server.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import sys
+import warnings
 
 import aiohttp_apispec
 from aiohttp_apispec import validation_middleware
@@ -77,6 +78,10 @@ def init_swagger_documentation(app):
     """Makes swagger documentation available at /api/docs for any endpoints
     marked for aiohttp_apispec documentation.
     """
+    warnings.filterwarnings(
+        "ignore",
+        message="Multiple schemas resolved to the name"
+    )
     aiohttp_apispec.setup_aiohttp_apispec(
         app=app,
         title='CALDERA',

templates/agents.html

@@ -144,7 +144,7 @@ <h2>Agents</h2>
                             </div>
                             <div class="box container">
                                 <pre class="box" x-text="getCommandField(command.command)"></pre>
-                                <a class="button is-small is-outlined cmd-copy-button" @click="copyCommandToClipboard(getCommandField(command.command))">
+                                <a class="button is-small is-outlined cmd-copy-button" @click="copyCommandToClipboard(getCommandField(command.command))" x-show="window.isSecureContext">
                                     <span class="icon"><i class="far fa-lg fa-copy"></i></span>
                                     <span>Copy</span>
                                 </a>
@@ -163,7 +163,7 @@ <h2>Agents</h2>
                                     </div>
                                     <div class="box container">
                                         <pre class="box" x-text="getCommandField(variation.command).replaceAll(';', ';\n')"></pre>
-                                        <a class="button is-small is-outlined cmd-copy-button" @click="copyCommandToClipboard(getCommandField(variation.command))">
+                                        <a class="button is-small is-outlined cmd-copy-button" @click="copyCommandToClipboard(getCommandField(variation.command))" x-show="window.isSecureContext">
                                             <span class="icon"><i class="far fa-lg fa-copy"></i></span>
                                             <span>Copy</span>
                                         </a>