Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Load schemas dynamically #1135

Merged
merged 39 commits into from
Sep 19, 2024
Merged
Changes from 1 commit
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
f6df03c
Load observables dynamically
udgover Sep 14, 2024
76e4a38
Ruff format
udgover Sep 14, 2024
12a0b97
Remove end of file observables imports
udgover Sep 16, 2024
9ed714b
Add dependencies to poetry / pyproject
udgover Sep 16, 2024
b15222e
Add load_observables function and handle guess type
udgover Sep 16, 2024
888ede4
Rename GenericObservable with Generic in tests
udgover Sep 16, 2024
2eb59c4
Remove validators and replace with static method is_valid
udgover Sep 16, 2024
1c5e3a9
Implement validators for previously handled observables
udgover Sep 16, 2024
df542ff
Ruff format
udgover Sep 16, 2024
bb0faeb
Add entity loader
udgover Sep 17, 2024
b03b7a2
Move entities under entities folder
udgover Sep 17, 2024
16e6d0d
Use enum.Enum for ObservableType definition
udgover Sep 17, 2024
ce32276
Add guess ObservableType if not already part of enum
udgover Sep 17, 2024
c6f3f8d
Remove unused declarations
udgover Sep 17, 2024
2a9fc2b
Remove Field(discriminator=type) which is only relevant when subclass…
udgover Sep 17, 2024
71ecd7e
Update tests to reflect entities changes
udgover Sep 17, 2024
9a09504
Ruff format
udgover Sep 17, 2024
93a501d
Add __init__.py in entities/observables private folder
udgover Sep 17, 2024
002c3b1
Add indicators and update type literal for all objects
udgover Sep 17, 2024
a2ea66c
Ruff format
udgover Sep 17, 2024
e209466
Update analytics to import relevant entities
udgover Sep 17, 2024
bf83486
Update analytics to correctly import schemas
udgover Sep 17, 2024
120d299
Update feeds to correctly import schemas
udgover Sep 17, 2024
59260a4
global naming change
tomchop Sep 18, 2024
d99a42e
Formatting
tomchop Sep 18, 2024
bc8c401
Ruff formatting
tomchop Sep 18, 2024
580acfd
Register entities, observables and indicators object accordingly to s…
udgover Sep 18, 2024
8afe499
Revert tests to main version
udgover Sep 18, 2024
0a6c3b1
Revert analytics to main version
udgover Sep 18, 2024
0c074dc
Replace ssdeep.SsdeepHash with ssdeep.Ssdeep
udgover Sep 18, 2024
913f8e3
Revert feeds to main version
udgover Sep 18, 2024
fff01fd
Replace ssdeep.SsdeepHash with ssdeep.Ssdeep
udgover Sep 18, 2024
1c221ba
Restore apiv2 to main version
udgover Sep 18, 2024
1169561
Revert dfiq to main version
udgover Sep 18, 2024
f4ccb2c
create objects from indicator. Replace indicator variable with regex
udgover Sep 18, 2024
decbf6f
Ruff format
udgover Sep 18, 2024
0dcd89b
Remove path_validator function and return regex matches in is_valid
udgover Sep 18, 2024
d920b00
Update comments to detail where types are populated
udgover Sep 19, 2024
d809ec2
Give more details about conversion from filename to entity name
udgover Sep 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Ruff format
  • Loading branch information
udgover committed Sep 17, 2024
commit a2ea66c497d1afbcbba954231a9545086d1b2162
15 changes: 12 additions & 3 deletions core/schemas/__init__.py
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@

logger = logging.getLogger(__name__)


def load_entities():
logger.info("Registering entities")
modules = dict()
@@ -55,7 +56,10 @@ def load_indicators():
if indicator_file.stem not in indicator.IndicatorType.__members__:
aenum.extend_enum(indicator.IndicatorType, indicator_file.stem, enum_value)
modules[module_name] = enum_value
indicator.TYPE_MAPPING = {"indicator": indicator.Indicator, "indicators": indicator.Indicator}
indicator.TYPE_MAPPING = {
"indicator": indicator.Indicator,
"indicators": indicator.Indicator,
}
for module_name, enum_value in modules.items():
module = importlib.import_module(module_name)
for _, obj in inspect.getmembers(module, inspect.isclass):
@@ -70,6 +74,7 @@ def load_indicators():
else:
indicator.IndicatorTypes |= cls


def load_observables():
logger.info("Registering observables")
modules = dict()
@@ -88,7 +93,10 @@ def load_observables():
modules[module_name] = observable_file.stem
if "guess" not in observable.ObservableType.__members__:
aenum.extend_enum(observable.ObservableType, "guess", "guess")
observable.TYPE_MAPPING = {"observable": observable.Observable, "observables": observable.Observable}
observable.TYPE_MAPPING = {
"observable": observable.Observable,
"observables": observable.Observable,
}
for module_name, enum_value in modules.items():
module = importlib.import_module(module_name)
for _, obj in inspect.getmembers(module, inspect.isclass):
@@ -103,6 +111,7 @@ def load_observables():
else:
observable.ObservableTypes |= cls


load_observables()
load_entities()
load_indicators()
load_indicators()
4 changes: 3 additions & 1 deletion core/schemas/entities/course_of_action.py
Original file line number Diff line number Diff line change
@@ -5,4 +5,6 @@

class CourseOfAction(entity.Entity):
_type_filter: ClassVar[str] = entity.EntityType.course_of_action
type: Literal[entity.EntityType.course_of_action] = entity.EntityType.course_of_action
type: Literal[
entity.EntityType.course_of_action
] = entity.EntityType.course_of_action
2 changes: 1 addition & 1 deletion core/schemas/entity.py
Original file line number Diff line number Diff line change
@@ -78,4 +78,4 @@ def add_context(
else:
context["source"] = source
self.context.append(context)
return self.save()
return self.save()
5 changes: 3 additions & 2 deletions core/schemas/indicator.py
Original file line number Diff line number Diff line change
@@ -18,15 +18,16 @@ def future():

DEFAULT_INDICATOR_VALIDITY_DAYS = 30


# forward declarations
class IndicatorType(str, Enum):
...


IndicatorTypes = ()
TYPE_MAPPING = {}



class IndicatorMatch(BaseModel):
name: str
match: str
@@ -87,4 +88,4 @@ def search(cls, observables: list[str]) -> list[tuple[str, "Indicator"]]:
except NotImplementedError as error:
logging.error(
f"Indicator type {indicator.type} has not implemented match(): {error}"
)
)
8 changes: 6 additions & 2 deletions core/schemas/indicators/forensicartifact.py
Original file line number Diff line number Diff line change
@@ -19,7 +19,9 @@ class ForensicArtifact(indicator.Indicator):
"""

_type_filter: ClassVar[str] = indicator.IndicatorType.forensicartifact
type: Literal[indicator.IndicatorType.forensicartifact] = indicator.IndicatorType.forensicartifact
type: Literal[
indicator.IndicatorType.forensicartifact
] = indicator.IndicatorType.forensicartifact

sources: list[dict] = []
aliases: list[str] = []
@@ -169,7 +171,9 @@ def save_indicators(self, create_links: bool = False):
regex_indicator.save()
if create_links:
for indicator_obj in indicators:
indicator_obj.link_to(self, "indicates", f"Indicates {indicator_obj.name}")
indicator_obj.link_to(
self, "indicates", f"Indicates {indicator_obj.name}"
)
return indicators


2 changes: 1 addition & 1 deletion core/schemas/indicators/regex.py
Original file line number Diff line number Diff line change
@@ -30,4 +30,4 @@ def match(self, value: str) -> indicator.IndicatorMatch | None:
result = self.compiled_pattern.search(value)
if result:
return indicator.IndicatorMatch(name=self.name, match=result.group())
return None
return None
2 changes: 2 additions & 0 deletions core/schemas/observable.py
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@
class ObservableType(str, Enum):
...


ObservableTypes = ()
TYPE_MAPPING = {}

@@ -127,6 +128,7 @@ def delete_context(
break
return self.save()


def find_type(value: str) -> ObservableType | None:
for obs_type, obj in TYPE_MAPPING.items():
if obj.is_valid(value):
4 changes: 3 additions & 1 deletion core/schemas/observables/certificate.py
Original file line number Diff line number Diff line change
@@ -22,7 +22,9 @@ class Certificate(observable.Observable):
fingerprint: the certificate fingerprint.
"""

type: Literal[observable.ObservableType.certificate] = observable.ObservableType.certificate
type: Literal[
observable.ObservableType.certificate
] = observable.ObservableType.certificate
last_seen: datetime.datetime = Field(default_factory=now)
first_seen: datetime.datetime = Field(default_factory=now)
issuer: str | None = None
4 changes: 3 additions & 1 deletion core/schemas/observables/command_line.py
Original file line number Diff line number Diff line change
@@ -4,4 +4,6 @@


class CommandLine(observable.Observable):
type: Literal[observable.ObservableType.command_line] = observable.ObservableType.command_line
type: Literal[
observable.ObservableType.command_line
] = observable.ObservableType.command_line
4 changes: 3 additions & 1 deletion core/schemas/observables/docker_image.py
Original file line number Diff line number Diff line change
@@ -4,4 +4,6 @@


class DockerImage(observable.Observable):
type: Literal[observable.ObservableType.docker_image] = observable.ObservableType.docker_image
type: Literal[
observable.ObservableType.docker_image
] = observable.ObservableType.docker_image
4 changes: 3 additions & 1 deletion core/schemas/observables/hostname.py
Original file line number Diff line number Diff line change
@@ -6,7 +6,9 @@


class Hostname(observable.Observable):
type: Literal[observable.ObservableType.hostname] = observable.ObservableType.hostname
type: Literal[
observable.ObservableType.hostname
] = observable.ObservableType.hostname

@staticmethod
def is_valid(value: str) -> bool:
4 changes: 3 additions & 1 deletion core/schemas/observables/mac_address.py
Original file line number Diff line number Diff line change
@@ -4,4 +4,6 @@


class MacAddress(observable.Observable):
type: Literal[observable.ObservableType.mac_address] = observable.ObservableType.mac_address
type: Literal[
observable.ObservableType.mac_address
] = observable.ObservableType.mac_address
4 changes: 3 additions & 1 deletion core/schemas/observables/named_pipe.py
Original file line number Diff line number Diff line change
@@ -4,4 +4,6 @@


class NamedPipe(observable.Observable):
type: Literal[observable.ObservableType.named_pipe] = observable.ObservableType.named_pipe
type: Literal[
observable.ObservableType.named_pipe
] = observable.ObservableType.named_pipe
4 changes: 3 additions & 1 deletion core/schemas/observables/registry_key.py
Original file line number Diff line number Diff line change
@@ -26,7 +26,9 @@ class RegistryKey(observable.Observable):
path_file: The filesystem path to the file that contains the registry key value.
"""

type: Literal[observable.ObservableType.registry_key] = observable.ObservableType.registry_key
type: Literal[
observable.ObservableType.registry_key
] = observable.ObservableType.registry_key
key: str
data: bytes
hive: RegistryHive
4 changes: 3 additions & 1 deletion core/schemas/observables/user_account.py
Original file line number Diff line number Diff line change
@@ -14,7 +14,9 @@ class UserAccount(observable.Observable):
Value should to be in the form <ACCOUNT_TYPE>:<ACCOUNT_LOGIN>.
"""

type: Literal[observable.ObservableType.user_account] = observable.ObservableType.user_account
type: Literal[
observable.ObservableType.user_account
] = observable.ObservableType.user_account
user_id: str | None = None
credential: str | None = None
account_login: str | None = None
4 changes: 3 additions & 1 deletion core/schemas/observables/user_agent.py
Original file line number Diff line number Diff line change
@@ -4,4 +4,6 @@


class UserAgent(observable.Observable):
type: Literal[observable.ObservableType.user_agent] = observable.ObservableType.user_agent
type: Literal[
observable.ObservableType.user_agent
] = observable.ObservableType.user_agent
2 changes: 1 addition & 1 deletion core/web/apiv2/graph.py
Original file line number Diff line number Diff line change
@@ -111,7 +111,7 @@ class GraphSearchResponse(BaseModel):
| entity.EntityTypes
| indicator.IndicatorTypes
| tag.Tag
| dfiq.DFIQTypes
| dfiq.DFIQTypes,
]
paths: list[list[graph.Relationship | graph.TagRelationship]]
total: int
4 changes: 3 additions & 1 deletion core/web/apiv2/indicators.py
Original file line number Diff line number Diff line change
@@ -79,7 +79,9 @@ async def patch(request: PatchIndicatorRequest, indicator_id) -> IndicatorTypes:

if db_indicator.type == IndicatorType.forensicartifact:
if db_indicator.pattern != request.indicator.pattern:
return forensicartifact.ForensicArtifact.from_yaml_string(request.indicator.pattern)[0]
return forensicartifact.ForensicArtifact.from_yaml_string(
request.indicator.pattern
)[0]

update_data = request.indicator.model_dump(exclude_unset=True)
updated_indicator = db_indicator.model_copy(update=update_data)
1 change: 1 addition & 0 deletions core/web/apiv2/observables.py
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@ class PatchObservableRequest(BaseModel):

observable: ObservableTypes = Field(discriminator="type")


class NewBulkObservableAddRequest(BaseModel):
model_config = ConfigDict(extra="forbid")

8 changes: 6 additions & 2 deletions tests/schemas/indicator.py
Original file line number Diff line number Diff line change
@@ -38,7 +38,9 @@ def test_filter_entities_different_types(self) -> None:

self.assertEqual(len(all_entities), 1)
self.assertEqual(len(regex_indicators), 1)
self.assertEqual(regex_indicators[0].model_dump_json(), regex_indicator.model_dump_json())
self.assertEqual(
regex_indicators[0].model_dump_json(), regex_indicator.model_dump_json()
)

def test_create_indicator_same_name_diff_types(self) -> None:
regex1 = regex.Regex(
@@ -242,7 +244,9 @@ def test_forensic_artifacts_parent_extraction(self):
- blah3
"""

artifacts = forensicartifact.ForensicArtifact.from_yaml_string(pattern, update_parents=True)
artifacts = forensicartifact.ForensicArtifact.from_yaml_string(
pattern, update_parents=True
)
self.assertEqual(len(artifacts), 3)

vertices, _, total = artifacts[0].neighbors()
Loading