From 53ffe032e13191329eab5dd0ff67540ae6aef9bf Mon Sep 17 00:00:00 2001 From: Matthias Fey Date: Wed, 10 Nov 2021 16:54:54 +0100 Subject: [PATCH] malnet tiny (#3472) --- torch_geometric/datasets/__init__.py | 2 + torch_geometric/datasets/hgb_dataset.py | 10 ++-- torch_geometric/datasets/malnet_tiny.py | 79 +++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 torch_geometric/datasets/malnet_tiny.py diff --git a/torch_geometric/datasets/__init__.py b/torch_geometric/datasets/__init__.py index 6a2838231f33..1229024f8516 100644 --- a/torch_geometric/datasets/__init__.py +++ b/torch_geometric/datasets/__init__.py @@ -64,6 +64,7 @@ from .twitch import Twitch from .airports import Airports from .ba_shapes import BAShapes +from .malnet_tiny import MalNetTiny __all__ = [ 'KarateClub', @@ -134,6 +135,7 @@ 'Twitch', 'Airports', 'BAShapes', + 'MalNetTiny', ] classes = __all__ diff --git a/torch_geometric/datasets/hgb_dataset.py b/torch_geometric/datasets/hgb_dataset.py index 4dcfb8ddc789..511be7888efa 100644 --- a/torch_geometric/datasets/hgb_dataset.py +++ b/torch_geometric/datasets/hgb_dataset.py @@ -49,8 +49,7 @@ class HGBDataset(InMemoryDataset): def __init__(self, root: str, name: str, transform: Optional[Callable] = None, - pre_transform: Optional[Callable] = None, - pre_filter: Optional[Callable] = None): + pre_transform: Optional[Callable] = None): self.name = name.lower() assert self.name in set(self.names.keys()) super().__init__(root, transform, pre_transform) @@ -114,7 +113,7 @@ def process(self): src, dst = n_types[int(src)], n_types[int(dst)] rel = rel.split('-')[1] e_types[key] = (src, rel, dst) - else: + else: # Link prediction: raise NotImplementedError # Extract node information: @@ -180,9 +179,12 @@ def process(self): n_id, n_type = mapping_dict[int(y[0])], n_types[int(y[2])] data[n_type].test_mask[n_id] = True - else: + else: # Link prediction: raise NotImplementedError + if self.pre_transform is not None: + data = self.pre_transform(data) + torch.save(self.collate([data]), self.processed_paths[0]) def __repr__(self) -> str: diff --git a/torch_geometric/datasets/malnet_tiny.py b/torch_geometric/datasets/malnet_tiny.py new file mode 100644 index 000000000000..1ea72761a18d --- /dev/null +++ b/torch_geometric/datasets/malnet_tiny.py @@ -0,0 +1,79 @@ +from typing import Optional, Callable, List + +import os +import glob +import os.path as osp + +import torch +from torch_geometric.data import (InMemoryDataset, Data, download_url, + extract_tar) + + +class MalNetTiny(InMemoryDataset): + r"""The MalNet Tiny dataset from the + `"A Large-Scale Database for Graph Representation Learning" + `_ paper. + :class:`MalNetTiny` contains 5,000 malicious and benign software function + call graphs across 5 different types. + + Args: + root (string): Root directory where the dataset should be saved. + transform (callable, optional): A function/transform that takes in an + :obj:`torch_geometric.data.Data` object and returns a transformed + version. The data object will be transformed before every access. + (default: :obj:`None`) + pre_transform (callable, optional): A function/transform that takes in + an :obj:`torch_geometric.data.Data` object and returns a + transformed version. The data object will be transformed before + being saved to disk. (default: :obj:`None`) + pre_filter (callable, optional): A function that takes in an + :obj:`torch_geometric.data.Data` object and returns a boolean + value, indicating whether the data object should be included in the + final dataset. (default: :obj:`None`) + """ + + url = 'http://malnet.cc.gatech.edu/graph-data/malnet-graphs-tiny.tar.gz' + + def __init__(self, root: str, transform: Optional[Callable] = None, + pre_transform: Optional[Callable] = None, + pre_filter: Optional[Callable] = None): + super().__init__(root, transform, pre_transform, pre_filter) + self.data, self.slices = torch.load(self.processed_paths[0]) + + @property + def raw_file_names(self) -> List[str]: + folders = ['addisplay', 'adware', 'benign', 'downloader', 'trojan'] + return [osp.join('malnet-graphs-tiny', folder) for folder in folders] + + @property + def processed_file_names(self) -> str: + return 'data.pt' + + def download(self): + path = download_url(self.url, self.raw_dir) + extract_tar(path, self.raw_dir) + os.unlink(path) + + def process(self): + data_list = [] + + for y, raw_path in enumerate(self.raw_paths): + raw_path = osp.join(raw_path, os.listdir(raw_path)[0]) + filenames = glob.glob(osp.join(raw_path, '*.edgelist')) + + for filename in filenames: + with open(filename, 'r') as f: + edges = f.read().split('\n')[5:-1] + edge_index = [[int(edge[0]), int(edge[-1])] for edge in edges] + edge_index = torch.tensor(edge_index).t().contiguous() + num_nodes = int(edge_index.max()) + 1 + data = Data(edge_index=edge_index, y=y, num_nodes=num_nodes) + data_list.append(data) + + if self.pre_filter is not None: + data_list = [data for data in data_list if self.pre_filter(data)] + + if self.pre_transform is not None: + data_list = [self.pre_transform(data) for data in data_list] + + torch.save(self.collate(data_list), self.processed_paths[0])