From 6d0bd1a535c8bf2a18cfec4350dc1d19e04e9c90 Mon Sep 17 00:00:00 2001 From: CoreLeader Date: Mon, 28 Mar 2022 22:12:00 +0800 Subject: [PATCH 01/37] Fix bugs for feature engineers but issues of solver still exists --- autogl/module/feature/__init__.py | 8 +++++++- autogl/module/feature/_auto_feature.py | 4 ++-- .../_base_feature_engineer/_base_feature_engineer.py | 6 +++++- .../_base_feature_engineer/_base_feature_engineer_dgl.py | 2 +- .../_base_feature_engineer/_base_feature_engineer_pyg.py | 2 +- autogl/module/feature/_selectors/_basic.py | 8 ++------ autogl/module/feature/_selectors/_gbdt.py | 3 ++- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/autogl/module/feature/__init__.py b/autogl/module/feature/__init__.py index 7f3746d4..dbdea868 100644 --- a/autogl/module/feature/__init__.py +++ b/autogl/module/feature/__init__.py @@ -33,6 +33,9 @@ from ._selectors import ( FilterConstant, GBDTFeatureSelector ) +from ._auto_feature import ( + IdentityFeature, OnlyConstFeature, AutoFeatureEngineer +) __all__ = [ "BaseFeatureEngineer", @@ -61,5 +64,8 @@ "NXGlobalEfficiency", "NXIsEulerian", "FilterConstant", - "GBDTFeatureSelector" + "GBDTFeatureSelector", + "IdentityFeature", + "OnlyConstFeature", + "AutoFeatureEngineer" ] \ No newline at end of file diff --git a/autogl/module/feature/_auto_feature.py b/autogl/module/feature/_auto_feature.py index 113eb652..8e4e58ad 100644 --- a/autogl/module/feature/_auto_feature.py +++ b/autogl/module/feature/_auto_feature.py @@ -166,7 +166,7 @@ def __init__( verbosity: int = 0, *args, **kwargs ): - super(AutoFeatureEngineer, self).__init__(multi_graph=False) + super(AutoFeatureEngineer, self).__init__() self._ops = [op_sum, op_mean, op_max, op_min] self._sim = cosine_similarity self._fixlen = fix_length @@ -208,7 +208,7 @@ def _fit(self, homogeneous_static_graph: autogl.data.graph.GeneralStaticGraph): for u, v in homogeneous_static_graph.edges.connections.t().numpy(): neighbours[u].append(v) self.__neighbours: _typing.Sequence[np.ndarray] = tuple( - [np.ndarray(v) for v in neighbours] + [np.array(v) for v in neighbours] ) x: np.ndarray = _original_features.numpy() diff --git a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py index 4920566d..e3b2f972 100644 --- a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py +++ b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py @@ -29,7 +29,11 @@ def transform(self, dataset, inplace: bool = True): return dataset def fit_transform(self, dataset, inplace: bool = True): - return self.fit_transform(dataset, inplace) + for fe in self.fe_components: + dataset = fe.fit(dataset) + for fe in self.fe_components: + dataset = fe.fit(dataset) + return dataset def __init__(self, feature_engineers: _typing.Iterable[_AbstractBaseFeatureEngineer]): self.__fe_components: _typing.List[_AbstractBaseFeatureEngineer] = [] diff --git a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_dgl.py b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_dgl.py index 695099b1..cf292843 100644 --- a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_dgl.py +++ b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_dgl.py @@ -40,7 +40,7 @@ def fit(self, dataset): with torch.no_grad(): for i, data in enumerate(dataset): dataset[i] = self.__postprocess( - self._postprocess(self._transform(self._fit(self._preprocess(self.__preprocess(data))))) + self._postprocess(self._fit(self._preprocess(self.__preprocess(data)))) ) return dataset diff --git a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_pyg.py b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_pyg.py index 5b213ebb..74d42ee5 100644 --- a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_pyg.py +++ b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer_pyg.py @@ -25,7 +25,7 @@ def fit(self, dataset): with torch.no_grad(): for i, data in enumerate(dataset): dataset[i] = self.__postprocess( - self._postprocess(self._transform(self._fit(self._preprocess(self.__preprocess(data))))) + self._postprocess(self._fit(self._preprocess(self.__preprocess(data)))) ) return dataset diff --git a/autogl/module/feature/_selectors/_basic.py b/autogl/module/feature/_selectors/_basic.py index d7d93db3..413d3a20 100644 --- a/autogl/module/feature/_selectors/_basic.py +++ b/autogl/module/feature/_selectors/_basic.py @@ -16,16 +16,12 @@ def __transform_homogeneous_static_graph( ) -> GeneralStaticGraph: if ( 'x' in static_graph.nodes.data and - self._selection not in (Ellipsis, None) and - isinstance(self._selection, torch.Tensor) and - torch.is_tensor(self._selection) and self._selection.dim() == 1 + isinstance(self._selection, (torch.Tensor, np.ndarray)) ): static_graph.nodes.data['x'] = static_graph.nodes.data['x'][:, self._selection] if ( 'feat' in static_graph.nodes.data and - self._selection not in (Ellipsis, None) and - isinstance(self._selection, torch.Tensor) and - torch.is_tensor(self._selection) and self._selection.dim() == 1 + isinstance(self._selection, (torch.Tensor, np.ndarray)) ): static_graph.nodes.data['feat'] = static_graph.nodes.data['feat'][:, self._selection] return static_graph diff --git a/autogl/module/feature/_selectors/_gbdt.py b/autogl/module/feature/_selectors/_gbdt.py index 88cb9c61..f35974f9 100644 --- a/autogl/module/feature/_selectors/_gbdt.py +++ b/autogl/module/feature/_selectors/_gbdt.py @@ -41,6 +41,7 @@ def _gbdt_generator( ) num_classes: int = torch.max(data.y).item() + 1 + parameters["num_class"] = num_classes __optimizer_parameters = { "num_boost_round": 100, "early_stopping_rounds": 5, @@ -78,7 +79,7 @@ def _gbdt_generator( train_x = pd.DataFrame(x, columns=feature_index, index=None) dtrain = lightgbm.Dataset(train_x, label=label) clf = lightgbm.train( - train_set=dtrain, params=params, + train_set=dtrain, params=parameters, **__optimizer_parameters ) From 7bce26e03054932e26d00f128fb4aa17599b6865 Mon Sep 17 00:00:00 2001 From: CoreLeader Date: Mon, 28 Mar 2022 22:26:00 +0800 Subject: [PATCH 02/37] Fix bugs for feature engineers --- .../feature/_base_feature_engineer/_base_feature_engineer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py index e3b2f972..0df049bc 100644 --- a/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py +++ b/autogl/module/feature/_base_feature_engineer/_base_feature_engineer.py @@ -32,7 +32,7 @@ def fit_transform(self, dataset, inplace: bool = True): for fe in self.fe_components: dataset = fe.fit(dataset) for fe in self.fe_components: - dataset = fe.fit(dataset) + dataset = fe.transform(dataset) return dataset def __init__(self, feature_engineers: _typing.Iterable[_AbstractBaseFeatureEngineer]): From daa31fb7f8e1e201450734b46c35cbdbb11f4112 Mon Sep 17 00:00:00 2001 From: Generall Date: Fri, 1 Apr 2022 16:18:03 +0800 Subject: [PATCH 03/37] add tutorial_cn --- examples/quickstart.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 examples/quickstart.py diff --git a/examples/quickstart.py b/examples/quickstart.py new file mode 100644 index 00000000..915222ec --- /dev/null +++ b/examples/quickstart.py @@ -0,0 +1,21 @@ +from autogl.datasets import build_dataset_from_name +cora_dataset = build_dataset_from_name('cora', path = '/home/qinyj/AGL/') + +import torch +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +from autogl.solver import AutoNodeClassifier +solver = AutoNodeClassifier( + graph_models=['gcn', 'gat'], + hpo_module='anneal', + ensemble_module='voting', + device=device +) + +solver.fit(cora_dataset, time_limit=3600) +solver.get_leaderboard().show() + +from autogl.module.train import Acc +predicted = solver.predict_proba() +print('Test accuracy: ', Acc.evaluate(predicted, + cora_dataset.data.y[cora_dataset.data.test_mask].cpu().numpy())) + From da7a17901e54728a47894f0074545a626d7dc4ca Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 14:49:27 +0800 Subject: [PATCH 04/37] add tutorial_cn/t_hetero_node_clf.rst --- .../docfile/tutorial_cn/t_hetero_node_clf.rst | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 docs/docfile/tutorial_cn/t_hetero_node_clf.rst diff --git a/docs/docfile/tutorial_cn/t_hetero_node_clf.rst b/docs/docfile/tutorial_cn/t_hetero_node_clf.rst new file mode 100644 index 00000000..ce1f0c3e --- /dev/null +++ b/docs/docfile/tutorial_cn/t_hetero_node_clf.rst @@ -0,0 +1,159 @@ +.. _hetero_node_clf: + +.. Node Classification for Heterogeneous Graph +异质图上的节点分类 +=========================================== + +.. This tutorial introduces how to use AutoGL to automate the learning of heterogeneous graphs in Deep Graph Library (DGL). +本教程指导如何利用AutoGL来对Deep Graph Library (DGL)中的异质图进行自动学习。 + +.. Creating a Heterogeneous Graph +创建一个异质图 +------------------------------ +.. AutoGL supports datasets created in DGL. We provide two datasets named "hetero-acm-han" and "hetero-acm-hgt" for HAN and HGT models, respectively [1]. +AutoGL支持DGL内的数据集。我们分别对HAN和HGT这两个模型提供了两个数据集，分别叫做"hetero-acm-han"和"hetero-acm-hgt" [1]。 +.. The following code snippet is an example for loading a heterogeneous graph. +下面的代码片断提供了一个加载异质图的例子。 + +.. code-block:: python + + from autogl.datasets import build_dataset_from_name + dataset = build_dataset_from_name("hetero-acm-han") + +.. You can also access to data stored in the dataset object for more details: +你也可以通过访问存储在数据集对象中的数据来了解更多细节。 + +.. code-block:: python + + g = dataset[0] + + node_type = dataset.schema["target_node_type"] + labels = g.nodes[node_type].data['label'] + num_classes = labels.max().item() + 1 + num_features=g.nodes[node_type].data['feat'].shape[1] + + train_mask = g.nodes[node_type].data['train_mask'] + val_mask = g.nodes[node_type].data['val_mask'] + test_mask = g.nodes[node_type].data['test_mask'] + +.. You can also build your own dataset and do feature engineering by adding files in the location AutoGL/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py. We suggest users create a data object of type torch_geometric.data.HeteroData refering to the official documentation of DGL. +你也可以通过在 AutoGL/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py 目录下添加文件来建立自己的数据集并进行特征工程。我们建议用户参考DGL的官方文档，创建一个类型为torch_geometric.data.HeteroData的数据对象。 + +.. Building Heterogeneous GNN Modules +构建异质图神经网络模块 +---------------------------------- +.. AutoGL integrates commonly used heterogeneous graph neural network models such as HeteroRGCN (Schlichtkrull et al., 2018) [2], HAN (Wang et al., 2019) [3] and HGT (Hu et al., 2029) [4]. +AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlichtkrull et al., 2018) [2]，HAN (Wang et al., 2019) [3]和HGT (Hu et al., 2029) [4]。 + +.. code-block:: python + + from autogl.module.model.dgl import AutoHAN + model = AutoHAN( + dataset=dataset, + num_features=num_features, + num_classes=num_classes, + device = args['device'], + init=True + ).model + +.. Then you can train the model for 100 epochs. +然后你可以对模型进行100次的训练。 + +.. code-block:: python + + # Define the loss function. + loss_fcn = torch.nn.CrossEntropyLoss() + # Define the loss optimizer. + optimizer = torch.optim.Adam(model.parameters(), lr=1e-2, + weight_decay=1e-2) + + # Training. + for epoch in range(100): + model.train() + logits = model(g) + loss = loss_fcn(logits[train_mask], labels[train_mask]) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + val_loss, val_acc, _, _ = evaluate(model, g, labels, val_mask, loss_fcn) + +.. Finally, evaluate the model. +最后可以评估该模型。 + +.. code-block:: python + + _, test_acc, _, _ = evaluate(model, g, labels, test_mask, loss_fcn) + +.. You can also define your own heterogeneous graph neural network models by adding files in the location AutoGL/autogl/module/model/dgl/hetero. +你也可以通过在 AutoGL/autogl/module/model/dgl/hetero 目录下添加文件来定义自己的异质图神经网络模型。 + +.. Automatic Search for Node Classification Tasks +节点分类任务的自动搜索 +---------------------------------------------- +.. On top of the modules mentioned above, we provide a high-level API Solver to control the overall pipeline. We encapsulated the training process in the Building Heterogeneous GNN Modules part in the solver AutoHeteroNodeClassifier that supports automatic hyperparametric optimization as well as feature engineering and ensemble. +.. In this part, we will show you how to use AutoHeteroNodeClassifier to automatically predict the publishing conference of a paper using the ACM academic graph dataset. +在上述模块的基础上，我们提供了一个高级API求解器来控制整个流水线。我们将构建异质图神经网络模块部分的训练过程封装在求解器AutoHeteroNodeClassifier中，它支持自动超参数优化以，特征工程及集成。 +在这一部分，我们将使用ACM学术图数据集，来向你展示如何使用AutoHeteroNodeClassifier自动预测一篇论文发表在哪个会议上。 + +.. Firstly, we get the pre-defined model hyperparameter. +首先，我们得到预先定义的模型超参数。 + +.. code-block:: python + + from helper import get_encoder_decoder_hp + model_hp, _ = get_encoder_decoder_hp(args.model) + +.. You can also define your own model hyperparameters in a dict: +你也可以在一个字典（dict）中定义你自己的模型超参数。 + +.. code-block:: python + + model_hp = { + "num_layers": 2, + "hidden": [256], + "heads": 4, + "dropout": 0.2, + "act": "leaky_relu", + } + +.. Secondly, use AutoHeteroNodeClassifier directly to bulid automatic heterogeneous GNN models in the following example: +其次，在下面的例子中，直接使用AutoHeteroNodeClassifier来构建自动异质图神经网络模型。 + +.. code-block:: python + + from autogl.solver import AutoHeteroNodeClassifier + solver = AutoHeteroNodeClassifier( + graph_models=["han"], + hpo_module="random", + ensemble_module=None, + max_evals=1, + device=args.device, + trainer_hp_space=fixed( + max_epoch=100, + early_stopping_round=101, + lr=1e-3, + weight_decay=1e-2 + ), + model_hp_spaces=[fixed(**model_hp)] + ) + +.. Finally, fit and evlauate the model. +最后，对模型进行拟合和评估。 + +.. code-block:: python + + solver.fit(dataset) + acc = solver.evaluate() + +.. References: +参考文献： + +[1] https://data.dgl.ai/dataset/ACM.mat + +[2] Schlichtkrull, Michael, et al. "Modeling relational data with graph convolutional networks." European semantic web conference. Springer, Cham, 2018. + +[3] Wang, Xiao, et al. "Heterogeneous graph attention network." The World Wide Web Conference. 2019. + +[4] Yun, Seongjun, et al. "Graph transformer networks." Advances in Neural Information Processing Systems 32 (2019): 11983-11993. From b6965053af783fa85c8ee5b63ea67cbd9f381482 Mon Sep 17 00:00:00 2001 From: BeiniXie Date: Tue, 12 Apr 2022 15:08:33 +0800 Subject: [PATCH 05/37] model tutorial --- docs/docfile/tutorial/t_model_chinese.rst | 215 ++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 docs/docfile/tutorial/t_model_chinese.rst diff --git a/docs/docfile/tutorial/t_model_chinese.rst b/docs/docfile/tutorial/t_model_chinese.rst new file mode 100644 index 00000000..4e438ce6 --- /dev/null +++ b/docs/docfile/tutorial/t_model_chinese.rst @@ -0,0 +1,215 @@ +.. _model: + +AutoGL 模型 +============ +在AutoGL中，我们使用``model``和``automodel``类定义图神经网络模型，并让它们和超参数优化(hyper parameter optimization, HPO)模块兼容。 + +当前版本下，我们支持节点分类、图分类和链接预测三种任务任务，支持的具体模型如下： + ++----------------------+----------------------------+ +| 任务 | 模型 | ++======================+============================+ +| 节点分类 | ``gcn``, ``gat``, ``sage`` | ++----------------------+----------------------------+ +| 图分类 | ``gin``, ``topk`` | ++----------------------+----------------------------+ +| 链接预测 | ``gcn``, ``gat``, ``sage`` | ++----------------------+----------------------------+ + +自定义模型和自动模型 +------------------ +我们强烈建议您同时定义``model``类和``automodel``类。 +其中，``model``类来管理参数的初始化与模型前向传播逻辑，``automodel``类组织超参数相关的搜索。 +``automodel``在``solver``和``trainer``模块会被调用。 + +示例 +^^^^ +以一个用于节点分类任务的多层感知机(MLP)为例。您可以使用AutoGL来帮您找到最合适的超参数。 + +首先，您可以定义一个MLP模型，并假设所有超参数已经给定。 + +.. code-block:: python + + import torch + + class MyMLP(torch.nn.Module): + # 假定所有超参数可获得 + def __init__(self, args): + super().__init__() + in_channels, num_classes = args['in_channels'], args['num_classes'] + layer_num, dim = args['layer_num'], int(args['dim']) + + if layer_num == 1: + ops = [torch.nn.Linear(in_channels, num_classes)] + else: + ops = [torch.nn.Linear(in_channels, dim)] + for i in range(layer_num - 2): + ops.append(torch.nn.Linear(dim, dim)) + ops.append(torch.nn.Linear(dim, num_classes)) + + self.core = torch.nn.Sequential(*ops) + + # 必须利用forward函数定义模型的前向传播逻辑 + def forward(self, data): + assert hasattr(data, 'x'), 'MLP only support graph data with features' + x = data.x + return torch.nn.functional.log_softmax(self.core(x)) + + +接下来，您可以定义自动模型``automodel``类以更好管理您的超参数。 +对于来自于数据集的参数如输入维度与输出维度，可以直接传入``automodel``类中的初始化函数中``__init__()``。 +而对于需要搜索的其他超参数，需要自定义搜索空间。 + +.. code-block:: python + + from autogl.module.model import BaseAutoModel + + # 定义自动模型类，需要从BaseAutoModel类继承 + class MyAutoMLP(BaseAutoModel): + def __init__(self, num_features=None, num_classes=None, device=None, **args + ): + super().__init__(num_features, num_classes, device, **args) + + # (required) 需要定义搜索空间（包含超参数、超参数的类型以及搜索范围） + self.space = [ + {'parameterName': 'layer_num', 'type': 'INTEGER', 'minValue': 1, 'maxValue': 5, 'scalingType': 'LINEAR'}, + {'parameterName': 'dim', 'type': 'INTEGER', 'minValue': 64, 'maxValue': 128, 'scalingType': 'LINEAR'} + ] + + # 设置默认超参数 + self.hyper_parameters = { + "layer_num": 2, + "dim": 72, + } + + + # # (required) since we don't know the num_classes and num_features until we see the dataset, + # # we cannot initialize the models when instantiated. the initialized will be set to False. + # self.initialized = False + + + # (required) instantiate the core MLP model using corresponding hyper-parameters + def _initialize(self): + # (required) you need to make sure the core model is named as `self.model` + self.model = MyMLP({ + "in_channels": self.input_dimension, + "num_classes": self.output_dimension, + **self.hyper_parameters + } + ).to(self.device) + + + +接着，只需要将定义好的自动图模型输入自动图分类任务的``solver``中，就可以利用它完成节点分类任务。 +具体代码示例如下： +.. code-block :: python + + from autogl.solver import AutoNodeClassifier + + solver = AutoNodeClassifier(graph_models=(MyAutoMLP(num_features, num_classes,device=torch.device('cuda')),)) + + + +图分类任务的模型定义和整个流程和节点分类任务相似。详情参考图分类模型的tutorial。 + + +用于链接预测任务的模型 +^^^^^^^^^^^^^^^^^^^^ + +对于链接预测任务，模型的定义在``forward()``函数中略有不同。 +为了更好地和链接预测训练器``LinkPredictionTrainer``与自动链接预测器``AutoLinkPredictor``交互，您需要定义编码函数``lp_encode(self, data)``与解码函数``lp_decode(self, x, pos_edge_index, neg_edge_index)`` + +用同样的多层感知机作为示例，如果您想要将其用于链接预测任务，那么您不必再定义``forward()``函数，而是定义``lp_encode(self, data)``与``lp_decode(self, x, pos_edge_index, neg_edge_index)``两个函数。具体代码示例如下： + +.. code-block:: python + + class MyMLPForLP(torch.nn.Module): + def __init__(self, in_channels, layer_num, dim): + super().__init__() + ops = [torch.nn.Linear(in_channels, dim)] + for i in range(layer_num - 1): + ops.append(torch.nn.Linear(dim, dim)) + + self.core = torch.nn.Sequential(*ops) + + # (required) 和trainer与solver模块交互 + def lp_encode(self, data): + return self.core(data.x) + + # (required) 和trainer与solver模块交互 + def lp_decode(self, x, pos_edge_index, neg_edge_index): + # 首先得到所有需要的正样本边与负样本边集合 + edge_index = torch.cat([pos_edge_index, neg_edge_index], dim=-1) + # 利用点积计算logits，或者使用其他decode方法 + logits = (x[edge_index[0]] * x[edge_index[1]]).sum(dim=-1) + return logits + + class MyAutoMLPForLP(MyAutoMLP): + def initialize(self): + self.model = MyMLPForLP( + in_channels = self.num_features, + layer_num = self.layer_num, + dim = self.dim + ).to(self.device) + + + +支持采样的模型 +^^^^^^^^^^^^^ +为了高效地实现大规模图上表示学习，AutoGL目前支持使用节点级别(node-wise)的采样、层级别(layer-wise)的采样和子图级别(subgraph-wise)的采样等采样技术进行节点分类。 +有关采样的更多信息，请参阅：:ref:`trainer`。 + +根据图神经网络中的消息传递机制，一个节点的表达由它多跳邻居构成的子图决定。 +但是，节点的邻居数量随着神经网络层数的增加呈现指数级增长，计算并储存所有节点的表达会占用许多的计算资源。 +因此，在得到节点表达时，我们可以在每层神经网络输入不同的采样后的子图以达到高效计算的目的。 +以torch_geometric的data为例，一个图包含节点特征x和边集合edge_index，在AutoGL的采样技巧中，我们会为data提供edge_indexes属性以表示不同的图卷积层采样出来的不同子图。 + +.. code-block:: python + + import autogl + from autogl.module.model import ClassificationSupportedSequentialModel + + # 重新定义接收图作为输入的Linear类 + class Linear(torch.nn.Linear): + def forward(self, data): + return super().forward(data.x) + + class MyMLPSampling(ClassificationSupportedSequentialModel): + def __init__(self, in_channels, num_classes, layer_num, dim): + super().__init__() + if layer_num == 1: + ops = [Linear(in_channels, num_classes)] + else: + ops = [Linear(in_channels, dim)] + for i in range(layer_num - 2): + ops.append(Linear(dim, dim)) + ops.append(Linear(dim, num_classes)) + + self.core = torch.nn.ModuleList(ops) + + # (required) 覆盖序列编码层sequential_encoding_layers()，和sampling交互 + @property + def sequential_encoding_layers(self) -> torch.nn.ModuleList: + return self.core + + # (required) define the encode logic of classification for sampling + def cls_encode(self, data): + if hasattr(data, 'edge_indexes'): + # edge_indexes是由edge_index组成的列表，每个edge_index代表每层图卷积所使用的边 + edge_indexes = data.edge_indexes + edge_weights = [None] * len(self.core) if getattr(data, 'edge_weights', None) is None else data.edge_weights + else: + # 默认edge_index和edge_weight是相同的 + edge_indexes = [data.edge_index] * len(self.core) + edge_weights = [getattr(data, 'edge_weight', None)] * len(self.core) + + x = data.x + for i in range(len(self.core)): + data = autogl.data.Data(x=x, edge_index=edge_indexes[i]) + data.edge_weight = edge_weights[i] + x = self.sequential_encoding_layers[i](data) + return x + + def cls_decode(self, x): + return torch.nn.functional.log_softmax(x) + From ec1f23164805866d25a087ba12544c770f92c29d Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 15:16:11 +0800 Subject: [PATCH 06/37] add tutorial_cn/t_homo_graph_classification_gin.rst --- .../docfile/tutorial_cn/t_hetero_node_clf.rst | 27 +- .../t_homo_graph_classification_gin.rst | 328 ++++++++++++++++++ 2 files changed, 342 insertions(+), 13 deletions(-) create mode 100644 docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst diff --git a/docs/docfile/tutorial_cn/t_hetero_node_clf.rst b/docs/docfile/tutorial_cn/t_hetero_node_clf.rst index ce1f0c3e..2eb67ebf 100644 --- a/docs/docfile/tutorial_cn/t_hetero_node_clf.rst +++ b/docs/docfile/tutorial_cn/t_hetero_node_clf.rst @@ -11,9 +11,10 @@ 创建一个异质图 ------------------------------ .. AutoGL supports datasets created in DGL. We provide two datasets named "hetero-acm-han" and "hetero-acm-hgt" for HAN and HGT models, respectively [1]. -AutoGL支持DGL内的数据集。我们分别对HAN和HGT这两个模型提供了两个数据集，分别叫做"hetero-acm-han"和"hetero-acm-hgt" [1]。 +AutoGL支持DGL内的数据集。我们分别对HAN和HGT这两个模型提供了两个数据集，分别叫做 ``hetero-acm-han`` 和 ``hetero-acm-hgt`` [1]。 + .. The following code snippet is an example for loading a heterogeneous graph. -下面的代码片断提供了一个加载异质图的例子。 +下面的代码片断提供了一个加载异质图的例子： .. code-block:: python @@ -21,7 +22,7 @@ AutoGL支持DGL内的数据集。我们分别对HAN和HGT这两个模型提供 dataset = build_dataset_from_name("hetero-acm-han") .. You can also access to data stored in the dataset object for more details: -你也可以通过访问存储在数据集对象中的数据来了解更多细节。 +你也可以通过访问存储在数据集对象中的数据来了解更多细节： .. code-block:: python @@ -37,13 +38,13 @@ AutoGL支持DGL内的数据集。我们分别对HAN和HGT这两个模型提供 test_mask = g.nodes[node_type].data['test_mask'] .. You can also build your own dataset and do feature engineering by adding files in the location AutoGL/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py. We suggest users create a data object of type torch_geometric.data.HeteroData refering to the official documentation of DGL. -你也可以通过在 AutoGL/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py 目录下添加文件来建立自己的数据集并进行特征工程。我们建议用户参考DGL的官方文档，创建一个类型为torch_geometric.data.HeteroData的数据对象。 +你也可以通过在 AutoGL/autogl/datasets/_heterogeneous_datasets/_dgl_heterogeneous_datasets.py 目录下添加文件来建立自己的数据集并进行特征工程。我们建议用户参考DGL的官方文档，创建一个类型为 ``torch_geometric.data.HeteroData`` 的数据对象。 .. Building Heterogeneous GNN Modules 构建异质图神经网络模块 ---------------------------------- .. AutoGL integrates commonly used heterogeneous graph neural network models such as HeteroRGCN (Schlichtkrull et al., 2018) [2], HAN (Wang et al., 2019) [3] and HGT (Hu et al., 2029) [4]. -AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlichtkrull et al., 2018) [2]，HAN (Wang et al., 2019) [3]和HGT (Hu et al., 2029) [4]。 +AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlichtkrull et al., 2018) [2]，HAN (Wang et al., 2019) [3]和HGT (Hu et al., 2029) [4]： .. code-block:: python @@ -57,7 +58,7 @@ AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlicht ).model .. Then you can train the model for 100 epochs. -然后你可以对模型进行100次的训练。 +然后你可以对模型进行100次的训练： .. code-block:: python @@ -80,7 +81,7 @@ AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlicht val_loss, val_acc, _, _ = evaluate(model, g, labels, val_mask, loss_fcn) .. Finally, evaluate the model. -最后可以评估该模型。 +最后，你可以评估该模型： .. code-block:: python @@ -94,11 +95,11 @@ AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlicht ---------------------------------------------- .. On top of the modules mentioned above, we provide a high-level API Solver to control the overall pipeline. We encapsulated the training process in the Building Heterogeneous GNN Modules part in the solver AutoHeteroNodeClassifier that supports automatic hyperparametric optimization as well as feature engineering and ensemble. .. In this part, we will show you how to use AutoHeteroNodeClassifier to automatically predict the publishing conference of a paper using the ACM academic graph dataset. -在上述模块的基础上，我们提供了一个高级API求解器来控制整个流水线。我们将构建异质图神经网络模块部分的训练过程封装在求解器AutoHeteroNodeClassifier中，它支持自动超参数优化以，特征工程及集成。 -在这一部分，我们将使用ACM学术图数据集，来向你展示如何使用AutoHeteroNodeClassifier自动预测一篇论文发表在哪个会议上。 +在上述模块的基础上，我们提供了一个高级API求解器来控制整个流水线。我们将构建异质图神经网络模块部分的训练过程封装在求解器 ``AutoHeteroNodeClassifier`` 中，它支持自动超参数优化，特征工程及集成。 +在这一部分，我们将使用ACM学术图数据集，来向你展示如何使用 ``AutoHeteroNodeClassifier`` 自动预测一篇论文发表在哪个会议上。 .. Firstly, we get the pre-defined model hyperparameter. -首先，我们得到预先定义的模型超参数。 +首先，我们得到预先定义的模型超参数： .. code-block:: python @@ -106,7 +107,7 @@ AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlicht model_hp, _ = get_encoder_decoder_hp(args.model) .. You can also define your own model hyperparameters in a dict: -你也可以在一个字典（dict）中定义你自己的模型超参数。 +你也可以在一个字典（dict）中定义你自己的模型超参数： .. code-block:: python @@ -119,7 +120,7 @@ AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlicht } .. Secondly, use AutoHeteroNodeClassifier directly to bulid automatic heterogeneous GNN models in the following example: -其次，在下面的例子中，直接使用AutoHeteroNodeClassifier来构建自动异质图神经网络模型。 +然后，在下面的例子中，直接使用 ``AutoHeteroNodeClassifier`` 来构建自动异质图神经网络模型： .. code-block:: python @@ -140,7 +141,7 @@ AutoGL集成了常用的异质图神经网络模型，例如HeteroRGCN (Schlicht ) .. Finally, fit and evlauate the model. -最后，对模型进行拟合和评估。 +最后，对模型进行拟合和评估： .. code-block:: python diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst new file mode 100644 index 00000000..4334a67d --- /dev/null +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -0,0 +1,328 @@ +========================== +.. Graph Classification Model +图分类模型 +========================== + +.. Building Graph Classification Modules +构建图分类模块 +===================================== + +.. In AutoGL, we support two graph classification models, ``gin`` and ``topk``. +在AutoGL中，我们支持两种图分类模型：``gin`` and ``topk`` 。 + +AutoGIN +>>>>>>> + +.. The graph isomorphism operator from the “How Powerful are Graph Neural Networks?” paper +图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中 + +.. Graph Isomorphism Network (GIN) is one graph classification model from `"How Powerful are Graph Neural Networks" paper `_. +图同构网络（Graph Isomorphism Network (GIN)是出自论文`"How Powerful are Graph Neural Networks" paper `_中的一种图神经网络。 + +.. The layer is +层间更新方式为： + +.. math:: + + \mathbf{x}^{\prime}_i = h_{\mathbf{\Theta}} \left( (1 + \epsilon) \cdot + \mathbf{x}_i + \sum_{j \in \mathcal{N}(i)} \mathbf{x}_j \right) + +或者： + +.. math:: + + \mathbf{X}^{\prime} = h_{\mathbf{\Theta}} \left( \left( \mathbf{A} + + (1 + \epsilon) \cdot \mathbf{I} \right) \cdot \mathbf{X} \right), + +.. here :math:`h_{\mathbf{\Theta}}` denotes a neural network, *.i.e.* an MLP. +这里 :math:`h_{\mathbf{\Theta}}`代表一个, *.i.e.* 一个多层感知机. + +.. PARAMETERS: +参数包括： +.. - num_features: `int` - The dimension of features. +- num_features: `int` - 特征的维度. + +.. - num_classes: `int` - The number of classes. +- num_classes: `int` - 类别的数量. + +.. - device: `torch.device` or `str` - The device where model will be running on. +- device: `torch.device` or `str` - 用于运行模型的设备. + +.. - init: `bool` - If True(False), the model will (not) be initialized. +- init: `bool` - 如果设为True（False），模型会（不会）被初始化. + +.. code-block:: python + + class AutoGIN(BaseModel): + r""" + AutoGIN. The model used in this automodel is GIN, i.e., the graph isomorphism network from the `"How Powerful are + Graph Neural Networks?" `_ paper. The layer is + + .. math:: + \mathbf{x}^{\prime}_i = h_{\mathbf{\Theta}} \left( (1 + \epsilon) \cdot + \mathbf{x}_i + \sum_{j \in \mathcal{N}(i)} \mathbf{x}_j \right) + + or + + .. math:: + \mathbf{X}^{\prime} = h_{\mathbf{\Theta}} \left( \left( \mathbf{A} + + (1 + \epsilon) \cdot \mathbf{I} \right) \cdot \mathbf{X} \right), + + here :math:`h_{\mathbf{\Theta}}` denotes a neural network, *.i.e.* an MLP. + + Parameters + ---------- + num_features: `int`. + The dimension of features. + + num_classes: `int`. + The number of classes. + + device: `torch.device` or `str` + The device where model will be running on. + + init: `bool`. + If True(False), the model will (not) be initialized. + """ + + def __init__( + self, + num_features=None, + num_classes=None, + device=None, + init=False, + num_graph_features=None, + **args + ): + + super(AutoGIN, self).__init__() + self.num_features = num_features if num_features is not None else 0 + self.num_classes = int(num_classes) if num_classes is not None else 0 + self.num_graph_features = ( + int(num_graph_features) if num_graph_features is not None else 0 + ) + self.device = device if device is not None else "cpu" + + self.params = { + "features_num": self.num_features, + "num_class": self.num_classes, + "num_graph_features": self.num_graph_features, + } + self.space = [ + { + "parameterName": "num_layers", + "type": "DISCRETE", + "feasiblePoints": "4,5,6", + }, + { + "parameterName": "hidden", + "type": "NUMERICAL_LIST", + "numericalType": "INTEGER", + "length": 5, + "minValue": [8, 8, 8, 8, 8], + "maxValue": [64, 64, 64, 64, 64], + "scalingType": "LOG", + "cutPara": ("num_layers",), + "cutFunc": lambda x: x[0] - 1, + }, + { + "parameterName": "dropout", + "type": "DOUBLE", + "maxValue": 0.9, + "minValue": 0.1, + "scalingType": "LINEAR", + }, + { + "parameterName": "act", + "type": "CATEGORICAL", + "feasiblePoints": ["leaky_relu", "relu", "elu", "tanh"], + }, + { + "parameterName": "eps", + "type": "CATEGORICAL", + "feasiblePoints": ["True", "False"], + }, + { + "parameterName": "mlp_layers", + "type": "DISCRETE", + "feasiblePoints": "2,3,4", + }, + { + "parameterName": "neighbor_pooling_type", + "type": "CATEGORICAL", + "feasiblePoints": ["sum", "mean", "max"], + }, + { + "parameterName": "graph_pooling_type", + "type": "CATEGORICAL", + "feasiblePoints": ["sum", "mean", "max"], + }, + ] + + self.hyperparams = { + "num_layers": 5, + "hidden": [64,64,64,64], + "dropout": 0.5, + "act": "relu", + "eps": "False", + "mlp_layers": 2, + "neighbor_pooling_type": "sum", + "graph_pooling_type": "sum" + } + + self.initialized = False + if init is True: + self.initialize() + +.. Hyperparameters in GIN: +GIN中的超参数： + +.. - num_layers: `int` - number of GIN layers. + +.. - hidden: `List[int]` - hidden size for each hidden layer. + +.. - dropout: `float` - dropout probability. + +.. - act: `str` - type of activation function. + +.. - eps: `str` - whether to train parameter :math:`epsilon` in the GIN layer. + +.. - mlp_layers: `int` - number of MLP layers in the GIN layer. + +.. - neighbor_pooling_type: `str` - pooling type in the GIN layer. + +.. - graph_pooling_type: `str` - graph pooling type following the last GIN layer. +- num_layers: `int` - GIN的层数. + +- hidden: `List[int]` - 每个隐藏层的大小. + +- dropout: `float` - 随机失活（Dropout）的概率. + +- act: `str` - 激活函数的类型. + +- eps: `str` - 是否在GIN层中训练参数 :math:`epsilon` . + +- mlp_layers: `int` - GIN中的多层感知机（MLP）层数. + +- neighbor_pooling_type: `str` - GIN中的池化（pooling）层类型. + +- graph_pooling_type: `str` - GIN最后一层之后的图池化（graph pooling）类型. + + +.. You could get define your own ``gin`` model by using ``from_hyper_parameter`` function and specify the hyperpameryers. +You could get define your own ``gin`` model by using ``from_hyper_parameter`` function and specify the hyperpameryers. +你可以通过使用 ``from_hyper_parameter`` 函数定义你自己的 ``gin`` 模型，并对其指定超参数。 + +.. code-block:: python + + # pyg version + from autogl.module.model.pyg import AutoGIN + # from autogl.module.model.dgl import AutoGIN # dgl version + model = AutoGIN( + num_features=dataset.num_node_features, + num_classes=dataset.num_classes, + num_graph_features=0, + init=False + ).from_hyper_parameter({ + # hp from model + "num_layers": 5, + "hidden": [64,64,64,64], + "dropout": 0.5, + "act": "relu", + "eps": "False", + "mlp_layers": 2, + "neighbor_pooling_type": "sum", + "graph_pooling_type": "sum" + }).model + + +.. Then you can train the model for 100 epochs. +然后你可以对模型进行100次的训练： + +.. code-block:: python + + import torch.nn.functional as F + + # Define the loss optimizer. + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + # Training + for epoch in range(100): + model.train() + for data in train_loader: + data = data.to(args.device) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, data.y) + loss.backward() + optimizer.step() + +.. Finally, evaluate the trained model. +最后，你可以评估该模型： + +.. code-block:: python + + def test(model, loader, args): + model.eval() + + correct = 0 + for data in loader: + data = data.to(args.device) + output = model(data) + pred = output.max(dim=1)[1] + correct += pred.eq(data.y).sum().item() + return correct / len(loader.dataset) + + acc = test(model, test_loader, args) + + +.. Automatic Search for Graph Classification Tasks +图分类任务的自动搜索 +=============================================== + +.. In AutoGL, we also provide a high-level API Solver to control the overall pipeline. We encapsulated the training process in the Building GNN Modules part for graph classification tasks in the solver ``AutoGraphClassifier`` that supports automatic hyperparametric optimization as well as feature engineering and ensemble. In this part, we will show you how to use ``AutoGraphClassifier``. +.. In AutoGL, we also provide a high-level API Solver to control the overall pipeline. We encapsulated the training process in the Building GNN Modules part for graph classification tasks in the solver ``AutoGraphClassifier`` that supports automatic hyperparametric optimization as well as feature engineering and ensemble. In this part, we will show you how to use ``AutoGraphClassifier``. +在AutoGL中，我们还提供了一个高级的API求解器来控制整个流水线。我们将构建图神经网络模块部分的训练过程封装在求解器 ``AutoGraphClassifier`` 中以用于图分类任务，它支持自动超参数优化，特征工程及集成。 +在这一部分，我们提供了一个例子来指导如何使用 ``AutoGraphClassifier`` ： + +.. code-block:: python + + solver = AutoGraphClassifier( + feature_module=None, + graph_models=[args.model], + hpo_module='random', + ensemble_module=None, + device=args.device, max_evals=1, + trainer_hp_space = fixed( + **{ + # hp from trainer + "max_epoch": args.epoch, + "batch_size": args.batch_size, + "early_stopping_round": args.epoch + 1, + "lr": args.lr, + "weight_decay": 0, + } + ), + model_hp_spaces=[ + fixed(**{ + # hp from model + "num_layers": 5, + "hidden": [64,64,64,64], + "dropout": 0.5, + "act": "relu", + "eps": "False", + "mlp_layers": 2, + "neighbor_pooling_type": "sum", + "graph_pooling_type": "sum" + }) if args.model == 'gin' else fixed(**{ + "ratio": 0.8, + "dropout": 0.5, + "act": "relu" + }), + ] + ) + + # fit auto model + solver.fit(dataset, evaluation_method=['acc']) + # prediction + out = solver.predict(dataset, mask='test') From 83fd19484a4aee09298ebe1204328726186d55fa Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 15:22:44 +0800 Subject: [PATCH 07/37] fix bug in t_homo_graph_classification_gin.rst --- .../tutorial_cn/t_homo_graph_classification_gin.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst index 4334a67d..e207201d 100644 --- a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -1,23 +1,19 @@ ========================== -.. Graph Classification Model 图分类模型 ========================== -.. Building Graph Classification Modules 构建图分类模块 ===================================== - .. In AutoGL, we support two graph classification models, ``gin`` and ``topk``. -在AutoGL中，我们支持两种图分类模型：``gin`` and ``topk`` 。 +在AutoGL中，我们支持两种图分类模型： ``gin`` and ``topk`` 。 AutoGIN >>>>>>> .. The graph isomorphism operator from the “How Powerful are Graph Neural Networks?” paper -图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中 +图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中， -.. Graph Isomorphism Network (GIN) is one graph classification model from `"How Powerful are Graph Neural Networks" paper `_. -图同构网络（Graph Isomorphism Network (GIN)是出自论文`"How Powerful are Graph Neural Networks" paper `_中的一种图神经网络。 +图同构网络（Graph Isomorphism Network (GIN)是的一种图神经网,络出自论文`"How Powerful are Graph Neural Networks" paper `_。 .. The layer is 层间更新方式为： From 34cf1f3b5526eeb014888988860c101483e02704 Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 15:26:16 +0800 Subject: [PATCH 08/37] fix bug in t_homo_graph_classification_gin.rst --- docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst index e207201d..e50a3236 100644 --- a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -13,7 +13,7 @@ AutoGIN .. The graph isomorphism operator from the “How Powerful are Graph Neural Networks?” paper 图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中， -图同构网络（Graph Isomorphism Network (GIN)是的一种图神经网,络出自论文`"How Powerful are Graph Neural Networks" paper `_。 +图同构网络（Graph Isomorphism Network (GIN)是一种图神经网络，出自论文`"How Powerful are Graph Neural Networks" `_. .. The layer is 层间更新方式为： @@ -31,10 +31,11 @@ AutoGIN (1 + \epsilon) \cdot \mathbf{I} \right) \cdot \mathbf{X} \right), .. here :math:`h_{\mathbf{\Theta}}` denotes a neural network, *.i.e.* an MLP. -这里 :math:`h_{\mathbf{\Theta}}`代表一个, *.i.e.* 一个多层感知机. +这里 :math:`h_{\mathbf{\Theta}}`代表一个神经网络, *.i.e.* 一个多层感知机. .. PARAMETERS: 参数包括： + .. - num_features: `int` - The dimension of features. - num_features: `int` - 特征的维度. From 8fb1415a63084dadb486ae1a2ad7b03cbb6f38e1 Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 15:31:05 +0800 Subject: [PATCH 09/37] fix bug in t_homo_graph_classification_gin.rst --- .../t_homo_graph_classification_gin.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst index e50a3236..51600289 100644 --- a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -13,7 +13,7 @@ AutoGIN .. The graph isomorphism operator from the “How Powerful are Graph Neural Networks?” paper 图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中， -图同构网络（Graph Isomorphism Network (GIN)是一种图神经网络，出自论文`"How Powerful are Graph Neural Networks" `_. +图同构网络（Graph Isomorphism Network (GIN)）是一种图神经网络，出自论文`“How Powerful are Graph Neural Networks” `_ 。 .. The layer is 层间更新方式为： @@ -189,21 +189,21 @@ GIN中的超参数： .. - neighbor_pooling_type: `str` - pooling type in the GIN layer. .. - graph_pooling_type: `str` - graph pooling type following the last GIN layer. -- num_layers: `int` - GIN的层数. +- num_layers: `int` - GIN的层数。 -- hidden: `List[int]` - 每个隐藏层的大小. +- hidden: `List[int]` - 每个隐藏层的大小。 -- dropout: `float` - 随机失活（Dropout）的概率. +- dropout: `float` - 随机失活（Dropout）的概率。 -- act: `str` - 激活函数的类型. +- act: `str` - 激活函数的类型。 -- eps: `str` - 是否在GIN层中训练参数 :math:`epsilon` . +- eps: `str` - 是否在GIN层中训练参数 :math:`epsilon` 。 -- mlp_layers: `int` - GIN中的多层感知机（MLP）层数. +- mlp_layers: `int` - GIN中的多层感知机（MLP）层数。 -- neighbor_pooling_type: `str` - GIN中的池化（pooling）层类型. +- neighbor_pooling_type: `str` - GIN中的池化（pooling）层类型。 -- graph_pooling_type: `str` - GIN最后一层之后的图池化（graph pooling）类型. +- graph_pooling_type: `str` - GIN最后一层之后的图池化（graph pooling）类型。 .. You could get define your own ``gin`` model by using ``from_hyper_parameter`` function and specify the hyperpameryers. From e0fa52a2080fc6ab960de93730fe318784238ade Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 15:31:45 +0800 Subject: [PATCH 10/37] fix bug in t_homo_graph_classification_gin.rst --- docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst index 51600289..8aada8f7 100644 --- a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -13,7 +13,7 @@ AutoGIN .. The graph isomorphism operator from the “How Powerful are Graph Neural Networks?” paper 图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中， -图同构网络（Graph Isomorphism Network (GIN)）是一种图神经网络，出自论文`“How Powerful are Graph Neural Networks” `_ 。 +图同构网络（Graph Isomorphism Network (GIN)）是一种图神经网络，出自论文`“How Powerful are Graph Neural Networks” `_. .. The layer is 层间更新方式为： From 9613d26d45281e84a995886bc232fad29515dc3e Mon Sep 17 00:00:00 2001 From: SwiftieH Date: Tue, 12 Apr 2022 15:33:44 +0800 Subject: [PATCH 11/37] fix bug in t_homo_graph_classification_gin.rst --- docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst index 8aada8f7..25b568b8 100644 --- a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -13,7 +13,7 @@ AutoGIN .. The graph isomorphism operator from the “How Powerful are Graph Neural Networks?” paper 图同构算子（Graph Isomorphism Operator）出自论文“How Powerful are Graph Neural Networks?”中， -图同构网络（Graph Isomorphism Network (GIN)）是一种图神经网络，出自论文`“How Powerful are Graph Neural Networks” `_. +图同构网络（Graph Isomorphism Network (GIN)）是一种图神经网络，出自论文 `“How Powerful are Graph Neural Networks” `_ 。 .. The layer is 层间更新方式为： @@ -31,7 +31,7 @@ AutoGIN (1 + \epsilon) \cdot \mathbf{I} \right) \cdot \mathbf{X} \right), .. here :math:`h_{\mathbf{\Theta}}` denotes a neural network, *.i.e.* an MLP. -这里 :math:`h_{\mathbf{\Theta}}`代表一个神经网络, *.i.e.* 一个多层感知机. +这里 :math:`h_{\mathbf{\Theta}}` 代表一个神经网络, 例如一个多层感知机（MLP）. .. PARAMETERS: 参数包括： From dca18c2ca2eb17fd1fb068c4a1dc9a64642df183 Mon Sep 17 00:00:00 2001 From: BeiniXie Date: Tue, 12 Apr 2022 20:44:02 +0800 Subject: [PATCH 12/37] model_cn --- docs/docfile/tutorial/t_model_chinese.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/docfile/tutorial/t_model_chinese.rst b/docs/docfile/tutorial/t_model_chinese.rst index 4e438ce6..5a76e8f3 100644 --- a/docs/docfile/tutorial/t_model_chinese.rst +++ b/docs/docfile/tutorial/t_model_chinese.rst @@ -2,7 +2,7 @@ AutoGL 模型 ============ -在AutoGL中，我们使用``model``和``automodel``类定义图神经网络模型，并让它们和超参数优化(hyper parameter optimization, HPO)模块兼容。 +在AutoGL中，我们使用 ``model``和 ``automodel``类定义图神经网络模型，并让它们和超参数优化(hyper parameter optimization, HPO)模块兼容。 当前版本下，我们支持节点分类、图分类和链接预测三种任务任务，支持的具体模型如下： @@ -18,8 +18,8 @@ AutoGL 模型 自定义模型和自动模型 ------------------ -我们强烈建议您同时定义``model``类和``automodel``类。 -其中，``model``类来管理参数的初始化与模型前向传播逻辑，``automodel``类组织超参数相关的搜索。 +我们强烈建议您同时定义 ``model``类和 ``automodel``类。 +其中， ``model`` 类来管理参数的初始化与模型前向传播逻辑， ``automodel``类组织超参数相关的搜索。 ``automodel``在``solver``和``trainer``模块会被调用。 示例 @@ -56,8 +56,8 @@ AutoGL 模型 return torch.nn.functional.log_softmax(self.core(x)) -接下来，您可以定义自动模型``automodel``类以更好管理您的超参数。 -对于来自于数据集的参数如输入维度与输出维度，可以直接传入``automodel``类中的初始化函数中``__init__()``。 +接下来，您可以定义自动模型 ``automodel`` 类以更好管理您的超参数。 +对于来自于数据集的参数如输入维度与输出维度，可以直接传入 ``automodel`` 类中的初始化函数中 ``__init__()`` 。 而对于需要搜索的其他超参数，需要自定义搜索空间。 .. code-block:: python @@ -100,7 +100,7 @@ AutoGL 模型 -接着，只需要将定义好的自动图模型输入自动图分类任务的``solver``中，就可以利用它完成节点分类任务。 +接着，只需要将定义好的自动图模型输入自动图分类任务的 ``solver`` 中，就可以利用它完成节点分类任务。 具体代码示例如下： .. code-block :: python @@ -116,10 +116,10 @@ AutoGL 模型 用于链接预测任务的模型 ^^^^^^^^^^^^^^^^^^^^ -对于链接预测任务，模型的定义在``forward()``函数中略有不同。 -为了更好地和链接预测训练器``LinkPredictionTrainer``与自动链接预测器``AutoLinkPredictor``交互，您需要定义编码函数``lp_encode(self, data)``与解码函数``lp_decode(self, x, pos_edge_index, neg_edge_index)`` +对于链接预测任务，模型的定义在 ``forward()`` 函数中略有不同。 +为了更好地和链接预测训练器 ``LinkPredictionTrainer`` 与自动链接预测器 ``AutoLinkPredictor`` 交互，您需要定义编码函数 ``lp_encode(self, data)`` 与解码函数 ``lp_decode(self, x, pos_edge_index, neg_edge_index)`` 。 -用同样的多层感知机作为示例，如果您想要将其用于链接预测任务，那么您不必再定义``forward()``函数，而是定义``lp_encode(self, data)``与``lp_decode(self, x, pos_edge_index, neg_edge_index)``两个函数。具体代码示例如下： +用同样的多层感知机作为示例，如果您想要将其用于链接预测任务，那么您不必再定义 ``forward()`` 函数，而是定义 ``lp_encode(self, data)`` 与 ``lp_decode(self, x, pos_edge_index, neg_edge_index)`` 两个函数。具体代码示例如下： .. code-block:: python From 67b0bb05f813386dff3b589cde9bf923b828576c Mon Sep 17 00:00:00 2001 From: BeiniXie Date: Tue, 12 Apr 2022 20:49:21 +0800 Subject: [PATCH 13/37] model_cn --- docs/docfile/tutorial/t_model_chinese.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/docfile/tutorial/t_model_chinese.rst b/docs/docfile/tutorial/t_model_chinese.rst index 5a76e8f3..7d58fe86 100644 --- a/docs/docfile/tutorial/t_model_chinese.rst +++ b/docs/docfile/tutorial/t_model_chinese.rst @@ -2,10 +2,11 @@ AutoGL 模型 ============ -在AutoGL中，我们使用 ``model``和 ``automodel``类定义图神经网络模型，并让它们和超参数优化(hyper parameter optimization, HPO)模块兼容。 +在AutoGL中，我们使用 ``model`` 和 ``automodel`` 类定义图神经网络模型，并让它们和超参数优化(hyper parameter optimization, HPO)模块兼容。 当前版本下，我们支持节点分类、图分类和链接预测三种任务任务，支持的具体模型如下： + +----------------------+----------------------------+ | 任务 | 模型 | +======================+============================+ @@ -16,11 +17,12 @@ AutoGL 模型 | 链接预测 | ``gcn``, ``gat``, ``sage`` | +----------------------+----------------------------+ + 自定义模型和自动模型 ------------------ -我们强烈建议您同时定义 ``model``类和 ``automodel``类。 -其中， ``model`` 类来管理参数的初始化与模型前向传播逻辑， ``automodel``类组织超参数相关的搜索。 -``automodel``在``solver``和``trainer``模块会被调用。 +我们强烈建议您同时定义 ``model`` 类和 ``automodel`` 类。 +其中， ``model`` 类来管理参数的初始化与模型前向传播逻辑， ``automodel`` 类组织超参数相关的搜索。 +``automodel`` 在 ``solver`` 和 ``trainer`` 模块会被调用。 示例 ^^^^ From 73e6b87ee985caf0871de0dfc92098c6a3436105 Mon Sep 17 00:00:00 2001 From: BeiniXie Date: Tue, 12 Apr 2022 20:52:02 +0800 Subject: [PATCH 14/37] model_cn --- docs/docfile/tutorial/t_model_chinese.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/docfile/tutorial/t_model_chinese.rst b/docs/docfile/tutorial/t_model_chinese.rst index 7d58fe86..801347ae 100644 --- a/docs/docfile/tutorial/t_model_chinese.rst +++ b/docs/docfile/tutorial/t_model_chinese.rst @@ -8,13 +8,13 @@ AutoGL 模型 +----------------------+----------------------------+ -| 任务 | 模型 | +|任务 | 模型 | +======================+============================+ -| 节点分类 | ``gcn``, ``gat``, ``sage`` | +|节点分类 | ``gcn``, ``gat``, ``sage`` | +----------------------+----------------------------+ -| 图分类 | ``gin``, ``topk`` | +|图分类 | ``gin``, ``topk`` | +----------------------+----------------------------+ -| 链接预测 | ``gcn``, ``gat``, ``sage`` | +|链接预测 | ``gcn``, ``gat``, ``sage`` | +----------------------+----------------------------+ From 9184df249386efc36f19f1a082c5e84921be8d6b Mon Sep 17 00:00:00 2001 From: ZW-ZHANG Date: Wed, 13 Apr 2022 11:19:04 +0800 Subject: [PATCH 15/37] add README_cn.md --- README.md | 2 + README_cn.md | 170 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 README_cn.md diff --git a/README.md b/README.md index aee4d743..a9f57dff 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Auto Graph Learning +[Chinese Introduction](https://github.com/THUMNLab/AutoGL/blob/main/README_cn.md) + An autoML framework & toolkit for machine learning on graphs. *Actively under development by @THUMNLab* diff --git a/README_cn.md b/README_cn.md new file mode 100644 index 00000000..db083407 --- /dev/null +++ b/README_cn.md @@ -0,0 +1,170 @@ +# 智图 (AutoGL) +[English Introduction](https://github.com/THUMNLab/AutoGL) + +用于图数据的自动机器学习框架和工具包。 + +*由清华大学媒体与网络实验室进行开发与维护* + +若有任何意见或建议，欢迎通过issues 或邮件autogl@tsinghua.edu.cn与我们联系。 + + + +## 最新消息 + +- 2021.12.31 v0.3.0-pre版本更新! + - 智图目前支持[__Deep Graph Library (DGL)__](https://www.dgl.ai/)作为后端，以方便DGL的用户使用。目前在DGL后端已经支持同构图的节点分类、链接预测以及图分类等任务。智图现在也可兼容PyG 2.0版本。 + - 智图可以支持__异构图__节点分类任务!详情请参考[异构图教程](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_hetero_node_clf.html)。 + - 为了使智图算法更灵活，`model`模块目前支持__解耦__为两个子模块，即编码器`encoder`和解码器`decoder`。在__解耦__设计中，一个`encoder`可以被用来处理不同任务，以减少重复开发的负担， + - 我们扩展了支持的[神经架构搜索算法](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_nas.html)，例如[AutoAttend](https://proceedings.mlr.press/v139/guan21a.html)，[GASSO](https://proceedings.neurips.cc/paper/2021/hash/8c9f32e03aeb2e3000825c8c875c4edd-Abstract.html)， [硬件感知算法](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/documentation/nas.html#autogl.module.nas.estimator.OneShotEstimator_HardwareAware)等。 +- 2021.07.11 智图更新v0.2.0-pre版本! 在新版本中，智图支持[神经架构搜索(NAS)](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_nas.html)，可以对给定的数据集和架构定制化神经网络架构。智图也支持了[采样](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_trainer.html#node-classification-with-sampling)功能以处理大规模图数据集，包括节点采样、层采样和子图采样。链接预测任务也已经支持。详情请参考我们的[教程](http://mn.cs.tsinghua.edu.cn/autogl/documentation/index.html). +- 2021.04.16 我们关于图自动机器学习的综述文章已经被IJCAI 2021接受! 详情见[这里](http://arxiv.org/abs/2103.00742)。 +- 2021.04.10 我们的论文[__AutoGL: A Library for Automated Graph Learning__](https://arxiv.org/abs/2104.04987)已经被 _ICLR 2021 Workshop on Geometrical and Topological Representation Learning_ 接受! + +## 介绍 + +智图的设计目标是可以简单、快速地对图数据集和任务进行自动机器学习，可供研究者和开发者使用。更多详细信息，可以参阅我们的文档。 + +下图是智图的整体框架。 + + + +智图通过 `datasets` 类以支持图数据集，其基于 PyTorch Geometric 和 Deep Graph Library 的数据集，并添加了一些函数以支持自动机器学习框架。 + +智图通过 `AutoGL solvers` 以处理不同的图机器学习任务，利用五个主要模块自动解决给定的任务，即自动特征工程 `auto feature engineer`，神经架构搜索 `neural architecture search`，自动模型 `auto model`，超参数优化 `hyperparameter optimization`，和自动模型集成 `auto ensemble`。 + +目前，智图支持以下算法： + + + + + + + + + + + + + + + + + + +
特征工程图模型神经架构搜索超参数优化模型集成
生成器
Graphlets
EigenGNN
更多 ...

选择器
SeFilterConstant
gbdt

全图特征
Netlsd
NxAverageClustering
更多 ...		同构图编码器
GCNEncoder
GATEncoder
SAGEEncoder
GINEncoder

解码器
LogSoftmaxDecoder
DotProductDecoder
SumPoolMLPDecoder
JKSumPoolDecoder 		+ 搜索算法
+ Random
+ RL
+ Evolution
+ GASSO
+ 更多 ...

+ 搜索空间
+ SinglePath
+ GraphNas
+ AutoAttend
+ 更多 ...

+ 模型评估
+ Oneshot
+ Scratch
+ 		Grid
Random
Anneal
Bayes
CAMES
MOCAMES
Quasi random
TPE
AutoNE 		Voting
Stacking
+ +此工具包还可作为一个框架供用户实现和测试自己的自动机器学习或图机器学习模型。 + +## 安装 + +### 依赖 + +在安装智图之前，请首先安装以下依赖项。 + +1. Python >= 3.6.0 + +2. PyTorch (>=1.6.0) + + 详细信息请参考。 + +3. 图机器学习工具包 + + 智图需要 PyTorch Geometric（PyG）或 Deep Graph Library（DGL）作为后端。若两者均安装，可在运行时选择任一后端，参考[这里](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_backend.html)。 + + 3.1 PyTorch Geometric (>=1.7.0) + + 详细信息请参考。 + + 3.2 Deep Graph Library (>=0.7.0) + + 详细信息请参考。 + + +### 安装 + +#### 通过pip进行安装 + +运行以下命令以通过`pip`安装智图。 + +``` +pip install autogl +``` + +#### 从源代码安装 + +运行以下命令以从源安装智图。 + +``` +git clone https://github.com/THUMNLab/AutoGL.git +cd AutoGL +python setup.py install +``` + +#### 开发者安装 + +如果您想以开发者方式安装智图，请运行以下命令以创建软链接，然后即可修改本地程序后而无需重复安装。 +``` +pip install -e . +``` + +## 文档 + +您可参考文档页面 以参阅我们的详细文档。 + +文档也可以进行本地编译。首先，请安装 sphinx 和 sphinx-rtd-theme: +``` +pip install -U Sphinx +pip install sphinx-rtd-theme +``` + +然后，通过以下方式创建 html 文档： +``` +cd docs +make clean && make html +``` +文档将在如下路径自动生成：`docs/_build/html` + +## 引用 + +如果您使用了智图代码，请按如下方式引用我们的[论文](https://openreview.net/forum?id=0yHwpLeInDn): +``` +@inproceedings{guan2021autogl, + title={Auto{GL}: A Library for Automated Graph Learning}, + author={Chaoyu Guan and Ziwei Zhang and Haoyang Li and Heng Chang and Zeyang Zhang and Yijian Qin and Jiyan Jiang and Xin Wang and Wenwu Zhu}, + booktitle={ICLR 2021 Workshop on Geometrical and Topological Representation Learning}, + year={2021}, + url={https://openreview.net/forum?id=0yHwpLeInDn} +} +``` + +或许您也会发现我们的[综述](http://arxiv.org/abs/2103.00742)有帮助: +``` +@article{zhang2021automated, + title={Automated Machine Learning on Graphs: A Survey}, + author={Zhang, Ziwei and Wang, Xin and Zhu, Wenwu}, + booktitle = {Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence, {IJCAI-21}}, + year={2021}, + note={Survey track} +} +``` + +## 版权相关 +从v0.2版本开始，智图的所有代码采用[Apache license](LICENSE)。 + From 566a34af30bbe37dc862be47d4f3e52abe328ec5 Mon Sep 17 00:00:00 2001 From: ZW-ZHANG Date: Wed, 13 Apr 2022 11:27:10 +0800 Subject: [PATCH 16/37] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a9f57dff..81d5d8fa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Auto Graph Learning -[Chinese Introduction](https://github.com/THUMNLab/AutoGL/blob/main/README_cn.md) +[Chinese Introduction](README_cn.md) An autoML framework & toolkit for machine learning on graphs. From bc6bb59cf5d6f87de61d6668f5f8bbccd5aefcf6 Mon Sep 17 00:00:00 2001 From: Generall <465383637@qq.com> Date: Wed, 13 Apr 2022 11:31:53 +0800 Subject: [PATCH 17/37] Update README_cn.md --- README_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_cn.md b/README_cn.md index db083407..f87fc498 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ # 智图 (AutoGL) -[English Introduction](https://github.com/THUMNLab/AutoGL) +[English Introduction]() 用于图数据的自动机器学习框架和工具包。 From 2f576e6fc3e59e89e305a8d85331747e410f747c Mon Sep 17 00:00:00 2001 From: Generall <465383637@qq.com> Date: Wed, 13 Apr 2022 11:33:15 +0800 Subject: [PATCH 18/37] Update README_cn.md --- README_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_cn.md b/README_cn.md index f87fc498..6f9c9d51 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ # 智图 (AutoGL) -[English Introduction]() +[English Introduction](.) 用于图数据的自动机器学习框架和工具包。 From 19bb6f58647a4d3009c064a60bf2b83faf4a9dfd Mon Sep 17 00:00:00 2001 From: Generall <465383637@qq.com> Date: Wed, 13 Apr 2022 11:33:35 +0800 Subject: [PATCH 19/37] Update README_cn.md --- README_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_cn.md b/README_cn.md index 6f9c9d51..8549812e 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ # 智图 (AutoGL) -[English Introduction](.) +[English Introduction](..) 用于图数据的自动机器学习框架和工具包。 From b2a27888a96142aac9bf127ac10d1be8b770a0a7 Mon Sep 17 00:00:00 2001 From: wondergo2017 Date: Wed, 13 Apr 2022 14:28:31 +0800 Subject: [PATCH 20/37] ensemble cn --- docs/docfile/tutorial/t_ensemble_cn.rst | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/docfile/tutorial/t_ensemble_cn.rst diff --git a/docs/docfile/tutorial/t_ensemble_cn.rst b/docs/docfile/tutorial/t_ensemble_cn.rst new file mode 100644 index 00000000..3b8ec8fe --- /dev/null +++ b/docs/docfile/tutorial/t_ensemble_cn.rst @@ -0,0 +1,54 @@ +.. _ensemble: + +Ensemble +======== + +我们现在支持 voting 和 stacking 方法 + +Voting +------ + +Voter本质上构建了base learner预测的加权和。给定一个评估指标，Voter以某种方式确定base learner的权重，使得验证集指标分数最大化。 + +我们采用Rich Caruana的权重确定方法。该方法首先通过贪婪搜索找到权重相等的(可能是冗余的)base learner集合，然后通过集合中出现的次数指定Voter中的权重。 + +您可以通过重写 ``_specificy_weights`` 方法来定制自己的权重确定方法。 + +.. code-block :: python + + # 例子: 对所有base learner 使用同样的权重 + class EqualWeightVoting(Voting): + def _specify_weights(self, predictions, label, feval): + return np.ones(self.n_models)/self.n_models + # 对所有base learner 赋予相同的权重 + +Stacking +-------- + +Stacker将Base Learner的预测作为输入来训练元模型，以找到这些base learner的最佳组合。 + +目前我们支持广义线性模型(GLM)和梯度推进模型(GBM)作为元模型。 + +创建一个新的ensemble +---------------------- + +您可以通过继承base ensember,重载``fit``和``ensemble``方法来创建自己的ensember。 + +.. code-block :: python + + # 例子 : 使用当前可用的最佳模型 + from autogl.module.ensemble.base import BaseEnsembler + import numpy as np + class BestModel(BaseEnsembler): + def fit(self, predictions, label, identifiers, feval): + if not isinstance(feval, list): + feval = [feval] + scores = np.array([feval[0].evaluate(pred, label) for pred in predictions]) * (1 if feval[0].is_higher_better else -1) + self.scores = dict(zip(identifiers, scores)) # record validation score of base learners + ensemble_pred = predictions[np.argmax(scores)] + return [fx.evaluate(ensemble_pred, label) for fx in feval] + + def ensemble(self, predictions, identifiers): + best_idx = np.argmax([self.scores[model_name] for model_name in identifiers]) # choose the currently best model in the identifiers + return predictions[best_idx] + From e87e7b39aa56f41b3be61f88081c61fe22bc0f46 Mon Sep 17 00:00:00 2001 From: wondergo2017 Date: Wed, 13 Apr 2022 15:19:35 +0800 Subject: [PATCH 21/37] add fe_tutorial unit test --- test/fe/fe_tutorial.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/fe/fe_tutorial.py diff --git a/test/fe/fe_tutorial.py b/test/fe/fe_tutorial.py new file mode 100644 index 00000000..eac51f39 --- /dev/null +++ b/test/fe/fe_tutorial.py @@ -0,0 +1,35 @@ +# 1. Choose a dataset. +from autogl.datasets import build_dataset_from_name +data = build_dataset_from_name('cora') + +# 2. Compose a feature engineering pipeline +from autogl.module.feature._base_feature_engineer._base_feature_engineer import _ComposedFeatureEngineer +from autogl.module.feature import EigenFeatureGenerator +from autogl.module.feature import NetLSD + +# you may compose feature engineering bases through autogl.module.feature._base_feature_engineer +fe = _ComposedFeatureEngineer([ + EigenFeatureGenerator(size=32), + NetLSD() +]) + +# 3. Fit and transform the data +fe.fit(data) +data1=fe.transform(data,inplace=False) + +import autogl +import torch +from autogl.module.feature._generators._basic import BaseFeatureGenerator + +class OneHotFeatureGenerator(BaseFeatureGenerator): + # if overrider_features==False , concat the features with original features; otherwise override. + def __init__(self, override_features: bool = False): + super(BaseFeatureGenerator, self).__init__(override_features) + + def _extract_nodes_feature(self, data: autogl.data.Data) -> torch.Tensor: + num_nodes: int = ( + data.x.size(0) + if data.x is not None and isinstance(data.x, torch.Tensor) + else (data.edge_index.max().item() + 1) + ) + return torch.eye(num_nodes) \ No newline at end of file From c7a06ceac8b5087d452adef7c5f16532e6eee53c Mon Sep 17 00:00:00 2001 From: wondergo2017 Date: Wed, 13 Apr 2022 15:20:27 +0800 Subject: [PATCH 22/37] update fe tutorial doc --- docs/docfile/tutorial/t_fe.rst | 76 ++++++++++++++++------------------ 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/docs/docfile/tutorial/t_fe.rst b/docs/docfile/tutorial/t_fe.rst index df928095..bf76e791 100644 --- a/docs/docfile/tutorial/t_fe.rst +++ b/docs/docfile/tutorial/t_fe.rst @@ -4,8 +4,7 @@ AutoGL Feature Engineering ========================== We provide a series of node and graph feature engineers for -you to compose within a feature engineering pipeline. An automatic -feature engineering algorithm is also provided. +you to compose within a feature engineering pipeline. Quick Start ----------- @@ -16,18 +15,15 @@ Quick Start data = build_dataset_from_name('cora') # 2. Compose a feature engineering pipeline - from autogl.module.feature import BaseFeature,AutoFeatureEngineer - from autogl.module.feature.generators import GeEigen - from autogl.module.feature.selectors import SeGBDT - from autogl.module.feature.graph import SgNetLSD - # you may compose feature engineering bases through BaseFeature.compose - fe = BaseFeature.compose([ - GeEigen(size=32) , - SeGBDT(fixlen=100), - SgNetLSD() + from autogl.module.feature._base_feature_engineer._base_feature_engineer import _ComposedFeatureEngineer + from autogl.module.feature import EigenFeatureGenerator + from autogl.module.feature import NetLSD + + # you may compose feature engineering bases through autogl.module.feature._base_feature_engineer + fe = _ComposedFeatureEngineer([ + EigenFeatureGenerator(size=32), + NetLSD() ]) - # or just through '&' operator - fe = fe & AutoFeatureEngineer(fixlen=200,max_epoch=3) # 3. Fit and transform the data fe.fit(data) @@ -44,19 +40,19 @@ in configurations or as arguments of the autogl solver. +---------------------------+-------------------------------------------------+ | Base | Description | +===========================+=================================================+ -| ``graphlet`` | concatenate local graphlet numbers as features. | +| ``GraphletGenerator`` | concatenate local graphlet numbers as features. | +---------------------------+-------------------------------------------------+ -| ``eigen`` | concatenate Eigen features. | +| ``EigenFeatureGenerator`` | concatenate Eigen features. | +---------------------------+-------------------------------------------------+ -| ``pagerank`` | concatenate Pagerank scores. | +| ``PageRankFeatureGenerator`` | concatenate Pagerank scores. | +---------------------------+-------------------------------------------------+ -| ``PYGLocalDegreeProfile`` | concatenate Local Degree Profile features. | +| `` LocalDegreeProfileGenerator `` | concatenate Local Degree Profile features. | +---------------------------+-------------------------------------------------+ -| ``PYGNormalizeFeatures`` | Normalize all node features | +| ``NormalizeFeatures`` | Normalize all node features | +---------------------------+-------------------------------------------------+ -| ``PYGOneHotDegree`` | concatenate degree one-hot encoding. | +| ``OneHotDegreeGenerator`` | concatenate degree one-hot encoding. | +---------------------------+-------------------------------------------------+ -| ``onehot`` | concatenate node id one-hot encoding. | +| ``OneHotFeatureGenerator`` | concatenate node id one-hot encoding. | +---------------------------+-------------------------------------------------+ 2. ``selectors`` @@ -64,14 +60,14 @@ in configurations or as arguments of the autogl solver. +----------------------+--------------------------------------------------------------------------------+ | Base | Description | +======================+================================================================================+ -| ``SeFilterConstant`` | delete all constant and one-hot encoding node features. | +| ``FilterConstant`` | delete all constant and one-hot encoding node features. | +----------------------+--------------------------------------------------------------------------------+ -| ``gbdt`` | select top-k important node features ranked by Gradient Descent Decision Tree. | +| ``GBDTFeatureSelector`` | select top-k important node features ranked by Gradient Descent Decision Tree. | +----------------------+--------------------------------------------------------------------------------+ 3. ``graph`` -``netlsd`` is a graph feature generation method. please refer to the according document. +``NetLSD`` is a graph feature generation method. please refer to the according document. A set of graph feature extractors implemented in NetworkX are wrapped, please refer to NetworkX for details. (``NxLargeCliqueSize``, ``NxAverageClusteringApproximate``, ``NxDegreeAssortativityCoefficient``, ``NxDegreePearsonCorrelationCoefficient``, ``NxHasBridge`` ,``NxGraphCliqueNumber``, ``NxGraphNumberOfCliques``, ``NxTransitivity``, ``NxAverageClustering``, ``NxIsConnected``, ``NxNumberConnectedComponents``, @@ -87,24 +83,24 @@ Of course, you can directly inherit the ``BaseFeature`` as well. Create Your Own FE ------------------ You can create your own feature engineering object by simply inheriting one of feature engineering base types ,namely ``generators``, ``selectors`` , ``graph``, -and overloading methods ``_fit`` and ``_transform``. +and overloading methods ``extract_xx_features``. .. code-block :: python # for example : create a node one-hot feature. - from autogl.module.feature.generators.base import BaseGenerator - import numpy as np - class GeOnehot(BaseGenerator): - def __init__(self): - super(GeOnehot,self).__init__(data_t='np',multigraph=True,subgraph=False) - # data type in mid is 'numpy', - # and it can be used for multigraph, - # but not suitable for subgraph feature extraction. - - def _fit(self): - pass # nothing to train or memorize - - def _transform(self, data): - fe=np.eye(data.x.shape[0]) - data.x=np.concatenate([data.x,fe],axis=1) - return data + import autogl + import torch + from autogl.module.feature._generators._basic import BaseFeatureGenerator + + class OneHotFeatureGenerator(BaseFeatureGenerator): + # if overrider_features==False , concat the features with original features; otherwise override. + def __init__(self, override_features: bool = False): + super(BaseFeatureGenerator, self).__init__(override_features) + + def _extract_nodes_feature(self, data: autogl.data.Data) -> torch.Tensor: + num_nodes: int = ( + data.x.size(0) + if data.x is not None and isinstance(data.x, torch.Tensor) + else (data.edge_index.max().item() + 1) + ) + return torch.eye(num_nodes) From 7cce7cc535816fe5de1f56821eed8b266ec69134 Mon Sep 17 00:00:00 2001 From: wondergo2017 Date: Wed, 13 Apr 2022 15:42:09 +0800 Subject: [PATCH 23/37] update fe_cn tutorial doc --- docs/docfile/tutorial/t_fe_cn.rst | 100 ++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 docs/docfile/tutorial/t_fe_cn.rst diff --git a/docs/docfile/tutorial/t_fe_cn.rst b/docs/docfile/tutorial/t_fe_cn.rst new file mode 100644 index 00000000..9b1b2466 --- /dev/null +++ b/docs/docfile/tutorial/t_fe_cn.rst @@ -0,0 +1,100 @@ +.. _fe: + +AutoGL 特征工程 +========================== + +我们提供了一系列的节点和图的特征工程方法。您可以挑选需要的特征工程方法，并在一个特性工程管道中编写。 + +快速开始 +----------- +.. code-block :: python + + # 1. 选择一个数据集. + from autogl.datasets import build_dataset_from_name + data = build_dataset_from_name('cora') + + # 2. 选择特征工程方法 + from autogl.module.feature._base_feature_engineer._base_feature_engineer import _ComposedFeatureEngineer + from autogl.module.feature import EigenFeatureGenerator + from autogl.module.feature import NetLSD + + # 可以通过以下方式将多个特征工程方法组合起来 + fe = _ComposedFeatureEngineer([ + EigenFeatureGenerator(size=32), + NetLSD() + ]) + + # 3.拟合变换数据 + fe.fit(data) + data1=fe.transform(data,inplace=False) + + +特征工程方法 +--------------------- +现在支持3种类型的特征工程方法, 分别是 ``generators``, ``selectors`` , ``graph``. 你可以像在 ``快速开始`` 部分一样引入对应的模块，或者可以直接在Config或者Solver中传入需要的方法名称。 + +1. ``generators`` + ++---------------------------+-------------------------------------------------+ +| 方法名 | 描述 | ++===========================+=================================================+ +| ``GraphletGenerator`` | 生成local graphlet 数量作为节点特征 | ++---------------------------+-------------------------------------------------+ +| ``EigenFeatureGenerator`` | 生成特征向量作为节点特征 | ++---------------------------+-------------------------------------------------+ +| ``PageRankFeatureGenerator`` | 生成Pagerank 分数作为节点特征 | ++---------------------------+-------------------------------------------------+ +| `` LocalDegreeProfileGenerator `` | 生成Local Degree Profile作为节点特征 | ++---------------------------+-------------------------------------------------+ +| ``NormalizeFeatures`` | 归一化所有节点特征 | ++---------------------------+-------------------------------------------------+ +| ``OneHotDegreeGenerator`` | 生成节点度的独热编码作为节点特征 | ++---------------------------+-------------------------------------------------+ +| ``OneHotFeatureGenerator`` | 生成节点ID的独热编码作为节点特征 | ++---------------------------+-------------------------------------------------+ + +2. ``selectors`` + ++----------------------+--------------------------------------------------------------------------------+ +| 方法名 | 描述 | ++======================+================================================================================+ +| ``FilterConstant`` | 删除所有常量和独热编码节点特征 | ++----------------------+--------------------------------------------------------------------------------+ +| ``GBDTFeatureSelector`` | 通过梯度下降决策树对节点特征进行重要性排序，选择最重要的K个重要的节点特征 | ++----------------------+--------------------------------------------------------------------------------+ + +3. ``graph`` + +``NetLSD`` 是一种图特征生成方法。 + +一系列Networkx中的图特征生成方法被集成到库中, 若想了解详情，请查阅NetworkX的相关文档。 (``NxLargeCliqueSize``, ``NxAverageClusteringApproximate``, ``NxDegreeAssortativityCoefficient``, ``NxDegreePearsonCorrelationCoefficient``, ``NxHasBridge`` +,``NxGraphCliqueNumber``, ``NxGraphNumberOfCliques``, ``NxTransitivity``, ``NxAverageClustering``, ``NxIsConnected``, ``NxNumberConnectedComponents``, +``NxIsDistanceRegular``, ``NxLocalEfficiency``, ``NxGlobalEfficiency``, ``NxIsEulerian``) + +特征工程类型根据变化特征的方法进行分类。 ``generators`` 生成新特征并拼接或覆盖原始的特征。 而 ``selectors`` 选择原始特征中有用的部分。 +前两种可以节点或者边的层级使用（更改节点或边的特征）, 而 ``graph`` 关注图级别的特征工程(在图特征上进行修改)。 +如果您需要进一步开发使用，可以通过继承其中一种基础类进行修改；或者可以直接继承更加底层的``BaseFeature``类。 + +构建您自己的特征工程方法 +------------------ +您可以继承其中一种特征工程基础类 ``BaseFeatureGenerator``或 ``BaseFeatureSelector`` 进行修改， 重载方法 ``extract_xx_features``。对于图层级特征工程，可以参考 ``_NetworkXGraphFeatureEngineer`` 的实现。 + +.. code-block :: python + + # 例子：创建节点ID独热编码特征 + import autogl + import torch + from autogl.module.feature._generators._basic import BaseFeatureGenerator + + class OneHotFeatureGenerator(BaseFeatureGenerator): + # 设置 overrider_features 为False , 则将原始特征拼接起来; 否则直接覆盖原始特征。 + def __init__(self, override_features: bool = False): + super(BaseFeatureGenerator, self).__init__(override_features) + + def _extract_nodes_feature(self, data: autogl.data.Data) -> torch.Tensor: + num_nodes: int = ( + data.x.size(0) + if data.x is not None and isinstance(data.x, torch.Tensor) + else (data.edge_index.max().item() + 1) + ) + return torch.eye(num_nodes) From fd7ed457996b879cadf5b1e0e9e13774051adf08 Mon Sep 17 00:00:00 2001 From: wondergo2017 Date: Wed, 13 Apr 2022 15:53:46 +0800 Subject: [PATCH 24/37] fix nas device problem --- autogl/module/nas/algorithm/enas.py | 1 - autogl/module/nas/algorithm/gasso.py | 1 - autogl/module/nas/algorithm/rl.py | 2 -- autogl/module/nas/algorithm/spos.py | 1 - 4 files changed, 5 deletions(-) diff --git a/autogl/module/nas/algorithm/enas.py b/autogl/module/nas/algorithm/enas.py index 122ee2db..23bb1fe1 100644 --- a/autogl/module/nas/algorithm/enas.py +++ b/autogl/module/nas/algorithm/enas.py @@ -82,7 +82,6 @@ def __init__( device="auto", ): super().__init__(device) - self.device = device self.num_epochs = num_epochs self.log_frequency = log_frequency self.entropy_weight = entropy_weight diff --git a/autogl/module/nas/algorithm/gasso.py b/autogl/module/nas/algorithm/gasso.py index c3f5ba8d..c4453867 100644 --- a/autogl/module/nas/algorithm/gasso.py +++ b/autogl/module/nas/algorithm/gasso.py @@ -61,7 +61,6 @@ def __init__( device="auto", ): super().__init__(device=device) - self.device = device self.num_epochs = num_epochs self.warmup_epochs = warmup_epochs self.model_lr = model_lr diff --git a/autogl/module/nas/algorithm/rl.py b/autogl/module/nas/algorithm/rl.py index 95530a01..06ad6d85 100644 --- a/autogl/module/nas/algorithm/rl.py +++ b/autogl/module/nas/algorithm/rl.py @@ -265,7 +265,6 @@ def __init__( disable_progress=False, ): super().__init__(device) - self.device = device self.num_epochs = num_epochs self.log_frequency = log_frequency self.entropy_weight = entropy_weight @@ -447,7 +446,6 @@ def __init__( hardware_metric_limit=None, ): super().__init__(device) - self.device = device self.num_epochs = num_epochs self.log_frequency = log_frequency self.entropy_weight = entropy_weight diff --git a/autogl/module/nas/algorithm/spos.py b/autogl/module/nas/algorithm/spos.py index 0e03fa66..5bcaf714 100644 --- a/autogl/module/nas/algorithm/spos.py +++ b/autogl/module/nas/algorithm/spos.py @@ -228,7 +228,6 @@ def __init__( device="cuda", ): super().__init__(device) - self.device = device self.model_lr=5e-3 self.model_wd=5e-4 self.n_warmup = n_warmup From 921edf2cafb7312e150b618cab18e41aa745048b Mon Sep 17 00:00:00 2001 From: wondergo2017 Date: Wed, 13 Apr 2022 16:12:55 +0800 Subject: [PATCH 25/37] fix nas nclf backend --- autogl/module/nas/space/autoattend.py | 3 +++ test/nas/node_classification.py | 17 +++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/autogl/module/nas/space/autoattend.py b/autogl/module/nas/space/autoattend.py index 51d12eb0..c0e64772 100644 --- a/autogl/module/nas/space/autoattend.py +++ b/autogl/module/nas/space/autoattend.py @@ -57,6 +57,9 @@ def __init__( ): super().__init__() + + from autogl.backend import DependentBackend;assert not DependentBackend.is_dgl(),"Now AutoAttend only support pyg" + self.layer_number = layer_number self.hidden_dim = hidden_dim self.input_dim = input_dim diff --git a/test/nas/node_classification.py b/test/nas/node_classification.py index 4b5e3c23..e28bbaca 100644 --- a/test/nas/node_classification.py +++ b/test/nas/node_classification.py @@ -20,7 +20,7 @@ elif DependentBackend.is_pyg(): from torch_geometric.datasets import Planetoid from autogl.module.model.pyg import BaseAutoModel - +from autogl.datasets import build_dataset_from_name import torch import torch.nn.functional as F from autogl.module.nas.space.single_path import SinglePathNodeClassificationSpace @@ -71,12 +71,13 @@ def test_model(model, data=None, check_children=False): if __name__ == "__main__": print("Testing backend: {}".format("dgl" if DependentBackend.is_dgl() else "pyg")) - if DependentBackend.is_dgl(): - dataset = CoraGraphDataset() + from autogl.datasets.utils.conversion._to_dgl_dataset import to_dgl_dataset as convert_dataset else: - dataset = Planetoid(os.path.expanduser("~/.cache-autogl"), "Cora") + from autogl.datasets.utils.conversion._to_pyg_dataset import to_pyg_dataset as convert_dataset + dataset = build_dataset_from_name('cora') + dataset = convert_dataset(dataset) data = dataset[0] di = bk_feat(data).shape[1] @@ -125,14 +126,6 @@ def test_model(model, data=None, check_children=False): model = algo.search(space, dataset, esti) test_model(model, data, True) - print("darts + graphnas ") - space = AutoAttendNodeClassificationSpace().cuda() - space.instantiate(input_dim=di, output_dim=do) - esti = OneShotEstimator() - algo = Darts(num_epochs=10) - model = algo.search(space, dataset, esti) - test_model(model, data, True) - print("Random search + graphnas ") space = GraphNasNodeClassificationSpace().cuda() space.instantiate(input_dim=di, output_dim=do) From 925c517828a7d3fd2c122ef914d4abee2648e6cb Mon Sep 17 00:00:00 2001 From: Generall <465383637@qq.com> Date: Wed, 13 Apr 2022 16:58:36 +0800 Subject: [PATCH 26/37] Update README_cn.md --- README_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_cn.md b/README_cn.md index 8549812e..8c0cc077 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ # 智图 (AutoGL) -[English Introduction](..) +[English Introduction](...) 用于图数据的自动机器学习框架和工具包。 From 3e78ec6ddf6f9a16810616abd511a36a38327f8b Mon Sep 17 00:00:00 2001 From: Generall <465383637@qq.com> Date: Wed, 13 Apr 2022 16:58:53 +0800 Subject: [PATCH 27/37] Update README_cn.md --- README_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_cn.md b/README_cn.md index 8c0cc077..0def3f07 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@ # 智图 (AutoGL) -[English Introduction](...) +[English Introduction](../..) 用于图数据的自动机器学习框架和工具包。 From dde3d3aef5dba695f80d27c191758ad23eb5e2f4 Mon Sep 17 00:00:00 2001 From: CoreLeader Date: Wed, 13 Apr 2022 18:05:00 +0800 Subject: [PATCH 28/37] Revise tutorial for backend and datasets --- docs/docfile/tutorial/t_backend-cn.rst | 33 ++++ docs/docfile/tutorial/t_backend.rst | 4 +- docs/docfile/tutorial/t_dataset-cn.rst | 100 ++++++++++++ docs/docfile/tutorial/t_dataset.rst | 218 ++++++++++--------------- 4 files changed, 221 insertions(+), 134 deletions(-) create mode 100644 docs/docfile/tutorial/t_backend-cn.rst create mode 100644 docs/docfile/tutorial/t_dataset-cn.rst diff --git a/docs/docfile/tutorial/t_backend-cn.rst b/docs/docfile/tutorial/t_backend-cn.rst new file mode 100644 index 00000000..574abe87 --- /dev/null +++ b/docs/docfile/tutorial/t_backend-cn.rst @@ -0,0 +1,33 @@ +.. _backend: + +Backend Support +=============== + +目前，AutoGL支持使用PyTorch-Geometric或Deep Graph Library作为后端，以便熟悉两者之一的用户均可受益于自动图学习。 + +为指定特定的后端，用户可以使用环境变量``AUTOGL_BACKEND``进行声明，例如： + +.. code-block:: python + + AUTOGL_BACKEND=pyg python xxx.py + +或 + +.. code-block:: python + + import os + os.environ["AUTOGL_BACKEND"] = "pyg" + import autogl + + ... + + +如果环境变量``AUTOGL_BACKEND``未声明，AutoGL会根据用户的Python运行环境中所安装的图学习库自动选择。 +如果PyTorch-Geometric和Deep Graph Library均已安装，则Deep Graph Library将被作为默认的后端。 + +可以以编程方式获得当前使用的后端： + +.. code-block:: python + + from autogl.backend import DependentBackend + print(DependentBackend.get_backend_name()) diff --git a/docs/docfile/tutorial/t_backend.rst b/docs/docfile/tutorial/t_backend.rst index ae68c6ef..4ec39ec6 100644 --- a/docs/docfile/tutorial/t_backend.rst +++ b/docs/docfile/tutorial/t_backend.rst @@ -9,13 +9,13 @@ enable users from both end benifiting the automation of graph learning. To specify one specific backend, you can declare the backend using environment variables ``AUTOGL_BACKEND``. For example: -.. code-block :: shell +.. code-block:: python AUTOGL_BACKEND=pyg python xxx.py or -.. code-block :: python +.. code-block:: python import os os.environ["AUTOGL_BACKEND"] = "pyg" diff --git a/docs/docfile/tutorial/t_dataset-cn.rst b/docs/docfile/tutorial/t_dataset-cn.rst new file mode 100644 index 00000000..a2106b51 --- /dev/null +++ b/docs/docfile/tutorial/t_dataset-cn.rst @@ -0,0 +1,100 @@ +.. _dataset: + +AutoGL 数据集 +============== + +我们基于PyTorch-Geometric (PyG)，Deep Graph Learning (DGL)及Open Graph Benchmark (OGB)等图学习库提供了多种多样的常用数据集。 +同时，用户可以使用AutoGL所提供的统一静态图容器``GeneralStaticGraph``自定义静态同构图及异构图，例如： + +.. code-block:: python + from autogl.data.graph import GeneralStaticGraph, GeneralStaticGraphGenerator + + ''' 创建同构图 ''' + custom_static_homogeneous_graph = GeneralStaticGraphGenerator.create_homogeneous_static_graph( + {'x': torch.rand(2708, 3), 'y': torch.rand(2708, 1)}, torch.randint(0, 1024, (2, 10556)) + ) + + ''' 创建异构图 ''' + custom_static_heterogeneous_graph = GeneralStaticGraphGenerator.create_heterogeneous_static_graph( + { + 'author': {'x': torch.rand(1024, 3), 'y': torch.rand(1024, 1)}, + 'paper': {'feat': torch.rand(2048, 10), 'z': torch.rand(2048, 13)} + }, + { + ('author', 'writing', 'paper'): (torch.randint(0, 1024, (2, 5120)), torch.rand(5120, 10)), + ('author', 'reading', 'paper'): torch.randint(0, 1024, (2, 3840)), + } + ) + + +提供的常用数据集 +---------------- +AutoGL目前提供如下多种常用基准数据集： + +半监督节点分类： + ++------------------+------------+-----------+--------------------------------+ +| 数据集 | PyG | DGL | 默认train/val/test划分 | ++==================+============+===========+================================+ +| Cora | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| Citeseer | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| Pubmed | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| Amazon Computers | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Amazon Photo | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Coauthor CS | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Coauthor Physics | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Reddit | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-products | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-proteins | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-arxiv | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-papers100M | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ + + +图分类任务： MUTAG, IMDB-Binary, IMDB-Multi, PROTEINS, COLLAB等 + ++-------------+------------+------------+--------------+------------+--------------------+ +| 数据集 | PyG | DGL | 节点特征 | 标签 | 边特征 | ++=============+============+============+==============+============+====================+ +| MUTAG | ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| IMDB-Binary | ✓ | ✓ | | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| IMDB-Multi | ✓ | ✓ | | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| PROTEINS | ✓ | ✓ | ✓ | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| COLLAB | ✓ | ✓ | | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-molhiv | ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-molpcba| ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-ppa | ✓ | ✓ | | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-code2 | ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ + + +链接预测任务：目前AutoGL可以使用针对节点分类任务的多种图数据进行自动链接预测。 + +通过GeneralStaticGraph序列构建自定义数据集 +---------------------------------------------------------------- +如下代码片段展示了通过一个由``GeneralStaticGraph``序列构建自定义数据集的方法。 + +.. code-block:: python + from autogl.data import InMemoryDataset + ''' graphs变量是一个由GeneralStaticGraph实例所构成的序列 ''' + graphs = [ ... ] + custom_dataset = InMemoryDataset(graphs) diff --git a/docs/docfile/tutorial/t_dataset.rst b/docs/docfile/tutorial/t_dataset.rst index 6cb6bd49..7ec0df75 100644 --- a/docs/docfile/tutorial/t_dataset.rst +++ b/docs/docfile/tutorial/t_dataset.rst @@ -3,144 +3,98 @@ AutoGL Dataset ============== -We import the module of datasets from `CogDL` and `PyTorch Geometric` and add support for datasets from `OGB`. One can refer to the usage of creating and building datasets via the tutorial of `CogDL`_, `PyTorch Geometric`_, and `OGB`_. +We provide various common datasets based on ``PyTorch-Geometric``, ``Deep Graph Library`` and ``OGB``. +Besides, users are able to leverage a unified abstraction provided in AutoGL, ``GeneralStaticGraph``, which is towards both static homogeneous graph and static heterogeneous graph. -.. _CogDL: https://cogdl.readthedocs.io/en/latest/tutorial.html -.. _PyTorch Geometric: https://pytorch-geometric.readthedocs.io/en/latest/notes/create_dataset.html -.. _OGB: https://ogb.stanford.edu/docs/dataset_overview/ +A basic example to construct an instance of ``GeneralStaticGraph`` is shown as follows. + +.. code-block:: python + from autogl.data.graph import GeneralStaticGraph, GeneralStaticGraphGenerator + + ''' Construct a custom homogeneous graph ''' + custom_static_homogeneous_graph: GeneralStaticGraph = GeneralStaticGraphGenerator.create_homogeneous_static_graph( + {'x': torch.rand(2708, 3), 'y': torch.rand(2708, 1)}, torch.randint(0, 1024, (2, 10556)) + ) + + ''' Construct a custom heterogemneous graph ''' + custom_static_heterogeneous_graph: GeneralStaticGraph = GeneralStaticGraphGenerator.create_heterogeneous_static_graph( + { + 'author': {'x': torch.rand(1024, 3), 'y': torch.rand(1024, 1)}, + 'paper': {'feat': torch.rand(2048, 10), 'z': torch.rand(2048, 13)} + }, + { + ('author', 'writing', 'paper'): (torch.randint(0, 1024, (2, 5120)), torch.rand(5120, 10)), + ('author', 'reading', 'paper'): torch.randint(0, 1024, (2, 3840)), + } + ) Supporting datasets ------------------- AutoGL now supports the following benchmarks for different tasks: -Semi-supervised node classification: Cora, Citeseer, Pubmed, Amazon Computers\*, Amazon Photo\*, Coauthor CS\*, Coauthor Physics\*, Reddit （\*: using `utils.random_splits_mask_class` for splitting dataset is recommended.). -For detailed information for supporting datasets, please kindly refer to `PyTorch Geometric Dataset`_. - -.. _PyTorch Geometric Dataset: https://pytorch-geometric.readthedocs.io/en/latest/modules/datasets.html - -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Dataset | PyG | CogDL | x | y | edge_index| edge_attr | train/val/test node | train/val/test mask | -+==================+============+===========+============+============+===========+============+====================+=====================+ -| Cora | ✓ | | ✓ | ✓ | ✓ | ✓ | | ✓ | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Citeseer | ✓ | | ✓ | ✓ | ✓ | ✓ | | ✓ | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Pubmed | ✓ | | ✓ | ✓ | ✓ | ✓ | | ✓ | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Amazon Computers | ✓ | | ✓ | ✓ | ✓ | ✓ | | | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Amazon Photo | ✓ | | ✓ | ✓ | ✓ | ✓ | | | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Coauthor CS | ✓ | | ✓ | ✓ | ✓ | ✓ | | | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Coauthor Physics | ✓ | | ✓ | ✓ | ✓ | ✓ | | | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ -| Reddit | ✓ | | ✓ | ✓ | ✓ | ✓ | | ✓ | -+------------------+------------+-----------+------------+------------+-----------+------------+--------------------+---------------------+ - -Graph classification: MUTAG, IMDB-B, IMDB-M, PROTEINS, COLLAB - -+-----------+------------+------------+-----------+------------+------------+-----------+ -| Dataset | PyG | CogDL | x | y | edge_index | edge_attr | -+===========+============+============+===========+============+============+===========+ -| MUTAG | ✓ | | ✓ | ✓ | ✓ | ✓ | -+-----------+------------+------------+-----------+------------+------------+-----------+ -| IMDB-B | ✓ | | | ✓ | ✓ | | -+-----------+------------+------------+-----------+------------+------------+-----------+ -| IMDB-M | ✓ | | | ✓ | ✓ | | -+-----------+------------+------------+-----------+------------+------------+-----------+ -| PROTEINS | ✓ | | ✓ | ✓ | ✓ | | -+-----------+------------+------------+-----------+------------+------------+-----------+ -| COLLAB | ✓ | | | ✓ | ✓ | | -+-----------+------------+------------+-----------+------------+------------+-----------+ - -TODO: Supporting all datasets from `PyTorch Geometric`. - -OGB datasets ------------- -AutoGL also supports the popular benchmark on `OGB` for node classification and graph classification tasks. For the summary of `OGB` datasets, please kindly refer to the their `docs`_. - -.. _docs: https://ogb.stanford.edu/docs/nodeprop/ - -Since the loss and evaluation metric used for `OGB` datasets vary among different tasks, we also add `string` properties of datasets for identification: - -+-----------------+----------------+-------------------+ -| Dataset | dataset.metric | datasets.loss | -+=================+================+===================+ -| ogbn-products | Accuracy | nll_loss | -+-----------------+----------------+-------------------+ -| ogbn-proteins | ROC-AUC | BCEWithLogitsLoss | -+-----------------+----------------+-------------------+ -| ogbn-arxiv | Accuracy | nll_loss | -+-----------------+----------------+-------------------+ -| ogbn-papers100M | Accuracy | nll_loss | -+-----------------+----------------+-------------------+ -| ogbn-mag | Accuracy | nll_loss | -+-----------------+----------------+-------------------+ -| ogbg-molhiv | ROC-AUC | BCEWithLogitsLoss | -+-----------------+----------------+-------------------+ -| ogbg-molpcba | AP | BCEWithLogitsLoss | -+-----------------+----------------+-------------------+ -| ogbg-ppa | Accuracy | CrossEntropyLoss | -+-----------------+----------------+-------------------+ -| ogbg-code | F1 score | CrossEntropyLoss | -+-----------------+----------------+-------------------+ - - -Create a dataset via URL ------------------------- - -If your dataset is the same as the 'ppi' dataset, which contains two matrices: 'network' and 'group', you can register your dataset directly use the above code. The default root for downloading dataset is `~/.cache-autogl`, you can also specify the root by passing the string to the `path` in `build_dataset(args, path)` or `build_dataset_from_name(dataset, path)`. - -.. code-block:: python - - # following code-snippet is from autogl/datasets/matlab_matrix.py - - @register_dataset("ppi") - class PPIDataset(MatlabMatrix): - def __init__(self, path): - dataset, filename = "ppi", "Homo_sapiens" - url = "http://snap.stanford.edu/node2vec/" - super(PPIDataset, self).__init__(path, filename, url) - -You should declare the name of the dataset, the name of the file, and the URL, where our script can download the resource. Then you can use either `build_dataset(args, path)` or `build_dataset_from_name(dataset, path)` in your task to build a dataset with corresponding parameters. - -Create a dataset locally ------------------------- - -If you want to test your local dataset, we recommend you to refer to the docs on `creating PyTorch Geometric dataset`_. - -.. _creating PyTorch Geometric dataset: https://pytorch-geometric.readthedocs.io/en/latest/notes/create_dataset.html - - -You can simply inherit from `torch_geometric.data.InMemoryDataset` to create an empty `dataset`, then create some `torch_geometric.data.Data` objects for your data and pass a regular python list holding them, then pass them to `torch_geometric.data.Dataset` or `torch_geometric.data.DataLoader`. -Let’s see this process in a simplified example: +Semi-supervised node classification: Cora, Citeseer, Pubmed, Amazon Computers, Amazon Photo, Coauthor CS, Coauthor Physics, Reddit, etc. + ++------------------+------------+-----------+--------------------------------+ +| Dataset | PyG | DGL | default train/val/test split | ++==================+============+===========+================================+ +| Cora | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| Citeseer | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| Pubmed | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| Amazon Computers | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Amazon Photo | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Coauthor CS | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Coauthor Physics | ✓ | ✓ | | ++------------------+------------+-----------+--------------------------------+ +| Reddit | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-products | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-proteins | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-arxiv | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ +| ogbn-papers100M | ✓ | ✓ | ✓ | ++------------------+------------+-----------+--------------------------------+ + +Graph classification: MUTAG, IMDB-Binary, IMDB-Multi, PROTEINS, COLLAB, etc. + ++-------------+------------+------------+--------------+------------+--------------------+ +| Dataset | PyG | DGL | Node Feature | Label | Edge Features | ++=============+============+============+==============+============+====================+ +| MUTAG | ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| IMDB-Binary | ✓ | ✓ | | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| IMDB-Multi | ✓ | ✓ | | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| PROTEINS | ✓ | ✓ | ✓ | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| COLLAB | ✓ | ✓ | | ✓ | | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-molhiv | ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-molpcba| ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-ppa | ✓ | ✓ | | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ +| ogbg-code2 | ✓ | ✓ | ✓ | ✓ | ✓ | ++-------------+------------+------------+--------------+------------+--------------------+ + +Link Prediction: At present, AutoGL utilizes various homogeneous graphs towards node classification to conduct automatic link prediction. + +Construct custom dataset by instances of GeneralStaticGraph +------------------------------------------------------------ +The following example shows the way to compose a custom dataset by a sequence of instances of ``GeneralStaticGraph``. .. code-block:: python - - from typing import Iterable - from torch_geometric.data.data import Data - from autogl.datasets import build_dataset_from_name - from torch_geometric.data import InMemoryDataset - - class MyDataset(InMemoryDataset): - def __init__(self, datalist) -> None: - super().__init__() - self.data, self.slices = self.collate(datalist) - - # Create your own Data objects - - # for example, if you have edge_index, features and labels - # you can create a Data as follows - # See pytorch geometric more info of Data - data = Data() - data.edge_index = edge_index - data.x = features - data.y = labels - - # create a list of Data object - data_list = [data, Data(...), ..., Data(...)] - - # Initialize AutoGL Dataset with your own data - myData = MyDataset(data_list) + from autogl.data import InMemoryDataset + ''' Suppose the graphs is a sequence of instances of GeneralStaticGraph ''' + graphs = [ ... ] + custom_dataset = InMemoryDataset(graphs) From 17b5cdbc008b3e0812adb473b8b5640c123dd4ff Mon Sep 17 00:00:00 2001 From: Generall Date: Wed, 13 Apr 2022 20:16:21 +0800 Subject: [PATCH 29/37] solver_cn and nas_bug_fixed --- docs/docfile/tutorial/t_nas.rst | 14 +- docs/docfile/tutorial_cn/t_solver.rst | 246 ++++++++++++++++++++++++++ 2 files changed, 254 insertions(+), 6 deletions(-) create mode 100644 docs/docfile/tutorial_cn/t_solver.rst diff --git a/docs/docfile/tutorial/t_nas.rst b/docs/docfile/tutorial/t_nas.rst index 9cd3ffb7..3d50ed1e 100644 --- a/docs/docfile/tutorial/t_nas.rst +++ b/docs/docfile/tutorial/t_nas.rst @@ -43,9 +43,9 @@ solver. Following shows an example: graph_models = (), hpo = 'tpe', ensemble = None, - nas_algorithms='rl', + nas_algorithms=['rl'], nas_spaces='graphnasmacro', - nas_estimators='scratch' + nas_estimators=['scratch'] ) cora = build_dataset_from_name('cora') @@ -96,16 +96,17 @@ Here is an example. self.output_dim = output_dim # Instantiate the super network - def instantiate(self, input_dim, output_dim): + def instantiate(self, input_dim = None, output_dim = None): # must call super in this function super().instantiate() self.input_dim = input_dim or self.input_dim self.output_dim = output_dim or self.output_dim # define two layers with order 0 and 1 - self.layer0 = self.setLayerChoice(0, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']]) - self.layer1 = self.setLayerChoice(1, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']]) + setattr(self, 'layer0', self.setLayerChoice(0, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']], key = 'layer0') + setattr(self, 'layer1', self.setLayerChoice(1, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']], key = 'layer1') # define an input choice two choose from the result of the two layer - self.input_layer = self.setInputChoice(2, n_candidates = 2) + setattr(self, 'input_layer', self.setInputChoice(2, choose_from = ['layer0', 'layer1'], n_chosen = 1, returen_mask = False, key = 'input_layer')) + self._initialized = True # Define the forward process def forward(self, data): @@ -113,6 +114,7 @@ Here is an example. x_0 = self.layer0(x, edges) x_1 = self.layer1(x, edges) y = self.input_layer([x_0, x_1]) + y = F.log_fostmax(y, dim = 1) return y # For one-shot fashion, you can directly use following scheme in ``parse_model`` diff --git a/docs/docfile/tutorial_cn/t_solver.rst b/docs/docfile/tutorial_cn/t_solver.rst new file mode 100644 index 00000000..3deaa5be --- /dev/null +++ b/docs/docfile/tutorial_cn/t_solver.rst @@ -0,0 +1,246 @@ +.. _solver: + +AutoGL Solver +============= + +AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前，我们已经支持了一下任务： + +* ``AutoNodeClassifier`` for semi-supervised node classification +* ``AutoGraphClassifier`` for supervised graph classification +* ``AutoLinkPredictor`` for link prediction + +初始化 +-------------- + +一个 solver 可以通过 ``__init__()`` 来进行初始化，也可以通过一个配置字典或文件初始化. + +通过 ``__init__()`` 初始化 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +如果你想通过 ``__init__()`` 初始化一个 solver, 你需要向其传入关键的模块。你可以选择传入相关模块的关键词，也可以传入初始化后的实例： + +.. code-block:: python + + from autogl.solver import AutoNodeClassifier + + # 1. initialize from keywords + solver = AutoNodeClassifier( + feature_module='deepgl', + graph_models=['gat','gcn'], + hpo_module='anneal', + ensemble_module='voting', + device='auto' + ) + + # 2. initialize using instances + from autogl.module import AutoFeatureEngineer, AutoGCN, AutoGAT, AnnealAdvisorHPO, Voting + solver = AutoNodeClassifier( + feature_module=AutoFeatureEngineer(), + graph_models=[AutoGCN(device='cuda'), AutoGAT(device='cuda')], + hpo_module=AnnealAdvisorHPO(max_evals=10), + ensemble_module=Voting(size=2), + device='cuda' + ) + +这里，参数 ``device`` 表示进行训练和搜索的平台。如果设置为 ``auto``，则会在 ``cuda`` 可以使用的时候使用 ``cuda`` 。 + +如果你想禁用某个模块，你可以将其设为 ``None``: + +.. code-block:: python + + solver = AutoNodeClassifier(feature_module=None, hpo_module=None, ensemble_module=None) + +你也可以直接向 solver 传入一些模块的重要参数，它们将会自动为你设置模块： + +.. code-block:: python + + solver = AutoNodeClassifier(hpo_module='anneal', max_evals=10) + +参考 :ref:`solver documentation` 以获得关于默认参数值和重要参数列表的更多细节。 + +通过配置字典或文件进行初始化 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +你也可以直接通过一个配置字典或文件进行初始化。目前，solver 支持 ``yaml`` 或者 ``json`` 类型的文件。如果你想通过这种方式初始化，你需要使用 ``from_config()``： + +.. code-block:: python + + # initialize from config file + path_to_config = 'your/path/to/config' + solver = AutoNodeClassifier.from_config(path_to_config) + + # initialize from a dictionary + config = { + 'models':{'gcn': None, 'gat': None}, + 'hpo': {'name': 'tpe', 'max_evals': 10}, + 'ensemble': {'name': 'voting', 'size': 2} + } + solver = AutoNodeClassifier.from_config(config) + +参考配置字典的描述 :ref:`config` 以获取更多细节。 + +优化 +------------ + +初始化 solver 之后，你可以在给定的数据集上进行优化（请参考 :ref:`dataset` 和 :ref:`dataset documentation` 以创建数据集）。 + +你可以使用 ``fit()`` 或 ``fit_predict()`` 来进行优化，它们有相似的参数列表： + +.. code-block:: python + + # load your dataset here + dataset = some_dataset() + solver.fit(dataset, inplace=True) + + +如果设置 “inplace” 参数为真，它将会在特征工程步骤中把你的数据集设置为原地替换的模式以节省空间。 + +你也可以指定 ``train_split`` 和 ``val_split`` 参数来使 solver 自动分割给定的数据集。如果给定了这些参数，将会使用自动分割的数据集而不是数据集的默认分割。所有的模型都会在 ``train dataset`` 上进行训练。它们的超参数将会根据在 ``valid dataset`` 上的表现进行优化，包括最后的模型集成方式。例如 + +.. code-block:: python + + # split 0.2 of total nodes/graphs for train and 0.4 of nodes/graphs for validation, + # the rest 0.4 is left for test. + solver.fit(dataset, train_split=0.2, val_split=0.4) + + # split 600 nodes/graphs for train and 400 nodes/graphs for validation, + # the rest nodes are left for test. + solver.fit(dataset, train_split=600, val_split=400) + +对于点分类问题，我们同样支持对训练和测试集的平衡采样：强制不同类别的节点数量相同。这种平衡模式可以通过在 ``fit()`` 使 ``balanced=True`` 来进行设置，而其默认值也是 ``True``。 + +.. note:: Solver 会维护每个你传入的模型（初始化时的 ``graph_models``）的最好的超参数。这些模型将会在集成模块中进行集成。 + +``fit()`` 操作之后，solver 在一个榜单中维护每个单独的模型以及集成模型的性能。你可以通过一下代码输出验证集的性能： + +.. code-block:: python + + # get current leaderboard of the solver + lb = solver.get_leaderboard() + # show the leaderboard info + lb.show() + +你可以参考榜单的文档 :ref:`solver documentation` 以获取更多使用细节。 + +预测 +---------- + +在给定的数据集常优化之后，你可以通过 ``solver`` 来进行预测。 + +使用集成模型预测 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +你可以使用 slover 生成的集成模型来进行预测，这也是默认的选项，我们也推荐这样做： + +.. code-block:: python + + solver.predict() + +如果你没有传入任何数据集，那么用于拟合的数据集将会被用于预测。 + +你也可以在预测时传入数据集，请确认已经合理地设置了 ``inplaced`` 参数。 + +.. code-block:: python + + solver.predict(dataset, inplaced=True, inplace=True) + +``predict()`` 函数也有 ``inplace`` 参数，这与在 ``fit()`` 中是一样的。至于 ``inplaced``，意味着无论传入的数据集是否被修改过（也许被 ``fit()`` 函数）。如果你之前使用过 ``fit()``，请确认 ``predict()`` 和 ``fit()`` 中的 ``inplaced`` 参数值是相同的。 + +使用单个模型预测 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +你也可以使用 solver 维护的最好的单个模型来进行预测： + +.. code-block:: python + + solver.predict(use_ensemble=False, use_best=True) + +同样的，你可以为 solver 维护的单个模型命名。 + +.. code-block:: python + + solver.predict(use_ensemble=False, use_best=False, name=the_name_of_model) + +模型的名字可以通过 ``solver.trained_models.keys()`` 来进行调用，这与 solver 榜单中维护的名字是类似的。 + +.. note:: + 默认地，solver 只会在数据集的 ``test`` 部分进行预测。请确认传入的数据集在预测时有 ``test`` 部分。你也可以通过将 ``mask`` 设置为 ``train`` 或者 ``valid`` 来改变默认预测对象。 + +附录 +-------- + +.. _config: + +配置文件格式 +~~~~~~~~~~~~~~~~ +这里介绍了配置文件的结构。配置是一个有五个键的字典，分别是 ``feature`` ，``models``，``trainer``，``hpo`` 和 ``ensemble``。如果使用默认配置，你可以你添加其中的某些键。模块的默认参数与 ``__init__()`` 中的值相同。 + +对于 ``feature``，``hpo`` 和 ``ensemble``，它们对应的值都是字典，里面至少有一个键是 ``name``， 其它的参数则用于初始化对应模型。``name`` 指定了所使用的的算法，如果你不想使用某个模块，你可以传入 ``None``。 + +对于 ``trainer``，你需要制指定它的超参数空间。请参考 :ref:`trainer` 或者 :ref:`train documentation` 来获取不同 trainer 的详细超参数空间信息。 + +对于 ``models``，其值是另一个字典，它的键是需要优化的模型，值是对应模型的超参数空间。参考 :ref:`model` 或者 :ref:`model documentation` 来获取不同模型的详细超参数信息。 + +下面展示了配置所需字典的一个例子。 + +.. code-block:: python + + config_for_node_classification = { + 'feature': { + 'name': 'deepgl', # name of auto feature module + # following are the deepgl specified auto feature engineer arguments + 'fixlen': 100, + 'max_epoch': 5 + }, + 'models': { + 'gcn': + # specify the hp space of gcn + [ + {'parameterName': 'num_layers', 'type': 'DISCRETE', 'feasiblePoints': '2,3,4'}, + {'parameterName': 'hidden', 'type': 'NUMERICAL_LIST', 'numericalType': 'INTEGER', 'length': 3, + 'minValue': [8, 8, 8], 'maxValue': [64, 64, 64], 'scalingType': 'LOG'}, + {'parameterName': 'dropout', 'type': 'DOUBLE', 'maxValue': 0.9, 'minValue': 0.1, 'scalingType': 'LINEAR'}, + {'parameterName': 'act', 'type': 'CATEGORICAL', 'feasiblePoints': ['leaky_relu', 'relu', 'elu', 'tanh']} + ], + 'gat': None, # set to None to use default hp space + 'gin': None + } + 'trainer': [ + # trainer hp space + {'parameterName': 'max_epoch', 'type': 'INTEGER', 'maxValue': 300, 'minValue': 10, 'scalingType': 'LINEAR'}, + {'parameterName': 'early_stopping_round', 'type': 'INTEGER', 'maxValue': 30, 'minValue': 10, 'scalingType': 'LINEAR'}, + {'parameterName': 'lr', 'type': 'DOUBLE', 'maxValue': 0.001, 'minValue': 0.0001, 'scalingType': 'LOG'}, + {'parameterName': 'weight_decay', 'type': 'DOUBLE', 'maxValue': 0.005, 'minValue': 0.0005, 'scalingType': 'LOG'} + ], + 'hpo': { + 'name': 'autone', # name of hpo module + # following are the autone specified auto hpo arguments + 'max_evals': 10, + 'subgraphs': 10, + 'sub_evals': 5 + }, + 'ensemble': { + 'name': 'voting', # name of ensemble module + # following are the voting specified auto ensemble arguments + 'size': 2 + } + } + + config_for_graph_classification = { + 'feature': None, # set to None to disable this module + # do not add field `model` to use default settings of solver + 'trainer': [ + # trainer hp space + {'parameterName': 'max_epoch', 'type': 'INTEGER', 'maxValue': 300, 'minValue': 10, 'scalingType': 'LINEAR'}, + {'parameterName': 'batch_size', 'type': 'INTEGER', 'maxValue': 128, 'minValue': 32, 'scalingType': 'LOG'}, + {'parameterName': 'early_stopping_round', 'type': 'INTEGER', 'maxValue': 30, 'minValue': 10, 'scalingType': 'LINEAR'}, + {'parameterName': 'lr', 'type': 'DOUBLE', 'maxValue': 1e-3, 'minValue': 1e-4, 'scalingType': 'LOG'}, + {'parameterName': 'weight_decay', 'type': 'DOUBLE', 'maxValue': 5e-3, 'minValue': 5e-4, 'scalingType': 'LOG'}, + ], + 'hpo': { + 'name': 'random', # name of hpo module + # following are the random specified auto hpo arguments + 'max_evals': 10 + }, + 'ensemble': None # set to None to disable this module + } \ No newline at end of file From 07436a2e1117a8884ff52fe7aea6dd490551580b Mon Sep 17 00:00:00 2001 From: Generall Date: Wed, 13 Apr 2022 21:36:43 +0800 Subject: [PATCH 30/37] fix quickstart --- autogl/solver/classifier/node_classifier.py | 1 + examples/quickstart.py | 23 +++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 examples/quickstart.py diff --git a/autogl/solver/classifier/node_classifier.py b/autogl/solver/classifier/node_classifier.py index 6caf0d4b..04660cc5 100644 --- a/autogl/solver/classifier/node_classifier.py +++ b/autogl/solver/classifier/node_classifier.py @@ -313,6 +313,7 @@ def fit( # check whether the dataset has features. # currently we only support graph classification with features. + graph_data = get_graph_from_dataset(dataset, 0) feat = get_graph_node_features(graph_data) assert feat is not None, ( "Does not support fit on non node-feature dataset!" diff --git a/examples/quickstart.py b/examples/quickstart.py new file mode 100644 index 00000000..7551a986 --- /dev/null +++ b/examples/quickstart.py @@ -0,0 +1,23 @@ +import autogl +from autogl.datasets import build_dataset_from_name +cora_dataset = build_dataset_from_name('cora', path = '/home/qinyj/AGL/') + +import torch +device = torch.device('cuda:5' if torch.cuda.is_available() else 'cpu') +#device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +from autogl.solver import AutoNodeClassifier +solver = AutoNodeClassifier( + feature_module='deepgl', + graph_models=['gcn', 'gat'], + hpo_module='anneal', + ensemble_module='voting', + device=device +) + +solver.fit(cora_dataset, time_limit=3600) +solver.get_leaderboard().show() + +from autogl.module.train import Acc +predicted = solver.predict_proba() +print('Test accuracy: ', Acc.evaluate(predicted, + cora_dataset.data.y[cora_dataset.data.test_mask].cpu().numpy())) From a6b8408a6c8bcf381857953543b69665cf3999f3 Mon Sep 17 00:00:00 2001 From: Generall Date: Sat, 16 Apr 2022 17:33:00 +0800 Subject: [PATCH 31/37] chinese tutorial --- docs/docfile/tutorial/t_nas.rst | 8 +- .../t_backend.rst} | 2 +- .../t_dataset.rst} | 2 +- .../t_ensemble.rst} | 2 +- .../t_fe_cn.rst => tutorial_cn/t_fe.rst} | 2 +- .../docfile/tutorial_cn/t_hetero_node_clf.rst | 2 +- .../t_homo_graph_classification_gin.rst | 2 + docs/docfile/tutorial_cn/t_hpo.rst | 150 ++++++++ .../t_model.rst} | 4 +- docs/docfile/tutorial_cn/t_nas.rst | 323 ++++++++++++++++++ docs/docfile/tutorial_cn/t_quickstart.rst | 63 ++++ docs/docfile/tutorial_cn/t_solver.rst | 60 ++-- docs/docfile/tutorial_cn/t_trainer.rst | 182 ++++++++++ docs/index.rst | 20 ++ 14 files changed, 781 insertions(+), 41 deletions(-) rename docs/docfile/{tutorial/t_backend-cn.rst => tutorial_cn/t_backend.rst} (98%) rename docs/docfile/{tutorial/t_dataset-cn.rst => tutorial_cn/t_dataset.rst} (99%) rename docs/docfile/{tutorial/t_ensemble_cn.rst => tutorial_cn/t_ensemble.rst} (99%) rename docs/docfile/{tutorial/t_fe_cn.rst => tutorial_cn/t_fe.rst} (99%) create mode 100644 docs/docfile/tutorial_cn/t_hpo.rst rename docs/docfile/{tutorial/t_model_chinese.rst => tutorial_cn/t_model.rst} (99%) create mode 100644 docs/docfile/tutorial_cn/t_nas.rst create mode 100644 docs/docfile/tutorial_cn/t_quickstart.rst create mode 100644 docs/docfile/tutorial_cn/t_trainer.rst diff --git a/docs/docfile/tutorial/t_nas.rst b/docs/docfile/tutorial/t_nas.rst index 3d50ed1e..ab9264f6 100644 --- a/docs/docfile/tutorial/t_nas.rst +++ b/docs/docfile/tutorial/t_nas.rst @@ -6,11 +6,11 @@ Neural Architecture Search We support different neural architecture search algorithm in variant search space. Neural architecture search is usually constructed by three modules: search space, search strategy and estimation strategy. -The search space describes all possible architectures to be searched. There are mainly two parts of the space formulated, the operations(e.g. GCNconv, GATconv) and the input-ouput realations. +The search space describes all possible architectures to be searched. There are mainly two parts of the space formulated, the operations(e.g. GCNconv, GATconv) and the input-ouput relations. A large space may have better optimal architecture but demands more effect to explore. Human knowledge can help to design a reasonable search space to reduce the efforts of search strategy. -The search strategy controls how to explore the search sapce. +The search strategy controls how to explore the search space. It encompasses the classical exploration-exploitation trade-off since. On the one hand, it is desirable to find well-performing architectures quickly, while on the other hand, premature convergence to a region of suboptimal architectures should be avoided. @@ -104,7 +104,7 @@ Here is an example. # define two layers with order 0 and 1 setattr(self, 'layer0', self.setLayerChoice(0, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']], key = 'layer0') setattr(self, 'layer1', self.setLayerChoice(1, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']], key = 'layer1') - # define an input choice two choose from the result of the two layer + # define an input choice to choose from the result of the two layer setattr(self, 'input_layer', self.setInputChoice(2, choose_from = ['layer0', 'layer1'], n_chosen = 1, returen_mask = False, key = 'input_layer')) self._initialized = True @@ -287,7 +287,7 @@ If you want to define more complex strategy, you can refer to Darts, Enas or oth Different search strategies should be combined with different search spaces and estimators in usage. +----------------+-------------+-------------+------------------+ -| Sapce | single path | GraphNAS[1] | GraphNAS-macro[1]| +| Space | single path | GraphNAS[1] | GraphNAS-macro[1]| +================+=============+=============+==================+ | Random | ✓ | ✓ | ✓ | +----------------+-------------+-------------+------------------+ diff --git a/docs/docfile/tutorial/t_backend-cn.rst b/docs/docfile/tutorial_cn/t_backend.rst similarity index 98% rename from docs/docfile/tutorial/t_backend-cn.rst rename to docs/docfile/tutorial_cn/t_backend.rst index 574abe87..e43f6306 100644 --- a/docs/docfile/tutorial/t_backend-cn.rst +++ b/docs/docfile/tutorial_cn/t_backend.rst @@ -1,4 +1,4 @@ -.. _backend: +.. _backend_cn: Backend Support =============== diff --git a/docs/docfile/tutorial/t_dataset-cn.rst b/docs/docfile/tutorial_cn/t_dataset.rst similarity index 99% rename from docs/docfile/tutorial/t_dataset-cn.rst rename to docs/docfile/tutorial_cn/t_dataset.rst index a2106b51..fc0f7052 100644 --- a/docs/docfile/tutorial/t_dataset-cn.rst +++ b/docs/docfile/tutorial_cn/t_dataset.rst @@ -1,4 +1,4 @@ -.. _dataset: +.. _dataset_cn: AutoGL 数据集 ============== diff --git a/docs/docfile/tutorial/t_ensemble_cn.rst b/docs/docfile/tutorial_cn/t_ensemble.rst similarity index 99% rename from docs/docfile/tutorial/t_ensemble_cn.rst rename to docs/docfile/tutorial_cn/t_ensemble.rst index 3b8ec8fe..957648bb 100644 --- a/docs/docfile/tutorial/t_ensemble_cn.rst +++ b/docs/docfile/tutorial_cn/t_ensemble.rst @@ -1,4 +1,4 @@ -.. _ensemble: +.. _ensemble_cn: Ensemble ======== diff --git a/docs/docfile/tutorial/t_fe_cn.rst b/docs/docfile/tutorial_cn/t_fe.rst similarity index 99% rename from docs/docfile/tutorial/t_fe_cn.rst rename to docs/docfile/tutorial_cn/t_fe.rst index 9b1b2466..2e9f2be1 100644 --- a/docs/docfile/tutorial/t_fe_cn.rst +++ b/docs/docfile/tutorial_cn/t_fe.rst @@ -1,4 +1,4 @@ -.. _fe: +.. _fe_cn: AutoGL 特征工程 ========================== diff --git a/docs/docfile/tutorial_cn/t_hetero_node_clf.rst b/docs/docfile/tutorial_cn/t_hetero_node_clf.rst index 2eb67ebf..855be28c 100644 --- a/docs/docfile/tutorial_cn/t_hetero_node_clf.rst +++ b/docs/docfile/tutorial_cn/t_hetero_node_clf.rst @@ -1,4 +1,4 @@ -.. _hetero_node_clf: +.. _hetero_node_clf_cn: .. Node Classification for Heterogeneous Graph 异质图上的节点分类 diff --git a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst index 25b568b8..76a4ca77 100644 --- a/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst +++ b/docs/docfile/tutorial_cn/t_homo_graph_classification_gin.rst @@ -1,3 +1,5 @@ +: _homo_cn: + ========================== 图分类模型 ========================== diff --git a/docs/docfile/tutorial_cn/t_hpo.rst b/docs/docfile/tutorial_cn/t_hpo.rst new file mode 100644 index 00000000..b3c50a1a --- /dev/null +++ b/docs/docfile/tutorial_cn/t_hpo.rst @@ -0,0 +1,150 @@ +.. _hpo_cn: + +超参数优化 +============================ + +我们支持不同的搜索空间下的黑盒超参数优化。 + +搜索空间 +------------ + +我们支持三种类型的搜索空间，请使用python字典``dict``来定义你的搜索空间。 +对于数字列表形式的搜索空间，你可以为列表指定一个固定的长度，如果是这样，你不需要提供``cutPara``和``cutFunc``。 +或者你可以把列表切成一个特定的长度，这个长度取决于参数``cutPara``和``cutFunc``。你应该在``cutPara``中提供参数名并在``cutFunc``中提供计算剪切长度的函数。 + +.. code-block:: python + + # 数值搜索空间： + { + "parameterName": "xxx", + "type": "DOUBLE" / "INTEGER", + "minValue": xx, + "maxValue": xx, + "scalingType": "LINEAR" / "LOG" + } + + # 数值列表搜索空间： + { + "parameterName": "xxx", + "type": "NUMERICAL_LIST", + "numericalType": "DOUBLE" / "INTEGER", + "length": 3, + "cutPara": ("para_a", "para_b"), + "cutFunc": lambda x: x[0] - 1, + "minValue": [xx,xx,xx], + "maxValue": [xx,xx,xx], + "scalingType": "LINEAR" / "LOG" + } + + # 类别搜索空间： + { + "parameterName": xxx, + "type": "CATEGORICAL" + "feasiblePoints": [a,b,c] + } + + # 固定参数作为搜索空间： + { + "parameterName": xxx, + "type": "FIXED", + "value": xxx + } + +下表列出了超参数优化算法所支持的搜索空间形式： + ++------------------+------------+--------------+-----------+------------+ +| 算法 | 数值 | 数值列表 | 类别 | 固定的 | ++==================+============+==============+===========+============+ +| Grid | | | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| Random | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| Anneal | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| Bayes | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| TPE [1]_ | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| CMAES [2]_ | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| MOCMAES [3]_ | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +|Quasi random [4]_ | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ +| AutoNE [5]_ | ✓ | ✓ | ✓ | ✓ | ++------------------+------------+--------------+-----------+------------+ + +添加你自己的超参数优化器（HPOptimizer） +-------------------- + +如果你想添加你自己的 HPOptimizer, 你只需要完成 HPOptimizer 中的``optimize`` 函数: + +.. code-block:: python + + # 例如，创建一个随机超参数优化算法 + import random + from autogl.module.hpo.base import BaseHPOptimizer + class RandomOptimizer(BaseHPOptimizer): + # 在初始化时获取基本参数 + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_evals = kwargs.get("max_evals", 2) + + # 你应该做的最重要的事情是完成优化函数 + def optimize(self, trainer, dataset, time_limit=None, memory_limit=None): + # 1. 从训练器得到搜索空间。 + space = trainer.hyper_parameter_space + trainer.model.hyper_parameter_space + # 可选的：使用 self._encode_para (在 BaseOptimizer) 来对搜索空间进行预处理 + # 如果使用 _encode_para, NUMERICAL_LIST 将扩展为 双精度浮点数 或 整数，LOG尺度类型将改为线性，CATEGORICAL中的可行点将改为离散数。 + # 您还应该使用_decode_para将参数类型转换回来。 + current_space = self._encode_para(space) + + # 2. 定义你自己的性能函数。 + def fn(dset, para): + current_trainer = trainer.duplicate_from_hyper_parameter(para) + current_trainer.train(dset) + loss, self.is_higher_better = current_trainer.get_valid_score(dset) + # 为了方便起见，损失分数越高越好；如果是负数，那么我们就应该把损失分数降到最低。 + if self.is_higher_better: + loss = -loss + return current_trainer, loss + + # 3. 定义如何获得超参数建议，它应该返回一个参数字典。你可以使用历史实验来得到新的建议。 + def get_random(history_trials): + hps = {} + for para in current_space: + # 因为我们之前使用了_encode_para函数，所以我们应该只处理DOUBLE、INTEGER和DISCRETE函数 + if para["type"] == "DOUBLE" or para["type"] == "INTEGER": + hp = random.random() * (para["maxValue"] - para["minValue"]) + para["minValue"] + if para["type"] == "INTEGER": + hp = round(hp) + hps[para["parameterName"]] = hp + elif para["type"] == "DISCRETE": + feasible_points = para["feasiblePoints"].split(",") + hps[para["parameterName"]] = random.choice(feasible_points) + return hps + + # 4. 运行算法。对于每个回合，根据历史信息获得一组参数并进行评估。 + best_trainer, best_para, best_perf = None, None, None + self.trials = [] + for i in range(self.max_evals): + # 在这个例子中，我们不需要历史追踪。因此我们为history_trails传入None + new_hp = get_random(None) + # 可选的：如果你使用参数 _encode_para，也要提供参数 _decode_para 。 para_for_trainer 撤销 _encode_para 中的所有转换，并在需要时将双精度浮点数转换为整数。para_for_hpo 只将双精度浮点数转换为整数。 + para_for_trainer, para_for_hpo = self._decode_para(new_hp) + current_trainer, perf = fn(dataset, para_for_trainer) + self.trials.append((para_for_hpo, perf)) + if not best_perf or perf < best_perf: + best_perf = perf + best_trainer = current_trainer + best_para = para_for_trainer + + # 5. 返回最优训练器和参数。 + return best_trainer, best_para + + +.. [1] Bergstra, James S., et al. "Algorithms for hyper-parameter optimization." Advances in neural information processing systems. 2011. +.. [2] Arnold, Dirk V., and Nikolaus Hansen. "Active covariance matrix adaptation for the (1+ 1)-CMA-ES." Proceedings of the 12th annual conference on Genetic and evolutionary computation. 2010. +.. [3] Voß, Thomas, Nikolaus Hansen, and Christian Igel. "Improved step size adaptation for the MO-CMA-ES." Proceedings of the 12th annual conference on Genetic and evolutionary computation. 2010. +.. [4] Bratley, Paul, Bennett L. Fox, and Harald Niederreiter. "Programs to generate Niederreiter's low-discrepancy sequences." ACM Transactions on Mathematical Software (TOMS) 20.4 (1994): 494-495. +.. [5] Tu, Ke, et al. "Autone: Hyperparameter optimization for massive network embedding." Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. 2019. diff --git a/docs/docfile/tutorial/t_model_chinese.rst b/docs/docfile/tutorial_cn/t_model.rst similarity index 99% rename from docs/docfile/tutorial/t_model_chinese.rst rename to docs/docfile/tutorial_cn/t_model.rst index 801347ae..e5462667 100644 --- a/docs/docfile/tutorial/t_model_chinese.rst +++ b/docs/docfile/tutorial_cn/t_model.rst @@ -1,4 +1,4 @@ -.. _model: +.. _model_cn: AutoGL 模型 ============ @@ -159,7 +159,7 @@ AutoGL 模型 支持采样的模型 ^^^^^^^^^^^^^ 为了高效地实现大规模图上表示学习，AutoGL目前支持使用节点级别(node-wise)的采样、层级别(layer-wise)的采样和子图级别(subgraph-wise)的采样等采样技术进行节点分类。 -有关采样的更多信息，请参阅：:ref:`trainer`。 +有关采样的更多信息，请参阅：:ref:`trainer_cn`。 根据图神经网络中的消息传递机制，一个节点的表达由它多跳邻居构成的子图决定。 但是，节点的邻居数量随着神经网络层数的增加呈现指数级增长，计算并储存所有节点的表达会占用许多的计算资源。 diff --git a/docs/docfile/tutorial_cn/t_nas.rst b/docs/docfile/tutorial_cn/t_nas.rst new file mode 100644 index 00000000..b1212043 --- /dev/null +++ b/docs/docfile/tutorial_cn/t_nas.rst @@ -0,0 +1,323 @@ +.. _nas_cn： + +神经架构搜索 +============================ + +我们在不同的搜索空间中支持不同的神经架构搜索算法。 +神经架构搜索通常由三个模块构成：搜索空间、搜索策略和评估策略。 + +搜索空间描述了所有可能要搜索的架构。空间主要由两部分组成：操作（如GCNconv、GATconv）和输入输出关系。 +大空间可能有更好的最优架构，但需要更多的算力来探索。 +人类知识可以帮助设计合理的搜索空间，减少搜索策略的开销。 + +搜索策略则控制着如何来探索搜索空间。 +它包含了经典的探索-利用困境。 +一方面，人们希望快速找到性能良好的架构， +而另一方面，也需要避免过早收敛到次优架构区域。 + +评估策略在探索时给出架构的性能。 +最简单的方法是对数据执行标准的架构训练和验证。 +但由于在搜索过程中有很多架构需要评估，因此需要非常有效的评估策略来节省计算资源。 + +.. image:: ../../../resources/nas.svg + :align: center + +为了更加灵活易用，我们将神经架构搜索过程分解为三部分：算法、空间和评估器，分别对应于三个模块：搜索空间、搜索策略和评估策略。 +不同部分的不同模型可以在满足一定条件的时候组合起来。 +如果您想设计自己的神经架构搜索流程，您可以根据需要更改其中的任意部分。 + +用法 +----- + +您可以通过将算法、空间与评估器直接传递给求解器来在节点分类任务上启用架构搜索。 +下面是一个例子： + +.. code-block:: python + + # 在cora数据集上使用图神经架构搜索 + from autogl.datasets import build_dataset_from_name + from autogl.solver import AutoNodeClassifier + + solver = AutoNodeClassifier( + feature = 'PYGNormalizeFeatures', + graph_models = (), + hpo = 'tpe', + ensemble = None, + nas_algorithms=['rl'], + nas_spaces='graphnasmacro', + nas_estimators=['scratch'] + ) + + cora = build_dataset_from_name('cora') + solver.fit(cora) + +上面的代码将先用 ``rl`` 搜索算法在空间 ``GraphnaSmaro`` 中找到最优架构。 +然后通过超参数优化方法 ``tpe`` 进一步优化搜索到的架构。 + +.. note:: ``graph_models`` 参数与神经架构搜索模块不冲突。您可以将 ``graph_models`` 设置为 + 神经架构搜索发现的模型外的，其他手工设计的模型。而神经架构搜索模块派生出来的架构 + 的作用也与直接通过图模型模块传递的手工设计的模型相同。 + +搜索空间 +------------ + +空间定义基于开源工具包NNI中使用的可变方式，定义为继承基础空间的模型。 +定义搜索空间的方法主要有两种，一种支持单样本方法，而另一种则不支持。 +目前，我们支持如下搜索空间： + ++------------------------+-----------------------------------------------------------------+ +| 空间 | 描述 | ++========================+=================================================================+ +| ``singlepath`` [4]_ | 多个连续层构成的架构，其中每层 | +| | 只能有一个选择 | ++------------------------+-----------------------------------------------------------------+ +| ``graphnas`` [1]_ | 为监督节点分类问题的模型设计的 | +| | 图神经架构搜索微搜索空间 | ++------------------------+-----------------------------------------------------------------+ +| ``graphnasmacro`` [1]_ | 为半监督节点分类问题的模型设计的 | +| | 图神经架构搜索宏搜索空间 | ++------------------------+-----------------------------------------------------------------+ + +您也可以定义自己的神经架构搜索空间。 +如果需要支持单样本方式，可以使用函数 ``setLayerChoice`` 与 ``setInputChoice`` 来构建超网络。 +下面是一个例子。 + +.. code-block:: python + + # 创建一个神经架构搜索空间的例子 + from autogl.module.nas.space.base import BaseSpace + from autogl.module.nas.space.operation import gnn_map + class YourOneShotSpace(BaseSpace): + # 在初始化时获取基本参数 + def __init__(self, input_dim = None, output_dim = None): + super().__init__() + # 必须在空间中设定输入维度与输出维度，也可以在函数 `instantiate` 中初始化这两个参数` + self.input_dim = input_dim + self.output_dim = output_dim + + # 实例化超网络 + def instantiate(self, input_dim = None, output_dim = None): + # 必须在函数中调用父类的实例化 + super().instantiate() + self.input_dim = input_dim or self.input_dim + self.output_dim = output_dim or self.output_dim + # 按照顺序定义两层网络 + setattr(self, 'layer0', self.setLayerChoice(0, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']], key = 'layer0') + setattr(self, 'layer1', self.setLayerChoice(1, [gnn_map(op,self.input_dim,self.output_dim)for op in ['gcn', 'gat']], key = 'layer1') + # 定义一个从两层的结果中选择的输入选项 + setattr(self, 'input_layer', self.setInputChoice(2, choose_from = ['layer0', 'layer1'], n_chosen = 1, returen_mask = False, key = 'input_layer')) + self._initialized = True + + # 定义前向传播过程 + def forward(self, data): + x, edges = data.x, data.edge_index + x_0 = self.layer0(x, edges) + x_1 = self.layer1(x, edges) + y = self.input_layer([x_0, x_1]) + y = F.log_fostmax(y, dim = 1) + return y + + # 对于单样本范式，您可以使用如 ``parse_model`` 函数中的方法 + def parse_model(self, selection, device) -> BaseModel: + return self.wrap().fix(selection) + +您也可以使用不支持单样本范式的方式。 +这样的话，您可以直接复制模型，并进行少量更改。 +但相应的，您也只能使用基于样本的搜索策略。 + +.. code-block:: python + + # 创建一个神经架构搜索空间的例子 + from autogl.module.nas.space.base import BaseSpace, map_nn + from autogl.module.nas.space.operation import gnn_map + # 在这里，我们以 `head` 作为参数，在三种图卷积上进行搜索 + # 在搜索 `heads` 时，我们在搜索图卷积 + from torch_geometric.nn import GATConv, FeaStConv, TransformerConv + class YourNonOneShotSpace(BaseSpace): + # 在初始化时获取基本参数 + def __init__(self, input_dim = None, output_dim = None): + super().__init__() + # 必须在空间中设定输入维度与输出维度，也可以在函数 `instantiate` 中初始化这两个参数` + self.input_dim = input_dim + self.output_dim = output_dim + + # 实例化超网络 + def instantiate(self, input_dim, output_dim): + # 必须在函数中调用父类的实例化 + super().instantiate() + self.input_dim = input_dim or self.input_dim + self.output_dim = output_dim or self.output_dim + # 设置你每一层的选择 + self.choice0 = self.setLayerChoice(0, map_nn(["gat", "feast", "transformer"]), key="conv") + self.choice1 = self.setLayerChoice(1, map_nn([1, 2, 4, 8]), key="head") + + # 不要忘记在这里定义前向传播过程 + # 对于非单样本范式，您可以直接返回选择下的模型 + # ``YourModel`` 也就是您的模型必须继承基础空间 + def parse_model(self, selection, device) -> BaseModel: + model = YourModel(selection, self.input_dim, self.output_dim).wrap() + return model + +.. code-block:: python + + # ``YourModel`` 也就是您的模型定义如下 + class YourModel(BaseSpace): + def __init__(self, selection, input_dim, output_dim): + self.input_dim = input_dim + self.output_dim = output_dim + if selection["conv"] == "gat": + conv = GATConv + elif selection["conv"] == "feast": + conv = FeaStConv + elif selection["conv"] == "transformer": + conv = TransformerConv + self.layer = conv(input_dim, output_dim, selection["head"]) + + def forward(self, data): + x, edges = data.x, data.edge_index + y = self.layer(x, edges) + return y + +性能评估器 +--------------------- + +性能评估器用于评估一个架构的优劣. 目前我们支持如下一些评估器: + ++-------------------------+-------------------------------------------------------+ +| 评估器 | 描述 | ++=========================+=======================================================+ +| ``oneshot`` | 对于给定的子架构，无需训练地直接评估其效果 | ++-------------------------+-------------------------------------------------------+ +| ``scratch`` | 对于给定的子架构，从头开始训练直到收敛之后再评估其效果 | ++-------------------------+-------------------------------------------------------+ + +您也可以自己定义一个评估器. 下面是一个无需训练即可评估架构效果的评估器的例子 (通常应用于one-shot space). + +.. code-block:: python + + # 例如，您也可以自己定义一个estimator + from autogl.module.nas.estimator.base import BaseEstimator + class YourOneShotEstimator(BaseEstimator): + # 您所需要做的只是定义``infer``这个方法 + def infer(self, model: BaseSpace, dataset, mask="train"): + device = next(model.parameters()).device + dset = dataset[0].to(device) + # 对架构直接进行前向传播 + pred = model(dset)[getattr(dset, f"{mask}_mask")] + y = dset.y[getattr(dset, f'{mask}_mask')] + # 使用默认的损失函数和评价指标来评估架构效果，当然，在这里您也可以选择其他的损失函数和评价指标 + loss = getattr(F, self.loss_f)(pred, y) + probs = F.softmax(pred, dim = 1) + metrics = [eva.evaluate(probs, y) for eva in self.evaluation] + return metrics, loss + +搜索策略 +--------------- + +搜索策略定义了如何去搜索一个好的子架构. 目前我们支持如下一些搜索策略: + ++-------------------------+-------------------------------------------------------+ +| 策略 | 描述 | ++=========================+=======================================================+ +| ``random`` | 通过均匀采样进行随机搜索 | ++-------------------------+-------------------------------------------------------+ +| ``rl`` [1]_ | 通过强化学习方法来进行架构搜索 | ++-------------------------+-------------------------------------------------------+ +| ``enas`` [2]_ | 通过共享参数等方法，更高效地进行架构搜索 | ++-------------------------+-------------------------------------------------------+ +| ``darts`` [3]_ | 通过可微方法来进行架构搜索 | ++-------------------------+-------------------------------------------------------+ + +基于采样的非共享权重的搜索策略在实现上更加简单 +接下来，我们将向您展示如何自定义一个基于DFS的搜索策略来作为一个例子 +如果您想要自定义更多复杂的搜索策略，您可以去参考NNI中Darts、Enas或者其他搜索策略的实现 + +.. code-block:: python + + from autogl.module.nas.algorithm.base import BaseNAS + class RandomSearch(BaseNAS): + # 接收需要采样的数量作为初始化 + def __init__(self, n_sample): + super().__init__() + self.n_sample = n_sample + + # NAS算法流程中的关键步骤，这个方法会根据给定的search space、dataset和estimator去搜索一个合理的架构 + def search(self, space: BaseSpace, dset, estimator): + self.estimator=estimator + self.dataset=dset + self.space=space + + self.nas_modules = [] + k2o = get_module_order(self.space) + # 寻找并存储search space中所有的mutables，这些mutables就是您在search space中定义的可搜索的部分 + replace_layer_choice(self.space, PathSamplingLayerChoice, self.nas_modules) + replace_input_choice(self.space, PathSamplingInputChoice, self.nas_modules) + # 根据给定的orders对mutables进行排序 + self.nas_modules = sort_replaced_module(k2o, self.nas_modules) + # 得到包含所有可能选择的一个字典 + selection_range={} + for k,v in self.nas_modules: + selection_range[k]=len(v) + self.selection_dict=selection_range + + arch_perfs=[] + # 定义DFS的流程 + self.selection = {} + last_k = list(self.selection_dict.keys())[-1] + def dfs(): + for k,v in self.selection_dict.items(): + if not k in self.selection: + for i in range(v): + self.selection[k] = i + if k == last_k: + # 评估一个架构的效果 + self.arch=space.parse_model(self.selection,self.device) + metric,loss=self._infer(mask='val') + arch_perfs.append([metric, self.selection.copy()]) + else: + dfs() + del self.selection[k] + break + dfs() + + # 得到在搜索过程中拥有最好效果的架构 + selection=arch_perfs[np.argmax([x[0] for x in arch_perfs])][1] + arch=space.parse_model(selection,self.device) + return arch + +不同的搜索策略需要与特定的搜索空间与评估器搭配使用 +这与它们的实现相关，如非one-shot的搜索空间不能与one-shot的搜索策略搭配使用 +下面的表格中给出了我们目前所支持的搭配组合 + ++----------------+-------------+-------------+------------------+ +| Space | single path | GraphNAS[1] | GraphNAS-macro[1]| ++================+=============+=============+==================+ +| Random | ✓ | ✓ | ✓ | ++----------------+-------------+-------------+------------------+ +| RL | ✓ | ✓ | ✓ | ++----------------+-------------+-------------+------------------+ +| GraphNAS [1]_ | ✓ | ✓ | ✓ | ++----------------+-------------+-------------+------------------+ +| ENAS [2]_ | ✓ | | | ++----------------+-------------+-------------+------------------+ +| DARTS [3]_ | ✓ | | | ++----------------+-------------+-------------+------------------+ + ++----------------+-------------+-------------+ +| Estimator | one-shot | Train | ++================+=============+=============+ +| Random | | ✓ | ++----------------+-------------+-------------+ +| RL | | ✓ | ++----------------+-------------+-------------+ +| GraphNAS [1]_ | | ✓ | ++----------------+-------------+-------------+ +| ENAS [2]_ | ✓ | | ++----------------+-------------+-------------+ +| DARTS [3]_ | ✓ | | ++----------------+-------------+-------------+ + +.. [1] Gao, Yang, et al. "Graph neural architecture search." IJCAI. Vol. 20. 2020. +.. [2] Pham, Hieu, et al. "Efficient neural architecture search via parameters sharing." International Conference on Machine Learning. PMLR, 2018. +.. [3] Liu, Hanxiao, Karen Simonyan, and Yiming Yang. "DARTS: Differentiable Architecture Search." International Conference on Learning Representations. 2018. +.. [4] Guo, Zichao, et al. “Single Path One-Shot Neural Architecture Search with Uniform Sampling.” European Conference on Computer Vision, 2019, pp. 544–560. \ No newline at end of file diff --git a/docs/docfile/tutorial_cn/t_quickstart.rst b/docs/docfile/tutorial_cn/t_quickstart.rst new file mode 100644 index 00000000..dca8e970 --- /dev/null +++ b/docs/docfile/tutorial_cn/t_quickstart.rst @@ -0,0 +1,63 @@ +.. _quickstart_cn: + +快速开始 +=========== + +本教程将帮助您快速了解智图（AutoGL）中重要类的概念和用法。在本教程中，您将在数据集`Cora`_上进行快速的自动图学习。 + +.. _Cora: https://graphsandnetworks.com/the-cora-dataset/ + +智图 +--------------- + +基于自动机器学习（AutoML）的概念，自动图学习的目标是用图数据**自动地**解决任务。与传统的学习框架不同，自动图学习不需要人工参与实验循环。您只需要向智图求解器提供数据集和任务，本框架将自动为您找到合适的方法和超参数。 + +下图描述了智图框架的工作流程。 + +为了达到自动机器学习的目标，我们提出的自动图学习框架组织如下。我们用 ``数据集（dataset）`` 来维护由用户给出的图形数据集。为了指定目标任务，需要构建一个``求解器（solver）``对象。``求解器``内部，有五个子模型来帮助完成完成自动图学习任务，即``自动特征工程``,``自动模型``，``神经架构搜索``，``超参数优化``和``自动集成学习``来根据您的需求自动预处理/增强您的数据，选择和优化深度模型并集成。 +假设你想在数据集``Cora``上进行自动图学习。首先，您可以通过 ``dataset``模块得到``Cora``数据集： + +.. code-block:: python + + from autogl.datasets import build_dataset_from_name + cora_dataset = build_dataset_from_name('cora') + +数据集将自动下载给您。请参考:ref:`dataset`或:ref:`dataset documentation`获取更多关于数据集构造、可用数据集、添加本地数据集等详细信息。 + +在导出数据集之后，您可以构建一个``节点分类求解器（node classification solver）``来处理自动训练过程： + +.. code-block:: python + + import torch + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + from autogl.solver import AutoNodeClassifier + solver = AutoNodeClassifier( + feature_module='deepgl', + graph_models=['gcn', 'gat'], + hpo_module='anneal', + ensemble_module='voting', + device=device + ) + +通过这种方式，我们构建了一个``节点分类求解器``，它使用``deepgl``进行特征工程，并使用``anneal``超参数优化器对给定的三个模型“``['gcn','gat']``进行优化。派生的模型将使用``voting``集成器进行集成。请参考相应的教程或文档，以了解可用子模块的定义和用法。 + +Then, you can fit the solver and then check the leaderboard: +接下来，你可以安装求解器，然后查看排行榜： + +.. code-block:: python + + solver.fit(cora_dataset, time_limit=3600) + solver.get_leaderboard().show() + +``time_limit``设置为3600，这样整个自动绘图过程不会超过1小时。``solver.show()``将显示由``solver``维护的模型，以及它们在验证数据集上的性能。 + +然后，你可以使用提供的评估函数进行预测和结果评估： + +.. code-block:: python + + from autogl.module.train import Acc + predicted = solver.predict_proba() + print('Test accuracy: ', Acc.evaluate(predicted, + cora_dataset.data.y[cora_dataset.data.test_mask].cpu().numpy())) + +.. 注意:: 当预测时，你不需要再次传递``cora_dataset``，因为数据集被``求解器``**记住**，预测时如果没有传递数据集将被重用。然而，您也可以在预测时传递一个新的数据集，新的数据集将被使用，而不是被记住的数据集。详情请参考:ref:`solver`或:ref:`solver documentation`。 \ No newline at end of file diff --git a/docs/docfile/tutorial_cn/t_solver.rst b/docs/docfile/tutorial_cn/t_solver.rst index 3deaa5be..abe3b81e 100644 --- a/docs/docfile/tutorial_cn/t_solver.rst +++ b/docs/docfile/tutorial_cn/t_solver.rst @@ -1,4 +1,4 @@ -.. _solver: +.. _solver_cn: AutoGL Solver ============= @@ -23,7 +23,7 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 from autogl.solver import AutoNodeClassifier - # 1. initialize from keywords + # 1. 通过关键词初始化 solver = AutoNodeClassifier( feature_module='deepgl', graph_models=['gat','gcn'], @@ -32,7 +32,7 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 device='auto' ) - # 2. initialize using instances + # 2. 使用实例初始化 from autogl.module import AutoFeatureEngineer, AutoGCN, AutoGAT, AnnealAdvisorHPO, Voting solver = AutoNodeClassifier( feature_module=AutoFeatureEngineer(), @@ -65,11 +65,11 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 .. code-block:: python - # initialize from config file + # 通过配置文件初始化 path_to_config = 'your/path/to/config' solver = AutoNodeClassifier.from_config(path_to_config) - # initialize from a dictionary + # 通过字典初始化 config = { 'models':{'gcn': None, 'gat': None}, 'hpo': {'name': 'tpe', 'max_evals': 10}, @@ -82,13 +82,13 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 优化 ------------ -初始化 solver 之后，你可以在给定的数据集上进行优化（请参考 :ref:`dataset` 和 :ref:`dataset documentation` 以创建数据集）。 +初始化 solver 之后，你可以在给定的数据集上进行优化（请参考 :ref:`dataset_cn` 和 :ref:`dataset documentation` 以创建数据集）。 你可以使用 ``fit()`` 或 ``fit_predict()`` 来进行优化，它们有相似的参数列表： .. code-block:: python - # load your dataset here + # 加载数据集 dataset = some_dataset() solver.fit(dataset, inplace=True) @@ -99,12 +99,12 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 .. code-block:: python - # split 0.2 of total nodes/graphs for train and 0.4 of nodes/graphs for validation, - # the rest 0.4 is left for test. + .. # 分割 20% 的节点/图用于训练，40% 的节点/图用于验证 + # 剩余 40% 用于测试 solver.fit(dataset, train_split=0.2, val_split=0.4) - # split 600 nodes/graphs for train and 400 nodes/graphs for validation, - # the rest nodes are left for test. + # 分割 600 个节点/图用于训练，400 个节点/图用于验证 + # 剩余的用于测试 solver.fit(dataset, train_split=600, val_split=400) 对于点分类问题，我们同样支持对训练和测试集的平衡采样：强制不同类别的节点数量相同。这种平衡模式可以通过在 ``fit()`` 使 ``balanced=True`` 来进行设置，而其默认值也是 ``True``。 @@ -115,9 +115,9 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 .. code-block:: python - # get current leaderboard of the solver + # 获取当前榜单 lb = solver.get_leaderboard() - # show the leaderboard info + # 展示榜单信息 lb.show() 你可以参考榜单的文档 :ref:`solver documentation` 以获取更多使用细节。 @@ -177,9 +177,9 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 对于 ``feature``，``hpo`` 和 ``ensemble``，它们对应的值都是字典，里面至少有一个键是 ``name``， 其它的参数则用于初始化对应模型。``name`` 指定了所使用的的算法，如果你不想使用某个模块，你可以传入 ``None``。 -对于 ``trainer``，你需要制指定它的超参数空间。请参考 :ref:`trainer` 或者 :ref:`train documentation` 来获取不同 trainer 的详细超参数空间信息。 +对于 ``trainer``，你需要制指定它的超参数空间。请参考 :ref:`trainer_cn` 或者 :ref:`train documentation` 来获取不同 trainer 的详细超参数空间信息。 -对于 ``models``，其值是另一个字典，它的键是需要优化的模型，值是对应模型的超参数空间。参考 :ref:`model` 或者 :ref:`model documentation` 来获取不同模型的详细超参数信息。 +对于 ``models``，其值是另一个字典，它的键是需要优化的模型，值是对应模型的超参数空间。参考 :ref:`model_cn` 或者 :ref:`model documentation` 来获取不同模型的详细超参数信息。 下面展示了配置所需字典的一个例子。 @@ -187,14 +187,14 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 config_for_node_classification = { 'feature': { - 'name': 'deepgl', # name of auto feature module - # following are the deepgl specified auto feature engineer arguments + 'name': 'deepgl', # 自动特征工程模块的名字 + # 下面是 deepgl 特征工程模块的专有参数 'fixlen': 100, 'max_epoch': 5 }, 'models': { 'gcn': - # specify the hp space of gcn + # 指定 gcn 的超参数空间 [ {'parameterName': 'num_layers', 'type': 'DISCRETE', 'feasiblePoints': '2,3,4'}, {'parameterName': 'hidden', 'type': 'NUMERICAL_LIST', 'numericalType': 'INTEGER', 'length': 3, @@ -202,35 +202,35 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 {'parameterName': 'dropout', 'type': 'DOUBLE', 'maxValue': 0.9, 'minValue': 0.1, 'scalingType': 'LINEAR'}, {'parameterName': 'act', 'type': 'CATEGORICAL', 'feasiblePoints': ['leaky_relu', 'relu', 'elu', 'tanh']} ], - 'gat': None, # set to None to use default hp space + 'gat': None, # 设置为空则使用默认的超参数空间 'gin': None } 'trainer': [ - # trainer hp space + # trainer 超参数空间 {'parameterName': 'max_epoch', 'type': 'INTEGER', 'maxValue': 300, 'minValue': 10, 'scalingType': 'LINEAR'}, {'parameterName': 'early_stopping_round', 'type': 'INTEGER', 'maxValue': 30, 'minValue': 10, 'scalingType': 'LINEAR'}, {'parameterName': 'lr', 'type': 'DOUBLE', 'maxValue': 0.001, 'minValue': 0.0001, 'scalingType': 'LOG'}, {'parameterName': 'weight_decay', 'type': 'DOUBLE', 'maxValue': 0.005, 'minValue': 0.0005, 'scalingType': 'LOG'} ], 'hpo': { - 'name': 'autone', # name of hpo module - # following are the autone specified auto hpo arguments + 'name': 'autone', # 超参数优化模块的名字 + # 下面是 autone 超参数优化模块的专有参数 'max_evals': 10, 'subgraphs': 10, 'sub_evals': 5 }, 'ensemble': { - 'name': 'voting', # name of ensemble module - # following are the voting specified auto ensemble arguments + 'name': 'voting', # 集成模块的名字 + # 下面是 voting 集成模块的专有参数 'size': 2 } } config_for_graph_classification = { - 'feature': None, # set to None to disable this module - # do not add field `model` to use default settings of solver + 'feature': None, # 设置为空会禁用该模块 + # 不添加 `model` 域以使用默认设置 'trainer': [ - # trainer hp space + # trainer 超参数空间 {'parameterName': 'max_epoch', 'type': 'INTEGER', 'maxValue': 300, 'minValue': 10, 'scalingType': 'LINEAR'}, {'parameterName': 'batch_size', 'type': 'INTEGER', 'maxValue': 128, 'minValue': 32, 'scalingType': 'LOG'}, {'parameterName': 'early_stopping_round', 'type': 'INTEGER', 'maxValue': 30, 'minValue': 10, 'scalingType': 'LINEAR'}, @@ -238,9 +238,9 @@ AutoGL 项目用 ``solver`` 类来控制整个自动机器学习流程。目前 {'parameterName': 'weight_decay', 'type': 'DOUBLE', 'maxValue': 5e-3, 'minValue': 5e-4, 'scalingType': 'LOG'}, ], 'hpo': { - 'name': 'random', # name of hpo module - # following are the random specified auto hpo arguments + 'name': 'random', # 超参数优化模块的名字 + # 下面是 random 超参数优化模块的专有参数 'max_evals': 10 }, - 'ensemble': None # set to None to disable this module + 'ensemble': None # 设置为空以禁用该模块 } \ No newline at end of file diff --git a/docs/docfile/tutorial_cn/t_trainer.rst b/docs/docfile/tutorial_cn/t_trainer.rst new file mode 100644 index 00000000..0ad3118a --- /dev/null +++ b/docs/docfile/tutorial_cn/t_trainer.rst @@ -0,0 +1,182 @@ +.. _trainer_cn: + +AutoGL训练器 +============== + +AutoGL项目使用``训练器``来处理任务的自动训练。目前，我们支持以下任务： + +* ``NodeClassificationTrainer`` 用于半监督的节点分类 +* ``GraphClassificationTrainer`` 用于有监督的图分类 +* ``LinkPredictionTrainer`` 用于链接预测 + + +简单初始化 +------------------- +类似于模型，我们也对所有训练器使用简单的初始化。当调用`__init__()'时，只有（部分）超参数会被设置。只有在明确调用`initialize()`后，`训练器'才会有其核心`模型'，在所有的超参数被正确设置后，这将在`solver`和`duplicate_from_hyper_parameter()`中自动完成， + + +训练和预测 +----------------- +初始化训练器后，你可以在给定的数据集上训练它。 + +到目前为止，我们已经给出了节点分类、图分类和链接预测等任务的训练和测试函数。你也可以按照与我们相似的模式创建你的任务。对于训练，你需要定义``train_only()``并在``train()``中使用。对于测试，你需要定义`predict_proba()`并在`predict()`中使用它。 + +评价函数在``evaluate()``中定义，你可以使用你的评价指标和方法。 + + +基于采样而进行的节点分类 +--------------------------------- +根据目前的各种研究，用空间采样训练已被证明作为一种有效的技术在大规模图上进行表示学习。 +我们提供了各种代表性的采样机制的实现，包括邻居采样、依赖层重要性采样（LADIES）和GraphSAINT。 +利用各种有效的采样机制，用户可以在大规模的图数据集上利用这个库，例如Reddit。 + +具体来说，由于各种采样技术通常需要模型支持在转发中的一些分层处理，现在只有提供的GCN和GraphSAGE模型可以用于节点取样（邻居取样）和层级取样（LADIES）。 +更多的模型和任务计划在未来的版本中支持采样。 + + +* 节点采样（GraphSAGE) + 支持 ``GCN ``和 ``GraphSAGE``两种模型。 + +* 分层取样（依赖层的重要性取样） + 当前版本只支持``GCN``模型。 + +* 子图式采样（GraphSAINT） + 由于GraphSAINT采样技术对采用的模型没有具体要求。大多数可用的模型对于采用GraphSAINT技术是可行的。 + 然而，在GraphSAINT技术实际应用时，预测过程是一个潜在的瓶颈，甚至是障碍。 + 当GraphSAINT技术实际应用于大规模图时，预测过程是一个潜在的瓶颈甚至障碍。 + 因此，要采用的模型最好是支持分层预测的。 + 而我们提供的``GCN ``模型已经满足了这一强化要求。 + 根据实验GraphSAINT的实现现在有了杠杆作用，可以支持小于*Flickr*规模的整体图数据。 + +采样技术可以通过采用相应的训练器来利用 +``NodeClassificationGraphSAINTTrainer``, +``NodeClassificationLayerDependentImportanceSamplingTrainer``, +和 ``NodeClassificationNeighborSamplingTrainer``. +你可以在YAML配置文件中指定训练器的相应名称 +或者用特定训练器的实例来实例化解算器`AutoNodeClassifier`。 +的实例。然而，请确保正确管理一些关键的 +超参数空间内的一些关键超参数。具体来说： + + +对于 ``NodeClassificationLayerDependentImportanceSamplingTrainer``, 你需要设置 +超参数 ``sampled_node_sizes``。 ``sampled_node_sizes``的空间应该是一个和**Sequential Model**大小相同的列表. 例如，如果你有一个 +层数为4的模型，你需要正确传递超参数空间： + +.. code-block:: python + + solver = AutoNodeClassifier( + graph_models=(A_MODEL_WITH_4_LAYERS,), + default_trainer='NodeClassificationLayerDependentImportanceSamplingTrainer', + trainer_hp_space=[ + # (required) you need to set the trainer_hp_space properly. + { + 'parameterName': 'sampled_node_sizes', + 'type': 'NUMERICAL_LIST', + "numericalType": "INTEGER", + "length": 4, # same with the layer number of your model + "minValue": [200,200,200,200], + "maxValue": [1000,1000,1000,1000], + "scalingType": "LOG" + }, + ... + ] + ) + +如果你的模型的层数是一个可搜索的超参数，你也可以设置 ``cutPara`` +和 ``cutFunc`` ， 使其与你的模型的层数超参数相联系。 + +.. code-block:: python + + ''' + Suppose the layer number of your model is of the following forms: + { + 'parameterName': 'layer_number', + 'type': 'INTEGER', + 'minValue': 2, + 'maxValue': 4, + 'scalingType': 'LOG' + } + ''' + + solver = AutoNodeClassifier( + graph_models=(A_MODEL_WITH_DYNAMIC_LAYERS,), + default_trainer='NodeClassificationLayerDependentImportanceSamplingTrainer', + trainer_hp_space=[ + # (required) you need to set the trainer_hp_space properly. + { + 'parameterName': 'sampled_node_sizes', + 'type': 'NUMERICAL_LIST', + "numericalType": "INTEGER", + "length": 4, # max length + "cutPara": ("layer_number", ), # link with layer_number + "cutFunc": lambda x:x[0], # link with layer_number + "minValue": [200,200,200,200], + "maxValue": [1000,1000,1000,1000], + "scalingType": "LOG" + }, + ... + ] + ) + + +类似地, 如果你想使用 ``NodeClassificationNeighborSamplingTrainer``, 你需要保证超参 ``sampling_sizes`` 和模型层数相同，例如: + +.. code-block:: python + + ''' + Suppose the layer number of your model is of the following forms: + { + 'parameterName': 'layer_number', + 'type': 'INTEGER', + 'minValue': 2, + 'maxValue': 4, + 'scalingType': 'LOG' + } + ''' + + solver = AutoNodeClassifier( + graph_models=(A_MODEL_WITH_DYNAMIC_LAYERS,), + default_trainer='NodeClassificationNeighborSamplingTrainer', + trainer_hp_space=[ + # (required) you need to set the trainer_hp_space properly. + { + 'parameterName': 'sampling_sizes', + 'type': 'NUMERICAL_LIST', + "numericalType": "INTEGER", + "length": 4, # max length + "cutPara": ("layer_number", ), # link with layer_number + "cutFunc": lambda x:x[0], # link with layer_number + "minValue": [20,20,20,20], + "maxValue": [100,100,100,100], + "scalingType": "LOG" + }, + ... + ] + ) + + +你也可以直接在模型列表中传递一个训练器。一个简单的例子演示如下： + +.. code-block:: python + + ladies_sampling_trainer = NodeClassificationLayerDependentImportanceSamplingTrainer( + model='gcn', num_features=dataset.num_features, num_classes=dataset.num_classes, ... + ) + + ladies_sampling_trainer.hyper_parameter_space = [ + # (required) you need to set the trainer_hp_space properly. + { + 'parameterName': 'sampled_node_sizes', + 'type': 'NUMERICAL_LIST', + "numericalType": "INTEGER", + "length": 4, # max length + "cutPara": ("num_layers", ), # link with layer_number + "cutFunc": lambda x:x[0], # link with layer_number + "minValue": [200,200,200,200], + "maxValue": [1000,1000,1000,1000], + "scalingType": "LOG" + }, + ... + ] + + AutoNodeClassifier(graph_models=(ladies_sampling_trainer,), ...) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 57720d3b..face3162 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -109,6 +109,26 @@ In AutoGL, the tasks are solved by corresponding solvers, which in general do th docfile/tutorial/t_ensemble docfile/tutorial/t_solver +.. toctree:: + :maxdepth: 2 + :caption: 中文教程 + + docfile/tutorial_cn/t_quickstart + docfile/tutorial_cn/t_hetero_node_clf + docfile/tutorial_cn/t_homo_graph_classification_gin + docfile/tutorial_cn/t_backend + + .. + docfile/tutorial/t_dataset + docfile/tutorial/t_fe + + docfile/tutorial_cn/t_model + docfile/tutorial_cn/t_trainer + docfile/tutorial_cn/t_hpo + docfile/tutorial_cn/t_nas + docfile/tutorial_cn/t_ensemble + docfile/tutorial_cn/t_solver + .. toctree:: :maxdepth: 2 :caption: Documentation From da0fb53e1b6f900ec450904de34a5c9b18bd283f Mon Sep 17 00:00:00 2001 From: Generall Date: Sat, 16 Apr 2022 17:48:42 +0800 Subject: [PATCH 32/37] add hetero init --- autogl/module/model/dgl/hetero/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 autogl/module/model/dgl/hetero/__init__.py diff --git a/autogl/module/model/dgl/hetero/__init__.py b/autogl/module/model/dgl/hetero/__init__.py new file mode 100644 index 00000000..e69de29b From 9386d0682e56e05073f8a7e48a85857bbc84296d Mon Sep 17 00:00:00 2001 From: Frozenmad <351549709@sjtu.edu.cn> Date: Sun, 17 Apr 2022 14:44:58 +0800 Subject: [PATCH 33/37] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index ec87ad73..644d7e89 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -3,7 +3,7 @@ name: Bug report about: Create a report to help us improve title: "[BUG]" labels: bug -assignees: Frozenmad +assignees: general502570 --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index c6b696da..68b86de7 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -3,7 +3,7 @@ name: Feature request about: Suggest an idea for this project title: "[FEATURE]" labels: enhancement -assignees: Frozenmad +assignees: general502570 --- From ab25531ab89b755ce1a3294ac5c8d5c225e59422 Mon Sep 17 00:00:00 2001 From: Generall <465383637@qq.com> Date: Mon, 18 Apr 2022 19:27:56 +0800 Subject: [PATCH 34/37] Update README_cn.md --- README_cn.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README_cn.md b/README_cn.md index 0def3f07..ad989010 100644 --- a/README_cn.md +++ b/README_cn.md @@ -168,3 +168,10 @@ make clean && make html ## 版权相关 从v0.2版本开始，智图的所有代码采用[Apache license](LICENSE)。 +## 用户交流微信群 +![weixin](https://user-images.githubusercontent.com/17705534/163801846-2aa1bdf8-919e-4636-b7d9-e5346f352c15.jpg) + + + + + From 941b7c66fc725eabed34e433c8171eec870f7f10 Mon Sep 17 00:00:00 2001 From: Generall Date: Tue, 19 Apr 2022 11:15:02 +0800 Subject: [PATCH 35/37] fix some tutorial --- docs/docfile/tutorial/t_dataset.rst | 1 + docs/docfile/tutorial_cn/t_dataset.rst | 5 +++-- docs/docfile/tutorial_cn/t_quickstart.rst | 10 +++++----- docs/index.rst | 14 ++++---------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/docs/docfile/tutorial/t_dataset.rst b/docs/docfile/tutorial/t_dataset.rst index 7ec0df75..d10f5e88 100644 --- a/docs/docfile/tutorial/t_dataset.rst +++ b/docs/docfile/tutorial/t_dataset.rst @@ -10,6 +10,7 @@ Besides, users are able to leverage a unified abstraction provided in AutoGL, `` A basic example to construct an instance of ``GeneralStaticGraph`` is shown as follows. .. code-block:: python + from autogl.data.graph import GeneralStaticGraph, GeneralStaticGraphGenerator ''' Construct a custom homogeneous graph ''' diff --git a/docs/docfile/tutorial_cn/t_dataset.rst b/docs/docfile/tutorial_cn/t_dataset.rst index fc0f7052..bf6d0694 100644 --- a/docs/docfile/tutorial_cn/t_dataset.rst +++ b/docs/docfile/tutorial_cn/t_dataset.rst @@ -7,6 +7,7 @@ AutoGL 数据集 同时，用户可以使用AutoGL所提供的统一静态图容器``GeneralStaticGraph``自定义静态同构图及异构图，例如： .. code-block:: python + from autogl.data.graph import GeneralStaticGraph, GeneralStaticGraphGenerator ''' 创建同构图 ''' @@ -34,7 +35,7 @@ AutoGL目前提供如下多种常用基准数据集： 半监督节点分类： +------------------+------------+-----------+--------------------------------+ -| 数据集 | PyG | DGL | 默认train/val/test划分 | +| 数据集 | PyG | DGL | 默认train/val/test划分 | +==================+============+===========+================================+ | Cora | ✓ | ✓ | ✓ | +------------------+------------+-----------+--------------------------------+ @@ -65,7 +66,7 @@ AutoGL目前提供如下多种常用基准数据集： 图分类任务： MUTAG, IMDB-Binary, IMDB-Multi, PROTEINS, COLLAB等 +-------------+------------+------------+--------------+------------+--------------------+ -| 数据集 | PyG | DGL | 节点特征 | 标签 | 边特征 | +| 数据集 | PyG | DGL | 节点特征 | 标签 | 边特征 | +=============+============+============+==============+============+====================+ | MUTAG | ✓ | ✓ | ✓ | ✓ | ✓ | +-------------+------------+------------+--------------+------------+--------------------+ diff --git a/docs/docfile/tutorial_cn/t_quickstart.rst b/docs/docfile/tutorial_cn/t_quickstart.rst index dca8e970..30147efb 100644 --- a/docs/docfile/tutorial_cn/t_quickstart.rst +++ b/docs/docfile/tutorial_cn/t_quickstart.rst @@ -14,8 +14,8 @@ 下图描述了智图框架的工作流程。 -为了达到自动机器学习的目标，我们提出的自动图学习框架组织如下。我们用 ``数据集（dataset）`` 来维护由用户给出的图形数据集。为了指定目标任务，需要构建一个``求解器（solver）``对象。``求解器``内部，有五个子模型来帮助完成完成自动图学习任务，即``自动特征工程``,``自动模型``，``神经架构搜索``，``超参数优化``和``自动集成学习``来根据您的需求自动预处理/增强您的数据，选择和优化深度模型并集成。 -假设你想在数据集``Cora``上进行自动图学习。首先，您可以通过 ``dataset``模块得到``Cora``数据集： +为了达到自动机器学习的目标，我们提出的自动图学习框架组织如下。我们用 ``数据集（dataset）``来维护由用户给出的图形数据集。为了指定目标任务，需要构建一个 ``求解器（solver）``对象。 ``求解器``内部，有五个子模型来帮助完成完成自动图学习任务，即 ``自动特征工程``， ``自动模型``， ``神经架构搜索``， ``超参数优化``和 ``自动集成学习``来根据您的需求自动预处理/增强您的数据，选择和优化深度模型并集成。 +假设你想在数据集 ``Cora``上进行自动图学习。首先，您可以通过 ``dataset``模块得到 ``Cora``数据集： .. code-block:: python @@ -24,7 +24,7 @@ 数据集将自动下载给您。请参考:ref:`dataset`或:ref:`dataset documentation`获取更多关于数据集构造、可用数据集、添加本地数据集等详细信息。 -在导出数据集之后，您可以构建一个``节点分类求解器（node classification solver）``来处理自动训练过程： +在导出数据集之后，您可以构建一个 ``节点分类求解器（node classification solver）``来处理自动训练过程： .. code-block:: python @@ -39,7 +39,7 @@ device=device ) -通过这种方式，我们构建了一个``节点分类求解器``，它使用``deepgl``进行特征工程，并使用``anneal``超参数优化器对给定的三个模型“``['gcn','gat']``进行优化。派生的模型将使用``voting``集成器进行集成。请参考相应的教程或文档，以了解可用子模块的定义和用法。 +通过这种方式，我们构建了一个 ``节点分类求解器``，它使用 ``deepgl``进行特征工程，并使用 ``anneal``超参数优化器对给定的三个模型“ ``['gcn','gat']``进行优化。派生的模型将使用 ``voting``集成器进行集成。请参考相应的教程或文档，以了解可用子模块的定义和用法。 Then, you can fit the solver and then check the leaderboard: 接下来，你可以安装求解器，然后查看排行榜： @@ -49,7 +49,7 @@ Then, you can fit the solver and then check the leaderboard: solver.fit(cora_dataset, time_limit=3600) solver.get_leaderboard().show() -``time_limit``设置为3600，这样整个自动绘图过程不会超过1小时。``solver.show()``将显示由``solver``维护的模型，以及它们在验证数据集上的性能。 +``time_limit``设置为3600，这样整个自动绘图过程不会超过1小时。 ``solver.show()``将显示由 ``solver``维护的模型，以及它们在验证数据集上的性能。 然后，你可以使用提供的评估函数进行预测和结果评估： diff --git a/docs/index.rst b/docs/index.rst index face3162..62bde455 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -97,11 +97,8 @@ In AutoGL, the tasks are solved by corresponding solvers, which in general do th docfile/tutorial/t_hetero_node_clf docfile/tutorial/t_homo_graph_classification_gin docfile/tutorial/t_backend - - .. - docfile/tutorial/t_dataset - docfile/tutorial/t_fe - + docfile/tutorial/t_dataset + docfile/tutorial/t_fe docfile/tutorial/t_model docfile/tutorial/t_trainer docfile/tutorial/t_hpo @@ -117,11 +114,8 @@ In AutoGL, the tasks are solved by corresponding solvers, which in general do th docfile/tutorial_cn/t_hetero_node_clf docfile/tutorial_cn/t_homo_graph_classification_gin docfile/tutorial_cn/t_backend - - .. - docfile/tutorial/t_dataset - docfile/tutorial/t_fe - + docfile/tutorial_cn/t_dataset + docfile/tutorial_cn/t_fe docfile/tutorial_cn/t_model docfile/tutorial_cn/t_trainer docfile/tutorial_cn/t_hpo From 487f2b2f798b9b1363ad5dc100fb410b12222e06 Mon Sep 17 00:00:00 2001 From: Generall Date: Tue, 19 Apr 2022 11:40:55 +0800 Subject: [PATCH 36/37] v0.3.1 --- README.md | 1 + README_cn.md | 1 + autogl/__init__.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 81d5d8fa..9c76bdf2 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Feel free to open issues ## News! +- 2022.4.19 New version v0.3.1！We have released Chinese tutorial for the first time! - 2021.12.31 New Version! v0.3.0-pre is here! - AutoGL now support [__Deep Graph Library (DGL)__](https://www.dgl.ai/) backend to be interface-friendly for DGL users! All the homogeneous node classification task, link prediction task, and graph classification task are currently supported under DGL backend. AutoGL is also compatible with PyG 2.0 now. - The __heterogeneous__ node classification tasks are now supported! See [hetero tutorial](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_hetero_node_clf.html) for more details. diff --git a/README_cn.md b/README_cn.md index ad989010..f52b34f1 100644 --- a/README_cn.md +++ b/README_cn.md @@ -13,6 +13,7 @@ ## 最新消息 +- 2022.4.19 v0.3.1版本更新！首次更新中文教程！ - 2021.12.31 v0.3.0-pre版本更新! - 智图目前支持[__Deep Graph Library (DGL)__](https://www.dgl.ai/)作为后端，以方便DGL的用户使用。目前在DGL后端已经支持同构图的节点分类、链接预测以及图分类等任务。智图现在也可兼容PyG 2.0版本。 - 智图可以支持__异构图__节点分类任务!详情请参考[异构图教程](http://mn.cs.tsinghua.edu.cn/autogl/documentation/docfile/tutorial/t_hetero_node_clf.html)。 diff --git a/autogl/__init__.py b/autogl/__init__.py index 00a5ff99..f5c02390 100644 --- a/autogl/__init__.py +++ b/autogl/__init__.py @@ -16,4 +16,4 @@ train, ) -__version__ = "0.3.0-pre" +__version__ = "0.3.1" diff --git a/setup.py b/setup.py index afaa49d2..ea4be0e3 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ ''' https://setuptools.readthedocs.io/en/latest/ ''' setup( name='autogl', - version='0.3.0-pre', + version='0.3.1', author='THUMNLab/aglteam', maintainer='THUMNLab/aglteam', author_email='autogl@tsinghua.edu.cn', From 5901d35718bc859f9e3ddcecf79b3fc30718c04a Mon Sep 17 00:00:00 2001 From: BeiniXie Date: Tue, 28 Jun 2022 15:58:40 +0800 Subject: [PATCH 37/37] gat_pool --- autogl/module/model/pyg/__init__.py | 2 + autogl/module/model/pyg/gin_gat_glf.py | 262 +++++++++++++++++++++++++ ppi/test_model.py | 207 +++++++++++++++++++ 3 files changed, 471 insertions(+) create mode 100644 autogl/module/model/pyg/gin_gat_glf.py create mode 100644 ppi/test_model.py diff --git a/autogl/module/model/pyg/__init__.py b/autogl/module/model/pyg/__init__.py index cc17cb23..462fbee6 100644 --- a/autogl/module/model/pyg/__init__.py +++ b/autogl/module/model/pyg/__init__.py @@ -8,6 +8,7 @@ from .gcn import AutoGCN from .gat import AutoGAT from .gin import AutoGIN +from .gin_gat_glf import AutoGATPooling __all__ = [ "ModelUniversalRegistry", @@ -19,4 +20,5 @@ "AutoGCN", "AutoGAT", "AutoGIN", + "AutoGATPooling" ] diff --git a/autogl/module/model/pyg/gin_gat_glf.py b/autogl/module/model/pyg/gin_gat_glf.py new file mode 100644 index 00000000..551a8a87 --- /dev/null +++ b/autogl/module/model/pyg/gin_gat_glf.py @@ -0,0 +1,262 @@ +import torch +import torch.nn.functional as F +from torch_geometric.nn import GATConv, GINConv +from torch.nn import BatchNorm1d, Linear, ReLU, Sequential +from torch_scatter import scatter + +from . import register_model +from .base import BaseAutoModel, activate_func +from ....utils import get_logger + +LOGGER = get_logger("GATModel") + + +def set_default(args, d): + for k, v in d.items(): + if k not in args: + args[k] = v + return args + + + +def construct_batch(batch_size,num_nodes): + batch=torch.cat([torch.ones(num_nodes).long()*i for i in range(batch_size)],dim=0) + return batch + +def construct_inner_edge(edge_index,batch_size,num_nodes): + shift=torch.LongTensor([[num_nodes,num_nodes]]).T.to(edge_index.device) + e=torch.cat([edge_index+shift*i for i in range(batch_size)],dim=-1) + return e + +def construct_cross_edge(edge_index,batch_size,num_nodes1,num_nodes2): + + shift=torch.LongTensor([[num_nodes1,num_nodes2]]).T.to(edge_index.device) + e=torch.cat([edge_index+shift*i for i in range(batch_size)],dim=-1) + return e + +def construct_cross_edge_both(edge_index,batch_size,num_nodes1,num_nodes2): + shift0 = num_nodes1+num_nodes2 + shift1=torch.LongTensor([[0,num_nodes1]]).T.to(edge_index.device) + edge_index=torch.cat([edge_index+shift0*i for i in range(batch_size)],dim=-1) + e = torch.cat([edge_index+shift1 for i in range(batch_size)],dim=-1) + return e + +class GATPool(torch.nn.Module): + def __init__(self, args): + super(GATPool, self).__init__() + self.args = args + self.edge_index,self.inner_edge_indexs,self.cross_edge_indexs,self.num_nodes=self.args["metadata"] + + missing_keys = list( + set( + [ + "features_num", + "num_classes", + "hidden", + "heads", + "dropout", + "act", + ] + ) + - set(self.args.keys()) + ) + if len(missing_keys) > 0: + raise Exception("Missing keys: %s." % ",".join(missing_keys)) + + def crossconv0(): + def pool(x,edge_index,num_nodes): + res = x[edge_index[0]] + res = scatter(res, edge_index[1], dim=0, dim_size=num_nodes, reduce='mean') + return res + return pool + + self.inner_convs = torch.nn.ModuleList() + self.cross_convs1 = torch.nn.ModuleList() + + self.conv1 = GINConv(Sequential(Linear(self.args["features_num"], self.args["hidden"][0]), BatchNorm1d(self.args["hidden"][0]), ReLU(), Linear(self.args["hidden"][0], self.args["hidden"][0]), ReLU())) + + self.cross_convs0=[crossconv0() for i in range(len(self.num_nodes)-2)] + + for l in range(len(self.num_nodes)-2): + + self.cross_convs1.append( + GATConv(self.args["hidden"][l],self.args["hidden"][l],heads=self.args["heads"], concat=False, dropout=self.args["dropout"], add_self_loops=False) + ) + if l==len(self.num_nodes)-3: + self.inner_convs.append( + GINConv(Sequential(Linear(self.args["hidden"][l], self.args["hidden"][l]), BatchNorm1d(self.args["hidden"][l]), ReLU(), Linear(self.args["hidden"][l], self.args["hidden"][l]), ReLU())) + ) + else: + self.inner_convs.append( + GINConv(Sequential(Linear(self.args["hidden"][l], self.args["hidden"][l+1]), BatchNorm1d(self.args["hidden"][l+1]), ReLU(), Linear(self.args["hidden"][l+1], self.args["hidden"][l+1]), ReLU())) + ) + + + self.lin1 = Linear(self.num_nodes[-2]*self.args["hidden"][-1], self.args["hidden"][-1]) + self.lin2 = Linear(self.args["hidden"][-1], self.args["num_classes"]) + + self.cache={} + + # @timing + def forward(self, x, batch, return_attention_weights=False): + batch_size=batch[-1].item()+1 + attentions = [] + + # self.edge_index,self.inner_edge_indexs,self.cross_edge_indexs,self.num_nodes=metadata + edge_index=self.construct_inner_edge(self.edge_index,batch_size,self.num_nodes[0],-1) + + x = self.conv1(x, edge_index) + + for i in range(len(self.num_nodes)-2): + num_nodes1=self.num_nodes[i] + num_nodes2=self.num_nodes[i+1] + cross_edge_index=self.cross_edge_indexs[i] + cross_edge_index0=self.construct_cross_edge(cross_edge_index,batch_size,num_nodes1,num_nodes2,i) + cross_edge_index1=self.construct_cross_edge_both(cross_edge_index,batch_size,num_nodes1,num_nodes2,i) + + inner_edge_index=self.inner_edge_indexs[i] + inner_edge_index=self.construct_inner_edge(inner_edge_index,batch_size,num_nodes2,i) + + x0 = x # last layerx + x0 = x0.reshape(batch_size, num_nodes1,-1) + x1 = self.cross_convs0[i](x,cross_edge_index0,num_nodes2*batch_size) # get new layer x初始值 + x1 = x1.reshape(batch_size, num_nodes2,-1) + x = torch.cat([x0,x1], dim=1) + x = x.reshape(batch_size*(num_nodes1+num_nodes2),-1) + + if return_attention_weights: + x, (edge_index_, alpha) = self.cross_convs1[i](x, cross_edge_index1, return_attention_weights=return_attention_weights) + attentions.append((edge_index_, alpha)) + else: + print('cross_edge_index1:',cross_edge_index1) + print('x:',x.size()) + x = self.cross_convs1[i](x, cross_edge_index1) + + # 生成新一层数据的mask + mask = (torch.arange(num_nodes2+num_nodes1) >= num_nodes1) + mask = mask.repeat(batch_size) + x = x[mask] + # x = F.elu(x) + x = activate_func(x, self.args["act"]) + + x = self.inner_convs[i](x,inner_edge_index) + + x = x.view(batch_size,-1) + x = activate_func(self.lin1(x), self.args["act"]) + x = self.lin2(x) + + if return_attention_weights: + return x,attentions + else: + return x + + # @timing + def construct_cross_edge(self,edge_index,batch_size,num_nodes1,num_nodes2,index): + cindex = ('c0',batch_size,index) + if cindex not in self.cache: + edge_index=construct_cross_edge(edge_index,batch_size,num_nodes1,num_nodes2) + self.cache[cindex]=edge_index + else: + edge_index=self.cache[cindex] + return edge_index + # @timing + def construct_cross_edge_both(self,edge_index,batch_size,num_nodes1,num_nodes2,index): + cindex = ('c1',batch_size,index) + if cindex not in self.cache: + edge_index=construct_cross_edge_both(edge_index,batch_size,num_nodes1,num_nodes2) + self.cache[cindex]=edge_index + else: + edge_index=self.cache[cindex] + return edge_index + # @timing + def construct_inner_edge(self,edge_index,batch_size,num_nodes,index): + cindex = ('i',batch_size,index) + if cindex not in self.cache: + edge_index=construct_inner_edge(edge_index,batch_size,num_nodes) + self.cache[cindex]=edge_index + else: + edge_index=self.cache[cindex] + return edge_index + + # def lp_encode(self, data): + # x = data.x + # for i in range(self.num_layer - 1): + # x = self.convs[i](x, data.edge_index) # Jie + # if i != self.num_layer - 2: + # x = activate_func(x, self.args["act"]) + # # x = F.dropout(x, p=self.args["dropout"], training=self.training) + # return x + + + +@register_model("gat-pooling-model") +class AutoGATPooling(BaseAutoModel): + r""" + + Parameters + ---------- + num_features: `int`. + The dimension of features. + + num_classes: `int`. + The number of classes. + + device: `torch.device` or `str` + The device where model will be running on. + + init: `bool`. + If True(False), the model will (not) be initialized. + + args: Other parameters. + """ + + def __init__( + self, num_features=None, num_classes=None, device=None, **args + ): + super().__init__(num_features, num_classes, device, **args) + self.hyper_parameter_space = [ + { + "parameterName": "hidden", + "type": "NUMERICAL_LIST", + "numericalType": "INTEGER", + "length": 3, + "minValue": [8, 8, 8], + "maxValue": [64, 64, 64], + "scalingType": "LOG", + "cutPara": ("num_layers",), + "cutFunc": lambda x: x[0] - 1, + }, + { + "parameterName": "dropout", + "type": "DOUBLE", + "maxValue": 0.6, + "minValue": 0, + "scalingType": "LINEAR", + }, + { + "parameterName": "heads", + "type": "DISCRETE", + "feasiblePoints": "2,4,8,16", + }, + { + "parameterName": "act", + "type": "CATEGORICAL", + "feasiblePoints": ["leaky_relu", "relu", "elu", "tanh"], + }, + ] + + self.hyper_parameters = { + "hidden": [16,16,16], + "heads": 4, + "dropout": 0, + "act": "relu", + } + + def _initialize(self): + # """Initialize model.""" + self._model = GATPool({ + "features_num": self.input_dimension, + "num_classes": self.output_dimension, + "metadata":None, + **self.hyper_parameters + }).to(self.device) diff --git a/ppi/test_model.py b/ppi/test_model.py new file mode 100644 index 00000000..4f44f78b --- /dev/null +++ b/ppi/test_model.py @@ -0,0 +1,207 @@ +""" +Performance check of AutoGL model + PYG (trainer + dataset) +""" +import os +import pickle +import numpy as np +from tqdm import tqdm +from torch_geometric.loader import DataLoader + +os.environ["AUTOGL_BACKEND"] = "pyg" + +import torch +import torch.nn.functional as F +import torch_geometric.transforms as T +from autogl.module.model.pyg import AutoGCN, AutoGAT, AutoSAGE, AutoGATPooling +from autogl.datasets import utils +from autogl.solver.utils import set_seed +from helper import get_encoder_decoder_hp +import logging +from datetime import datetime +from torch_geometric.data import Data +from torch.utils.tensorboard import SummaryWriter +from utils import * +from sklearn.metrics import f1_score +from sklearn.metrics import roc_curve, auc, roc_auc_score +from sklearn.metrics import (accuracy_score) + + + +def train(model,train_loader,optimizer,device): + model.train() + total_loss = 0 + for data in train_loader: + data = data.to(device) + optimizer.zero_grad() + output = model(data.x, data.batch) + loss = F.cross_entropy(output, data.y) + loss.backward() + optimizer.step() + total_loss += float(loss) * data.num_graphs + return total_loss / len(train_loader.dataset) + + +@torch.no_grad() +def test(model,loader,device): + model.eval() + ys = [] + preds = [] + for data in loader: + ys.append(data.y) + data = data.to(device) + out = model(data.x, data.batch).cpu() + + preds.append(out) + # print("edge_index:",edge_index, alpha) + + ys = torch.cat(ys).numpy() + preds = F.softmax(torch.cat(preds), dim = 1).numpy() + preds = np.argmax(preds, axis=1) + acc=accuracy_score(ys,preds) + + return acc + + +def get_model_att(test_dataset, model): + """output attentions for test_dataset of current model + """ + + all_atts = [] + for data in test_dataset: + data = data.to(device) + data.batch = torch.LongTensor([0]*data.num_nodes).to(device) + _, attentions = model(data.x, data.batch, return_attention_weights=True) + all_atts.append(attentions) + return all_atts + +def train_till_end(fold,device,foldi, args, metadata): + train_loader=DataLoader(fold[0], args.batch_size, shuffle = True) + val_loader=DataLoader(fold[1], args.batch_size) + test_loader=DataLoader(fold[2], args.batch_size) + test_dataset = DataLoader(fold[2], 1) + model = AutoGATPooling( + num_features=dataset.num_features, + num_classes=dataset.num_classes, + device=device, + init=False, + num_nodes=num_nodes + ).from_hyper_parameter( + { + "hidden": [16,16,16], + "dropout": 0., + "act": "relu", + "heads":4, + "metadata":metadata, + } + ).model + + model.to(device) + # model = Net(num_features,hid_dim,num_classes,metadata).to(device) + + + optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) + earlystop = EarlyStopping(mode="max", patience=patience) + best_val=0 + best_test=0 + best_attentions = None + with tqdm(range(args.max_epochs)) as bar: + for epoch in bar: + loss = train(model,train_loader,optimizer,device) + # train_metric=test(model,train_loader,device) + train_metric=0 + if val_ratio>0: + val_metric=test(model,val_loader,device,) + else: + val_metric=test(model,train_loader,device) + test_metric=test(model,test_loader,device) + + if val_metric>best_val: + best_val=val_metric + best_test=test_metric + best_attentions=get_model_att(test_dataset, model) + bar.set_postfix(loss=loss,btest=best_test,bval=best_val,test_metric=test_metric,val_metric=val_metric,train_metric=train_metric) + + if writer: + writer.add_scalar(f"Model{foldi}/train_loss", loss, epoch) + writer.add_scalar(f"Model{foldi}/train_metric", train_metric, epoch) + writer.add_scalar(f"Model{foldi}/test_metric", test_metric, epoch) + writer.add_scalar(f"Model{foldi}/val_metric", val_metric, epoch) + + if earlystop.step(val_metric): + break + print(f'Acc :{best_test}') + + return best_test, best_attentions + + +if __name__ == '__main__': + + import argparse + parser = argparse.ArgumentParser('pyg model') + parser.add_argument('--device', type=str, default='cuda') + parser.add_argument('--repeat', type=int, default=1) + parser.add_argument('--lr', type=float, default=5e-5) + parser.add_argument('--weight_decay', type=float, default=0.0) + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--max_epochs', type=int, default=500) + + args = parser.parse_args() + + device=torch.device('cuda') + logging.basicConfig(level=logging.ERROR) + + seed_everything(0) + log_dir='logs/' + timestamp = datetime.now().strftime('%m%d-%H-%M-%S') + writer = SummaryWriter(log_dir) + + # read data + datafile='./data/processed2-PPI-3' + data=torch.load(datafile) # x y ei ef inner_links cross_links hid2ids + x=data['x'] + y=data['y'] + edge_index=data['ei'].to(device) + # edge_weight=data['ef'] + inner_edge_index=data['inner_links'] + cross_edge_index=data['cross_links'] + inner_edge_index=[i.to(device) for i in inner_edge_index] + cross_edge_index=[i.to(device) for i in inner_edge_index] + + num_nodes=[len(_) for _ in data['hid2ids']] # num of nodes from 1 to last layer + metadata=[edge_index,inner_edge_index,cross_edge_index,num_nodes] + + datas=[Data(x=x[i],y=int(y[i])) for i in range(x.shape[0])] + + train_dataset=dataset=MPPIDataset(datas) + num_features=dataset.num_features + num_classes=dataset.num_classes + + #### hyper + # device='cpu' + + # hid_dim=16 + # max_epochs = 500 + # batch_size = 32 #32 + batch_size = 32 + patience = 50 + val_ratio=0 + # lr=1e-4 + # lr = 5e-5 + #### folds + folds=get_fold(datas,5,val_ratio=val_ratio) + print('folds:',folds) + + accs = [] + + for seed in tqdm(range(args.repeat)): + set_seed(seed) + results = [] + for i,fold in enumerate(folds): + f1,best_attentions=train_till_end(fold,device,i, args,metadata) + results.append(f1) + + accs.append(np.mean(results)) + + + + print('{:.4f} ~ {:.4f}'.format(np.mean(accs), np.std(accs)))