From 95ff433bb3958bc55dd012753b489e971ff4267a Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 20 Oct 2023 08:35:46 +0800 Subject: [PATCH 1/6] Add hpinns hydra --- docs/zh/examples/hpinns.md | 76 ++++++---- examples/hpinns/conf/hpinns.yaml | 63 ++++++++ examples/hpinns/holography.py | 237 +++++++++++++++++-------------- 3 files changed, 243 insertions(+), 133 deletions(-) create mode 100644 examples/hpinns/conf/hpinns.yaml diff --git a/docs/zh/examples/hpinns.md b/docs/zh/examples/hpinns.md index 2385b9942..3d3f10399 100644 --- a/docs/zh/examples/hpinns.md +++ b/docs/zh/examples/hpinns.md @@ -2,6 +2,30 @@ AI Studio快速体验 +=== "模型训练命令" + + ``` sh + # linux + wget -P ./datasets/ https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_train.mat + wget -P ./datasets/ https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_valid.mat + # windows + # curl https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_train.mat --output ./datasets/hpinns_holo_train.mat + # curl https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_valid.mat --output ./datasets/hpinns_holo_valid.mat + python holography.py + ``` + +=== "模型评估命令" + + ``` sh + # linux + wget -P ./datasets/ https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_train.mat + wget -P ./datasets/ https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_valid.mat + # windows + # curl https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_train.mat --output ./datasets/hpinns_holo_train.mat + # curl https://paddle-org.bj.bcebos.com/paddlescience/datasets/hPINNs/hpinns_holo_valid.mat --output ./datasets/hpinns_holo_valid.mat + python holography.py mode=eval EVAL.pretrained_model_path=https://paddle-org.bj.bcebos.com/paddlescience/models/hpinns/hpinns_pretrained.pdparams + ``` + ## 1. 背景简介 求解偏微分方程(PDE) 是一类基础的物理问题，在过去几十年里，以有限差分(FDM)、有限体积(FVM)、有限元(FEM)为代表的多种偏微分方程组数值解法趋于成熟。随着人工智能技术的高速发展，利用深度学习求解偏微分方程成为新的研究趋势。PINNs(Physics-informed neural networks) 是一种加入物理约束的深度学习网络，因此与纯数据驱动的神经网络学习相比，PINNs 可以用更少的数据样本学习到更具泛化能力的模型，其应用范围包括但不限于流体力学、热传导、电磁场、量子力学等领域。 @@ -84,9 +108,9 @@ $$ 上式中 $f_1,f_2,f_3$ 分别为一个 MLP 模型，三者共同构成了一个 Model List，用 PaddleScience 代码表示如下 -``` py linenums="43" +``` py linenums="42" --8<-- -examples/hpinns/holography.py:43:51 +examples/hpinns/holography.py:42:50 --8<-- ``` @@ -105,9 +129,9 @@ examples/hpinns/functions.py:49:92 需要对每个 MLP 模型分别注册相应的 transform ，然后将 3 个 MLP 模型组成 Model List -``` py linenums="59" +``` py linenums="57" --8<-- -examples/hpinns/holography.py:59:68 +examples/hpinns/holography.py:57:66 --8<-- ``` @@ -117,15 +141,15 @@ examples/hpinns/holography.py:59:68 我们需要指定问题相关的参数，如通过 `train_mode` 参数指定应用增强的拉格朗日方法的硬约束进行训练 -``` py linenums="31" +``` py linenums="35" --8<-- -examples/hpinns/holography.py:31:41 +examples/hpinns/holography.py:35:40 --8<-- ``` -``` py linenums="53" +``` py linenums="52" --8<-- -examples/hpinns/holography.py:53:57 +examples/hpinns/holography.py:52:55 --8<-- ``` @@ -141,15 +165,9 @@ $\mu_k = \beta \mu_{k-1}$, $\lambda_k = \beta \lambda_{k-1}$ 同时需要指定训练轮数和学习率等超参数 -``` py linenums="70" ---8<-- -examples/hpinns/holography.py:70:72 ---8<-- -``` - -``` py linenums="212" +``` py linenums="53" --8<-- -examples/hpinns/holography.py:212:214 +examples/hpinns/conf/hpinns.yaml:53:57 --8<-- ``` @@ -159,13 +177,13 @@ examples/hpinns/holography.py:212:214 ``` py linenums="74" --8<-- -examples/hpinns/holography.py:74:75 +examples/hpinns/holography.py:68:71 --8<-- ``` -``` py linenums="216" +``` py linenums="212" --8<-- -examples/hpinns/holography.py:216:219 +examples/hpinns/holography.py:212:215 --8<-- ``` @@ -175,9 +193,9 @@ examples/hpinns/holography.py:216:219 虽然我们不是以监督学习方式进行训练，但此处仍然可以采用监督约束 `SupervisedConstraint`，在定义约束之前，需要给监督约束指定文件路径等数据读取配置，因为数据集中没有标签数据，因此在数据读取时我们需要使用训练数据充当标签数据，并注意在之后不要使用这部分“假的”标签数据。 -``` py linenums="113" +``` py linenums="109" --8<-- -examples/hpinns/holography.py:113:118 +examples/hpinns/holography.py:109:114 --8<-- ``` @@ -185,9 +203,9 @@ examples/hpinns/holography.py:113:118 下面是约束等具体内容，要注意上述提到的给定“假的”标签数据： -``` py linenums="77" +``` py linenums="73" --8<-- -examples/hpinns/holography.py:77:138 +examples/hpinns/holography.py:73:134 --8<-- ``` @@ -207,9 +225,9 @@ examples/hpinns/holography.py:77:138 在约束构建完毕之后，以我们刚才的命名为关键字，封装到一个字典中，方便后续访问。 -``` py linenums="139" +``` py linenums="135" --8<-- -examples/hpinns/holography.py:139:142 +examples/hpinns/holography.py:135:138 --8<-- ``` @@ -217,9 +235,9 @@ examples/hpinns/holography.py:139:142 与约束同理，虽然本问题使用无监督学习，但仍可以使用 `ppsci.validate.SupervisedValidator` 构建评估器。 -``` py linenums="144" +``` py linenums="140" --8<-- -examples/hpinns/holography.py:144:192 +examples/hpinns/holography.py:140:188 --8<-- ``` @@ -249,9 +267,9 @@ examples/hpinns/functions.py:320:336 完成上述设置之后，只需要将上述实例化的对象按顺序传递给 `ppsci.solver.Solver`，然后启动训练、评估。 -``` py linenums="194" +``` py linenums="191" --8<-- -examples/hpinns/holography.py:194:210 +examples/hpinns/holography.py:191:208 --8<-- ``` diff --git a/examples/hpinns/conf/hpinns.yaml b/examples/hpinns/conf/hpinns.yaml new file mode 100644 index 000000000..c3a7bad03 --- /dev/null +++ b/examples/hpinns/conf/hpinns.yaml @@ -0,0 +1,63 @@ +hydra: + run: + # dynamic output directory according to running time and override name + dir: outputs_hpinns/${now:%Y-%m-%d}/${now:%H-%M-%S}/${hydra.job.override_dirname} + job: + name: ${mode} # name of logfile + chdir: false # keep current working direcotry unchanged + config: + override_dirname: + exclude_keys: + - TRAIN.checkpoint_path + - TRAIN.pretrained_model_path + - EVAL.pretrained_model_path + - mode + - output_dir + - log_freq + sweep: + # output directory for multirun + dir: ${hydra.run.dir} + subdir: ./ + +# general settings +mode: train # running mode: train/eval +seed: 42 +output_dir: ${hydra:run.dir} +DATASET_PATH: ./datasets/hpinns_holo_train.mat +DATASET_PATH_VALID: ./datasets/hpinns_holo_valid.mat + +# set working condition +TRAIN_MODE: aug_lag # "soft", "penalty", "aug_lag" +TRAIN_K: 9 + +# model settings +MODEL: + re_net: + output_keys: ["e_re"] + num_layers: 4 + hidden_size: 48 + activation: "tanh" + im_net: + output_keys: ["e_im"] + num_layers: 4 + hidden_size: 48 + activation: "tanh" + eps_net: + output_keys: ["eps"] + num_layers: 4 + hidden_size: 48 + activation: "tanh" + +# training settings +TRAIN: + epochs: 20000 + iters_per_epoch: 1 + eval_during_train: false + learning_rate: 0.001 + max_iter: 15000 + pretrained_model_path: null + checkpoint_path: null + +# evaluation settings +EVAL: + pretrained_model_path: null diff --git a/examples/hpinns/holography.py b/examples/hpinns/holography.py index f7dcc2f1b..a0cf81d53 100644 --- a/examples/hpinns/holography.py +++ b/examples/hpinns/holography.py @@ -16,29 +16,28 @@ This module is heavily adapted from https://github.com/lululxvi/hpinn """ +from os import path as osp + import functions as func_module +import hydra import numpy as np import paddle import plotting as plot_module +from omegaconf import DictConfig import ppsci from ppsci.autodiff import hessian from ppsci.autodiff import jacobian -from ppsci.utils import config from ppsci.utils import logger -if __name__ == "__main__": + +def train_or_evaluate(cfg: DictConfig): # open FLAG for higher order differential operator paddle.framework.core.set_prim_eager_enabled(True) - args = config.parse_args() - ppsci.utils.misc.set_random_seed(42) - DATASET_PATH = "./datasets/hpinns_holo_train.mat" - DATASET_PATH_VALID = "./datasets/hpinns_holo_valid.mat" - OUTPUT_DIR = "./output_hpinns/" if args.output_dir is None else args.output_dir - + ppsci.utils.misc.set_random_seed(cfg.seed) # initialize logger - logger.init_logger("ppsci", f"{OUTPUT_DIR}/train.log", "info") + logger.init_logger("ppsci", osp.join(cfg.output_dir, f"{cfg.mode}.log"), "info") # initialize models in_keys = () @@ -46,14 +45,13 @@ in_keys += (f"x_cos_{t}", f"x_sin_{t}") in_keys += ("y", "y_cos_1", "y_sin_1") - model_re = ppsci.arch.MLP(in_keys, ("e_re",), 4, 48, "tanh") - model_im = ppsci.arch.MLP(in_keys, ("e_im",), 4, 48, "tanh") - model_eps = ppsci.arch.MLP(in_keys, ("eps",), 4, 48, "tanh") + model_re = ppsci.arch.MLP(in_keys, **cfg.MODEL.re_net) + model_im = ppsci.arch.MLP(in_keys, **cfg.MODEL.im_net) + model_eps = ppsci.arch.MLP(in_keys, **cfg.MODEL.eps_net) # intialize params - train_mode = "aug_lag" # "soft", "penalty", "aug_lag" - k = 9 - func_module.train_mode = train_mode + k = cfg.TRAIN_K + func_module.train_mode = cfg.TRAIN_MODE loss_log_obj = [] # register transform @@ -67,12 +65,10 @@ model_list = ppsci.arch.ModelList((model_re, model_im, model_eps)) - # set training hyper-parameters - ITERS_PER_EPOCH = 1 - EPOCHS = 20000 if args.epochs is None else args.epochs - # initialize Adam optimizer - optimizer_adam = ppsci.optimizer.Adam(1e-3)((model_re, model_im, model_eps)) + optimizer_adam = ppsci.optimizer.Adam(cfg.TRAIN.learning_rate)( + (model_re, model_im, model_eps) + ) # manually build constraint(s) label_keys = ("x", "y", "bound", "e_real", "e_imaginary", "epsilon") @@ -107,7 +103,7 @@ { "dataset": { "name": "IterableMatDataset", - "file_path": DATASET_PATH, + "file_path": cfg.DATASET_PATH, "input_keys": ("x", "y", "bound"), "label_keys": label_keys + label_keys_derivative, "alias_dict": { @@ -126,7 +122,7 @@ { "dataset": { "name": "IterableMatDataset", - "file_path": DATASET_PATH, + "file_path": cfg.DATASET_PATH, "input_keys": ("x", "y", "bound"), "label_keys": label_keys, "alias_dict": {"e_real": "x", "e_imaginary": "x", "epsilon": "x"}, @@ -146,7 +142,7 @@ { "dataset": { "name": "IterableMatDataset", - "file_path": DATASET_PATH_VALID, + "file_path": cfg.DATASET_PATH_VALID, "input_keys": ("x", "y", "bound"), "label_keys": label_keys + label_keys_derivative, "alias_dict": { @@ -168,7 +164,7 @@ { "dataset": { "name": "IterableMatDataset", - "file_path": DATASET_PATH_VALID, + "file_path": cfg.DATASET_PATH_VALID, "input_keys": ("x", "y", "bound"), "label_keys": label_keys + label_keys_derivative, "alias_dict": { @@ -191,45 +187,19 @@ sup_validator_val.name: sup_validator_val, } - # initialize solver - solver = ppsci.solver.Solver( - model_list, - constraint, - OUTPUT_DIR, - optimizer_adam, - None, - EPOCHS, - ITERS_PER_EPOCH, - eval_during_train=False, - validator=validator, - ) - - # train model - solver.train() - # evaluate after finished training - solver.eval() - - # set training hyper-parameters - EPOCHS_LBFGS = 1 - MAX_ITER = 15000 - - # initialize LBFGS optimizer - optimizer_lbfgs = ppsci.optimizer.LBFGS(max_iter=MAX_ITER)( - (model_re, model_im, model_eps) - ) - - # train: soft constraint, epoch=1 for lbfgs - if train_mode == "soft": + if cfg.mode == "train": + # initialize solver solver = ppsci.solver.Solver( model_list, constraint, - OUTPUT_DIR, - optimizer_lbfgs, + cfg.output_dir, + optimizer_adam, None, - EPOCHS_LBFGS, - ITERS_PER_EPOCH, - eval_during_train=False, + cfg.TRAIN.epochs, + cfg.TRAIN.iters_per_epoch, + eval_during_train=cfg.TRAIN.eval_during_train, validator=validator, + checkpoint_path=cfg.TRAIN.checkpoint_path, ) # train model @@ -237,64 +207,123 @@ # evaluate after finished training solver.eval() - # append objective loss for plot - loss_log_obj.append(func_module.loss_obj) - - # penalty and augmented Lagrangian, difference between the two is updating of lambda - if train_mode != "soft": - train_dict = ppsci.utils.reader.load_mat_file(DATASET_PATH, ("x", "y", "bound")) - in_dict = {"x": train_dict["x"], "y": train_dict["y"]} - expr_dict = output_expr.copy() - expr_dict.pop("bound") - - func_module.init_lambda(in_dict, int(train_dict["bound"])) - func_module.lambda_log.append( - [ - func_module.lambda_re.copy().squeeze(), - func_module.lambda_im.copy().squeeze(), - ] + # set training hyper-parameters + EPOCHS_LBFGS = 1 + # initialize LBFGS optimizer + optimizer_lbfgs = ppsci.optimizer.LBFGS(max_iter=cfg.TRAIN.max_iter)( + (model_re, model_im, model_eps) ) - for i in range(1, k + 1): - pred_dict = solver.predict( - in_dict, - expr_dict, - batch_size=np.shape(train_dict["x"])[0], - no_grad=False, - ) - func_module.update_lambda(pred_dict, int(train_dict["bound"])) - - func_module.update_mu() - logger.message(f"Iteration {i}: mu = {func_module.mu}\n") - + # train: soft constraint, epoch=1 for lbfgs + if cfg.TRAIN_MODE == "soft": solver = ppsci.solver.Solver( model_list, constraint, - OUTPUT_DIR, + cfg.output_dir, optimizer_lbfgs, None, EPOCHS_LBFGS, - ITERS_PER_EPOCH, - eval_during_train=False, + cfg.TRAIN.iters_per_epoch, + eval_during_train=cfg.TRAIN.eval_during_train, validator=validator, + checkpoint_path=cfg.TRAIN.checkpoint_path, ) # train model solver.train() # evaluate after finished training solver.eval() - # append objective loss for plot - loss_log_obj.append(func_module.loss_obj) - - ################# plotting ################### - # log of loss - loss_log = np.array(func_module.loss_log).reshape(-1, 3) - - plot_module.set_params(train_mode, OUTPUT_DIR, DATASET_PATH, DATASET_PATH_VALID) - plot_module.plot_6a(loss_log) - if train_mode != "soft": - plot_module.prepare_data(solver, expr_dict) - plot_module.plot_6b(loss_log_obj) - plot_module.plot_6c7c(func_module.lambda_log) - plot_module.plot_6d(func_module.lambda_log) - plot_module.plot_6ef(func_module.lambda_log) + + # append objective loss for plot + loss_log_obj.append(func_module.loss_obj) + + # penalty and augmented Lagrangian, difference between the two is updating of lambda + if cfg.TRAIN_MODE != "soft": + train_dict = ppsci.utils.reader.load_mat_file( + cfg.DATASET_PATH, ("x", "y", "bound") + ) + in_dict = {"x": train_dict["x"], "y": train_dict["y"]} + expr_dict = output_expr.copy() + expr_dict.pop("bound") + + func_module.init_lambda(in_dict, int(train_dict["bound"])) + func_module.lambda_log.append( + [ + func_module.lambda_re.copy().squeeze(), + func_module.lambda_im.copy().squeeze(), + ] + ) + + for i in range(1, k + 1): + pred_dict = solver.predict( + in_dict, + expr_dict, + batch_size=np.shape(train_dict["x"])[0], + no_grad=False, + ) + func_module.update_lambda(pred_dict, int(train_dict["bound"])) + + func_module.update_mu() + logger.message(f"Iteration {i}: mu = {func_module.mu}\n") + + solver = ppsci.solver.Solver( + model_list, + constraint, + cfg.output_dir, + optimizer_lbfgs, + None, + EPOCHS_LBFGS, + cfg.TRAIN.iters_per_epoch, + eval_during_train=cfg.TRAIN.eval_during_train, + validator=validator, + checkpoint_path=cfg.TRAIN.checkpoint_path, + ) + + # train model + solver.train() + # evaluate after finished training + solver.eval() + # append objective loss for plot + loss_log_obj.append(func_module.loss_obj) + + ################# plotting ################### + # log of loss + loss_log = np.array(func_module.loss_log).reshape(-1, 3) + + plot_module.set_params( + cfg.TRAIN_MODE, cfg.output_dir, cfg.DATASET_PATH, cfg.DATASET_PATH_VALID + ) + plot_module.plot_6a(loss_log) + if cfg.TRAIN_MODE != "soft": + plot_module.prepare_data(solver, expr_dict) + plot_module.plot_6b(loss_log_obj) + plot_module.plot_6c7c(func_module.lambda_log) + plot_module.plot_6d(func_module.lambda_log) + plot_module.plot_6ef(func_module.lambda_log) + elif cfg.mode == "eval": + solver = ppsci.solver.Solver( + model_list, + constraint, + cfg.output_dir, + optimizer_adam, + None, + validator=validator, + pretrained_model_path=cfg.EVAL.pretrained_model_path, + ) + + # train model + solver.train() + # evaluate after finished training + solver.eval() + + +@hydra.main(version_base=None, config_path="./conf", config_name="hpinns.yaml") +def main(cfg: DictConfig): + if cfg.mode == "train" or cfg.mode == "eval": + train_or_evaluate(cfg) + else: + raise ValueError(f"cfg.mode should in ['train', 'eval'], but got '{cfg.mode}'") + + +if __name__ == "__main__": + main() From 75608d90a740f93d5d1c7f9084efcdeac602d11e Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 20 Oct 2023 08:46:36 +0800 Subject: [PATCH 2/6] Fix --- docs/zh/examples/hpinns.md | 8 ++++---- examples/hpinns/conf/hpinns.yaml | 1 + examples/hpinns/holography.py | 6 ++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/zh/examples/hpinns.md b/docs/zh/examples/hpinns.md index 3d3f10399..4ef8aabe9 100644 --- a/docs/zh/examples/hpinns.md +++ b/docs/zh/examples/hpinns.md @@ -181,9 +181,9 @@ examples/hpinns/holography.py:68:71 --8<-- ``` -``` py linenums="212" +``` py linenums="210" --8<-- -examples/hpinns/holography.py:212:215 +examples/hpinns/holography.py:210:213 --8<-- ``` @@ -279,9 +279,9 @@ examples/hpinns/holography.py:191:208 PaddleScience 中提供了可视化器，但由于本问题图片数量较多且较为复杂，代码中自定义了可视化函数，调用自定义函数即可实现可视化 -``` py linenums="289" +``` py linenums="287" --8<-- -examples/hpinns/holography.py:289: +examples/hpinns/holography.py:287: --8<-- ``` diff --git a/examples/hpinns/conf/hpinns.yaml b/examples/hpinns/conf/hpinns.yaml index c3a7bad03..dfa8476e8 100644 --- a/examples/hpinns/conf/hpinns.yaml +++ b/examples/hpinns/conf/hpinns.yaml @@ -55,6 +55,7 @@ TRAIN: eval_during_train: false learning_rate: 0.001 max_iter: 15000 + epochs_lbfgs: 1 pretrained_model_path: null checkpoint_path: null diff --git a/examples/hpinns/holography.py b/examples/hpinns/holography.py index a0cf81d53..d6a74d0bb 100644 --- a/examples/hpinns/holography.py +++ b/examples/hpinns/holography.py @@ -207,8 +207,6 @@ def train_or_evaluate(cfg: DictConfig): # evaluate after finished training solver.eval() - # set training hyper-parameters - EPOCHS_LBFGS = 1 # initialize LBFGS optimizer optimizer_lbfgs = ppsci.optimizer.LBFGS(max_iter=cfg.TRAIN.max_iter)( (model_re, model_im, model_eps) @@ -222,7 +220,7 @@ def train_or_evaluate(cfg: DictConfig): cfg.output_dir, optimizer_lbfgs, None, - EPOCHS_LBFGS, + cfg.TRAIN.epochs_lbfgs, cfg.TRAIN.iters_per_epoch, eval_during_train=cfg.TRAIN.eval_during_train, validator=validator, @@ -272,7 +270,7 @@ def train_or_evaluate(cfg: DictConfig): cfg.output_dir, optimizer_lbfgs, None, - EPOCHS_LBFGS, + cfg.TRAIN.epochs_lbfgs, cfg.TRAIN.iters_per_epoch, eval_during_train=cfg.TRAIN.eval_during_train, validator=validator, From fe16eb1adf37d09eb754f213c255067b811e2aba Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 20 Oct 2023 16:14:43 +0800 Subject: [PATCH 3/6] Fix --- examples/hpinns/holography.py | 289 +++++++++++++++++++++++----------- 1 file changed, 199 insertions(+), 90 deletions(-) diff --git a/examples/hpinns/holography.py b/examples/hpinns/holography.py index d6a74d0bb..8207a6c54 100644 --- a/examples/hpinns/holography.py +++ b/examples/hpinns/holography.py @@ -31,7 +31,7 @@ from ppsci.utils import logger -def train_or_evaluate(cfg: DictConfig): +def train(cfg: DictConfig): # open FLAG for higher order differential operator paddle.framework.core.set_prim_eager_enabled(True) @@ -187,15 +187,39 @@ def train_or_evaluate(cfg: DictConfig): sup_validator_val.name: sup_validator_val, } - if cfg.mode == "train": - # initialize solver + # initialize solver + solver = ppsci.solver.Solver( + model_list, + constraint, + cfg.output_dir, + optimizer_adam, + None, + cfg.TRAIN.epochs, + cfg.TRAIN.iters_per_epoch, + eval_during_train=cfg.TRAIN.eval_during_train, + validator=validator, + checkpoint_path=cfg.TRAIN.checkpoint_path, + ) + + # train model + solver.train() + # evaluate after finished training + solver.eval() + + # initialize LBFGS optimizer + optimizer_lbfgs = ppsci.optimizer.LBFGS(max_iter=cfg.TRAIN.max_iter)( + (model_re, model_im, model_eps) + ) + + # train: soft constraint, epoch=1 for lbfgs + if cfg.TRAIN_MODE == "soft": solver = ppsci.solver.Solver( model_list, constraint, cfg.output_dir, - optimizer_adam, + optimizer_lbfgs, None, - cfg.TRAIN.epochs, + cfg.TRAIN.epochs_lbfgs, cfg.TRAIN.iters_per_epoch, eval_during_train=cfg.TRAIN.eval_during_train, validator=validator, @@ -207,13 +231,38 @@ def train_or_evaluate(cfg: DictConfig): # evaluate after finished training solver.eval() - # initialize LBFGS optimizer - optimizer_lbfgs = ppsci.optimizer.LBFGS(max_iter=cfg.TRAIN.max_iter)( - (model_re, model_im, model_eps) + # append objective loss for plot + loss_log_obj.append(func_module.loss_obj) + + # penalty and augmented Lagrangian, difference between the two is updating of lambda + if cfg.TRAIN_MODE != "soft": + train_dict = ppsci.utils.reader.load_mat_file( + cfg.DATASET_PATH, ("x", "y", "bound") + ) + in_dict = {"x": train_dict["x"], "y": train_dict["y"]} + expr_dict = output_expr.copy() + expr_dict.pop("bound") + + func_module.init_lambda(in_dict, int(train_dict["bound"])) + func_module.lambda_log.append( + [ + func_module.lambda_re.copy().squeeze(), + func_module.lambda_im.copy().squeeze(), + ] ) - # train: soft constraint, epoch=1 for lbfgs - if cfg.TRAIN_MODE == "soft": + for i in range(1, k + 1): + pred_dict = solver.predict( + in_dict, + expr_dict, + batch_size=np.shape(train_dict["x"])[0], + no_grad=False, + ) + func_module.update_lambda(pred_dict, int(train_dict["bound"])) + + func_module.update_mu() + logger.message(f"Iteration {i}: mu = {func_module.mu}\n") + solver = ppsci.solver.Solver( model_list, constraint, @@ -231,94 +280,154 @@ def train_or_evaluate(cfg: DictConfig): solver.train() # evaluate after finished training solver.eval() + # append objective loss for plot + loss_log_obj.append(func_module.loss_obj) - # append objective loss for plot - loss_log_obj.append(func_module.loss_obj) + ################# plotting ################### + # log of loss + loss_log = np.array(func_module.loss_log).reshape(-1, 3) - # penalty and augmented Lagrangian, difference between the two is updating of lambda - if cfg.TRAIN_MODE != "soft": - train_dict = ppsci.utils.reader.load_mat_file( - cfg.DATASET_PATH, ("x", "y", "bound") - ) - in_dict = {"x": train_dict["x"], "y": train_dict["y"]} - expr_dict = output_expr.copy() - expr_dict.pop("bound") - - func_module.init_lambda(in_dict, int(train_dict["bound"])) - func_module.lambda_log.append( - [ - func_module.lambda_re.copy().squeeze(), - func_module.lambda_im.copy().squeeze(), - ] - ) + plot_module.set_params( + cfg.TRAIN_MODE, cfg.output_dir, cfg.DATASET_PATH, cfg.DATASET_PATH_VALID + ) + plot_module.plot_6a(loss_log) + if cfg.TRAIN_MODE != "soft": + plot_module.prepare_data(solver, expr_dict) + plot_module.plot_6b(loss_log_obj) + plot_module.plot_6c7c(func_module.lambda_log) + plot_module.plot_6d(func_module.lambda_log) + plot_module.plot_6ef(func_module.lambda_log) - for i in range(1, k + 1): - pred_dict = solver.predict( - in_dict, - expr_dict, - batch_size=np.shape(train_dict["x"])[0], - no_grad=False, - ) - func_module.update_lambda(pred_dict, int(train_dict["bound"])) - - func_module.update_mu() - logger.message(f"Iteration {i}: mu = {func_module.mu}\n") - - solver = ppsci.solver.Solver( - model_list, - constraint, - cfg.output_dir, - optimizer_lbfgs, - None, - cfg.TRAIN.epochs_lbfgs, - cfg.TRAIN.iters_per_epoch, - eval_during_train=cfg.TRAIN.eval_during_train, - validator=validator, - checkpoint_path=cfg.TRAIN.checkpoint_path, - ) - - # train model - solver.train() - # evaluate after finished training - solver.eval() - # append objective loss for plot - loss_log_obj.append(func_module.loss_obj) - - ################# plotting ################### - # log of loss - loss_log = np.array(func_module.loss_log).reshape(-1, 3) - - plot_module.set_params( - cfg.TRAIN_MODE, cfg.output_dir, cfg.DATASET_PATH, cfg.DATASET_PATH_VALID - ) - plot_module.plot_6a(loss_log) - if cfg.TRAIN_MODE != "soft": - plot_module.prepare_data(solver, expr_dict) - plot_module.plot_6b(loss_log_obj) - plot_module.plot_6c7c(func_module.lambda_log) - plot_module.plot_6d(func_module.lambda_log) - plot_module.plot_6ef(func_module.lambda_log) - elif cfg.mode == "eval": - solver = ppsci.solver.Solver( - model_list, - constraint, - cfg.output_dir, - optimizer_adam, - None, - validator=validator, - pretrained_model_path=cfg.EVAL.pretrained_model_path, - ) - # train model - solver.train() - # evaluate after finished training - solver.eval() +def evaluate(cfg: DictConfig): + # open FLAG for higher order differential operator + paddle.framework.core.set_prim_eager_enabled(True) + + ppsci.utils.misc.set_random_seed(cfg.seed) + # initialize logger + logger.init_logger("ppsci", osp.join(cfg.output_dir, f"{cfg.mode}.log"), "info") + + # initialize models + in_keys = () + for t in range(1, 7): + in_keys += (f"x_cos_{t}", f"x_sin_{t}") + in_keys += ("y", "y_cos_1", "y_sin_1") + + model_re = ppsci.arch.MLP(in_keys, **cfg.MODEL.re_net) + model_im = ppsci.arch.MLP(in_keys, **cfg.MODEL.im_net) + model_eps = ppsci.arch.MLP(in_keys, **cfg.MODEL.eps_net) + + # intialize params + func_module.train_mode = cfg.TRAIN_MODE + + # register transform + model_re.register_input_transform(func_module.transform_in) + model_im.register_input_transform(func_module.transform_in) + model_eps.register_input_transform(func_module.transform_in) + + model_re.register_output_transform(func_module.transform_out_real_part) + model_im.register_output_transform(func_module.transform_out_imaginary_part) + model_eps.register_output_transform(func_module.transform_out_epsilon) + + model_list = ppsci.arch.ModelList((model_re, model_im, model_eps)) + + # manually build constraint(s) + label_keys = ("x", "y", "bound", "e_real", "e_imaginary", "epsilon") + label_keys_derivative = ( + "de_re_x", + "de_re_y", + "de_re_xx", + "de_re_yy", + "de_im_x", + "de_im_y", + "de_im_xx", + "de_im_yy", + ) + output_expr = { + "x": lambda out: out["x"], + "y": lambda out: out["y"], + "bound": lambda out: out["bound"], + "e_real": lambda out: out["e_real"], + "e_imaginary": lambda out: out["e_imaginary"], + "epsilon": lambda out: out["epsilon"], + "de_re_x": lambda out: jacobian(out["e_real"], out["x"]), + "de_re_y": lambda out: jacobian(out["e_real"], out["y"]), + "de_re_xx": lambda out: hessian(out["e_real"], out["x"]), + "de_re_yy": lambda out: hessian(out["e_real"], out["y"]), + "de_im_x": lambda out: jacobian(out["e_imaginary"], out["x"]), + "de_im_y": lambda out: jacobian(out["e_imaginary"], out["y"]), + "de_im_xx": lambda out: hessian(out["e_imaginary"], out["x"]), + "de_im_yy": lambda out: hessian(out["e_imaginary"], out["y"]), + } + + # manually build validator + sup_validator_opt = ppsci.validate.SupervisedValidator( + { + "dataset": { + "name": "IterableMatDataset", + "file_path": cfg.DATASET_PATH_VALID, + "input_keys": ("x", "y", "bound"), + "label_keys": label_keys + label_keys_derivative, + "alias_dict": { + "x": "x_opt", + "y": "y_opt", + "e_real": "x_opt", + "e_imaginary": "x_opt", + "epsilon": "x_opt", + **{k: "x_opt" for k in label_keys_derivative}, + }, + }, + }, + ppsci.loss.FunctionalLoss(func_module.eval_loss_fun), + output_expr, + {"mse": ppsci.metric.FunctionalMetric(func_module.eval_metric_fun)}, + name="opt_sup", + ) + sup_validator_val = ppsci.validate.SupervisedValidator( + { + "dataset": { + "name": "IterableMatDataset", + "file_path": cfg.DATASET_PATH_VALID, + "input_keys": ("x", "y", "bound"), + "label_keys": label_keys + label_keys_derivative, + "alias_dict": { + "x": "x_val", + "y": "y_val", + "e_real": "x_val", + "e_imaginary": "x_val", + "epsilon": "x_val", + **{k: "x_val" for k in label_keys_derivative}, + }, + }, + }, + ppsci.loss.FunctionalLoss(func_module.eval_loss_fun), + output_expr, + {"mse": ppsci.metric.FunctionalMetric(func_module.eval_metric_fun)}, + name="val_sup", + ) + validator = { + sup_validator_opt.name: sup_validator_opt, + sup_validator_val.name: sup_validator_val, + } + + solver = ppsci.solver.Solver( + model_list, + output_dir=cfg.output_dir, + seed=cfg.seed, + validator=validator, + pretrained_model_path=cfg.EVAL.pretrained_model_path, + ) + + # evaluate after finished training + solver.eval() @hydra.main(version_base=None, config_path="./conf", config_name="hpinns.yaml") def main(cfg: DictConfig): - if cfg.mode == "train" or cfg.mode == "eval": - train_or_evaluate(cfg) + if cfg.mode == "train": + train(cfg) + elif cfg.mode == "eval": + evaluate(cfg) else: raise ValueError(f"cfg.mode should in ['train', 'eval'], but got '{cfg.mode}'") From 1c8ae38b178b33169037207ba44afc9ccc6f42c9 Mon Sep 17 00:00:00 2001 From: co63oc Date: Fri, 20 Oct 2023 16:21:49 +0800 Subject: [PATCH 4/6] Fix --- docs/zh/examples/hpinns.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/zh/examples/hpinns.md b/docs/zh/examples/hpinns.md index 4ef8aabe9..8d360a5fa 100644 --- a/docs/zh/examples/hpinns.md +++ b/docs/zh/examples/hpinns.md @@ -267,9 +267,9 @@ examples/hpinns/functions.py:320:336 完成上述设置之后，只需要将上述实例化的对象按顺序传递给 `ppsci.solver.Solver`，然后启动训练、评估。 -``` py linenums="191" +``` py linenums="190" --8<-- -examples/hpinns/holography.py:191:208 +examples/hpinns/holography.py:190:207 --8<-- ``` @@ -279,9 +279,9 @@ examples/hpinns/holography.py:191:208 PaddleScience 中提供了可视化器，但由于本问题图片数量较多且较为复杂，代码中自定义了可视化函数，调用自定义函数即可实现可视化 -``` py linenums="287" +``` py linenums="286" --8<-- -examples/hpinns/holography.py:287: +examples/hpinns/holography.py:286:299 --8<-- ``` From db3c9783ea600a21c0fac1a202a5e67f673add55 Mon Sep 17 00:00:00 2001 From: co63oc Date: Wed, 25 Oct 2023 07:56:53 +0800 Subject: [PATCH 5/6] Fix --- docs/zh/examples/hpinns.md | 42 ++++++++++++++++---------------- examples/hpinns/conf/hpinns.yaml | 3 +++ examples/hpinns/holography.py | 27 ++++++-------------- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/docs/zh/examples/hpinns.md b/docs/zh/examples/hpinns.md index 8d360a5fa..b54eac3d8 100644 --- a/docs/zh/examples/hpinns.md +++ b/docs/zh/examples/hpinns.md @@ -110,7 +110,7 @@ $$ ``` py linenums="42" --8<-- -examples/hpinns/holography.py:42:50 +examples/hpinns/holography.py:42:44 --8<-- ``` @@ -129,9 +129,9 @@ examples/hpinns/functions.py:49:92 需要对每个 MLP 模型分别注册相应的 transform ，然后将 3 个 MLP 模型组成 Model List -``` py linenums="57" +``` py linenums="50" --8<-- -examples/hpinns/holography.py:57:66 +examples/hpinns/holography.py:50:59 --8<-- ``` @@ -147,9 +147,9 @@ examples/hpinns/holography.py:35:40 --8<-- ``` -``` py linenums="52" +``` py linenums="46" --8<-- -examples/hpinns/holography.py:52:55 +examples/hpinns/holography.py:46:48 --8<-- ``` @@ -175,15 +175,15 @@ examples/hpinns/conf/hpinns.yaml:53:57 训练分为两个阶段，先使用 Adam 优化器进行大致训练，再使用 LBFGS 优化器逼近最优点，因此需要两个优化器，这也对应了上一部分超参数中的两种 `EPOCHS` 值 -``` py linenums="74" +``` py linenums="62" --8<-- -examples/hpinns/holography.py:68:71 +examples/hpinns/holography.py:62:64 --8<-- ``` -``` py linenums="210" +``` py linenums="203" --8<-- -examples/hpinns/holography.py:210:213 +examples/hpinns/holography.py:203:205 --8<-- ``` @@ -193,9 +193,9 @@ examples/hpinns/holography.py:210:213 虽然我们不是以监督学习方式进行训练，但此处仍然可以采用监督约束 `SupervisedConstraint`，在定义约束之前，需要给监督约束指定文件路径等数据读取配置，因为数据集中没有标签数据，因此在数据读取时我们需要使用训练数据充当标签数据，并注意在之后不要使用这部分“假的”标签数据。 -``` py linenums="109" +``` py linenums="102" --8<-- -examples/hpinns/holography.py:109:114 +examples/hpinns/holography.py:102:107 --8<-- ``` @@ -203,9 +203,9 @@ examples/hpinns/holography.py:109:114 下面是约束等具体内容，要注意上述提到的给定“假的”标签数据： -``` py linenums="73" +``` py linenums="66" --8<-- -examples/hpinns/holography.py:73:134 +examples/hpinns/holography.py:66:127 --8<-- ``` @@ -225,9 +225,9 @@ examples/hpinns/holography.py:73:134 在约束构建完毕之后，以我们刚才的命名为关键字，封装到一个字典中，方便后续访问。 -``` py linenums="135" +``` py linenums="128" --8<-- -examples/hpinns/holography.py:135:138 +examples/hpinns/holography.py:128:131 --8<-- ``` @@ -235,9 +235,9 @@ examples/hpinns/holography.py:135:138 与约束同理，虽然本问题使用无监督学习，但仍可以使用 `ppsci.validate.SupervisedValidator` 构建评估器。 -``` py linenums="140" +``` py linenums="133" --8<-- -examples/hpinns/holography.py:140:188 +examples/hpinns/holography.py:133:181 --8<-- ``` @@ -267,9 +267,9 @@ examples/hpinns/functions.py:320:336 完成上述设置之后，只需要将上述实例化的对象按顺序传递给 `ppsci.solver.Solver`，然后启动训练、评估。 -``` py linenums="190" +``` py linenums="183" --8<-- -examples/hpinns/holography.py:190:207 +examples/hpinns/holography.py:183:200 --8<-- ``` @@ -279,9 +279,9 @@ examples/hpinns/holography.py:190:207 PaddleScience 中提供了可视化器，但由于本问题图片数量较多且较为复杂，代码中自定义了可视化函数，调用自定义函数即可实现可视化 -``` py linenums="286" +``` py linenums="279" --8<-- -examples/hpinns/holography.py:286:299 +examples/hpinns/holography.py:279:292 --8<-- ``` diff --git a/examples/hpinns/conf/hpinns.yaml b/examples/hpinns/conf/hpinns.yaml index dfa8476e8..96f600515 100644 --- a/examples/hpinns/conf/hpinns.yaml +++ b/examples/hpinns/conf/hpinns.yaml @@ -33,16 +33,19 @@ TRAIN_K: 9 # model settings MODEL: re_net: + input_keys: ['x_cos_1', 'x_sin_1', 'x_cos_2', 'x_sin_2', 'x_cos_3', 'x_sin_3', 'x_cos_4', 'x_sin_4', 'x_cos_5', 'x_sin_5', 'x_cos_6', 'x_sin_6', 'y', 'y_cos_1', 'y_sin_1'] output_keys: ["e_re"] num_layers: 4 hidden_size: 48 activation: "tanh" im_net: + input_keys: ${MODEL.re_net.input_keys} output_keys: ["e_im"] num_layers: 4 hidden_size: 48 activation: "tanh" eps_net: + input_keys: ${MODEL.re_net.input_keys} output_keys: ["eps"] num_layers: 4 hidden_size: 48 diff --git a/examples/hpinns/holography.py b/examples/hpinns/holography.py index 8207a6c54..fbb92b2b0 100644 --- a/examples/hpinns/holography.py +++ b/examples/hpinns/holography.py @@ -39,18 +39,11 @@ def train(cfg: DictConfig): # initialize logger logger.init_logger("ppsci", osp.join(cfg.output_dir, f"{cfg.mode}.log"), "info") - # initialize models - in_keys = () - for t in range(1, 7): - in_keys += (f"x_cos_{t}", f"x_sin_{t}") - in_keys += ("y", "y_cos_1", "y_sin_1") - - model_re = ppsci.arch.MLP(in_keys, **cfg.MODEL.re_net) - model_im = ppsci.arch.MLP(in_keys, **cfg.MODEL.im_net) - model_eps = ppsci.arch.MLP(in_keys, **cfg.MODEL.eps_net) + model_re = ppsci.arch.MLP(**cfg.MODEL.re_net) + model_im = ppsci.arch.MLP(**cfg.MODEL.im_net) + model_eps = ppsci.arch.MLP(**cfg.MODEL.eps_net) # intialize params - k = cfg.TRAIN_K func_module.train_mode = cfg.TRAIN_MODE loss_log_obj = [] @@ -251,7 +244,7 @@ def train(cfg: DictConfig): ] ) - for i in range(1, k + 1): + for i in range(1, cfg.TRAIN_K + 1): pred_dict = solver.predict( in_dict, expr_dict, @@ -307,15 +300,9 @@ def evaluate(cfg: DictConfig): # initialize logger logger.init_logger("ppsci", osp.join(cfg.output_dir, f"{cfg.mode}.log"), "info") - # initialize models - in_keys = () - for t in range(1, 7): - in_keys += (f"x_cos_{t}", f"x_sin_{t}") - in_keys += ("y", "y_cos_1", "y_sin_1") - - model_re = ppsci.arch.MLP(in_keys, **cfg.MODEL.re_net) - model_im = ppsci.arch.MLP(in_keys, **cfg.MODEL.im_net) - model_eps = ppsci.arch.MLP(in_keys, **cfg.MODEL.eps_net) + model_re = ppsci.arch.MLP(**cfg.MODEL.re_net) + model_im = ppsci.arch.MLP(**cfg.MODEL.im_net) + model_eps = ppsci.arch.MLP(**cfg.MODEL.eps_net) # intialize params func_module.train_mode = cfg.TRAIN_MODE From 4de81e280ef897071924df0dd8a9e370ea5fdfdc Mon Sep 17 00:00:00 2001 From: co63oc Date: Wed, 25 Oct 2023 15:03:01 +0800 Subject: [PATCH 6/6] Fix --- docs/zh/examples/hpinns.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/examples/hpinns.md b/docs/zh/examples/hpinns.md index b54eac3d8..f0b1372a8 100644 --- a/docs/zh/examples/hpinns.md +++ b/docs/zh/examples/hpinns.md @@ -167,7 +167,7 @@ $\mu_k = \beta \mu_{k-1}$, $\lambda_k = \beta \lambda_{k-1}$ ``` py linenums="53" --8<-- -examples/hpinns/conf/hpinns.yaml:53:57 +examples/hpinns/conf/hpinns.yaml:53:61 --8<-- ```