From 6343d4ab91b5e1ab77161bf9c3fa7ca9d3c6e369 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 23 Sep 2016 15:14:29 +0800 Subject: [PATCH 1/2] Complete unittest of TrainerConfigHelper * Refine img_conv_op * Add conv_shift layer * Add tensor layer, refine selective fc * Try fix some styles * simple rnn layers * Add image layers * util_layers * Use collection.Sequence instead of list|tuple * BugFix for print_layer * Unitest for poolings. * Add test_grumemory_layer * add last_first_seq to unittest * Add test expand layer, test ntm layers * add *args for outputs * Add trans layer * Add test for hsigmoid * also add interface to set param attr * sampling_id, linear_comb * ctc_layer, crf_layer * cost_layers * beam_search --- python/paddle/trainer/config_parser.py | 12 +- .../trainer_config_helpers/activations.py | 9 +- .../paddle/trainer_config_helpers/layers.py | 585 +++++++++++------- .../paddle/trainer_config_helpers/networks.py | 9 +- .../paddle/trainer_config_helpers/poolings.py | 7 +- .../tests/CMakeLists.txt | 5 + .../tests/configs/.gitignore | 1 + .../tests/configs/beam_search.py | 30 + .../tests/configs/check.md5 | 18 + .../tests/configs/generate_protostr.sh | 17 + .../tests/configs/img_layers.py | 20 + .../tests/configs/last_first_seq.py | 26 + .../tests/configs/layer_activations.py | 21 + .../tests/configs/projections.py | 47 ++ .../tests/configs/run_tests.sh | 5 + .../tests/configs/simple_rnn_layers.py | 36 ++ .../tests/configs/test_cost_layers.py | 26 + .../tests/configs/test_expand_layer.py | 14 + .../tests/configs/test_fc.py | 20 + .../tests/configs/test_grumemory_layer.py | 11 + .../tests/configs/test_hsigmoid.py | 11 + .../tests/configs/test_lstmemory_layer.py | 11 + .../tests/configs/test_ntm_layers.py | 23 + .../tests/configs/test_print_layer.py | 12 + .../tests/configs/test_rnn_group.py | 35 ++ .../tests/configs/test_sequence_pooling.py | 30 + .../tests/configs/unused_layers.py | 14 + .../tests/configs/util_layers.py | 15 + 28 files changed, 834 insertions(+), 236 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/.gitignore create mode 100644 python/paddle/trainer_config_helpers/tests/configs/beam_search.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/check.md5 create mode 100755 python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh create mode 100644 python/paddle/trainer_config_helpers/tests/configs/img_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/layer_activations.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/projections.py create mode 100755 python/paddle/trainer_config_helpers/tests/configs/run_tests.sh create mode 100644 python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_fc.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/unused_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/util_layers.py diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index a57e9065c6f98..1f55298f24f07 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1279,7 +1279,7 @@ def create_input_parameter( size, dims=None, sparse = None, - format = "csr"): + format = None): if dims is None: # TODO(yuyang18): print warning and callstack here! dims = list() @@ -2074,7 +2074,7 @@ def __init__( active_type='linear', device=None, bias=False, - output_max_index=False): + output_max_index=None): super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') self.config.trans_type = trans_type @@ -2083,7 +2083,8 @@ def __init__( input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) - self.config.output_max_index=output_max_index + if output_max_index is not None: + self.config.output_max_index = output_max_index @config_layer('maxid') @@ -2440,7 +2441,7 @@ def __init__( inputs, size=0, bias=True, - error_clipping_threshold=0.0, + error_clipping_threshold=None, **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( @@ -2510,7 +2511,8 @@ def __init__( self.create_bias_parameter(bias, self.config.size) - self.config.error_clipping_threshold = error_clipping_threshold + if error_clipping_threshold is not None: + self.config.error_clipping_threshold = error_clipping_threshold # like MixedLayer, but no bias parameter @config_func diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 85534675199e7..292014519374e 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -15,8 +15,10 @@ __all__ = ["TanhActivation", "SigmoidActivation", "SoftmaxActivation", "IdentityActivation", "LinearActivation", 'SequenceSoftmaxActivation', 'ExpActivation', - "ReluActivation", "BReluActivation", "SoftReluActivation", "STanhActivation", - "AbsActivation", "SquareActivation", "BaseActivation"] + "ReluActivation", "BReluActivation", "SoftReluActivation", + "STanhActivation", + "AbsActivation", "SquareActivation", + "BaseActivation"] class BaseActivation(object): @@ -36,6 +38,9 @@ def __init__(self, name, support_hppl): self.name = name self.support_hppl = support_hppl + def __repr__(self): + return self.name + class TanhActivation(BaseActivation): """ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index dabf8e2953054..7c575b9baaf0e 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -13,6 +13,7 @@ # limitations under the License. import functools +import collections from paddle.trainer.config_parser import * from .activations import LinearActivation, SigmoidActivation, TanhActivation, \ @@ -21,6 +22,7 @@ from .poolings import MaxPooling, AvgPooling, BasePoolingType from .attrs import * from .default_decorators import * + try: import cPickle as pickle except ImportError: @@ -51,7 +53,8 @@ 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', 'out_prod_layer', 'print_layer' + # 'block_expand_layer', # TODO(yuyang18): this layer is not correct + 'out_prod_layer', 'print_layer' ] @@ -165,11 +168,12 @@ class LayerOutput(object): :param activation: Layer Activation. :type activation: BaseActivation. :param parents: Layer's parents. - :type parents: list|tuple + :type parents: list|tuple|collection.Sequence """ def __init__(self, name, layer_type, parents=None, activation=None, - num_filters=None, img_norm_type=None, size=None, outputs=None): + num_filters=None, img_norm_type=None, size=None, outputs=None, + reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) assert LayerType.is_layer_type(layer_type) @@ -185,6 +189,7 @@ def __init__(self, name, layer_type, parents=None, activation=None, if outputs is None: outputs = ['default'] self.outputs = outputs + self.reverse = reverse def __repr__(self): """ @@ -204,25 +209,6 @@ def __str__(self): DEVICE = 'device' -def check_input(input): - """ - Check input is a LayerOutput or list of LayerOutput or tuple of LayerOutput - if is a LayerOutput, - - :param input: The input layer. Could be a list/tuple of input layer. - :type input: LayerOutput|list|tuple - :return: list of LayerOutput - :rtype: list of LayerOutput - """ - - if isinstance(input, LayerOutput): - return [input] - assert isinstance(input, list) - for inp in input: - assert isinstance(inp, LayerOutput) - return list(input) - - def layer_support(*attrs): attrs_list = list(attrs) attrs_list.append(DEVICE) @@ -292,6 +278,43 @@ def full_matrix_projection(input, size=0, param_attr=None): return proj +@wrap_param_attr_default() +def trans_full_matrix_projection(input, size=0, param_attr=None): + """ + Different from full_matrix_projection, this projection performs matrix + multiplication, using transpose of weight. + + .. math:: + out.row[i] += in.row[i] * w^\mathrm{T} + + :math:`w^\mathrm{T}` means transpose of weight. + The simply usage is: + + .. code-block:: python + + proj = trans_full_matrix_projection(input=layer, + size=100, + param_attr=ParamAttr( + name='_proj', + initial_mean=0.0, + initial_std=0.01)) + + :param input: input layer + :type input: LayerOutput + :param size: The parameter size. Means the width of parameter. + :type size: int + :param param_attr: Parameter config, None if use default. + :type param_attr: ParameterAttribute + :return: A TransposedFullMatrixProjection Object. + :rtype: TransposedFullMatrixProjection + """ + proj = TransposedFullMatrixProjection(input_layer_name=input.name, + size=size, + **param_attr.attr) + proj.origin = input + return proj + + @wrap_param_attr_default() def table_projection(input, size=0, param_attr=None): """ @@ -369,7 +392,7 @@ def identity_projection(input, offset=None): Note that both of two projections should not have any parameter. :param input: Input Layer. - :type input: LayerOutput. + :type input: LayerOutput :param offset: Offset, None if use default. :type offset: int :return: A IdentityProjection or IdentityOffsetProjection Object @@ -412,9 +435,10 @@ def dotmul_projection(input, param_attr=None): proj = DotMulProjection(input_layer_name=input.name, size=input.size, **param_attr.attr) - proj.origin = input + proj.origin = input return proj + def dotmul_operator(x, y, scale=1): """ DotMulOperator takes two inputs and performs element-wise multiplication: @@ -442,11 +466,15 @@ def dotmul_operator(x, y, scale=1): """ assert isinstance(x, LayerOutput) assert isinstance(y, LayerOutput) + if x.size is not None and y.size is not None: + assert x.size == y.size + op = DotMulOperator(input_layer_names=[x.name, y.name], scale=scale) op.origin = [x, y] return op + @wrap_bias_attr_default(['padding_attr']) def context_projection(input, context_len, context_start=None, padding_attr=False): @@ -615,7 +643,7 @@ def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, else: with mixed_layer(name=name, size=size, act=act, bias_attr=bias_attr, layer_attr=layer_attr) as m: - if isinstance(input, list) or isinstance(input, tuple): + if isinstance(input, collections.Sequence): for each in input: m += each else: @@ -725,23 +753,19 @@ def fc_layer(input, size, act=None, name=None, """ if isinstance(input, LayerOutput): input = [input] - assert not isinstance(param_attr, list) + assert not isinstance(param_attr, collections.Sequence) param_attr = [param_attr] else: - if isinstance(param_attr, list) or isinstance(param_attr, tuple): + if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] - assert isinstance(input, list) - - def __idx_to_input__(i): - attr = param_attr[i] - assert isinstance(attr, ParameterAttribute) - return Input(input[i].name, **attr.attr) + assert isinstance(input, collections.Sequence) Layer( - inputs=map(__idx_to_input__, range(len(input))), + inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( + input, param_attr)], name=name, type=LayerType.FC_LAYER, size=size, @@ -762,16 +786,20 @@ def print_layer(input, name=None): :type name: basestring :param input: The input layer. Could be a list/tuple of input layer. :type input: LayerOutput|list|tuple - :return: No return + :return: LayerOutput """ - input = check_input(input) + if isinstance(input, LayerOutput): + input = [input] + assert isinstance(input, collections.Sequence) # list or tuple + for each in input: + assert isinstance(each, LayerOutput) Layer( name=name, type=LayerType.PRINT_LAYER, inputs=[l.name for l in input], ) - LayerOutput(name, LayerType.PRINT_LAYER, input) + return LayerOutput(name, LayerType.PRINT_LAYER, input) @wrap_name_default("seq_pooling") @@ -810,8 +838,13 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, :rtype: LayerType """ extra_dict = dict() + # noinspection PyUnresolvedReferences if isinstance(pooling_type, AvgPooling): extra_dict['average_strategy'] = pooling_type.strategy + elif isinstance(pooling_type, MaxPooling) and \ + pooling_type.output_max_index is not None: + assert isinstance(pooling_type.output_max_index, bool) + extra_dict['output_max_index'] = pooling_type.output_max_index extra_dict.update(ExtraLayerAttribute.to_kwargs(layer_attr)) Layer( @@ -835,7 +868,7 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, @wrap_name_default("lstmemory") @layer_support(DROPOUT) def lstmemory(input, name=None, reverse=False, act=None, - gate_act=None, + gate_act=None, size=None, state_act=None, bias_attr=None, param_attr=None, layer_attr=None): """ @@ -900,6 +933,12 @@ def lstmemory(input, name=None, reverse=False, act=None, assert gate_act.support_hppl assert state_act.support_hppl assert act.support_hppl + assert input.size is not None and input.size % 4 == 0 + if size is not None: + logger.warning("NOTE: The lstmemory layer[%s]'s size is set by previous" + " input layer. The lstm size should be equal with input" + " layer size/4. The size which is set explicitly will" + " be ignored." % name) Layer(name=name, type=LayerType.LSTMEMORY, @@ -911,8 +950,9 @@ def lstmemory(input, name=None, reverse=False, act=None, inputs=[Input(input.name, **param_attr.attr)], **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.LSTMEMORY, [input], - size=input.size / 4 if input.size is not None else None) + return LayerOutput(name, LayerType.LSTMEMORY, [input], size=input.size / 4, + reverse=reverse) + @wrap_bias_attr_default() @wrap_param_attr_default() @@ -922,7 +962,7 @@ def lstmemory(input, name=None, reverse=False, act=None, @wrap_name_default("gru") @layer_support(DROPOUT) def grumemory(input, name=None, reverse=False, act=None, - gate_act=None, + gate_act=None, size=None, bias_attr=None, param_attr=None, layer_attr=None): """ @@ -980,7 +1020,7 @@ def grumemory(input, name=None, reverse=False, act=None, :type name: None|basestring :param input: input layer. :type input: LayerOutput. - :param reverse: Wether sequence process is reversed or not. + :param reverse: Whether sequence process is reversed or not. :type reverse: bool :param act: activation type, TanhActivation by default. This activation affects the :math:`{\\tilde{h_t}}`. @@ -996,12 +1036,19 @@ def grumemory(input, name=None, reverse=False, act=None, :type param_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer attribute :type layer_attr: ExtraLayerAttribute|None + :param size: Stub parameter of size, but actually not used. If set this size + will get a warning. + :type size: None :return: LayerOutput object. :rtype: LayerOutput """ - assert act.support_hppl assert gate_act.support_hppl + assert input.size is not None and input.size % 3 == 0 + if size is not None: + logger.warning("NOTE: the gru memory layer's size is set by previous " + "input layer, and should be input size / 3. Set size " + "explicitly will be ignored.") Layer(name=name, type=LayerType.GRUMEMORY, @@ -1013,8 +1060,9 @@ def grumemory(input, name=None, reverse=False, act=None, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.GRUMEMORY, [input], - size=input.size / 3 if input.size is not None else None) + return LayerOutput(name, LayerType.GRUMEMORY, [input], size=input.size / 3, + reverse=reverse) + @wrap_name_default() @layer_support() @@ -1033,6 +1081,12 @@ def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, :return: LayerOutput object. :rtype: LayerOutput """ + if input.reverse is not None and input.reverse: + logger.warning("You are getting the last instance of a sequence that" + " is a output of a REVERSED layer. There is no time" + " series information at all. Maybe you want to use" + " first_seq instead.") + Layer( name=name, type=LayerType.SEQUENCE_LAST_INSTANCE, @@ -1061,6 +1115,13 @@ def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, :return: LayerOutput object. :rtype: LayerOutput """ + + if input.reverse is not None and not input.reverse: + logger.warning('You are getting the first instance for a time series,' + ' and it is a normal recurrent layer output. There is no' + ' time series information at all. Maybe you want to use' + ' last_seq instead.') + Layer( name=name, type=LayerType.SEQUENCE_FIRST_INSTANCE, @@ -1076,6 +1137,7 @@ class ExpandLevel(object): FROM_TIMESTEP = AggregateLevel.EACH_TIMESTEP FROM_SEQUENCE = AggregateLevel.EACH_SEQUENCE + @wrap_name_default() @layer_support() def expand_layer(input, expand_as, @@ -1126,7 +1188,6 @@ def expand_layer(input, expand_as, parents=[input, expand_as]) - @wrap_name_default() @layer_support() def interpolation_layer(input, weight, name=None, layer_attr=None): @@ -1158,10 +1219,15 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) assert len(input) == 2 - assert input[0].size == input[1].size - assert weight.size == 1 + assert isinstance(input[0], LayerOutput) and isinstance(input[1], + LayerOutput) + if input[0].size is not None and input[1].size is not None: + assert input[0].size == input[1].size + assert isinstance(weight, LayerOutput) + if weight.size is not None: + assert weight.size == 1 Layer( name=name, type=LayerType.INTERPOLATION_LAYER, @@ -1203,11 +1269,13 @@ def power_layer(input, weight, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - assert weight.size == 1 + assert isinstance(input, LayerOutput) and isinstance(weight, LayerOutput) + if weight.size is not None: + assert weight.size == 1 Layer( name=name, type=LayerType.POWER_LAYER, - inputs=[input.name, weight.name], + inputs=[weight.name, input.name], **ExtraAttr.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.POWER_LAYER, @@ -1246,7 +1314,9 @@ def scaling_layer(input, weight, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - assert weight.size == 1 + assert isinstance(weight, LayerOutput) and isinstance(input, LayerOutput) + if weight.size is not None: + assert weight.size == 1 Layer( name=name, type=LayerType.SCALING_LAYER, @@ -1325,6 +1395,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ + assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput) if size == 1: Layer( name=name, @@ -1334,6 +1405,8 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): **ExtraLayerAttribute.to_kwargs(layer_attr) ) else: + if a.size is not None and b.size is not None: + assert size == b.size / a.size Layer( name=name, type=LayerType.COSINE_SIM_VEC, @@ -1344,11 +1417,13 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): ) return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b]) + @wrap_name_default() @wrap_bias_attr_default(has_bias=True) +@wrap_param_attr_default() @layer_support() def hsigmoid(input, label, num_classes, name=None, bias_attr=None, - layer_attr=None): + param_attr=None, layer_attr=None): """ Organize the classes into a binary tree. At each node, a sigmoid function is used to calculate the probability of belonging to the right branch. @@ -1382,15 +1457,23 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, """ if isinstance(input, LayerOutput): input = [input] - assert isinstance(input, list) or isinstance(input, tuple) + if not isinstance(param_attr, collections.Sequence): + param_attr = [param_attr] + else: + if not isinstance(param_attr, collections.Sequence): + param_attr = [param_attr] * len(input) + else: + assert len(param_attr) == len(input) + + assert isinstance(input, collections.Sequence) assert isinstance(label, LayerOutput) assert label.layer_type == LayerType.DATA ipts_for_layer = [] parents = [] - for each_input in input: + for each_input, each_param_attr in zip(input, param_attr): assert isinstance(each_input, LayerOutput) - ipts_for_layer.append(each_input.name) + ipts_for_layer.append(Input(each_input.name, **each_param_attr.attr)) parents.append(each_input) ipts_for_layer.append(label.name) parents.append(label) @@ -1405,6 +1488,7 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, ) return LayerOutput(name, LayerType.HSIGMOID, parents=parents) + @wrap_name_default("conv") @wrap_param_attr_default() @wrap_bias_attr_default() @@ -1438,23 +1522,26 @@ def img_conv_layer(input, filter_size, num_filters, :type name: basestring :param input: Layer Input. :type input: LayerOutput - :param filter_size: The x dimension of a filter kernel. - :type filter_size: int + :param filter_size: The x dimension of a filter kernel. Or input a tuple for + two image dimension. + :type filter_size: int|tuple|list :param filter_size_y: The y dimension of a filter kernel. Since PaddlePaddle currently supports rectangular filters, the filter's shape will be (filter_size, filter_size_y). - :type filter_size_y: int + :type filter_size_y: int|None :param num_filters: Each filter group's number of filter :param act: Activation type. Default is tanh :type act: BaseActivation :param groups: Group size of filters. :type groups: int - :param stride: The x dimension of the stride. - :type stride: int + :param stride: The x dimension of the stride. Or input a tuple for two image + dimension. + :type stride: int|tuple|list :param stride_y: The y dimension of the stride. :type stride_y: int - :param padding: The x dimension of the padding. - :type padding: int + :param padding: The x dimension of the padding. Or input a tuple for two + image dimension + :type padding: int|tuple|list :param padding_y: The y dimension of the padding. :type padding_y: int :param bias_attr: Convolution bias attribute. None means default bias. @@ -1475,13 +1562,30 @@ def img_conv_layer(input, filter_size, num_filters, if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters + if filter_size_y is None: - filter_size_y = filter_size + if isinstance(filter_size, collections.Sequence): + assert len(filter_size) == 2 + filter_size, filter_size_y = filter_size + else: + filter_size_y = filter_size + if stride_y is None: - stride_y = stride + if isinstance(stride, collections.Sequence): + assert len(stride) == 2 + stride, stride_y = stride + else: + stride_y = stride + if padding_y is None: - padding_y = padding - if param_attr.attr.get('initial_smart') == True: # special initial for conv layers. + if isinstance(padding, collections.Sequence): + assert len(padding) == 2 + padding, padding_y = padding + else: + padding_y = padding + + if param_attr.attr.get('initial_smart'): + # special initial for conv layers. init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w @@ -1492,8 +1596,9 @@ def img_conv_layer(input, filter_size, num_filters, inputs=Input(input.name, conv=Conv( filter_size=filter_size, padding=padding, stride=stride, channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, stride_y=stride_y), - **param_attr.attr), + filter_size_y=filter_size_y, padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), active_type=act.name, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), @@ -1553,7 +1658,7 @@ def img_pool_layer(input, pool_size, name=None, type=LayerType.POOL_LAYER, inputs=[Input(input.name, pool=Pool( - pool_type=pool_type.name + '-projection', + pool_type=''.join([pool_type.name, '-projection']), channels=num_channels, size_x=pool_size, start=start, @@ -1607,7 +1712,6 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, :type power: float :param num_channels: input layer's filers number or channels. If num_channels is None, it will be set automatically. - :param blocked: namely normalize in number of blocked feature maps. :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -1660,7 +1764,7 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, batch_norm for CPU. Otherwise, select batch norm type based on the specified type. If you use cudnn_batch_norm, we suggested you use latest version, such as v5.1. - :type type: None|string, None or "batch_norm" or "cudnn_batch_norm" + :type batch_norm_type: None|string, None or "batch_norm" or "cudnn_batch_norm" :param act: Activation Type. Better be relu. Because batch normalization will normalize input near zero. :type act: BaseActivation @@ -1821,7 +1925,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, if isinstance(input, LayerOutput): input = [input] - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) ipts_for_layer = [] for each_input in input: assert isinstance(each_input, LayerOutput) @@ -1835,7 +1939,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, active_type=act.name, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - assert isinstance(input, list) or isinstance(input, tuple) + return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, activation=act, num_filters=num_filters) @@ -1851,7 +1955,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None): :param name: Layer name. :type name: basestring :param input: input layers or projections - :type input: list|tuple + :type input: list|tuple|collection.Sequence :param act: Activation type. :type act: BaseActivation :param layer_attr: Extra Layer Attribute. @@ -1865,10 +1969,10 @@ def concat_layer(input, act=None, name=None, layer_attr=None): elif isinstance(input, Projection): input = [input] else: - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) def __is_type__(o, tp): - if not isinstance(o, list) and not isinstance(o, tuple): + if not isinstance(o, collections.Sequence): if o == tp: return True elif len(o.__bases__) == 0: @@ -2150,28 +2254,36 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): @wrap_param_attr_default() @layer_support() def recurrent_layer(input, act=None, bias_attr=None, - param_attr=None, name=None, layer_attr=None): + param_attr=None, name=None, reverse=False, layer_attr=None): """ - TODO(yuyang18): Add docs + Simple recurrent unit layer. It is just a fully connect layer through both + time and neural network. - :param input: - :param size: - :param act: - :param bias_attr: - :param param_attr: - :param name: - :param layer_attr: + :param input: Input Layer + :type input: LayerOutput + :param act: activation. + :type act: BaseActivation + :param bias_attr: bias attribute. + :type bias_attr: ParameterAttribute + :param param_attr: parameter attribute. + :type param_attr: ParameterAttribute + :param name: name of the layer + :type name: basestring + :param layer_attr: Layer Attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. + :rtype: LayerOutput """ Layer(name=name, type=LayerType.RECURRENT_LAYER, inputs=Input(input.name, **param_attr.attr), active_type=act.name, - size=input.size, bias=ParamAttr.to_bias(bias_attr), + reversed=reverse, **ExtraAttr.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.RECURRENT_LAYER, - parents=[input], size=input.size, activation=act) + parents=[input], size=input.size, activation=act, + reverse=reverse) class StaticInput(object): @@ -2179,6 +2291,7 @@ class StaticInput(object): StaticInput is only used in recurrent_group which defines a read-only memory that can be a sequence or non-sequence. """ + def __init__(self, input, is_seq=False, size=None): assert isinstance(input, LayerOutput) self.input = input @@ -2198,6 +2311,7 @@ class SubsequenceInput(object): input = SubsequenceInput(layer) """ + def __init__(self, input): assert isinstance(input, LayerOutput) assert input.size is not None @@ -2270,7 +2384,7 @@ def is_single_input(x): if is_single_input(input): input = [input] - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) def is_in_links(x): return isinstance(x, LayerOutput) or isinstance(x, SubsequenceInput) @@ -2314,6 +2428,7 @@ def map_in_links(x): for ot in layer_outs: assert isinstance(ot, LayerOutput) + ot.reverse = reverse if contains_sub_seq[0]: RecurrentLayerGroupSetOutLink(Link(ot.name, has_subseq=True)) else: @@ -2326,6 +2441,7 @@ def map_in_links(x): else: return layer_outs + class BaseGeneratedInput(object): def __init__(self): self.bos_id = None @@ -2354,6 +2470,7 @@ def before_real_step(self): return trg_emb def __init__(self, size, embedding_name, embedding_size): + super(GeneratedInput, self).__init__() self.size = size self.embedding_name = embedding_name self.embedding_size = embedding_size @@ -2390,6 +2507,7 @@ def maxid_layer(input, name=None, layer_attr=None): layer_type=LayerType.MAXID_LAYER, parents=[input]) + @wrap_name_default() def out_prod_layer(input1, input2, name=None, layer_attr=None): """ @@ -2422,7 +2540,8 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1,input2]) + parents=[input1, input2]) + @wrap_name_default() def eos_layer(input, eos_id, name=None, layer_attr=None): @@ -2475,14 +2594,14 @@ def beam_search(step, input, bos_id, eos_id, beam_size, def rnn_step(input): last_time_step_output = memory(name='rnn', size=512) - with mixed_layer(size=512) as simple_rnn: + with mixed_layer(size=512, name='rnn') as simple_rnn: simple_rnn += full_matrix_projection(input) simple_rnn += last_time_step_output return simple_rnn beam_gen = beam_search(name="decoder", step=rnn_step, - input=[StaticInput("encoder_last")], + input=[StaticInput(encoder_last)], bos_id=0, eos_id=1, beam_size=5, @@ -2496,18 +2615,18 @@ def rnn_step(input): :param name: Name of the recurrent unit that generates sequences. :type name: base string :param step: A callable function that defines the calculation in a time - step, and it is appled to sequences with arbitrary length by + step, and it is applied to sequences with arbitrary length by sharing a same set of weights. You can refer to the first parameter of recurrent_group, or demo/seqToseq/seqToseq_net.py for more details. :type step: callable :param input: Input data for the recurrent unit - :type input: StaticInput|GeneratedInput + :type input: list :param bos_id: Index of the start symbol in the dictionary. The start symbol is a special token for NLP task, which indicates the beginning of a sequence. In the generation task, the start - symbol is ensential, since it is used to initialize the RNN + symbol is essential, since it is used to initialize the RNN internal state. :type bos_id: int :param eos_id: Index of the end symbol in the dictionary. The end symbol is @@ -2516,6 +2635,8 @@ def rnn_step(input): symbol is generated, or a pre-defined max iteration number is exceeded. :type eos_id: int + :param max_length: Max generated sequence length. + :type max_length: int :param beam_size: Beam search for sequence generation is an iterative search algorithm. To maintain tractability, every iteration only only stores a predetermined number, called the beam_size, @@ -2556,8 +2677,8 @@ def rnn_step(input): real_input = [] for i, each_input in enumerate(input): # print type(each_input) - assert isinstance(each_input, StaticInput) or isinstance(each_input, - BaseGeneratedInput) + assert isinstance(each_input, StaticInput) or isinstance( + each_input, BaseGeneratedInput) if isinstance(each_input, BaseGeneratedInput): assert generated_input_index == -1 generated_input_index = i @@ -2667,7 +2788,7 @@ def __add_evaluator__(e): e(name=e.__name__, input=input, label=label) - if not isinstance(evaluator, list) and not isinstance(evaluator, tuple): + if not isinstance(evaluator, collections.Sequence): evaluator = [evaluator] for each_evaluator in evaluator: @@ -2675,8 +2796,9 @@ def __add_evaluator__(e): return LayerOutput(name, LayerType.COST, parents=[input, label]) + def conv_operator(img, filter, filter_size, num_filters, - num_channel=None, stride=1, padding=0, groups=1, + num_channel=None, stride=1, padding=0, filter_size_y=None, stride_y=None, padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used @@ -2690,7 +2812,7 @@ def conv_operator(img, filter, filter_size, num_filters, op = conv_operator(img=input1, filter=input2, - filter_size=3.0, + filter_size=3, num_filters=64, num_channels=64) @@ -2704,8 +2826,8 @@ def conv_operator(img, filter, filter_size, num_filters, PaddlePaddle now supports rectangular filters, the filter's shape can be (filter_size, filter_size_y). :type filter_size_y: int - :param num_filter: channel of output data. - :type num_filter: int + :param num_filters: channel of output data. + :type num_filters: int :param num_channel: channel of input data. :type num_channel: int :param stride: The x dimension of the stride. @@ -2725,8 +2847,16 @@ def conv_operator(img, filter, filter_size, num_filters, stride_y = stride if padding_y is None: padding_y = padding + + if num_channel is None: + num_channel = img.num_filters + + assert isinstance(filter, LayerOutput) + if filter.size is not None: + filter.size = filter_size * filter_size_y * num_filters * num_channel + op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters = num_filter, + num_filters=num_filters, conv_conf=Conv(filter_size=filter_size, padding=padding, stride=stride, @@ -2734,13 +2864,13 @@ def conv_operator(img, filter, filter_size, num_filters, filter_size_y=filter_size_y, padding_y=padding_y, stride_y=stride_y, - groups=groups)) + groups=1)) op.origin = [img, filter] return op @wrap_name_default() -def conv_shift_layer(input, name=None): +def conv_shift_layer(a, b, name=None): """ This layer performs cyclic convolution for two input. For example: - a[in]: contains M elements. @@ -2752,37 +2882,44 @@ def conv_shift_layer(input, name=None): c[i] = \sum_{j=-(N-1)/2}^{(N-1)/2}a_{i+j} * b_{j} In this formular: - - a's index is computed modulo M. - - b's index is computed modulo N. + - a's index is computed modulo M. When it is negative, then get item from + the right side (which is the end of array) to the left. + - b's index is computed modulo N. When it is negative, then get item from + the right size (which is the end of array) to the left. The example usage is: .. code-block:: python - conv_shift = conv_shif_layer(input=[layer1, layer2]) + conv_shift = conv_shift_layer(input=[layer1, layer2]) :param name: layer name :type name: basestring - :param input: Input layer. - :type input: LayerOutput|list|tuple. + :param a: Input layer a. + :type a: LayerOutput + :param b: input layer b + :type b: LayerOutput :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput) + assert b.size is None or b.size % 2 == 1 # size of b must be odd. Layer( name=name, type=LayerType.CONV_SHIFT_LAYER, - inputs=[x.name for x in input], + inputs=[a.name, b.name], ) - return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=input) + return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], + size=a.size) @wrap_name_default() @wrap_param_attr_default() @wrap_bias_attr_default() +@wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(input, size, act=None, name=None, +def tensor_layer(x1, x2, size, act=None, name=None, param_attr=None, bias_attr=None, layer_attr=None): """ This layer performs tensor operation for two input. @@ -2802,18 +2939,20 @@ def tensor_layer(input, size, act=None, name=None, .. code-block:: python - tensor = tensor_layer(input=[layer1, layer2]) + tensor = tensor_layer(x1=layer1, x2=layer2, size=1000) :param name: layer name :type name: basestring - :param input: Input layer. - :type input: LayerOutput|list|tuple. + :param x1: Input layer x_1. + :type x1: LayerOutput + :param x2: input layer x_2. + :type x2: LayerOutput :param size: the layer dimension. :type size: int. :param act: Activation Type. Default is tanh. :type act: BaseActivation :param param_attr: The Parameter Attribute. - :type param_attr: ParameterAttribute|list + :type param_attr: ParameterAttribute :param bias_attr: The Bias Attribute. If no bias, then pass False or something not type of ParameterAttribute. None will get a default Bias. @@ -2823,65 +2962,26 @@ def tensor_layer(input, size, act=None, name=None, :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) - assert len(input) == 2 + assert isinstance(x1, LayerOutput) and isinstance(x2, LayerOutput) Layer( name=name, size=size, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input[0].name, **param_attr.attr), - Input(input[1].name)], + inputs=[Input(x1.name, **param_attr.attr), + Input(x2.name)], **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=input, + return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[x1, x2], activation=act, size=size) -@wrap_param_attr_default() -def trans_full_matrix_projection(input, size=0, param_attr=None): - """ - Different from full_matrix_projection, this projection performs matrix - multiplication, using transpose of weight. - - .. math:: - out.row[i] += in.row[i] * w^\mathrm{T} - - :math:`w^\mathrm{T}` means transpose of weight. - The simply usage is: - - .. code-block:: python - - proj = trans_full_matrix_projection(input=layer, - size=100, - param_attr=ParamAttr( - name='_proj', - initial_mean=0.0, - initial_std=0.01)) - - :param input: input layer - :type input: LayerOutput - :param size: The parameter size. Means the width of parameter. - :type size: int - :param param_attr: Parameter config, None if use default. - :type param_attr: ParameterAttribute - :return: A TransposedFullMatrixProjection Object. - :rtype: TransposedFullMatrixProjection - """ - proj = TransposedFullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) - proj.origin = input - proj.origin.projection = "trans_matrix" - return proj - - @wrap_name_default() @wrap_param_attr_default() @wrap_bias_attr_default() @wrap_act_default() -def selective_fc_layer(input, size, act=None, name=None, +def selective_fc_layer(input, select, size, act=None, name=None, pass_generation=False, has_selected_colums=True, mul_ratio=0.02, @@ -2896,12 +2996,15 @@ def selective_fc_layer(input, size, act=None, name=None, .. code-block:: python - sel_fc = selective_fc_layer(input=input, 128, act=TanhActivation()) + sel_fc = selective_fc_layer(input=input, size=128, act=TanhActivation()) :param name: The Layer Name. :type name: basestring :param input: The input layer. :type input: LayerOutput|list|tuple + :param select: The select layer. The output of select layer should be a + sparse binary matrix, and treat as the mask of selective fc. + :type select: LayerOutput :param size: The layer dimension. :type size: int :param act: Activation Type. Default is tanh. @@ -2919,33 +3022,33 @@ def selective_fc_layer(input, size, act=None, name=None, """ if isinstance(input, LayerOutput): input = [input] - assert not isinstance(param_attr, list) + assert not isinstance(param_attr, collections.Sequence) param_attr = [param_attr] else: - if isinstance(param_attr, list) or isinstance(param_attr, tuple): + if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] - assert isinstance(input, list) - - def __idx_to_input__(i): - attr = param_attr[i] - assert isinstance(attr, ParameterAttribute) - return Input(input[i].name, **attr.attr) - + assert isinstance(input, collections.Sequence) + assert isinstance(select, LayerOutput) + if select.size is not None: + assert select.size == size Layer( - inputs=map(__idx_to_input__, range(len(input))), + inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( + input, param_attr)] + [select.name], name=name, type=LayerType.SEL_FC_LAYER, size=size, + bias=ParameterAttribute.to_bias(bias_attr), active_type=act.name, selective_fc_pass_generation=pass_generation, has_selected_colums=has_selected_colums, selective_fc_full_mul_ratio=mul_ratio, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.SEL_FC_LAYER, input, activation=act, + return LayerOutput(name, LayerType.SEL_FC_LAYER, list(input) + [select], + activation=act, size=size) @@ -3013,7 +3116,7 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): @wrap_name_default() -def linear_comb_layer(weights, vectors, size, name=None): +def linear_comb_layer(weights, vectors, size=None, name=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3043,11 +3146,13 @@ def linear_comb_layer(weights, vectors, size, name=None): .. code-block:: python - linear_comb = linear_comb_layer(weighs=weight, vectors=vectors, + linear_comb = linear_comb_layer(weights=weight, vectors=vectors, size=elem_dim) - :param input: The input layers. - :type input: LayerOutput + :param weights: The weight layer. + :type weights: LayerOutput + :param vectors: The vector layer. + :type vectors: LayerOutput :param size: the dimension of this layer. :type size: int :param name: The Layer Name. @@ -3055,7 +3160,13 @@ def linear_comb_layer(weights, vectors, size, name=None): :return: LayerOutput object. :rtype: LayerOutput """ - + assert isinstance(weights, LayerOutput) and isinstance(vectors, LayerOutput) + if vectors.size is not None and weights.size is not None: + assert vectors.size % weights.size == 0 + if size is None: + size = vectors.size / weights.size + else: + assert size == vectors.size / weights.size Layer( name=name, type=LayerType.LINEAR_COMBINATION_LAYER, @@ -3065,8 +3176,10 @@ def linear_comb_layer(weights, vectors, size, name=None): return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) + convex_comb_layer = linear_comb_layer + @wrap_name_default() def block_expand_layer(input, channel=0, @@ -3128,22 +3241,22 @@ def block_expand_layer(input, """ Layer(name=name, input=Input(input.name, - block_expand=BlockExpand(channel=channel, + block_expand=BlockExpand(channels=channel, block_x=block_x, block_y=block_y, stride_x=stride_x, stride_y=stride_y, padding_x=padding_x, padding_y=padding_y) - ), + ), type=LayerType.BLOCK_EXPAND, - ) + ) + + return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) - return LayerOutput(name, LayerType.BLOCK_EXPAND, - parents=[input], size=size) @wrap_name_default() -def ctc_layer(input, label, size, name=None, norm_by_times=False): +def ctc_layer(input, label, size=None, name=None, norm_by_times=False): """ Connectionist Temporal Classification (CTC) is designed for temporal classication task. That is, for sequence labeling problems where the @@ -3151,7 +3264,8 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): More details can be found by referring to `Connectionist Temporal Classification: Labelling Unsegmented Sequence Data with Recurrent - Neural Networks `_ + Neural Networks `_ Note: Considering the 'blank' label needed by CTC, you need to use @@ -3169,14 +3283,14 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): size=9055, norm_by_times=True) - :param input: The input layers. + :param input: The input layer. :type input: LayerOutput :param label: The data layer of label with variable length. :type label: LayerOutput :param size: category numbers + 1. :type size: int - :param name: The name of this layer, which can not specify. - :type name: string|None + :param name: The name of this layer + :type name: basestring|None :param norm_by_times: Whether to normalization by times. False by default. :type norm_by_times: bool :return: LayerOutput object. @@ -3184,18 +3298,24 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): """ assert isinstance(input, LayerOutput) assert isinstance(label, LayerOutput) + if label.size is not None: + if size is not None: + assert size == label.size + 1 + else: + size = label.size + 1 Layer( - name = name, - type = LayerType.CTC_LAYER, - size = size, - norm_by_times = norm_by_times, - inputs = [input.name, label.name] + name=name, + type=LayerType.CTC_LAYER, + size=size, + norm_by_times=norm_by_times, + inputs=[input.name, label.name] ) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) + @wrap_name_default() @wrap_param_attr_default() -def crf_layer(input, label, size, weight=None, param_attr=None, name=None): +def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): """ A layer for calculating the cost of sequential conditional random field model. @@ -3211,7 +3331,7 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): :param input: The first input layer is the feature. :type input: LayerOutput :param label: The second input layer is label. - :type input: LayerOutput + :type label: LayerOutput :param size: The category number. :type size: int :param weight: The third layer is "weight" of each sample, which is an @@ -3227,6 +3347,12 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): assert isinstance(input, LayerOutput) assert isinstance(label, LayerOutput) assert weight is None or isinstance(weight, LayerOutput) + if input.size is not None and label.size is not None: + assert input.size == label.size + if size is None: + size = input.size + else: + assert size == input.size ipts = [Input(input.name, **param_attr.attr), Input(label.name)] @@ -3234,16 +3360,17 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): ipts.append(Input(weight.name)) Layer( - name = name, - type = LayerType.CRF_LAYER, - size = size, - inputs = ipts, + name=name, + type=LayerType.CRF_LAYER, + size=size, + inputs=ipts, ) parents = [input, label] if weight is not None: parents.append(weight) return LayerOutput(name, LayerType.CRF_LAYER, parents, size=size) + @wrap_name_default() @wrap_param_attr_default() def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): @@ -3276,24 +3403,28 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): ipts.append(Input(label.name)) Layer( - name = name, - type = LayerType.CRF_DECODING_LAYER, - size = size, - inputs = ipts, + name=name, + type=LayerType.CRF_DECODING_LAYER, + size=size, + inputs=ipts, ) parents = [input] if label is not None: parents.append(label) return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=size) + """ following are cost Layers. """ + + @wrap_name_default() -def rank_cost(left, right, lable, weight=None, name=None, coeff=1.0): +def rank_cost(left, right, label, weight=None, name=None, coeff=1.0): """ A cost Layer for learning to rank using gradient descent. Details can refer - to `papers `_. + to `papers `_. This layer contains at least three inputs. The weight is an optional argument, which affects the cost. @@ -3350,12 +3481,13 @@ def rank_cost(left, right, lable, weight=None, name=None, coeff=1.0): type=LayerType.RANK_COST, inputs=ipts, coeff=coeff, - ) + ) return LayerOutput(name, LayerType.RANK_COST, parents=parents) + @wrap_name_default() -def lambda_cost(input, score, NDCG_num=5, max_sort_size=-1, coeff=1.0): +def lambda_cost(input, score, name, NDCG_num=5, max_sort_size=-1): """ lambdaCost for lambdaRank LTR approach. @@ -3368,9 +3500,7 @@ def lambda_cost(input, score, NDCG_num=5, max_sort_size=-1, coeff=1.0): NDCG_num=8, max_sort_size=-1) - :param input: The 1st input. Samples of the same query should be loaded - as sequence. User should provided socres for each sample. - The score should be the 2nd input of this layer. + :param input: Samples of the same query should be loaded as sequence. :type input: LayerOutput :param score: The 2nd input. Score of each sample. :type input: LayerOutput @@ -3388,21 +3518,22 @@ def lambda_cost(input, score, NDCG_num=5, max_sort_size=-1, coeff=1.0): :type max_sort_size: int :param name: The name of this layers. It is not necessary. :type name: None|basestring - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float :return: LayerOutput object. :rtype: LayerOutput """ + assert isinstance(input, LayerOutput) and isinstance(score, LayerOutput) + if score.size is not None: + assert score.size == 1 Layer(name=name, type=LayerType.LAMBDA_COST, inputs=[input.name, score.name], NDCG_num=NDCG_num, - max_sort_size=max_sort_size, - coeff=coeff, - ) + max_sort_size=max_sort_size + ) return LayerOutput(name, LayerType.LAMBDA_COST, parents=[input, score]) + @wrap_name_default() def cross_entropy(input, label, name=None, coeff=1.0): """ @@ -3430,9 +3561,10 @@ def cross_entropy(input, label, name=None, coeff=1.0): type=LayerType.CROSS_ENTROPY, inputs=[input.name, label.name], coeff=coeff, - ) + ) return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=[input, label]) + @wrap_name_default() def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, softmax_selfnorm_alpha=0.1): @@ -3463,12 +3595,13 @@ def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, inputs=[input.name, label.name], coeff=coeff, softmax_selfnorm_alpha=softmax_selfnorm_alpha, - ) + ) return LayerOutput(name, LayerType.CROSS_ENTROPY_WITH_SELFNORM, parents=[input, label]) + @wrap_name_default() def huber_cost(input, label, name=None, coeff=1.0): """ @@ -3482,8 +3615,6 @@ def huber_cost(input, label, name=None, coeff=1.0): :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param type: The type of cost. - :type type: basestring. :param name: The name of this layers. It is not necessary. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. @@ -3491,14 +3622,17 @@ def huber_cost(input, label, name=None, coeff=1.0): :return: LayerOutput object. :rtype: LayerOutput. """ - + assert isinstance(input, LayerOutput) + if input.size is not None: + assert input.size == 1 Layer(name=name, type=LayerType.HUBER, inputs=[input.name, label.name], coeff=coeff, - ) + ) return LayerOutput(name, LayerType.HUBER, parents=[input, label]) + @wrap_name_default() def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0): """ @@ -3522,15 +3656,16 @@ def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0): :rtype: LayerOutput """ - if not isinstance(input.act, SigmoidActivation): + if input.activation is None or \ + not isinstance(input.activation, SigmoidActivation): logger.log(logging.WARN, "%s is not recommend for batch normalization's activation, " - "maybe the relu is better" % act.name) + "maybe the relu is better" % repr(input.activation)) Layer(name=name, type=LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY, inputs=[input.name, label.name], coeff=coeff, - ) + ) return LayerOutput(name, LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY, parents=[input, label]) diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index 93c4261cc48c5..e59e93acbe33a 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -616,7 +616,7 @@ def lstmemory_group(input, size=None, name=None, cell states, or hidden states in every time step are accessible to for the user. This is especially useful in attention model. If you do not need to access to the internal states of the lstm, but merely use its outputs, - it is recommanded to use the lstmemory, which is relatively faster than + it is recommended to use the lstmemory, which is relatively faster than lstmemory_group. NOTE: In PaddlePaddle's implementation, the following input-to-hidden @@ -1052,7 +1052,7 @@ def dropout_layer(input, dropout_rate, name=None): layer_attr=ExtraAttr(drop_rate=dropout_rate)) -def outputs(layers): +def outputs(layers, *args): """ Declare the end of network. Currently it will only calculate the input/output order of network. It will calculate the predict network or @@ -1089,9 +1089,12 @@ def __dfs_travel__(layer, if isinstance(layers, LayerOutput): layers = [layers] + if len(args) != 0: + layers.extend(args) + assert len(layers) > 0 if len(layers) != 1: - logger.warning("EndOfNetwork routine try to calculate network's" + logger.warning("`outputs` routine try to calculate network's" " inputs and outputs order. It might not work well." "Please see follow log carefully.") inputs = [] diff --git a/python/paddle/trainer_config_helpers/poolings.py b/python/paddle/trainer_config_helpers/poolings.py index 5e06d82005841..d627daab0c496 100644 --- a/python/paddle/trainer_config_helpers/poolings.py +++ b/python/paddle/trainer_config_helpers/poolings.py @@ -47,9 +47,14 @@ class MaxPooling(BasePoolingType): .. math:: max(samples\\_of\\_a\\_sequence) + + :param output_max_index: True if output sequence max index instead of max + value. None means use default value in proto. + :type output_max_index: bool|None """ - def __init__(self): + def __init__(self, output_max_index=None): BasePoolingType.__init__(self, "max") + self.output_max_index = output_max_index class AvgPooling(BasePoolingType): diff --git a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt index 611fb855a8c9a..cf52b06bfea06 100644 --- a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt +++ b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt @@ -3,3 +3,8 @@ add_test(NAME layers_test COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) + +add_test(NAME test_layerHelpers + COMMAND + ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh +) diff --git a/python/paddle/trainer_config_helpers/tests/configs/.gitignore b/python/paddle/trainer_config_helpers/tests/configs/.gitignore new file mode 100644 index 0000000000000..52378fe7a4865 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/.gitignore @@ -0,0 +1 @@ +*protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/beam_search.py b/python/paddle/trainer_config_helpers/tests/configs/beam_search.py new file mode 100644 index 0000000000000..dc81940761af5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/beam_search.py @@ -0,0 +1,30 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +dat = data_layer(name='data_in', size=100) + + +def rnn_step(input, embedding): + last_time_step_output = memory(name='rnn', size=128) + with mixed_layer(size=128, name='rnn') as simple_rnn: + simple_rnn += full_matrix_projection(embedding) + simple_rnn += full_matrix_projection(input) + simple_rnn += full_matrix_projection(last_time_step_output) + return simple_rnn + + +beam_gen = beam_search(name="decoder", + step=rnn_step, + input=[StaticInput(input=dat), GeneratedInput( + embedding_name='emb', embedding_size=128, size=128 + )], + bos_id=0, + eos_id=1, + beam_size=5, + result_file="./generated_sequences.txt") + +outputs(beam_gen) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 new file mode 100644 index 0000000000000..6f1fca144a7b3 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -0,0 +1,18 @@ +8e6bd468063c6908aa8903fc5f43af04 beam_search.protostr +7e6919d17562516e9a1d9a88de1fb3b9 img_layers.protostr +a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr +9c038249ec8ff719753a746cdb04c026 layer_activations.protostr +5913f87b39cee3b2701fa158270aca26 projections.protostr +6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr +0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr +144bc6d3a509de74115fa623741797ed test_expand_layer.protostr +2378518bdb71e8c6e888b1842923df58 test_fc.protostr +8bb44e1e5072d0c261572307e7672bda test_grumemory_layer.protostr +1f3510672dce7a9ed25317fc58579ac7 test_hsigmoid.protostr +d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr +251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr +e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr +2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr +67d6fde3afb54f389d0ce4ff14726fe1 test_sequence_pooling.protostr +f586a548ef4350ba1ed47a81859a64cb unused_layers.protostr +8122477f4f65244580cec09edc590041 util_layers.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh new file mode 100755 index 0000000000000..2b551f73c4f7b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +cd `dirname $0` +export PYTHONPATH=$PWD/../../../../ + +configs=(test_fc layer_activations projections test_print_layer +test_sequence_pooling test_lstmemory_layer test_grumemory_layer +last_first_seq test_expand_layer test_ntm_layers test_hsigmoid +img_layers util_layers simple_rnn_layers unused_layers test_cost_layers +test_rnn_group beam_search) + + +for conf in ${configs[*]} +do + python -m paddle.utils.dump_config $conf.py > $conf.protostr +done \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py new file mode 100644 index 0000000000000..6c8ba8be846e5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py @@ -0,0 +1,20 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-3, + batch_size=1000 +) + +img = data_layer(name='image', size=256*256) + +img_conv = img_conv_layer(input=img, num_channels=1, num_filters=64, + filter_size=(32, 64), padding=(1, 0), stride=(1, 1), + act=LinearActivation()) +img_bn = batch_norm_layer(input=img_conv, act=ReluActivation()) + +img_norm = img_cmrnorm_layer(input=img_bn, size=32) + +img_pool = img_pool_layer(input=img_conv, pool_size=32, pool_type=MaxPooling()) + + +outputs(img_pool, img_norm) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py new file mode 100644 index 0000000000000..d54a1c49fd3fd --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py @@ -0,0 +1,26 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=30) + +seq_op = [ + first_seq, + last_seq +] + +agg_level = [ + AggregateLevel.EACH_SEQUENCE, + AggregateLevel.EACH_TIMESTEP +] + +opts = [] + +for op in seq_op: + for al in agg_level: + opts.append(op(input=din, agg_level=al)) + +outputs(opts) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py new file mode 100644 index 0000000000000..ba10dc78e1e3b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py @@ -0,0 +1,21 @@ +''' +Test all activations. +''' + +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='input', size=100) + +acts = [ + TanhActivation, SigmoidActivation, SoftmaxActivation, IdentityActivation, + LinearActivation, ExpActivation, ReluActivation, BReluActivation, + SoftReluActivation, STanhActivation, AbsActivation, SquareActivation] + +outputs( + [fc_layer(input=din, size=100, act=act(), name="layer_%d" % i) for i, act in + enumerate(acts)]) diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.py b/python/paddle/trainer_config_helpers/tests/configs/projections.py new file mode 100644 index 0000000000000..b6da4f99f1cbc --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.py @@ -0,0 +1,47 @@ +''' +Test mixed layer, projections and operators. +''' +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='test', size=100) + +din = embedding_layer(input=din, size=256) + +with mixed_layer(size=100) as m1: + m1 += full_matrix_projection(input=din) + +with mixed_layer(size=100) as m2: + m2 += table_projection(input=m1) + +with mixed_layer(size=100) as m3: + m3 += identity_projection(input=m2) + +with mixed_layer(size=100) as m4: + m4 += dotmul_projection(input=m3) + +with mixed_layer() as m5: + m5 += context_projection(input=m4, context_len=3) + +with mixed_layer() as m6: + m6 += dotmul_operator(m3, m4) + +img = data_layer(name='img', size=32*32) +flt = data_layer(name='filter', size=3*3*1*64) + +with mixed_layer() as m7: + m7 += conv_operator(img=img, filter=flt, num_filters=64, + num_channel=1, filter_size=3) + +end = mixed_layer(input=[full_matrix_projection(input=m5), + trans_full_matrix_projection(input=m6), + full_matrix_projection(input=m7)], + size=100, + layer_attr=ExtraAttr(drop_rate=0.5, + error_clipping_threshold=40)) + +outputs(end) diff --git a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh new file mode 100755 index 0000000000000..78114ce32b019 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd `dirname $0` +set -e +./generate_protostr.sh +md5sum -c check.md5 diff --git a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py new file mode 100644 index 0000000000000..87c2a85cf92dd --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py @@ -0,0 +1,36 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='data', size=200) + +hidden = fc_layer(input=din, size=200, act=SigmoidActivation()) + +rnn = recurrent_layer(input=hidden, act=SigmoidActivation()) + +rnn2 = recurrent_layer(input=hidden, act=SigmoidActivation(), reverse=True) + +lstm1_param = fc_layer(input=hidden, size=200*4, act=LinearActivation(), + bias_attr=False) + +lstm1 = lstmemory(input=lstm1_param, act=SigmoidActivation()) + +lstm2_param = fc_layer(input=hidden, size=200*4, act=LinearActivation(), + bias_attr=False) + +lstm2 = lstmemory(input=lstm2_param, act=SigmoidActivation(), reverse=True) + +gru1_param = fc_layer(input=hidden, size=200*3, act=LinearActivation(), + bias_attr=False) +gru1 = grumemory(input=gru1_param, act=SigmoidActivation()) + +gru2_param = fc_layer(input=hidden, size=200*3, act=LinearActivation(), + bias_attr=False) +gru2 = grumemory(input=gru2_param, act=SigmoidActivation(), reverse=True) + +outputs(last_seq(input=rnn), first_seq(input=rnn2), + last_seq(input=lstm1), first_seq(input=lstm2), + last_seq(input=gru1), first_seq(gru2)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py new file mode 100644 index 0000000000000..64b45f4ded10b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -0,0 +1,26 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +seq_in = data_layer(name='input', size=200) +labels = data_layer(name='labels', size=5000) + +probs = data_layer(name='probs', size=10) +xe_label = data_layer(name='xe-label', size=10) + +outputs(ctc_layer(input=seq_in, label=labels), + crf_layer(input=fc_layer(input=seq_in, size=4), + label=data_layer(name='crf_label', size=4)), + rank_cost(left=data_layer(name='left', size=1), + right=data_layer(name='right', size=1), + label=data_layer(name='label', size=1)), + lambda_cost(input=data_layer(name='list_feature', size=100), + score=data_layer(name='list_scores', size=1)), + cross_entropy(input=probs, label=xe_label), + cross_entropy_with_selfnorm(input=probs, label=xe_label), + huber_cost(input=data_layer(name='huber_probs', size=1), + label=data_layer(name='huber_label', size=1)), + multi_binary_label_cross_entropy(input=probs, label=xe_label)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py new file mode 100644 index 0000000000000..d9c841ab277e1 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py @@ -0,0 +1,14 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=30) +data_seq = data_layer(name='data_seq', size=30) + +outputs(expand_layer(input=din, expand_as=data_seq, + expand_level=ExpandLevel.FROM_SEQUENCE), + expand_layer(input=din, expand_as=data_seq, + expand_level=ExpandLevel.FROM_TIMESTEP)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_fc.py b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py new file mode 100644 index 0000000000000..a6d033f291d2c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py @@ -0,0 +1,20 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=100) + +trans = trans_layer(input=din) + +hidden = fc_layer(input=trans, size=100, + bias_attr=False) + +mask = data_layer(name='mask', size=100) + +hidden_sel = selective_fc_layer(input=din, select=mask, size=100, + act=SigmoidActivation()) + +outputs(hidden, hidden_sel) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py new file mode 100644 index 0000000000000..8d9fd9df5179c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='data', size=120) + +outputs(grumemory(input=din, size=40, reverse=True, gate_act=TanhActivation(), + act=SigmoidActivation())) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py new file mode 100644 index 0000000000000..46069074ded56 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='data', size=100) +label = data_layer(name='label', size=10) + +outputs(hsigmoid(input=din, label=label, num_classes=10)) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py new file mode 100644 index 0000000000000..56304addb17b2 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=128) + +outputs(lstmemory(input=din, reverse=True, gate_act=TanhActivation(), + act=TanhActivation(), size=32)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py new file mode 100644 index 0000000000000..662b19145b376 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py @@ -0,0 +1,23 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +weight = data_layer(name='w', size=1) +a = data_layer(name='a', size=100) +b = data_layer(name='b', size=100) +c = data_layer(name='c', size=200) +d = data_layer(name='d', size=31) + +outputs(interpolation_layer(input=[a, b], weight=weight), + power_layer(input=a, weight=weight), + scaling_layer(input=a, weight=weight), + cos_sim(a=a, b=b), + cos_sim(a=a, b=c, size=2), + sum_to_one_norm_layer(input=a), + conv_shift_layer(a=a, b=d), + tensor_layer(x1=a, x2=b, size=1000), + slope_intercept_layer(input=a, slope=0.7, intercept=0.9), + linear_comb_layer(weights=b, vectors=c)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py new file mode 100644 index 0000000000000..f6b2661c7b9e8 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py @@ -0,0 +1,12 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='input', size=100) + +print_layer(input=din) + +outputs(din) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py new file mode 100644 index 0000000000000..53f5c5d2499f9 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py @@ -0,0 +1,35 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +seq = data_layer(name='seq_input', size=100) +sub_seq = data_layer(name='sub_seq_input', size=100) +lbl = data_layer(name='label', size=1) + + +def generate_rnn_simple(name): + def rnn_simple(s): + m = memory(name=name, size=200) + fc = fc_layer(input=[s, m], size=200, name=name) + return fc + + return rnn_simple + + +with mixed_layer() as lstm_param: # test lstm unit, rnn group + lstm_param += full_matrix_projection(input=seq, size=100 * 4) + +with mixed_layer() as gru_param: + gru_param += full_matrix_projection(input=seq, size=100 * 3) + +outputs(last_seq(input=recurrent_group(step=generate_rnn_simple('rnn_forward'), + input=seq)), + first_seq(input=recurrent_group(step=generate_rnn_simple('rnn_back'), + input=seq, reverse=True)), + last_seq(input=recurrent_group(step=generate_rnn_simple( + 'rnn_subseq_forward'), input=SubsequenceInput(input=sub_seq))), + last_seq(input=lstmemory_group(input=lstm_param, size=100)), + last_seq(input=gru_group(input=gru_param, size=100))) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py new file mode 100644 index 0000000000000..2e24164b5578c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py @@ -0,0 +1,30 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='dat_in', size=100) + +POOL_TYPE = [ + MaxPooling, + AvgPooling, + SumPooling +] + +AGG_LEVEL = [ + AggregateLevel.EACH_SEQUENCE, + AggregateLevel.EACH_TIMESTEP +] + +opts = [] + +for pt in POOL_TYPE: + for al in AGG_LEVEL: + opts.append(pooling_layer(input=din, agg_level=al, pooling_type=pt())) + +opts.append(pooling_layer(input=din, + pooling_type=MaxPooling(output_max_index=True))) + +outputs(opts) diff --git a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py new file mode 100644 index 0000000000000..a6a3d09a4315a --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py @@ -0,0 +1,14 @@ +from paddle.trainer_config_helpers import * +settings( + batch_size=1000, + learning_rate=1e-4 +) + +probs = data_layer(name='probs', size=100) + +outputs( + sampling_id_layer(input=probs), # It seems not support training + + # It seems this layer is not correct, and should be rewrite. + # block_expand_layer(input=probs, channel=1, block_x=1, block_y=3), +) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/util_layers.py b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py new file mode 100644 index 0000000000000..aadb3f3f5e799 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py @@ -0,0 +1,15 @@ +from paddle.trainer_config_helpers import * + +settings(learning_rate=1e-4, batch_size=1000) + +a = data_layer(name='a', size=10) +b = data_layer(name='b', size=10) + +result = addto_layer(input=[a, b]) +concat1 = concat_layer(input=[a, b]) +concat2 = concat_layer(input=[ + identity_projection(input=a), + identity_projection(input=b) +]) + +outputs(result, concat1, concat2) \ No newline at end of file From 584c0fcc13b323251c496ba1e27b3c6b58d849bc Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 28 Sep 2016 13:54:01 +0800 Subject: [PATCH 2/2] Follow comments * Fix a unittest bug before. * dotmul inputs' sizes should be equal * add equation to recurrent_layer * unify x,y x1,x2 => a,b * log fatal when lstm/gru set wrong size --- .../paddle/trainer_config_helpers/layers.py | 96 ++++++++++++------- .../tests/configs/beam_search.py | 30 ------ .../tests/configs/check.md5 | 1 - .../tests/configs/generate_protostr.sh | 5 +- .../tests/configs/projections.py | 2 +- .../tests/configs/test_ntm_layers.py | 2 +- .../tests/layers_test_config.py | 2 +- 7 files changed, 68 insertions(+), 70 deletions(-) delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/beam_search.py diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 7c575b9baaf0e..b28dd02b70946 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -439,7 +439,7 @@ def dotmul_projection(input, param_attr=None): return proj -def dotmul_operator(x, y, scale=1): +def dotmul_operator(a=None, b=None, scale=1, **kwargs): """ DotMulOperator takes two inputs and performs element-wise multiplication: @@ -455,23 +455,28 @@ def dotmul_operator(x, y, scale=1): op = dotmul_operator(x=layer1, y=layer2, scale=0.5) - :param x: Input layer1 - :type x: LayerOutput - :param y: Input layer2 - :type y: LayerOutput + :param a: Input layer1 + :type a: LayerOutput + :param b: Input layer2 + :type b: LayerOutput :param scale: config scalar, default value is one. :type scale: float :return: A DotMulOperator Object. :rtype: DotMulOperator """ - assert isinstance(x, LayerOutput) - assert isinstance(y, LayerOutput) - if x.size is not None and y.size is not None: - assert x.size == y.size - - op = DotMulOperator(input_layer_names=[x.name, y.name], + if 'x' in kwargs or 'y' in kwargs: + logger.warning('x and y arguments for dotmul_operator is deprecated. ' + 'Please use a and b as parameter.') + a = kwargs.get('x', a) # For Backward capacity. + b = kwargs.get('y', b) + assert isinstance(a, LayerOutput) + assert isinstance(b, LayerOutput) + if a.size is not None and b.size is not None: + assert a.size == b.size + + op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale) - op.origin = [x, y] + op.origin = [a, b] return op @@ -799,7 +804,7 @@ def print_layer(input, name=None): type=LayerType.PRINT_LAYER, inputs=[l.name for l in input], ) - return LayerOutput(name, LayerType.PRINT_LAYER, input) + # this layer don't return anything, can not be input of other layer. @wrap_name_default("seq_pooling") @@ -935,10 +940,14 @@ def lstmemory(input, name=None, reverse=False, act=None, assert act.support_hppl assert input.size is not None and input.size % 4 == 0 if size is not None: - logger.warning("NOTE: The lstmemory layer[%s]'s size is set by previous" - " input layer. The lstm size should be equal with input" - " layer size/4. The size which is set explicitly will" - " be ignored." % name) + if input.size / 4 == size: + plog = logger.warning + else: + plog = logger.fatal + + plog("NOTE: The lstmemory layer[%s]'s size is set by previous input " + "layer. The lstm size should be equal with input layer size/4. The" + " size which is set explicitly will be ignored." % name) Layer(name=name, type=LayerType.LSTMEMORY, @@ -1046,9 +1055,13 @@ def grumemory(input, name=None, reverse=False, act=None, assert gate_act.support_hppl assert input.size is not None and input.size % 3 == 0 if size is not None: - logger.warning("NOTE: the gru memory layer's size is set by previous " - "input layer, and should be input size / 3. Set size " - "explicitly will be ignored.") + if input.size / 3 == size: + plog = logger.warning + else: + plog = logger.fatal + plog("NOTE: the gru memory layer's size is set by previous input layer," + " and should be input size / 3. Set size explicitly will be " + "ignored.") Layer(name=name, type=LayerType.GRUMEMORY, @@ -2259,6 +2272,21 @@ def recurrent_layer(input, act=None, bias_attr=None, Simple recurrent unit layer. It is just a fully connect layer through both time and neural network. + For each sequence [start, end] it performs the following computation\: + + .. math:: + + out_{i} = act(in_{i}) \\ \\ \\text{for} \\ i = start \\\\ + out_{i} = act(in_{i} + out_{i-1} * W) \\ \\ \\text{for} \\ start < i <= end + + If reversed is true, the order is reversed\: + + .. math:: + + out_{i} = act(in_{i}) \\ \\ \\text{for} \\ i = end \\\\ + out_{i} = act(in_{i} + out_{i+1} * W) \\ \\ \\text{for} \\ start <= i < end + + :param input: Input Layer :type input: LayerOutput :param act: activation. @@ -2919,34 +2947,34 @@ def conv_shift_layer(a, b, name=None): @wrap_bias_attr_default() @wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(x1, x2, size, act=None, name=None, +def tensor_layer(a, b, size, act=None, name=None, param_attr=None, bias_attr=None, layer_attr=None): """ This layer performs tensor operation for two input. For example, each sample: .. math:: - y_{i} = x_{1} * W_{i} * {x_{2}^\mathrm{T}}, i=0,1,...,K-1 + y_{i} = a * W_{i} * {b^\mathrm{T}}, i=0,1,...,K-1 In this formular: - - :math:`x_{1}`: the first input contains M elements. - - :math:`x_{2}`: the second input contains N elements. + - :math:`a`: the first input contains M elements. + - :math:`b`: the second input contains N elements. - :math:`y_{i}`: the i-th element of y. - :math:`W_{i}`: the i-th learned weight, shape if [M, N] - - :math:`{x_{2}}^\mathrm{T}`: the transpose of :math:`x_{2}`. + - :math:`b^\mathrm{T}`: the transpose of :math:`b_{2}`. The simple usage is: .. code-block:: python - tensor = tensor_layer(x1=layer1, x2=layer2, size=1000) + tensor = tensor_layer(a=layer1, b=layer2, size=1000) :param name: layer name :type name: basestring - :param x1: Input layer x_1. - :type x1: LayerOutput - :param x2: input layer x_2. - :type x2: LayerOutput + :param a: Input layer a. + :type a: LayerOutput + :param b: input layer b. + :type b: LayerOutput :param size: the layer dimension. :type size: int. :param act: Activation Type. Default is tanh. @@ -2962,18 +2990,18 @@ def tensor_layer(x1, x2, size, act=None, name=None, :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(x1, LayerOutput) and isinstance(x2, LayerOutput) + assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput) Layer( name=name, size=size, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(x1.name, **param_attr.attr), - Input(x2.name)], + inputs=[Input(a.name, **param_attr.attr), + Input(b.name)], **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[x1, x2], + return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size) diff --git a/python/paddle/trainer_config_helpers/tests/configs/beam_search.py b/python/paddle/trainer_config_helpers/tests/configs/beam_search.py deleted file mode 100644 index dc81940761af5..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/beam_search.py +++ /dev/null @@ -1,30 +0,0 @@ -from paddle.trainer_config_helpers import * - -settings( - learning_rate=1e-4, - batch_size=1000 -) - -dat = data_layer(name='data_in', size=100) - - -def rnn_step(input, embedding): - last_time_step_output = memory(name='rnn', size=128) - with mixed_layer(size=128, name='rnn') as simple_rnn: - simple_rnn += full_matrix_projection(embedding) - simple_rnn += full_matrix_projection(input) - simple_rnn += full_matrix_projection(last_time_step_output) - return simple_rnn - - -beam_gen = beam_search(name="decoder", - step=rnn_step, - input=[StaticInput(input=dat), GeneratedInput( - embedding_name='emb', embedding_size=128, size=128 - )], - bos_id=0, - eos_id=1, - beam_size=5, - result_file="./generated_sequences.txt") - -outputs(beam_gen) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 6f1fca144a7b3..29928b6f7b423 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -1,4 +1,3 @@ -8e6bd468063c6908aa8903fc5f43af04 beam_search.protostr 7e6919d17562516e9a1d9a88de1fb3b9 img_layers.protostr a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 9c038249ec8ff719753a746cdb04c026 layer_activations.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 2b551f73c4f7b..fc2acbd41ed90 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -8,10 +8,11 @@ configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers -test_rnn_group beam_search) +test_rnn_group) for conf in ${configs[*]} do + echo "Generating " $conf python -m paddle.utils.dump_config $conf.py > $conf.protostr -done \ No newline at end of file +done diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.py b/python/paddle/trainer_config_helpers/tests/configs/projections.py index b6da4f99f1cbc..4066c5bc6e0f0 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/projections.py +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.py @@ -28,7 +28,7 @@ m5 += context_projection(input=m4, context_len=3) with mixed_layer() as m6: - m6 += dotmul_operator(m3, m4) + m6 += dotmul_operator(a=m3, b=m4) img = data_layer(name='img', size=32*32) flt = data_layer(name='filter', size=3*3*1*64) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py index 662b19145b376..4d8e1fdc6b598 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py @@ -18,6 +18,6 @@ cos_sim(a=a, b=c, size=2), sum_to_one_norm_layer(input=a), conv_shift_layer(a=a, b=d), - tensor_layer(x1=a, x2=b, size=1000), + tensor_layer(a=a, b=b, size=1000), slope_intercept_layer(input=a, slope=0.7, intercept=0.9), linear_comb_layer(weights=b, vectors=c)) diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index cd368d6b12cbd..faaab9107d8fb 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -47,7 +47,7 @@ outputs(classification_cost(out, data_layer(name="label", size=num_classes))) -dotmul = mixed_layer(input=[dotmul_operator(x=x1, y=y1), +dotmul = mixed_layer(input=[dotmul_operator(a=x1, b=x1), dotmul_projection(input=y1)]) proj_with_attr_init = mixed_layer(input=full_matrix_projection(input=y1,