From 53c219c385e57ce95ab22263279c1902369d6882 Mon Sep 17 00:00:00 2001 From: yoyoIcy Date: Sat, 9 Sep 2023 11:40:34 +0800 Subject: [PATCH 1/7] [Doctest]fix No.330, test=docs_preview --- python/paddle/base/data_feed_desc.py | 220 +++++++++++++-------------- 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index 3987440ecc6da..aa4614cf2cdf5 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -33,26 +33,26 @@ class DataFeedDesc: .. code-block:: python - import paddle.base as base - f = open("data.proto", "w") - print >> f, 'name: "MultiSlotDataFeed"' - print >> f, 'batch_size: 2' - print >> f, 'multi_slot_desc {' - print >> f, ' slots {' - print >> f, ' name: "words"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, ' slots {' - print >> f, ' name: "label"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, '}' - f.close() - data_feed = base.DataFeedDesc('data.proto') + >>> import paddle.base as base + >>> f = open("data.proto", "w") + >>> print >> f, 'name: "MultiSlotDataFeed"' + >>> print >> f, 'batch_size: 2' + >>> print >> f, 'multi_slot_desc {' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "words"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "label"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, '}' + >>> f.close() + >>> data_feed = base.DataFeedDesc('data.proto') However, users usually shouldn't care about the message format; instead, they are encouraged to use :code:`Data Generator` as a tool to generate a @@ -64,17 +64,17 @@ class DataFeedDesc: .. code-block:: python - import paddle.base as base - data_feed = base.DataFeedDesc('data.proto') - data_feed.set_batch_size(128) - data_feed.set_dense_slots('wd') # The slot named 'wd' will be dense - data_feed.set_use_slots('wd') # The slot named 'wd' will be used + >>> import paddle.base as base + >>> data_feed = base.DataFeedDesc('data.proto') + >>> data_feed.set_batch_size(128) + >>> data_feed.set_dense_slots('wd') # The slot named 'wd' will be dense + >>> data_feed.set_use_slots('wd') # The slot named 'wd' will be used Finally, the content can be dumped out for debugging purpose: .. code-block:: python - print(data_feed.desc()) + >>> print(data_feed.desc()) Args: proto_file(string): Disk file containing a data feed description. @@ -99,27 +99,27 @@ def set_batch_size(self, batch_size): Example: .. code-block:: python - import paddle.base as base - f = open("data.proto", "w") - print >> f, 'name: "MultiSlotDataFeed"' - print >> f, 'batch_size: 2' - print >> f, 'multi_slot_desc {' - print >> f, ' slots {' - print >> f, ' name: "words"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, ' slots {' - print >> f, ' name: "label"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, '}' - f.close() - data_feed = base.DataFeedDesc('data.proto') - data_feed.set_batch_size(128) + >>> import paddle.base as base + >>> f = open("data.proto", "w") + >>> print >> f, 'name: "MultiSlotDataFeed"' + >>> print >> f, 'batch_size: 2' + >>> print >> f, 'multi_slot_desc {' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "words"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "label"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, '}' + >>> f.close() + >>> data_feed = base.DataFeedDesc('data.proto') + >>> data_feed.set_batch_size(128) Args: batch_size (int): The number of batch size. @@ -140,27 +140,27 @@ def set_dense_slots(self, dense_slots_name): Example: .. code-block:: python - import paddle.base as base - f = open("data.proto", "w") - print >> f, 'name: "MultiSlotDataFeed"' - print >> f, 'batch_size: 2' - print >> f, 'multi_slot_desc {' - print >> f, ' slots {' - print >> f, ' name: "words"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, ' slots {' - print >> f, ' name: "label"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, '}' - f.close() - data_feed = base.DataFeedDesc('data.proto') - data_feed.set_dense_slots(['words']) + >>> import paddle.base as base + >>> f = open("data.proto", "w") + >>> print >> f, 'name: "MultiSlotDataFeed"' + >>> print >> f, 'batch_size: 2' + >>> print >> f, 'multi_slot_desc {' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "words"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "label"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, '}' + >>> f.close() + >>> data_feed = base.DataFeedDesc('data.proto') + >>> data_feed.set_dense_slots(['words']) Args: dense_slots_name (list(str)): a list of slot names which will be set dense. @@ -187,27 +187,27 @@ def set_use_slots(self, use_slots_name): Example: .. code-block:: python - import paddle.base as base - f = open("data.proto", "w") - print >> f, 'name: "MultiSlotDataFeed"' - print >> f, 'batch_size: 2' - print >> f, 'multi_slot_desc {' - print >> f, ' slots {' - print >> f, ' name: "words"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, ' slots {' - print >> f, ' name: "label"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, '}' - f.close() - data_feed = base.DataFeedDesc('data.proto') - data_feed.set_use_slots(['words']) + >>> import paddle.base as base + >>> f = open("data.proto", "w") + >>> print >> f, 'name: "MultiSlotDataFeed"' + >>> print >> f, 'batch_size: 2' + >>> print >> f, 'multi_slot_desc {' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "words"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "label"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, '}' + >>> f.close() + >>> data_feed = base.DataFeedDesc('data.proto') + >>> data_feed.set_use_slots(['words']) Args: use_slots_name: a list of slot names which will be used in training @@ -231,27 +231,27 @@ def desc(self): Example: .. code-block:: python - import paddle.base as base - f = open("data.proto", "w") - print >> f, 'name: "MultiSlotDataFeed"' - print >> f, 'batch_size: 2' - print >> f, 'multi_slot_desc {' - print >> f, ' slots {' - print >> f, ' name: "words"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, ' slots {' - print >> f, ' name: "label"' - print >> f, ' type: "uint64"' - print >> f, ' is_dense: false' - print >> f, ' is_used: true' - print >> f, ' }' - print >> f, '}' - f.close() - data_feed = base.DataFeedDesc('data.proto') - print(data_feed.desc()) + >>> import paddle.base as base + >>> f = open("data.proto", "w") + >>> print >> f, 'name: "MultiSlotDataFeed"' + >>> print >> f, 'batch_size: 2' + >>> print >> f, 'multi_slot_desc {' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "words"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, ' slots {' + >>> print >> f, ' name: "label"' + >>> print >> f, ' type: "uint64"' + >>> print >> f, ' is_dense: false' + >>> print >> f, ' is_used: true' + >>> print >> f, ' }' + >>> print >> f, '}' + >>> f.close() + >>> data_feed = base.DataFeedDesc('data.proto') + >>> print(data_feed.desc()) Returns: A string message From afc17fb2f8897a8f06dcf24e42229eb58517d1ed Mon Sep 17 00:00:00 2001 From: yoyoIcy <142126786+yoyoIcy@users.noreply.github.com> Date: Sat, 9 Sep 2023 03:00:25 -0500 Subject: [PATCH 2/7] Update python/paddle/base/data_feed_desc.py Co-authored-by: Nyakku Shigure --- python/paddle/base/data_feed_desc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index aa4614cf2cdf5..657a9dd37e595 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -96,7 +96,7 @@ def set_batch_size(self, batch_size): """ Set :attr:`batch_size` in :ref:`api_base_DataFeedDesc` . :attr:`batch_size` can be changed during training. - Example: + Examples: .. code-block:: python >>> import paddle.base as base From 1a7b69d11ad84b0cce24107b6ac39c4e9dd5765d Mon Sep 17 00:00:00 2001 From: yoyoIcy Date: Sun, 10 Sep 2023 08:12:39 +0800 Subject: [PATCH 3/7] [Doctest]fix No.330, test=docs_preview --- python/paddle/base/data_feed_desc.py | 228 +++++++++++++-------------- 1 file changed, 112 insertions(+), 116 deletions(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index aa4614cf2cdf5..af49f3c0ab517 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -31,50 +31,50 @@ class DataFeedDesc: See :code:`paddle/base/framework/data_feed.proto` for message definition. A typical message might look like: - .. code-block:: python - - >>> import paddle.base as base - >>> f = open("data.proto", "w") - >>> print >> f, 'name: "MultiSlotDataFeed"' - >>> print >> f, 'batch_size: 2' - >>> print >> f, 'multi_slot_desc {' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "words"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "label"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, '}' - >>> f.close() - >>> data_feed = base.DataFeedDesc('data.proto') - - However, users usually shouldn't care about the message format; instead, - they are encouraged to use :code:`Data Generator` as a tool to generate a - valid data description, in the process of converting their raw log files to - training files acceptable to AsyncExecutor. - - DataFeedDesc can also be changed during runtime. Once you got familiar with - what each field mean, you can modify it to better suit your need. E.g.: - - .. code-block:: python - - >>> import paddle.base as base - >>> data_feed = base.DataFeedDesc('data.proto') - >>> data_feed.set_batch_size(128) - >>> data_feed.set_dense_slots('wd') # The slot named 'wd' will be dense - >>> data_feed.set_use_slots('wd') # The slot named 'wd' will be used - - Finally, the content can be dumped out for debugging purpose: - - .. code-block:: python - - >>> print(data_feed.desc()) + Examples: + .. code-block:: python + + >>> import paddle.base as base + >>> with open("data.proto", "w") as f: + ... f.write('name: "MultiSlotDataFeed"') + ... f.write('batch_size: 2') + ... f.write('multi_slot_desc {') + ... f.write(' slots {') + ... f.write(' name: "words"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write(' slots {') + ... f.write(' name: "label"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write('}') + >>> data_feed = base.DataFeedDesc('data.proto') + + However, users usually shouldn't care about the message format; instead, + they are encouraged to use :code:`Data Generator` as a tool to generate a + valid data description, in the process of converting their raw log files to + training files acceptable to AsyncExecutor. + + DataFeedDesc can also be changed during runtime. Once you got familiar with + what each field mean, you can modify it to better suit your need. E.g.: + + .. code-block:: python + + >>> import paddle.base as base + >>> data_feed = base.DataFeedDesc('data.proto') + >>> data_feed.set_batch_size(128) + >>> data_feed.set_dense_slots('wd') # The slot named 'wd' will be dense + >>> data_feed.set_use_slots('wd') # The slot named 'wd' will be used + + Finally, the content can be dumped out for debugging purpose: + + .. code-block:: python + + >>> print(data_feed.desc()) Args: proto_file(string): Disk file containing a data feed description. @@ -100,24 +100,23 @@ def set_batch_size(self, batch_size): .. code-block:: python >>> import paddle.base as base - >>> f = open("data.proto", "w") - >>> print >> f, 'name: "MultiSlotDataFeed"' - >>> print >> f, 'batch_size: 2' - >>> print >> f, 'multi_slot_desc {' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "words"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "label"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, '}' - >>> f.close() + >>> with open("data.proto", "w") as f: + ... f.write('name: "MultiSlotDataFeed"') + ... f.write('batch_size: 2') + ... f.write('multi_slot_desc {') + ... f.write(' slots {') + ... f.write(' name: "words"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write(' slots {') + ... f.write(' name: "label"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_batch_size(128) @@ -141,24 +140,23 @@ def set_dense_slots(self, dense_slots_name): .. code-block:: python >>> import paddle.base as base - >>> f = open("data.proto", "w") - >>> print >> f, 'name: "MultiSlotDataFeed"' - >>> print >> f, 'batch_size: 2' - >>> print >> f, 'multi_slot_desc {' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "words"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "label"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, '}' - >>> f.close() + >>> with open("data.proto", "w") as f: + ... f.write('name: "MultiSlotDataFeed"') + ... f.write('batch_size: 2') + ... f.write('multi_slot_desc {') + ... f.write(' slots {') + ... f.write(' name: "words"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write(' slots {') + ... f.write(' name: "label"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_dense_slots(['words']) @@ -188,24 +186,23 @@ def set_use_slots(self, use_slots_name): .. code-block:: python >>> import paddle.base as base - >>> f = open("data.proto", "w") - >>> print >> f, 'name: "MultiSlotDataFeed"' - >>> print >> f, 'batch_size: 2' - >>> print >> f, 'multi_slot_desc {' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "words"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "label"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, '}' - >>> f.close() + >>> with open("data.proto", "w") as f: + ... f.write('name: "MultiSlotDataFeed"') + ... f.write('batch_size: 2') + ... f.write('multi_slot_desc {') + ... f.write(' slots {') + ... f.write(' name: "words"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write(' slots {') + ... f.write(' name: "label"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_use_slots(['words']) @@ -232,24 +229,23 @@ def desc(self): .. code-block:: python >>> import paddle.base as base - >>> f = open("data.proto", "w") - >>> print >> f, 'name: "MultiSlotDataFeed"' - >>> print >> f, 'batch_size: 2' - >>> print >> f, 'multi_slot_desc {' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "words"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, ' slots {' - >>> print >> f, ' name: "label"' - >>> print >> f, ' type: "uint64"' - >>> print >> f, ' is_dense: false' - >>> print >> f, ' is_used: true' - >>> print >> f, ' }' - >>> print >> f, '}' - >>> f.close() + >>> with open("data.proto", "w") as f: + ... f.write('name: "MultiSlotDataFeed"') + ... f.write('batch_size: 2') + ... f.write('multi_slot_desc {') + ... f.write(' slots {') + ... f.write(' name: "words"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write(' slots {') + ... f.write(' name: "label"') + ... f.write(' type: "uint64"') + ... f.write(' is_dense: false') + ... f.write(' is_used: true') + ... f.write(' }') + ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> print(data_feed.desc()) From 90beb05c348808b45aca20e0a22bef2af2ff214f Mon Sep 17 00:00:00 2001 From: yoyoIcy Date: Sun, 10 Sep 2023 08:16:33 +0800 Subject: [PATCH 4/7] [Doctest]fix No.330, test=docs_preview --- python/paddle/base/data_feed_desc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index 5faa342941e0b..d0cb4aa45b731 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -136,7 +136,7 @@ def set_dense_slots(self, dense_slots_name): Features for a dense slot will be fed into a Tensor, while those for a sparse slot will be fed into a LoDTensor. - Example: + Examples: .. code-block:: python >>> import paddle.base as base @@ -182,7 +182,7 @@ def set_use_slots(self, use_slots_name): contain a lot of features, through this function one can select which ones will be used for a specific model. - Example: + Examples: .. code-block:: python >>> import paddle.base as base @@ -225,7 +225,7 @@ def desc(self): """ Returns a protobuf message for this DataFeedDesc - Example: + Examples: .. code-block:: python >>> import paddle.base as base From d856970e3f2fd15f5a8664dd169664e644db05c6 Mon Sep 17 00:00:00 2001 From: yoyoIcy Date: Sun, 10 Sep 2023 21:22:34 +0800 Subject: [PATCH 5/7] [Doctest]fix No.330, test=docs_preview --- python/paddle/base/data_feed_desc.py | 150 +++++++++++++-------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index d0cb4aa45b731..e4eae3312a552 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -36,21 +36,21 @@ class DataFeedDesc: >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"') - ... f.write('batch_size: 2') - ... f.write('multi_slot_desc {') - ... f.write(' slots {') - ... f.write(' name: "words"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') - ... f.write(' slots {') - ... f.write(' name: "label"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') + ... f.write('name: "MultiSlotDataFeed"\\n') + ... f.write('batch_size: 2\\n') + ... f.write('multi_slot_desc {\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "words"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "label"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') @@ -101,21 +101,21 @@ def set_batch_size(self, batch_size): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"') - ... f.write('batch_size: 2') - ... f.write('multi_slot_desc {') - ... f.write(' slots {') - ... f.write(' name: "words"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') - ... f.write(' slots {') - ... f.write(' name: "label"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') + ... f.write('name: "MultiSlotDataFeed"\\n') + ... f.write('batch_size: 2\\n') + ... f.write('multi_slot_desc {\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "words"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "label"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_batch_size(128) @@ -141,21 +141,21 @@ def set_dense_slots(self, dense_slots_name): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"') - ... f.write('batch_size: 2') - ... f.write('multi_slot_desc {') - ... f.write(' slots {') - ... f.write(' name: "words"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') - ... f.write(' slots {') - ... f.write(' name: "label"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') + ... f.write('name: "MultiSlotDataFeed"\\n') + ... f.write('batch_size: 2\\n') + ... f.write('multi_slot_desc {\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "words"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "label"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_dense_slots(['words']) @@ -187,21 +187,21 @@ def set_use_slots(self, use_slots_name): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"') - ... f.write('batch_size: 2') - ... f.write('multi_slot_desc {') - ... f.write(' slots {') - ... f.write(' name: "words"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') - ... f.write(' slots {') - ... f.write(' name: "label"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') + ... f.write('name: "MultiSlotDataFeed"\\n') + ... f.write('batch_size: 2\\n') + ... f.write('multi_slot_desc {\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "words"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "label"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_use_slots(['words']) @@ -230,21 +230,21 @@ def desc(self): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"') - ... f.write('batch_size: 2') - ... f.write('multi_slot_desc {') - ... f.write(' slots {') - ... f.write(' name: "words"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') - ... f.write(' slots {') - ... f.write(' name: "label"') - ... f.write(' type: "uint64"') - ... f.write(' is_dense: false') - ... f.write(' is_used: true') - ... f.write(' }') + ... f.write('name: "MultiSlotDataFeed"\\n') + ... f.write('batch_size: 2\\n') + ... f.write('multi_slot_desc {\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "words"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') + ... f.write(' slots {\\n') + ... f.write(' name: "label"\\n') + ... f.write(' type: "uint64"\\n') + ... f.write(' is_dense: false\\n') + ... f.write(' is_used: true\\n') + ... f.write(' }\\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> print(data_feed.desc()) From 1997cb40114354d0f93d93bc8aced0a57a955d77 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Tue, 12 Sep 2023 09:52:13 +0800 Subject: [PATCH 6/7] `\\n` -> `\n` --- python/paddle/base/data_feed_desc.py | 158 +++++++++++++-------------- 1 file changed, 78 insertions(+), 80 deletions(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index e4eae3312a552..95f3357a4f02e 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -36,21 +36,21 @@ class DataFeedDesc: >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"\\n') - ... f.write('batch_size: 2\\n') - ... f.write('multi_slot_desc {\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "words"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "label"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') + ... f.write('name: "MultiSlotDataFeed"\n') + ... f.write('batch_size: 2\n') + ... f.write('multi_slot_desc {\n') + ... f.write(' slots {\n') + ... f.write(' name: "words"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') + ... f.write(' slots {\n') + ... f.write(' name: "label"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') @@ -67,12 +67,10 @@ class DataFeedDesc: >>> import paddle.base as base >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_batch_size(128) - >>> data_feed.set_dense_slots('wd') # The slot named 'wd' will be dense - >>> data_feed.set_use_slots('wd') # The slot named 'wd' will be used + >>> data_feed.set_dense_slots(['words']) # The slot named 'words' will be dense + >>> data_feed.set_use_slots(['words']) # The slot named 'words' will be used - Finally, the content can be dumped out for debugging purpose: - - .. code-block:: python + >>> # Finally, the content can be dumped out for debugging purpose: >>> print(data_feed.desc()) @@ -101,21 +99,21 @@ def set_batch_size(self, batch_size): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"\\n') - ... f.write('batch_size: 2\\n') - ... f.write('multi_slot_desc {\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "words"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "label"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') + ... f.write('name: "MultiSlotDataFeed"\n') + ... f.write('batch_size: 2\n') + ... f.write('multi_slot_desc {\n') + ... f.write(' slots {\n') + ... f.write(' name: "words"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') + ... f.write(' slots {\n') + ... f.write(' name: "label"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_batch_size(128) @@ -141,21 +139,21 @@ def set_dense_slots(self, dense_slots_name): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"\\n') - ... f.write('batch_size: 2\\n') - ... f.write('multi_slot_desc {\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "words"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "label"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') + ... f.write('name: "MultiSlotDataFeed"\n') + ... f.write('batch_size: 2\n') + ... f.write('multi_slot_desc {\n') + ... f.write(' slots {\n') + ... f.write(' name: "words"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') + ... f.write(' slots {\n') + ... f.write(' name: "label"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_dense_slots(['words']) @@ -187,21 +185,21 @@ def set_use_slots(self, use_slots_name): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"\\n') - ... f.write('batch_size: 2\\n') - ... f.write('multi_slot_desc {\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "words"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "label"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') + ... f.write('name: "MultiSlotDataFeed"\n') + ... f.write('batch_size: 2\n') + ... f.write('multi_slot_desc {\n') + ... f.write(' slots {\n') + ... f.write(' name: "words"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') + ... f.write(' slots {\n') + ... f.write(' name: "label"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> data_feed.set_use_slots(['words']) @@ -230,21 +228,21 @@ def desc(self): >>> import paddle.base as base >>> with open("data.proto", "w") as f: - ... f.write('name: "MultiSlotDataFeed"\\n') - ... f.write('batch_size: 2\\n') - ... f.write('multi_slot_desc {\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "words"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') - ... f.write(' slots {\\n') - ... f.write(' name: "label"\\n') - ... f.write(' type: "uint64"\\n') - ... f.write(' is_dense: false\\n') - ... f.write(' is_used: true\\n') - ... f.write(' }\\n') + ... f.write('name: "MultiSlotDataFeed"\n') + ... f.write('batch_size: 2\n') + ... f.write('multi_slot_desc {\n') + ... f.write(' slots {\n') + ... f.write(' name: "words"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') + ... f.write(' slots {\n') + ... f.write(' name: "label"\n') + ... f.write(' type: "uint64"\n') + ... f.write(' is_dense: false\n') + ... f.write(' is_used: true\n') + ... f.write(' }\n') ... f.write('}') >>> data_feed = base.DataFeedDesc('data.proto') >>> print(data_feed.desc()) From f760ac7a3bc0dec20a26205a98aa905b85501a79 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Tue, 12 Sep 2023 10:33:37 +0800 Subject: [PATCH 7/7] use raw string --- python/paddle/base/data_feed_desc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/base/data_feed_desc.py b/python/paddle/base/data_feed_desc.py index 95f3357a4f02e..8aa69890f1933 100644 --- a/python/paddle/base/data_feed_desc.py +++ b/python/paddle/base/data_feed_desc.py @@ -19,7 +19,7 @@ class DataFeedDesc: - """ + r""" :api_attr: Static Graph Datafeed descriptor, describing input training data format. This class is @@ -91,7 +91,7 @@ def __init__(self, proto_file): } def set_batch_size(self, batch_size): - """ + r""" Set :attr:`batch_size` in :ref:`api_base_DataFeedDesc` . :attr:`batch_size` can be changed during training. Examples: @@ -128,7 +128,7 @@ def set_batch_size(self, batch_size): self.proto_desc.batch_size = batch_size def set_dense_slots(self, dense_slots_name): - """ + r""" Set slots in :attr:`dense_slots_name` as dense slots. **Note: In default, all slots are sparse slots.** Features for a dense slot will be fed into a Tensor, while those for a @@ -175,7 +175,7 @@ def set_dense_slots(self, dense_slots_name): ].is_dense = True def set_use_slots(self, use_slots_name): - """ + r""" Set if a specific slot will be used for training. A dataset shall contain a lot of features, through this function one can select which ones will be used for a specific model. @@ -220,7 +220,7 @@ def set_use_slots(self, use_slots_name): ].is_used = True def desc(self): - """ + r""" Returns a protobuf message for this DataFeedDesc Examples: