Split nn layers (PaddlePaddle#892)

* split layers,test=develop

* split nn layers

* remove nn header, test=develop

doc/fluid/api/gen_doc.py

@@ -16,63 +16,48 @@
 import argparse
 import sys
 import types
+import os
+import contextlib
 
 import paddle.fluid as fluid
 
-
 def parse_arg():
     parser = argparse.ArgumentParser()
     parser.add_argument('--submodules', nargs="*")
     parser.add_argument(
-        '--module', type=str, help='Generate the documentation of which module')
+        '--module_name', type=str, help='Generate the documentation of which module')
     parser.add_argument(
         '--module_prefix', type=str, help='Generate the prefix of module')
+    parser.add_argument(
+        '--output', type=str, help='Output file or output directory for output rst')
+    parser.add_argument(
+        '--to_multiple_files', type=bool, default=False, help='Whether to separate to multiple files')
     return parser.parse_args()
 
-
-class DocGenerator(object):
-    def __init__(self, module_name=None, module_prefix=None, stream=sys.stdout):
-        if module_name == "":
-            module_name = None
-
-        if module_prefix == "":
-            module_prefix = None
-
-        self.stream = stream
-        if module_name is None:
-            self.module_name = "fluid"
-        else:
-            self.module_name = "fluid." + module_name
-        if module_name is None:
-            self.module = fluid
-        else: 
-            self.module = fluid
-            for each_module_name in module_name.split('.'): 
-                if not hasattr(self.module, each_module_name):
-                    raise ValueError("Cannot find fluid.{0}".format(module_name))
-                else:
-                    self.module = getattr(self.module, each_module_name)
-
-        if module_prefix is None:
-            self.module_prefix = self.module_name 
+    def print_item(self, name):
+        item = getattr(self.module, name, None)
+        if item is None:
+            return
+        if isinstance(item, types.TypeType):
+            self.print_class(name)
+        elif isinstance(item, types.FunctionType):
+            self.print_method(name)
         else:
-            self.module_prefix = "fluid." + module_prefix
-
-        self.stream.write('''..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
-    !DO NOT EDIT THIS FILE MANUALLY!
-
-''')            
+            pass
 
-        header_name = self.module_name
-        if module_prefix is not None:
-            prefix_len = len(self.module_prefix)
-            assert self.module_prefix == self.module_name[0:prefix_len],    \
-                "module_prefix must be prefix of module_name"
-            diff_name = self.module_name[prefix_len+1:]
-            if diff_name != "":
-                header_name = diff_name
-
-        self._print_header_(header_name, dot='=', is_title=True)
+class DocGenerator(object):
+    def __init__(self, module_name=None, module_prefix=None): 
+        self.module_name = module_name
+        self.module_prefix = module_prefix
+        self.stream = None
+
+    @contextlib.contextmanager
+    def guard(self, filename):
+        assert self.stream is None, "stream must be None"
+        self.stream = open(filename, 'w') 
+        yield
+        self.stream.close()
+        self.stream = None
 
     def print_submodule(self, submodule_name):
         submodule = getattr(self.module, submodule_name)
@@ -118,6 +103,12 @@ def print_method(self, name):
 
 '''.format(self.module_prefix, name))
 
+    def print_header_reminder(self):
+        self.stream.write('''..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+''')
+
     def _print_header_(self, name, dot, is_title):
         dot_line = dot * len(name)
         if is_title:
@@ -133,15 +124,66 @@ def _print_ref_(self, name):
         self.stream.write(".. _api_{0}_{1}:\n\n".format("_".join(
             self.module_prefix.split(".")), name))
 
+def generate_doc(module_name, module_prefix, output, to_multiple_files):
+    if module_name == "":
+        module_name = None
+
+    if module_prefix == "":
+        module_prefix = None
+
+    gen = DocGenerator()
+
+    if module_name is None:
+        gen.module = fluid
+        gen.module_name = 'fluid'
+    else:
+        gen.module = fluid
+        for each_module_name in module_name.split('.'):
+            if not hasattr(gen.module, each_module_name):
+                raise ValueError("Cannot find fluid.{0}".format(module_name))
+            else:
+                gen.module = getattr(gen.module, each_module_name)
+
+        gen.module_name = "fluid." + module_name
+
+    if module_prefix is None:
+        gen.module_prefix = gen.module_name
+    else:
+        gen.module_prefix = "fluid." + module_prefix
+
+    dirname = output if to_multiple_files else os.path.dirname(output) 
+    if len(dirname) > 0 and (not os.path.exists(dirname) or not os.path.isdir(dirname)): 
+        os.makedirs(dirname)
+
+    if not to_multiple_files:
+        header_name = gen.module_name
+        if module_prefix is not None:
+            prefix_len = len(gen.module_prefix)
+            assert gen.module_prefix == gen.module_name[0:prefix_len],    \
+                "module_prefix must be prefix of module_name"
+            diff_name = gen.module_name[prefix_len+1:]
+            if diff_name != "":
+                header_name = diff_name
+    else:
+        header_name = None
+
+    if not to_multiple_files:
+        with gen.guard(output):
+            gen.print_header_reminder()
+            gen._print_header_(header_name, dot='=', is_title=True)
+            gen.print_current_module()
+    else:
+        apis = sorted(gen.module.__all__,key=str.lower)
+        for api in apis:
+            header_name = api
+            with gen.guard(os.path.join(output, api + '.rst')):
+                gen.print_header_reminder()
+                gen.print_item(api)
+
 
 def main():
     args = parse_arg()
-    gen = DocGenerator(args.module, args.module_prefix)
-    if args.submodules is None:
-        gen.print_current_module()
-    else:
-        for submodule_name in args.submodules:
-            gen.print_submodule(submodule_name)
+    generate_doc(args.module_name, args.module_prefix, args.output, args.to_multiple_files)
 
 
 if __name__ == '__main__':

doc/fluid/api/gen_doc.sh

@@ -1,17 +1,24 @@
 #!/bin/bash
 
-mkdir -p layers
+for module in nn
+do
+  python gen_doc.py --module_name layers.${module} --module_prefix layers --output layers/${module} --to_multiple_files True
+done
 
-for module in control_flow io nn ops tensor learning_rate_scheduler detection metric_op
+for module in control_flow io ops tensor learning_rate_scheduler detection metric_op
 do
-  python gen_doc.py --module layers.${module} --module_prefix layers > layers/${module}.rst
+  python gen_doc.py --module_name layers.${module} --module_prefix layers --output layers/${module}.rst
 done 
 
 for module in data_feeder dataset clip metrics executor initializer io nets optimizer profiler regularizer transpiler recordio_writer backward average profiler unique_name
 do
-  python gen_doc.py --module ${module} --module_prefix ${module} > ${module}.rst
+  python gen_doc.py --module_name ${module} --module_prefix ${module} --output ${module}.rst
 done
 
-python gen_doc.py --module "" --module_prefix "" > fluid.rst
+python gen_doc.py --module_name "" --module_prefix "" --output fluid.rst
+
+python gen_module_index.py layers.nn nn
+python gen_module_index.py layers fluid.layers
 
 python gen_index.py
+

doc/fluid/api/gen_module_index.py

@@ -0,0 +1,39 @@
+import glob 
+import sys
+import os
+
+def print_module_index(module, header):
+    modules = module.split('.')
+    if len(modules) > 1:
+        os.chdir('/'.join(modules[0:-1]))
+        pattern = modules[-1] + '/*.rst'
+        stream = open(modules[-1] + '.rst', 'w')
+    else:
+        pattern = modules[0] + '/*.rst'
+        stream = open(modules[0] + '.rst', 'w')
+
+    stream.write('=' * len(header) + '\n')
+    stream.write(header + '\n')
+    stream.write('=' * len(header) + '\n')
+    stream.write('''
+..  toctree::
+    :maxdepth: 1
+
+''')
+
+    blank_num = 4 
+    files = sorted(glob.glob(pattern), key=str.lower)
+
+    for f in files: 
+        stream.write(' ' * blank_num)
+        stream.write(f)
+        stream.write('\n')
+
+    stream.close()
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print('Usage: python gen_module_index.py [module_name] [header_name]')
+        sys.exit(-1)
+
+    print_module_index(sys.argv[1], sys.argv[2])

doc/fluid/api/layers.rst

@@ -7,7 +7,6 @@ fluid.layers
 
     layers/control_flow.rst
     layers/detection.rst
-    layers/device.rst
     layers/io.rst
     layers/learning_rate_scheduler.rst
     layers/metric_op.rst