google · ianspektor · Jun 23, 2023 · Jun 19, 2023 · Jun 19, 2023 · Jun 19, 2023
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -7,7 +7,8 @@
   },
   "search.exclude": {
     "**/.venv/": true,
-    "**/bazel-*/**": true
+    "**/bazel-*/**": true,
+    "**/reference/**/*.md": true
   },
   "files.watcherExclude": {
     "**/.venv/**": true,

diff --git a/docs/BUILD b/docs/BUILD
@@ -15,3 +15,8 @@ py_test(
         "//temporian",
     ],
 )
+
+filegroup(
+    name = "reference",
+    srcs = glob(["src/reference/**/*.md"]),
+)
diff --git a/docs/gen_ref_pages.py b/docs/gen_ref_pages.py
@@ -1,111 +1,58 @@
 """
 Generate the code reference pages.
 
-Source: https://mkdocstrings.github.io/recipes/
+This script traverses the markdown files under docs/src/reference and for each:
+- If the file is not empty it will appear in the docs as-is (after mkdocstrings
+    has filled in any identifiers inside it).
+- If the file is empty, it is interpreted as a placeholder for the top-level
+    symbol with its same name and its reference page is generated in its same
+    path. E.g., if an empty docs/src/reference/temporian/io/to_csv.md file
+    exists, the reference page for `temporian.to_csv` will be generated under
+    reference/temporian/io/to_csv/ in the docs.
+
+Related: https://mkdocstrings.github.io/recipes/
 """
 
 from pathlib import Path
-from typing import Set, Tuple
 
 import mkdocs_gen_files
 
 nav = mkdocs_gen_files.Nav()
 
-SRC_PATH = Path("temporian")
+REFERENCE = Path("docs/src/reference")
 
-# Stores symbol and path of each public API member
-members: Set[Tuple[str, Path]] = set()
+# Sort files by depth and name for alphabetical order and subdirs at the bottom
+paths = sorted(
+    REFERENCE.rglob("*.md"), key=lambda p: str(len(p.parts)) + str(p)
+)
 
-non_parsable_imports = []
+non_empty_files = []
 
-# We need to be able to parse other files to allow wildcard imports
-# Storing pair of (prefix, path) to parse in a stack
-files_to_parse = [(None, SRC_PATH / "__init__.py")]
+for path in paths:
+    path = Path(path)
+    ref_path = path.relative_to(REFERENCE)
+    nav_path = ref_path.with_suffix("")
+    full_ref_path = "reference" / ref_path
 
-while files_to_parse:
-    prefix, file = files_to_parse.pop()
+    nav[list(nav_path.parts)] = ref_path.as_posix()
 
-    with open(file, "r", encoding="utf8") as f:
-        lines = f.read().splitlines()
+    # If file is empty we assume it's a top-level symbol and auto
+    # generate the mkdocstring identifier for it
+    if path.stat().st_size == 0:
+        with mkdocs_gen_files.open(full_ref_path, "w") as fd:
+            ident = "temporian." + nav_path.name
+            fd.write(f"::: {ident}")
 
-    for line in lines:
-        words = line.split(" ")
+    else:
+        non_empty_files.append(str(ref_path))
 
-        # It is an import statement
-        if words[0] == "from":
-            # Remove trailing "as <name>" if it exists and save symbol's name
-            symbol = None
-            if words[-2] == "as":
-                # If symbol was renamed to a private name, skip it
-                if words[-1].startswith("_"):
-                    continue
-
-                symbol = words[-1]
-                words = words[:-2]
-
-            # `words` is now in the form "from module.submodule import symbol"
-            if words[-2] == "import":
-                name = words[-1]
-
-                # We only allow wildcard imports from modules explicitly named
-                # api_symbols to prevent unwanted names in the public API
-                if name == "*":
-                    module_path = Path(words[1].replace(".", "/")).with_suffix(
-                        ".py"
-                    )
-                    if module_path.stem == "api_symbols":
-                        new_prefix = (
-                            (prefix + ".") if prefix else ""
-                        ) + module_path.parent.name
-                        files_to_parse.append((new_prefix, module_path))
-                        continue
-
-                    non_parsable_imports.append(line)
-                    continue
-
-                # If symbol wasn't renamed, use its imported name
-                if symbol is None:
-                    symbol = name
-
-                path = Path(words[1].replace(".", "/")) / name
-
-                if prefix:
-                    symbol = prefix + "." + symbol
-
-                members.add((symbol, path))
-
-            # It is a multi-symbol import statement, error will be raised below
-            else:
-                non_parsable_imports.append(line)
-
-if non_parsable_imports:
-    raise RuntimeError(
-        "`gen_ref_pages` failed to parse the following import statements in"
-        f" the top-level __init__.py file: {non_parsable_imports}. Import"
-        " statements in the top-level module must import a single symbol each,"
-        " in the form `from <module> import <symbol>`, `from <module> import"
-        " <symbol> as <name>`, or `from <module> import *`."
-    )
-
-nav["temporian"] = "index.md"
-
-for symbol, path in sorted(members):
-    symbol_path = Path(symbol.replace(".", "/"))
-    symbol_name = symbol_path.name
-    src_path = SRC_PATH / symbol_name
-
-    doc_path = SRC_PATH / symbol_path
-    parts = list(doc_path.parts)
-    doc_path = doc_path.with_suffix(".md")
-    full_doc_path = Path("reference", doc_path)
-
-    nav[parts] = doc_path.as_posix()
-
-    with mkdocs_gen_files.open(full_doc_path, "w") as fd:
-        identifier = ".".join(list(src_path.parts))
-        print("::: " + identifier, file=fd)
-
-    mkdocs_gen_files.set_edit_path(full_doc_path, path)
+print(
+    (
+        "These md files in docs/src/reference are not empty and will be"
+        " rendered my mkdocs as-is:"
+    ),
+    non_empty_files,
+)
 
 with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as nav_file:
     nav_file.writelines(nav.build_literate_nav())
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -59,7 +59,7 @@ plugins:
       scripts:
         # Generate the index page from the README.md.
         - gen_index.py
-        # Generate the reference pages from docstrings.
+        # Generate the ref pages and navigation.
         - gen_ref_pages.py
   - literate-nav:
       nav_file: SUMMARY.md

diff --git a/docs/src/reference/index.md b/docs/src/reference/index.md
@@ -18,7 +18,7 @@ Check the index on the left for a more detailed description of any symbol.
 | ----------------------------------------- | -------------------------------------------------------------------------------------- |
 | [`tp.evaluate()`][temporian.evaluate]     | Evaluates [`Nodes`][temporian.Node] on [`EventSets`][temporian.EventSet].              |
 | [`tp.plot()`][temporian.plot]             | Plots [`EventSets`][temporian.EventSet].                                               |
-| [`tp.event_set()`][temporian.event_set]   | Creates an [`EventSet`][temporian.EventSet] from arrays (list, numpy, pandas).         |
+| [`tp.event_set()`][temporian.event_set]   | Creates an [`EventSet`][temporian.EventSet] from arrays (lists, NumPy, Pandas Series.) |
 | [`tp.input_node()`][temporian.input_node] | Creates an input [`Node`][temporian.Node], that can be used to feed data into a graph. |
 
 ## Input/output
@@ -60,11 +60,11 @@ Check the index on the left for a more detailed description of any symbol.
 
 ### Binary operators
 
-| Symbols                                                                                                                                                                                                                                           | Description                                                                   |
-| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
-| [`tp.add()`][temporian.add] [`tp.subtract()`][temporian.subtract] [`tp.multiply()`][temporian.multiply] [`tp.divide()`][temporian.divide] [`tp.floordiv()`][temporian.floordiv] [`tp.modulo()`][temporian.modulo] [`tp.power()`][temporian.power] | Compute an arithmetic binary operation between two [`Nodes`][temporian.Node]. |
-| [`tp.equal()`][temporian.equal] [`tp.not_equal()`][temporian.not_equal] [`tp.greater()`][temporian.greater] [`tp.greater_equal()`][temporian.greater_equal] [`tp.less()`][temporian.less] [`tp.less_equal()`][temporian.less_equal]               | Compute a relational binary operator between two [`Nodes`][temporian.Node].   |
-| [`tp.logical_and()`][temporian.logical_and] [`tp.logical_or()`][temporian.logical_or] [`tp.logical_xor()`][temporian.logical_xor]                                                                                                                 | Compute a logical binary operation between two [`Nodes`][temporian.Node].     |
+| Symbols                                                                                                                                                                                                                                           | Description                                                                                                         |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- |
+| [`tp.add()`][temporian.add] [`tp.subtract()`][temporian.subtract] [`tp.multiply()`][temporian.multiply] [`tp.divide()`][temporian.divide] [`tp.floordiv()`][temporian.floordiv] [`tp.modulo()`][temporian.modulo] [`tp.power()`][temporian.power] | Compute an arithmetic binary operation between two [`Nodes`][temporian.Node]. Aliases for `+`, `-`, `\*`, `/`, etc. |
+| [`tp.equal()`][temporian.equal] [`tp.not_equal()`][temporian.not_equal] [`tp.greater()`][temporian.greater] [`tp.greater_equal()`][temporian.greater_equal] [`tp.less()`][temporian.less] [`tp.less_equal()`][temporian.less_equal]               | Compute a relational binary operator between two [`Nodes`][temporian.Node].                                         |
+| [`tp.logical_and()`][temporian.logical_and] [`tp.logical_or()`][temporian.logical_or] [`tp.logical_xor()`][temporian.logical_xor]                                                                                                                 | Compute a logical binary operation between two [`Nodes`][temporian.Node].                                           |
 
 ### Unary operators
 

diff --git a/docs/src/reference/temporian/EventSet.md b/docs/src/reference/temporian/EventSet.md
diff --git a/docs/src/reference/temporian/Node.md b/docs/src/reference/temporian/Node.md
diff --git a/docs/src/reference/temporian/Schema.md b/docs/src/reference/temporian/Schema.md
diff --git a/docs/src/reference/temporian/dtypes/bool_.md b/docs/src/reference/temporian/dtypes/bool_.md
diff --git a/docs/src/reference/temporian/dtypes/float32.md b/docs/src/reference/temporian/dtypes/float32.md
diff --git a/docs/src/reference/temporian/dtypes/float64.md b/docs/src/reference/temporian/dtypes/float64.md
diff --git a/docs/src/reference/temporian/dtypes/int32.md b/docs/src/reference/temporian/dtypes/int32.md
diff --git a/docs/src/reference/temporian/dtypes/int64.md b/docs/src/reference/temporian/dtypes/int64.md
diff --git a/docs/src/reference/temporian/dtypes/str_.md b/docs/src/reference/temporian/dtypes/str_.md
diff --git a/docs/src/reference/temporian/duration.md b/docs/src/reference/temporian/duration.md
diff --git a/docs/src/reference/temporian/evaluate.md b/docs/src/reference/temporian/evaluate.md
diff --git a/docs/src/reference/temporian/event_set.md b/docs/src/reference/temporian/event_set.md
diff --git a/docs/src/reference/temporian/input_node.md b/docs/src/reference/temporian/input_node.md
diff --git a/docs/src/reference/temporian/io/from_csv.md b/docs/src/reference/temporian/io/from_csv.md
diff --git a/docs/src/reference/temporian/io/from_pandas.md b/docs/src/reference/temporian/io/from_pandas.md
diff --git a/docs/src/reference/temporian/io/to_csv.md b/docs/src/reference/temporian/io/to_csv.md
diff --git a/docs/src/reference/temporian/io/to_pandas.md b/docs/src/reference/temporian/io/to_pandas.md
diff --git a/docs/src/reference/temporian/operators/add_index.md b/docs/src/reference/temporian/operators/add_index.md
diff --git a/docs/src/reference/temporian/operators/begin.md b/docs/src/reference/temporian/operators/begin.md
diff --git a/docs/src/reference/temporian/operators/binary/add.md b/docs/src/reference/temporian/operators/binary/add.md
diff --git a/docs/src/reference/temporian/operators/binary/divide.md b/docs/src/reference/temporian/operators/binary/divide.md
diff --git a/docs/src/reference/temporian/operators/binary/equal.md b/docs/src/reference/temporian/operators/binary/equal.md
diff --git a/docs/src/reference/temporian/operators/binary/floordiv.md b/docs/src/reference/temporian/operators/binary/floordiv.md
diff --git a/docs/src/reference/temporian/operators/binary/greater.md b/docs/src/reference/temporian/operators/binary/greater.md
diff --git a/docs/src/reference/temporian/operators/binary/greater_equal.md b/docs/src/reference/temporian/operators/binary/greater_equal.md
diff --git a/docs/src/reference/temporian/operators/binary/less.md b/docs/src/reference/temporian/operators/binary/less.md
diff --git a/docs/src/reference/temporian/operators/binary/less_equal.md b/docs/src/reference/temporian/operators/binary/less_equal.md
diff --git a/docs/src/reference/temporian/operators/binary/logical_and.md b/docs/src/reference/temporian/operators/binary/logical_and.md
diff --git a/docs/src/reference/temporian/operators/binary/logical_or.md b/docs/src/reference/temporian/operators/binary/logical_or.md
diff --git a/docs/src/reference/temporian/operators/binary/logical_xor.md b/docs/src/reference/temporian/operators/binary/logical_xor.md
diff --git a/docs/src/reference/temporian/operators/binary/modulo.md b/docs/src/reference/temporian/operators/binary/modulo.md
diff --git a/docs/src/reference/temporian/operators/binary/multiply.md b/docs/src/reference/temporian/operators/binary/multiply.md
diff --git a/docs/src/reference/temporian/operators/binary/not_equal.md b/docs/src/reference/temporian/operators/binary/not_equal.md
diff --git a/docs/src/reference/temporian/operators/binary/power.md b/docs/src/reference/temporian/operators/binary/power.md
diff --git a/docs/src/reference/temporian/operators/binary/subtract.md b/docs/src/reference/temporian/operators/binary/subtract.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_day_of_month.md b/docs/src/reference/temporian/operators/calendar/calendar_day_of_month.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_day_of_week.md b/docs/src/reference/temporian/operators/calendar/calendar_day_of_week.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_day_of_year.md b/docs/src/reference/temporian/operators/calendar/calendar_day_of_year.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_hour.md b/docs/src/reference/temporian/operators/calendar/calendar_hour.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_iso_week.md b/docs/src/reference/temporian/operators/calendar/calendar_iso_week.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_minute.md b/docs/src/reference/temporian/operators/calendar/calendar_minute.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_month.md b/docs/src/reference/temporian/operators/calendar/calendar_month.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_second.md b/docs/src/reference/temporian/operators/calendar/calendar_second.md
diff --git a/docs/src/reference/temporian/operators/calendar/calendar_year.md b/docs/src/reference/temporian/operators/calendar/calendar_year.md
diff --git a/docs/src/reference/temporian/operators/cast.md b/docs/src/reference/temporian/operators/cast.md
diff --git a/docs/src/reference/temporian/operators/drop_index.md b/docs/src/reference/temporian/operators/drop_index.md
diff --git a/docs/src/reference/temporian/operators/end.md b/docs/src/reference/temporian/operators/end.md
diff --git a/docs/src/reference/temporian/operators/filter.md b/docs/src/reference/temporian/operators/filter.md
diff --git a/docs/src/reference/temporian/operators/glue.md b/docs/src/reference/temporian/operators/glue.md
diff --git a/docs/src/reference/temporian/operators/lag.md b/docs/src/reference/temporian/operators/lag.md
diff --git a/docs/src/reference/temporian/operators/leak.md b/docs/src/reference/temporian/operators/leak.md
diff --git a/docs/src/reference/temporian/operators/prefix.md b/docs/src/reference/temporian/operators/prefix.md
diff --git a/docs/src/reference/temporian/operators/propagate.md b/docs/src/reference/temporian/operators/propagate.md
diff --git a/docs/src/reference/temporian/operators/rename.md b/docs/src/reference/temporian/operators/rename.md
diff --git a/docs/src/reference/temporian/operators/resample.md b/docs/src/reference/temporian/operators/resample.md
diff --git a/docs/src/reference/temporian/operators/scalar/add_scalar.md b/docs/src/reference/temporian/operators/scalar/add_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/divide_scalar.md b/docs/src/reference/temporian/operators/scalar/divide_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/equal_scalar.md b/docs/src/reference/temporian/operators/scalar/equal_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/floordiv_scalar.md b/docs/src/reference/temporian/operators/scalar/floordiv_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/greater_equal_scalar.md b/docs/src/reference/temporian/operators/scalar/greater_equal_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/greater_scalar.md b/docs/src/reference/temporian/operators/scalar/greater_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/less_equal_scalar.md b/docs/src/reference/temporian/operators/scalar/less_equal_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/less_scalar.md b/docs/src/reference/temporian/operators/scalar/less_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/modulo_scalar.md b/docs/src/reference/temporian/operators/scalar/modulo_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/multiply_scalar.md b/docs/src/reference/temporian/operators/scalar/multiply_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/not_equal_scalar.md b/docs/src/reference/temporian/operators/scalar/not_equal_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/power_scalar.md b/docs/src/reference/temporian/operators/scalar/power_scalar.md
diff --git a/docs/src/reference/temporian/operators/scalar/subtract_scalar.md b/docs/src/reference/temporian/operators/scalar/subtract_scalar.md
diff --git a/docs/src/reference/temporian/operators/select.md b/docs/src/reference/temporian/operators/select.md
diff --git a/docs/src/reference/temporian/operators/set_index.md b/docs/src/reference/temporian/operators/set_index.md
diff --git a/docs/src/reference/temporian/operators/since_last.md b/docs/src/reference/temporian/operators/since_last.md
diff --git a/docs/src/reference/temporian/operators/tick.md b/docs/src/reference/temporian/operators/tick.md
diff --git a/docs/src/reference/temporian/operators/unary/abs.md b/docs/src/reference/temporian/operators/unary/abs.md
diff --git a/docs/src/reference/temporian/operators/unary/invert.md b/docs/src/reference/temporian/operators/unary/invert.md
diff --git a/docs/src/reference/temporian/operators/unary/isnan.md b/docs/src/reference/temporian/operators/unary/isnan.md
diff --git a/docs/src/reference/temporian/operators/unary/log.md b/docs/src/reference/temporian/operators/unary/log.md
diff --git a/docs/src/reference/temporian/operators/unary/notnan.md b/docs/src/reference/temporian/operators/unary/notnan.md
diff --git a/docs/src/reference/temporian/operators/unique_timestamps.md b/docs/src/reference/temporian/operators/unique_timestamps.md
diff --git a/docs/src/reference/temporian/operators/window/cumsum.md b/docs/src/reference/temporian/operators/window/cumsum.md
diff --git a/docs/src/reference/temporian/operators/window/moving_count.md b/docs/src/reference/temporian/operators/window/moving_count.md
diff --git a/docs/src/reference/temporian/operators/window/moving_max.md b/docs/src/reference/temporian/operators/window/moving_max.md
diff --git a/docs/src/reference/temporian/operators/window/moving_min.md b/docs/src/reference/temporian/operators/window/moving_min.md
diff --git a/docs/src/reference/temporian/operators/window/moving_standard_deviation.md b/docs/src/reference/temporian/operators/window/moving_standard_deviation.md
diff --git a/docs/src/reference/temporian/operators/window/moving_sum.md b/docs/src/reference/temporian/operators/window/moving_sum.md
diff --git a/docs/src/reference/temporian/operators/window/simple_moving_average.md b/docs/src/reference/temporian/operators/window/simple_moving_average.md
diff --git a/docs/src/reference/temporian/plot.md b/docs/src/reference/temporian/plot.md
diff --git a/docs/src/reference/temporian/serialization/load.md b/docs/src/reference/temporian/serialization/load.md
diff --git a/docs/src/reference/temporian/serialization/save.md b/docs/src/reference/temporian/serialization/save.md
diff --git a/temporian/BUILD b/temporian/BUILD
@@ -14,16 +14,37 @@ py_library(
         "//temporian/core:evaluation",
         "//temporian/core:graph",
         "//temporian/core:operator_lib",
+        "//temporian/core:serialization",
+        "//temporian/core/data:dtype",
         "//temporian/core/data:duration",
         "//temporian/core/data:node",
         "//temporian/core/data:schema",
-        "//temporian/core/data/dtypes:api_symbols",
-        "//temporian/core/operators:api_symbols",
+        "//temporian/core/operators:add_index",
         "//temporian/core/operators:base",
-        "//temporian/core/serialization:api_symbols",
+        "//temporian/core/operators:begin",
+        "//temporian/core/operators:cast",
+        "//temporian/core/operators:drop_index",
+        "//temporian/core/operators:end",
+        "//temporian/core/operators:filter",
+        "//temporian/core/operators:glue",
+        "//temporian/core/operators:lag",
+        "//temporian/core/operators:leak",
+        "//temporian/core/operators:prefix",
+        "//temporian/core/operators:propagate",
+        "//temporian/core/operators:rename",
+        "//temporian/core/operators:resample",
+        "//temporian/core/operators:select",
+        "//temporian/core/operators:since_last",
+        "//temporian/core/operators:tick",
+        "//temporian/core/operators:unary",
+        "//temporian/core/operators:unique_timestamps",
+        "//temporian/core/operators/binary",
+        "//temporian/core/operators/calendar",
+        "//temporian/core/operators/scalar",
+        "//temporian/core/operators/window",
         "//temporian/implementation/numpy/data:event_set",
         "//temporian/implementation/numpy/data:io",
         "//temporian/implementation/numpy/data:plotter",
-        "//temporian/io:api_symbols",
+        "//temporian/io",
     ],
 )