From 68e301ceb398e629b6323e060ac8a7a9721e08b1 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Mon, 8 Jan 2024 16:15:39 -0500 Subject: [PATCH 1/4] add custom tag for conditional jekyll includes --- docs/_plugins/include_relative_if_exists.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/_plugins/include_relative_if_exists.rb diff --git a/docs/_plugins/include_relative_if_exists.rb b/docs/_plugins/include_relative_if_exists.rb new file mode 100644 index 000000000000..e5d5b2141e2c --- /dev/null +++ b/docs/_plugins/include_relative_if_exists.rb @@ -0,0 +1,15 @@ +module Jekyll + class IncludeRelativeIfExistsTag < Tags::IncludeRelativeTag + def render(context) + super + rescue IOError => e + puts e + "placeholder for missing file: `#{@file}`" + end + end +end + +Liquid::Template.register_tag( + 'include_relative_if_exists', + Jekyll::IncludeRelativeIfExistsTag, +) From 81a40944b952a6997582f5e59d341c908c37b5b5 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Tue, 9 Jan 2024 14:16:09 -0500 Subject: [PATCH 2/4] add include_api_gen tag --- docs/_plugins/conditonal_includes.rb | 55 +++++++++++++++++++++ docs/_plugins/include_relative_if_exists.rb | 15 ------ 2 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 docs/_plugins/conditonal_includes.rb delete mode 100644 docs/_plugins/include_relative_if_exists.rb diff --git a/docs/_plugins/conditonal_includes.rb b/docs/_plugins/conditonal_includes.rb new file mode 100644 index 000000000000..39280cbe5bec --- /dev/null +++ b/docs/_plugins/conditonal_includes.rb @@ -0,0 +1,55 @@ +module Jekyll + # Tag for including a file if it exists. + class IncludeRelativeIfExistsTag < Tags::IncludeRelativeTag + def render(context) + super + rescue IOError + "" + end + end + + # Tag for including files generated as part of the various language APIs. + # If a SKIP_ flag is set, tolerate missing files. If not, raise an error. + class IncludeApiGenTag < Tags::IncludeRelativeTag + @@displayed_warning = false + + def render(context) + super + rescue IOError => e + skip_flags = [ + 'SKIP_API', + 'SKIP_SCALADOC', + 'SKIP_PYTHONDOC', + 'SKIP_RDOC', + 'SKIP_SQLDOC', + ] + set_flags = skip_flags.select { |flag| ENV[flag] } + # A more sophisticated approach would be to accept a tag parameter + # specifying the relevant API so we tolerate missing files only for + # APIs that are explicitly skipped. But this is unnecessary for now. + # Instead, we simply tolerate missing files if _any_ skip flag is set. + if set_flags.any? then + set_flags_string = set_flags.join(', ') + if !@@displayed_warning then + STDERR.puts "Warning: Tolerating missing API files because the " \ + "following skip flags are set: #{set_flags_string}" + @@displayed_warning = true + end + # "skip flags set: `#{set_flags_string}`; " \ + "placeholder for missing API include: `#{@file}`" + else + raise e + end + end + end +end + +Liquid::Template.register_tag( + 'include_relative_if_exists', + Jekyll::IncludeRelativeIfExistsTag, +) + +Liquid::Template.register_tag( + 'include_api_gen', + Jekyll::IncludeApiGenTag, +) diff --git a/docs/_plugins/include_relative_if_exists.rb b/docs/_plugins/include_relative_if_exists.rb deleted file mode 100644 index e5d5b2141e2c..000000000000 --- a/docs/_plugins/include_relative_if_exists.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Jekyll - class IncludeRelativeIfExistsTag < Tags::IncludeRelativeTag - def render(context) - super - rescue IOError => e - puts e - "placeholder for missing file: `#{@file}`" - end - end -end - -Liquid::Template.register_tag( - 'include_relative_if_exists', - Jekyll::IncludeRelativeIfExistsTag, -) From 14294e343186a0d7987828c359a121cdca8ab9f2 Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Tue, 9 Jan 2024 14:17:06 -0500 Subject: [PATCH 3/4] migrate sql-ref-functions-builtin to include_api_gen --- docs/sql-ref-functions-builtin.md | 180 +++++++----------------------- 1 file changed, 40 insertions(+), 140 deletions(-) diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md index 0ff1432fabf8..c5f4e44dec0d 100644 --- a/docs/sql-ref-functions-builtin.md +++ b/docs/sql-ref-functions-builtin.md @@ -17,202 +17,102 @@ license: | limitations under the License. --- -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-agg-funcs-table.html' %} ### Aggregate Functions -{% include_relative generated-agg-funcs-table.html %} +{% include_api_gen generated-agg-funcs-table.html %} #### Examples -{% include_relative generated-agg-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-agg-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-window-funcs-table.html' %} ### Window Functions -{% include_relative generated-window-funcs-table.html %} +{% include_api_gen generated-window-funcs-table.html %} #### Examples -{% include_relative generated-window-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-window-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-array-funcs-table.html' %} ### Array Functions -{% include_relative generated-array-funcs-table.html %} +{% include_api_gen generated-array-funcs-table.html %} #### Examples -{% include_relative generated-array-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-array-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-collection-funcs-table.html' %} ### Collection Functions -{% include_relative generated-collection-funcs-table.html %} +{% include_api_gen generated-collection-funcs-table.html %} #### Examples -{% include_relative generated-collection-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-collection-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-struct-funcs-table.html' %} ### STRUCT Functions -{% include_relative generated-struct-funcs-table.html %} +{% include_api_gen generated-struct-funcs-table.html %} #### Examples -{% include_relative generated-struct-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-struct-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-map-funcs-table.html' %} ### Map Functions -{% include_relative generated-map-funcs-table.html %} +{% include_api_gen generated-map-funcs-table.html %} #### Examples -{% include_relative generated-map-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-map-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-datetime-funcs-table.html' %} ### Date and Timestamp Functions -{% include_relative generated-datetime-funcs-table.html %} +{% include_api_gen generated-datetime-funcs-table.html %} #### Examples -{% include_relative generated-datetime-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-datetime-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-math-funcs-table.html' %} ### Mathematical Functions -{% include_relative generated-math-funcs-table.html %} +{% include_api_gen generated-math-funcs-table.html %} #### Examples -{% include_relative generated-math-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-math-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-string-funcs-table.html' %} ### String Functions -{% include_relative generated-string-funcs-table.html %} +{% include_api_gen generated-string-funcs-table.html %} #### Examples -{% include_relative generated-string-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-string-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-conditional-funcs-table.html' %} ### Conditional Functions -{% include_relative generated-conditional-funcs-table.html %} +{% include_api_gen generated-conditional-funcs-table.html %} #### Examples -{% include_relative generated-conditional-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-conditional-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-hash-funcs-table.html' %} ### Hash Functions -{% include_relative generated-hash-funcs-table.html %} +{% include_api_gen generated-hash-funcs-table.html %} #### Examples -{% include_relative generated-hash-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-hash-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-csv-funcs-table.html' %} ### CSV Functions -{% include_relative generated-csv-funcs-table.html %} +{% include_api_gen generated-csv-funcs-table.html %} #### Examples -{% include_relative generated-csv-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-csv-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-json-funcs-table.html' %} ### JSON Functions -{% include_relative generated-json-funcs-table.html %} +{% include_api_gen generated-json-funcs-table.html %} #### Examples -{% include_relative generated-json-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-json-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-xml-funcs-table.html' %} ### XML Functions -{% include_relative generated-xml-funcs-table.html %} +{% include_api_gen generated-xml-funcs-table.html %} #### Examples -{% include_relative generated-xml-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-xml-funcs-examples.html %} -{% for static_file in site.static_files %} -{% if static_file.name == 'generated-url-funcs-table.html' %} ### URL Functions -{% include_relative generated-url-funcs-table.html %} +{% include_api_gen generated-url-funcs-table.html %} #### Examples -{% include_relative generated-url-funcs-examples.html %} -{% break %} -{% endif %} -{% endfor %} +{% include_api_gen generated-url-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-bitwise-funcs-table.html' %} ### Bitwise Functions -{% include_relative generated-bitwise-funcs-table.html %} +{% include_api_gen generated-bitwise-funcs-table.html %} #### Examples -{% include_relative generated-bitwise-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-bitwise-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-conversion-funcs-table.html' %} ### Conversion Functions -{% include_relative generated-conversion-funcs-table.html %} +{% include_api_gen generated-conversion-funcs-table.html %} #### Examples -{% include_relative generated-conversion-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-conversion-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-predicate-funcs-table.html' %} ### Predicate Functions -{% include_relative generated-predicate-funcs-table.html %} +{% include_api_gen generated-predicate-funcs-table.html %} #### Examples -{% include_relative generated-predicate-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-predicate-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-misc-funcs-table.html' %} ### Misc Functions -{% include_relative generated-misc-funcs-table.html %} +{% include_api_gen generated-misc-funcs-table.html %} #### Examples -{% include_relative generated-misc-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-misc-funcs-examples.html %} -{% for static_file in site.static_files %} - {% if static_file.name == 'generated-generator-funcs-table.html' %} ### Generator Functions -{% include_relative generated-generator-funcs-table.html %} +{% include_api_gen generated-generator-funcs-table.html %} #### Examples -{% include_relative generated-generator-funcs-examples.html %} - {% break %} - {% endif %} -{% endfor %} +{% include_api_gen generated-generator-funcs-examples.html %} From 5dd01737df0a695b398e01353e7c1da86d04ff3a Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Tue, 9 Jan 2024 14:30:38 -0500 Subject: [PATCH 4/4] fix broken link --- docs/sql-ref-functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md index cc9edd61f41e..b4891fe72eb3 100644 --- a/docs/sql-ref-functions.md +++ b/docs/sql-ref-functions.md @@ -20,7 +20,7 @@ license: | --- Spark SQL provides two function features to meet a wide range of user needs: built-in functions and user-defined functions (UDFs). -Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task. +Built-in functions are commonly used routines that Spark SQL predefines and a complete list of the functions can be found in the [Built-in Functions](api/sql/index.html) API document. UDFs allow users to define their own functions when the system’s built-in functions are not enough to perform the desired task. ### Built-in Functions