Docs upcoming v26.0.0 (#34136)

Signed-off-by: Petros Angelatos <petrosagg@gmail.com>
Co-authored-by: Jesse Luehrs <doy@materialize.com>
Co-authored-by: doy-materialize <110416385+doy-materialize@users.noreply.github.com>
Co-authored-by: alex-hunt-materialize <86254996+alex-hunt-materialize@users.noreply.github.com>
Co-authored-by: Pranshu Maheshwari <maheshwarip@users.noreply.github.com>
Co-authored-by: Sang Jun Bak <jun@materialize.com>
Co-authored-by: Petros Angelatos <petrosagg@gmail.com>
Co-authored-by: Justin Bradfield <justin@materialize.com>
Co-authored-by: Dennis Felsing <dennis@felsing.org>
Co-authored-by: Ben Kirwin <bkirwi@materialize.com>

Author: 9 people

Cargo.lock

@@ -69,6 +69,10 @@ rm -f src/{clusterd,environmentd,materialized,persist-client,testdrive,catalog-d
 
 cargo update --workspace
 
+crd_descriptions_json=doc/user/data/self_managed/materialize_crd_descriptions.json
+cargo run -p mz-cloud-resources --bin crd-writer > "${crd_descriptions_json}"
+git add "${crd_descriptions_json}"
+
 bin/helm-chart-version-bump --bump-orchestratord-version "v$version"
 
 helm_docs_version=1.14.2
@@ -83,7 +87,11 @@ if [ "$(helm-docs --version)" != "helm-docs version $helm_docs_version" ]; then
     echo "Install helm-docs $helm_docs_version from https://github.com/norwoodj/helm-docs/releases/tag/v$helm_docs_version"
     exit 1
 fi
+# update README.md
 helm-docs misc/helm-charts
+# update values yaml descriptions
+helm-docs misc/helm-charts -t misc/helm-charts/operator/value_descriptions.yml.gotmpl -o value_descriptions.yml
+mv misc/helm-charts/operator/value_descriptions.yml doc/user/data/self_managed/materialize_operator_chart_parameter.yml
 
 if $sbom; then
     if ! inspector-sbomgen --version > /dev/null; then

bin/bump-version

@@ -34,7 +34,7 @@ $rem-scale: 0.585;
     --milli: #{rem(2)};
     --xx-small: #{rem(2.5)};
     --x-small: #{rem(3)};
-    --small: #{rem(4)};
+    --small: #{rem(4.5)};
     --medium: #{rem(5)};
     --large: #{rem(6)};
 

doc/user/assets/sass/_base.scss

@@ -10,10 +10,10 @@
 .content-wrapper {
     display: flex;
     justify-content: center;
-    padding-top: var(--nav-height);
+    padding-top: calc(var(--nav-height) + var(--small));
 
     @media(max-width: 800px) {
-        padding-top: calc(var(--nav-height) + var(--xx-small));
+        padding-top: calc(var(--nav-height) + var(--small));
     }
 
     @media(max-width: 380px) {
@@ -139,7 +139,7 @@ table.inline-headings {
         z-index: 100;
         top: var(--nav-height);
         left: 0;
-        height: calc(100vh - var(--nav-height));
+        height: calc(120vh - var(--nav-height));
         border-right: 1px solid var(--divider-light);
         transform: translateX(-100%);
         transition: all .2s ease-out;
@@ -193,6 +193,7 @@ table.inline-headings {
     @media(max-width: 850px) {
         padding-bottom: var(--large);
         height: calc(100vh - var(--nav-height));
+        margin-top: 80px;
     }
 
     a {

doc/user/assets/sass/_layout.scss

@@ -149,7 +149,7 @@
 
 button.show-topics {
     display: none;
-    margin: 0 0 var(--small) !important;
+    margin: var(--small) 0 var(--small) !important;
     text-align: left;
     font-weight: 300;
     font-size: var(--base);
@@ -171,6 +171,7 @@ button.close-topics {
     position: absolute;
     justify-content: center;
     align-items: center;
+    margin-top: 80px;
     top: 0;
     right: 0;
     transform: translateX(50%) translateY(50%);

doc/user/assets/sass/_nav.scss

@@ -7,7 +7,7 @@ disableKinds = ['taxonomy']
 
 [params]
 repo = "//github.com/MaterializeInc/materialize"
-bannerMessage = ""
+bannerMessage = "**New: Self-Managed v26.0!** New features include license keys, SCRAM-SHA support, and more! For details on v26.0 features, see [Release Notes](/releases/).<br>**Note**: To upgrade from v25.2 to v26.0, you must [upgrade first to v25.2.15](https://materialize.com/docs/self-managed/v25.2/release-notes/#v25215)."
 bannerLink = ""
 
 [frontmatter]

doc/user/config.toml

@@ -38,15 +38,11 @@ Materialize gives you the following benefits:
 The PostgreSQL source requires **PostgreSQL 11+** and is compatible with most
 common PostgreSQL hosted services.
 
-| Integration guides                          |
-| ------------------------------------------- |
-| {{% ingest-data/postgres-native-support %}} |
-
-If there is a hosted service or PostgreSQL distribution that is not listed above
-but you would like to use with Materialize, please submit a [feature
-request](https://github.com/MaterializeInc/materialize/discussions/new?category=feature-requests&labels=A-integration)
-or reach out in the Materialize [Community
-Slack](https://materialize.com/s/chat).
+## Integration guides
+
+The following integration guides are available:
+
+{{% include-md file="shared-content/postgresql-ingest-data-guides.md" %}}
 
 ## Considerations
 

doc/user/content/ingest-data/postgres/_index.md

@@ -0,0 +1,203 @@
+---
+title: "Guide: Handle upstream schema changes with zero downtime"
+description: "How to add a column, or drop a column, from your source PostgreSQL database, without any downtime in Materialize"
+
+menu:
+  main:
+    parent: "postgresql"
+    weight: 85
+
+---
+
+{{< private-preview />}}
+{{< note >}}
+Changing column types is currently unsupported.
+{{< /note >}}
+
+Starting in v26.0.0, Materialize allows you to handle certain types of upstream
+table schema changes seamlessly, specifically:
+
+- Adding a column in the upstream database.
+- Dropping a column in the upstream database.
+
+This guide walks you through how to handle these changes without any downtime in Materialize.
+
+## Prerequisites
+
+Some familiarity with Materialize. If you've never used Materialize before,
+start with our [guide to getting started](/get-started/quickstart/) to learn
+how to connect a database to Materialize.
+
+### Set up a PostgreSQL database
+
+For this guide, setup a PostgreSQL 11+ database. In your PostgreSQL, create a
+table `T` and populate:
+
+```sql
+CREATE TABLE T (
+    A INT
+);
+
+INSERT INTO T (A) VALUES
+    (10);
+```
+
+### Connect your source database to Materialize
+
+{{< include-md file="shared-content/postgres-source-prereq.md" >}}
+
+## Create a source using the new syntax
+
+In Materialize, create a source using the updated [`CREATE SOURCE`
+syntax](/sql/create-source/postgres-v2/).
+
+```sql
+CREATE SOURCE IF NOT EXISTS my_source
+    FROM POSTGRES CONNECTION my_connection (PUBLICATION 'mz_source');
+```
+
+Unlike the [legacy syntax](/sql/create-source/postgres/), the new syntax does
+not include the `FOR [[ALL] TABLES|SCHEMAS]` clause; i.e., the new syntax does
+not create corresponding subsources in Materialize automatically. Instead, the
+new syntax requires a separate [`CREATE TABLE ... FROM
+SOURCE`](/sql/create-table/), which will create the corresponding tables and
+start the snapshotting process. See [Create a table from the
+source](#create-a-table-from-the-source).
+
+{{< note >}}
+The [legacy syntax](/sql/create-source/postgres/) is still supported. However,
+the legacy syntax doesn't support upstream schema changes.
+{{< /note >}}
+
+## Create a table from the source
+To start ingesting specific tables from your source database, you can create a
+table in Materialize. We'll add it into the v1 schema in Materialize.
+
+```sql
+CREATE SCHEMA v1;
+
+CREATE TABLE v1.T
+    FROM SOURCE my_source(REFERENCE public.T);
+```
+
+Once you've created a table from source, the [initial
+snapshot](/ingest-data/#snapshotting) of table `v1.T` will begin.
+
+{{< note >}}
+
+During the snapshotting, the data ingestion for the other tables associated with
+the source is temporarily blocked. As before, you can monitor progress for the
+snapshot operation on the overview page for the source in the Materialize
+console.
+
+{{< /note >}}
+
+## Create a view on top of the table.
+
+For this guide, add a materialized view `matview` (also in schema `v1`) that
+sums column `A` from table `T`.
+
+```sql
+CREATE MATERIALIZED VIEW v1.matview AS
+    SELECT SUM(A) from v1.T;
+```
+
+## Handle upstream column addition
+
+### A. Add a column in your upstream PostgreSQL database
+
+In your upstream PostgreSQL database, add a new column `B` to the table `T`:
+
+```sql
+ALTER TABLE T
+    ADD COLUMN B BOOLEAN DEFAULT false;
+
+INSERT INTO T (A, B) VALUES
+    (20, true);
+```
+
+This operation will have no immediate effect in Materialize. In Materialize,
+`v1.T` will continue to ingest only column `A`. The materialized view
+`v1.matview` will continue to have access to column `A` as well.
+
+### B. Incorporate the new column in Materialize
+
+To incorporate the new column into Materialize, create a new `v2` schema and
+recreate the table in the new schema:
+
+```sql
+CREATE SCHEMA v2;
+
+CREATE TABLE v2.T
+    FROM SOURCE my_source(REFERENCE public.T);
+```
+
+The [snapshotting](/ingest-data/#snapshotting) of table `v2.T` will begin.
+`v2.T` will include columns `A` and `B`.
+
+{{< note >}}
+
+During the snapshotting, the data ingestion for the other tables associated with
+the source is temporarily blocked. As before, you can monitor progress for the
+snapshot operation on the overview page for the source in the Materialize
+console.
+
+{{< /note >}}
+
+
+When the new `v2.T` table has finished snapshotting, create a new materialized
+view `matview` in the new schema.  Since the new `v2.matview` is referencing the
+new `v2.T`, it can reference column `B`:
+
+```sql {hl_lines="4"}
+CREATE MATERIALIZED VIEW v2.matview AS
+    SELECT SUM(A)
+    FROM v2.T
+    WHERE B = true;
+```
+
+## Handle upstream column drop
+
+### A. Exclude the column in Materialize
+
+To drop a column safely, in Materialize, first, create a new `v3` schema, and
+recreate table `T` in the new schema but exclude the column to drop. In this
+example, we'll drop the column B.
+
+```sql
+CREATE SCHEMA v3;
+CREATE TABLE v3.T
+    FROM SOURCE my_source(REFERENCE public.T) WITH (EXCLUDE COLUMNS (B));
+```
+
+{{< note >}}
+
+During the snapshotting, the data ingestion for the other tables associated with
+the source is temporarily blocked. As before, you can monitor progress for the
+snapshot operation on the overview page for the source in the Materialize
+console.
+
+{{< /note >}}
+
+### B. Drop a column in your upstream PostgreSQL database
+
+In your upstream PostgreSQL database, drop the column `B` from the table `T`:
+
+```sql
+ALTER TABLE T DROP COLUMN B;
+```
+
+Dropping the column B will have no effect on `v3.T`. However, the drop affects
+`v2.T` and `v2.matview` from our earlier examples. When the user attempts to
+read from either, Materialize will report an error that the source table schema
+has been altered.
+
+## Optional: Swap schemas
+
+When you're ready to fully cut over to the new source version, you can optionally swap the schemas and drop the old objects.
+
+```sql
+ALTER SCHEMA v1 SWAP WITH v3;
+
+DROP SCHEMA v3 CASCADE;
+```