Managed DB: Add MongoDB and Kafka Configurations to docs (#864)

* adding kafka and mongo advanced config api docs

* fixing formatting issues in kafka docs

* add examples to spec

specification/resources/databases/models/database_config.yml

@@ -6,3 +6,5 @@ properties:
       - $ref: './mysql.yml'
       - $ref: './postgres.yml'
       - $ref: './redis.yml'
+      - $ref: './mongo.yml'
+      - $ref: './kafka.yml'

specification/resources/databases/models/kafka.yml

@@ -0,0 +1,328 @@
+type: object
+
+properties:
+  compression_type:
+    description: >-
+      Specify the final compression type for a given topic. This
+      configuration accepts the standard compression codecs ('gzip', 'snappy',
+      'lz4', 'zstd'). It additionally accepts 'uncompressed' which is equivalent
+      to no compression; and 'producer' which means retain the original
+      compression codec set by the producer.
+    type: string
+    enum:
+    - gzip
+    - snappy
+    - lz4
+    - zstd
+    - uncompressed
+    - producer
+    example: gzip
+  group_initial_rebalance_delay_ms:
+    description: >-
+      The amount of time, in milliseconds, the group coordinator
+      will wait for more consumers to join a new group before performing
+      the first rebalance. A longer delay means potentially fewer rebalances,
+      but increases the time until processing begins. The default value
+      for this is 3 seconds. During development and testing it might be
+      desirable to set this to 0 in order to not delay test execution time.
+    type: integer
+    example: 3000
+    minimum: 0
+    maximum: 300000
+  group_min_session_timeout_ms:
+    description: >-
+      The minimum allowed session timeout for registered consumers.
+      Longer timeouts give consumers more time to process messages in between
+      heartbeats at the cost of a longer time to detect failures.
+    type: integer
+    example: 6000
+    minimum: 0
+    maximum: 60000
+  group_max_session_timeout_ms:
+    description: >-
+      The maximum allowed session timeout for registered consumers.
+      Longer timeouts give consumers more time to process messages in between
+      heartbeats at the cost of a longer time to detect failures.
+    type: integer
+    example: 1800000
+    minimum: 0
+    maximum: 1800000
+  connections_max_idle_ms:
+    description: >-
+      Idle connections timeout: the server socket processor
+      threads close the connections that idle for longer than this.
+    type: integer
+    minimum: 1000
+    example: 540000
+    maximum: 3600000
+  max_incremental_fetch_session_cache_slots:
+    description: >-
+      The maximum number of incremental fetch sessions that the
+      broker will maintain.
+    type: integer
+    example: 1000
+    minimum: 1000
+    maximum: 10000
+  message_max_bytes:
+    description: >-
+      The maximum size of message that the server can receive.
+    type: integer
+    example: 1048588
+    minimum: 0
+    maximum: 100001200
+  offsets_retention_minutes:
+    description: >-
+      Log retention window in minutes for offsets topic
+    type: integer
+    example: 10080
+    minimum: 1
+    maximum: 2147483647
+  log_cleaner_delete_retention_ms:
+    description: >-
+      How long are delete records retained?
+    type: integer
+    minimum: 0
+    maximum: 315569260000
+    example: 86400000
+  log_cleaner_min_cleanable_ratio:
+    description: >-
+      Controls log compactor frequency. Larger value means more
+      frequent compactions but also more space wasted for logs. Consider
+      setting log_cleaner_max_compaction_lag_ms to enforce compactions sooner,
+      instead of setting a very high value for this option.
+    type: number
+    minimum: 0.2
+    maximum: 0.9
+    example: 0.5
+  log_cleaner_max_compaction_lag_ms:
+    description: >-
+      The maximum amount of time message will remain uncompacted.
+      Only applicable for logs that are being compacted
+    type: integer
+    minimum: 30000
+    maximum: 9223372036854776000
+    example: 60000
+  log_cleaner_min_compaction_lag_ms:
+    description: >-
+      The minimum time a message will remain uncompacted in the
+      log. Only applicable for logs that are being compacted.
+    type: integer
+    minimum: 0
+    maximum: 9223372036854776000
+    example: 100000
+  log_cleanup_policy:
+    description: >-
+      The default cleanup policy for segments beyond the retention
+      window
+    type: string
+    enum:
+    - delete
+    - compact
+    - compact,delete
+    example: delete
+  log_flush_interval_messages:
+    description: >-
+      The number of messages accumulated on a log partition before
+      messages are flushed to disk
+    type: integer
+    minimum: 1
+    maximum: 9223372036854776000
+    example: 9223372036854776000
+  log_flush_interval_ms:
+    description: >-
+      The maximum time in ms that a message in any topic is kept
+      in memory before flushed to disk. If not set, the value in log.flush.scheduler.interval.ms
+      is used
+    type: integer
+    minimum: 0
+    maximum: 9223372036854776000
+    example: 1000000
+  log_index_interval_bytes:
+    description: >-
+      The interval with which Kafka adds an entry to the offset
+      index
+    type: integer
+    minimum: 0
+    maximum: 104857600
+    example: 4096
+  log_index_size_max_bytes:
+    description: >-
+      The maximum size in bytes of the offset index
+    type: integer
+    minimum: 1048576
+    maximum: 104857600
+    example: 10485760
+  log_message_downconversion_enable:
+    description: >-
+      This configuration controls whether down-conversion of
+      message formats is enabled to satisfy consume requests.
+    type: boolean
+    example: true
+  log_message_timestamp_type:
+    description: >-
+      Define whether the timestamp in the message is message
+      create time or log append time.
+    type: string
+    enum:
+    - CreateTime
+    - LogAppendTime
+    example: CreateTime
+  log_message_timestamp_difference_max_ms:
+    description: >-
+      The maximum difference allowed between the timestamp when
+      a broker receives a message and the timestamp specified in the message
+    type: integer
+    minimum: 0
+    maximum: 9223372036854776000
+    example: 1000000
+  log_preallocate:
+    description: >-
+      Controls whether to preallocate a file when creating a new segment
+    type: boolean
+    example: false
+  log_retention_bytes:
+    description: >-
+      The maximum size of the log before deleting messages
+    type: integer
+    minimum: -1
+    maximum: 9223372036854776000
+    example: 1000000
+  log_retention_hours:
+    description: >-
+      The number of hours to keep a log file before deleting it
+    type: integer
+    minimum: -1
+    maximum: 2147483647
+    example: 1000000
+  log_retention_ms:
+    description: >-
+      The number of milliseconds to keep a log file before deleting
+      it (in milliseconds), If not set, the value in log.retention.minutes
+      is used. If set to -1, no time limit is applied.
+    type: integer
+    minimum: -1
+    maximum: 9223372036854776000
+    example: 100000000
+  log_roll_jitter_ms:
+    description: >-
+      The maximum jitter to subtract from logRollTimeMillis (in
+      milliseconds). If not set, the value in log.roll.jitter.hours is used
+    type: integer
+    minimum: 0
+    maximum: 9223372036854776000
+    example: 10000000
+  log_roll_ms:
+    description: >-
+      The maximum time before a new log segment is rolled out
+      (in milliseconds).
+    type: integer
+    minimum: 1
+    maximum: 9223372036854776000
+    example: 1000000
+  log_segment_bytes:
+    description: >-
+      The maximum size of a single log file
+    type: integer
+    minimum: 10485760
+    maximum: 1073741824
+    example: 100000000
+  log_segment_delete_delay_ms:
+    description: >-
+      The amount of time to wait before deleting a file from
+      the filesystem
+    type: integer
+    minimum: 0
+    maximum: 3600000
+    example: 60000
+  auto_create_topics_enable:
+    description: >-
+      Enable auto creation of topics
+    type: boolean
+    example: true
+  min_insync_replicas:
+    description: >-
+      When a producer sets acks to 'all' (or '-1'), min_insync_replicas
+      specifies the minimum number of replicas that must acknowledge a write
+      for the write to be considered successful.
+    type: integer
+    minimum: 1
+    maximum: 7
+    example: 1
+  num_partitions:
+    description: >-
+      Number of partitions for autocreated topics
+    type: integer
+    minimum: 1
+    maximum: 1000
+    example: 10
+  default_replication_factor:
+    description: >-
+      Replication factor for autocreated topics
+    type: integer
+    minimum: 1
+    maximum: 10
+    example: 2
+  replica_fetch_max_bytes:
+    description: >-
+      The number of bytes of messages to attempt to fetch for
+      each partition (defaults to 1048576). This is not an absolute maximum,
+      if the first record batch in the first non-empty partition of the
+      fetch is larger than this value, the record batch will still be returned
+      to ensure that progress can be made.
+    type: integer
+    minimum: 1048576
+    maximum: 104857600
+    example: 2097152
+  replica_fetch_response_max_bytes:
+    description: >-
+      Maximum bytes expected for the entire fetch response (defaults
+      to 10485760). Records are fetched in batches, and if the first record
+      batch in the first non-empty partition of the fetch is larger than
+      this value, the record batch will still be returned to ensure that
+      progress can be made. As such, this is not an absolute maximum.
+    type: integer
+    minimum: 10485760
+    maximum: 1048576000
+    example: 20971520
+  max_connections_per_ip:
+    description: >-
+      The maximum number of connections allowed from each ip
+      address (defaults to 2147483647).
+    type: integer
+    minimum: 256
+    maximum: 2147483647
+    example: 512
+  producer_purgatory_purge_interval_requests:
+    description: >-
+      The purge interval (in number of requests) of the producer
+      request purgatory (defaults to 1000).
+    type: integer
+    minimum: 10
+    maximum: 10000
+    example: 100
+  socket_request_max_bytes:
+    description: >-
+      The maximum number of bytes in a socket request (defaults
+      to 104857600).
+    type: integer
+    minimum: 10485760
+    maximum: 209715200
+    example: 20971520
+  transaction_state_log_segment_bytes:
+    description: >-
+      The transaction topic segment bytes should be kept relatively
+      small in order to facilitate faster log compaction and cache loads
+      (defaults to 104857600 (100 mebibytes)).
+    type: integer
+    minimum: 1048576
+    maximum: 2147483647
+    example: 104857600
+  transaction_remove_expired_transaction_cleanup_interval_ms:
+    description: >-
+      The interval at which to remove transactions that have
+      expired due to transactional.id.expiration.ms passing (defaults to
+      3600000 (1 hour)).
+    type: integer
+    minimum: 600000
+    maximum: 3600000
+    example: 3600000

specification/resources/databases/models/mongo.yml

@@ -0,0 +1,52 @@
+type: object
+
+properties:
+  default_read_concern:
+    description: >-
+      Specifies the default consistency behavior of reads from the database. Data that is returned from the query with may
+      or may not have been acknowledged by all nodes in the replicaset depending on this value. 
+      Learn more [here](https://www.mongodb.com/docs/manual/reference/read-concern/).
+    type: string
+    enum: [local, available, majority]
+    default: local
+    example: local
+  default_write_concern:
+    description: >-
+      Describes the level of acknowledgment requested from MongoDB for write operations clusters. This field can set to either `majority`
+      or a number `0...n` which will describe the number of nodes that must acknowledge the write operation before it is fully accepted.
+      Setting to `0` will request no acknowledgement of the write operation. 
+      Learn more [here](https://www.mongodb.com/docs/manual/reference/write-concern/).
+    type: string
+    default: majority
+    example: majority
+  transaction_lifetime_limit_seconds:
+    description: >-
+        Specifies the lifetime of multi-document transactions. Transactions that exceed this limit are considered expired and will be 
+        aborted by a periodic cleanup process. The cleanup process runs every `transactionLifetimeLimitSeconds/2 seconds` or at least 
+        once every 60 seconds. *Changing this parameter will lead to a restart of the MongoDB service.*
+        Learn more [here](https://www.mongodb.com/docs/manual/reference/parameters/#mongodb-parameter-param.transactionLifetimeLimitSeconds).
+    type: integer
+    minimum: 1
+    default: 60
+    example: 100
+  slow_op_threshold_ms:
+    description: >-
+        Operations that run for longer than this threshold are considered slow which are then recorded to the diagnostic logs. 
+        Higher log levels (verbosity) will record all operations regardless of this threshold on the primary node. 
+        *Changing this parameter will lead to a restart of the MongoDB service.*
+        Learn more [here](https://www.mongodb.com/docs/manual/reference/configuration-options/#mongodb-setting-operationProfiling.slowOpThresholdMs).
+    type: integer
+    minimum: 0
+    default: 100
+    example: 200
+  verbosity:
+    description: >-
+        The log message verbosity level. The verbosity level determines the amount of Informational and Debug messages MongoDB outputs.
+        0 includes informational messages while 1...5 increases the level to include debug messages.
+        *Changing this parameter will lead to a restart of the MongoDB service.*
+        Learn more [here](https://www.mongodb.com/docs/manual/reference/configuration-options/#mongodb-setting-systemLog.verbosity).
+    type: integer
+    minimum: 0
+    maximum: 5
+    default: 0
+    example: 3