new file: .user.yml

.user.yml

@@ -0,0 +1 @@
+id: 93533b73-9644-414a-a42e-347ce27a2d5a

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "githubPullRequests.ignoredPullRequestBranches": [
+        "main"
+    ]
+}

Dockerfile

@@ -1,5 +1,5 @@
-ARG DBT_VERSION=1.0.0
-FROM fishtownanalytics/dbt:${DBT_VERSION}
+ARG DBT_VERSION=1.8.3
+FROM ghcr.io/dbt-labs/dbt-core:${DBT_VERSION}
 
 # Terraform configuration file
 # COPY terraformrc root/.terraformrc

dbt_project.yml

@@ -1,4 +1,3 @@
-
 # Name your project! Project names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
@@ -40,4 +39,4 @@ models:
             +tags: ['staging']
         marts:
             +materialized: table
-            +tags: ['marts']
+            +tags: ['marts']

macros/init_s3_sources.sql

@@ -13,7 +13,7 @@
                 C_MKTSEGMENT    LowCardinality(String),
                 C_COMMENT       String   
         )
-        ENGINE = S3(\'https://storage.yandexcloud.net/otus-dwh/tpch-dbgen-1g/customer.tbl\', \'CustomSeparated\')
+        ENGINE = S3(\'https://storage.yandexcloud.net/ch-data-course/customer.tbl\', \'CustomSeparated\')
         SETTINGS
             format_custom_field_delimiter=\'|\'
             ,format_custom_escaping_rule=\'CSV\'
@@ -32,7 +32,7 @@
             O_SHIPPRIORITY           UInt8,  
             O_COMMENT               String
         )
-        ENGINE = S3(\'https://storage.yandexcloud.net/otus-dwh/tpch-dbgen-1g/orders.tbl\', \'CustomSeparated\')
+        ENGINE = S3(\'https://storage.yandexcloud.net/ch-data-course/orders.tbl\', \'CustomSeparated\')
         SETTINGS
             format_custom_field_delimiter=\'|\'
             ,format_custom_escaping_rule=\'CSV\'
@@ -58,7 +58,7 @@
             L_SHIPMODE             LowCardinality(String),
             L_COMMENT               String
         )
-        ENGINE = S3(\'https://storage.yandexcloud.net/otus-dwh/tpch-dbgen-1g/lineitem.tbl\', \'CustomSeparated\')
+        ENGINE = S3(\'https://storage.yandexcloud.net/ch-data-course/lineitem.tbl\', \'CustomSeparated\')
         SETTINGS
             format_custom_field_delimiter=\'|\'
             ,format_custom_escaping_rule=\'CSV\'
@@ -77,7 +77,7 @@
                 P_RETAILPRICE       Decimal(15,2),
                 P_COMMENT       String
         )
-        ENGINE = S3(\'https://storage.yandexcloud.net/otus-dwh/tpch-dbgen-1g/part.tbl\', \'CustomSeparated\')
+        ENGINE = S3(\'https://storage.yandexcloud.net/ch-data-course/part.tbl\', \'CustomSeparated\')
         SETTINGS
             format_custom_field_delimiter=\'|\'
             ,format_custom_escaping_rule=\'CSV\'
@@ -94,7 +94,7 @@
                 S_ACCTBAL       Decimal(15,2),
                 S_COMMENT       String
         )
-        ENGINE = S3(\'https://storage.yandexcloud.net/otus-dwh/tpch-dbgen-1g/supplier.tbl\', \'CustomSeparated\')
+        ENGINE = S3(\'https://storage.yandexcloud.net/ch-data-course/supplier.tbl\', \'CustomSeparated\')
         SETTINGS
             format_custom_field_delimiter=\'|\'
             ,format_custom_escaping_rule=\'CSV\'

main.tf

@@ -8,17 +8,22 @@ terraform {
 }
 
 provider "yandex" {
+  token              = "t1.9euelZqPjcudic6JlpeZlpKazJTGkO3rnpWalpDMioycm5jIz8jKnZ6Sjs_l9PdBBUNK-e8Dbwev3fT3ATRASvnvA28Hr83n9euelZqdjcjKiszMy5THl4_Kzcadju_8xeuelZqdjcjKiszMy5THl4_Kzcadjg.fCgEd3SWL-ehzJihzWkH_NNZDQQvRfMEenDT-PeHqJfAR1c3pOi3ydQmxLIEeI5cY75ngJ7sPPBqXL8-vCtJCA"
+  cloud_id           = "b1g38qb1ddfldjnj15m9"
+  folder_id          = "b1gcaq5kiuko0f1f02j4"
+  storage_access_key = "YCAJE8F0PQP6DUrWLKF-DQzkI"
+  storage_secret_key = "YCM1kZ4lXdVQWWouZ39wJmhYfgTH0JA0Ra2W1O9P"
+  zone               = "ru-central1-d"
 }
 
-
 resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" {
   name                    = "clickhouse_starschema"
-  environment             = "PRESTABLE"
+  environment             = "PRODUCTION"
   network_id              = yandex_vpc_network.default_network.id
   sql_database_management = true
   sql_user_management     = true
   admin_password          = var.clickhouse_password
-  version                 = "23.3"
+  version                 = "24.4"
 
   clickhouse {
     resources {
@@ -39,21 +44,21 @@ resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" {
       timezone                        = "UTC"
       geobase_uri                     = ""
       query_log_retention_size        = 1073741824
-      query_log_retention_time        = 2592000
+      query_log_retention_time        = 2592000000
       query_thread_log_enabled        = true
       query_thread_log_retention_size = 536870912
-      query_thread_log_retention_time = 2592000
+      query_thread_log_retention_time = 2592000000
       part_log_retention_size         = 536870912
-      part_log_retention_time         = 2592000
+      part_log_retention_time         = 2592000000
       metric_log_enabled              = true
       metric_log_retention_size       = 536870912
-      metric_log_retention_time       = 2592000
+      metric_log_retention_time       = 2592000000
       trace_log_enabled               = true
       trace_log_retention_size        = 536870912
-      trace_log_retention_time        = 2592000
+      trace_log_retention_time        = 2592000000
       text_log_enabled                = true
       text_log_retention_size         = 536870912
-      text_log_retention_time         = 2592000
+      text_log_retention_time         = 2592000000
       text_log_level                  = "TRACE"
       background_pool_size            = 16
       background_schedule_pool_size   = 16
@@ -137,7 +142,7 @@ resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" {
 
   host {
     type             = "CLICKHOUSE"
-    zone             = "ru-central1-b"
+    zone             = "ru-central1-d"
     subnet_id        = yandex_vpc_subnet.foo.id
     assign_public_ip = true
   }
@@ -150,11 +155,9 @@ resource "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" {
     type = "ANYTIME"
   }
 }
-
 resource "yandex_vpc_network" "default_network" {}
-
 resource "yandex_vpc_subnet" "foo" {
-  zone           = "ru-central1-b"
+  zone           = "ru-central1-d"
   network_id     = yandex_vpc_network.default_network.id
   v4_cidr_blocks = ["10.5.0.0/24"]
 }

models/marts/f_orders_stats.sql

@@ -5,4 +5,16 @@
     )
 }}
 
-SELECT 1
+SELECT
+    toYear(O_ORDERDATE) AS O_ORDERYEAR
+    , O_ORDERSTATUS
+    , O_ORDERPRIORITY
+    , count(DISTINCT O_ORDERKEY) AS num_orders
+    , count(DISTINCT C_CUSTKEY) AS num_customers
+    , sum(L_EXTENDEDPRICE * L_DISCOUNT) AS revenue
+FROM {{ ref('f_lineorder_flat') }}
+WHERE 1=1
+GROUP BY
+    toYear(O_ORDERDATE)
+    , O_ORDERSTATUS
+    , O_ORDERPRIORITY

package-lock.yml

@@ -0,0 +1,4 @@
+packages:
+  - package: dbt-labs/dbt_utils
+    version: 1.2.0
+sha1_hash: 375b38987c2346580b2f54a91a56cbc9dadd6f1f

packages.yml

@@ -1,3 +1,3 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: [">=1.0.0", "<1.1.0"]
+    version: [">=1.0.0", "<1.3.0"]

profiles.yml

@@ -1,17 +1,12 @@
-config:
-  send_anonymous_usage_stats: False
-  use_colors: True
-  partial_parse: True
-
 clickhouse_starschema:
   outputs:
     dev:
       type: clickhouse
       schema: default
-      host: "{{ env_var('DBT_HOST') }}"
+      host: "rc1d-mb0nbushgtr6bvn8.mdb.yandexcloud.net"
       port: 8443
-      user: "{{ env_var('DBT_USER') }}"
-      password: "{{ env_var('DBT_PASSWORD') }}"
+      user: "admin"
+      password: "jxtymlkbyysqgfhjkm"
       secure: True
       verify: False
     ci:

variables.tf

@@ -1,13 +1,13 @@
 variable "clickhouse_password" {
-  description = "Clickhouse admin password"
+  description = "jxtymlkbyysqgfhjkm"
   type        = string
   sensitive   = true
 }
 
-# data "yandex_mdb_clickhouse_cluster" "clickhouse_starschema" {
-#   name = "clickhouse_starschema"
+# data "yandex_mdb_clickhouse_cluster" "clickhouse_starschema"" {
+#  name = "clickhouse_starschema""
 # }
 
 output "clickhouse_host_fqdn" {
   value = resource.yandex_mdb_clickhouse_cluster.clickhouse_starschema.host[0].fqdn
-}
+}