From 104a54c63bd8e788d7695445709995a4b1bb6ca6 Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 09:55:07 -0500 Subject: [PATCH 1/8] commit after getting S3 setup --- .github/workflows/feast_apply_gcp.yml | 37 -------------- .github/workflows/feast_plan_gcp.yml | 48 ------------------- module_0/client_aws/feature_store.yaml | 2 +- module_0/feature_repo_aws/data_sources.py | 4 +- module_0/feature_repo_aws/entities.py | 4 +- module_0/feature_repo_aws/feature_services.py | 6 ++- module_0/feature_repo_aws/feature_store.yaml | 2 +- module_0/feature_repo_aws/features.py | 2 +- module_0/infra/aws/main.tf | 16 ++++--- module_0/infra/aws/variables.tf | 2 +- 10 files changed, 22 insertions(+), 101 deletions(-) delete mode 100644 .github/workflows/feast_apply_gcp.yml delete mode 100644 .github/workflows/feast_plan_gcp.yml diff --git a/.github/workflows/feast_apply_gcp.yml b/.github/workflows/feast_apply_gcp.yml deleted file mode 100644 index 02928cc..0000000 --- a/.github/workflows/feast_apply_gcp.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Feast apply (GCP) - -on: - push: - branches: - - main - -jobs: - feast_apply: - runs-on: ubuntu-latest - steps: - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: "3.7" - architecture: x64 - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Use gcloud CLI - run: gcloud info - - # Run `feast apply` - - uses: actions/checkout@v2 - - name: Install feast - run: pip install "feast[aws]" - - name: Run feast apply - env: - FEAST_USAGE: "False" - IS_TEST: "True" - run: | - cd module_0/feature_repo_aws - feast apply \ No newline at end of file diff --git a/.github/workflows/feast_plan_gcp.yml b/.github/workflows/feast_plan_gcp.yml deleted file mode 100644 index a77e7b2..0000000 --- a/.github/workflows/feast_plan_gcp.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Feast plan (GCP) - -on: [pull_request] - -jobs: - feast_plan: - runs-on: ubuntu-latest - steps: - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: "3.7" - architecture: x64 - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_SA_KEY }} - export_default_credentials: true - - name: Use gcloud CLI - run: gcloud info - - # Run `feast plan` - - uses: actions/checkout@v2 - - name: Install feast - run: pip install "feast[aws]" - - name: Capture `feast plan` in a variable - id: feast_plan - env: - FEAST_USAGE: "False" - FEAST_FORCE_USAGE_UUID: None - IS_TEST: "True" - run: | - body=$(cd module_0/feature_repo_gcp; feast plan) - body="${body//'%'/'%25'}" - body="${body//$'\n'/'%0A'}" - body="${body//$'\r'/'%0D'}" - echo "::set-output name=body::$body" - - # Post a comment on the PR with the results of `feast plan` - - name: Create comment - uses: peter-evans/create-or-update-comment@v1 - if: ${{ steps.feast_plan.outputs.body }} - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - ${{ steps.feast_plan.outputs.body }} \ No newline at end of file diff --git a/module_0/client_aws/feature_store.yaml b/module_0/client_aws/feature_store.yaml index cf04a36..7dde609 100644 --- a/module_0/client_aws/feature_store.yaml +++ b/module_0/client_aws/feature_store.yaml @@ -1,6 +1,6 @@ project: feast_demo_aws provider: aws -registry: s3://feast-workshop-danny/registry.pb # TODO: Replace with your bucket +registry: s3://feast-repo-gyerli/feast-workshop-module-0/registry.pb # TODO: Replace with your bucket online_store: null offline_store: type: file diff --git a/module_0/feature_repo_aws/data_sources.py b/module_0/feature_repo_aws/data_sources.py index b46a8ea..3e02a49 100644 --- a/module_0/feature_repo_aws/data_sources.py +++ b/module_0/feature_repo_aws/data_sources.py @@ -2,10 +2,10 @@ driver_stats = FileSource( name="driver_stats_source", - path="s3://feast-workshop-danny/driver_stats.parquet", # TODO: Replace with your bucket + path="s3://feast-workshop-gyerli/feast-workshop-module-0/driver_stats.parquet", # TODO: Replace with your bucket s3_endpoint_override="http://s3.us-west-2.amazonaws.com", # Needed since s3fs defaults to us-east-1 timestamp_field="event_timestamp", created_timestamp_column="created", description="A table describing the stats of a driver based on hourly logs", - owner="test2@gmail.com", + owner="gursoy@yerli.org" ) diff --git a/module_0/feature_repo_aws/entities.py b/module_0/feature_repo_aws/entities.py index b93c50a..3186610 100644 --- a/module_0/feature_repo_aws/entities.py +++ b/module_0/feature_repo_aws/entities.py @@ -7,5 +7,5 @@ name="driver", join_keys=["driver_id"], value_type=ValueType.INT64, - description="driver id", -) + description="driver id" +) \ No newline at end of file diff --git a/module_0/feature_repo_aws/feature_services.py b/module_0/feature_repo_aws/feature_services.py index 0a47fec..9f71695 100644 --- a/module_0/feature_repo_aws/feature_services.py +++ b/module_0/feature_repo_aws/feature_services.py @@ -5,9 +5,11 @@ feature_service = FeatureService( name="model_v1", features=[driver_hourly_stats_view[["conv_rate"]]], - owner="test3@gmail.com", + owner="gursoy@yerli.org", ) feature_service_2 = FeatureService( - name="model_v2", features=[driver_hourly_stats_view], owner="test3@gmail.com", + name="model_v2", + features=[driver_hourly_stats_view], + owner="gursoy@yerli.org", ) diff --git a/module_0/feature_repo_aws/feature_store.yaml b/module_0/feature_repo_aws/feature_store.yaml index cf04a36..7dde609 100644 --- a/module_0/feature_repo_aws/feature_store.yaml +++ b/module_0/feature_repo_aws/feature_store.yaml @@ -1,6 +1,6 @@ project: feast_demo_aws provider: aws -registry: s3://feast-workshop-danny/registry.pb # TODO: Replace with your bucket +registry: s3://feast-repo-gyerli/feast-workshop-module-0/registry.pb # TODO: Replace with your bucket online_store: null offline_store: type: file diff --git a/module_0/feature_repo_aws/features.py b/module_0/feature_repo_aws/features.py index 7c4ef4b..e23e371 100644 --- a/module_0/feature_repo_aws/features.py +++ b/module_0/feature_repo_aws/features.py @@ -12,7 +12,7 @@ driver_hourly_stats_view = FeatureView( name="driver_hourly_stats", description="Hourly features", - entities=["driver"], + entities=[driver], ttl=timedelta(seconds=8640000000), schema=[ Field(name="conv_rate", dtype=Float32), diff --git a/module_0/infra/aws/main.tf b/module_0/infra/aws/main.tf index 4fe1cca..ad37e3d 100644 --- a/module_0/infra/aws/main.tf +++ b/module_0/infra/aws/main.tf @@ -3,17 +3,21 @@ provider "aws" { } resource "aws_s3_bucket" "feast_bucket" { - bucket = "feast-workshop-${var.project_name}" + bucket = "feast-repo-gyerli" force_destroy = true } -resource "aws_s3_bucket_acl" "feast_bucket_acl" { - bucket = aws_s3_bucket.feast_bucket.bucket - acl = "private" +# resource "aws_s3_bucket_acl" "feast_bucket_acl" { +# bucket = aws_s3_bucket.feast_bucket.bucket +# } + +resource "aws_s3_bucket_object" "feast_workshop_folder" { + bucket = aws_s3_bucket.feast_bucket.id + key = "feast-workshop-${var.project_name}/" } resource "aws_s3_object" "driver_stats_upload" { bucket = aws_s3_bucket.feast_bucket.bucket - key = "driver_stats.parquet" + key = "feast-workshop-${var.project_name}/driver_stats.parquet" source = "${path.module}/../driver_stats.parquet" -} +} \ No newline at end of file diff --git a/module_0/infra/aws/variables.tf b/module_0/infra/aws/variables.tf index e5bba56..8789c85 100644 --- a/module_0/infra/aws/variables.tf +++ b/module_0/infra/aws/variables.tf @@ -1,4 +1,4 @@ variable "project_name" { type = string - description = "The project identifier is used to uniquely namespace resources" + description = "The project identifier is used to uniquely namespace resources - gyerli" } From 18e1bdb7d0dec5d26fb29f91abbe07ebdc38d90c Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:08:02 -0500 Subject: [PATCH 2/8] secrets --- .github/workflows/feast_apply_aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index 7ad9b27..83405f3 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -15,7 +15,7 @@ jobs: with: python-version: "3.7" architecture: x64 - - name: Set up AWS SDK + - name: Set up AWS SDK w/ Secrets uses: aws-actions/configure-aws-credentials@v1 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} From 3cca09a21df027d83c0aa305a940197577dd8ed3 Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:15:18 -0500 Subject: [PATCH 3/8] pip re-install --- .github/workflows/feast_apply_aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index 83405f3..fc2d87d 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -25,7 +25,7 @@ jobs: # Run `feast apply` - uses: actions/checkout@v2 - name: Install feast - run: pip install "feast[aws]" + run: pip install --no-build-isolation --force-reinstall "feast[aws]"" - name: Run feast apply env: FEAST_USAGE: "False" From 22ec9ede0a908945870d6cfca5fcec955a012450 Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:16:35 -0500 Subject: [PATCH 4/8] fixed syntax on apply workflow --- .github/workflows/feast_apply_aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index fc2d87d..0915b27 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -25,7 +25,7 @@ jobs: # Run `feast apply` - uses: actions/checkout@v2 - name: Install feast - run: pip install --no-build-isolation --force-reinstall "feast[aws]"" + run: pip install --no-build-isolation --force-reinstall "feast[aws]" - name: Run feast apply env: FEAST_USAGE: "False" From a454f84038d8980553f479edbeee72aa0f610116 Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:21:39 -0500 Subject: [PATCH 5/8] added upgrade pip --- .github/workflows/feast_apply_aws.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index 0915b27..070da20 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -22,6 +22,11 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: us-west-2 + # Upgrade pip + - uses: actions/checkout@v2 + - name: Upgrade pip + run : pip install --upgrade pip + # Run `feast apply` - uses: actions/checkout@v2 - name: Install feast From 3d93fb2a86b96c9509fd9b4507bbc808dda77ba0 Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:26:16 -0500 Subject: [PATCH 6/8] changed to python 3.8 --- .github/workflows/feast_apply_aws.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index 070da20..d42e443 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -13,7 +13,7 @@ jobs: id: setup-python uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" architecture: x64 - name: Set up AWS SDK w/ Secrets uses: aws-actions/configure-aws-credentials@v1 From 80c6e9b6a5f6cc275413e4e6f1f82cbff25e7dd3 Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:29:11 -0500 Subject: [PATCH 7/8] setuptools --- .github/workflows/feast_apply_aws.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index d42e443..3bf070c 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -25,7 +25,9 @@ jobs: # Upgrade pip - uses: actions/checkout@v2 - name: Upgrade pip - run : pip install --upgrade pip + run: pip install --upgrade pip + - name: Upgrade setuptools + run: pip install --upgrade setuptools # Run `feast apply` - uses: actions/checkout@v2 From 0db3e930ff2e427cd371be15a9b0b134f734351d Mon Sep 17 00:00:00 2001 From: Gursoy Yerli Date: Mon, 15 May 2023 10:36:42 -0500 Subject: [PATCH 8/8] install wheel --- .github/workflows/feast_apply_aws.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/feast_apply_aws.yml b/.github/workflows/feast_apply_aws.yml index 3bf070c..3022b9c 100644 --- a/.github/workflows/feast_apply_aws.yml +++ b/.github/workflows/feast_apply_aws.yml @@ -28,6 +28,8 @@ jobs: run: pip install --upgrade pip - name: Upgrade setuptools run: pip install --upgrade setuptools + - name: Install wheel + run: pip install wheel # Run `feast apply` - uses: actions/checkout@v2