diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 0e9ef00db8..49669aa5b1 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -72,16 +72,6 @@ jobs: CIBW_BUILD: "cp3*_x86_64" CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64" CIBW_ARCHS: "native" - CIBW_ENVIRONMENT: > - COMPILE_GO=True PATH=$PATH:/usr/local/go/bin - CIBW_BEFORE_ALL_LINUX: | - curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz - tar -C /usr/local -xzf go.tar.gz - go version - yum -y update && - yum install -y epel-release || yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1).noarch.rpm && - yum install -y https://apache.jfrog.io/artifactory/arrow/centos/$(cut -d: -f5 /etc/system-release-cpe | cut -d. -f1)/apache-arrow-release-latest.rpm && - yum install -y --enablerepo=epel arrow-devel # For C++ CIBW_BEFORE_ALL_MACOS: | brew install apache-arrow brew install pkg-config @@ -90,8 +80,6 @@ jobs: # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. CIBW_BEFORE_BUILD: | make install-protoc-dependencies - make install-go-proto-dependencies - make install-go-ci-dependencies git status git restore go.mod go.sum git restore sdk/python/feast/ui/yarn.lock @@ -139,8 +127,6 @@ jobs: run: | pip install -U pip setuptools wheel twine make install-protoc-dependencies - make install-go-proto-dependencies - make install-go-ci-dependencies make build-ui git status git restore go.mod go.sum @@ -203,9 +189,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - uses: actions/setup-go@v3 - with: - go-version: '>=1.17.0' - uses: actions/download-artifact@v2 with: name: wheels @@ -217,33 +200,6 @@ jobs: cd dist/ pip install wheel for f in *.whl; do pip install $f || true; done - - name: Install apache-arrow on ubuntu - if: ${{ matrix.from-source && matrix.os == 'ubuntu-latest' }} - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: ${{ matrix.from-source && matrix.os == 'macos-10.15' && matrix.python-version != '3.10' }} - run: | - brew install apache-arrow - brew install pkg-config - - name: Install dist with go - if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}} - env: - COMPILE_GO: "True" - run: | - pip install 'grpcio-tools==1.47.0' 'pybindgen==0.22.0' - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 - pip install dist/*tar.gz - # py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881 - - name: Install dist w/o go - if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}} - run: pip install dist/*tar.gz - name: Install OS X dependencies if: matrix.os == 'macos-10.15' run: brew install coreutils @@ -269,13 +225,3 @@ jobs: echo "$TEST_SCRIPT" > run-and-wait.sh bash run-and-wait.sh feast serve bash run-and-wait.sh feast ui - # We disable this test for the Python 3.10 binary since it does not include Go. - - name: Smoke test with go - if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest' - run: | - cd test_repo/feature_repo - feast apply - echo "$TEST_SCRIPT" > run-and-wait.sh - pip install cffi - printf "\ngo_feature_serving: True" >> feature_store.yaml - bash run-and-wait.sh feast serve \ No newline at end of file diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 6229051db3..f4c280d682 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -112,11 +112,6 @@ jobs: with: python-version: 3.8 architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - name: Upgrade pip version run: | pip install --upgrade pip @@ -136,14 +131,7 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev + - name: Install Python dependencies run: make install-python-ci-dependencies - uses: actions/cache@v2 diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 58a765c79c..ad8700c072 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -148,11 +148,6 @@ jobs: with: python-version: 3.8 architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - name: Upgrade pip version run: | pip install --upgrade pip @@ -172,14 +167,6 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - name: Install Python dependencies run: make install-python-ci-dependencies - name: Run integration tests diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 96b2412d9f..31657d3dfc 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -15,11 +15,6 @@ jobs: with: python-version: "3.8" architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - name: Upgrade pip version run: | pip install --upgrade pip @@ -39,45 +34,8 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - name: Install dependencies run: | - make compile-protos-go make install-python-ci-dependencies - name: Lint python run: make lint-python - - lint-go: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - name: Upgrade pip version - run: | - pip install --upgrade pip - - name: Install apache-arrow on ubuntu - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - - name: Lint go - run: make lint-go \ No newline at end of file diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 976df16b60..49d6fa4f85 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -58,7 +58,7 @@ jobs: docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} outputs: DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} - integration-test-python-and-go: + integration-test-python: if: github.repository == 'feast-dev/feast' needs: build-lambda-docker-image runs-on: ${{ matrix.os }} @@ -66,7 +66,6 @@ jobs: fail-fast: false matrix: python-version: [ "3.8", "3.9", "3.10" ] - go-version: [ 1.17.0 ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} @@ -89,11 +88,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' with: @@ -131,20 +125,6 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' - run: | - brew install apache-arrow - brew install pkg-config - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 65362bcf33..1fd49f08af 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -115,11 +115,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - name: Authenticate to Google Cloud uses: 'google-github-actions/auth@v1' with: @@ -157,20 +152,6 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' - run: | - brew install apache-arrow - brew install pkg-config - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index d197901805..41df3aefff 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -57,15 +57,6 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - name: Install dependencies run: make install-python-ci-dependencies - name: Test local integration tests diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index ffd436f774..e057201572 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -25,11 +25,6 @@ jobs: with: python-version: ${{ matrix.python-version }} architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - name: Install mysql on macOS if: startsWith(matrix.os, 'macOS') run: | @@ -54,53 +49,11 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools - - name: Install apache-arrow on ubuntu - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - - name: Install apache-arrow on macos - if: matrix.os == 'macOS-latest' - run: | - brew install apache-arrow - brew install pkg-config - name: Install dependencies run: make install-python-ci-dependencies - name: Test Python run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests - unit-test-go: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=22.1,<23" - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.18.0 - - name: Install apache-arrow on ubuntu - run: | - sudo apt update - sudo apt install -y -V ca-certificates lsb-release wget - wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb - sudo apt update - sudo apt install -y -V libarrow-dev - sudo apt install -y -V pkg-config - - name: Test - run: make test-go unit-test-ui: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 8b7eb39c60..e1fd342881 100644 --- a/Makefile +++ b/Makefile @@ -24,19 +24,19 @@ TRINO_VERSION ?= 376 # General -format: format-python format-java format-go +format: format-python format-java -lint: lint-python lint-java lint-go +lint: lint-python lint-java -test: test-python test-java test-go +test: test-python test-java -protos: compile-protos-go compile-protos-python compile-protos-docs +protos: compile-protos-python compile-protos-docs build: protos build-java build-docker # Python SDK -install-python-ci-dependencies: install-go-proto-dependencies install-go-ci-dependencies +install-python-ci-dependencies: python -m piptools sync sdk/python/requirements/py$(PYTHON)-ci-requirements.txt COMPILE_GO=true python setup.py develop @@ -281,9 +281,6 @@ test-python-universal-cassandra-no-cloud-providers: test-python-universal: FEAST_USAGE=False IS_TEST=True python -m pytest -n 8 --integration sdk/python/tests -test-python-go-server: compile-go-lib - FEAST_USAGE=False IS_TEST=True pytest --integration --goserver sdk/python/tests - format-python: # Sort cd ${ROOT_DIR}/sdk/python; python -m isort feast/ tests/ @@ -334,48 +331,15 @@ test-trino-plugin-locally: kill-trino-locally: cd ${ROOT_DIR}; docker stop trino -# Go SDK & embedded - -install-go-proto-dependencies: - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 - -install-go-ci-dependencies: - # TODO: currently gopy installation doesn't work w/o explicit go get in the next line - # TODO: there should be a better way to install gopy - go get github.com/go-python/gopy@v0.4.4 - go install golang.org/x/tools/cmd/goimports - # The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy` - # but the following command is needed to install it for some reason. - go install github.com/go-python/gopy - python -m pip install pybindgen==0.22.0 protobuf==3.20.1 - install-protoc-dependencies: pip install --ignore-installed protobuf grpcio-tools==1.47.0 mypy-protobuf==3.1.0 -compile-protos-go: install-go-proto-dependencies install-protoc-dependencies - python setup.py build_go_protos - -compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - CGO_LDFLAGS_ALLOW=".*" COMPILE_GO=True python setup.py build_ext --inplace - install-feast-ci-locally: pip install -e ".[ci]" -# Needs feast package to setup the feature store -# CGO flag is due to this issue: https://github.com/golang/go/wiki/InvalidFlag -test-go: compile-protos-go compile-protos-python compile-go-lib install-feast-ci-locally - CGO_LDFLAGS_ALLOW=".*" go test -tags cgo,ccalloc ./... - -format-go: - gofmt -s -w go/ - -lint-go: compile-protos-go compile-go-lib - go vet -tags cgo,ccalloc ./go/internal/feast ./go/embedded - # Docker -build-docker: build-feature-server-python-docker build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker +build-docker: build-feature-server-python-aws-docker build-feature-transformation-server-docker build-feature-server-java-docker push-ci-docker: docker push $(REGISTRY)/feast-ci:$(VERSION) diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index cf3d9f214d..70b4ee2a92 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -58,6 +58,7 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): annotations: dict = {} """ (optional) Annotations to apply to the job container. Useful for linking the service account to IAM roles, operational metadata, etc """ + class BytewaxMaterializationEngine(BatchMaterializationEngine): def __init__( self, @@ -350,7 +351,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): "name": f"feast-{job_id}", }, ], - } + }, }, }, } diff --git a/sdk/python/tests/integration/e2e/test_go_feature_server.py b/sdk/python/tests/integration/e2e/test_go_feature_server.py deleted file mode 100644 index 0f972e45df..0000000000 --- a/sdk/python/tests/integration/e2e/test_go_feature_server.py +++ /dev/null @@ -1,263 +0,0 @@ -import threading -import time -from datetime import datetime -from typing import List - -import grpc -import pandas as pd -import pytest -import pytz -import requests - -from feast.embedded_go.online_features_service import EmbeddedOnlineFeatureServer -from feast.feast_object import FeastObject -from feast.feature_logging import LoggingConfig -from feast.feature_service import FeatureService -from feast.infra.feature_servers.base_config import FeatureLoggingConfig -from feast.protos.feast.serving.ServingService_pb2 import ( - FieldStatus, - GetOnlineFeaturesRequest, - GetOnlineFeaturesResponse, -) -from feast.protos.feast.serving.ServingService_pb2_grpc import ServingServiceStub -from feast.protos.feast.types.Value_pb2 import RepeatedValue -from feast.type_map import python_values_to_proto_values -from feast.value_type import ValueType -from feast.wait import wait_retry_backoff -from tests.integration.feature_repos.repo_configuration import ( - construct_universal_feature_views, -) -from tests.integration.feature_repos.universal.entities import ( - customer, - driver, - location, -) -from tests.utils.http_server import check_port_open, free_port -from tests.utils.test_log_creator import generate_expected_logs, get_latest_rows - - -@pytest.mark.integration -@pytest.mark.goserver -def test_go_grpc_server(grpc_client): - resp: GetOnlineFeaturesResponse = grpc_client.GetOnlineFeatures( - GetOnlineFeaturesRequest( - feature_service="driver_features", - entities={ - "driver_id": RepeatedValue( - val=python_values_to_proto_values( - [5001, 5002], feature_type=ValueType.INT64 - ) - ) - }, - full_feature_names=True, - ) - ) - assert list(resp.metadata.feature_names.val) == [ - "driver_id", - "driver_stats__conv_rate", - "driver_stats__acc_rate", - "driver_stats__avg_daily_trips", - ] - for vector in resp.results: - assert all([s == FieldStatus.PRESENT for s in vector.statuses]) - - -@pytest.mark.integration -@pytest.mark.goserver -def test_go_http_server(http_server_port): - response = requests.post( - f"http://localhost:{http_server_port}/get-online-features", - json={ - "feature_service": "driver_features", - "entities": {"driver_id": [5001, 5002]}, - "full_feature_names": True, - }, - ) - assert response.status_code == 200, response.text - response = response.json() - assert set(response.keys()) == {"metadata", "results"} - metadata = response["metadata"] - results = response["results"] - assert response["metadata"] == { - "feature_names": [ - "driver_id", - "driver_stats__conv_rate", - "driver_stats__acc_rate", - "driver_stats__avg_daily_trips", - ] - }, metadata - assert len(results) == 4, results - assert all( - set(result.keys()) == {"event_timestamps", "statuses", "values"} - for result in results - ), results - assert all( - result["statuses"] == ["PRESENT", "PRESENT"] for result in results - ), results - assert results[0]["values"] == [5001, 5002], results - for result in results[1:]: - assert len(result["values"]) == 2, result - assert all(value is not None for value in result["values"]), result - - -@pytest.mark.integration -@pytest.mark.goserver -@pytest.mark.universal_offline_stores -@pytest.mark.parametrize("full_feature_names", [True, False], ids=lambda v: str(v)) -def test_feature_logging( - grpc_client, environment, universal_data_sources, full_feature_names -): - fs = environment.feature_store - feature_service = fs.get_feature_service("driver_features") - log_start_date = datetime.now().astimezone(pytz.UTC) - driver_ids = list(range(5001, 5011)) - - for driver_id in driver_ids: - # send each driver id in separate request - grpc_client.GetOnlineFeatures( - GetOnlineFeaturesRequest( - feature_service="driver_features", - entities={ - "driver_id": RepeatedValue( - val=python_values_to_proto_values( - [driver_id], feature_type=ValueType.INT64 - ) - ) - }, - full_feature_names=full_feature_names, - ) - ) - # with some pause - time.sleep(0.1) - - _, datasets, _ = universal_data_sources - latest_rows = get_latest_rows(datasets.driver_df, "driver_id", driver_ids) - feature_view = fs.get_feature_view("driver_stats") - features = [ - feature.name - for proj in feature_service.feature_view_projections - for feature in proj.features - ] - expected_logs = generate_expected_logs( - latest_rows, feature_view, features, ["driver_id"], "event_timestamp" - ) - - def retrieve(): - retrieval_job = fs._get_provider().retrieve_feature_service_logs( - feature_service=feature_service, - start_date=log_start_date, - end_date=datetime.now().astimezone(pytz.UTC), - config=fs.config, - registry=fs._registry, - ) - try: - df = retrieval_job.to_df() - except Exception: - # Table or directory was not created yet - return None, False - - return df, df.shape[0] == len(driver_ids) - - persisted_logs = wait_retry_backoff( - retrieve, timeout_secs=60, timeout_msg="Logs retrieval failed" - ) - - persisted_logs = persisted_logs.sort_values(by="driver_id").reset_index(drop=True) - persisted_logs = persisted_logs[expected_logs.columns] - pd.testing.assert_frame_equal(expected_logs, persisted_logs, check_dtype=False) - - -""" -Start go feature server either on http or grpc based on the repo configuration for testing. -""" - - -def _server_port(environment, server_type: str): - if not environment.test_repo_config.go_feature_serving: - pytest.skip("Only for Go path") - - fs = environment.feature_store - - embedded = EmbeddedOnlineFeatureServer( - repo_path=str(fs.repo_path.absolute()), - repo_config=fs.config, - feature_store=fs, - ) - port = free_port() - if server_type == "grpc": - target = embedded.start_grpc_server - elif server_type == "http": - target = embedded.start_http_server - else: - raise ValueError("Server Type must be either 'http' or 'grpc'") - - t = threading.Thread( - target=target, - args=("127.0.0.1", port), - kwargs=dict( - enable_logging=True, - logging_options=FeatureLoggingConfig( - enabled=True, - queue_capacity=100, - write_to_disk_interval_secs=1, - flush_interval_secs=1, - emit_timeout_micro_secs=10000, - ), - ), - ) - t.start() - - wait_retry_backoff( - lambda: (None, check_port_open("127.0.0.1", port)), timeout_secs=15 - ) - - yield port - if server_type == "grpc": - embedded.stop_grpc_server() - else: - embedded.stop_http_server() - - # wait for graceful stop - time.sleep(5) - - -# Go test fixtures - - -@pytest.fixture -def initialized_registry(environment, universal_data_sources): - fs = environment.feature_store - - _, _, data_sources = universal_data_sources - feature_views = construct_universal_feature_views(data_sources) - - feature_service = FeatureService( - name="driver_features", - features=[feature_views.driver], - logging_config=LoggingConfig( - destination=environment.data_source_creator.create_logged_features_destination(), - sample_rate=1.0, - ), - ) - feast_objects: List[FeastObject] = [feature_service] - feast_objects.extend(feature_views.values()) - feast_objects.extend([driver(), customer(), location()]) - - fs.apply(feast_objects) - fs.materialize(environment.start_date, environment.end_date) - - -@pytest.fixture -def grpc_server_port(environment, initialized_registry): - yield from _server_port(environment, "grpc") - - -@pytest.fixture -def http_server_port(environment, initialized_registry): - yield from _server_port(environment, "http") - - -@pytest.fixture -def grpc_client(grpc_server_port): - ch = grpc.insecure_channel(f"localhost:{grpc_server_port}") - yield ServingServiceStub(ch) diff --git a/setup.py b/setup.py index 1ee9f8e8cb..b0e0e7603b 100644 --- a/setup.py +++ b/setup.py @@ -129,10 +129,6 @@ GE_REQUIRED = ["great_expectations>=0.15.41,<0.16.0"] -GO_REQUIRED = [ - "cffi~=1.15.0", -] - AZURE_REQUIRED = [ "azure-storage-blob>=0.37.0", "azure-identity>=1.6.1", @@ -316,93 +312,12 @@ def run(self): file.write(filedata) -def _generate_path_with_gopath(): - go_path = subprocess.check_output(["go", "env", "GOPATH"]).decode("utf-8") - go_path = go_path.strip() - path_val = os.getenv("PATH") - path_val = f"{path_val}:{go_path}/bin" - - return path_val - - -def _ensure_go_and_proto_toolchain(): - try: - version = subprocess.check_output(["go", "version"]) - except Exception as e: - raise RuntimeError("Unable to find go toolchain") from e - - semver_string = re.search(r"go[\S]+", str(version)).group().lstrip("go") - parts = semver_string.split(".") - if not (int(parts[0]) >= 1 and int(parts[1]) >= 16): - raise RuntimeError(f"Go compiler too old; expected 1.16+ found {semver_string}") - - path_val = _generate_path_with_gopath() - - try: - subprocess.check_call(["protoc-gen-go", "--version"], env={"PATH": path_val}) - subprocess.check_call( - ["protoc-gen-go-grpc", "--version"], env={"PATH": path_val} - ) - except Exception as e: - raise RuntimeError("Unable to find go/grpc extensions for protoc") from e - - -class BuildGoProtosCommand(Command): - description = "Builds the proto files into Go files." - user_options = [] - - def initialize_options(self): - self.go_protoc = [ - sys.executable, - "-m", - "grpc_tools.protoc", - ] # find_executable("protoc") - self.proto_folder = os.path.join(repo_root, "protos") - self.go_folder = os.path.join(repo_root, "go/protos") - self.sub_folders = PROTO_SUBDIRS - self.path_val = _generate_path_with_gopath() - - def finalize_options(self): - pass - - def _generate_go_protos(self, path: str): - proto_files = glob.glob(os.path.join(self.proto_folder, path)) - - try: - subprocess.check_call( - self.go_protoc - + [ - "-I", - self.proto_folder, - "--go_out", - self.go_folder, - "--go_opt=module=github.com/feast-dev/feast/go/protos", - "--go-grpc_out", - self.go_folder, - "--go-grpc_opt=module=github.com/feast-dev/feast/go/protos", - ] - + proto_files, - env={"PATH": self.path_val}, - ) - except CalledProcessError as e: - print(f"Stderr: {e.stderr}") - print(f"Stdout: {e.stdout}") - - def run(self): - go_dir = Path(repo_root) / "go" / "protos" - go_dir.mkdir(exist_ok=True) - for sub_folder in self.sub_folders: - self._generate_go_protos(f"feast/{sub_folder}/*.proto") - class BuildCommand(build_py): """Custom build command.""" def run(self): self.run_command("build_python_protos") - if os.getenv("COMPILE_GO", "false").lower() == "true": - _ensure_go_and_proto_toolchain() - self.run_command("build_go_protos") self.run_command("build_ext") build_py.run(self) @@ -414,99 +329,10 @@ class DevelopCommand(develop): def run(self): self.reinitialize_command("build_python_protos", inplace=1) self.run_command("build_python_protos") - if os.getenv("COMPILE_GO", "false").lower() == "true": - _ensure_go_and_proto_toolchain() - self.run_command("build_go_protos") develop.run(self) -class build_ext(_build_ext): - def finalize_options(self) -> None: - super().finalize_options() - if os.getenv("COMPILE_GO", "false").lower() == "false": - self.extensions = [e for e in self.extensions if not self._is_go_ext(e)] - - def _is_go_ext(self, ext: Extension): - return any( - source.endswith(".go") or source.startswith("github") - for source in ext.sources - ) - - def build_extension(self, ext: Extension): - print(f"Building extension {ext}") - if not self._is_go_ext(ext): - # the base class may mutate `self.compiler` - compiler = copy.deepcopy(self.compiler) - self.compiler, compiler = compiler, self.compiler - try: - return _build_ext.build_extension(self, ext) - finally: - self.compiler, compiler = compiler, self.compiler - - bin_path = _generate_path_with_gopath() - go_env = json.loads( - subprocess.check_output(["go", "env", "-json"]).decode("utf-8").strip() - ) - - print(f"Go env: {go_env}") - print(f"CWD: {os.getcwd()}") - - destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) - subprocess.check_call( - ["go", "install", "golang.org/x/tools/cmd/goimports"], - env={"PATH": bin_path, **go_env}, - ) - subprocess.check_call( - ["go", "get", "github.com/go-python/gopy@v0.4.4"], - env={"PATH": bin_path, **go_env}, - ) - subprocess.check_call( - ["go", "install", "github.com/go-python/gopy"], - env={"PATH": bin_path, **go_env}, - ) - subprocess.check_call( - [ - "gopy", - "build", - "-output", - destination, - "-vm", - sys.executable, - "--build-tags", - "cgo,ccalloc", - "--dynamic-link=True", - "-no-make", - *ext.sources, - ], - env={ - "PATH": bin_path, - "CGO_LDFLAGS_ALLOW": ".*", - **go_env, - }, - ) - - def copy_extensions_to_source(self): - build_py = self.get_finalized_command("build_py") - for ext in self.extensions: - fullname = self.get_ext_fullname(ext.name) - modpath = fullname.split(".") - package = ".".join(modpath[:-1]) - package_dir = build_py.get_package_dir(package) - - src_dir = dest_dir = package_dir - - if src_dir.startswith(PYTHON_CODE_PREFIX): - src_dir = package_dir[len(PYTHON_CODE_PREFIX) :] - src_dir = src_dir.lstrip("/") - - src_dir = os.path.join(self.build_lib, src_dir) - - # copy whole directory - print(f"Copying from {src_dir} to {dest_dir}") - copy_tree(src_dir, dest_dir) - - setup( name=NAME, author=AUTHOR, @@ -537,7 +363,6 @@ def copy_extensions_to_source(self): "mysql": MYSQL_REQUIRED, "ge": GE_REQUIRED, "hbase": HBASE_REQUIRED, - "go": GO_REQUIRED, "docs": DOCS_REQUIRED, "cassandra": CASSANDRA_REQUIRED, }, @@ -562,15 +387,7 @@ def copy_extensions_to_source(self): ], cmdclass={ "build_python_protos": BuildPythonProtosCommand, - "build_go_protos": BuildGoProtosCommand, "build_py": BuildCommand, "develop": DevelopCommand, - "build_ext": build_ext, }, - ext_modules=[ - Extension( - "feast.embedded_go.lib._embedded", - ["github.com/feast-dev/feast/go/embedded"], - ) - ], )