From ca9aa21486b03c76676ec226a2d73edd553aece4 Mon Sep 17 00:00:00 2001 From: Enya-Yx <108409954+enya-yx@users.noreply.github.com> Date: Sun, 6 Nov 2022 12:01:42 +0800 Subject: [PATCH 1/9] Add exception if materialize features defined on 'INPUT_CONTEXT' (#785) * Add checking and error message if materialize features defined on INPUT_CONTEXT --- feathr_project/feathr/client.py | 12 +++++++++-- .../test/test_feature_materialization.py | 21 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py index b14bf868e..63cd07c1e 100644 --- a/feathr_project/feathr/client.py +++ b/feathr_project/feathr/client.py @@ -32,6 +32,7 @@ from feathr.utils._file_utils import write_to_file from feathr.utils.feature_printer import FeaturePrinter from feathr.utils.spark_job_params import FeatureGenerationJobParams, FeatureJoinJobParams +from feathr.definition.source import InputContext class FeathrClient(object): @@ -619,8 +620,15 @@ def materialize_features(self, settings: MaterializationSettings, execution_conf allow_materialize_non_agg_feature: Materializing non-aggregated features (the features without WindowAggTransformation) doesn't output meaningful results so it's by default set to False, but if you really want to materialize non-aggregated features, set this to True. """ feature_list = settings.feature_names - if len(feature_list) > 0 and not self._valid_materialize_keys(feature_list): - raise RuntimeError(f"Invalid materialization features: {feature_list}, since they have different keys. Currently Feathr only supports materializing features of the same keys.") + if len(feature_list) > 0: + if 'anchor_list' in dir(self): + anchors = [anchor for anchor in self.anchor_list if isinstance(anchor.source, InputContext)] + anchor_feature_names = set(feature.name for anchor in anchors for feature in anchor.features) + for feature in feature_list: + if feature in anchor_feature_names: + raise RuntimeError(f"Materializing features that are defined on INPUT_CONTEXT is not supported. {feature} is defined on INPUT_CONTEXT so you should remove it from the feature list in MaterializationSettings.") + if not self._valid_materialize_keys(feature_list): + raise RuntimeError(f"Invalid materialization features: {feature_list}, since they have different keys. Currently Feathr only supports materializing features of the same keys.") if not allow_materialize_non_agg_feature: # Check if there are non-aggregation features in the list diff --git a/feathr_project/test/test_feature_materialization.py b/feathr_project/test/test_feature_materialization.py index e8100578c..754c12ebb 100644 --- a/feathr_project/test/test_feature_materialization.py +++ b/feathr_project/test/test_feature_materialization.py @@ -18,6 +18,8 @@ from test_fixture import basic_test_setup from test_fixture import get_online_test_table_name from test_utils.constants import Constants +from logging import raiseExceptions +import pytest def test_feature_materialization_config(): backfill_time = BackfillTime(start=datetime(2020, 5, 20), end=datetime(2020, 5,20), step=timedelta(days=1)) @@ -255,4 +257,21 @@ def test_delete_feature_from_redis(): res = client.get_online_features(online_test_table, '265', ['f_location_avg_fare']) assert len(res) == 1 - assert res[0] == None \ No newline at end of file + assert res[0] == None + +def test_feature_list_on_input_context(): + with pytest.raises(RuntimeError) as e_info: + test_workspace_dir = Path(__file__).parent.resolve() / "test_user_workspace" + + client: FeathrClient = basic_test_setup(os.path.join(test_workspace_dir, "feathr_config.yaml")) + online_test_table = get_online_test_table_name('nycTaxiCITableDeletion') + redisSink = RedisSink(table_name=online_test_table) + settings = MaterializationSettings(name="py_udf", + sinks=[redisSink], + feature_names=[ + "f_location_avg_fare", + "f_day_of_week" + ]) + client.materialize_features(settings, allow_materialize_non_agg_feature=True) + assert e_info is not None + assert e_info.value.args[0] == "Materializing features that are defined on INPUT_CONTEXT is not supported. f_day_of_week is defined on INPUT_CONTEXT so you should remove it from the feature list in MaterializationSettings." \ No newline at end of file From b03932d3a301b90b76098b9d762eebbc8ec2c6c2 Mon Sep 17 00:00:00 2001 From: Boli Guan Date: Mon, 7 Nov 2022 17:26:12 +0800 Subject: [PATCH 2/9] Fix only first Key will show even if multiple keys are added (#837) * Fix only first Key will show even if multiple keys are added Signed-off-by: Boli Guan * remove react-custom-scrollbars-2 dependencie Signed-off-by: Boli Guan Signed-off-by: Boli Guan --- ui/package-lock.json | 102 ++++++++++++++++-- ui/src/components/CardDescriptions/index.tsx | 4 +- ui/src/models/model.ts | 2 +- .../NodeDetails/FeatureNodeDetail.tsx | 22 ++-- .../components/NodeDetails/index.module.less | 10 ++ .../feature/components/NodeDetails/index.tsx | 5 +- ui/src/pages/feature/featureDetails.tsx | 27 +++-- ui/src/utils/attributesMapping.ts | 2 +- ui/src/utils/utils.tsx | 10 ++ 9 files changed, 159 insertions(+), 25 deletions(-) create mode 100644 ui/src/pages/feature/components/NodeDetails/index.module.less diff --git a/ui/package-lock.json b/ui/package-lock.json index 480dfdc62..b9dbe29b9 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "feathr-ui", - "version": "0.1.0", + "version": "0.9.0-rc2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "feathr-ui", - "version": "0.1.0", + "version": "0.9.0-rc2", "dependencies": { "@ant-design/icons": "^4.7.0", "@azure/msal-browser": "^2.24.0", @@ -4446,6 +4446,11 @@ "node": ">=0.4.0" } }, + "node_modules/add-px-to-style": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/add-px-to-style/-/add-px-to-style-1.0.0.tgz", + "integrity": "sha512-YMyxSlXpPjD8uWekCQGuN40lV4bnZagUwqa2m/uFv1z/tNImSk9fnXVMUI5qwME/zzI3MMQRvjZ+69zyfSSyew==" + }, "node_modules/address": { "version": "1.2.0", "dev": true, @@ -6803,6 +6808,16 @@ "utila": "~0.4" } }, + "node_modules/dom-css": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/dom-css/-/dom-css-2.1.0.tgz", + "integrity": "sha512-w9kU7FAbaSh3QKijL6n59ofAhkkmMJ31GclJIz/vyQdjogfyxcB6Zf8CZyibOERI5o0Hxz30VmJS7+7r5fEj2Q==", + "dependencies": { + "add-px-to-style": "1.0.0", + "prefix-style": "2.0.1", + "to-camel-case": "1.0.0" + } + }, "node_modules/dom-serializer": { "version": "0.2.2", "dev": true, @@ -12287,7 +12302,6 @@ }, "node_modules/performance-now": { "version": "2.1.0", - "dev": true, "license": "MIT" }, "node_modules/picocolors": { @@ -13598,6 +13612,11 @@ "dev": true, "license": "MIT" }, + "node_modules/prefix-style": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/prefix-style/-/prefix-style-2.0.1.tgz", + "integrity": "sha512-gdr1MBNVT0drzTq95CbSNdsrBDoHGlb2aDJP/FoY+1e+jSDPOb1Cv554gH2MGiSr2WTcXi/zu+NaFzfcHQkfBQ==" + }, "node_modules/prelude-ls": { "version": "1.2.1", "dev": true, @@ -13810,7 +13829,6 @@ }, "node_modules/raf": { "version": "3.4.1", - "dev": true, "license": "MIT", "dependencies": { "performance-now": "^2.1.0" @@ -16376,6 +16394,14 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/to-camel-case": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-camel-case/-/to-camel-case-1.0.0.tgz", + "integrity": "sha512-nD8pQi5H34kyu1QDMFjzEIYqk0xa9Alt6ZfrdEMuHCFOfTLhDG5pgTu/aAM9Wt9lXILwlXmWP43b8sav0GNE8Q==", + "dependencies": { + "to-space-case": "^1.0.0" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "dev": true, @@ -16384,6 +16410,11 @@ "node": ">=4" } }, + "node_modules/to-no-case": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/to-no-case/-/to-no-case-1.0.2.tgz", + "integrity": "sha512-Z3g735FxuZY8rodxV4gH7LxClE4H0hTIyHNIHdk+vpQxjLm0cwnKXq/OFVZ76SOQmto7txVcwSCwkU5kqp+FKg==" + }, "node_modules/to-regex-range": { "version": "5.0.1", "dev": true, @@ -16395,6 +16426,14 @@ "node": ">=8.0" } }, + "node_modules/to-space-case": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-space-case/-/to-space-case-1.0.0.tgz", + "integrity": "sha512-rLdvwXZ39VOn1IxGL3V6ZstoTbwLRckQmn/U8ZDLuWwIXNpuZDhQ3AiRUlhTbOXFVE9C+dR51wM0CBDhk31VcA==", + "dependencies": { + "to-no-case": "^1.0.0" + } + }, "node_modules/toggle-selection": { "version": "1.0.6", "license": "MIT" @@ -20394,6 +20433,11 @@ "version": "7.2.0", "dev": true }, + "add-px-to-style": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/add-px-to-style/-/add-px-to-style-1.0.0.tgz", + "integrity": "sha512-YMyxSlXpPjD8uWekCQGuN40lV4bnZagUwqa2m/uFv1z/tNImSk9fnXVMUI5qwME/zzI3MMQRvjZ+69zyfSSyew==" + }, "address": { "version": "1.2.0", "dev": true @@ -21911,6 +21955,16 @@ "utila": "~0.4" } }, + "dom-css": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/dom-css/-/dom-css-2.1.0.tgz", + "integrity": "sha512-w9kU7FAbaSh3QKijL6n59ofAhkkmMJ31GclJIz/vyQdjogfyxcB6Zf8CZyibOERI5o0Hxz30VmJS7+7r5fEj2Q==", + "requires": { + "add-px-to-style": "1.0.0", + "prefix-style": "2.0.1", + "to-camel-case": "1.0.0" + } + }, "dom-serializer": { "version": "0.2.2", "dev": true, @@ -25479,8 +25533,7 @@ "dev": true }, "performance-now": { - "version": "2.1.0", - "dev": true + "version": "2.1.0" }, "picocolors": { "version": "1.0.0", @@ -26181,6 +26234,11 @@ "version": "4.2.0", "dev": true }, + "prefix-style": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/prefix-style/-/prefix-style-2.0.1.tgz", + "integrity": "sha512-gdr1MBNVT0drzTq95CbSNdsrBDoHGlb2aDJP/FoY+1e+jSDPOb1Cv554gH2MGiSr2WTcXi/zu+NaFzfcHQkfBQ==" + }, "prelude-ls": { "version": "1.2.1", "dev": true @@ -26306,7 +26364,6 @@ }, "raf": { "version": "3.4.1", - "dev": true, "requires": { "performance-now": "^2.1.0" } @@ -26746,6 +26803,16 @@ "whatwg-fetch": "^3.6.2" } }, + "react-custom-scrollbars-2": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/react-custom-scrollbars-2/-/react-custom-scrollbars-2-4.5.0.tgz", + "integrity": "sha512-/z0nWAeXfMDr4+OXReTpYd1Atq9kkn4oI3qxq3iMXGQx1EEfwETSqB8HTAvg1X7dEqcCachbny1DRNGlqX5bDQ==", + "requires": { + "dom-css": "^2.0.0", + "prop-types": "^15.5.10", + "raf": "^3.1.0" + } + }, "react-dev-utils": { "version": "12.0.1", "dev": true, @@ -28009,10 +28076,23 @@ "version": "1.0.5", "dev": true }, + "to-camel-case": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-camel-case/-/to-camel-case-1.0.0.tgz", + "integrity": "sha512-nD8pQi5H34kyu1QDMFjzEIYqk0xa9Alt6ZfrdEMuHCFOfTLhDG5pgTu/aAM9Wt9lXILwlXmWP43b8sav0GNE8Q==", + "requires": { + "to-space-case": "^1.0.0" + } + }, "to-fast-properties": { "version": "2.0.0", "dev": true }, + "to-no-case": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/to-no-case/-/to-no-case-1.0.2.tgz", + "integrity": "sha512-Z3g735FxuZY8rodxV4gH7LxClE4H0hTIyHNIHdk+vpQxjLm0cwnKXq/OFVZ76SOQmto7txVcwSCwkU5kqp+FKg==" + }, "to-regex-range": { "version": "5.0.1", "dev": true, @@ -28020,6 +28100,14 @@ "is-number": "^7.0.0" } }, + "to-space-case": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-space-case/-/to-space-case-1.0.0.tgz", + "integrity": "sha512-rLdvwXZ39VOn1IxGL3V6ZstoTbwLRckQmn/U8ZDLuWwIXNpuZDhQ3AiRUlhTbOXFVE9C+dR51wM0CBDhk31VcA==", + "requires": { + "to-no-case": "^1.0.0" + } + }, "toggle-selection": { "version": "1.0.6" }, diff --git a/ui/src/components/CardDescriptions/index.tsx b/ui/src/components/CardDescriptions/index.tsx index 9c0d41498..dffdec77d 100644 --- a/ui/src/components/CardDescriptions/index.tsx +++ b/ui/src/components/CardDescriptions/index.tsx @@ -1,6 +1,8 @@ import React from "react"; import { Card, Descriptions } from "antd"; +import { isEmpty } from "@/utils/utils"; + export interface CardDescriptionsProps { title?: string; mapping: any[]; @@ -10,7 +12,7 @@ export interface CardDescriptionsProps { const CardDescriptions = (props: CardDescriptionsProps) => { const { title, mapping, descriptions } = props; - return descriptions ? ( + return !isEmpty(descriptions) ? ( {mapping.reduce((list: any, item) => { diff --git a/ui/src/models/model.ts b/ui/src/models/model.ts index c45d0632f..b6da49361 100644 --- a/ui/src/models/model.ts +++ b/ui/src/models/model.ts @@ -20,7 +20,7 @@ export interface FeatureAttributes { key: FeatureKey[]; name: string; qualifiedName: string; - tags: string[]; + tags: Object; transformation: FeatureTransformation; type: FeatureType; } diff --git a/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx b/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx index 868c866a7..9aa3e533e 100644 --- a/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx +++ b/ui/src/pages/feature/components/NodeDetails/FeatureNodeDetail.tsx @@ -7,6 +7,7 @@ import { FeatureKeyMap, TypeMap, } from "@/utils/attributesMapping"; +import { getJSONMap } from "@/utils/utils"; export interface FeatureNodeDetialProps { feature: Feature; @@ -16,8 +17,9 @@ const FeatureNodeDetial = (props: FeatureNodeDetialProps) => { const { feature } = props; const { attributes } = feature; - const { transformation, key, type } = attributes; - const FeatureKey = key?.[0]; + const { transformation, key, type, tags } = attributes; + + const tagsMap = getJSONMap(tags); return ( { mapping={TransformationMap} descriptions={transformation} /> - + {key.map((item, index) => { + return ( + + ); + })} + ); }; diff --git a/ui/src/pages/feature/components/NodeDetails/index.module.less b/ui/src/pages/feature/components/NodeDetails/index.module.less new file mode 100644 index 000000000..9e9815d75 --- /dev/null +++ b/ui/src/pages/feature/components/NodeDetails/index.module.less @@ -0,0 +1,10 @@ +.wrap { + :global { + .ant-space { + margin-bottom: 16px; + } + .card { + box-shadow: none; + } + } +} diff --git a/ui/src/pages/feature/components/NodeDetails/index.tsx b/ui/src/pages/feature/components/NodeDetails/index.tsx index 8a3391cfd..edbce587d 100644 --- a/ui/src/pages/feature/components/NodeDetails/index.tsx +++ b/ui/src/pages/feature/components/NodeDetails/index.tsx @@ -8,6 +8,8 @@ import { FeatureType } from "@/utils/utils"; import FeatureNodeDetail from "./FeatureNodeDetail"; import SourceNodeDetial from "./SourceNodeDetial"; +import styles from "./index.module.less"; + const { Paragraph } = Typography; const NodeDetails = () => { @@ -37,10 +39,11 @@ const NodeDetails = () => { return ( } > -
+
{data ? ( isSource ? ( diff --git a/ui/src/pages/feature/featureDetails.tsx b/ui/src/pages/feature/featureDetails.tsx index fdecb7505..24f782160 100644 --- a/ui/src/pages/feature/featureDetails.tsx +++ b/ui/src/pages/feature/featureDetails.tsx @@ -22,6 +22,7 @@ import { TransformationMap, TypeMap, } from "@/utils/attributesMapping"; +import { getJSONMap } from "@/utils/utils"; const contentStyle = { marginRight: 16 }; @@ -134,8 +135,9 @@ const FeatureDetails = () => { } ); const { attributes } = data; - const { transformation, key, type, name } = attributes; - const FeatureKey = key?.[0]; + const { transformation, key, type, name, tags } = attributes; + + const tagsMap = getJSONMap(tags); return (
@@ -175,16 +177,27 @@ const FeatureDetails = () => { mapping={TransformationMap} descriptions={transformation} /> - + {key.map((item, index) => { + return ( + + ); + })} + + diff --git a/ui/src/utils/attributesMapping.ts b/ui/src/utils/attributesMapping.ts index 09e7459b7..9e888d1d0 100644 --- a/ui/src/utils/attributesMapping.ts +++ b/ui/src/utils/attributesMapping.ts @@ -42,7 +42,7 @@ export const SourceAttributesMap: Array<{ { label: "Path", key: "path" }, { label: "Preprocessing", key: "preprocessing" }, { label: "Event Timestamp Column", key: "event_timestamp_column" }, - { label: "Timestamp Forma", key: "timestamp_format" }, + { label: "Timestamp Format", key: "timestamp_format" }, { label: "Qualified Name", key: "qualified_name" }, { label: "Tags", key: "tags" }, ]; diff --git a/ui/src/utils/utils.tsx b/ui/src/utils/utils.tsx index 9cd2c959b..28b066233 100644 --- a/ui/src/utils/utils.tsx +++ b/ui/src/utils/utils.tsx @@ -48,3 +48,13 @@ export const getFeatureDetailUrl = (project: string, feature: Feature) => { return; } }; + +export const getJSONMap = (json: any = {}) => { + return Object.keys(json).map((key) => { + return { label: key, key }; + }); +}; + +export const isEmpty = (obj: any = {}) => { + return !obj || Object.getOwnPropertyNames(obj).length === 0; +}; From a0f91eb3127efdec65c744d09b2ab9da2fc21b8a Mon Sep 17 00:00:00 2001 From: Boli Guan Date: Tue, 8 Nov 2022 13:44:51 +0800 Subject: [PATCH 3/9] Move the version information to the bottom of the sidemenu. (#832) Signed-off-by: Boli Guan Signed-off-by: Boli Guan --- ui/src/app.tsx | 4 +- ui/src/components/footer/index.module.less | 12 ---- ui/src/components/footer/index.tsx | 25 -------- ui/src/components/header/header.css | 31 ---------- ui/src/components/header/header.tsx | 21 ++++--- ui/src/components/header/index.module.less | 48 +++++++++++++++ ui/src/components/sidemenu/VersionBar.tsx | 22 +++++++ ui/src/components/sidemenu/index.module.less | 19 ++++++ ui/src/components/sidemenu/siteMenu.tsx | 65 +++++++++++++++----- ui/src/site.css | 1 - 10 files changed, 149 insertions(+), 99 deletions(-) delete mode 100644 ui/src/components/footer/index.module.less delete mode 100644 ui/src/components/footer/index.tsx delete mode 100644 ui/src/components/header/header.css create mode 100644 ui/src/components/header/index.module.less create mode 100644 ui/src/components/sidemenu/VersionBar.tsx create mode 100644 ui/src/components/sidemenu/index.module.less diff --git a/ui/src/app.tsx b/ui/src/app.tsx index 8b43cc70a..b3d2b317a 100644 --- a/ui/src/app.tsx +++ b/ui/src/app.tsx @@ -20,7 +20,6 @@ import RoleManagement from "./pages/management/roleManagement"; import Home from "./pages/home/home"; import Projects from "./pages/project/projects"; import { getMsalConfig } from "./utils/utils"; -import Footer from "@/components/footer"; const queryClient = new QueryClient(); @@ -34,7 +33,7 @@ const App = () => { - +
@@ -69,7 +68,6 @@ const App = () => { /> -