Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ML] Nodes overview for the Model Management page #116361

Merged
merged 6 commits into from
Oct 27, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion x-pack/plugins/ml/common/constants/locator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ export const ML_PAGES = {
SINGLE_METRIC_VIEWER: 'timeseriesexplorer',
DATA_FRAME_ANALYTICS_JOBS_MANAGE: 'data_frame_analytics',
DATA_FRAME_ANALYTICS_CREATE_JOB: 'data_frame_analytics/new_job',
DATA_FRAME_ANALYTICS_MODELS_MANAGE: 'data_frame_analytics/models',
TRAINED_MODELS_MANAGE: 'trained_models',
TRAINED_MODELS_NODES: 'trained_models/nodes',
DATA_FRAME_ANALYTICS_EXPLORATION: 'data_frame_analytics/exploration',
DATA_FRAME_ANALYTICS_MAP: 'data_frame_analytics/map',
/**
Expand Down
12 changes: 11 additions & 1 deletion x-pack/plugins/ml/common/types/locator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,10 @@ export interface DataFrameAnalyticsQueryState {
globalState?: MlCommonGlobalState;
}

export interface TrainedModelsQueryState {
modelId?: string;
}

export type DataFrameAnalyticsUrlState = MLPageState<
| typeof ML_PAGES.DATA_FRAME_ANALYTICS_JOBS_MANAGE
| typeof ML_PAGES.DATA_FRAME_ANALYTICS_MAP
Expand Down Expand Up @@ -250,8 +254,14 @@ export type MlLocatorState =
| DataFrameAnalyticsExplorationUrlState
| CalendarEditUrlState
| FilterEditUrlState
| MlGenericUrlState;
| MlGenericUrlState
| TrainedModelsUrlState;

export type MlLocatorParams = MlLocatorState & SerializableRecord;

export type MlLocator = LocatorPublic<MlLocatorParams>;

export type TrainedModelsUrlState = MLPageState<
typeof ML_PAGES.TRAINED_MODELS_MANAGE,
TrainedModelsQueryState | undefined
>;
81 changes: 81 additions & 0 deletions x-pack/plugins/ml/common/types/trained_models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ export interface TrainedModelStat {
}
>;
};
deployment_stats?: Omit<TrainedModelDeploymentStatsResponse, 'model_id'>;
}

type TreeNode = object;
Expand Down Expand Up @@ -95,6 +96,7 @@ export interface TrainedModelConfigResponse {
model_aliases?: string[];
} & Record<string, unknown>;
model_id: string;
model_type: 'tree_ensemble' | 'pytorch' | 'lang_ident';
tags: string[];
version: string;
inference_config?: Record<string, any>;
Expand All @@ -117,3 +119,82 @@ export interface ModelPipelines {
export interface InferenceConfigResponse {
trained_model_configs: TrainedModelConfigResponse[];
}

export interface TrainedModelDeploymentStatsResponse {
model_id: string;
model_size_bytes: number;
inference_threads: number;
model_threads: number;
state: string;
allocation_status: { target_allocation_count: number; state: string; allocation_count: number };
nodes: Array<{
node: Record<
string,
{
transport_address: string;
roles: string[];
name: string;
attributes: {
'ml.machine_memory': string;
'xpack.installed': string;
'ml.max_open_jobs': string;
'ml.max_jvm_size': string;
};
ephemeral_id: string;
}
>;
inference_count: number;
routing_state: { routing_state: string };
average_inference_time_ms: number;
last_access: number;
}>;
}

export interface NodeDeploymentStatsResponse {
id: string;
name: string;
transport_address: string;
attributes: Record<string, string>;
roles: string[];
allocated_models: Array<{
inference_threads: number;
allocation_status: {
target_allocation_count: number;
state: string;
allocation_count: number;
};
model_id: string;
state: string;
model_threads: number;
model_size_bytes: number;
}>;
memory_overview: {
machine_memory: {
/** Total machine memory in bytes */
total: number;
jvm: number;
};
/** Open anomaly detection jobs + hardcoded overhead */
anomaly_detection: {
/** Total size in bytes */
total: number;
};
/** DFA jobs currently in training + hardcoded overhead */
dfa_training: {
total: number;
};
/** Allocated trained models */
trained_models: {
total: number;
by_model: Array<{
model_id: string;
model_size: number;
}>;
};
};
}

export interface NodesOverviewResponse {
count: number;
nodes: NodeDeploymentStatsResponse[];
}
7 changes: 6 additions & 1 deletion x-pack/plugins/ml/public/application/app.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,11 @@ import { MlRouter } from './routing';
import { mlApiServicesProvider } from './services/ml_api_service';
import { HttpService } from './services/http_service';
import { ML_APP_LOCATOR, ML_PAGES } from '../../common/constants/locator';
export type MlDependencies = Omit<MlSetupDependencies, 'share' | 'indexPatternManagement'> &

export type MlDependencies = Omit<
MlSetupDependencies,
'share' | 'indexPatternManagement' | 'fieldFormats'
> &
MlStartDependencies;

interface AppProps {
Expand Down Expand Up @@ -84,6 +88,7 @@ const App: FC<AppProps> = ({ coreStart, deps, appMountParams }) => {
triggersActionsUi: deps.triggersActionsUi,
dataVisualizer: deps.dataVisualizer,
usageCollection: deps.usageCollection,
fieldFormats: deps.fieldFormats,
...coreStart,
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import React, { FC, useState, useEffect } from 'react';

import { EuiPageHeader } from '@elastic/eui';
import { EuiPageHeader, EuiBetaBadge } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { TabId } from './navigation_menu';
import { useMlKibana, useMlLocator, useNavigateToPath } from '../../contexts/kibana';
Expand All @@ -20,6 +20,7 @@ export interface Tab {
id: TabId;
name: any;
disabled: boolean;
betaTag?: JSX.Element;
}

interface Props {
Expand Down Expand Up @@ -50,6 +51,27 @@ function getTabs(disableLinks: boolean): Tab[] {
}),
disabled: disableLinks,
},
{
id: 'trained_models',
name: i18n.translate('xpack.ml.navMenu.trainedModelsTabLinkText', {
defaultMessage: 'Model Management',
}),
disabled: disableLinks,
betaTag: (
<EuiBetaBadge
label={i18n.translate('xpack.ml.navMenu.trainedModelsTabBetaLabel', {
defaultMessage: 'Experimental',
})}
size="m"
color="hollow"
iconType="beaker"
tooltipContent={i18n.translate('xpack.ml.navMenu.trainedModelsTabBetaTooltipContent', {
defaultMessage:
"Model Management is an experimental feature and subject to change. We'd love to hear your feedback.",
})}
/>
),
},
{
id: 'datavisualizer',
name: i18n.translate('xpack.ml.navMenu.dataVisualizerTabLinkText', {
Expand Down Expand Up @@ -93,6 +115,12 @@ const TAB_DATA: Record<TabId, TabData> = {
defaultMessage: 'Data Frame Analytics',
}),
},
trained_models: {
testSubject: 'mlMainTab modelManagement',
name: i18n.translate('xpack.ml.trainedModelsTabLabel', {
defaultMessage: 'Trained Models',
}),
},
datavisualizer: {
testSubject: 'mlMainTab dataVisualizer',
name: i18n.translate('xpack.ml.dataVisualizerTabLabel', {
Expand Down Expand Up @@ -173,6 +201,7 @@ export const MainTabs: FC<Props> = ({ tabId, disableLinks }) => {
},
'data-test-subj': testSubject + (id === selectedTabId ? ' selected' : ''),
isSelected: id === selectedTabId,
append: tab.betaTag,
};
})}
/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export type TabId =
| 'access-denied'
| 'anomaly_detection'
| 'data_frame_analytics'
| 'trained_models'
| 'datavisualizer'
| 'overview'
| 'settings';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import type { EmbeddableStart } from '../../../../../../../src/plugins/embeddabl
import type { MapsStartApi } from '../../../../../maps/public';
import type { DataVisualizerPluginStart } from '../../../../../data_visualizer/public';
import type { TriggersAndActionsUIPublicPluginStart } from '../../../../../triggers_actions_ui/public';
import type { FieldFormatsRegistry } from '../../../../../../../src/plugins/field_formats/common';

interface StartPlugins {
data: DataPublicPluginStart;
Expand All @@ -32,6 +33,7 @@ interface StartPlugins {
triggersActionsUi?: TriggersAndActionsUIPublicPluginStart;
dataVisualizer?: DataVisualizerPluginStart;
usageCollection?: UsageCollectionSetup;
fieldFormats: FieldFormatsRegistry;
}
export type StartServices = CoreStart &
StartPlugins & {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { useMlKibana } from './kibana_context';

export function useFieldFormatter(fieldType: 'bytes') {
const {
services: { fieldFormats },
} = useMlKibana();

const fieldFormatter = fieldFormats.deserialize({ id: fieldType });
return fieldFormatter.convert.bind(fieldFormatter);
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,6 @@ export const AnalyticsNavigationBar: FC<{
path: '/data_frame_analytics',
testSubj: 'mlAnalyticsJobsTab',
},
{
id: 'models',
name: i18n.translate('xpack.ml.dataframe.modelsTabLabel', {
defaultMessage: 'Models',
}),
path: '/data_frame_analytics/models',
testSubj: 'mlTrainedModelsTab',
},
];
if (jobId !== undefined || modelId !== undefined) {
navTabs.push({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import { NodeAvailableWarning } from '../../../components/node_available_warning
import { SavedObjectsWarning } from '../../../components/saved_objects_warning';
import { UpgradeWarning } from '../../../components/upgrade';
import { AnalyticsNavigationBar } from './components/analytics_navigation_bar';
import { ModelsList } from './components/models_management';
import { JobMap } from '../job_map';
import { usePageUrlState } from '../../../util/url_state';
import { ListingPageUrlState } from '../../../../../common/types/common';
Expand Down Expand Up @@ -125,7 +124,6 @@ export const Page: FC = () => {
updatePageState={setDfaPageState}
/>
)}
{selectedTabId === 'models' && <ModelsList />}
</EuiPageContent>
</EuiPageBody>
</EuiPage>
Expand Down
8 changes: 8 additions & 0 deletions x-pack/plugins/ml/public/application/routing/breadcrumbs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,13 @@ export const DATA_FRAME_ANALYTICS_BREADCRUMB: ChromeBreadcrumb = Object.freeze({
href: '/data_frame_analytics',
});

export const TRAINED_MODELS: ChromeBreadcrumb = Object.freeze({
text: i18n.translate('xpack.ml.trainedModelsLabel', {
defaultMessage: 'Trained Models',
}),
href: '/trained_models',
});

export const DATA_VISUALIZER_BREADCRUMB: ChromeBreadcrumb = Object.freeze({
text: i18n.translate('xpack.ml.datavisualizerBreadcrumbLabel', {
defaultMessage: 'Data Visualizer',
Expand Down Expand Up @@ -74,6 +81,7 @@ const breadcrumbs = {
SETTINGS_BREADCRUMB,
ANOMALY_DETECTION_BREADCRUMB,
DATA_FRAME_ANALYTICS_BREADCRUMB,
TRAINED_MODELS,
DATA_VISUALIZER_BREADCRUMB,
CREATE_JOB_BREADCRUMB,
CALENDAR_MANAGEMENT_BREADCRUMB,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,4 @@
export * from './analytics_jobs_list';
export * from './analytics_job_exploration';
export * from './analytics_job_creation';
export * from './models_list';
export * from './analytics_map';
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ export * from './data_frame_analytics';
export { timeSeriesExplorerRouteFactory } from './timeseriesexplorer';
export * from './explorer';
export * from './access_denied';
export * from './trained_models';
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

export * from './models_list';
export * from './nodes_list';
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,20 @@ import { NavigateToPath } from '../../../contexts/kibana';
import { MlRoute, PageLoader, PageProps } from '../../router';
import { useResolver } from '../../use_resolver';
import { basicResolvers } from '../../resolvers';
import { Page } from '../../../data_frame_analytics/pages/analytics_management';
import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs';
import { Page } from '../../../trained_models';

export const modelsListRouteFactory = (
navigateToPath: NavigateToPath,
basePath: string
): MlRoute => ({
path: '/data_frame_analytics/models',
path: '/trained_models',
render: (props, deps) => <PageWrapper {...props} deps={deps} />,
breadcrumbs: [
getBreadcrumbWithUrlForApp('ML_BREADCRUMB', navigateToPath, basePath),
getBreadcrumbWithUrlForApp('DATA_FRAME_ANALYTICS_BREADCRUMB', navigateToPath, basePath),
getBreadcrumbWithUrlForApp('TRAINED_MODELS', navigateToPath, basePath),
{
text: i18n.translate('xpack.ml.dataFrameAnalyticsBreadcrumbs.modelsListLabel', {
text: i18n.translate('xpack.ml.trainedModelsBreadcrumbs.modelsListLabel', {
defaultMessage: 'Model Management',
}),
href: '',
Expand Down
Loading