[Ingest Manager] Fix and improve agent status

x-pack/plugins/ingest_manager/common/services/agent_status.ts

@@ -5,63 +5,52 @@
  */
 
 import {
-  AGENT_TYPE_TEMPORARY,
   AGENT_POLLING_THRESHOLD_MS,
   AGENT_TYPE_PERMANENT,
-  AGENT_TYPE_EPHEMERAL,
   AGENT_SAVED_OBJECT_TYPE,
 } from '../constants';
 import { Agent, AgentStatus } from '../types';
 
 export function getAgentStatus(agent: Agent, now: number = Date.now()): AgentStatus {
-  const { type, last_checkin: lastCheckIn } = agent;
-  const msLastCheckIn = new Date(lastCheckIn || 0).getTime();
-  const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn;
-  const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS);
+  const { last_checkin: lastCheckIn } = agent;
+
   if (!agent.active) {
     return 'inactive';
   }
+  if (!agent.last_checkin) {
+    return 'enrolling';
+  }
   if (agent.unenrollment_started_at && !agent.unenrolled_at) {
     return 'unenrolling';
   }
-  if (agent.current_error_events.length > 0) {
+
+  const msLastCheckIn = new Date(lastCheckIn || 0).getTime();
+  const msSinceLastCheckIn = new Date().getTime() - msLastCheckIn;
+  const intervalsSinceLastCheckIn = Math.floor(msSinceLastCheckIn / AGENT_POLLING_THRESHOLD_MS);
+
+  if (agent.last_checkin_status === 'error') {
     return 'error';
   }
-  switch (type) {
-    case AGENT_TYPE_PERMANENT:
-      if (intervalsSinceLastCheckIn >= 4) {
-        return 'error';
-      }
-    case AGENT_TYPE_TEMPORARY:
-      if (intervalsSinceLastCheckIn >= 3) {
-        return 'offline';
-      }
-    case AGENT_TYPE_EPHEMERAL:
-      if (intervalsSinceLastCheckIn >= 3) {
-        return 'inactive';
-      }
+  if (agent.last_checkin_status === 'degraded') {
+    return 'degraded';
+  }
+  if (intervalsSinceLastCheckIn >= 4) {
+    return 'offline';
   }
+
   return 'online';
 }
 
 export function buildKueryForOnlineAgents() {
-  return `(${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
-    (4 * AGENT_POLLING_THRESHOLD_MS) / 1000
-  }s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
-    (3 * AGENT_POLLING_THRESHOLD_MS) / 1000
-  }s) or (${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_EPHEMERAL} and ${AGENT_SAVED_OBJECT_TYPE}.last_checkin >= now-${
-    (3 * AGENT_POLLING_THRESHOLD_MS) / 1000
-  }s)`;
+  return `not (${buildKueryForOfflineAgents()}) AND not (${buildKueryForErrorAgents()})`;
 }
 
-export function buildKueryForOfflineAgents() {
-  return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_TEMPORARY} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
-    (3 * AGENT_POLLING_THRESHOLD_MS) / 1000
-  }s`;
+export function buildKueryForErrorAgents() {
+  return `( ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:error or ${AGENT_SAVED_OBJECT_TYPE}.last_checkin_status:degraded )`;
 }
 
-export function buildKueryForErrorAgents() {
-  return `${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
+export function buildKueryForOfflineAgents() {
+  return `((${AGENT_SAVED_OBJECT_TYPE}.type:${AGENT_TYPE_PERMANENT} AND ${AGENT_SAVED_OBJECT_TYPE}.last_checkin < now-${
     (4 * AGENT_POLLING_THRESHOLD_MS) / 1000
-  }s`;
+  }s) AND not ( ${buildKueryForErrorAgents()} ))`;
 }

x-pack/plugins/ingest_manager/common/types/models/agent.ts

@@ -11,7 +11,16 @@ export type AgentType =
   | typeof AGENT_TYPE_PERMANENT
   | typeof AGENT_TYPE_TEMPORARY;
 
-export type AgentStatus = 'offline' | 'error' | 'online' | 'inactive' | 'warning' | 'unenrolling';
+export type AgentStatus =
+  | 'offline'
+  | 'error'
+  | 'online'
+  | 'inactive'
+  | 'warning'
+  | 'enrolling'
+  | 'unenrolling'
+  | 'degraded';
+
 export type AgentActionType = 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE' | 'UNENROLL';
 export interface NewAgentAction {
   type: AgentActionType;
@@ -82,6 +91,7 @@ interface AgentBase {
   config_id?: string;
   config_revision?: number | null;
   last_checkin?: string;
+  last_checkin_status?: 'error' | 'online' | 'degraded';
   user_provided_metadata: AgentMetadata;
   local_metadata: AgentMetadata;
 }

x-pack/plugins/ingest_manager/common/types/rest_spec/agent.ts

@@ -47,6 +47,7 @@ export interface PostAgentCheckinRequest {
     agentId: string;
   };
   body: {
+    status?: 'online' | 'error' | 'degraded';
     local_metadata?: Record<string, any>;
     events?: NewAgentEvent[];
   };

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/agent_list_page/index.tsx

@@ -178,11 +178,7 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
   }
 
   if (selectedStatus.length) {
-    if (kuery) {
-      kuery = `(${kuery}) and`;
-    }
-
-    kuery = selectedStatus
+    const kueryStatus = selectedStatus
       .map((status) => {
         switch (status) {
           case 'online':
@@ -196,6 +192,12 @@ export const AgentListPage: React.FunctionComponent<{}> = () => {
         return '';
       })
       .join(' or ');
+
+    if (kuery) {
+      kuery = `(${kuery}) and ${kueryStatus}`;
+    } else {
+      kuery = kueryStatus;
+    }
   }
 
   const agentsRequest = useGetAgents(

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/fleet/components/agent_health.tsx

@@ -53,6 +53,22 @@ const Status = {
       />
     </EuiHealth>
   ),
+  Degraded: (
+    <EuiHealth color="danger">
+      <FormattedMessage
+        id="xpack.ingestManager.agentHealth.degradedStatusText"
+        defaultMessage="Degraded"
+      />
+    </EuiHealth>
+  ),
+  Enrolling: (
+    <EuiHealth color="warning">
+      <FormattedMessage
+        id="xpack.ingestManager.agentHealth.enrollingStatusText"
+        defaultMessage="Enrolling"
+      />
+    </EuiHealth>
+  ),
   Unenrolling: (
     <EuiHealth color="warning">
       <FormattedMessage
@@ -67,6 +83,8 @@ function getStatusComponent(agent: Agent): React.ReactElement {
   switch (agent.status) {
     case 'error':
       return Status.Error;
+    case 'degraded':
+      return Status.Degraded;
     case 'inactive':
       return Status.Inactive;
     case 'offline':
@@ -75,6 +93,8 @@ function getStatusComponent(agent: Agent): React.ReactElement {
       return Status.Warning;
     case 'unenrolling':
       return Status.Unenrolling;
+    case 'enrolling':
+      return Status.Enrolling;
     default:
       return Status.Online;
   }

x-pack/plugins/ingest_manager/server/routes/agent/handlers.ts

@@ -178,8 +178,11 @@ export const postAgentCheckinHandler: RequestHandler<
     const { actions } = await AgentService.agentCheckin(
       soClient,
       agent,
-      request.body.events || [],
-      request.body.local_metadata,
+      {
+        events: request.body.events || [],
+        localMetadata: request.body.local_metadata,
+        status: request.body.status,
+      },
       { signal }
     );
     const body: PostAgentCheckinResponse = {

x-pack/plugins/ingest_manager/server/saved_objects/index.ts

@@ -63,6 +63,7 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
         config_id: { type: 'keyword' },
         last_updated: { type: 'date' },
         last_checkin: { type: 'date' },
+        last_checkin_status: { type: 'keyword' },
         config_revision: { type: 'integer' },
         default_api_key_id: { type: 'keyword' },
         default_api_key: { type: 'binary', index: false },
@@ -310,6 +311,7 @@ export function registerEncryptedSavedObjects(
       'config_id',
       'last_updated',
       'last_checkin',
+      'last_checkin_status',
       'config_revision',
       'config_newest_revision',
       'updated_at',

x-pack/plugins/ingest_manager/server/services/agents/checkin/index.ts

@@ -11,7 +11,6 @@ import {
   AgentEvent,
   AgentSOAttributes,
   AgentEventSOAttributes,
-  AgentMetadata,
 } from '../../../types';
 
 import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../constants';
@@ -21,20 +20,24 @@ import { getAgentActionsForCheckin } from '../actions';
 export async function agentCheckin(
   soClient: SavedObjectsClientContract,
   agent: Agent,
-  events: NewAgentEvent[],
-  localMetadata?: any,
+  data: {
+    events: NewAgentEvent[];
+    localMetadata?: any;
+    status?: 'online' | 'error' | 'degraded';
+  },
   options?: { signal: AbortSignal }
 ) {
-  const updateData: {
-    local_metadata?: AgentMetadata;
-    current_error_events?: string;
-  } = {};
-  const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, events);
+  const updateData: Partial<AgentSOAttributes> = {};
+  const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, data.events);
   if (updatedErrorEvents) {
     updateData.current_error_events = JSON.stringify(updatedErrorEvents);
   }
-  if (localMetadata) {
-    updateData.local_metadata = localMetadata;
+  if (data.localMetadata) {
+    updateData.local_metadata = data.localMetadata;
+  }
+
+  if (data.status !== agent.last_checkin_status) {
+    updateData.last_checkin_status = data.status;
   }
   if (Object.keys(updateData).length > 0) {
     await soClient.update<AgentSOAttributes>(AGENT_SAVED_OBJECT_TYPE, agent.id, updateData);

x-pack/plugins/ingest_manager/server/services/agents/checkin/state_connected_agents.ts

@@ -59,7 +59,7 @@ export function agentCheckinStateConnectedAgentsFactory() {
     const internalSOClient = getInternalUserSOClient();
     const now = new Date().toISOString();
     const updates: Array<SavedObjectsBulkUpdateObject<AgentSOAttributes>> = [
-      ...connectedAgentsIds.values(),
+      ...agentToUpdate.values(),
     ].map((agentId) => ({
       type: AGENT_SAVED_OBJECT_TYPE,
       id: agentId,

x-pack/plugins/ingest_manager/server/services/agents/status.test.ts

@@ -33,11 +33,44 @@ describe('Agent status service', () => {
       type: AGENT_TYPE_PERMANENT,
       attributes: {
         active: true,
+        last_checkin: new Date().toISOString(),
         local_metadata: {},
         user_provided_metadata: {},
       },
     } as SavedObject<AgentSOAttributes>);
     const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
     expect(status).toEqual('online');
   });
+
+  it('should return enrolling when agent is active but never checkin', async () => {
+    const mockSavedObjectsClient = savedObjectsClientMock.create();
+    mockSavedObjectsClient.get = jest.fn().mockReturnValue({
+      id: 'id',
+      type: AGENT_TYPE_PERMANENT,
+      attributes: {
+        active: true,
+        local_metadata: {},
+        user_provided_metadata: {},
+      },
+    } as SavedObject<AgentSOAttributes>);
+    const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
+    expect(status).toEqual('enrolling');
+  });
+
+  it('should return unenrolling when agent is unenrolling', async () => {
+    const mockSavedObjectsClient = savedObjectsClientMock.create();
+    mockSavedObjectsClient.get = jest.fn().mockReturnValue({
+      id: 'id',
+      type: AGENT_TYPE_PERMANENT,
+      attributes: {
+        active: true,
+        last_checkin: new Date().toISOString(),
+        unenrollment_started_at: new Date().toISOString(),
+        local_metadata: {},
+        user_provided_metadata: {},
+      },
+    } as SavedObject<AgentSOAttributes>);
+    const status = await getAgentStatusById(mockSavedObjectsClient, 'id');
+    expect(status).toEqual('unenrolling');
+  });
 });

x-pack/plugins/ingest_manager/server/types/rest_spec/agent.ts

@@ -32,6 +32,9 @@ export const PostAgentCheckinRequestSchema = {
     agentId: schema.string(),
   }),
   body: schema.object({
+    status: schema.maybe(
+      schema.oneOf([schema.literal('online'), schema.literal('error'), schema.literal('degraded')])
+    ),
     local_metadata: schema.maybe(schema.recordOf(schema.string(), schema.any())),
     events: schema.maybe(schema.arrayOf(NewAgentEventSchema)),
   }),