Skip to content

Commit a268837

Browse files
jphickeyjphickey
committed
Fix nasa#456, Perf log threading and concurrency issues
Replaces the `OS_IntLock` with a standard OSAL mutex for protecting the shared/global perflog data structure. This may introduce unexpected task switches when contention occurs, but it ensures proper exclusion with respect to the data structures. Removes the temporary child worker task that was spawned for writing the log data to a file, and replace with a more generic CFE ES background task. The background task is started at boot and pends on a semaphore until there is work to do. The background performance log dump is implemented as a state machine which is called repeatedly over time from the background job task. This performs a limited amount of work on each invocation, and resumes where it left from the previous invocation.
1 parent 62252d1 commit a268837

11 files changed

+1042
-357
lines changed

‎fsw/cfe-core/src/es/cfe_es_backgroundtask.c

+250
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
/*
2+
** GSC-18128-1, "Core Flight Executive Version 6.7"
3+
**
4+
** Copyright (c) 2006-2019 United States Government as represented by
5+
** the Administrator of the National Aeronautics and Space Administration.
6+
** All Rights Reserved.
7+
**
8+
** Licensed under the Apache License, Version 2.0 (the "License");
9+
** you may not use this file except in compliance with the License.
10+
** You may obtain a copy of the License at
11+
**
12+
** http://www.apache.org/licenses/LICENSE-2.0
13+
**
14+
** Unless required by applicable law or agreed to in writing, software
15+
** distributed under the License is distributed on an "AS IS" BASIS,
16+
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
** See the License for the specific language governing permissions and
18+
** limitations under the License.
19+
*/
20+
21+
/*
22+
** File: cfe_es_backgroundtask.c
23+
**
24+
** Purpose: This file contains the implementation of the ES "background task"
25+
**
26+
** This task sits idle most of the time, but is woken by the ES application
27+
** for various maintenance duties that may take time to execute, such as
28+
** writing status/log files.
29+
**
30+
*/
31+
32+
/*
33+
** Include Section
34+
*/
35+
36+
#include <string.h>
37+
38+
#include "osapi.h"
39+
#include "private/cfe_private.h"
40+
#include "cfe_es_perf.h"
41+
#include "cfe_es_global.h"
42+
#include "cfe_es_task.h"
43+
44+
#define CFE_ES_BACKGROUND_SEM_NAME "ES_BackgroundSem"
45+
#define CFE_ES_BACKGROUND_CHILD_NAME "ES_BackgroundTask"
46+
#define CFE_ES_BACKGROUND_CHILD_STACK_PTR NULL
47+
#define CFE_ES_BACKGROUND_CHILD_STACK_SIZE CFE_PLATFORM_ES_PERF_CHILD_STACK_SIZE
48+
#define CFE_ES_BACKGROUND_CHILD_PRIORITY CFE_PLATFORM_ES_PERF_CHILD_PRIORITY
49+
#define CFE_ES_BACKGROUND_CHILD_FLAGS 0
50+
#define CFE_ES_BACKGROUND_MAX_IDLE_DELAY 30000 /* 30 seconds */
51+
52+
53+
typedef struct
54+
{
55+
bool (*RunFunc)(uint32 ElapsedTime, void *Arg);
56+
void *JobArg;
57+
uint32 ActivePeriod; /**< max wait/delay time between calls when job is active */
58+
uint32 IdlePeriod; /**< max wait/delay time between calls when job is idle */
59+
} CFE_ES_BackgroundJobEntry_t;
60+
61+
/*
62+
* List of "background jobs"
63+
*
64+
* This is just a list of functions to periodically call from the context of the background task,
65+
* and can be added/extended as needed.
66+
*
67+
* Each Job function returns a boolean, and should return "true" if it is active, or "false" if it is idle.
68+
*
69+
* This uses "cooperative multitasking" -- the function should do some limited work, then return to the
70+
* background task. It will be called again after a delay period to do more work.
71+
*/
72+
const CFE_ES_BackgroundJobEntry_t CFE_ES_BACKGROUND_JOB_TABLE[] =
73+
{
74+
{ /* Performance Log Data Dump to file */
75+
.RunFunc = CFE_ES_RunPerfLogDump,
76+
.JobArg = &CFE_ES_TaskData.BackgroundPerfDumpState,
77+
.ActivePeriod = CFE_PLATFORM_ES_PERF_CHILD_MS_DELAY,
78+
.IdlePeriod = CFE_PLATFORM_ES_PERF_CHILD_MS_DELAY * 1000
79+
}
80+
};
81+
82+
#define CFE_ES_BACKGROUND_NUM_JOBS (sizeof(CFE_ES_BACKGROUND_JOB_TABLE) / sizeof(CFE_ES_BACKGROUND_JOB_TABLE[0]))
83+
84+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
85+
/* Name: CFE_ES_BackgroundTask */
86+
/* */
87+
/* Purpose: A helper task for low priority routines that may take time to */
88+
/* execute, such as writing log files. */
89+
/* */
90+
/* Assumptions and Notes: This is started from the ES initialization, and */
91+
/* pends on a semaphore until a work request comes in. This is intended to */
92+
/* avoid the need to create a child task "on demand" when work items arrive, */
93+
/* which is a form of dynamic allocation. */
94+
/* */
95+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
96+
void CFE_ES_BackgroundTask(void)
97+
{
98+
int32 status;
99+
uint32 JobTotal;
100+
uint32 NumJobsRunning;
101+
uint32 NextDelay;
102+
uint32 ElapsedTime;
103+
OS_time_t CurrTime;
104+
OS_time_t LastTime;
105+
const CFE_ES_BackgroundJobEntry_t *JobPtr;
106+
107+
status = CFE_ES_RegisterChildTask();
108+
if (status != CFE_SUCCESS)
109+
{
110+
/* should never occur */
111+
CFE_ES_WriteToSysLog("CFE_ES: Background Task Failed to register: %08lx\n", (unsigned long)status);
112+
return;
113+
}
114+
115+
CFE_PSP_GetTime(&LastTime);
116+
117+
while (true)
118+
{
119+
/*
120+
* compute the elapsed time (difference) between last
121+
* execution and now, in microseconds.
122+
*
123+
* Note this calculation is done as a uint32 which will overflow
124+
* after about 35 minutes, but the max delays ensure that this
125+
* executes at least every few seconds, so that should never happen.
126+
*/
127+
CFE_PSP_GetTime(&CurrTime);
128+
ElapsedTime = 1000000 * (CurrTime.seconds - LastTime.seconds);
129+
ElapsedTime += CurrTime.microsecs;
130+
ElapsedTime -= LastTime.microsecs;
131+
LastTime = CurrTime;
132+
133+
/*
134+
* convert to milliseconds.
135+
* we do not really need high precision
136+
* for background task timings
137+
*/
138+
ElapsedTime /= 1000;
139+
140+
NextDelay = CFE_ES_BACKGROUND_MAX_IDLE_DELAY; /* default; will be adjusted based on active jobs */
141+
JobPtr = CFE_ES_BACKGROUND_JOB_TABLE;
142+
JobTotal = CFE_ES_BACKGROUND_NUM_JOBS;
143+
NumJobsRunning = 0;
144+
145+
while (JobTotal > 0)
146+
{
147+
/*
148+
* call the background job -
149+
* if it returns "true" that means it is active,
150+
* if it returns "false" that means it is idle
151+
*/
152+
if (JobPtr->RunFunc != NULL && JobPtr->RunFunc(ElapsedTime, JobPtr->JobArg))
153+
{
154+
++NumJobsRunning;
155+
156+
if (JobPtr->ActivePeriod != 0 && NextDelay > JobPtr->ActivePeriod)
157+
{
158+
/* next delay is based on this active job wait time */
159+
NextDelay = JobPtr->ActivePeriod;
160+
}
161+
}
162+
else if (JobPtr->IdlePeriod != 0 && NextDelay > JobPtr->IdlePeriod)
163+
{
164+
/* next delay is based on this idle job wait time */
165+
NextDelay = JobPtr->IdlePeriod;
166+
}
167+
--JobTotal;
168+
++JobPtr;
169+
}
170+
171+
CFE_ES_Global.BackgroundTask.NumJobsRunning = NumJobsRunning;
172+
173+
status = OS_BinSemTimedWait(CFE_ES_Global.BackgroundTask.WorkSem, NextDelay);
174+
if (status != OS_SUCCESS && status != OS_SEM_TIMEOUT)
175+
{
176+
/* should never occur */
177+
CFE_ES_WriteToSysLog("CFE_ES: Failed to take background sem: %08lx\n", (unsigned long)status);
178+
break;
179+
}
180+
181+
}
182+
}
183+
184+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
185+
/* Name: CFE_ES_BackgroundInit */
186+
/* */
187+
/* Purpose: Initialize the background task */
188+
/* */
189+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
190+
int32 CFE_ES_BackgroundInit(void)
191+
{
192+
int32 status;
193+
194+
status = OS_BinSemCreate(&CFE_ES_Global.BackgroundTask.WorkSem, CFE_ES_BACKGROUND_SEM_NAME, 0, 0);
195+
if (status != OS_SUCCESS)
196+
{
197+
CFE_ES_WriteToSysLog("CFE_ES: Failed to create background sem: %08lx\n", (unsigned long)status);
198+
return status;
199+
}
200+
201+
/* Spawn a task to write the performance data to a file */
202+
status = CFE_ES_CreateChildTask(&CFE_ES_Global.BackgroundTask.TaskID,
203+
CFE_ES_BACKGROUND_CHILD_NAME,
204+
CFE_ES_BackgroundTask,
205+
CFE_ES_BACKGROUND_CHILD_STACK_PTR,
206+
CFE_ES_BACKGROUND_CHILD_STACK_SIZE,
207+
CFE_ES_BACKGROUND_CHILD_PRIORITY,
208+
CFE_ES_BACKGROUND_CHILD_FLAGS);
209+
210+
if (status != OS_SUCCESS)
211+
{
212+
CFE_ES_WriteToSysLog("CFE_ES: Failed to create background task: %08lx\n", (unsigned long)status);
213+
return status;
214+
}
215+
216+
return CFE_SUCCESS;
217+
}
218+
219+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
220+
/* Name: CFE_ES_BackgroundCleanup */
221+
/* */
222+
/* Purpose: Exit/Stop the background task */
223+
/* */
224+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
225+
void CFE_ES_BackgroundCleanup(void)
226+
{
227+
CFE_ES_DeleteChildTask(CFE_ES_Global.BackgroundTask.TaskID);
228+
OS_BinSemDelete(CFE_ES_Global.BackgroundTask.WorkSem);
229+
230+
CFE_ES_Global.BackgroundTask.TaskID = 0;
231+
CFE_ES_Global.BackgroundTask.WorkSem = 0;
232+
}
233+
234+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
235+
/* Name: CFE_ES_BackgroundWakeup */
236+
/* */
237+
/* Purpose: Wake up the background task */
238+
/* Notifies the background task to perform an extra poll for new work */
239+
/* */
240+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
241+
void CFE_ES_BackgroundWakeup(void)
242+
{
243+
/* wake up the background task by giving the sem.
244+
* This is "informational" and not strictly required,
245+
* but it will make the task immediately wake up and check for new
246+
* work if it was idle. */
247+
OS_BinSemGive(CFE_ES_Global.BackgroundTask.WorkSem);
248+
}
249+
250+

‎fsw/cfe-core/src/es/cfe_es_global.h

+31-9
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
*/
2020

2121
/*
22-
** File:
23-
** cfe_es_global.h
22+
** File:
23+
** cfe_es_global.h
2424
**
2525
** Purpose:
2626
** This file contains the ES global data definitions.
@@ -66,13 +66,23 @@ typedef struct
6666
{
6767
bool RecordUsed; /* Is the record used(1) or available(0) ? */
6868
uint32 Counter;
69-
char CounterName[OS_MAX_API_NAME]; /* Counter Name */
69+
char CounterName[OS_MAX_API_NAME]; /* Counter Name */
7070
} CFE_ES_GenCounterRecord_t;
7171

72+
/*
73+
* Encapsulates the state of the ES background task
74+
*/
75+
typedef struct
76+
{
77+
uint32 TaskID; /**< OSAL ID of the background task */
78+
uint32 WorkSem; /**< Semaphore that is given whenever background work is pending */
79+
uint32 NumJobsRunning; /**< Current Number of active jobs (updated by background task) */
80+
} CFE_ES_BackgroundTaskState_t;
81+
7282

7383
/*
7484
** Executive Services Global Memory Data
75-
** This is the regular global data that is not preserved on a
85+
** This is the regular global data that is not preserved on a
7686
** processor reset.
7787
*/
7888
typedef struct
@@ -81,20 +91,25 @@ typedef struct
8191
** Debug Variables
8292
*/
8393
CFE_ES_DebugVariables_t DebugVars;
84-
94+
8595
/*
8696
** Shared Data Semaphore
8797
*/
8898
uint32 SharedDataMutex;
89-
99+
100+
/*
101+
** Performance Data Mutex
102+
*/
103+
uint32 PerfDataMutex;
104+
90105
/*
91106
** Startup Sync
92107
*/
93108
uint32 SystemState;
94109

95110
/*
96111
** ES Task Table
97-
*/
112+
*/
98113
uint32 RegisteredTasks;
99114
CFE_ES_TaskRecord_t TaskTable[OS_MAX_TASKS];
100115

@@ -104,7 +119,7 @@ typedef struct
104119
uint32 RegisteredCoreApps;
105120
uint32 RegisteredExternalApps;
106121
CFE_ES_AppRecord_t AppTable[CFE_PLATFORM_ES_MAX_APPLICATIONS];
107-
122+
108123
/*
109124
** ES Shared Library Table
110125
*/
@@ -121,12 +136,19 @@ typedef struct
121136
*/
122137
CFE_ES_CDSVariables_t CDSVars;
123138

139+
/*
140+
* Background task for handling long-running, non real time tasks
141+
* such as maintenance, file writes, and other items.
142+
*/
143+
CFE_ES_BackgroundTaskState_t BackgroundTask;
144+
145+
124146
} CFE_ES_Global_t;
125147

126148
/*
127149
** The Executive Services Global Data declaration
128150
*/
129-
extern CFE_ES_Global_t CFE_ES_Global;
151+
extern CFE_ES_Global_t CFE_ES_Global;
130152

131153
/*
132154
** The Executive Services Nonvolatile Data declaration

0 commit comments

Comments
 (0)