Skip to content

Enhance x86 apic timer driver #72839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Enhance x86 apic timer driver #72839

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2d04ccf
drivers: apic_timer: Add 64bit cycle counter support
kwd-doodling Dec 4, 2023
8d26d8a
drivers: apic_timer: Improve accuracy of last_announcement assignment
kwd-doodling May 15, 2024
1765352
drivers: apic_timer: Improve TSC mode accuracy by using TSC count
kwd-doodling May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions drivers/timer/Kconfig.x86
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ config APIC_TIMER
select LOAPIC
select TICKLESS_CAPABLE
select SYSTEM_CLOCK_LOCK_FREE_COUNT
select TIMER_HAS_64BIT_CYCLE_COUNTER
help
Use the x86 local APIC in one-shot mode as the system time
source. NOTE: this probably isn't what you want except on
Expand Down Expand Up @@ -67,13 +68,12 @@ config APIC_TIMER_IRQ
a different mechanism.

config APIC_TIMER_TSC
bool "Use invariant TSC for sys_clock_cycle_get_32()"
select TIMER_HAS_64BIT_CYCLE_COUNTER
bool "Use invariant TSC for Local APIC timer's cycle count"
help
If your CPU supports invariant TSC, and you know the ratio of the
TSC frequency to CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC (the local APIC
timer frequency), then enable this for a much faster and more
accurate sys_clock_cycle_get_32().
accurate Local APIC timer's cycle count.

if APIC_TIMER_TSC

Expand Down
63 changes: 44 additions & 19 deletions drivers/timer/apic_timer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <zephyr/spinlock.h>
#include <zephyr/drivers/interrupt_controller/loapic.h>
#include <zephyr/irq.h>
#include <limits.h>

BUILD_ASSERT(!IS_ENABLED(CONFIG_SMP), "APIC timer doesn't support SMP");

Expand Down Expand Up @@ -87,11 +88,17 @@ void sys_clock_set_timeout(int32_t n, bool idle)
{
ARG_UNUSED(idle);

uint32_t ccr;
int full_ticks; /* number of complete ticks we'll wait */
uint32_t full_cycles; /* full_ticks represented as cycles */
uint32_t partial_cycles; /* number of cycles to first tick boundary */

#ifdef CONFIG_APIC_TIMER_TSC
if (n == K_TICKS_FOREVER) {
x86_write_loapic(LOAPIC_TIMER_ICR, 0x0);
return;
}
#endif

if (n < 1) {
full_ticks = 0;
} else if ((n == K_TICKS_FOREVER) || (n > MAX_TICKS)) {
Expand All @@ -113,8 +120,11 @@ void sys_clock_set_timeout(int32_t n, bool idle)

k_spinlock_key_t key = k_spin_lock(&lock);

ccr = x86_read_loapic(LOAPIC_TIMER_CCR);
total_cycles += (cached_icr - ccr);
#ifdef CONFIG_APIC_TIMER_TSC
total_cycles = sys_clock_cycle_get_64();
#else
total_cycles += (cached_icr - x86_read_loapic(LOAPIC_TIMER_CCR));
#endif
partial_cycles = CYCLES_PER_TICK - (total_cycles % CYCLES_PER_TICK);
cached_icr = full_cycles + partial_cycles;
x86_write_loapic(LOAPIC_TIMER_ICR, cached_icr);
Expand All @@ -124,13 +134,15 @@ void sys_clock_set_timeout(int32_t n, bool idle)

uint32_t sys_clock_elapsed(void)
{
uint32_t ccr;
uint32_t ticks;
uint64_t ticks;

k_spinlock_key_t key = k_spin_lock(&lock);
ccr = x86_read_loapic(LOAPIC_TIMER_CCR);
#ifdef CONFIG_APIC_TIMER_TSC
ticks = sys_clock_cycle_get_64() - last_announcement;
#else
ticks = total_cycles - last_announcement;
ticks += cached_icr - ccr;
ticks += cached_icr - x86_read_loapic(LOAPIC_TIMER_CCR);
#endif
k_spin_unlock(&lock, key);
ticks /= CYCLES_PER_TICK;

Expand All @@ -141,8 +153,8 @@ static void isr(const void *arg)
{
ARG_UNUSED(arg);

uint32_t cycles;
int32_t ticks;
uint64_t ticks_u64;

k_spinlock_key_t key = k_spin_lock(&lock);

Expand All @@ -152,19 +164,24 @@ static void isr(const void *arg)
* a new counter. Just ignore it. See above for more info.
*/

if (x86_read_loapic(LOAPIC_TIMER_CCR) != 0) {
if ((x86_read_loapic(LOAPIC_TIMER_CCR) != 0)
|| (x86_read_loapic(LOAPIC_TIMER_ICR) == 0)) {
k_spin_unlock(&lock, key);
return;
}

#ifdef CONFIG_APIC_TIMER_TSC
total_cycles = sys_clock_cycle_get_64();
#else
/* Restart the timer as early as possible to minimize drift... */
x86_write_loapic(LOAPIC_TIMER_ICR, MAX_TICKS * CYCLES_PER_TICK);

cycles = cached_icr;
total_cycles += cached_icr;
cached_icr = MAX_TICKS * CYCLES_PER_TICK;
total_cycles += cycles;
ticks = (total_cycles - last_announcement) / CYCLES_PER_TICK;
last_announcement = total_cycles;
#endif
ticks_u64 = (total_cycles - last_announcement) / CYCLES_PER_TICK;
ticks = (ticks_u64 > INT_MAX) ? INT_MAX : (int32_t)ticks_u64;
last_announcement += ticks_u64 * CYCLES_PER_TICK;
k_spin_unlock(&lock, key);
sys_clock_announce(ticks);
}
Expand Down Expand Up @@ -192,20 +209,18 @@ uint32_t sys_clock_elapsed(void)

#ifdef CONFIG_APIC_TIMER_TSC

uint32_t sys_clock_cycle_get_32(void)
uint64_t sys_clock_cycle_get_64(void)
{
uint64_t tsc = z_tsc_read();
uint32_t cycles;

cycles = (tsc * CONFIG_APIC_TIMER_TSC_M) / CONFIG_APIC_TIMER_TSC_N;
return cycles;
return (tsc * CONFIG_APIC_TIMER_TSC_M) / CONFIG_APIC_TIMER_TSC_N;
}

#else

uint32_t sys_clock_cycle_get_32(void)
uint64_t sys_clock_cycle_get_64(void)
{
uint32_t ret;
uint64_t ret;
uint32_t ccr;

k_spinlock_key_t key = k_spin_lock(&lock);
Expand All @@ -218,6 +233,11 @@ uint32_t sys_clock_cycle_get_32(void)

#endif

uint32_t sys_clock_cycle_get_32(void)
{
return (uint32_t)sys_clock_cycle_get_64();
}

static int sys_clock_driver_init(void)
{
uint32_t val;
Expand All @@ -239,6 +259,11 @@ static int sys_clock_driver_init(void)
CONFIG_APIC_TIMER_IRQ_PRIORITY,
isr, 0, 0);

#ifdef CONFIG_APIC_TIMER_TSC
total_cycles = sys_clock_cycle_get_64();
last_announcement = total_cycles - (total_cycles % CYCLES_PER_TICK);
cached_icr = CYCLES_PER_TICK - (total_cycles % CYCLES_PER_TICK);
#endif
x86_write_loapic(LOAPIC_TIMER_ICR, cached_icr);
irq_enable(CONFIG_APIC_TIMER_IRQ);

Expand Down