From 2d04ccf4226512721d6a29d12ec3356680d8fb18 Mon Sep 17 00:00:00 2001 From: Dong Wang Date: Mon, 4 Dec 2023 11:54:03 +0800 Subject: [PATCH 1/3] drivers: apic_timer: Add 64bit cycle counter support Make k_cycle_get_64() call available for both TSC and none-TSC mode. Signed-off-by: Dong Wang --- drivers/timer/Kconfig.x86 | 2 +- drivers/timer/apic_timer.c | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/drivers/timer/Kconfig.x86 b/drivers/timer/Kconfig.x86 index ea6d4a216b7d4..34ddb9e991288 100644 --- a/drivers/timer/Kconfig.x86 +++ b/drivers/timer/Kconfig.x86 @@ -28,6 +28,7 @@ config APIC_TIMER select LOAPIC select TICKLESS_CAPABLE select SYSTEM_CLOCK_LOCK_FREE_COUNT + select TIMER_HAS_64BIT_CYCLE_COUNTER help Use the x86 local APIC in one-shot mode as the system time source. NOTE: this probably isn't what you want except on @@ -68,7 +69,6 @@ config APIC_TIMER_IRQ config APIC_TIMER_TSC bool "Use invariant TSC for sys_clock_cycle_get_32()" - select TIMER_HAS_64BIT_CYCLE_COUNTER help If your CPU supports invariant TSC, and you know the ratio of the TSC frequency to CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC (the local APIC diff --git a/drivers/timer/apic_timer.c b/drivers/timer/apic_timer.c index cf28acb04d93e..18f203b36c4e7 100644 --- a/drivers/timer/apic_timer.c +++ b/drivers/timer/apic_timer.c @@ -192,20 +192,18 @@ uint32_t sys_clock_elapsed(void) #ifdef CONFIG_APIC_TIMER_TSC -uint32_t sys_clock_cycle_get_32(void) +uint64_t sys_clock_cycle_get_64(void) { uint64_t tsc = z_tsc_read(); - uint32_t cycles; - cycles = (tsc * CONFIG_APIC_TIMER_TSC_M) / CONFIG_APIC_TIMER_TSC_N; - return cycles; + return (tsc * CONFIG_APIC_TIMER_TSC_M) / CONFIG_APIC_TIMER_TSC_N; } #else -uint32_t sys_clock_cycle_get_32(void) +uint64_t sys_clock_cycle_get_64(void) { - uint32_t ret; + uint64_t ret; uint32_t ccr; k_spinlock_key_t key = k_spin_lock(&lock); @@ -218,6 +216,11 @@ uint32_t sys_clock_cycle_get_32(void) #endif +uint32_t sys_clock_cycle_get_32(void) +{ + return (uint32_t)sys_clock_cycle_get_64(); +} + static int sys_clock_driver_init(void) { uint32_t val; From 8d26d8ae16e534a9f12e7a2e8cfa77cb3d5ac8ea Mon Sep 17 00:00:00 2001 From: Dong Wang Date: Wed, 15 May 2024 15:13:02 +0800 Subject: [PATCH 2/3] drivers: apic_timer: Improve accuracy of last_announcement assignment Change last_announcement to track the tick aligned cycle count of the last announcement rather than the absolute cycle count. This ensures the the number of ticks that have elapsed since last_announcement, provided by sys_clock_elapsed(), are accurate. This also makes annoucement calculation more accurate. Signed-off-by: Dong Wang --- drivers/timer/apic_timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/timer/apic_timer.c b/drivers/timer/apic_timer.c index 18f203b36c4e7..ca34cff30c6c0 100644 --- a/drivers/timer/apic_timer.c +++ b/drivers/timer/apic_timer.c @@ -164,7 +164,7 @@ static void isr(const void *arg) cached_icr = MAX_TICKS * CYCLES_PER_TICK; total_cycles += cycles; ticks = (total_cycles - last_announcement) / CYCLES_PER_TICK; - last_announcement = total_cycles; + last_announcement += ticks * CYCLES_PER_TICK; k_spin_unlock(&lock, key); sys_clock_announce(ticks); } From 1765352cd206935a29a6456ff0cc1084812c394f Mon Sep 17 00:00:00 2001 From: Dong Wang Date: Wed, 15 May 2024 14:05:51 +0800 Subject: [PATCH 3/3] drivers: apic_timer: Improve TSC mode accuracy by using TSC count Now the cycle count calculated from TSC is utilized for total_cycles. It's more accurate than the old value calculated out from previous total cycles saved plus APIC cycles passed since then, which has the interrrupt generation time missed. Additionally, stop the APIC timer when setting timeout as "forever" after TSC count is utilized. Signed-off-by: Dong Wang --- drivers/timer/Kconfig.x86 | 4 ++-- drivers/timer/apic_timer.c | 48 +++++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/drivers/timer/Kconfig.x86 b/drivers/timer/Kconfig.x86 index 34ddb9e991288..42973b0bcc299 100644 --- a/drivers/timer/Kconfig.x86 +++ b/drivers/timer/Kconfig.x86 @@ -68,12 +68,12 @@ config APIC_TIMER_IRQ a different mechanism. config APIC_TIMER_TSC - bool "Use invariant TSC for sys_clock_cycle_get_32()" + bool "Use invariant TSC for Local APIC timer's cycle count" help If your CPU supports invariant TSC, and you know the ratio of the TSC frequency to CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC (the local APIC timer frequency), then enable this for a much faster and more - accurate sys_clock_cycle_get_32(). + accurate Local APIC timer's cycle count. if APIC_TIMER_TSC diff --git a/drivers/timer/apic_timer.c b/drivers/timer/apic_timer.c index ca34cff30c6c0..ac39ef9120cb5 100644 --- a/drivers/timer/apic_timer.c +++ b/drivers/timer/apic_timer.c @@ -9,6 +9,7 @@ #include #include #include +#include BUILD_ASSERT(!IS_ENABLED(CONFIG_SMP), "APIC timer doesn't support SMP"); @@ -87,11 +88,17 @@ void sys_clock_set_timeout(int32_t n, bool idle) { ARG_UNUSED(idle); - uint32_t ccr; int full_ticks; /* number of complete ticks we'll wait */ uint32_t full_cycles; /* full_ticks represented as cycles */ uint32_t partial_cycles; /* number of cycles to first tick boundary */ +#ifdef CONFIG_APIC_TIMER_TSC + if (n == K_TICKS_FOREVER) { + x86_write_loapic(LOAPIC_TIMER_ICR, 0x0); + return; + } +#endif + if (n < 1) { full_ticks = 0; } else if ((n == K_TICKS_FOREVER) || (n > MAX_TICKS)) { @@ -113,8 +120,11 @@ void sys_clock_set_timeout(int32_t n, bool idle) k_spinlock_key_t key = k_spin_lock(&lock); - ccr = x86_read_loapic(LOAPIC_TIMER_CCR); - total_cycles += (cached_icr - ccr); +#ifdef CONFIG_APIC_TIMER_TSC + total_cycles = sys_clock_cycle_get_64(); +#else + total_cycles += (cached_icr - x86_read_loapic(LOAPIC_TIMER_CCR)); +#endif partial_cycles = CYCLES_PER_TICK - (total_cycles % CYCLES_PER_TICK); cached_icr = full_cycles + partial_cycles; x86_write_loapic(LOAPIC_TIMER_ICR, cached_icr); @@ -124,13 +134,15 @@ void sys_clock_set_timeout(int32_t n, bool idle) uint32_t sys_clock_elapsed(void) { - uint32_t ccr; - uint32_t ticks; + uint64_t ticks; k_spinlock_key_t key = k_spin_lock(&lock); - ccr = x86_read_loapic(LOAPIC_TIMER_CCR); +#ifdef CONFIG_APIC_TIMER_TSC + ticks = sys_clock_cycle_get_64() - last_announcement; +#else ticks = total_cycles - last_announcement; - ticks += cached_icr - ccr; + ticks += cached_icr - x86_read_loapic(LOAPIC_TIMER_CCR); +#endif k_spin_unlock(&lock, key); ticks /= CYCLES_PER_TICK; @@ -141,8 +153,8 @@ static void isr(const void *arg) { ARG_UNUSED(arg); - uint32_t cycles; int32_t ticks; + uint64_t ticks_u64; k_spinlock_key_t key = k_spin_lock(&lock); @@ -152,19 +164,24 @@ static void isr(const void *arg) * a new counter. Just ignore it. See above for more info. */ - if (x86_read_loapic(LOAPIC_TIMER_CCR) != 0) { + if ((x86_read_loapic(LOAPIC_TIMER_CCR) != 0) + || (x86_read_loapic(LOAPIC_TIMER_ICR) == 0)) { k_spin_unlock(&lock, key); return; } +#ifdef CONFIG_APIC_TIMER_TSC + total_cycles = sys_clock_cycle_get_64(); +#else /* Restart the timer as early as possible to minimize drift... */ x86_write_loapic(LOAPIC_TIMER_ICR, MAX_TICKS * CYCLES_PER_TICK); - cycles = cached_icr; + total_cycles += cached_icr; cached_icr = MAX_TICKS * CYCLES_PER_TICK; - total_cycles += cycles; - ticks = (total_cycles - last_announcement) / CYCLES_PER_TICK; - last_announcement += ticks * CYCLES_PER_TICK; +#endif + ticks_u64 = (total_cycles - last_announcement) / CYCLES_PER_TICK; + ticks = (ticks_u64 > INT_MAX) ? INT_MAX : (int32_t)ticks_u64; + last_announcement += ticks_u64 * CYCLES_PER_TICK; k_spin_unlock(&lock, key); sys_clock_announce(ticks); } @@ -242,6 +259,11 @@ static int sys_clock_driver_init(void) CONFIG_APIC_TIMER_IRQ_PRIORITY, isr, 0, 0); +#ifdef CONFIG_APIC_TIMER_TSC + total_cycles = sys_clock_cycle_get_64(); + last_announcement = total_cycles - (total_cycles % CYCLES_PER_TICK); + cached_icr = CYCLES_PER_TICK - (total_cycles % CYCLES_PER_TICK); +#endif x86_write_loapic(LOAPIC_TIMER_ICR, cached_icr); irq_enable(CONFIG_APIC_TIMER_IRQ);