* arch/x86_64/src/intel64/intel64_cache.c
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. The
* ASF licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
****************************************************************************/
* Included Files
****************************************************************************/
#include <nuttx/config.h>
#include <nuttx/cache.h>
#include <stdint.h>
#include "x86_64_internal.h"
* Pre-processor Definitions
****************************************************************************/
* Inline Functions
****************************************************************************/
* Private Functions
****************************************************************************/
* Name: x86_64_wbindv
****************************************************************************/
static inline void x86_64_wbindv(void)
{
__asm__ volatile("wbinvd" : : : "memory");
}
* Name: x86_64_wbnoinvd
****************************************************************************/
static inline void x86_64_wbnoinvd(void)
{
__asm__ volatile("wbnoinvd" : : : "memory");
}
* Name: x86_64_invd
****************************************************************************/
static inline void x86_64_invd(void)
{
__asm__ volatile("invd" : : : "memory");
}
* Name: x86_64_cache_linesize
****************************************************************************/
static size_t x86_64_cache_linesize(void)
{
#if CONFIG_ARCH_INTEL64_CACHE_LINESIZE == 0
unsigned long eax = 0;
unsigned long ebx = 0;
eax = 1;
__asm__ volatile("cpuid\n\t"
: "=b" (ebx)
: "a" (eax));
return ((ebx >> 8) & 0xff) * 8;
#else
return CONFIG_ARCH_INTEL64_CACHE_LINESIZE;
#endif
}
* Name: x86_64_cache_size
****************************************************************************/
static size_t x86_64_cache_size(int leaf)
{
unsigned long eax;
unsigned long ebx;
unsigned long ecx;
unsigned long edx;
eax = 4;
ecx = leaf;
__asm__ volatile("cpuid"
: "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
: "a"(eax), "c"(ecx));
return ((((ebx >> 22) & 0x3ff) + 1) *
(((ebx >> 12) & 0x3ff) + 1) *
((ebx & 0x7ff) + 1) *
(ecx + 1));
}
* Name: x86_64_is_cache_enabled
****************************************************************************/
static int x86_64_is_cache_enabled(void)
{
unsigned long cr0;
__asm__ volatile("\t mov %%cr0, %0\n"
: "=r" (cr0)
:: "memory");
return (cr0 & 0x60000000) == 0;
}
* Name: x86_64_cache_enable
****************************************************************************/
static void x86_64_cache_enable(void)
{
__asm__ volatile("\t mov %%cr0, %%rax\n"
"\t mov $0x9fffffff, %%rbx\n"
"\t and %%rbx, %%rax\n"
"\t mov %%rax, %%cr0\n"
::: "memory", "rax", "rbx");
}
* Name: x86_64_cache_disable
****************************************************************************/
static void x86_64_cache_disable(void)
{
__asm__ volatile("\t mov %%cr0, %%rax\n"
"\t mov $0x9fffffff, %%rbx \n"
"\t and %%rbx, %%rax \n"
"\t mov $0x60000000, %%rbx\n"
"\t or %%rbx, %%rax\n"
"\t mov %%rax, %%cr0\n"
:::"memory", "rax", "rbx");
up_flush_dcache_all();
write_msr(MSR_MTRR_DEF_TYPE, 0);
up_flush_dcache_all();
}
* Public Functions
****************************************************************************/
* Name: up_get_icache_linesize
*
* Description:
* Get icache linesize
*
* Input Parameters:
* None
*
* Returned Value:
* Cache line size
*
****************************************************************************/
#ifdef CONFIG_ARCH_ICACHE
size_t up_get_icache_linesize(void)
{
return x86_64_cache_linesize();
}
* Name: up_get_icache_size
*
* Description:
* Get icache size
*
* Input Parameters:
* None
*
* Returned Value:
* Cache size
*
****************************************************************************/
size_t up_get_icache_size(void)
{
#if CONFIG_ARCH_INTEL64_ICACHE_SIZE == 0
return x86_64_cache_size(1);
#else
return CONFIG_ARCH_INTEL64_ICACHE_SIZE;
#endif
}
#endif
* Name: up_enable_icache
*
* Description:
* Enable the I-Cache
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_ICACHE
void up_enable_icache(void)
{
x86_64_cache_enable();
}
#endif
* Name: up_disable_icache
*
* Description:
* Disable the I-Cache
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_ICACHE
void up_disable_icache(void)
{
x86_64_cache_disable();
}
#endif
* Name: up_invalidate_icache
*
* Description:
* Invalidate the instruction cache within the specified region.
*
* Input Parameters:
* start - virtual start address of region
* end - virtual end address of region + 1
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_ICACHE
void up_invalidate_icache(uintptr_t start, uintptr_t end)
{
up_invalidate_dcache(start, end);
}
#endif
* Name: up_invalidate_icache_all
*
* Description:
* Invalidate the entire contents of I cache.
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_ICACHE
void up_invalidate_icache_all(void)
{
up_invalidate_dcache_all();
}
#endif
* Name: up_get_dcache_linesize
*
* Description:
* Get dcache linesize
*
* Input Parameters:
* None
*
* Returned Value:
* Cache line size
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
size_t up_get_dcache_linesize(void)
{
return x86_64_cache_linesize();
}
* Name: up_get_dcache_size
*
* Description:
* Get icache size
*
* Input Parameters:
* None
*
* Returned Value:
* Cache size
*
****************************************************************************/
size_t up_get_dcache_size(void)
{
#if CONFIG_ARCH_INTEL64_DCACHE_SIZE == 0
return x86_64_cache_size(0);
#else
return CONFIG_ARCH_INTEL64_DCACHE_SIZE;
#endif
}
#endif
* Name: up_enable_dcache
*
* Description:
* Enable the D-Cache
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_enable_dcache(void)
{
if (x86_64_is_cache_enabled())
{
return;
}
up_invalidate_dcache_all();
x86_64_cache_enable();
}
#endif
* Name: up_disable_dcache
*
* Description:
* Disable the D-Cache
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_disable_dcache(void)
{
x86_64_cache_disable();
}
#endif
* Name: up_invalidae_dcache
*
* Description:
* Invalidate the data cache within the specified region; we will be
* performing a DMA operation in this region and we want to purge old data
* in the cache.
*
* Input Parameters:
* start - virtual start address of region
* end - virtual end address of region + 1
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_invalidate_dcache(uintptr_t start, uintptr_t end)
{
up_flush_dcache(start, end);
}
#endif
* Name: up_invalidate_dcache_all
*
* Description:
* Invalidate the entire contents of D cache.
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_invalidate_dcache_all(void)
{
x86_64_invd();
}
#endif
* Name: up_clean_dcache
*
* Description:
* Clean the data cache within the specified region by flushing the
* contents of the data cache to memory.
*
* Input Parameters:
* start - virtual start address of region
* end - virtual end address of region + 1
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_clean_dcache(uintptr_t start, uintptr_t end)
{
#ifdef CONFIG_ARCH_INTEL64_HAVE_CLWB
size_t lsize = up_get_dcache_linesize();
start &= ~(lsize - 1);
__asm__ volatile("mfence" : : : "memory");
do
{
__asm__ volatile("\tclwb %0;\n" : "+m" (start));
start += lsize;
}
while (start < end);
__asm__ volatile("mfence" : : : "memory");
#else
x86_64_wbnoinvd();
#endif
}
#endif
* Name: up_clean_dcache_all
*
* Description:
* Clean the entire data cache within the specified region by flushing the
* contents of the data cache to memory.
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_clean_dcache_all(void)
{
x86_64_wbnoinvd();
}
#endif
* Name: up_flush_dcache
*
* Description:
* Flush the data cache within the specified region by cleaning and
* invalidating the D cache.
*
* Input Parameters:
* start - virtual start address of region
* end - virtual end address of region + 1
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_flush_dcache(uintptr_t start, uintptr_t end)
{
size_t lsize = up_get_dcache_linesize();
start &= ~(lsize - 1);
__asm__ volatile("mfence" : : : "memory");
do
{
__asm__ volatile("\tclflush %0;\n" : "+m" (start));
start += lsize;
}
while (start < end);
__asm__ volatile("mfence" : : : "memory");
}
#endif
* Name: up_flush_dcache_all
*
* Description:
* Flush the entire data cache by cleaning and invalidating the D cache.
*
* Input Parameters:
* None
*
* Returned Value:
* None
*
****************************************************************************/
#ifdef CONFIG_ARCH_DCACHE
void up_flush_dcache_all(void)
{
x86_64_wbindv();
}
#endif
* Name: up_coherent_dcache
*
* Description:
* Ensure that the I and D caches are coherent within specified region
* by cleaning the D cache (i.e., flushing the D cache contents to memory
* and invalidating the I cache. This is typically used when code has been
* written to a memory region, and will be executed.
*
* Input Parameters:
* addr - virtual start address of region
* len - Size of the address region in bytes
*
* Returned Value:
* None
*
****************************************************************************/
#if defined(CONFIG_ARCH_ICACHE) && defined(CONFIG_ARCH_DCACHE)
void up_coherent_dcache(uintptr_t addr, size_t len)
{
uintptr_t end;
if (len > 0)
{
end = addr + len;
up_clean_dcache(addr, end);
up_invalidate_icache(addr, end);
}
}
#endif