* Copyright (c) 2023 Institute of Parallel And Distributed Systems (IPADS), Shanghai Jiao Tong University (SJTU)
* Licensed under the Mulan PSL v2.
* You can use this software according to the terms and conditions of the Mulan PSL v2.
* You may obtain a copy of Mulan PSL v2 at:
* http://license.coscl.org.cn/MulanPSL2
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
* PURPOSE.
* See the Mulan PSL v2 for more details.
*/
#ifdef CHCORE_ENABLE_FMAP
#include <arch/mmu.h>
#include <arch/mm/cache.h>
#include <ipc/notification.h>
#include <irq/irq.h>
#include <common/errno.h>
#include <common/radix.h>
#include <common/lock.h>
#include <common/types.h>
#include <sched/sched.h>
#include <mm/kmalloc.h>
#include <lib/printk.h>
#include <mm/vmspace.h>
#include <mm/kmalloc.h>
#include <mm/mm.h>
#include <object/object.h>
#include <object/user_fault.h>
#include <lib/ring_buffer.h>
struct lock fmap_fault_pool_list_lock;
struct list_head fmap_fault_pool_list;
static void user_fault_init(void)
{
static int inited = 0;
if (!atomic_cmpxchg_32(&inited, 0, 1)) {
lock_init(&fmap_fault_pool_list_lock);
init_list_head(&fmap_fault_pool_list);
}
}
static struct fmap_fault_pool *get_current_fault_pool(void)
{
badge_t badge;
struct fmap_fault_pool *pool_iter;
badge = current_cap_group->badge;
for_each_in_list (
pool_iter, struct fmap_fault_pool, node, &fmap_fault_pool_list) {
if (pool_iter->cap_group_badge == badge) {
return pool_iter;
}
}
return NULL;
}
static struct fault_pending_thread *
get_current_pending_thread(badge_t client_badge, vaddr_t fault_va)
{
struct fault_pending_thread *pt;
struct fmap_fault_pool *pool;
pool = get_current_fault_pool();
if (!pool)
return NULL;
for_each_in_list (
pt, struct fault_pending_thread, node, &pool->pending_threads) {
if (pt->fault_badge == client_badge && pt->fault_va == fault_va) {
return pt;
}
}
return NULL;
}
int sys_user_fault_register(cap_t notific_cap, vaddr_t msg_buffer)
{
int ret;
struct notification *notific;
struct ring_buffer *msg_buffer_kva;
* struct, so no need to initialize */
badge_t badge;
struct fmap_fault_pool *pool_iter;
user_fault_init();
badge = current_cap_group->badge;
notific = obj_get(current_cap_group, notific_cap, TYPE_NOTIFICATION);
if (notific == NULL) {
return -EINVAL;
}
ret = trans_uva_to_kva(msg_buffer, (vaddr_t *)&msg_buffer_kva);
if (ret != 0) {
return -EINVAL;
}
lock(&fmap_fault_pool_list_lock);
if (get_current_fault_pool() != NULL) {
unlock(&fmap_fault_pool_list_lock);
return -EINVAL;
}
pool_iter = (struct fmap_fault_pool *)kmalloc(sizeof(*pool_iter));
if (!pool_iter) {
unlock(&fmap_fault_pool_list_lock);
return -ENOMEM;
}
pool_iter->cap_group_badge = badge;
pool_iter->notific = notific;
pool_iter->msg_buffer_kva = msg_buffer_kva;
lock_init(&pool_iter->lock);
init_list_head(&pool_iter->pending_threads);
list_append(&pool_iter->node, &fmap_fault_pool_list);
unlock(&fmap_fault_pool_list_lock);
return 0;
}
int sys_user_fault_map(badge_t client_badge, vaddr_t fault_va, vaddr_t remap_va,
bool copy, unsigned long perm)
{
struct fmap_fault_pool *current_pool;
struct fault_pending_thread *pending_thread;
struct thread *thread_to_wake;
struct vmspace *handler_vmspace;
struct vmspace *fault_vmspace;
struct vmregion *fault_vmr;
struct pmobject *fault_pmo;
paddr_t pa, new_pa;
void *new_page;
int ret;
bool page_allocated = false;
long rss = 0;
current_pool = get_current_fault_pool();
lock(¤t_pool->lock);
pending_thread = get_current_pending_thread(client_badge, fault_va);
if (!pending_thread) {
unlock(¤t_pool->lock);
return -EINVAL;
}
list_del(&pending_thread->node);
unlock(¤t_pool->lock);
thread_to_wake = pending_thread->thread;
kfree(pending_thread);
if (remap_va) {
handler_vmspace =
obj_get(current_cap_group, VMSPACE_OBJ_ID, TYPE_VMSPACE);
if (handler_vmspace == NULL) {
return -EINVAL;
}
lock(&handler_vmspace->pgtbl_lock);
ret = query_in_pgtbl(handler_vmspace->pgtbl, remap_va, &pa, NULL);
if (ret) {
unlock(&handler_vmspace->pgtbl_lock);
obj_put(handler_vmspace);
return -EINVAL;
}
unlock(&handler_vmspace->pgtbl_lock);
obj_put(handler_vmspace);
}
if (!copy) {
if (!remap_va)
return -EINVAL;
new_pa = pa;
} else {
new_page = get_pages(0);
if (new_page == NULL)
return -EINVAL;
if (remap_va)
memcpy(new_page, (void *)phys_to_virt(pa), PAGE_SIZE);
else
memset(new_page, 0, PAGE_SIZE);
new_pa = (paddr_t)virt_to_phys(new_page);
page_allocated = true;
}
fault_vmspace =
obj_get(thread_to_wake->cap_group, VMSPACE_OBJ_ID, TYPE_VMSPACE);
if (fault_vmspace == NULL) {
return -EINVAL;
}
if (page_allocated) {
lock(&fault_vmspace->vmspace_lock);
fault_vmr = find_vmr_for_va(fault_vmspace, fault_va);
if (fault_vmr == NULL) {
unlock(&fault_vmspace->vmspace_lock);
obj_put(fault_vmspace);
return -EINVAL;
}
fault_pmo = fault_vmr->pmo;
commit_page_to_pmo(fault_pmo, new_pa, new_pa);
}
lock(&fault_vmspace->pgtbl_lock);
ret = map_range_in_pgtbl(
fault_vmspace->pgtbl, fault_va, new_pa, PAGE_SIZE, perm, &rss);
fault_vmspace->rss += rss;
BUG_ON(ret);
unlock(&fault_vmspace->pgtbl_lock);
if (page_allocated) {
unlock(&fault_vmspace->vmspace_lock);
}
obj_put(fault_vmspace);
switch_thread_vmspace_to(thread_to_wake);
if (perm & VMR_EXEC) {
arch_flush_cache(fault_va, PAGE_SIZE, SYNC_IDCACHE);
}
switch_thread_vmspace_to(current_thread);
thread_to_wake->thread_ctx->state = TS_INTER;
BUG_ON(sched_enqueue(thread_to_wake));
return 0;
}
void handle_user_fault(struct pmobject *pmo, vaddr_t fault_va)
{
struct fmap_fault_pool *fault_pool;
struct fault_pending_thread *pending_thread;
int ret;
fault_pool = (struct fmap_fault_pool *)pmo->private;
kdebug("pmo file fault: badge=%x, va=%lx\n",
fault_pool->cap_group_badge,
fault_va);
* Fault thread should pending until user handling finished.
* Record (fault_badge, fault_va) -> thread here.
*/
pending_thread =
(struct fault_pending_thread *)kmalloc(sizeof(*pending_thread));
if (!pending_thread) {
BUG_ON(1);
}
pending_thread->fault_badge = current_cap_group->badge;
pending_thread->fault_va = fault_va;
pending_thread->thread = current_thread;
lock(&fault_pool->lock);
if (if_buffer_full(fault_pool->msg_buffer_kva)) {
BUG_ON(1);
} else {
struct user_fault_msg tmp;
tmp.fault_badge = current_cap_group->badge;
tmp.fault_va = fault_va;
set_one_msg(fault_pool->msg_buffer_kva, &tmp);
}
list_append(&pending_thread->node, &fault_pool->pending_threads);
ret = signal_notific(fault_pool->notific);
BUG_ON(ret != 0);
* Give up the control flow here.
* The thread will wake up when map finished.
*/
current_thread->thread_ctx->state = TS_WAITING;
sched();
* To avoid sys_user_fault_map get pending thread too early,
* or modify thread->state early than here.
* Release lock here.
*/
unlock(&fault_pool->lock);
eret_to_thread(switch_context());
}
#endif