已合并
Remove the unnecessary heaplist rescan in svm VA alloc #60
tusdinbo创建于 4月3日
Remove the unnecessary heaplist rescan in svm VA alloc #60
已合并
共 4 个文件变更+41-14
| @@ -3256,6 +3256,7 @@ static void devmm_heap_close(struct devmm_virt_heap_mgmt *p_heap_mgmt, uint32_t | |||
| 3256 | (void)devmm_virt_destroy_heap(p_heap_mgmt, heap, true); | 3256 | (void)devmm_virt_destroy_heap(p_heap_mgmt, heap, true); |
| 3257 | heap_list->heap_cnt--; | 3257 | heap_list->heap_cnt--; |
| 3258 | } | 3258 | } |
| 3259 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 3259 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 3260 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 3260 | } | 3261 | } |
| 3261 | 3262 | ||
| @@ -212,6 +212,7 @@ STATIC virt_addr_t devmm_virt_set_alloced_mem_struct(struct devmm_virt_heap_mgmt | |||
| 212 | (void)pthread_rwlock_wrlock(&heap_list->list_lock); | 212 | (void)pthread_rwlock_wrlock(&heap_list->list_lock); |
| 213 | devmm_virt_list_add(&heap->list, &heap_list->heap_list); | 213 | devmm_virt_list_add(&heap->list, &heap_list->heap_list); |
| 214 | heap_list->heap_cnt++; | 214 | heap_list->heap_cnt++; |
| 215 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 215 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 216 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 216 | } | 217 | } |
| 217 | DEVMM_DRV_SWITCH("Devmm alloc heap. (ret_ptr=0x%lx; alloc_ptr=0x%lx; alloc_size=%lu; real_alloc_size=%lu)\n", | 218 | DEVMM_DRV_SWITCH("Devmm alloc heap. (ret_ptr=0x%lx; alloc_ptr=0x%lx; alloc_size=%lu; real_alloc_size=%lu)\n", |
| @@ -318,6 +319,7 @@ DVresult devmm_free_to_base_heap(struct devmm_virt_heap_mgmt *mgmt, struct devmm | |||
| 318 | } | 319 | } |
| 319 | devmm_virt_list_del_init(&heap->list); | 320 | devmm_virt_list_del_init(&heap->list); |
| 320 | heap_list->heap_cnt--; | 321 | heap_list->heap_cnt--; |
| 322 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 321 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 323 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 322 | 324 | ||
| 323 | if (devmm_virt_heap_free_ops(heap, ptr) != 0) { | 325 | if (devmm_virt_heap_free_ops(heap, ptr) != 0) { |
| @@ -402,6 +402,7 @@ static void devmm_virt_clear_sub_svm_type_heap(struct devmm_virt_heap_mgmt *mgmt | |||
| 402 | devmm_virt_list_del_init(&(heap->list)); | 402 | devmm_virt_list_del_init(&(heap->list)); |
| 403 | (void)devmm_virt_destroy_heap(mgmt, heap, true); /* will free heap, Cannot be accessed anymore */ | 403 | (void)devmm_virt_destroy_heap(mgmt, heap, true); /* will free heap, Cannot be accessed anymore */ |
| 404 | heap_list->heap_cnt--; | 404 | heap_list->heap_cnt--; |
| 405 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 405 | } | 406 | } |
| 406 | } | 407 | } |
| 407 | 408 | ||
| @@ -777,6 +778,7 @@ static void _devmm_virt_free_idle_heap(struct devmm_virt_heap_mgmt *mgmt, struct | |||
| 777 | devmm_virt_list_del_init(&(heap->list)); | 778 | devmm_virt_list_del_init(&(heap->list)); |
| 778 | (void)devmm_virt_destroy_heap(mgmt, heap, true); | 779 | (void)devmm_virt_destroy_heap(mgmt, heap, true); |
| 779 | heap_list->heap_cnt--; | 780 | heap_list->heap_cnt--; |
| 781 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 780 | } | 782 | } |
| 781 | } | 783 | } |
| 782 | 784 | ||
| @@ -955,6 +957,7 @@ STATIC DVresult devmm_virt_init_mgmt_queue_and_lists(struct devmm_virt_heap_mgmt | |||
| 955 | for (k = 0; k < DEVMM_MEM_TYPE_MAX; k++) { | 957 | for (k = 0; k < DEVMM_MEM_TYPE_MAX; k++) { |
| 956 | SVM_INIT_LIST_HEAD(&mgmt->normal_list[i][j][k].heap_list); | 958 | SVM_INIT_LIST_HEAD(&mgmt->normal_list[i][j][k].heap_list); |
| 957 | mgmt->normal_list[i][j][k].heap_cnt = 0; | 959 | mgmt->normal_list[i][j][k].heap_cnt = 0; |
| 960 | mgmt->normal_list[i][j][k].version = 0; | ||
| 958 | (void)pthread_rwlock_init(&mgmt->normal_list[i][j][k].list_lock, NULL); | 961 | (void)pthread_rwlock_init(&mgmt->normal_list[i][j][k].list_lock, NULL); |
| 959 | 962 | ||
| 960 | SVM_INIT_LIST_HEAD(&mgmt->huge_list[i][j][k].heap_list); | 963 | SVM_INIT_LIST_HEAD(&mgmt->huge_list[i][j][k].heap_list); |
| @@ -1132,6 +1135,12 @@ DVresult devmm_free_to_normal_heap(struct devmm_virt_heap_mgmt *p_heap_mgmt, str | |||
| 1132 | /* use heap list lock to ensure heap do not destroy when oper pg */ | 1135 | /* use heap list lock to ensure heap do not destroy when oper pg */ |
| 1133 | (void)pthread_rwlock_rdlock(&heap_list->list_lock); | 1136 | (void)pthread_rwlock_rdlock(&heap_list->list_lock); |
| 1134 | ret = devmm_free_mem(p, heap, free_len); | 1137 | ret = devmm_free_mem(p, heap, free_len); |
| 1138 | if (ret == DRV_ERROR_NONE) { | ||
| 1139 | /* increase the heaplist version to denote that there is memory space returned to the heap | ||
| 1140 | so the heaplist needs to be re-scanned before alloc a new heap and allocation path. | ||
| 1141 | */ | ||
| 1142 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 1143 | } | ||
| 1135 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 1144 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 1136 | if (ret != DRV_ERROR_NONE) { | 1145 | if (ret != DRV_ERROR_NONE) { |
| 1137 | DEVMM_DRV_ERR("Virt_heap_free_mem failed. (ret=%d; va=0x%llx)\n", ret, p); | 1146 | DEVMM_DRV_ERR("Virt_heap_free_mem failed. (ret=%d; va=0x%llx)\n", ret, p); |
| @@ -1394,6 +1403,7 @@ DVresult devmm_virt_init_heap_customize(struct devmm_virt_heap_mgmt *mgmt, struc | |||
| 1394 | 1403 | ||
| 1395 | devmm_virt_list_add(&heap->list, &heap_list->heap_list); | 1404 | devmm_virt_list_add(&heap->list, &heap_list->heap_list); |
| 1396 | heap_list->heap_cnt++; | 1405 | heap_list->heap_cnt++; |
| 1406 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 1397 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 1407 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 1398 | return DRV_ERROR_NONE; | 1408 | return DRV_ERROR_NONE; |
| 1399 | } | 1409 | } |
| @@ -1511,6 +1521,7 @@ static void devmm_add_heap_to_list(struct devmm_virt_com_heap *heap, struct devm | |||
| 1511 | { | 1521 | { |
| 1512 | devmm_virt_list_add(&heap->list, &heap_list->heap_list); | 1522 | devmm_virt_list_add(&heap->list, &heap_list->heap_list); |
| 1513 | heap_list->heap_cnt++; | 1523 | heap_list->heap_cnt++; |
| 1524 | (void)__sync_fetch_and_add(&heap_list->version, 1); | ||
| 1514 | DEVMM_DRV_SWITCH("Add heap list success. (heap_idx=%d)\n", heap->heap_idx); | 1525 | DEVMM_DRV_SWITCH("Add heap list success. (heap_idx=%d)\n", heap->heap_idx); |
| 1515 | } | 1526 | } |
| 1516 | 1527 | ||
| @@ -1581,26 +1592,30 @@ static DVdeviceptr _devmm_alloc_from_heaplist(struct devmm_heap_list *heap_list, | |||
| 1581 | 1592 | ||
| 1582 | /* If heaplist is out of virt mem, new heap to alloc */ | 1593 | /* If heaplist is out of virt mem, new heap to alloc */ |
| 1583 | static DVdeviceptr devmm_alloc_from_heaplist(struct devmm_heap_list *heap_list, struct devmm_virt_heap_type *heap_type, | 1594 | static DVdeviceptr devmm_alloc_from_heaplist(struct devmm_heap_list *heap_list, struct devmm_virt_heap_type *heap_type, |
| 1584 | size_t bytesize, DVmem_advise advise, uint64_t va) | 1595 | size_t bytesize, DVmem_advise advise, uint64_t va, uint64_t version) |
| 1585 | { | 1596 | { |
| 1586 | struct devmm_virt_com_heap *heap = NULL; | 1597 | struct devmm_virt_com_heap *heap = NULL; |
| 1587 | DVdeviceptr ptr; | 1598 | DVdeviceptr ptr; |
| 1588 | DVresult ret; | 1599 | DVresult ret; |
| 1589 | 1600 | ||
| 1590 | ptr = _devmm_alloc_from_heaplist(heap_list, bytesize, advise, va); | 1601 | uint64_t cur_version = __sync_fetch_and_add(&heap_list->version, 0); |
| 1591 | if (ptr_is_valid(ptr)) { | 1602 | if (unlikely(cur_version != version)) { |
| 1592 | return ptr; | 1603 | ptr = _devmm_alloc_from_heaplist(heap_list, bytesize, advise, va); |
| 1593 | } | 1604 | if (ptr_is_valid(ptr)) { |
| 1594 | 1605 | return ptr; | |
| 1595 | if (get_ptr_err(ptr) == DEVMM_OUT_OF_VIRT_MEM) { | 1606 | } |
| 1596 | ret = devmm_alloc_com_heap(heap_type, va, &heap); | 1607 | if (get_ptr_err(ptr) != DEVMM_OUT_OF_VIRT_MEM) { |
| 1597 | if (ret != DRV_ERROR_NONE) { | 1608 | return ptr; /* Device error */ |
| 1598 | return errcode_to_ptr(ret, DEVMM_INVALID_STOP); | ||
| 1599 | } | 1609 | } |
| 1600 | |||
| 1601 | ptr = devmm_alloc_from_tree(heap, bytesize, advise, DEVMM_IDLE_SIZE_TREE, va); | ||
| 1602 | devmm_add_heap_to_list(heap, heap_list); | ||
| 1603 | } | 1610 | } |
| 1611 | |||
| 1612 | ret = devmm_alloc_com_heap(heap_type, va, &heap); | ||
| 1613 | if (ret != DRV_ERROR_NONE) { | ||
| 1614 | return errcode_to_ptr(ret, DEVMM_INVALID_STOP); | ||
| 1615 | } | ||
| 1616 | |||
| 1617 | ptr = devmm_alloc_from_tree(heap, bytesize, advise, DEVMM_IDLE_SIZE_TREE, va); | ||
| 1618 | devmm_add_heap_to_list(heap, heap_list); | ||
| 1604 | return ptr; | 1619 | return ptr; |
| 1605 | } | 1620 | } |
| 1606 | 1621 | ||
| @@ -1610,6 +1625,7 @@ virt_addr_t devmm_alloc_from_normal_heap(struct devmm_virt_heap_mgmt *p_heap_mgm | |||
| 1610 | struct devmm_heap_list *heap_list = NULL; | 1625 | struct devmm_heap_list *heap_list = NULL; |
| 1611 | DVdeviceptr ptr; | 1626 | DVdeviceptr ptr; |
| 1612 | DVresult ret; | 1627 | DVresult ret; |
| 1628 | uint64_t version; | ||
| 1613 | 1629 | ||
| 1614 | ret = devmm_get_heap_list_by_type(p_heap_mgmt, heap_type, &heap_list); | 1630 | ret = devmm_get_heap_list_by_type(p_heap_mgmt, heap_type, &heap_list); |
| 1615 | if (ret != DRV_ERROR_NONE) { | 1631 | if (ret != DRV_ERROR_NONE) { |
| @@ -1618,6 +1634,7 @@ virt_addr_t devmm_alloc_from_normal_heap(struct devmm_virt_heap_mgmt *p_heap_mgm | |||
| 1618 | 1634 | ||
| 1619 | /* To improve the perf of concurrency, hold with read lock. */ | 1635 | /* To improve the perf of concurrency, hold with read lock. */ |
| 1620 | (void)pthread_rwlock_rdlock(&heap_list->list_lock); | 1636 | (void)pthread_rwlock_rdlock(&heap_list->list_lock); |
| 1637 | version = __sync_fetch_and_add(&heap_list->version, 0); | ||
| 1621 | ptr = _devmm_alloc_from_heaplist(heap_list, bytesize, advise, va); | 1638 | ptr = _devmm_alloc_from_heaplist(heap_list, bytesize, advise, va); |
| 1622 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 1639 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 1623 | if (ptr_is_valid(ptr)) { | 1640 | if (ptr_is_valid(ptr)) { |
| @@ -1630,7 +1647,7 @@ virt_addr_t devmm_alloc_from_normal_heap(struct devmm_virt_heap_mgmt *p_heap_mgm | |||
| 1630 | * otherwise alloc svm heap will fail if there are too many concurrent threads. | 1647 | * otherwise alloc svm heap will fail if there are too many concurrent threads. |
| 1631 | */ | 1648 | */ |
| 1632 | (void)pthread_rwlock_wrlock(&heap_list->list_lock); | 1649 | (void)pthread_rwlock_wrlock(&heap_list->list_lock); |
| 1633 | ptr = devmm_alloc_from_heaplist(heap_list, heap_type, bytesize, advise, va); | 1650 | ptr = devmm_alloc_from_heaplist(heap_list, heap_type, bytesize, advise, va, version); |
| 1634 | (void)pthread_rwlock_unlock(&heap_list->list_lock); | 1651 | (void)pthread_rwlock_unlock(&heap_list->list_lock); |
| 1635 | } | 1652 | } |
| 1636 | 1653 | ||
| @@ -44,6 +44,10 @@ | |||
| 44 | 44 | ||
| 45 | 45 | ||
| 46 | 46 | ||
| 47 | |||
| 48 | |||
| 49 | |||
| 50 | |||
| 47 | enum devmm_heap_list_type { | 51 | enum devmm_heap_list_type { |
| 48 | SVM_LIST, | 52 | SVM_LIST, |
| 49 | HOST_LIST, | 53 | HOST_LIST, |
| @@ -230,6 +234,9 @@ struct devmm_heap_list { | |||
| 230 | int heap_cnt; | 234 | int heap_cnt; |
| 231 | pthread_rwlock_t list_lock; | 235 | pthread_rwlock_t list_lock; |
| 232 | struct devmm_virt_list_head heap_list; | 236 | struct devmm_virt_list_head heap_list; |
| 237 | volatile uint64_t version; /* used for checking whether the heap_list has been modified or some memory spaces | ||
| 238 | have been return to heaps in the heap_list by another thread, if so the heap_list | ||
| 239 | needs to be re-scanned before alloc a new heap in allocation path */ | ||
| 233 | }; | 240 | }; |
| 234 | 241 | ||
| 235 | struct devmm_virt_heap_mgmt { | 242 | struct devmm_virt_heap_mgmt { |