/*
 * Copyright (c) 2014, Mentor Graphics Corporation
 * All rights reserved.
 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
 * Copyright (c) 2018 Linaro, Inc. All rights reserved.
 * Copyright (c) 2021 Nordic Semiconductor ASA
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */

#include <metal/alloc.h>
#include <metal/sleep.h>
#include <metal/sys.h>
#include <metal/utilities.h>
#include <openamp/rpmsg_virtio.h>
#include <openamp/remoteproc.h>
#include <openamp/virtqueue.h>

#include "rpmsg_internal.h"

#define RPMSG_NUM_VRINGS                        2

/**
 * struct vbuff_reclaimer_t - vring buffer recycler
 *
 * This structure is used by the rpmsg virtio to store unused virtio buffer, as the
 * virtqueue structure has been already updated and memory allocated.
 *
 * @param node	node in reclaimer list.
 * @param idx	virtio descriptor index containing the buffer information.
 */
struct vbuff_reclaimer_t {
	struct metal_list node;
	uint16_t idx;
};

/* Default configuration */
#if VIRTIO_ENABLED(VIRTIO_DRIVER_SUPPORT)
#define RPMSG_VIRTIO_DEFAULT_CONFIG                \
	(&(const struct rpmsg_virtio_config) {     \
		.h2r_buf_size = RPMSG_BUFFER_SIZE, \
		.r2h_buf_size = RPMSG_BUFFER_SIZE, \
		.split_shpool = false,             \
	})
#else
#define RPMSG_VIRTIO_DEFAULT_CONFIG          NULL
#endif

#if VIRTIO_ENABLED(VIRTIO_DRIVER_SUPPORT)
metal_weak void *
rpmsg_virtio_shm_pool_get_buffer(struct rpmsg_virtio_shm_pool *shpool,
				 size_t size)
{
	void *buffer = NULL;

	if (shpool && size && shpool->avail >= size) {
		buffer = (char *)shpool->base + shpool->size - shpool->avail;
		shpool->avail -= size;
	}

	return buffer;
}
#endif

void rpmsg_virtio_init_shm_pool(struct rpmsg_virtio_shm_pool *shpool,
				void *shb, size_t size)
{
	if (shpool && shb && size) {
		shpool->base = shb;
		shpool->size = size;
		shpool->avail = size;
	}
}

/**
 * @internal
 *
 * @brief Places the used buffer back on the virtqueue.
 *
 * @param rvdev		Pointer to remote core
 * @param buffer	Buffer pointer
 * @param len		Buffer length
 * @param idx		Buffer index
 */
static void rpmsg_virtio_return_buffer(struct rpmsg_virtio_device *rvdev,
				       void *buffer, uint32_t len,
				       uint16_t idx)
{
	int ret;

	BUFFER_INVALIDATE(buffer, len);

	if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev)) {
		struct virtqueue_buf vqbuf;

		(void)idx;
		/* Initialize buffer node */
		vqbuf.buf = buffer;
		vqbuf.len = len;
		ret = virtqueue_add_buffer(rvdev->rvq, &vqbuf, 0, 1, buffer);
		RPMSG_ASSERT(ret == VQUEUE_SUCCESS, "add buffer failed\r\n");
	}

	if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev)) {
		(void)buffer;
		ret = virtqueue_add_consumed_buffer(rvdev->rvq, idx, len);
		RPMSG_ASSERT(ret == VQUEUE_SUCCESS, "add consumed buffer failed\r\n");
	}
}

/**
 * @internal
 *
 * @brief Places buffer on the virtqueue for consumption by the other side.
 *
 * @param rvdev		Pointer to rpmsg virtio
 * @param buffer	Buffer pointer
 * @param len		Buffer length
 * @param idx		Buffer index
 *
 * @return Status of function execution
 */
static int rpmsg_virtio_enqueue_buffer(struct rpmsg_virtio_device *rvdev,
				       void *buffer, uint32_t len,
				       uint16_t idx)
{
	int ret = 0;

	BUFFER_FLUSH(buffer, len);

	if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev)) {
		struct virtqueue_buf vqbuf;
		(void)idx;

		/* Initialize buffer node */
		vqbuf.buf = buffer;
		vqbuf.len = len;
		ret = virtqueue_add_buffer(rvdev->svq, &vqbuf, 1, 0, buffer);
	}

	if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev)) {
		(void)buffer;
		ret = virtqueue_add_consumed_buffer(rvdev->svq, idx, len);
	}

	return ret;
}

/**
 * @internal
 *
 * @brief Provides buffer to transmit messages.
 *
 * @param rvdev	Pointer to rpmsg device
 * @param len	Length of returned buffer
 * @param idx	Buffer index
 *
 * @return Pointer to buffer.
 */
static void *rpmsg_virtio_get_tx_buffer(struct rpmsg_virtio_device *rvdev,
					uint32_t *len, uint16_t *idx)
{
	struct metal_list *node;
	struct vbuff_reclaimer_t *r_desc;
	void *data = NULL;

	/* Try first to recycle a buffer that has been freed without been used */
	node = metal_list_first(&rvdev->reclaimer);
	if (node) {
		r_desc = metal_container_of(node, struct vbuff_reclaimer_t, node);
		metal_list_del(node);
		data = r_desc;
		*idx = r_desc->idx;

		if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev))
			*len = rvdev->config.h2r_buf_size;
		if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev))
			*len = virtqueue_get_buffer_length(rvdev->svq, *idx);
	} else if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev)) {
		data = virtqueue_get_buffer(rvdev->svq, len, idx);
		if (!data && rvdev->svq->vq_free_cnt) {
			data = rpmsg_virtio_shm_pool_get_buffer(rvdev->shpool,
					rvdev->config.h2r_buf_size);
			*len = rvdev->config.h2r_buf_size;
			*idx = 0;
		}
	} else if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev)) {
		data = virtqueue_get_available_buffer(rvdev->svq, idx, len);
	}

	return data;
}

/**
 * @internal
 *
 * @brief Check if the remote is ready to start RPMsg communication
 *
 * @param vdev Pointer to virtio device
 *
 * @return 0 on success, otherwise error code.
 */
static int rpmsg_virtio_wait_remote_ready(struct virtio_device *vdev)
{
	uint8_t status;
	int ret;

	while (1) {
		ret = virtio_get_status(vdev, &status);
		if (ret)
			break;
		/* Busy wait until the remote is ready */
		if (status & VIRTIO_CONFIG_STATUS_NEEDS_RESET) {
			ret = virtio_set_status(vdev, 0);
			if (ret)
				break;
			/* TODO notify remote processor */
		} else if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) {
			break;
		}
		metal_yield();
	}

	return ret;
}

static void rpmsg_virtio_hold_rx_buffer(struct rpmsg_device *rdev, void *rxbuf)
{
	metal_mutex_acquire(&rdev->lock);
	RPMSG_BUF_HELD_INC(RPMSG_LOCATE_HDR(rxbuf));
	metal_mutex_release(&rdev->lock);
}

static void rpmsg_virtio_release_rx_buffer(struct rpmsg_device *rdev,
					   void *rxbuf)
{
	struct rpmsg_virtio_device *rvdev;
	struct rpmsg_hdr *rp_hdr;

	rvdev = metal_container_of(rdev, struct rpmsg_virtio_device, rdev);
	rp_hdr = RPMSG_LOCATE_HDR(rxbuf);

	metal_mutex_acquire(&rdev->lock);
	if (rpmsg_virtio_buf_held_dec_test(rp_hdr)) {
		rpmsg_virtio_release_rx_buffer_nolock(rvdev, rp_hdr);
		/* Tell peer we returned an rx buffer */
		virtqueue_kick(rvdev->rvq);
	}
	metal_mutex_release(&rdev->lock);
}

static int rpmsg_virtio_notify_wait(struct rpmsg_virtio_device *rvdev, struct virtqueue *vq)
{
	struct virtio_vring_info *vring_info;
	int ret = RPMSG_EOPNOTSUPP;

	vring_info = &rvdev->vdev->vrings_info[vq->vq_queue_index];

	if (rvdev->notify_wait_cb)
		ret = rvdev->notify_wait_cb(&rvdev->rdev, vring_info->notifyid);

	return ret;
}

static void *rpmsg_virtio_get_tx_payload_buffer(struct rpmsg_device *rdev,
						uint32_t *len, int wait,
						uint8_t priority)
{
	struct rpmsg_virtio_device *rvdev;
	struct rpmsg_hdr *rp_hdr;
	uint16_t idx;
	void *data = NULL;
	int tick_count;
	int status;

	metal_unused(priority);

	/* Get the associated remote device for channel. */
	rvdev = metal_container_of(rdev, struct rpmsg_virtio_device, rdev);

	if (wait)
		tick_count = RPMSG_TICK_COUNT / RPMSG_TICKS_PER_INTERVAL;
	else
		tick_count = 0;

	while (1) {
		/* Lock the device to enable exclusive access to virtqueues */
		metal_mutex_acquire(&rdev->lock);
		rp_hdr = rpmsg_virtio_get_tx_buffer(rvdev, len, &idx);
		metal_mutex_release(&rdev->lock);
		if (rp_hdr || !tick_count)
			break;

		/*
		 * Try to use wait loop implemented in the virtio dispatcher and
		 * use metal_sleep_usec() method by default.
		 */
		status = rpmsg_virtio_notify_wait(rvdev, rvdev->rvq);
		if (status == RPMSG_EOPNOTSUPP) {
			metal_sleep_usec(RPMSG_TICKS_PER_INTERVAL);
			tick_count--;
		} else if (status != RPMSG_SUCCESS) {
			break;
		}
	}

	RPMSG_ASSERT(!wait || rp_hdr != NULL, "Get tx buffer timeout\r\n");

	if (rp_hdr) {
		/* Store the index into the reserved field to be used when sending */
		rp_hdr->reserved = idx;

		/* Increase the held counter to hold this Tx buffer */
		RPMSG_BUF_HELD_INC(rp_hdr);

		/* Actual data buffer size is vring buffer size minus header length */
		*len -= sizeof(struct rpmsg_hdr);
		data = RPMSG_LOCATE_DATA(rp_hdr);
	}

	return data;
}

static int rpmsg_virtio_send_offchannel_nocopy(struct rpmsg_device *rdev,
					       uint32_t src, uint32_t dst,
					       const void *data, int len)
{
	struct rpmsg_virtio_device *rvdev;
	struct metal_io_region *io;
	struct rpmsg_hdr rp_hdr;
	struct rpmsg_hdr *hdr;
	uint32_t buff_len;
	uint16_t idx;
	int status;

	/* Get the associated remote device for channel. */
	rvdev = metal_container_of(rdev, struct rpmsg_virtio_device, rdev);

	hdr = RPMSG_LOCATE_HDR(data);
	/* The reserved field contains buffer index */
	idx = RPMSG_BUF_INDEX(hdr);

	/* Initialize RPMSG header. */
	rp_hdr.dst = dst;
	rp_hdr.src = src;
	rp_hdr.len = (uint16_t)len;
	rp_hdr.reserved = 0;
	rp_hdr.flags = 0;

	/* Copy data to rpmsg buffer. */
	io = rvdev->shbuf_io;
	status = metal_io_block_write(io, metal_io_virt_to_offset(io, hdr),
				      &rp_hdr, (int)sizeof(rp_hdr));
	RPMSG_ASSERT(status == (int)sizeof(rp_hdr), "failed to write header\r\n");

	metal_mutex_acquire(&rdev->lock);

	if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev))
		buff_len = rvdev->config.h2r_buf_size;
	else
		buff_len = virtqueue_get_buffer_length(rvdev->svq, idx);

	/* Enqueue buffer on virtqueue. */
	status = rpmsg_virtio_enqueue_buffer(rvdev, hdr, buff_len, idx);
	RPMSG_ASSERT(status == VQUEUE_SUCCESS, "failed to enqueue buffer\r\n");
	/* Let the other side know that there is a job to process. */
	virtqueue_kick(rvdev->svq);

	metal_mutex_release(&rdev->lock);

	return len;
}

static int rpmsg_virtio_release_tx_buffer(struct rpmsg_device *rdev, void *txbuf)
{
	struct rpmsg_virtio_device *rvdev;
	struct rpmsg_hdr *rp_hdr = RPMSG_LOCATE_HDR(txbuf);
	void *vbuff = rp_hdr;  /* only used to avoid warning on the cast of a packed structure */
	struct vbuff_reclaimer_t *r_desc = (struct vbuff_reclaimer_t *)vbuff;

	rvdev = metal_container_of(rdev, struct rpmsg_virtio_device, rdev);

	metal_mutex_acquire(&rdev->lock);

	/* Check whether to release the Tx buffer */
	if (rpmsg_virtio_buf_held_dec_test(rp_hdr)) {
		/*
		 * Reuse the RPMsg buffer to temporary store the vbuff_reclaimer_t structure.
		 * Store the index locally before overwriting the RPMsg header.
		 */
		r_desc->idx = RPMSG_BUF_INDEX(rp_hdr);
		metal_list_add_tail(&rvdev->reclaimer, &r_desc->node);
	}

	metal_mutex_release(&rdev->lock);

	return RPMSG_SUCCESS;
}

/**
 * @internal
 *
 * @brief This function sends rpmsg "message" to remote device.
 *
 * @param rdev	Pointer to rpmsg device
 * @param src	Source address of channel
 * @param dst	Destination address of channel
 * @param data	Data to transmit
 * @param len	Size of data
 * @param wait	Boolean, wait or not for buffer to become
 *		available
 *
 * @return Size of data sent or negative value for failure.
 */
static int rpmsg_virtio_send_offchannel_raw(struct rpmsg_device *rdev,
					    uint32_t src, uint32_t dst,
					    const void *data, int len,
					    int wait, uint8_t priority)
{
	struct rpmsg_virtio_device *rvdev;
	struct metal_io_region *io;
	uint32_t buff_len;
	void *buffer;
	int status = RPMSG_ERR_NO_BUFF;

	/* Get the associated remote device for channel. */
	rvdev = metal_container_of(rdev, struct rpmsg_virtio_device, rdev);

	/* Get the payload buffer. */
	buffer = rpmsg_virtio_get_tx_payload_buffer(rdev, &buff_len, wait, priority);
	if (buffer) {
		/* Copy data to rpmsg buffer. */
		if (len > (int)buff_len)
			len = (int)buff_len;
		io = rvdev->shbuf_io;
		status = metal_io_block_write(io, metal_io_virt_to_offset(io, buffer),
					      data, len);
		RPMSG_ASSERT(status == len, "failed to write buffer\r\n");

		status = rpmsg_virtio_send_offchannel_nocopy(rdev, src, dst, buffer, len);
	}

	return status;
}

/**
 * @internal
 *
 * @brief Tx callback function.
 *
 * @param vq	Pointer to virtqueue on which Tx is has been
 *		completed.
 */
static void rpmsg_virtio_tx_callback(struct virtqueue *vq)
{
	(void)vq;
}

/**
 * @internal
 *
 * @brief Rx callback function.
 *
 * @param vq	Pointer to virtqueue on which messages is received
 */
static void rpmsg_virtio_rx_callback(struct virtqueue *vq)
{
	struct virtio_device *vdev = vq->vq_dev;
	struct rpmsg_virtio_device *rvdev = vdev->priv;
	struct rpmsg_device *rdev = &rvdev->rdev;
	struct rpmsg_endpoint *ept;
	struct rpmsg_hdr *rp_hdr;
	uint32_t len;
	uint16_t idx;
	bool release = false;
	bool last = false;
	int status;

	metal_mutex_acquire(&rdev->lock);
	while (!last) {
		/* Process the received data from remote node */
		rp_hdr = rpmsg_virtio_get_rx_buffer(rvdev, &len, &idx, &last);
		if (!rp_hdr)
			break;

		rp_hdr->reserved = idx;

		/* Get the channel node from the remote device channels list. */
		ept = rpmsg_get_ept_from_addr(rdev, rp_hdr->dst);
		rpmsg_ept_incref(ept);
		RPMSG_BUF_HELD_INC(rp_hdr);
		metal_mutex_release(&rdev->lock);

		if (ept) {
			if (ept->dest_addr == RPMSG_ADDR_ANY) {
				/*
				 * First message received from the remote side,
				 * update channel destination address
				 */
				ept->dest_addr = rp_hdr->src;
			}
			status = ept->cb(ept, RPMSG_LOCATE_DATA(rp_hdr),
					 rp_hdr->len, rp_hdr->src, ept->priv);

			RPMSG_ASSERT(status >= 0 ||
				     status == RPMSG_SUCCESS_BUFFER_RELEASED,
				     "unexpected callback status\r\n");
		} else {
			status = RPMSG_SUCCESS;
		}

		rpmsg_ept_decref(ept);
		metal_mutex_acquire(&rdev->lock);
		if (status != RPMSG_SUCCESS_BUFFER_RELEASED &&
		    rpmsg_virtio_buf_held_dec_test(rp_hdr)) {
			rpmsg_virtio_release_rx_buffer_nolock(rvdev, rp_hdr);
			if (VIRTIO_ENABLED(VQ_RX_EMPTY_NOTIFY))
				/* Kick will be sent only when last buffer is released */
				release = true;
			else
				/* Tell peer we returned an rx buffer */
				virtqueue_kick(rvdev->rvq);
		}
	}

	if (release && VIRTIO_ENABLED(VQ_RX_EMPTY_NOTIFY))
		/* tell peer we return some rx buffer */
		virtqueue_kick(rvdev->rvq);

	metal_mutex_release(&rdev->lock);
}

/**
 * @internal
 *
 * @brief This callback handles name service announcement from the remote
 * device and creates/deletes rpmsg channels.
 *
 * @param ept	Pointer to server channel control block.
 * @param data	Pointer to received messages
 * @param len	Length of received data
 * @param priv	Any private data
 * @param src	Source address
 *
 * @return Rpmsg endpoint callback handled
 */
static int rpmsg_virtio_ns_callback(struct rpmsg_endpoint *ept, void *data,
				    size_t len, uint32_t src, void *priv)
{
	struct rpmsg_device *rdev = priv;
	struct rpmsg_virtio_device *rvdev = metal_container_of(rdev,
							       struct rpmsg_virtio_device,
							       rdev);
	struct metal_io_region *io = rvdev->shbuf_io;
	struct rpmsg_endpoint *_ept;
	struct rpmsg_ns_msg ns_msg = {0};
	uint32_t dest;
	bool ept_to_release;
	int status = RPMSG_SUCCESS;

	(void)ept;
	(void)src;

	if (len == sizeof(ns_msg)) {
		/*
		 * copy buffer to local ns_msg and release the rx buffer early to
		 * improve the buffer utilization.
		 */
		metal_io_block_read(io, metal_io_virt_to_offset(io, data),
				    &ns_msg, (int)sizeof(ns_msg));
		rpmsg_virtio_release_rx_buffer(rdev, data);

		dest = ns_msg.addr;

		/* check if a Ept has been locally registered */
		metal_mutex_acquire(&rdev->lock);
		_ept = rpmsg_get_endpoint(rdev, ns_msg.name, RPMSG_ADDR_ANY, dest);

		/*
		 * If ept-release callback is not implemented, ns_unbind_cb() can free the ept.
		 * Test _ept->release_cb before calling ns_unbind_cb() callbacks.
		 */
		ept_to_release = _ept && _ept->release_cb;

		if (ns_msg.flags == RPMSG_NS_DESTROY) {
			if (_ept)
				_ept->dest_addr = RPMSG_ADDR_ANY;
			if (ept_to_release)
				rpmsg_ept_incref(_ept);
			metal_mutex_release(&rdev->lock);
			if (_ept && _ept->ns_unbind_cb)
				_ept->ns_unbind_cb(_ept);
			if (rdev->ns_unbind_cb)
				rdev->ns_unbind_cb(rdev, ns_msg.name, dest);
			if (ept_to_release)
				rpmsg_ept_decref(_ept);
		} else if (ns_msg.flags == RPMSG_NS_CREATE) {
			if (!_ept) {
				/*
				 * send callback to application, that can
				 * - create the associated endpoints.
				 * - store information for future use.
				 * - just ignore the request as service not supported.
				 */
				metal_mutex_release(&rdev->lock);
				if (rdev->ns_bind_cb)
					rdev->ns_bind_cb(rdev, ns_msg.name, dest);
			} else if (_ept->dest_addr == RPMSG_ADDR_ANY) {
				_ept->dest_addr = dest;
				metal_mutex_release(&rdev->lock);
				if (_ept->name[0] && rdev->support_ack)
					rpmsg_send_ns_message(_ept, RPMSG_NS_CREATE_ACK);
				/* notify application that the endpoint has been bound */
				if (_ept->ns_bound_cb)
					_ept->ns_bound_cb(_ept);
			}
			else
				metal_mutex_release(&rdev->lock);
		} else { /* RPMSG_NS_CREATE_ACK */
			/* save the received destination address */
			if (_ept && _ept->dest_addr == RPMSG_ADDR_ANY) {
				_ept->dest_addr = dest;
				metal_mutex_release(&rdev->lock);
				/* notify application that the endpoint has been bound */
				if (_ept->ns_bound_cb)
					_ept->ns_bound_cb(_ept);
			}
			else
				metal_mutex_release(&rdev->lock);
		}

		status = RPMSG_SUCCESS_BUFFER_RELEASED;
	}

	return status;
}

static int rpmsg_virtio_rvdev_init(struct rpmsg_virtio_device *rvdev,
				   struct virtio_device *vdev,
				   rpmsg_ns_bind_cb ns_bind_cb,
				   struct metal_io_region *shm_io,
				   const struct rpmsg_virtio_config *config)
{
	const char *vq_names[RPMSG_NUM_VRINGS];
	vq_callback callback[RPMSG_NUM_VRINGS];
	struct fw_rsc_config fw_config = {0};
	struct rpmsg_device *rdev;
	uint64_t features;
	int status;
	int i;

	rdev = &rvdev->rdev;
	rvdev->notify_wait_cb = NULL;
	memset(rdev, 0, sizeof(*rdev));
	metal_mutex_init(&rdev->lock);
	rvdev->vdev = vdev;
	rdev->ns_bind_cb = ns_bind_cb;
	vdev->priv = rvdev;
	rdev->ops.send_offchannel_raw = rpmsg_virtio_send_offchannel_raw;
	rdev->ops.hold_rx_buffer = rpmsg_virtio_hold_rx_buffer;
	rdev->ops.release_rx_buffer = rpmsg_virtio_release_rx_buffer;
	rdev->ops.get_tx_payload_buffer = rpmsg_virtio_get_tx_payload_buffer;
	rdev->ops.send_offchannel_nocopy = rpmsg_virtio_send_offchannel_nocopy;
	rdev->ops.release_tx_buffer = rpmsg_virtio_release_tx_buffer;
	rdev->ops.get_rx_buffer_size = rpmsg_virtio_get_rx_buffer_size;
	rdev->ops.get_tx_buffer_size = rpmsg_virtio_get_tx_buffer_size;

	status = virtio_get_features(vdev, &features);
	if (!status) {
		rdev->support_ns = !!(features & (1U << VIRTIO_RPMSG_F_NS));
		rdev->support_ack = !!(features & (1U << VIRTIO_RPMSG_F_ACK));
		if (features & (1U << VIRTIO_RPMSG_F_BUFSZ)) {
			virtio_read_config(rvdev->vdev, 0, &fw_config, (int)sizeof(fw_config));
			rvdev->config.h2r_buf_size = fw_config.h2r_buf_size;
			rvdev->config.r2h_buf_size = fw_config.r2h_buf_size;
		}

		if (VIRTIO_ROLE_IS_DRIVER(vdev)) {
			vq_names[0] = "rx_vq";
			vq_names[1] = "tx_vq";
			callback[0] = config->rx_callback ? config->rx_callback :
							    rpmsg_virtio_rx_callback;
			callback[1] = config->tx_callback ? config->tx_callback :
							    rpmsg_virtio_tx_callback;
		}

		if (VIRTIO_ROLE_IS_DEVICE(vdev)) {
			vq_names[0] = "tx_vq";
			vq_names[1] = "rx_vq";
			callback[0] = config->tx_callback ? config->tx_callback :
							    rpmsg_virtio_tx_callback;
			callback[1] = config->rx_callback ? config->rx_callback :
							    rpmsg_virtio_rx_callback;
		}

		rvdev->shbuf_io = shm_io;
		metal_list_init(&rvdev->reclaimer);

		/* Create virtqueues for remote device */
		status = virtio_create_virtqueues(vdev, 0, RPMSG_NUM_VRINGS,
						  vq_names, callback, NULL);
	}

	if (!status) {
		/* Create virtqueue success, assign back the virtqueue */
		if (VIRTIO_ROLE_IS_DRIVER(vdev)) {
			rvdev->rvq  = vdev->vrings_info[0].vq;
			rvdev->svq  = vdev->vrings_info[1].vq;
		}

		if (VIRTIO_ROLE_IS_DEVICE(vdev)) {
			rvdev->rvq  = vdev->vrings_info[1].vq;
			rvdev->svq  = vdev->vrings_info[0].vq;
		}

		/*
		 * Suppress "tx-complete" interrupts
		 * since send method use busy loop when buffer pool exhaust
		 */
		virtqueue_disable_cb(rvdev->svq);

		/* TODO: can have a virtio function to set the shared memory I/O */
		for (i = 0; i < RPMSG_NUM_VRINGS; i++) {
			struct virtqueue *vq;

			vq = vdev->vrings_info[i].vq;
			vq->shm_io = shm_io;
		}

		rvdev->shbuf = NULL;
	}

	return status;
}

static int rpmsg_virtio_rx_fill(struct rpmsg_virtio_device *rvdev,
				struct metal_io_region *shm_io,
				struct rpmsg_virtio_shm_pool *shpool,
				const struct rpmsg_virtio_config *config)
{
	struct virtqueue_buf vqbuf;
	unsigned int idx;
	size_t shbufsz;
	void *buffer;
	int status = RPMSG_SUCCESS;

	if (!shpool) {
		status = RPMSG_ERR_PARAM;
	} else if (!shpool->size) {
		shbufsz = rvdev->config.h2r_buf_size * rvdev->svq->vq_nentries +
			  rvdev->config.r2h_buf_size * rvdev->rvq->vq_nentries;
		status = virtio_alloc_buf(rvdev->vdev, &rvdev->shbuf, shbufsz, 8);
		if (status < 0) {
			status = RPMSG_ERR_NO_MEM;
		} else {
			rpmsg_virtio_init_shm_pool(shpool, rvdev->shbuf, shbufsz);
			rvdev->shpool = shpool;
		}
	} else {
		rvdev->shpool = config->split_shpool ? shpool + 1 : shpool;
		if (!rvdev->shpool->size) {
			status = RPMSG_ERR_PARAM;
		}
	}

	if (!status) {
		vqbuf.len = rvdev->config.r2h_buf_size;
		for (idx = 0; idx < rvdev->rvq->vq_nentries; idx++) {
			/* Initialize TX virtqueue buffers for remote device */
			buffer = rpmsg_virtio_shm_pool_get_buffer(shpool,
					rvdev->config.r2h_buf_size);

			if (!buffer) {
				status = RPMSG_ERR_NO_BUFF;
				break;
			}

			vqbuf.buf = buffer;

			metal_io_block_set(shm_io, metal_io_virt_to_offset(shm_io,
									   buffer),
					   0x00, (int)rvdev->config.r2h_buf_size);
			status = virtqueue_add_buffer(rvdev->rvq, &vqbuf, 0, 1,
						      buffer);
		}
	}

	return status;
}

int rpmsg_virtio_get_tx_buffer_size(struct rpmsg_device *rdev)
{
	struct rpmsg_virtio_device *rvdev;
	int size = RPMSG_ERR_PARAM;

	if (rdev) {
		metal_mutex_acquire(&rdev->lock);
		rvdev = (struct rpmsg_virtio_device *)rdev;

		if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev)) {
			/*
			 * If device role is host then buffers are provided by us,
			 * so just provide the macro.
			 */
			size = (int)(rvdev->config.h2r_buf_size - sizeof(struct rpmsg_hdr));
		}

		if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev)) {
			/*
			 * If other core is host then buffers are provided by it,
			 * so get the buffer size from the virtqueue.
			 */
			size = (int)(rvdev->config.r2h_buf_size - sizeof(struct rpmsg_hdr));
		}

		if (size <= 0)
			size = RPMSG_ERR_NO_BUFF;

		metal_mutex_release(&rdev->lock);
	}

	return size;
}

void *rpmsg_virtio_get_rx_buffer(struct rpmsg_virtio_device *rvdev,
				 uint32_t *len, uint16_t *idx, bool *last)
{
	void *data = NULL;

	if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev)) {
		data = virtqueue_get_buffer(rvdev->rvq, len, idx);
		*last = virtqueue_nused(rvdev->rvq) == 0U;
	}

	if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev)) {
		data =
		    virtqueue_get_available_buffer(rvdev->rvq, idx, len);
		*last = virtqueue_navail(rvdev->rvq) == 0U;
	}

	/* Invalidate the buffer before returning it */
	if (data)
		BUFFER_INVALIDATE(data, *len);

	return data;
}

int rpmsg_virtio_get_rx_buffer_size(struct rpmsg_device *rdev)
{
	struct rpmsg_virtio_device *rvdev;
	int size = RPMSG_ERR_PARAM;

	if (rdev) {
		metal_mutex_acquire(&rdev->lock);
		rvdev = (struct rpmsg_virtio_device *)rdev;

		if (VIRTIO_ROLE_IS_DRIVER(rvdev->vdev)) {
			/*
			 * If device role is host then buffers are provided by us,
			 * so just provide the macro.
			 */
			size = (int)(rvdev->config.r2h_buf_size - sizeof(struct rpmsg_hdr));
		}

		if (VIRTIO_ROLE_IS_DEVICE(rvdev->vdev)) {
			/*
			 * If other core is host then buffers are provided by it,
			 * so get the buffer size from the virtqueue.
			 */
			size = (int)(rvdev->config.h2r_buf_size - sizeof(struct rpmsg_hdr));
		}

		if (size <= 0)
			size = RPMSG_ERR_NO_BUFF;

		metal_mutex_release(&rdev->lock);
	}

	return size;
}

bool rpmsg_virtio_release_rx_buffer_nolock(struct rpmsg_virtio_device *rvdev,
					   struct rpmsg_hdr *rp_hdr)
{
	uint16_t idx;
	uint32_t len;

	/* The reserved field contains buffer index */
	idx = RPMSG_BUF_INDEX(rp_hdr);
	/* Return buffer on virtqueue. */
	len = virtqueue_get_buffer_length(rvdev->rvq, idx);
	rpmsg_virtio_return_buffer(rvdev, rp_hdr, len, idx);

	return true;
}

bool rpmsg_virtio_buf_held_dec_test(struct rpmsg_hdr *rp_hdr)
{
	/* Check the held counter first */
	RPMSG_ASSERT(RPMSG_BUF_HELD_COUNTER(rp_hdr) > 0U,
		     "unexpected buffer held counter\r\n");

	/* Decrease the held counter */
	RPMSG_BUF_HELD_DEC(rp_hdr);

	/* Check whether to release the buffer */
	return RPMSG_BUF_HELD_COUNTER(rp_hdr) <= 0U;
}

int rpmsg_init_vdev(struct rpmsg_virtio_device *rvdev,
		    struct virtio_device *vdev,
		    rpmsg_ns_bind_cb ns_bind_cb,
		    struct metal_io_region *shm_io,
		    struct rpmsg_virtio_shm_pool *shpool)
{
	return rpmsg_init_vdev_with_config(rvdev, vdev, ns_bind_cb, shm_io,
			   shpool, RPMSG_VIRTIO_DEFAULT_CONFIG);
}

int rpmsg_init_vdev_with_config(struct rpmsg_virtio_device *rvdev,
				struct virtio_device *vdev,
				rpmsg_ns_bind_cb ns_bind_cb,
				struct metal_io_region *shm_io,
				struct rpmsg_virtio_shm_pool *shpool,
				const struct rpmsg_virtio_config *config)
{
	int status = RPMSG_SUCCESS;

	if (!rvdev || !vdev || !shm_io)
		status = RPMSG_ERR_PARAM;
	else if (VIRTIO_ROLE_IS_DRIVER(vdev)) {
		if (config)
			rvdev->config = *config;
		else
			status = RPMSG_ERR_PARAM;
	} else if (VIRTIO_ROLE_IS_DEVICE(vdev))
		status = rpmsg_virtio_wait_remote_ready(vdev);

	if (!status)
		status = rpmsg_virtio_rvdev_init(rvdev, vdev, ns_bind_cb, shm_io, config);

	if (!status && VIRTIO_ROLE_IS_DRIVER(vdev))
		status = rpmsg_virtio_rx_fill(rvdev, shm_io, shpool, config);

	if (!status) {
		struct rpmsg_device *rdev = &rvdev->rdev;

		/* Initialize channels and endpoints list */
		metal_list_init(&rdev->endpoints);

		/*
		 * Create name service announcement endpoint if device supports name
		 * service announcement feature.
		 */
		if (rdev->support_ns || rdev->support_ack) {
			rpmsg_register_endpoint(rdev, &rdev->ns_ept, "NS",
						RPMSG_NS_EPT_ADDR, RPMSG_NS_EPT_ADDR,
						rpmsg_virtio_ns_callback, NULL, rvdev);
		}

		if (VIRTIO_ROLE_IS_DRIVER(vdev))
			status = virtio_set_status(vdev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
	}

	if (status) {
		if (rvdev && rvdev->shbuf)
			virtio_free_buf(vdev, rvdev->shbuf);
		virtio_delete_virtqueues(vdev);
	}

	return status;
}

void rpmsg_deinit_vdev(struct rpmsg_virtio_device *rvdev)
{
	struct metal_list *node;
	struct rpmsg_device *rdev;
	struct rpmsg_endpoint *ept;

	if (rvdev) {
		rdev = &rvdev->rdev;
		while (!metal_list_is_empty(&rdev->endpoints)) {
			node = rdev->endpoints.next;
			ept = metal_container_of(node, struct rpmsg_endpoint, node);
			rpmsg_destroy_ept(ept);
		}

		if (rvdev->shbuf)
			virtio_free_buf(rvdev->vdev, rvdev->shbuf);

		rvdev->rvq = 0;
		rvdev->svq = 0;

		virtio_delete_virtqueues(rvdev->vdev);
		metal_mutex_deinit(&rdev->lock);
		rvdev->vdev = NULL;
	}
}