qemu/backends/iommufd.c

/*
 * iommufd container backend
 *
 * Copyright (C) 2023 Intel Corporation.
 * Copyright Red Hat, Inc. 2023
 *
 * Authors: Yi Liu <yi.l.liu@intel.com>
 *          Eric Auger <eric.auger@redhat.com>
 *
 * SPDX-License-Identifier: GPL-2.0-or-later
 */

#include "qemu/osdep.h"
#include "system/iommufd.h"
#include "qapi/error.h"
#include "qemu/module.h"
#include "qom/object_interfaces.h"
#include "qemu/error-report.h"
#include "migration/cpr.h"
#include "monitor/monitor.h"
#include "trace.h"
#include "hw/vfio/vfio-device.h"
#include <sys/ioctl.h>
#include <linux/iommufd.h>

static const char *iommufd_fd_name(IOMMUFDBackend *be)
{
    return object_get_canonical_path_component(OBJECT(be));
}

static void iommufd_backend_init(Object *obj)
{
    IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);

    be->fd = -1;
    be->users = 0;
    be->owned = true;
}

static void iommufd_backend_finalize(Object *obj)
{
    IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);

    if (be->owned) {
        close(be->fd);
        be->fd = -1;
    }
}

static void iommufd_backend_set_fd(Object *obj, const char *str, Error **errp)
{
    ERRP_GUARD();
    IOMMUFDBackend *be = IOMMUFD_BACKEND(obj);
    int fd = -1;

    fd = monitor_fd_param(monitor_cur(), str, errp);
    if (fd == -1) {
        error_prepend(errp, "Could not parse remote object fd %s:", str);
        return;
    }
    be->fd = fd;
    be->owned = false;
    trace_iommu_backend_set_fd(be->fd);
}

static bool iommufd_backend_prepare_delete(UserCreatable *uc, Error **errp)
{
    IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);

    if (be->users) {
        error_setg(errp, "Can not delete IOMMUFD backend '%s' with %d users",
                   object_get_canonical_path_component(OBJECT(uc)), be->users);
        return false;
    }
    return true;
}

static void iommufd_backend_complete(UserCreatable *uc, Error **errp)
{
    IOMMUFDBackend *be = IOMMUFD_BACKEND(uc);
    const char *name = iommufd_fd_name(be);

    if (!be->owned) {
        /* fd came from the command line. Fetch updated value from cpr state. */
        if (cpr_is_incoming()) {
            be->fd = cpr_find_fd(name, 0);
        } else {
            cpr_save_fd(name, 0, be->fd);
        }
    } else if (!g_file_test("/dev/iommu", G_FILE_TEST_EXISTS)) {
        error_setg(errp, "/dev/iommu does not exist"
                         " (is your kernel config missing CONFIG_IOMMUFD?)");
    }
}

static void iommufd_backend_class_init(ObjectClass *oc, const void *data)
{
    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);

    ucc->prepare_delete = iommufd_backend_prepare_delete;
    ucc->complete = iommufd_backend_complete;

    object_class_property_add_str(oc, "fd", NULL, iommufd_backend_set_fd);
}

bool iommufd_change_process_capable(IOMMUFDBackend *be)
{
    struct iommu_ioas_change_process args = {.size = sizeof(args)};

    /*
     * Call IOMMU_IOAS_CHANGE_PROCESS to verify it is a recognized ioctl.
     * This is a no-op if the process has not changed since DMA was mapped.
     */
    return !ioctl(be->fd, IOMMU_IOAS_CHANGE_PROCESS, &args);
}

bool iommufd_change_process(IOMMUFDBackend *be, Error **errp)
{
    struct iommu_ioas_change_process args = {.size = sizeof(args)};
    bool ret = !ioctl(be->fd, IOMMU_IOAS_CHANGE_PROCESS, &args);

    if (!ret) {
        error_setg_errno(errp, errno, "IOMMU_IOAS_CHANGE_PROCESS fd %d failed",
                         be->fd);
    }
    trace_iommufd_change_process(be->fd, ret);
    return ret;
}

bool iommufd_backend_connect(IOMMUFDBackend *be, Error **errp)
{
    int fd;

    if (be->owned && !be->users) {
        fd = cpr_open_fd("/dev/iommu", O_RDWR, iommufd_fd_name(be), 0, errp);
        if (fd < 0) {
            return false;
        }
        be->fd = fd;
    }
    if (!be->users && !vfio_iommufd_cpr_register_iommufd(be, errp)) {
        if (be->owned) {
            close(be->fd);
            be->fd = -1;
        }
        return false;
    }
    be->users++;

    trace_iommufd_backend_connect(be->fd, be->owned, be->users);
    return true;
}

void iommufd_backend_disconnect(IOMMUFDBackend *be)
{
    if (!be->users) {
        goto out;
    }
    be->users--;
    if (!be->users) {
        vfio_iommufd_cpr_unregister_iommufd(be);
        if (be->owned) {
            cpr_delete_fd(iommufd_fd_name(be), 0);
            close(be->fd);
            be->fd = -1;
        }
    }
out:
    trace_iommufd_backend_disconnect(be->fd, be->users);
}

bool iommufd_backend_alloc_ioas(IOMMUFDBackend *be, uint32_t *ioas_id,
                                Error **errp)
{
    int fd = be->fd;
    struct iommu_ioas_alloc alloc_data  = {
        .size = sizeof(alloc_data),
        .flags = 0,
    };

    if (ioctl(fd, IOMMU_IOAS_ALLOC, &alloc_data)) {
        error_setg_errno(errp, errno, "Failed to allocate ioas");
        return false;
    }

    *ioas_id = alloc_data.out_ioas_id;
    trace_iommufd_backend_alloc_ioas(fd, *ioas_id);

    return true;
}

void iommufd_backend_free_id(IOMMUFDBackend *be, uint32_t id)
{
    int ret, fd = be->fd;
    struct iommu_destroy des = {
        .size = sizeof(des),
        .id = id,
    };

    ret = ioctl(fd, IOMMU_DESTROY, &des);
    trace_iommufd_backend_free_id(fd, id, ret);
    if (ret) {
        error_report("Failed to free id: %u %m", id);
    }
}

int iommufd_backend_map_dma(IOMMUFDBackend *be, uint32_t ioas_id, hwaddr iova,
                            uint64_t size, void *vaddr, bool readonly)
{
    int ret, fd = be->fd;
    struct iommu_ioas_map map = {
        .size = sizeof(map),
        .flags = IOMMU_IOAS_MAP_READABLE |
                 IOMMU_IOAS_MAP_FIXED_IOVA,
        .ioas_id = ioas_id,
        .__reserved = 0,
        .user_va = (uintptr_t)vaddr,
        .iova = iova,
        .length = size,
    };

    if (!readonly) {
        map.flags |= IOMMU_IOAS_MAP_WRITEABLE;
    }

    ret = ioctl(fd, IOMMU_IOAS_MAP, &map);
    trace_iommufd_backend_map_dma(fd, ioas_id, iova, size,
                                  vaddr, readonly, ret);
    if (ret) {
        ret = -errno;

        /* TODO: Not support mapping hardware PCI BAR region for now. */
        if (errno == EFAULT) {
            warn_report("IOMMU_IOAS_MAP failed: %m, PCI BAR?");
        }
    }
    return ret;
}

int iommufd_backend_map_file_dma(IOMMUFDBackend *be, uint32_t ioas_id,
                                 hwaddr iova, uint64_t size,
                                 int mfd, unsigned long start, bool readonly)
{
    int ret, fd = be->fd;
    struct iommu_ioas_map_file map = {
        .size = sizeof(map),
        .flags = IOMMU_IOAS_MAP_READABLE |
                 IOMMU_IOAS_MAP_FIXED_IOVA,
        .ioas_id = ioas_id,
        .fd = mfd,
        .start = start,
        .iova = iova,
        .length = size,
    };

    if (cpr_is_incoming()) {
        return 0;
    }

    if (!readonly) {
        map.flags |= IOMMU_IOAS_MAP_WRITEABLE;
    }

    ret = ioctl(fd, IOMMU_IOAS_MAP_FILE, &map);
    trace_iommufd_backend_map_file_dma(fd, ioas_id, iova, size, mfd, start,
                                       readonly, ret);
    if (ret) {
        ret = -errno;

        /* TODO: Not support mapping hardware PCI BAR region for now. */
        if (errno == EFAULT) {
            warn_report("IOMMU_IOAS_MAP_FILE failed: %m, PCI BAR?");
        }
    }
    return ret;
}

int iommufd_backend_unmap_dma(IOMMUFDBackend *be, uint32_t ioas_id,
                              hwaddr iova, uint64_t size)
{
    int ret, fd = be->fd;
    struct iommu_ioas_unmap unmap = {
        .size = sizeof(unmap),
        .ioas_id = ioas_id,
        .iova = iova,
        .length = size,
    };

    if (cpr_is_incoming()) {
        return 0;
    }

    ret = ioctl(fd, IOMMU_IOAS_UNMAP, &unmap);
    /*
     * IOMMUFD takes mapping as some kind of object, unmapping
     * nonexistent mapping is treated as deleting a nonexistent
     * object and return ENOENT. This is different from legacy
     * backend which allows it. vIOMMU may trigger a lot of
     * redundant unmapping, to avoid flush the log, treat them
     * as succeess for IOMMUFD just like legacy backend.
     */
    if (ret && errno == ENOENT) {
        trace_iommufd_backend_unmap_dma_non_exist(fd, ioas_id, iova, size, ret);
        ret = 0;
    } else {
        trace_iommufd_backend_unmap_dma(fd, ioas_id, iova, size, ret);
    }

    if (ret) {
        ret = -errno;
    }
    return ret;
}

bool iommufd_backend_alloc_hwpt(IOMMUFDBackend *be, uint32_t dev_id,
                                uint32_t pt_id, uint32_t flags,
                                uint32_t data_type, uint32_t data_len,
                                void *data_ptr, uint32_t *out_hwpt,
                                Error **errp)
{
    int ret, fd = be->fd;
    struct iommu_hwpt_alloc alloc_hwpt = {
        .size = sizeof(struct iommu_hwpt_alloc),
        .flags = flags,
        .dev_id = dev_id,
        .pt_id = pt_id,
        .data_type = data_type,
        .data_len = data_len,
        .data_uptr = (uintptr_t)data_ptr,
    };

    ret = ioctl(fd, IOMMU_HWPT_ALLOC, &alloc_hwpt);
    trace_iommufd_backend_alloc_hwpt(fd, dev_id, pt_id, flags, data_type,
                                     data_len, (uintptr_t)data_ptr,
                                     alloc_hwpt.out_hwpt_id, ret);
    if (ret) {
        error_setg_errno(errp, errno, "Failed to allocate hwpt");
        return false;
    }

    *out_hwpt = alloc_hwpt.out_hwpt_id;
    return true;
}

bool iommufd_backend_set_dirty_tracking(IOMMUFDBackend *be,
                                        uint32_t hwpt_id, bool start,
                                        Error **errp)
{
    int ret;
    struct iommu_hwpt_set_dirty_tracking set_dirty = {
            .size = sizeof(set_dirty),
            .hwpt_id = hwpt_id,
            .flags = start ? IOMMU_HWPT_DIRTY_TRACKING_ENABLE : 0,
    };

    ret = ioctl(be->fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &set_dirty);
    trace_iommufd_backend_set_dirty(be->fd, hwpt_id, start, ret ? errno : 0);
    if (ret) {
        error_setg_errno(errp, errno,
                         "IOMMU_HWPT_SET_DIRTY_TRACKING(hwpt_id %u) failed",
                         hwpt_id);
        return false;
    }

    return true;
}

bool iommufd_backend_get_dirty_bitmap(IOMMUFDBackend *be,
                                      uint32_t hwpt_id,
                                      uint64_t iova, ram_addr_t size,
                                      uint64_t page_size, uint64_t *data,
                                      uint64_t flags, Error **errp)
{
    int ret;
    struct iommu_hwpt_get_dirty_bitmap get_dirty_bitmap = {
        .size = sizeof(get_dirty_bitmap),
        .hwpt_id = hwpt_id,
        .iova = iova,
        .length = size,
        .page_size = page_size,
        .data = (uintptr_t)data,
        .flags = flags,
    };

    ret = ioctl(be->fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &get_dirty_bitmap);
    trace_iommufd_backend_get_dirty_bitmap(be->fd, hwpt_id, iova, size,
                                           flags, page_size, ret ? errno : 0);
    if (ret) {
        error_setg_errno(errp, errno,
                         "IOMMU_HWPT_GET_DIRTY_BITMAP (iova: 0x%"HWADDR_PRIx
                         " size: 0x"RAM_ADDR_FMT") failed", iova, size);
        return false;
    }

    return true;
}

/*
 * @type can carry a desired HW info type defined in the uapi headers. If caller
 * doesn't have one, indicating it wants the default type, then @type should be
 * zeroed (i.e. IOMMU_HW_INFO_TYPE_DEFAULT).
 */
bool iommufd_backend_get_device_info(IOMMUFDBackend *be, uint32_t devid,
                                     uint32_t *type, void *data, uint32_t len,
                                     uint64_t *caps, uint8_t *max_pasid_log2,
                                     Error **errp)
{
    struct iommu_hw_info info = {
        .flags = (*type) ? IOMMU_HW_INFO_FLAG_INPUT_TYPE : 0,
        .size = sizeof(info),
        .dev_id = devid,
        .data_len = len,
        .data_uptr = (uintptr_t)data,
        .in_data_type = *type,
    };

    if (ioctl(be->fd, IOMMU_GET_HW_INFO, &info)) {
        error_setg_errno(errp, errno, "Failed to get hardware info");
        return false;
    }

    g_assert(type);
    *type = info.out_data_type;
    g_assert(caps);
    *caps = info.out_capabilities;

    if (max_pasid_log2) {
        *max_pasid_log2 = info.out_max_pasid_log2;
    }
    return true;
}

bool iommufd_backend_invalidate_cache(IOMMUFDBackend *be, uint32_t id,
                                      uint32_t data_type, uint32_t entry_len,
                                      uint32_t *entry_num, void *data,
                                      Error **errp)
{
    int ret, fd = be->fd;
    uint32_t total_entries = *entry_num;
    struct iommu_hwpt_invalidate cache = {
        .size = sizeof(cache),
        .hwpt_id = id,
        .data_type = data_type,
        .entry_len = entry_len,
        .entry_num = total_entries,
        .data_uptr = (uintptr_t)data,
    };

    ret = ioctl(fd, IOMMU_HWPT_INVALIDATE, &cache);
    trace_iommufd_backend_invalidate_cache(fd, id, data_type, entry_len,
                                           total_entries, cache.entry_num,
                                           (uintptr_t)data, ret ? errno : 0);
    *entry_num = cache.entry_num;

    if (ret) {
        error_setg_errno(errp, errno, "IOMMU_HWPT_INVALIDATE failed:"
                         " total %d entries, processed %d entries",
                         total_entries, cache.entry_num);
    } else if (total_entries != cache.entry_num) {
        error_setg(errp, "IOMMU_HWPT_INVALIDATE succeed but with unprocessed"
                         " entries: total %d entries, processed %d entries."
                         " Kernel BUG?!", total_entries, cache.entry_num);
        return false;
    }

    return !ret;
}

bool iommufd_backend_alloc_viommu(IOMMUFDBackend *be, uint32_t dev_id,
                                  uint32_t viommu_type, uint32_t hwpt_id,
                                  void *data_ptr, uint32_t data_len,
                                  uint32_t *out_viommu_id, Error **errp)
{
    int ret;
    struct iommu_viommu_alloc alloc_viommu = {
        .size = sizeof(alloc_viommu),
        .type = viommu_type,
        .dev_id = dev_id,
        .hwpt_id = hwpt_id,
        .data_len = data_len,
        .data_uptr = (uintptr_t)data_ptr,
    };

    ret = ioctl(be->fd, IOMMU_VIOMMU_ALLOC, &alloc_viommu);

    trace_iommufd_backend_alloc_viommu(be->fd, dev_id, viommu_type, hwpt_id,
                                       (uintptr_t)data_ptr, data_len,
                                       alloc_viommu.out_viommu_id, ret);
    if (ret) {
        error_setg_errno(errp, errno, "IOMMU_VIOMMU_ALLOC failed");
        return false;
    }

    g_assert(out_viommu_id);
    *out_viommu_id = alloc_viommu.out_viommu_id;
    return true;
}

bool iommufd_backend_alloc_vdev(IOMMUFDBackend *be, uint32_t dev_id,
                                uint32_t viommu_id, uint64_t virt_id,
                                uint32_t *out_vdev_id, Error **errp)
{
    int ret;
    struct iommu_vdevice_alloc alloc_vdev = {
        .size = sizeof(alloc_vdev),
        .viommu_id = viommu_id,
        .dev_id = dev_id,
        .virt_id = virt_id,
    };

    ret = ioctl(be->fd, IOMMU_VDEVICE_ALLOC, &alloc_vdev);

    trace_iommufd_backend_alloc_vdev(be->fd, dev_id, viommu_id, virt_id,
                                     alloc_vdev.out_vdevice_id, ret);

    if (ret) {
        error_setg_errno(errp, errno, "IOMMU_VDEVICE_ALLOC failed");
        return false;
    }

    g_assert(out_vdev_id);
    *out_vdev_id = alloc_vdev.out_vdevice_id;
    return true;
}

bool iommufd_backend_alloc_veventq(IOMMUFDBackend *be, uint32_t viommu_id,
                                   uint32_t type, uint32_t depth,
                                   uint32_t *out_veventq_id,
                                   uint32_t *out_veventq_fd, Error **errp)
{
    int ret;
    struct iommu_veventq_alloc alloc_veventq = {
        .size = sizeof(alloc_veventq),
        .flags = 0,
        .type = type,
        .veventq_depth = depth,
        .viommu_id = viommu_id,
    };

    ret = ioctl(be->fd, IOMMU_VEVENTQ_ALLOC, &alloc_veventq);

    trace_iommufd_viommu_alloc_eventq(be->fd, viommu_id, type,
                                      alloc_veventq.out_veventq_id,
                                      alloc_veventq.out_veventq_fd, ret);
    if (ret) {
        error_setg_errno(errp, errno, "IOMMU_VEVENTQ_ALLOC failed");
        return false;
    }

    g_assert(out_veventq_id);
    g_assert(out_veventq_fd);
    *out_veventq_id = alloc_veventq.out_veventq_id;
    *out_veventq_fd = alloc_veventq.out_veventq_fd;
    return true;
}

bool iommufd_backend_alloc_hw_queue(IOMMUFDBackend *be, uint32_t viommu_id,
                                    uint32_t queue_type, uint32_t index,
                                    uint64_t addr, uint64_t length,
                                    uint32_t *out_hw_queue_id, Error **errp)
{
    int ret;
    struct iommu_hw_queue_alloc alloc_hw_queue = {
        .size = sizeof(alloc_hw_queue),
        .flags = 0,
        .viommu_id = viommu_id,
        .type = queue_type,
        .index = index,
        .nesting_parent_iova = addr,
        .length = length,
    };

    ret = ioctl(be->fd, IOMMU_HW_QUEUE_ALLOC, &alloc_hw_queue);

    trace_iommufd_backend_alloc_hw_queue(be->fd, viommu_id, queue_type,
                                         index, addr, length,
                                         alloc_hw_queue.out_hw_queue_id, ret);
    if (ret) {
        error_setg_errno(errp, errno, "IOMMU_HW_QUEUE_ALLOC failed");
        return false;
    }

    g_assert(out_hw_queue_id);
    *out_hw_queue_id = alloc_hw_queue.out_hw_queue_id;
    return true;
}

/*
 * Helper to mmap HW MMIO regions exposed via iommufd for a vIOMMU instance.
 * The caller is responsible for unmapping the mapped region.
 */
bool iommufd_backend_viommu_mmap(IOMMUFDBackend *be, uint32_t viommu_id,
                                 uint64_t size, off_t offset, void **out_ptr,
                                 Error **errp)
{
    g_assert(viommu_id);
    g_assert(out_ptr);

    *out_ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, be->fd,
                   offset);
    trace_iommufd_backend_viommu_mmap(be->fd, viommu_id, size, offset);
    if (*out_ptr == MAP_FAILED) {
        error_setg_errno(errp, errno, "IOMMUFD vIOMMU mmap failed");
        return false;
    }

    return true;
}

bool host_iommu_device_iommufd_attach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi,
                                           uint32_t pasid, uint32_t hwpt_id,
                                           Error **errp)
{
    HostIOMMUDeviceIOMMUFDClass *hiodic =
        HOST_IOMMU_DEVICE_IOMMUFD_GET_CLASS(hiodi);

    g_assert(hiodic->attach_hwpt);
    return hiodic->attach_hwpt(hiodi, pasid, hwpt_id, errp);
}

bool host_iommu_device_iommufd_detach_hwpt(HostIOMMUDeviceIOMMUFD *hiodi,
                                           uint32_t pasid, Error **errp)
{
    HostIOMMUDeviceIOMMUFDClass *hiodic =
        HOST_IOMMU_DEVICE_IOMMUFD_GET_CLASS(hiodi);

    g_assert(hiodic->detach_hwpt);
    return hiodic->detach_hwpt(hiodi, pasid, errp);
}

static int hiod_iommufd_get_cap(HostIOMMUDevice *hiod, int cap, Error **errp)
{
    HostIOMMUDeviceCaps *caps = &hiod->caps;

    switch (cap) {
    case HOST_IOMMU_DEVICE_CAP_IOMMU_TYPE:
        return caps->type;
    case HOST_IOMMU_DEVICE_CAP_AW_BITS:
        return vfio_device_get_aw_bits(hiod->agent);
    default:
        error_setg(errp, "%s: unsupported capability %x", hiod->name, cap);
        return -EINVAL;
    }
}

static bool hiod_iommufd_support_ats(HostIOMMUDevice *hiod)
{
    HostIOMMUDeviceCaps *caps = &hiod->caps;

    return !(caps->hw_caps & IOMMU_HW_CAP_PCI_ATS_NOT_SUPPORTED);
}

static bool hiod_iommufd_get_pasid_info(HostIOMMUDevice *hiod,
                                        PasidInfo *pasid_info)
{
    HostIOMMUDeviceCaps *caps = &hiod->caps;

    if (!caps->max_pasid_log2) {
        return false;
    }

    g_assert(pasid_info);
    pasid_info->exec_perm = (caps->hw_caps & IOMMU_HW_CAP_PCI_PASID_EXEC);
    pasid_info->priv_mod = (caps->hw_caps & IOMMU_HW_CAP_PCI_PASID_PRIV);
    pasid_info->max_pasid_log2 = caps->max_pasid_log2;
    return true;
}

static void hiod_iommufd_class_init(ObjectClass *oc, const void *data)
{
    HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_CLASS(oc);

    hiodc->get_cap = hiod_iommufd_get_cap;
    hiodc->get_pasid_info = hiod_iommufd_get_pasid_info;
    hiodc->support_ats = hiod_iommufd_support_ats;
};

static const TypeInfo types[] = {
    {
        .name = TYPE_IOMMUFD_BACKEND,
        .parent = TYPE_OBJECT,
        .instance_size = sizeof(IOMMUFDBackend),
        .instance_init = iommufd_backend_init,
        .instance_finalize = iommufd_backend_finalize,
        .class_size = sizeof(IOMMUFDBackendClass),
        .class_init = iommufd_backend_class_init,
        .interfaces = (const InterfaceInfo[]) {
            { TYPE_USER_CREATABLE },
            { }
        }
    }, {
        .name = TYPE_HOST_IOMMU_DEVICE_IOMMUFD,
        .parent = TYPE_HOST_IOMMU_DEVICE,
        .instance_size = sizeof(HostIOMMUDeviceIOMMUFD),
        .class_size = sizeof(HostIOMMUDeviceIOMMUFDClass),
        .class_init = hiod_iommufd_class_init,
        .abstract = true,
    }
};

DEFINE_TYPES(types)