Minimal reproducer for failure to map Vulkan dma-buf into ibverbs

nvidia-vulkan-dmabuf.c

// Copyright 2026 National Research Foundation (SARAO)

#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <unistd.h>
#include <vulkan/vulkan.h>
#include <infiniband/verbs.h>

static void vk_error(const char *msg, VkResult result)
{
    fprintf(stderr, "%s: %d\n", msg, (int) result);
}

static void generic_error(const char *msg)
{
    fprintf(stderr, "%s\n", msg);
}

#define INIT_VK_DEVICE_PFN(name) do { name = (PFN_ ## name) vkGetDeviceProcAddr(device, #name); } while (0)

int main()
{
    VkResult vk_result;
    VkInstance instance;
    VkPhysicalDevice phys_device = VK_NULL_HANDLE;
    VkDevice device;
    VkDeviceMemory memory;
    PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR;
    int fd = -1;
    int ret = 1;
    const size_t size = 65536;

    const VkApplicationInfo application_info =
    {
        .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
        .applicationVersion = 1,
        .apiVersion = VK_API_VERSION_1_1
    };
    const VkInstanceCreateInfo instance_info =
    {
        .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
        .pApplicationInfo = &application_info
    };
    if ((vk_result = vkCreateInstance(&instance_info, NULL, &instance)) != VK_SUCCESS)
    {
        vk_error("vkCreateInstance failed", vk_result);
        goto failed;
    }

    uint32_t n_devices;
    if ((vk_result = vkEnumeratePhysicalDevices(instance, &n_devices, NULL)) != VK_SUCCESS)
    {
        vk_error("vkEnumeratePhysicalDevices failed", vk_result);
        goto destroy_instance;
    }
    VkPhysicalDevice *devices = calloc(n_devices, sizeof(VkPhysicalDevice));
    if (devices == NULL)
    {
        generic_error("failed to allocate memory for device list");
        goto destroy_instance;
    }
    if ((vk_result = vkEnumeratePhysicalDevices(instance, &n_devices, devices)) != VK_SUCCESS)
    {
        vk_error("vkEnumeratePhysicalDevices failed", vk_result);
        goto free_devices;
    }

    for (uint32_t i = 0; i < n_devices; i++)
    {
        VkPhysicalDeviceProperties2 properties2 =
        {
            .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
        };
        vkGetPhysicalDeviceProperties2(devices[i], &properties2);
        if (properties2.properties.vendorID == 0x10de
            && properties2.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU)
        {
            phys_device = devices[i];
            break;
        }
    }
    if (phys_device == VK_NULL_HANDLE)
    {
        generic_error("no NVIDIA discrete GPU Vulkan device found");
        goto free_devices;
    }

    const float queue_priorities[] = {1.0f};
    const VkDeviceQueueCreateInfo queue_infos[] =
    {
        {
            .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
            .queueFamilyIndex = 0,  // TODO: introspect the families
            .queueCount = 1,
            .pQueuePriorities = queue_priorities
        }
    };
    const char * const extensions[2] = {
        "VK_KHR_external_memory_fd",
        "VK_EXT_external_memory_dma_buf",
    };
    const VkDeviceCreateInfo device_info =
    {
        .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
        .queueCreateInfoCount = sizeof(queue_infos) / sizeof(queue_infos[0]),
        .pQueueCreateInfos = queue_infos,
        .enabledExtensionCount = sizeof(extensions) / sizeof(extensions[0]),
        .ppEnabledExtensionNames = extensions
    };
    if ((vk_result = vkCreateDevice(phys_device, &device_info, NULL, &device)) != VK_SUCCESS)
    {
        vk_error("vkCreateDevice failed", vk_result);
        goto free_devices;
    }

    INIT_VK_DEVICE_PFN(vkGetMemoryFdKHR);

    VkPhysicalDeviceMemoryProperties memory_properties = {};
    vkGetPhysicalDeviceMemoryProperties(phys_device, &memory_properties);
    uint32_t i;
    VkMemoryPropertyFlags require = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
    for (i = 0; i < memory_properties.memoryTypeCount; i++)
    {
        if ((memory_properties.memoryTypes[i].propertyFlags & require) == require)
            break;
    }
    if (i == memory_properties.memoryTypeCount)
    {
        generic_error("Vulkan device does not provide a suitable memory type");
        goto destroy_device;
    }

    VkExportMemoryAllocateInfo export_info =
    {
        .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,
        .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
    };
    VkMemoryAllocateInfo info =
    {
        .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
        .pNext = &export_info,
        .allocationSize = size,
        .memoryTypeIndex = i
    };
    if ((vk_result = vkAllocateMemory(device, &info, NULL, &memory)) != VK_SUCCESS)
    {
        vk_error("vkAllocateMemory failed", vk_result);
        goto free_devices;
    }
    VkMemoryGetFdInfoKHR fd_info =
    {
        .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
        .memory = memory,
        .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT
    };
    if ((vk_result = vkGetMemoryFdKHR(device, &fd_info, &fd)) != VK_SUCCESS)
    {
        vk_error("vkGetMemoryFdKHR failed", vk_result);
        goto free_memory;
    }

    printf("Successfully obtained dma-buf: fd = %d\n", fd);

    int n_ibv_devices;
    struct ibv_device **ibv_devices;
    struct ibv_device *ibv_device;
    struct ibv_context *context;
    struct ibv_pd *pd;
    struct ibv_mr *mr;

    ibv_devices = ibv_get_device_list(&n_ibv_devices);
    if (ibv_devices == NULL)
    {
        perror("ibv_get_device_list failed");
        goto close_fd;
    }
    if (n_ibv_devices == 0)
    {
        generic_error("no IBV devices found");
        goto free_ibv_devices;
    }
    // TODO: iterate over devices to find an appropriate one
    ibv_device = ibv_devices[0];
    printf("Using IBV device %s\n", ibv_get_device_name(ibv_device));
    if ((context = ibv_open_device(ibv_device)) == NULL)
    {
        perror("ibv_open_device failed");
        goto free_ibv_devices;
    }

    if ((pd = ibv_alloc_pd(context)) == NULL)
    {
        perror("ibv_alloc_pd failed");
        goto close_ibv_device;
    }

    if ((mr = ibv_reg_dmabuf_mr(pd, 0, size, 0, fd, IBV_ACCESS_LOCAL_WRITE)) == NULL)
    {
        perror("ibv_reg_dmabuf_mr failed");
        goto dealloc_pd;
    }

    printf("Successfully created MR from dmabuf\n");

    ibv_dereg_mr(mr);
dealloc_pd:
    ibv_dealloc_pd(pd);
close_ibv_device:
    ibv_close_device(context);
free_ibv_devices:
    free(ibv_devices);
close_fd:
    close(fd);
free_memory:
    vkFreeMemory(device, memory, NULL);
free_devices:
    free(devices);
destroy_device:
    vkDestroyDevice(device, NULL);
destroy_instance:
    vkDestroyInstance(instance, NULL);
failed:
    return ret;
}