Workaround to enable Vulkan validation layer in Vortek on Winlator

CMakeLists.txt

add_library(${CMAKE_PROJECT_NAME} SHARED
    # List C/C++ source files with relative paths to this CMakeLists.txt.
    dummyvk.cpp)

# Specifies libraries CMake should link to your target library. You
# can link libraries from various origins, such as libraries defined in this
# build script, prebuilt third-party libraries, or Android system libraries.
target_link_libraries(${CMAKE_PROJECT_NAME}
    # List libraries link to the target library
    android
    log
    vulkan)

dummyvk.cpp

#include <string.h>
#include <jni.h>
#include <vulkan/vulkan.h>
#include <dlfcn.h>
#include <android/log.h>
#include <stdio.h>
#include <__algorithm/find_if.h>
#include <assert.h>

extern "C"

// Intercepts vkCreateInstance _between_ Vortek and the underlying libvulkan.so
// and inject a single VK_LAYER_KHRONOS_validation layer
// TODO: this assumes that the application does not use any layers, need to make this more robust
VkResult my_vkCreateInstance(
        VkInstanceCreateInfo*                 pCreateInfo,
        const VkAllocationCallbacks*          pAllocator,
        VkInstance*                           pInstance) {
    __android_log_print(ANDROID_LOG_ERROR, "dummyvk", "Inside of my_vkCreateInstance.");

    void* libVulkan = dlopen("libvulkan.so", RTLD_NOW);
    PFN_vkCreateInstance original_vkCreateInstance_ptr = (PFN_vkCreateInstance)dlsym(libVulkan, "vkCreateInstance");

    // Enable just the Khronos validation layer.
    static const char *layers[] = {"VK_LAYER_KHRONOS_validation"};

    // Get the layer count using a null pointer as the last parameter.
    uint32_t instance_layer_present_count = 0;
    vkEnumerateInstanceLayerProperties(&instance_layer_present_count, nullptr);

    // Enumerate layers with a valid pointer in the last parameter.
    VkLayerProperties layer_props[instance_layer_present_count];
    vkEnumerateInstanceLayerProperties(&instance_layer_present_count, layer_props);

    for (const VkLayerProperties layerProperties : layer_props) {
        __android_log_print(ANDROID_LOG_ERROR, "dummyvk", "Layer found: %s\n%s", layerProperties.layerName, layerProperties.description);
    }

    // Make sure selected validation layers are available.
    VkLayerProperties *layer_props_end = layer_props + instance_layer_present_count;
    for (const char* layer:layers) {
        // TODO: make this modification conditional on this find, e.g. to support arbitrary layers
        // in /data/data/com.winlator/ as well
        assert(layer_props_end !=
               std::find_if(layer_props, layer_props_end, [layer](VkLayerProperties layerProperties) {
                   return strcmp(layerProperties.layerName, layer) == 0;
               }));
    }

    // Set the validation layer
    // TODO: inherit the existing layers
    pCreateInfo->enabledLayerCount = 1;
    pCreateInfo->ppEnabledLayerNames = layers;

    return original_vkCreateInstance_ptr(pCreateInfo, pAllocator, pInstance);
}

// Super hacky way to find the cached vkCreateInstance ptr within Vortek
// We need to do this (instead of directly patching libvulkan) in order to hijack the underlying
// call to Vk and inject our validation layer (see my_vkCreateInstance)
// Design:
// 1. Walk through the memory mapping of the current process (this would be under /proc/pid/maps
// 2. Find the base addr of the loaded image for libvortekrenderer
// 3. HACK: calculate the address for the cached pointer via a constant (liable to change, but
//    since Vortek doesn't export these symbols in the GOT, we can't easily find them)
void* find_vkCreateInstance_ptr() {
    // Open the library
    void* lib_handle = dlopen("libvortekrenderer.so", RTLD_LAZY);
    if (!lib_handle) return NULL;

    // Get the library base address from /proc/self/maps
    FILE* maps = fopen("/proc/self/maps", "r");
    char line[512];
    unsigned long lib_base = 0;

    while (fgets(line, sizeof(line), maps)) {
        if (strstr(line, "libvortekrenderer.so")) {
            sscanf(line, "%lx", &lib_base);
            break;
        }
    }
    fclose(maps);

    if (lib_base == 0) {
        __android_log_print(ANDROID_LOG_ERROR, "dummyvk", "Could not find the memory map.");
        dlclose(lib_handle);
        return nullptr;
    }

    // From the assembly analysis:
    // The vulkan wrapper structure base is loaded via adrp + ldr at e328-e32c
    // e328: adrp x8, 0x3a000  ; This gives us page-aligned address
    // e32c: ldr x8, [x8, #0x9d0] ; This loads from offset 0x9d0

    // Calculate the actual address where the vulkan wrapper pointer is stored
    void** wrapper_ptr_location = (void**)((char*)lib_base + 0x3a000 + 0x9d0);
    void* wrapper_base = *wrapper_ptr_location;

    if (!wrapper_base) {
        __android_log_print(ANDROID_LOG_ERROR, "dummyvk", "Could not find the Vortek cached Vulkan functions table.");
        dlclose(lib_handle);
        return nullptr;
    }

    // The vkCreateInstance pointer is at offset 0x18 from wrapper base
    // e388: str    x0, [x23, #0x18] ; stores the vkCreateInstance symbol from libvulkan at +0x18
    void** vkCreateInstance_ptr = (void**)((char*)wrapper_base + 0x18);

    dlclose(lib_handle);
    return vkCreateInstance_ptr;
}

extern "C"
void Java_com_winlator_xenvironment_components_VortekRendererComponent_patch( JNIEnv* env, jobject klass);
JNIEXPORT void Java_com_winlator_xenvironment_components_VortekRendererComponent_patch( JNIEnv* env, jobject klass ){
    __android_log_print(ANDROID_LOG_ERROR, "dummyvk", "Inside of patch.");
    // Calculate the actual address of the cache vkCreateInstance pointer within Vortek
    PFN_vkCreateInstance* vkCreateInstance_ptr = (PFN_vkCreateInstance*)find_vkCreateInstance_ptr();
    if (*vkCreateInstance_ptr != (PFN_vkCreateInstance)&my_vkCreateInstance) {
        __android_log_print(ANDROID_LOG_ERROR, "dummyvk", "Patching from %p to %p.", vkCreateInstance_ptr, &my_vkCreateInstance);
        *vkCreateInstance_ptr = (PFN_vkCreateInstance) &my_vkCreateInstance;
    }
}

VortekRendererComponent.smali

.class public Lcom/winlator/xenvironment/components/VortekRendererComponent;
.super Lcom/winlator/xenvironment/EnvironmentComponent;
.source "VortekRendererComponent.java"

# interfaces
.implements Lcom/winlator/xconnector/ConnectionHandler;
.implements Lcom/winlator/xconnector/RequestHandler;


# annotations
.annotation system Ldalvik/annotation/MemberClasses;
    value = {
        Lcom/winlator/xenvironment/components/VortekRendererComponent$Options;
    }
.end annotation


# static fields
.field public static final VK_MAX_VERSION:I


# instance fields
.field private connector:Lcom/winlator/xconnector/XConnectorEpoll;

.field private final options:Lcom/winlator/xenvironment/components/VortekRendererComponent$Options;

.field private final socketConfig:Lcom/winlator/xconnector/UnixSocketConfig;

.field private final xServer:Lcom/winlator/xserver/XServer;


# direct methods
.method static constructor <clinit>()V
    .locals 3

    .line 26
    const/4 v0, 0x1

    const/4 v1, 0x3

    const/16 v2, 0x80

    invoke-static {v0, v1, v2}, Lcom/winlator/core/GPUHelper;->vkMakeVersion(III)I

    move-result v0

    sput v0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->VK_MAX_VERSION:I

    .line 33
    const-string v0, "vortekrenderer"

    invoke-static {v0}, Ljava/lang/System;->loadLibrary(Ljava/lang/String;)V

    const-string v0, "dummyvk"

    invoke-static {v0}, Ljava/lang/System;->loadLibrary(Ljava/lang/String;)V

    .line 34
    return-void
.end method

.method public constructor <init>(Lcom/winlator/xserver/XServer;Lcom/winlator/xconnector/UnixSocketConfig;Lcom/winlator/xenvironment/components/VortekRendererComponent$Options;)V
    .locals 0
    .param p1, "xServer"    # Lcom/winlator/xserver/XServer;
    .param p2, "socketConfig"    # Lcom/winlator/xconnector/UnixSocketConfig;
    .param p3, "options"    # Lcom/winlator/xenvironment/components/VortekRendererComponent$Options;

    .line 61
    invoke-direct {p0}, Lcom/winlator/xenvironment/EnvironmentComponent;-><init>()V

    .line 62
    iput-object p1, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->xServer:Lcom/winlator/xserver/XServer;

    .line 63
    iput-object p2, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->socketConfig:Lcom/winlator/xconnector/UnixSocketConfig;

    .line 64
    iput-object p3, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->options:Lcom/winlator/xenvironment/components/VortekRendererComponent$Options;

    .line 65
    return-void
.end method

.method private native createVkContext(ILcom/winlator/xenvironment/components/VortekRendererComponent$Options;)J
.end method

.method private native destroyVkContext(J)V
.end method

.method private static native patch()V
.end method

.method private getWindowHardwareBuffer(I)J
    .locals 7
    .param p1, "windowId"    # I
    .annotation build Landroidx/annotation/Keep;
    .end annotation

    .line 98
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->xServer:Lcom/winlator/xserver/XServer;

    iget-object v0, v0, Lcom/winlator/xserver/XServer;->windowManager:Lcom/winlator/xserver/WindowManager;

    invoke-virtual {v0, p1}, Lcom/winlator/xserver/WindowManager;->getWindow(I)Lcom/winlator/xserver/Window;

    move-result-object v0

    .line 99
    .local v0, "window":Lcom/winlator/xserver/Window;
    if-eqz v0, :cond_1

    .line 100
    invoke-virtual {v0}, Lcom/winlator/xserver/Window;->getContent()Lcom/winlator/xserver/Drawable;

    move-result-object v1

    .line 101
    .local v1, "drawable":Lcom/winlator/xserver/Drawable;
    invoke-virtual {v1}, Lcom/winlator/xserver/Drawable;->getTexture()Lcom/winlator/renderer/Texture;

    move-result-object v2

    .line 103
    .local v2, "texture":Lcom/winlator/renderer/Texture;
    instance-of v3, v2, Lcom/winlator/renderer/GPUImage;

    if-nez v3, :cond_0

    .line 104
    iget-object v3, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->xServer:Lcom/winlator/xserver/XServer;

    invoke-virtual {v3}, Lcom/winlator/xserver/XServer;->getRenderer()Lcom/winlator/renderer/GLRenderer;

    move-result-object v3

    iget-object v3, v3, Lcom/winlator/renderer/GLRenderer;->xServerView:Lcom/winlator/widget/XServerView;

    invoke-static {v2}, Ljava/util/Objects;->requireNonNull(Ljava/lang/Object;)Ljava/lang/Object;

    new-instance v4, Lcom/winlator/xenvironment/components/VortekRendererComponent$$ExternalSyntheticLambda0;

    invoke-direct {v4, v2}, Lcom/winlator/xenvironment/components/VortekRendererComponent$$ExternalSyntheticLambda0;-><init>(Lcom/winlator/renderer/Texture;)V

    invoke-virtual {v3, v4}, Landroid/opengl/GLSurfaceView;->queueEvent(Ljava/lang/Runnable;)V

    .line 105
    new-instance v3, Lcom/winlator/renderer/GPUImage;

    iget-short v4, v1, Lcom/winlator/xserver/Drawable;->width:S

    iget-short v5, v1, Lcom/winlator/xserver/Drawable;->height:S

    const/4 v6, 0x0

    invoke-direct {v3, v4, v5, v6, v6}, Lcom/winlator/renderer/GPUImage;-><init>(SSZZ)V

    invoke-virtual {v1, v3}, Lcom/winlator/xserver/Drawable;->setTexture(Lcom/winlator/renderer/Texture;)V

    .line 108
    :cond_0
    invoke-virtual {v1}, Lcom/winlator/xserver/Drawable;->getTexture()Lcom/winlator/renderer/Texture;

    move-result-object v3

    check-cast v3, Lcom/winlator/renderer/GPUImage;

    invoke-virtual {v3}, Lcom/winlator/renderer/GPUImage;->getHardwareBufferPtr()J

    move-result-wide v3

    return-wide v3

    .line 111
    .end local v1    # "drawable":Lcom/winlator/xserver/Drawable;
    .end local v2    # "texture":Lcom/winlator/renderer/Texture;
    :cond_1
    const-wide/16 v1, 0x0

    return-wide v1
.end method

.method private getWindowHeight(I)I
    .locals 2
    .param p1, "windowId"    # I
    .annotation build Landroidx/annotation/Keep;
    .end annotation

    .line 92
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->xServer:Lcom/winlator/xserver/XServer;

    iget-object v0, v0, Lcom/winlator/xserver/XServer;->windowManager:Lcom/winlator/xserver/WindowManager;

    invoke-virtual {v0, p1}, Lcom/winlator/xserver/WindowManager;->getWindow(I)Lcom/winlator/xserver/Window;

    move-result-object v0

    .line 93
    .local v0, "window":Lcom/winlator/xserver/Window;
    if-eqz v0, :cond_0

    invoke-virtual {v0}, Lcom/winlator/xserver/Window;->getHeight()S

    move-result v1

    goto :goto_0

    :cond_0
    const/4 v1, 0x0

    :goto_0
    return v1
.end method

.method private getWindowWidth(I)I
    .locals 2
    .param p1, "windowId"    # I
    .annotation build Landroidx/annotation/Keep;
    .end annotation

    .line 86
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->xServer:Lcom/winlator/xserver/XServer;

    iget-object v0, v0, Lcom/winlator/xserver/XServer;->windowManager:Lcom/winlator/xserver/WindowManager;

    invoke-virtual {v0, p1}, Lcom/winlator/xserver/WindowManager;->getWindow(I)Lcom/winlator/xserver/Window;

    move-result-object v0

    .line 87
    .local v0, "window":Lcom/winlator/xserver/Window;
    if-eqz v0, :cond_0

    invoke-virtual {v0}, Lcom/winlator/xserver/Window;->getWidth()S

    move-result v1

    goto :goto_0

    :cond_0
    const/4 v1, 0x0

    :goto_0
    return v1
.end method

.method private updateWindowContent(I)V
    .locals 4
    .param p1, "windowId"    # I
    .annotation build Landroidx/annotation/Keep;
    .end annotation

    .line 116
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->xServer:Lcom/winlator/xserver/XServer;

    iget-object v0, v0, Lcom/winlator/xserver/XServer;->windowManager:Lcom/winlator/xserver/WindowManager;

    invoke-virtual {v0, p1}, Lcom/winlator/xserver/WindowManager;->getWindow(I)Lcom/winlator/xserver/Window;

    move-result-object v0

    .line 117
    .local v0, "window":Lcom/winlator/xserver/Window;
    if-eqz v0, :cond_0

    .line 118
    invoke-virtual {v0}, Lcom/winlator/xserver/Window;->getContent()Lcom/winlator/xserver/Drawable;

    move-result-object v1

    .line 119
    .local v1, "drawable":Lcom/winlator/xserver/Drawable;
    iget-object v2, v1, Lcom/winlator/xserver/Drawable;->renderLock:Ljava/lang/Object;

    monitor-enter v2

    .line 120
    :try_start_0
    invoke-virtual {v1}, Lcom/winlator/xserver/Drawable;->forceUpdate()V

    .line 121
    monitor-exit v2

    goto :goto_0

    :catchall_0
    move-exception v3

    monitor-exit v2
    :try_end_0
    .catchall {:try_start_0 .. :try_end_0} :catchall_0

    throw v3

    .line 123
    .end local v1    # "drawable":Lcom/winlator/xserver/Drawable;
    :cond_0
    :goto_0
    return-void
.end method


# virtual methods
.method public handleConnectionShutdown(Lcom/winlator/xconnector/Client;)V
    .locals 2
    .param p1, "client"    # Lcom/winlator/xconnector/Client;

    .line 127
    invoke-virtual {p1}, Lcom/winlator/xconnector/Client;->getTag()Ljava/lang/Object;

    move-result-object v0

    if-eqz v0, :cond_0

    .line 128
    invoke-virtual {p1}, Lcom/winlator/xconnector/Client;->getTag()Ljava/lang/Object;

    move-result-object v0

    check-cast v0, Ljava/lang/Long;

    invoke-virtual {v0}, Ljava/lang/Long;->longValue()J

    move-result-wide v0

    .line 129
    .local v0, "contextPtr":J
    invoke-direct {p0, v0, v1}, Lcom/winlator/xenvironment/components/VortekRendererComponent;->destroyVkContext(J)V

    .line 131
    .end local v0    # "contextPtr":J
    :cond_0
    return-void
.end method

.method public handleNewConnection(Lcom/winlator/xconnector/Client;)V
    .locals 0
    .param p1, "client"    # Lcom/winlator/xconnector/Client;

    .line 135
    invoke-virtual {p1}, Lcom/winlator/xconnector/Client;->createIOStreams()V

    .line 136
    return-void
.end method

.method public handleRequest(Lcom/winlator/xconnector/Client;)Z
    .locals 7
    .param p1, "client"    # Lcom/winlator/xconnector/Client;
    .annotation system Ldalvik/annotation/Throws;
        value = {
            Ljava/io/IOException;
        }
    .end annotation

    .line 140
    invoke-virtual {p1}, Lcom/winlator/xconnector/Client;->getInputStream()Lcom/winlator/xconnector/XInputStream;

    move-result-object v0

    .line 141
    .local v0, "inputStream":Lcom/winlator/xconnector/XInputStream;
    invoke-virtual {v0}, Lcom/winlator/xconnector/XInputStream;->available()I

    move-result v1

    const/4 v2, 0x1

    if-ge v1, v2, :cond_0

    const/4 v1, 0x0

    return v1

    .line 142
    :cond_0
    invoke-virtual {v0}, Lcom/winlator/xconnector/XInputStream;->readByte()B

    move-result v1

    .line 144
    .local v1, "requestCode":B
    if-ne v1, v2, :cond_2

    .line 145
    iget-object v3, p1, Lcom/winlator/xconnector/Client;->clientSocket:Lcom/winlator/xconnector/ClientSocket;

    iget v3, v3, Lcom/winlator/xconnector/ClientSocket;->fd:I

    iget-object v4, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->options:Lcom/winlator/xenvironment/components/VortekRendererComponent$Options;

    invoke-direct {p0, v3, v4}, Lcom/winlator/xenvironment/components/VortekRendererComponent;->createVkContext(ILcom/winlator/xenvironment/components/VortekRendererComponent$Options;)J

    move-result-wide v3

    invoke-static {}, Lcom/winlator/xenvironment/components/VortekRendererComponent;->patch()V

    .line 146
    .local v3, "contextPtr":J
    const-wide/16 v5, 0x0

    cmp-long v5, v3, v5

    if-lez v5, :cond_1

    .line 147
    invoke-static {v3, v4}, Ljava/lang/Long;->valueOf(J)Ljava/lang/Long;

    move-result-object v5

    invoke-virtual {p1, v5}, Lcom/winlator/xconnector/Client;->setTag(Ljava/lang/Object;)V

    goto :goto_0

    .line 149
    :cond_1
    iget-object v5, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    invoke-virtual {v5, p1}, Lcom/winlator/xconnector/XConnectorEpoll;->killConnection(Lcom/winlator/xconnector/Client;)V

    .line 152
    .end local v3    # "contextPtr":J
    :cond_2
    :goto_0
    return v2
.end method

.method public start()V
    .locals 2

    .line 69
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    if-eqz v0, :cond_0

    return-void

    .line 70
    :cond_0
    new-instance v0, Lcom/winlator/xconnector/XConnectorEpoll;

    iget-object v1, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->socketConfig:Lcom/winlator/xconnector/UnixSocketConfig;

    invoke-direct {v0, v1, p0, p0}, Lcom/winlator/xconnector/XConnectorEpoll;-><init>(Lcom/winlator/xconnector/UnixSocketConfig;Lcom/winlator/xconnector/ConnectionHandler;Lcom/winlator/xconnector/RequestHandler;)V

    iput-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    .line 71
    const/4 v1, 0x1

    invoke-virtual {v0, v1}, Lcom/winlator/xconnector/XConnectorEpoll;->setInitialInputBufferCapacity(I)V

    .line 72
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    const/4 v1, 0x0

    invoke-virtual {v0, v1}, Lcom/winlator/xconnector/XConnectorEpoll;->setInitialOutputBufferCapacity(I)V

    .line 73
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    invoke-virtual {v0}, Lcom/winlator/xconnector/XConnectorEpoll;->start()V

    .line 74
    return-void
.end method

.method public stop()V
    .locals 1

    .line 78
    iget-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    if-eqz v0, :cond_0

    .line 79
    invoke-virtual {v0}, Lcom/winlator/xconnector/XConnectorEpoll;->stop()V

    .line 80
    const/4 v0, 0x0

    iput-object v0, p0, Lcom/winlator/xenvironment/components/VortekRendererComponent;->connector:Lcom/winlator/xconnector/XConnectorEpoll;

    .line 82
    :cond_0
    return-void
.end method