I'm developing a custom Vulkan renderer and want to integrate a QML-based UI into it.
I already have a working Vulkan setup and also managed to render QML over Vulkan using a separate QQuickWindow and QQuickRenderControl, but this approach isn't ideal - the QML elements live in a distinct window, so they have their own focus and input handling. I can use QCoreApplication::sendEvent to send events, which works for Buttons, but doesn't work for TextField, because for TextFields the underlying window should be activated, but as I said currently my UI elements have distinct QQuickWindows and I don't want to activate them, so that my QWindow I use to render Vulkan onto won't lose focus.
What I want instead, is for QML elements to share focus and input events with my in-game window (i.e. no separate OS-level window, a single rendering surface that draws both game content and QML UI).
Is there a way to embed QML rendering directly into an existing Vulkan surface or swapchain image, so that both systems share focus and input?
Any examples or recommended architecture?
Current setup:
UserInterfaceElement.h
struct UserInterfaceElement
{
std::unique_ptr<UserInterfaceRenderer> renderer;
std::unique_ptr<UiCache> cache;
Texture texture;
Model model;
QString qmlPath;
};
UserInterfaceRenderer.h
class UserInterfaceRenderer : public QObject
{
Q_OBJECT
public:
UserInterfaceRenderer(QWindow* parentWindow = nullptr);
~UserInterfaceRenderer();
void loadQml(const QSize& size, const QString& qmlPath);
void render();
void resize(const QSize& size);
void createFbo(const QSize& size);
void deleteFbo();
QImage grabImage(); // taking screenshot
QOpenGLFramebufferObject* getFbo() const { return fbo; }
QQuickWindow* getQuickWindow() const { return quickWindow; }
void forwardEvent(QEvent* event);
QQuickItem* getRootItem() const { return rootItem; }
private:
QQuickRenderControl* renderControl = nullptr;
QQuickWindow* quickWindow = nullptr;
QQmlEngine* engine = nullptr;
QQmlComponent* component = nullptr;
QQuickItem* rootItem = nullptr;
QOpenGLFramebufferObject* fbo = nullptr;
QOpenGLContext* context = nullptr;
QOffscreenSurface* offscreenSurface = nullptr;
QSize surfaceSize;
};
UserInterfaceRenderer.cpp
UserInterfaceRenderer::UserInterfaceRenderer(QWindow* parentWindow) {
renderControl = new QQuickRenderControl(this);
quickWindow = new QQuickWindow(renderControl);
quickWindow->setGraphicsApi(QSGRendererInterface::OpenGL);
quickWindow->setFlags(
Qt::FramelessWindowHint | Qt::Tool |
Qt::WindowStaysOnTopHint | Qt::WindowTransparentForInput
);
quickWindow->setColor(Qt::transparent);
context = new QOpenGLContext(parentWindow);
context->setFormat(parentWindow->format());
context->create();
offscreenSurface = new QOffscreenSurface();
offscreenSurface->setFormat(context->format());
offscreenSurface->create();
context->makeCurrent(offscreenSurface);
quickWindow->setGraphicsDevice(QQuickGraphicsDevice::fromOpenGLContext(context));
renderControl->initialize();
quickWindow->create();
//quickWindow->setTransientParent(parentWindow);
//quickWindow->setParent(parentWindow);
engine = new QQmlEngine(this);
if (!engine->incubationController()) {
engine->setIncubationController(quickWindow->incubationController());
}
}
UserInterfaceRenderer::~UserInterfaceRenderer() {
if (rootItem) {
rootItem->setParentItem(nullptr);
delete rootItem;
}
delete renderControl;
delete quickWindow;
delete engine;
delete component;
deleteFbo();
if (context) {
context->doneCurrent();
delete context;
}
if (offscreenSurface) {
offscreenSurface->destroy();
delete offscreenSurface;
}
}
void UserInterfaceRenderer::forwardEvent(QEvent* event) {
QCoreApplication::sendEvent(quickWindow, event);
}
void UserInterfaceRenderer::loadQml(const QSize& size, const QString& qmlPath) {
component = new QQmlComponent(engine, QUrl::fromLocalFile(qmlPath));
rootItem = qobject_cast<QQuickItem*>(component->create());
if (!rootItem) {
qWarning() << "[UserInterfaceRenderer] Failed to load QML root item." << qmlPath;
return;
}
rootItem->setParentItem(quickWindow->contentItem());
rootItem->setSize(size);
quickWindow->resize(size);
surfaceSize = size;
deleteFbo();
createFbo(size);
}
void UserInterfaceRenderer::resize(const QSize& size) {
if (!rootItem) return;
rootItem->setSize(size);
quickWindow->resize(size);
surfaceSize = size;
deleteFbo();
createFbo(size);
}
void UserInterfaceRenderer::createFbo(const QSize& size) {
context->makeCurrent(offscreenSurface);
QOpenGLFramebufferObjectFormat format;
format.setAttachment(QOpenGLFramebufferObject::CombinedDepthStencil);
format.setTextureTarget(GL_TEXTURE_2D);
format.setInternalTextureFormat(GL_RGBA8);
fbo = new QOpenGLFramebufferObject(size, format);
QQuickRenderTarget renderTarget = QQuickRenderTarget::fromOpenGLTexture(
fbo->texture(),
surfaceSize
);
quickWindow->setRenderTarget(renderTarget);
}
void UserInterfaceRenderer::deleteFbo()
{
if (fbo) {
context->makeCurrent(offscreenSurface);
delete fbo;
fbo = nullptr;
}
}
void UserInterfaceRenderer::render() {
if (!rootItem || !quickWindow || !fbo) return;
if (!context->makeCurrent(offscreenSurface)) {
qWarning() << "Failed to make OpenGL context current!";
return;
}
QOpenGLFunctions* f = context->functions();
f->glViewport(0, 0, surfaceSize.width(), surfaceSize.height());
f->glClearColor(0, 0, 0, 0); // transparent clear
f->glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
renderControl->beginFrame();
renderControl->polishItems();
renderControl->sync();
renderControl->render();
renderControl->endFrame();
f->glFlush();
}
Model.h
struct Texture {
VkImage image = VK_NULL_HANDLE;
VmaAllocation vmaAllocation = VK_NULL_HANDLE;
VkImageView imageView = VK_NULL_HANDLE;
VkSampler sampler = VK_NULL_HANDLE;
uint32_t mipLevels = 0;
uint32_t width = 0;
uint32_t height = 0;
};
struct Material {
Texture diffuseTexture; // basic color
Texture normalTexture;
Texture specularTexture;
Texture emissiveTexture;
};
struct Mesh
{
std::vector<Vertex> vertices;
std::vector<uint32_t> indices;
glm::mat4 transform = glm::mat4(1.0f);
Material material;
VkBuffer vertexBuffer = VK_NULL_HANDLE;
VmaAllocation vertexBufferAllocation = VK_NULL_HANDLE;
VkBuffer indexBuffer = VK_NULL_HANDLE;
VmaAllocation indexBufferAllocation = VK_NULL_HANDLE;
std::vector<VkDescriptorSet> descriptorSets = std::vector<VkDescriptorSet>(MAX_FRAMES_IN_FLIGHT, VK_NULL_HANDLE);
};
struct Model {
std::vector<Mesh> meshes;
ModelType type = ModelType::OTHER;
glm::vec3 position;
glm::vec3 scale;
glm::quat rotation;
bool isCollidable = false;
};
called in recordCommandBuffer function, which is called each frame for each UI element
void AetherEngine::recordUiElementToCommandBuffer(UserInterfaceElement& uiElement, VkCommandBuffer commandBuffer)
{
renderQmlToTexture(uiElement.renderer.get(), uiElement.texture);
vkCmdBindPipeline(commandBuffer, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelines["ui"]);
recordModelToCommandBuffer(uiElement.model, commandBuffer);
}
void AetherEngine::renderQmlToTexture(UserInterfaceRenderer* renderer, Texture& texture)
{
renderer->render();
QImage image = renderer->getFbo()->toImage().convertToFormat(QImage::Format_RGBA8888);
if (texture.width != static_cast<uint32_t>(image.width()) ||
texture.height != static_cast<uint32_t>(image.height())) {
cleanupTexture(texture);
modelManager.createSolidColorTexture({ 0, 0, 0, 0 }, image.width(), image.height(), texture);
}
modelManager.uploadRawDataToTexture(image.bits(), image.width(), image.height(), texture);
}
