Desktop: Implement GPU accelerated offscreen rendering and improve rendering efficency (#3056)

* WIP accelerade offscreen canvas implementation

* Implement vulkan dmabuf import

* Add fps printing

* Add feature gates

* Forgot to add file

* Experimental windows support

* Cast ptr to isize

* Remove testing chrome://flags url

* Experimental macos support for texture import

* Cleanup code and improve latency / frame pacing

* Add path for importing textures on windows through dx12

* Update doc comment

* Import textures through metal on macos

* Review cleanup

---------

Co-authored-by: Timon Schelling <me@timon.zip>

Author: 2 people

Cargo.lock

@@ -9,9 +9,15 @@ edition = "2024"
 rust-version = "1.87"
 
 [features]
-default = ["gpu"]
+default = ["gpu", "accelerated_paint"]
 gpu = ["graphite-desktop-wrapper/gpu"]
 
+# Hardware acceleration features
+accelerated_paint = ["accelerated_paint_dmabuf", "accelerated_paint_d3d11", "accelerated_paint_iosurface"]
+accelerated_paint_dmabuf = ["libc", "ash"]
+accelerated_paint_d3d11 = ["windows", "ash"]
+accelerated_paint_iosurface = ["objc2-io-surface", "objc2-metal", "core-foundation"]
+
 [dependencies]
 # # Local dependencies
 graphite-desktop-wrapper = { path = "wrapper" }
@@ -32,3 +38,26 @@ vello = { workspace = true }
 derivative = { workspace = true }
 rfd = { workspace = true }
 open = { workspace = true }
+
+# Hardware acceleration dependencies
+ash = { version = "0.38", optional = true }
+
+# Windows-specific dependencies
+[target.'cfg(windows)'.dependencies]
+windows = { version = "0.58", features = [
+	"Win32_Graphics_Direct3D11",
+	"Win32_Graphics_Direct3D12",
+	"Win32_Graphics_Dxgi",
+	"Win32_Graphics_Dxgi_Common",
+	"Win32_Foundation"
+], optional = true }
+
+# macOS-specific dependencies
+[target.'cfg(target_os = "macos")'.dependencies]
+objc2-io-surface = { version = "0.3", optional = true }
+objc2-metal = { version = "0.3", optional = true }
+core-foundation = { version = "0.9", optional = true }
+
+# Linux-specific dependencies
+[target.'cfg(target_os = "linux")'.dependencies]
+libc = { version = "0.2", optional = true }

desktop/Cargo.toml

@@ -1,18 +1,19 @@
 use crate::CustomEvent;
 use crate::cef::WindowSize;
-use crate::consts::APP_NAME;
+use crate::consts::{APP_NAME, CEF_MESSAGE_LOOP_MAX_ITERATIONS};
 use crate::render::GraphicsState;
 use graphite_desktop_wrapper::messages::{DesktopFrontendMessage, DesktopWrapperMessage};
 use graphite_desktop_wrapper::{DesktopWrapper, NodeGraphExecutionResult, WgpuContext, serialize_frontend_messages};
+
 use rfd::AsyncFileDialog;
 use std::sync::Arc;
 use std::sync::mpsc::Sender;
+use std::sync::mpsc::SyncSender;
 use std::thread;
 use std::time::Duration;
 use std::time::Instant;
 use winit::application::ApplicationHandler;
 use winit::dpi::PhysicalSize;
-use winit::event::StartCause;
 use winit::event::WindowEvent;
 use winit::event_loop::ActiveEventLoop;
 use winit::event_loop::ControlFlow;
@@ -31,11 +32,23 @@ pub(crate) struct WinitApp {
 	wgpu_context: WgpuContext,
 	event_loop_proxy: EventLoopProxy<CustomEvent>,
 	desktop_wrapper: DesktopWrapper,
+	last_ui_update: Instant,
+	avg_frame_time: f32,
+	start_render_sender: SyncSender<()>,
 }
 
 impl WinitApp {
 	pub(crate) fn new(cef_context: cef::Context<cef::Initialized>, window_size_sender: Sender<WindowSize>, wgpu_context: WgpuContext, event_loop_proxy: EventLoopProxy<CustomEvent>) -> Self {
-		let desktop_wrapper = DesktopWrapper::new();
+		let rendering_loop_proxy = event_loop_proxy.clone();
+		let (start_render_sender, start_render_receiver) = std::sync::mpsc::sync_channel(1);
+		std::thread::spawn(move || {
+			loop {
+				let result = futures::executor::block_on(DesktopWrapper::execute_node_graph());
+				let _ = rendering_loop_proxy.send_event(CustomEvent::NodeGraphExecutionResult(result));
+				let _ = start_render_receiver.recv();
+			}
+		});
+
 		Self {
 			cef_context,
 			window: None,
@@ -44,7 +57,10 @@ impl WinitApp {
 			window_size_sender,
 			wgpu_context,
 			event_loop_proxy,
-			desktop_wrapper,
+			desktop_wrapper: DesktopWrapper::new(),
+			last_ui_update: Instant::now(),
+			avg_frame_time: 0.,
+			start_render_sender,
 		}
 	}
 
@@ -152,23 +168,20 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 		// Set a timeout in case we miss any cef schedule requests
 		let timeout = Instant::now() + Duration::from_millis(10);
 		let wait_until = timeout.min(self.cef_schedule.unwrap_or(timeout));
-		self.cef_context.work();
-
-		event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until));
-	}
-
-	fn new_events(&mut self, _event_loop: &ActiveEventLoop, cause: StartCause) {
 		if let Some(schedule) = self.cef_schedule
 			&& schedule < Instant::now()
 		{
 			self.cef_schedule = None;
-			self.cef_context.work();
-		}
-		if let StartCause::ResumeTimeReached { .. } = cause {
-			if let Some(window) = &self.window {
-				window.request_redraw();
+			// Poll cef message loop multiple times to avoid message loop starvation
+			for _ in 0..CEF_MESSAGE_LOOP_MAX_ITERATIONS {
+				self.cef_context.work();
 			}
 		}
+		if let Some(window) = &self.window.as_ref() {
+			window.request_redraw();
+		}
+
+		event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until));
 	}
 
 	fn resumed(&mut self, event_loop: &ActiveEventLoop) {
@@ -220,6 +233,11 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 				if let Some(graphics_state) = self.graphics_state.as_mut() {
 					graphics_state.resize(texture.width(), texture.height());
 					graphics_state.bind_ui_texture(texture);
+					let elapsed = self.last_ui_update.elapsed().as_secs_f32();
+					self.last_ui_update = Instant::now();
+					if elapsed < 0.5 {
+						self.avg_frame_time = (self.avg_frame_time * 3. + elapsed) / 4.;
+					}
 				}
 				if let Some(window) = &self.window {
 					window.request_redraw();
@@ -251,16 +269,18 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 			WindowEvent::RedrawRequested => {
 				let Some(ref mut graphics_state) = self.graphics_state else { return };
 				// Only rerender once we have a new ui texture to display
-
-				match graphics_state.render() {
-					Ok(_) => {}
-					Err(wgpu::SurfaceError::Lost) => {
-						tracing::warn!("lost surface");
-					}
-					Err(wgpu::SurfaceError::OutOfMemory) => {
-						event_loop.exit();
+				if let Some(window) = &self.window {
+					match graphics_state.render(window.as_ref()) {
+						Ok(_) => {}
+						Err(wgpu::SurfaceError::Lost) => {
+							tracing::warn!("lost surface");
+						}
+						Err(wgpu::SurfaceError::OutOfMemory) => {
+							event_loop.exit();
+						}
+						Err(e) => tracing::error!("{:?}", e),
 					}
-					Err(e) => tracing::error!("{:?}", e),
+					let _ = self.start_render_sender.try_send(());
 				}
 			}
 			// Currently not supported on wayland see https://github.com/rust-windowing/winit/issues/1881

desktop/src/app.rs

@@ -1,3 +1,18 @@
+//! CEF (Chromium Embedded Framework) integration for Graphite Desktop
+//!
+//! This module provides CEF browser integration with hardware-accelerated texture sharing.
+//!
+//! # Hardware Acceleration
+//!
+//! The texture import system supports platform-specific hardware acceleration:
+//!
+//! - **Linux**: DMA-BUF via Vulkan external memory (`accelerated_paint_dmabuf` feature)
+//! - **Windows**: D3D11 shared textures via either Vulkan or D3D12 interop (`accelerated_paint_d3d11` feature)
+//! - **macOS**: IOSurface via Metal/Vulkan interop (`accelerated_paint_iosurface` feature)
+//!
+//!
+//! The system gracefully falls back to CPU textures when hardware acceleration is unavailable.
+
 use crate::CustomEvent;
 use crate::render::FrameBufferRef;
 use graphite_desktop_wrapper::{WgpuContext, deserialize_editor_message};
@@ -10,15 +25,23 @@ mod dirs;
 mod input;
 mod internal;
 mod ipc;
+mod platform;
 mod scheme_handler;
 mod utility;
 
+#[cfg(feature = "accelerated_paint")]
+mod texture_import;
+#[cfg(feature = "accelerated_paint")]
+use texture_import::SharedTextureHandle;
+
 pub(crate) use context::{Context, InitError, Initialized, Setup, SetupError};
 use winit::event_loop::EventLoopProxy;
 
 pub(crate) trait CefEventHandler: Clone {
 	fn window_size(&self) -> WindowSize;
 	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>);
+	#[cfg(feature = "accelerated_paint")]
+	fn draw_gpu(&self, shared_texture: SharedTextureHandle);
 	/// Scheudule the main event loop to run the cef event loop after the timeout
 	///  [`_cef_browser_process_handler_t::on_schedule_message_pump_work`] for more documentation.
 	fn schedule_cef_message_loop_work(&self, scheduled_time: Instant);
@@ -128,4 +151,16 @@ impl CefEventHandler for CefHandler {
 		};
 		let _ = self.event_loop_proxy.send_event(CustomEvent::DesktopWrapperMessage(desktop_wrapper_message));
 	}
+
+	#[cfg(feature = "accelerated_paint")]
+	fn draw_gpu(&self, shared_texture: SharedTextureHandle) {
+		match shared_texture.import_texture(&self.wgpu_context.device) {
+			Ok(texture) => {
+				let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(texture));
+			}
+			Err(e) => {
+				tracing::error!("Failed to import shared texture: {}", e);
+			}
+		}
+	}
 }

desktop/src/cef.rs

@@ -86,14 +86,17 @@ impl Context<Setup> {
 		let mut client = Client::new(BrowserProcessClientImpl::new(render_handler, event_handler.clone()));
 
 		let url = CefString::from(format!("{GRAPHITE_SCHEME}://{FRONTEND_DOMAIN}/").as_str());
+		// let url = CefString::from("chrome://gpu");
 
 		let window_info = WindowInfo {
 			windowless_rendering_enabled: 1,
+			#[cfg(feature = "accelerated_paint")]
+			shared_texture_enabled: if crate::cef::platform::should_enable_hardware_acceleration() { 1 } else { 0 },
 			..Default::default()
 		};
 
 		let settings = BrowserSettings {
-			windowless_frame_rate: 60,
+			windowless_frame_rate: crate::consts::CEF_WINDOWLESS_FRAME_RATE,
 			background_color: 0x0,
 			..Default::default()
 		};

desktop/src/cef/context.rs

@@ -2,7 +2,7 @@ mod browser_process_app;
 mod browser_process_client;
 mod browser_process_handler;
 mod browser_process_life_span_handler;
-mod render_handler;
+pub mod render_handler;
 mod render_process_app;
 mod render_process_handler;
 mod render_process_v8_handler;

desktop/src/cef/internal.rs

@@ -34,12 +34,28 @@ impl<H: CefEventHandler + Clone> ImplApp for BrowserProcessAppImpl<H> {
 
 	fn on_before_command_line_processing(&self, _process_type: Option<&cef::CefString>, command_line: Option<&mut cef::CommandLine>) {
 		if let Some(cmd) = command_line {
-			// Disable GPU acceleration, because it is not supported for Offscreen Rendering and can cause crashes.
-			cmd.append_switch(Some(&CefString::from("disable-gpu")));
-			cmd.append_switch(Some(&CefString::from("disable-gpu-compositing")));
+			#[cfg(not(feature = "accelerated_paint"))]
+			{
+				// Disable GPU acceleration when accelerated_paint feature is not enabled
+				cmd.append_switch(Some(&CefString::from("disable-gpu")));
+				cmd.append_switch(Some(&CefString::from("disable-gpu-compositing")));
+			}
+
+			#[cfg(feature = "accelerated_paint")]
+			{
+				// Enable GPU acceleration switches for better performance
+				cmd.append_switch(Some(&CefString::from("enable-gpu-rasterization")));
+				cmd.append_switch(Some(&CefString::from("enable-accelerated-2d-canvas")));
+			}
+
+			#[cfg(all(feature = "accelerated_paint", target_os = "linux"))]
+			{
+				// Use Vulkan for accelerated painting
+				cmd.append_switch_with_value(Some(&CefString::from("use-angle")), Some(&CefString::from("vulkan")));
+			}
 
 			// Tell CEF to use Wayland if available
-			#[cfg(not(any(target_os = "macos", target_os = "windows")))]
+			#[cfg(target_os = "linux")]
 			{
 				let use_wayland = env::var("WAYLAND_DISPLAY")
 					.ok()