commit 53d39543b3cee4c17ba904b72fe1a74b6cbfad5c
parent a7df5a07b57d08fb4bf8a2b34a88030bd14382fb
Author: Jack Mordaunt <jackmordaunt.dev@gmail.com>
Date: Wed, 16 Sep 2026 21:30:06 -0400
shell-extension: recover from panics in COM callbacks
A Go panic escaping a syscall.NewCallback trampoline unwinds through C frames
and kills the host, which in per-user mode is explorer.exe. Malformed files or
GDI failures must instead surface to the shell as E_FAIL, so every entry point
and method body now runs under a recovering guard.
Diffstat:
3 files changed, 54 insertions(+), 29 deletions(-)
diff --git a/cmd/shell-extension/internal/com/factory.go b/cmd/shell-extension/internal/com/factory.go
@@ -3,7 +3,9 @@
package com
import (
+ "log/slog"
"runtime"
+ "runtime/debug"
"syscall"
"unsafe"
)
@@ -55,12 +57,27 @@ func (f *ClassFactory) CreateInstance(outer unsafe.Pointer, riid *GUID, ppv *uns
return f.create(riid, ppv)
}
+// Guard runs a COM method body and converts a panic into E_FAIL.
+//
+// A panic that unwinds out of a syscall.NewCallback trampoline crosses C
+// frames and takes the host process down with it, which in the shell's case
+// means Explorer. COM callers expect failures as HRESULTs, so report it as one.
+func Guard(method string, body func() HRESULT) (hr HRESULT) {
+ defer func() {
+ if p := recover(); p != nil {
+ slog.Error("panic in COM method", "method", method, "panic", p, "stack", string(debug.Stack()))
+ hr = E_FAIL
+ }
+ }()
+ return body()
+}
+
// classFactoryVtbl is shared by all factories. The trampolines recover the
// concrete factory from the `this` pointer COM passes back to us.
var classFactoryVtbl = &IClassFactoryVtbl{
IUnknownVtbl: IUnknownVtbl{
QueryInterface: syscall.NewCallback(func(this *ClassFactory, riid *GUID, ppv *unsafe.Pointer) uintptr {
- return this.QueryInterface(riid, ppv)
+ return Guard("IClassFactory::QueryInterface", func() HRESULT { return this.QueryInterface(riid, ppv) })
}),
AddRef: syscall.NewCallback(func(this *ClassFactory) uintptr {
return 1
@@ -70,7 +87,7 @@ var classFactoryVtbl = &IClassFactoryVtbl{
}),
},
CreateInstance: syscall.NewCallback(func(this *ClassFactory, outer unsafe.Pointer, riid *GUID, ppv *unsafe.Pointer) uintptr {
- return this.CreateInstance(outer, riid, ppv)
+ return Guard("IClassFactory::CreateInstance", func() HRESULT { return this.CreateInstance(outer, riid, ppv) })
}),
LockServer: syscall.NewCallback(func(this *ClassFactory, lock uintptr) uintptr {
// The Go runtime can never be unloaded from a host process (see
diff --git a/cmd/shell-extension/internal/provider/provider.go b/cmd/shell-extension/internal/provider/provider.go
@@ -230,39 +230,41 @@ func fromThumb(this unsafe.Pointer) *Provider {
return (*Provider)(unsafe.Add(this, -int(unsafe.Offsetof(Provider{}.thumbVtbl))))
}
-// Shared vtables. Every trampoline must return a single uintptr.
+// Shared vtables. Every trampoline must return a single uintptr, and every
+// body runs under com.Guard so a panic (a malformed file, a GDI failure)
+// surfaces as E_FAIL instead of unwinding into the host process.
var (
initVtbl = &initializeWithStreamVtbl{
IUnknownVtbl: com.IUnknownVtbl{
QueryInterface: syscall.NewCallback(func(this *Provider, riid *com.GUID, ppv *unsafe.Pointer) uintptr {
- return this.QueryInterface(riid, ppv)
+ return com.Guard("IInitializeWithStream::QueryInterface", func() com.HRESULT { return this.QueryInterface(riid, ppv) })
}),
AddRef: syscall.NewCallback(func(this *Provider) uintptr {
- return uintptr(this.AddRef())
+ return com.Guard("IInitializeWithStream::AddRef", func() com.HRESULT { return uintptr(this.AddRef()) })
}),
Release: syscall.NewCallback(func(this *Provider) uintptr {
- return uintptr(this.Release())
+ return com.Guard("IInitializeWithStream::Release", func() com.HRESULT { return uintptr(this.Release()) })
}),
},
Initialize: syscall.NewCallback(func(this *Provider, stream *com.IStream, grfMode uint32) uintptr {
- return this.Initialize(stream, grfMode)
+ return com.Guard("IInitializeWithStream::Initialize", func() com.HRESULT { return this.Initialize(stream, grfMode) })
}),
}
thumbVtbl = &thumbnailProviderVtbl{
IUnknownVtbl: com.IUnknownVtbl{
QueryInterface: syscall.NewCallback(func(this unsafe.Pointer, riid *com.GUID, ppv *unsafe.Pointer) uintptr {
- return fromThumb(this).QueryInterface(riid, ppv)
+ return com.Guard("IThumbnailProvider::QueryInterface", func() com.HRESULT { return fromThumb(this).QueryInterface(riid, ppv) })
}),
AddRef: syscall.NewCallback(func(this unsafe.Pointer) uintptr {
- return uintptr(fromThumb(this).AddRef())
+ return com.Guard("IThumbnailProvider::AddRef", func() com.HRESULT { return uintptr(fromThumb(this).AddRef()) })
}),
Release: syscall.NewCallback(func(this unsafe.Pointer) uintptr {
- return uintptr(fromThumb(this).Release())
+ return com.Guard("IThumbnailProvider::Release", func() com.HRESULT { return uintptr(fromThumb(this).Release()) })
}),
},
GetThumbnail: syscall.NewCallback(func(this unsafe.Pointer, cx uint32, phbmp *windows.Handle, pdwAlpha *uint32) uintptr {
- return fromThumb(this).GetThumbnail(cx, phbmp, pdwAlpha)
+ return com.Guard("IThumbnailProvider::GetThumbnail", func() com.HRESULT { return fromThumb(this).GetThumbnail(cx, phbmp, pdwAlpha) })
}),
}
)
diff --git a/cmd/shell-extension/main.go b/cmd/shell-extension/main.go
@@ -35,14 +35,16 @@ var factory = com.NewClassFactory(provider.Create)
//
//export DllGetClassObject
func DllGetClassObject(rclsid, riid unsafe.Pointer, ppv *unsafe.Pointer) uint32 {
- if ppv == nil {
- return uint32(com.E_POINTER)
- }
- *ppv = nil
- if !com.IsEqualGUID((*com.GUID)(rclsid), provider.CLSID) {
- return uint32(com.CLASS_E_CLASSNOTAVAILABLE)
- }
- return uint32(factory.QueryInterface((*com.GUID)(riid), ppv))
+ return uint32(com.Guard("DllGetClassObject", func() com.HRESULT {
+ if ppv == nil {
+ return com.E_POINTER
+ }
+ *ppv = nil
+ if !com.IsEqualGUID((*com.GUID)(rclsid), provider.CLSID) {
+ return com.CLASS_E_CLASSNOTAVAILABLE
+ }
+ return factory.QueryInterface((*com.GUID)(riid), ppv)
+ }))
}
// DllCanUnloadNow always refuses: a Go runtime cannot be torn down and
@@ -59,22 +61,26 @@ func DllCanUnloadNow() uint32 {
//
//export DllRegisterServer
func DllRegisterServer() uint32 {
- if err := register(); err != nil {
- logError("registering", err)
- return uint32(com.E_FAIL)
- }
- return uint32(com.S_OK)
+ return uint32(com.Guard("DllRegisterServer", func() com.HRESULT {
+ if err := register(); err != nil {
+ logError("registering", err)
+ return com.E_FAIL
+ }
+ return com.S_OK
+ }))
}
// DllUnregisterServer removes the current user's registration.
//
//export DllUnregisterServer
func DllUnregisterServer() uint32 {
- if err := unregister(); err != nil {
- logError("unregistering", err)
- return uint32(com.E_FAIL)
- }
- return uint32(com.S_OK)
+ return uint32(com.Guard("DllUnregisterServer", func() com.HRESULT {
+ if err := unregister(); err != nil {
+ logError("unregistering", err)
+ return com.E_FAIL
+ }
+ return com.S_OK
+ }))
}
// DllInstall supports `regsvr32 /n /i[:cmdline]`. Registration is always
