Skip to content

Commit b2a400b

Browse files
iamnbutleriamnbutler
authored
feat(desktop): open one item through the native app handler (#143)
1 parent b9afde3 commit b2a400b

9 files changed

Lines changed: 217 additions & 58 deletions

File tree

‎apps/desktop/native/patches/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,14 @@ again immediately before mutation. Positional AXPress waits for its actual retur
124124
operation lane through cancellation; an uncertain return remains unsafe to retry. No input runs in
125125
the detached read, no failed action falls back to another route, and no caret position is promised.
126126

127+
`peekaboo-open.patch` distinguishes accepted document/URL delivery from verified application launch
128+
for [native computer use](https://github.com/githubnext/ace2/issues/8). The existing launch service
129+
returns `dispatched_unverified` with `delivery_accepted` when its request contains items to open.
130+
The original native delivery, accepted unit count, global lane ownership and signed process target
131+
are retained. Readiness and activation do not prove that the app loaded the item. Launches without
132+
items and all refusal/uncertainty paths are unchanged; the existing bridge contract already permits
133+
this accepted outcome.
134+
127135
The desktop build resolves only `Package.resolved` versions, checks the pin and checkout revision,
128136
and assembles the ordered patch stack in a private Git index. A build compares the checkout
129137
with each complete ordered prefix, since later patches can change earlier patch contexts.

‎apps/desktop/native/patches/peekaboo-open.patch‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
diff --git a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift
2+
--- a/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift
3+
+++ b/Core/PeekabooAutomationKit/Sources/PeekabooAutomationKit/Services/System/ApplicationService+Lifecycle.swift
4+
@@ -174,6 +174,15 @@
5+
"Launched application process generation changed before its receipt could be returned")
6+
}
7+
let boundApplication = try await self.bindSelectorResolution(application, launch: launch)
8+
+ if !launch.openURLs.isEmpty {
9+
+ // LaunchServices acceptance does not establish the receiving app's document or URL effect.
10+
+ return DesktopActionResult(
11+
+ payload: boundApplication,
12+
+ outcome: .dispatchedUnverified(
13+
+ delivery: Self.applicationDelivery(mode: .foreground),
14+
+ evidence: .deliveryAccepted,
15+
+ unitCount: DesktopActionOutcome.DispatchUnitCount(acceptedNativeDispatchCount) ?? .one))
16+
+ }
17+
return DesktopActionResult(
18+
payload: boundApplication,
19+
outcome: .confirmedChange(

‎apps/desktop/native/sources/client/client.swift‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ private enum Client {
2929
|| (operation == "action" && args.count == 2)
3030
|| (operation == "management" && args.count == 2)
3131
|| (operation == "launch" && args.count == 2)
32+
|| (operation == "open" && args.count == 2)
3233
|| (operation == "clipboard" && args.count == 2)
3334
|| (operation == "menus" && args.count == 2)
3435
else { throw ClientError.usage }
@@ -59,6 +60,8 @@ private enum Client {
5960
return try await nativeManagement(client, handshake: handshake)
6061
case "launch":
6162
return try await nativeLaunch(client, handshake: handshake)
63+
case "open":
64+
return try await nativeLaunch(client, handshake: handshake, opensItem: true)
6265
case "apps":
6366
let inventory = try await client.listApplicationMutationInventory()
6467
var metadata: [ServiceApplicationInfo] = []

‎apps/desktop/native/sources/client/launch.swift‎

Lines changed: 68 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ private struct LaunchMessage: Encodable {
2424
}
2525

2626
private struct LaunchResult: Encodable {
27-
let action = "launch"
27+
let action: String
2828
var outcome = "refused"
2929
var native_outcome: DesktopActionOutcome?
3030
var application: LaunchedApplication?
3131
var requires_fresh_observation = false
32+
var message: String?
3233
var error: LaunchMessage?
3334
}
3435

@@ -38,25 +39,27 @@ private struct LaunchReply: Encodable {
3839
let target_receipt: Receipt?
3940
}
4041

41-
func nativeLaunch(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHandshakeResponse) async throws -> Data {
42-
var result = LaunchResult()
42+
func nativeLaunch(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHandshakeResponse, opensItem: Bool = false) async throws -> Data {
43+
var result = LaunchResult(action: opensItem ? "open" : "launch")
4344
var receipt: Receipt?
4445
var invoked = false
45-
let uncertain = "The app may still open later. Observe desktop_apps before any further action; do not blindly repeat the launch."
46+
let uncertain = opensItem
47+
? "The item may still open later. Observe desktop_apps before any further action; do not blindly repeat the open."
48+
: "The app may still open later. Observe desktop_apps before any further action; do not blindly repeat the launch."
4649
do {
47-
let request = try readLaunch()
50+
let request = try readLaunch(opensItem: opensItem)
4851
guard handshake.supportedOperations.contains(.launchApplicationWithOptions),
4952
(handshake.enabledOperations ?? handshake.supportedOperations).contains(.launchApplicationWithOptions)
5053
else { throw LaunchError("The desktop runtime does not support attested application launch. Update the runtime before acting.") }
5154
let session = CGSessionCopyCurrentDictionary() as NSDictionary?
5255
if session?["CGSSessionScreenIsLocked"] as? Bool == true {
5356
throw DesktopActionFailure.preDispatchRefusal(
5457
reason: .targetUnavailable,
55-
message: "The macOS GUI session is locked. Application launch was not dispatched.",
58+
message: "The macOS GUI session is locked. The opening operation was not dispatched.",
5659
hint: "Unlock the active user session before choosing a new action."
5760
)
5861
}
59-
// Verify signed operation support before a launch whose process does not exist yet.
62+
// The receiving process may not exist until LaunchServices returns it.
6063
_ = try await client.listApplicationMutationInventory()
6164
guard let preflight = await client.lastOperationReceipt(), preflight.payload.operation == PeekabooBridgeRequest.listApplicationMutationInventory.operation else {
6265
throw LaunchError("The native runtime cannot attest application targets. Update the desktop runtime before acting.")
@@ -75,10 +78,18 @@ func nativeLaunch(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHands
7578
result.application = LaunchedApplication(name: action.payload.name, bundle_id: action.payload.bundleIdentifier,
7679
path: action.payload.bundlePath, target: ManagementTarget(process))
7780
result.requires_fresh_observation = outcome.dispatchState.mutationDispatched
78-
switch outcome.state {
79-
case .confirmedChange, .confirmedNoChange: result.outcome = "completed"
80-
case .refused: result.outcome = "refused"
81-
default: result.outcome = "unknown"
81+
if opensItem {
82+
guard outcome.state == .dispatchedUnverified, outcome.evidence == .deliveryAccepted else {
83+
throw LaunchError("The native runtime did not distinguish accepted item delivery from its effect. Inspect the receiving app before acting again.")
84+
}
85+
result.outcome = "completed"
86+
result.message = "macOS accepted the item opening request. Its effect is unverified; inspect the receiving app before any further action."
87+
} else {
88+
switch outcome.state {
89+
case .confirmedChange, .confirmedNoChange: result.outcome = "completed"
90+
case .refused: result.outcome = "refused"
91+
default: result.outcome = "unknown"
92+
}
8293
}
8394
} catch let failure as DesktopActionFailure {
8495
let dispatched = failure.outcome.dispatchState.mutationDispatched
@@ -96,29 +107,57 @@ func nativeLaunch(_ client: PeekabooBridgeClient, handshake: PeekabooBridgeHands
96107
return try JSONEncoder().encode(LaunchReply(data: result, target_receipt: receipt))
97108
}
98109

99-
private func readLaunch() throws -> ApplicationLaunchRequest {
110+
private func readLaunch(opensItem: Bool) throws -> ApplicationLaunchRequest {
100111
var data = Data()
101112
while let chunk = try FileHandle.standardInput.read(upToCount: 4096), !chunk.isEmpty {
102113
data.append(chunk)
103-
guard data.count <= 16_384 else { throw LaunchError("The launch request exceeds 16 KiB.") }
114+
guard data.count <= 16_384 else { throw LaunchError("The opening request exceeds 16 KiB.") }
104115
}
116+
let operation = opensItem ? "open" : "launch"
117+
let keys: Set<String> = opensItem ? ["op", "item", "application"] : ["op", "application"]
105118
guard let request = try JSONSerialization.jsonObject(with: data) as? [String: Any],
106-
Set(request.keys) == ["op", "application"], request["op"] as? String == "launch",
107-
let application = request["application"] as? [String: Any], application.count == 1
108-
else { throw LaunchError("Launch accepts exactly one application path or bundle_id.") }
109-
let path = application["path"] as? String
110-
let bundleID = application["bundle_id"] as? String
111-
if let path {
112-
var directory: ObjCBool = false
113-
guard path.utf16.count <= 4096, path.hasPrefix("/"), path.lowercased().hasSuffix(".app"), !path.contains("\0"),
114-
FileManager.default.fileExists(atPath: path, isDirectory: &directory), directory.boolValue,
115-
Bundle(url: URL(fileURLWithPath: path))?.executableURL != nil
116-
else { throw LaunchError("Use an existing absolute .app path for launch.") }
117-
} else {
118-
guard let bundleID, bundleID.utf16.count <= 256,
119-
bundleID.range(of: "^[A-Za-z0-9.-]+$", options: .regularExpression) != nil
120-
else { throw LaunchError("Use an exact application bundle ID for launch.") }
119+
Set(request.keys).isSubset(of: keys), request["op"] as? String == operation
120+
else { throw LaunchError("Use one item and optional application for open, or one application for launch.") }
121+
var path: String?
122+
var bundleID: String?
123+
if !opensItem || request["application"] != nil {
124+
guard let application = request["application"] as? [String: Any], application.count == 1 else {
125+
throw LaunchError("Use exactly one application path or bundle_id.")
126+
}
127+
path = application["path"] as? String
128+
bundleID = application["bundle_id"] as? String
129+
if let path {
130+
var directory: ObjCBool = false
131+
guard path.utf16.count <= 4096, path.hasPrefix("/"), path.lowercased().hasSuffix(".app"), !path.contains("\0"),
132+
FileManager.default.fileExists(atPath: path, isDirectory: &directory), directory.boolValue,
133+
Bundle(url: URL(fileURLWithPath: path))?.executableURL != nil
134+
else { throw LaunchError("Use an existing absolute .app path for launch.") }
135+
} else {
136+
guard let bundleID, bundleID.utf16.count <= 256,
137+
bundleID.range(of: "^[A-Za-z0-9.-]+$", options: .regularExpression) != nil
138+
else { throw LaunchError("Use an exact application bundle ID for launch.") }
139+
}
140+
}
141+
var urls: [URL] = []
142+
if opensItem {
143+
guard let item = request["item"] as? [String: Any], item.count == 1 else {
144+
throw LaunchError("Open accepts exactly one item path or url.")
145+
}
146+
if let path = item["path"] as? String {
147+
guard path.utf16.count <= 4096, path.hasPrefix("/"), !path.contains("\0"),
148+
FileManager.default.fileExists(atPath: path)
149+
else { throw LaunchError("Use an existing absolute item path.") }
150+
urls = [URL(fileURLWithPath: path)]
151+
} else {
152+
guard let raw = item["url"] as? String, raw.utf16.count <= 4096,
153+
!raw.unicodeScalars.contains(where: { CharacterSet.controlCharacters.contains($0) }),
154+
let url = URL(string: raw, encodingInvalidCharacters: false), let scheme = url.scheme,
155+
scheme.range(of: "^[A-Za-z][A-Za-z0-9+.-]*$", options: .regularExpression) != nil,
156+
url.absoluteString == raw
157+
else { throw LaunchError("Use a correctly encoded absolute URL with an explicit scheme and no control characters.") }
158+
urls = [url]
159+
}
121160
}
122161
return ApplicationLaunchRequest(applicationIdentifier: path, applicationBundleIdentifier: bundleID,
123-
activates: true, waitUntilReady: true, waitForWindow: false, createsNewInstance: false)
162+
openURLs: urls, activates: true, waitUntilReady: true, waitForWindow: false, createsNewInstance: false)
124163
}

‎apps/desktop/scripts/helper.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ const patches = [
161161
"peekaboo-clipboard-files.patch",
162162
"peekaboo-point-focus.patch",
163163
"peekaboo-clipboard-image-write.patch",
164+
"peekaboo-open.patch",
164165
]
165166
.map((name) => join(native, "patches", name));
166167
const temporary = mkdtempSync(join(tmpdir(), "ace-native-patches-"));

‎apps/host/src/desktop.ts‎

Lines changed: 56 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@ import {
1414
DESKTOP_MODIFIERS,
1515
DESKTOP_SELECTIONS,
1616
type DesktopAction,
17+
type DesktopApplication,
1718
type DesktopAppTarget,
1819
type DesktopLaunch,
1920
type DesktopManagement,
21+
type DesktopOpen,
2022
type DesktopOutcome,
2123
type DesktopRequest,
2224
type DesktopResult,
@@ -377,24 +379,55 @@ async function availability(
377379
}
378380
}
379381

382+
function validateApplication(application: DesktopApplication) {
383+
if (!application || typeof application !== "object" || Object.keys(application).length !== 1) {
384+
throw new Error("Use exactly one application path or bundle_id.");
385+
}
386+
if ("path" in application) {
387+
const path = application.path;
388+
if (
389+
typeof path !== "string" || path.length > 4096 || !path.startsWith("/")
390+
|| !path.toLowerCase().endsWith(".app") || path.includes("\0")
391+
) throw new Error("Use an absolute .app path for launch.");
392+
} else if (
393+
!("bundle_id" in application) || typeof application.bundle_id !== "string"
394+
|| application.bundle_id.length > 256
395+
|| !/^[A-Za-z0-9.-]+$/.test(application.bundle_id)
396+
) throw new Error("Use an exact application bundle ID for launch.");
397+
}
398+
380399
function validateAction(request: DesktopAction) {
381400
if (request.op === "launch") {
382-
const application = request.application;
401+
if (Object.keys(request).some((key) => !["op", "application"].includes(key))) {
402+
throw new Error("Launch accepts exactly one application path or bundle_id.");
403+
}
404+
validateApplication(request.application);
405+
return;
406+
}
407+
if (request.op === "open") {
408+
const item = request.item;
383409
if (
384-
Object.keys(request).some((key) => !["op", "application"].includes(key))
385-
|| !application || typeof application !== "object" || Object.keys(application).length !== 1
386-
) throw new Error("Launch accepts exactly one application path or bundle_id.");
387-
if ("path" in application) {
388-
const path = application.path;
410+
Object.keys(request).some((key) => !["op", "item", "application"].includes(key))
411+
|| !item || typeof item !== "object" || Object.keys(item).length !== 1
412+
) throw new Error("Open accepts exactly one item path or url and an optional application.");
413+
if ("path" in item) {
389414
if (
390-
typeof path !== "string" || path.length > 4096 || !path.startsWith("/")
391-
|| !path.toLowerCase().endsWith(".app") || path.includes("\0")
392-
) throw new Error("Use an absolute .app path for launch.");
415+
typeof item.path !== "string" || item.path.length > 4096
416+
|| !item.path.startsWith("/") || item.path.includes("\0")
417+
) throw new Error("Use an existing absolute item path.");
393418
} else if (
394-
!("bundle_id" in application) || typeof application.bundle_id !== "string"
395-
|| application.bundle_id.length > 256
396-
|| !/^[A-Za-z0-9.-]+$/.test(application.bundle_id)
397-
) throw new Error("Use an exact application bundle ID for launch.");
419+
!("url" in item) || typeof item.url !== "string" || item.url.length > 4096
420+
|| !/^[A-Za-z][A-Za-z0-9+.-]*:/.test(item.url)
421+
|| /\p{Cc}/u.test(item.url)
422+
) {
423+
throw new Error(
424+
"Use a complete absolute URL with an explicit scheme and no control characters.",
425+
);
426+
}
427+
if (request.application !== undefined) validateApplication(request.application);
428+
if (Buffer.byteLength(JSON.stringify(request)) > 16_384) {
429+
throw new Error("The open request exceeds 16 KiB.");
430+
}
398431
return;
399432
}
400433
if (isDesktopManagement(request)) return validateManagement(request);
@@ -621,6 +654,8 @@ async function act(request: DesktopAction, signal: AbortSignal): Promise<Desktop
621654
? "clipboard"
622655
: request.op === "launch"
623656
? "launch"
657+
: request.op === "open"
658+
? "open"
624659
: isDesktopManagement(request)
625660
? "management"
626661
: "action";
@@ -640,6 +675,8 @@ async function act(request: DesktopAction, signal: AbortSignal): Promise<Desktop
640675
reason: (error instanceof Error ? error.message : String(error)).slice(0, 2000),
641676
message: dispatched && request.op === "launch"
642677
? "The launch may still finish and open the app later. Observe desktop_apps before any further action; do not blindly repeat the launch."
678+
: dispatched && request.op === "open"
679+
? "The item may still open later. Observe desktop_apps before any further action; do not blindly repeat the open."
643680
: dispatched
644681
? "The desktop action may have partially run. Inspect the current state before retrying. Stopping does not undo input already delivered."
645682
: "The desktop action was not sent to the native desktop.",
@@ -656,7 +693,7 @@ async function act(request: DesktopAction, signal: AbortSignal): Promise<Desktop
656693
if (outcome !== "completed" || request.op === "clipboard-write") {
657694
return actionResult(data, outcome);
658695
}
659-
if (isDesktopManagement(request) || request.op === "launch") {
696+
if (isDesktopManagement(request) || request.op === "launch" || request.op === "open") {
660697
return await observeManagement(request, data, signal);
661698
}
662699
// Observation is separate from delivery: its failure must not turn completed input into a retry.
@@ -700,20 +737,21 @@ async function act(request: DesktopAction, signal: AbortSignal): Promise<Desktop
700737
}
701738

702739
async function observeManagement(
703-
request: DesktopManagement | DesktopLaunch,
740+
request: DesktopManagement | DesktopLaunch | DesktopOpen,
704741
action: Record<string, unknown>,
705742
signal: AbortSignal,
706743
): Promise<DesktopResult> {
707744
const data: Record<string, unknown> = { action };
708745
const outcome = action.outcome === "unknown" ? "unknown" : "completed";
709746
const receipt = action.target_receipt as Reply["target_receipt"];
710-
const target = request.op === "launch"
747+
const target = request.op === "launch" || request.op === "open"
711748
? (action.application as { target?: DesktopAppTarget } | undefined)?.target
712749
: request.target;
713750
if (
714751
!receipt || !target || receipt.pid !== target.pid
715752
|| receipt.process_start_identity_decimal !== target.process_start_identity_decimal
716753
|| (request.op === "activate" || request.op === "quit" || request.op === "launch"
754+
|| request.op === "open"
717755
? receipt.window_id !== undefined
718756
: receipt.window_id !== request.target.window_id)
719757
) {
@@ -755,6 +793,7 @@ async function observeManagement(
755793
}
756794
if (
757795
request.op === "activate" || request.op === "quit" || request.op === "launch"
796+
|| request.op === "open"
758797
|| request.op === "close"
759798
) {
760799
return managementResult(data, action, outcome);
@@ -838,6 +877,7 @@ export const desktop: Desktop = async (request, context) => {
838877
"drag",
839878
"activate",
840879
"launch",
880+
"open",
841881
"quit",
842882
"close",
843883
"focus",

‎docs/architecture.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,12 @@ launch/readiness through the native global mutation lane. The signed response bi
221221
to the resulting process generation; later inventory failure preserves that result. Launch exposes
222222
no document/URL, extra-instance, or relaunch options. The app may still open after caller timeout
223223
or interruption, so uncertain launch is never replayed and native ownership lasts until it settles.
224+
Explicit item opening reuses that signed native launch route for one existing absolute path or
225+
complete URL, optionally selecting an app. Otherwise macOS chooses the default handler; no app
226+
is assumed before dispatch. The signed request includes the item URL and binds the returned
227+
process generation. Accepted delivery remains unverified item effect, including after successful
228+
readiness and activation. Later app/window inventory cannot prove document loading or navigation,
229+
and failures after native submission remain uncertain without retry or dialog handling.
224230
Minimize verifies native window state and returns fresh inventory without capturing the minimized
225231
window; restoring it remains an explicit action with a refreshed target.
226232
Move and resize use native background Accessibility, verify resulting geometry, and return refreshed

0 commit comments

Comments
 (0)