Leap0 Docs

Automate a full desktop sandbox with screenshots, mouse and keyboard input, recordings, and a built-in noVNC session.

Computer use endpoints let you programmatically control a full Linux desktop running inside a sandbox.

The runtime is based on Ubuntu 22.04 with an XFCE4 desktop at 1440x900 resolution. Firefox ESR, Google Chrome, and VS Code are pre-installed.

Sandbox template requirement

Computer Use requires a sandbox created with the system/desktop:v0.1.0 template. Sandboxes created with other templates do not include the desktop environment or VNC support unless you install the required packages. See Sandboxes for how to create one.

For a full endpoint reference, see the Computer Use (Desktop) API.

Desktop access

To open the desktop in a browser, see VNC Access.

Display info

Query the current display name and resolution. Useful for calculating coordinates before sending mouse or screenshot commands.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.display_info())

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.displayInfo());
await client.close();

Resize screen

Change the virtual display resolution. Width must be between 320 and 7680, height between 320 and 4320.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.resize_screen(width=1280, height=800))

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.resizeScreen({ width: 1280, height: 800 }));
await client.close();

List windows

Inspect all open windows on the desktop. Each window includes its position, size, class, title, and whether it is focused.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
for window in sandbox.desktop.windows():
    print(f"{window.title} ({window.width}x{window.height}) focused={window.focused}")

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

const windows = await sandbox.desktop.windows();
windows.forEach((window) =>
console.log(window.title, window.width + "x" + window.height, "focused=" + window.focused),
);
await client.close();

Screenshots

Full screen

Capture the entire screen. Supports png, jpg, or jpeg format via the format query parameter. JPEG quality can be set with quality (1--100).

from pathlib import Path
from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
screenshot = sandbox.desktop.screenshot(image_format="png")
Path("screen.png").write_bytes(screenshot)

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

const bytes = await sandbox.desktop.screenshot({ format: "png" });
console.log("screenshot bytes", bytes.byteLength);
await client.close();

Region

Capture a specific region of the screen.

from pathlib import Path
from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
region = sandbox.desktop.screenshot_region(
    x=0,
    y=0,
    width=800,
    height=600,
    image_format="jpg",
    quality=85,
)
Path("region.jpg").write_bytes(region)

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

const bytes = await sandbox.desktop.screenshotRegion({
x: 0,
y: 0,
width: 800,
height: 600,
format: "jpg",
quality: 85,
});

console.log("region bytes", bytes.byteLength);
await client.close();

Mouse

Get pointer position

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.pointer_position())

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.pointerPosition());
await client.close();

Move pointer

Move the cursor to absolute coordinates. Returns the new pointer position.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.move_pointer(x=320, y=240))

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.movePointer(320, 240));
await client.close();

Click

Click at the current pointer position or at optional x, y coordinates. The button field maps to X11 buttons: 1 (left, default), 2 (middle), 3 (right). Returns the pointer position after the click.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)

sandbox.desktop.click(x=320, y=240, button=1)
sandbox.desktop.click(button=3)

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

await sandbox.desktop.click({ x: 320, y: 240, button: 1 });

await sandbox.desktop.click({ button: 3 });
await client.close();

Drag

Drag from one coordinate to another. Uses button 1 (left) by default.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(
    sandbox.desktop.drag(
        from_x=100,
        from_y=200,
        to_x=400,
        to_y=200,
    )
)

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(
await sandbox.desktop.drag({
fromX: 100,
fromY: 200,
toX: 400,
toY: 200,
}),
);
await client.close();

Scroll

Scroll at the current pointer position. Direction must be up, down, left, or right. The amount field controls how many scroll steps to perform (1--100, defaults to 1).

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
sandbox.desktop.scroll(direction="down", amount=3)

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

await sandbox.desktop.scroll({ direction: "down", amount: 3 });
await client.close();

Keyboard

Type text

Type a string of text as if entered from the keyboard. Supports up to 50,000 characters.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
sandbox.desktop.type_text(text="hello from Leap0")

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

await sandbox.desktop.typeText("hello from Leap0");
await client.close();

Press key

Press a single key by its X11 keysym name (e.g. Return, Escape, Tab, BackSpace, space).

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
sandbox.desktop.press_key(key="Return")

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

await sandbox.desktop.pressKey("Return");
await client.close();

Hotkey

Press a multi-key shortcut. Pass an array of keys that will be joined with + and sent as a single xdotool key combination (e.g. ["ctrl", "l"] becomes ctrl+l). Accepts 1--10 keys.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
sandbox.desktop.hotkey(keys=["ctrl", "l"])

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

await sandbox.desktop.hotkey(["ctrl", "l"]);
await client.close();

Recordings

Screen recordings are captured as MP4 files (H.264/libx264, yuv420p) using ffmpeg at 12 fps. Files are saved inside the sandbox at ~/.cache/leap0-recordings by default (configurable via the LEAP0_RECORDINGS_DIR environment variable). Only one recording can be active at a time.

Start recording

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.start_recording())

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.startRecording());
await client.close();

Stop recording

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.stop_recording())

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.stopRecording());
await client.close();

Recording status

Get the state of the current or most recent recording.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.recording_status())

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.recordingStatus());
await client.close();

List recordings

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
for recording in sandbox.desktop.recordings():
    print(f"{recording.id}: {recording.size_bytes} bytes")

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

const recordings = await sandbox.desktop.recordings();
recordings.forEach((recording) => console.log(recording.id, recording.sizeBytes + " bytes"));
await client.close();

Get recording

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.get_recording("<recording_id>"))

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.getRecording("<recording_id>"));
await client.close();

Download recording

Returns the MP4 file as a binary download.

from pathlib import Path
from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
recording = sandbox.desktop.download_recording("<recording_id>")
Path("recording.mp4").write_bytes(recording)

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

const bytes = await sandbox.desktop.downloadRecording("<recording_id>");
console.log("recording bytes", bytes.byteLength);
await client.close();

Delete recording

Delete a saved recording. Active recordings cannot be deleted. Stop them first. Returns 204 No Content on success.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
sandbox.desktop.delete_recording("<recording_id>")

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

await sandbox.desktop.deleteRecording("<recording_id>");
await client.close();

Process management

The template runs five processes: xvfb (virtual framebuffer), xfce4 (desktop session), x11vnc (VNC server), novnc (web-based VNC client via websockify), and control (the HTTP API server). The four managed processes (xvfb, xfce4, x11vnc, novnc) can be inspected and restarted through the endpoints below.

Overall status

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
print(sandbox.desktop.process_status())

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.processStatus());
await client.close();

Status stream (SSE)

Subscribe to a live Server-Sent Events stream of process status. The server sends an event immediately on connect, whenever a process status changes (e.g. after a restart), and every 2 seconds as a heartbeat. Each event has the same payload as GET /api/status. The connection stays open until the client disconnects.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)
for status in sandbox.desktop.status_stream():
    print(status)
    break

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

for await (const status of sandbox.desktop.statusStream()) {
console.log(status);
break;
}

await client.close();

Per-process endpoints

Inspect or restart individual processes. Replace {name} with xvfb, xfce4, x11vnc, or novnc.

from leap0 import DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client

client = Leap0Client()
sandbox = client.sandboxes.create(template_name=DEFAULT_DESKTOP_TEMPLATE_NAME)

print(sandbox.desktop.get_process("x11vnc"))
print(sandbox.desktop.restart_process("xfce4"))
print(sandbox.desktop.process_logs("novnc"))

import { DEFAULT_DESKTOP_TEMPLATE_NAME, Leap0Client } from "leap0";

const client = new Leap0Client();
const sandbox = await client.sandboxes.create({ templateName: DEFAULT_DESKTOP_TEMPLATE_NAME });

console.log(await sandbox.desktop.getProcess("x11vnc"));
console.log(await sandbox.desktop.restartProcess("xfce4"));
console.log(await sandbox.desktop.processLogs("novnc"));
await client.close();

Was this page helpful?

Computer Use (Desktop)

On this page