Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion harness/clients/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ The defaults below are the current RTX 3090 starting points for
| Codex | `run_codex.sh` | `MAX_CTX=32768 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
| OpenCode | `run_opencode.sh` | `MAX_CTX=86016 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
| Hermes Agent | `run_hermes.sh` | `MAX_CTX=98304 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
| Pi | `run_pi.sh` | `MAX_CTX=65536 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
| Pi | `run_pi.sh` | `MAX_CTX=65536 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft PI_TIMEOUT=3600` |
| OpenClaw | `run_openclaw.sh` | `MAX_CTX=204800 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
| Open WebUI chat | `run_openwebui.sh` | `MAX_CTX=262144 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
| Open WebUI tools | `run_openwebui_tools.sh` | `MAX_CTX=65536 BUDGET=22 VERIFY_MODE=ddtree EXTRA_SERVER_ARGS=--lazy-draft` |
Expand All @@ -79,6 +79,27 @@ PROMPT='Explain the repo and end with lucebox-client-ok' harness/clients/run_ope
PROMPT_FILE=harness/clients/prompts/repo_inspection.txt harness/clients/run_hermes.sh
```

`PI_TIMEOUT` is Pi's total wall-clock limit in seconds. Its one-hour default
allows long-context prefill and long generations to finish; set
`PI_TIMEOUT=0` to run without a launcher deadline. The launcher also disables
Pi's separate five-minute HTTP idle timeout, which can otherwise terminate an
SSE connection during a long prefill. For a manually configured Pi install,
put the same setting in `~/.pi/agent/settings.json`:

```json
{"httpIdleTimeoutMs": 0}
```

The other real-client launchers also allow one hour by default. Override their
deadlines with `CLAUDE_TIMEOUT`, `CODEX_TIMEOUT`, `OPENCODE_TIMEOUT`,
`HERMES_TIMEOUT`, `OPENCLAW_TIMEOUT`, or (for Open WebUI's curl probe)
`CURL_MAX_TIME`. The CLI launcher timeouts accept `0` to disable the outer
deadline. OpenCode's provider-level request and chunk deadlines default to one
hour too and can be changed with `OPENCODE_REQUEST_TIMEOUT_MS` and
`OPENCODE_CHUNK_TIMEOUT_MS`. The server independently sends SSE heartbeat
comments during silent prefill and cancels backend work when a client
disconnects.

Claude Code uses the real Anthropic Messages client path. Lucebox trims
Claude-specific prompt boilerplate by default for local-model reliability. To
test the raw prompt, set:
Expand Down
17 changes: 17 additions & 0 deletions harness/clients/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,23 @@ require_client_binary() {
return 1
}

# Run a real client with a bounded wall-clock deadline. Each launcher exposes
# its own variable for backward compatibility, but shares validation and the
# `0 = unlimited` contract here.
run_with_timeout() {
local timeout_seconds="$1"
shift
if [[ ! "$timeout_seconds" =~ ^[0-9]+$ ]]; then
echo "client timeout must be a non-negative integer (seconds; 0 disables it)" >&2
return 2
fi
if [[ "$timeout_seconds" == "0" ]]; then
"$@"
else
timeout "${timeout_seconds}s" "$@"
fi
}

draft_enabled() {
[[ -n "${DRAFT:-}" && "$DRAFT" != "none" && "$DRAFT" != "off" && "$DRAFT" != "0" ]]
}
Expand Down
19 changes: 10 additions & 9 deletions harness/clients/run_claude_code.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${VERIFY_MODE:=ddtree}"
: "${EXTRA_SERVER_ARGS:=--lazy-draft}"
: "${CLAUDE_TOOLS:=default}"
: "${CLAUDE_TIMEOUT:=300}"
: "${CLAUDE_TIMEOUT:=3600}"
if [[ "${MODEL_SERVER:-}" == "llamacpp" ]]; then
: "${LLAMA_COMPAT_PROXY:=anthropic}"
fi
Expand Down Expand Up @@ -58,14 +58,15 @@ if [[ -n "${PFLASH_SESSION_ID:-}" ]]; then
fi

set +e
HOME="$HOME_DIR" \
ANTHROPIC_API_KEY="$API_KEY" \
ANTHROPIC_BASE_URL="$CLIENT_BASE_URL" \
CLAUDE_CODE_API_BASE_URL="$CLIENT_BASE_URL" \
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 \
CLAUDE_CODE_DISABLE_TELEMETRY=1 \
CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK=1 \
timeout "${CLAUDE_TIMEOUT}s" "$CLAUDE_BIN" \
run_with_timeout "$CLAUDE_TIMEOUT" env \
HOME="$HOME_DIR" \
ANTHROPIC_API_KEY="$API_KEY" \
ANTHROPIC_BASE_URL="$CLIENT_BASE_URL" \
CLAUDE_CODE_API_BASE_URL="$CLIENT_BASE_URL" \
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 \
CLAUDE_CODE_DISABLE_TELEMETRY=1 \
CLAUDE_CODE_DISABLE_NONSTREAMING_FALLBACK=1 \
"$CLAUDE_BIN" \
--print \
--output-format json \
--model "$MODEL_ID" \
Expand Down
10 changes: 6 additions & 4 deletions harness/clients/run_codex.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${BUDGET:=22}"
: "${VERIFY_MODE:=ddtree}"
: "${EXTRA_SERVER_ARGS:=--lazy-draft}"
: "${CODEX_TIMEOUT:=3600}"
if [[ "${MODEL_SERVER:-}" == "llamacpp" ]]; then
: "${LLAMA_COMPAT_PROXY:=responses}"
fi
Expand Down Expand Up @@ -38,10 +39,11 @@ trap stop_lucebox_server EXIT
wait_lucebox_server

set +e
HOME="$CODEX_HOME_DIR" \
CODEX_HOME="$CODEX_HOME_DIR" \
OPENAI_API_KEY="$API_KEY" \
timeout 420s "$CODEX_BIN" exec \
run_with_timeout "$CODEX_TIMEOUT" env \
HOME="$CODEX_HOME_DIR" \
CODEX_HOME="$CODEX_HOME_DIR" \
OPENAI_API_KEY="$API_KEY" \
"$CODEX_BIN" exec \
--skip-git-repo-check \
--sandbox "$CODEX_SANDBOX" \
--model "$MODEL_ID" \
Expand Down
24 changes: 13 additions & 11 deletions harness/clients/run_hermes.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${VERIFY_MODE:=ddtree}"
: "${EXTRA_SERVER_ARGS:=--lazy-draft}"
: "${HERMES_MAX_TURNS:=40}"
: "${HERMES_TIMEOUT:=3600}"
source "$SCRIPT_DIR/common.sh"

CLIENT_OUT="$LOG_DIR/hermes.out"
Expand Down Expand Up @@ -48,8 +49,8 @@ OPENAI_BASE_URL=$BASE_URL/v1
HERMES_INFERENCE_PROVIDER=lucebox
HERMES_INFERENCE_MODEL=$MODEL_ID
HERMES_ACCEPT_HOOKS=1
HERMES_API_TIMEOUT=600
HERMES_API_CALL_STALE_TIMEOUT=600
HERMES_API_TIMEOUT=$HERMES_TIMEOUT
HERMES_API_CALL_STALE_TIMEOUT=$HERMES_TIMEOUT
ENV

start_lucebox_server
Expand All @@ -58,15 +59,16 @@ wait_lucebox_server

set +e
cd "$REPO_DIR"
HOME="$HOME_DIR" \
HERMES_HOME="$HOME_DIR" \
OPENAI_API_KEY="$API_KEY" \
OPENAI_BASE_URL="$BASE_URL/v1" \
HERMES_INFERENCE_PROVIDER=lucebox \
HERMES_INFERENCE_MODEL="$MODEL_ID" \
HERMES_ACCEPT_HOOKS=1 \
NO_COLOR=1 \
timeout 420s "$HERMES_BIN" chat \
run_with_timeout "$HERMES_TIMEOUT" env \
HOME="$HOME_DIR" \
HERMES_HOME="$HOME_DIR" \
OPENAI_API_KEY="$API_KEY" \
OPENAI_BASE_URL="$BASE_URL/v1" \
HERMES_INFERENCE_PROVIDER=lucebox \
HERMES_INFERENCE_MODEL="$MODEL_ID" \
HERMES_ACCEPT_HOOKS=1 \
NO_COLOR=1 \
"$HERMES_BIN" chat \
--quiet \
--provider lucebox \
--model "$MODEL_ID" \
Expand Down
33 changes: 21 additions & 12 deletions harness/clients/run_openclaw.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${VERIFY_MODE:=ddtree}"
# FA_WINDOW stays 0 (common.sh): finite windows break tool calls.
: "${EXTRA_SERVER_ARGS:=--lazy-draft}"
: "${OPENCLAW_TIMEOUT:=3600}"
source "$SCRIPT_DIR/common.sh"

CLIENT_OUT="$LOG_DIR/openclaw.out"
Expand Down Expand Up @@ -81,28 +82,36 @@ cat > "$CONFIG_PATCH" <<JSON
}
JSON

HOME="$HOME_DIR" "$OPENCLAW_BIN" config patch --file "$CONFIG_PATCH" > "$LOG_DIR/openclaw-config.out" 2>&1
run_with_timeout "$OPENCLAW_TIMEOUT" env \
HOME="$HOME_DIR" \
"$OPENCLAW_BIN" config patch --file "$CONFIG_PATCH" \
> "$LOG_DIR/openclaw-config.out" 2>&1

start_lucebox_server
trap stop_lucebox_server EXIT
wait_lucebox_server

agent_args=()
openclaw_cmd=(
"$OPENCLAW_BIN" agent
--local
--json
--model "lucebox/$MODEL_ID"
--session-id "lucebox-client-harness"
)
if [[ "$OPENCLAW_TIMEOUT" != "0" ]]; then
openclaw_cmd+=(--timeout "$OPENCLAW_TIMEOUT")
fi
if [[ -n "$OPENCLAW_AGENT_ARGS" ]]; then
read -r -a agent_args <<< "$OPENCLAW_AGENT_ARGS"
openclaw_cmd+=("${agent_args[@]}")
fi
openclaw_cmd+=(--message "$PROMPT")

set +e
HOME="$HOME_DIR" \
OPENAI_API_KEY="$API_KEY" \
timeout 420s "$OPENCLAW_BIN" agent \
--local \
--json \
--model "lucebox/$MODEL_ID" \
--session-id "lucebox-client-harness" \
--timeout 300 \
"${agent_args[@]}" \
--message "$PROMPT" \
run_with_timeout "$OPENCLAW_TIMEOUT" env \
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
HOME="$HOME_DIR" \
OPENAI_API_KEY="$API_KEY" \
"${openclaw_cmd[@]}" \
< /dev/null > "$CLIENT_OUT" 2>&1
RC=$?
set -e
Expand Down
25 changes: 18 additions & 7 deletions harness/clients/run_opencode.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${BUDGET:=22}"
: "${VERIFY_MODE:=ddtree}"
: "${EXTRA_SERVER_ARGS:=--lazy-draft}"
: "${OPENCODE_TIMEOUT:=3600}"
: "${OPENCODE_REQUEST_TIMEOUT_MS:=3600000}"
: "${OPENCODE_CHUNK_TIMEOUT_MS:=3600000}"
source "$SCRIPT_DIR/common.sh"

for timeout_ms in "$OPENCODE_REQUEST_TIMEOUT_MS" "$OPENCODE_CHUNK_TIMEOUT_MS"; do
if [[ ! "$timeout_ms" =~ ^(0|[1-9][0-9]*)$ ]]; then
echo "OpenCode timeouts must be canonical non-negative integers (milliseconds)" >&2
exit 2
fi
done

CLIENT_OUT="$LOG_DIR/opencode.out"
EXPORT_OUT="$LOG_DIR/opencode-export.json"
OPENCODE_BIN="${OPENCODE_BIN:-$CLIENT_WORK_DIR/clients/opencode/npm/bin/opencode}"
Expand Down Expand Up @@ -36,8 +46,8 @@ cat > "$PROJECT_DIR/opencode.json" <<JSON
"options": {
"baseURL": "$BASE_URL/v1",
"apiKey": "$API_KEY",
"timeout": 600000,
"chunkTimeout": 60000
"timeout": $OPENCODE_REQUEST_TIMEOUT_MS,
"chunkTimeout": $OPENCODE_CHUNK_TIMEOUT_MS
},
"models": {
"$MODEL_ID": {
Expand All @@ -63,11 +73,12 @@ wait_lucebox_server

set +e
cd "$PROJECT_DIR"
HOME="$HOME_DIR" \
XDG_CONFIG_HOME="$HOME_DIR/.config" \
XDG_DATA_HOME="$HOME_DIR/.local/share" \
OPENAI_API_KEY="$API_KEY" \
timeout 300s "$OPENCODE_BIN" run \
run_with_timeout "$OPENCODE_TIMEOUT" env \
HOME="$HOME_DIR" \
XDG_CONFIG_HOME="$HOME_DIR/.config" \
XDG_DATA_HOME="$HOME_DIR/.local/share" \
OPENAI_API_KEY="$API_KEY" \
"$OPENCODE_BIN" run \
--pure \
--model "lucebox/$MODEL_ID" \
--format json \
Expand Down
2 changes: 1 addition & 1 deletion harness/clients/run_openwebui.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ AUTH_JSON="$LOG_DIR/openwebui-auth.json"
AUTH_OUT="$LOG_DIR/openwebui-auth.out"
WEBUI_ADMIN_EMAIL_VALUE="${WEBUI_ADMIN_EMAIL_VALUE:-admin@localhost}"
WEBUI_ADMIN_PASSWORD_VALUE="${WEBUI_ADMIN_PASSWORD_VALUE:-admin}"
CURL_MAX_TIME="${CURL_MAX_TIME:-300}"
CURL_MAX_TIME="${CURL_MAX_TIME:-3600}"
mkdir -p "$DATA_DIR"

start_lucebox_server
Expand Down
2 changes: 1 addition & 1 deletion harness/clients/run_openwebui_tools.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ AUTH_JSON="$LOG_DIR/openwebui-auth.json"
AUTH_OUT="$LOG_DIR/openwebui-auth.out"
WEBUI_ADMIN_EMAIL_VALUE="${WEBUI_ADMIN_EMAIL_VALUE:-admin@localhost}"
WEBUI_ADMIN_PASSWORD_VALUE="${WEBUI_ADMIN_PASSWORD_VALUE:-admin}"
CURL_MAX_TIME="${CURL_MAX_TIME:-300}"
CURL_MAX_TIME="${CURL_MAX_TIME:-3600}"
TOOL_ID="${TOOL_ID:-lucebox_harness_probe}"
if [[ "$OPENWEBUI_DISABLE_BUILTIN_TOOLS" == "1" ]]; then
WEBUI_MODEL_ID="${OPENWEBUI_MODEL_ID:-${MODEL_ID}-tools}"
Expand Down
47 changes: 32 additions & 15 deletions harness/clients/run_pi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
: "${VERIFY_MODE:=ddtree}"
: "${EXTRA_SERVER_ARGS:=--lazy-draft}"
: "${PI_TOOLS:=read,grep,find,ls}"
: "${PI_TIMEOUT:=3600}"
if [[ ! "$PI_TIMEOUT" =~ ^[0-9]+$ ]]; then
echo "PI_TIMEOUT must be a non-negative integer (seconds; 0 disables it)" >&2
exit 2
fi
source "$SCRIPT_DIR/common.sh"

CLIENT_OUT="$LOG_DIR/pi.out"
Expand All @@ -21,7 +26,8 @@ cat > "$AGENT_DIR/settings.json" <<JSON
{
"compaction": {
"enabled": false
}
},
"httpIdleTimeoutMs": 0
}
JSON

Expand Down Expand Up @@ -59,21 +65,32 @@ start_lucebox_server
trap stop_lucebox_server EXIT
wait_lucebox_server

pi_env=(
"HOME=$HOME_DIR"
"PI_CODING_AGENT_DIR=$AGENT_DIR"
"PI_CODING_AGENT_SESSION_DIR=$HOME_DIR/sessions"
"PI_OFFLINE=1"
)
pi_cmd=(
"$PI_BIN"
--provider lucebox
--model "$MODEL_ID"
--print
--mode json
--tools "$PI_TOOLS"
--no-session
--offline
"$PROMPT"
)

set +e
HOME="$HOME_DIR" \
PI_CODING_AGENT_DIR="$AGENT_DIR" \
PI_CODING_AGENT_SESSION_DIR="$HOME_DIR/sessions" \
PI_OFFLINE=1 \
timeout 300s "$PI_BIN" \
--provider lucebox \
--model "$MODEL_ID" \
--print \
--mode json \
--tools "$PI_TOOLS" \
--no-session \
--offline \
"$PROMPT" \
< /dev/null > "$CLIENT_OUT" 2>&1
if [[ "$PI_TIMEOUT" == "0" ]]; then
env "${pi_env[@]}" "${pi_cmd[@]}" \
< /dev/null > "$CLIENT_OUT" 2>&1
else
env "${pi_env[@]}" timeout "${PI_TIMEOUT}s" "${pi_cmd[@]}" \
< /dev/null > "$CLIENT_OUT" 2>&1
fi
RC=$?
set -e

Expand Down
Loading
Loading