commit 995ffd1fb819999feff3c84c4ac25ecde7f3c164
Author: Ben Sima <ben@bensima.com>
Date: Tue Dec 30 23:51:11 2025
pi-orchestrate: Add timeout and resource limits
- --coder-timeout=SECS (default 600 = 10 min per iteration)
- --reviewer-timeout=SECS (default 300 = 5 min per iteration)
- --total-timeout=SECS (default 1800 = 30 min overall)
Uses timeout command for per-phase limits, tracks elapsed time
for total timeout. Clear error messages distinguish timeout
(exit 124) from other failures.
Also updated Orchestrator.hs to recognize timeout messages.
Task-Id: t-298.8
diff --git a/Omni/Agent/Telegram/Orchestrator.hs b/Omni/Agent/Telegram/Orchestrator.hs
index d391309e..b7b348ff 100644
--- a/Omni/Agent/Telegram/Orchestrator.hs
+++ b/Omni/Agent/Telegram/Orchestrator.hs
@@ -239,6 +239,7 @@ streamOutput h onProgress resultVar = do
| "=== SUCCESS ===" `Text.isInfixOf` line = Just (OrchSuccess "")
| "=== FAILURE - Needs Help ===" `Text.isInfixOf` line = Just (OrchNeedsHelp "")
| "=== FAILURE - Max Iterations ===" `Text.isInfixOf` line = Just (OrchFailed "max iterations reached")
+ | "=== FAILURE - Total Timeout ===" `Text.isInfixOf` line = Just (OrchFailed "total timeout exceeded")
| "Commit: " `Text.isInfixOf` line =
let commit = Text.strip (Text.drop 8 (snd (Text.breakOn "Commit: " line)))
in case current of
@@ -257,7 +258,8 @@ streamOutput h onProgress resultVar = do
"Phase: REVIEWER",
"=== SUCCESS",
"=== FAILURE",
- "Commit:"
+ "Commit:",
+ "timed out after"
]
formatProgress :: Text -> Text
@@ -272,6 +274,7 @@ streamOutput h onProgress resultVar = do
| "Commit:" `Text.isInfixOf` line =
let commit = Text.strip (Text.drop 8 (snd (Text.breakOn "Commit: " line)))
in "📝 commit: " <> commit
+ | "timed out after" `Text.isInfixOf` line = "⏰ timeout!"
| otherwise = line
extractIterationNum :: Text -> Text
diff --git a/Omni/Ide/pi-orchestrate.sh b/Omni/Ide/pi-orchestrate.sh
index 604b2e7c..3d296744 100755
--- a/Omni/Ide/pi-orchestrate.sh
+++ b/Omni/Ide/pi-orchestrate.sh
@@ -6,6 +6,11 @@
# Usage:
# pi-orchestrate t-123 # Run coder+reviewer loop
# pi-orchestrate t-123 --max=3 # Max 3 iterations
+#
+# Timeouts:
+# --coder-timeout=SECS Per-iteration coder timeout (default: 600 = 10 min)
+# --reviewer-timeout=SECS Per-iteration reviewer timeout (default: 300 = 5 min)
+# --total-timeout=SECS Overall orchestrator timeout (default: 1800 = 30 min)
set -e
@@ -53,10 +58,13 @@ Arguments:
task-id Task ID to work on (e.g., t-123)
Options:
- --max=N Maximum iterations (default: 3)
- --dry-run Pass through to pi-review (no actual commits)
- -h, --help Show this help
- -v, --version Show version
+ --max=N Maximum iterations (default: 3)
+ --dry-run Pass through to pi-review (no actual commits)
+ --coder-timeout=SECS Timeout for each coder iteration (default: 600 = 10 min)
+ --reviewer-timeout=SECS Timeout for each reviewer iteration (default: 300 = 5 min)
+ --total-timeout=SECS Overall timeout for orchestrator (default: 1800 = 30 min)
+ -h, --help Show this help
+ -v, --version Show version
What it does:
1. Run pi-code on the task
@@ -89,6 +97,11 @@ MAX_ITERATIONS=3
DRY_RUN=false
TASK_ID=""
+# Timeout defaults (in seconds)
+CODER_TIMEOUT=600 # 10 minutes per coder iteration
+REVIEWER_TIMEOUT=300 # 5 minutes per reviewer iteration
+TOTAL_TIMEOUT=1800 # 30 minutes overall
+
while [ $# -gt 0 ]; do
case "$1" in
--max=*)
@@ -99,6 +112,18 @@ while [ $# -gt 0 ]; do
DRY_RUN=true
shift
;;
+ --coder-timeout=*)
+ CODER_TIMEOUT="${1#--coder-timeout=}"
+ shift
+ ;;
+ --reviewer-timeout=*)
+ REVIEWER_TIMEOUT="${1#--reviewer-timeout=}"
+ shift
+ ;;
+ --total-timeout=*)
+ TOTAL_TIMEOUT="${1#--total-timeout=}"
+ shift
+ ;;
-*)
echo "Unknown option: $1"
echo "Run 'pi-orchestrate --help' for usage."
@@ -137,9 +162,41 @@ if [ "$DRY_RUN" = true ]; then
REVIEW_ARGS="--dry-run"
fi
+# Record start time for total timeout tracking
+START_TIME=$(date +%s)
+
+# Function to check if total timeout exceeded
+check_total_timeout() {
+ local elapsed=$(($(date +%s) - START_TIME))
+ if [ "$elapsed" -ge "$TOTAL_TIMEOUT" ]; then
+ echo ""
+ phase "FAILURE - Total Timeout"
+ error "Exceeded total timeout of ${TOTAL_TIMEOUT}s ($(format_duration $TOTAL_TIMEOUT))"
+ info "Task $TASK_ID may need to be continued manually"
+ exit 1
+ fi
+ local remaining=$((TOTAL_TIMEOUT - elapsed))
+ info "Total time elapsed: $(format_duration $elapsed), remaining: $(format_duration $remaining)"
+}
+
+# Format seconds as human-readable duration
+format_duration() {
+ local secs=$1
+ local mins=$((secs / 60))
+ local remaining_secs=$((secs % 60))
+ if [ "$mins" -gt 0 ]; then
+ echo "${mins}m ${remaining_secs}s"
+ else
+ echo "${secs}s"
+ fi
+}
+
echo ""
phase "pi-orchestrate: $TASK_ID"
info "Max iterations: $MAX_ITERATIONS"
+info "Coder timeout: $(format_duration $CODER_TIMEOUT) per iteration"
+info "Reviewer timeout: $(format_duration $REVIEWER_TIMEOUT) per iteration"
+info "Total timeout: $(format_duration $TOTAL_TIMEOUT)"
if [ "$DRY_RUN" = true ]; then
warn "DRY RUN MODE - No commits will be made"
fi
@@ -150,20 +207,37 @@ ITERATION=1
while [ $ITERATION -le "$MAX_ITERATIONS" ]; do
phase "Iteration $ITERATION/$MAX_ITERATIONS"
- # Step 1: Run pi-code
+ # Check total timeout before starting iteration
+ check_total_timeout
+
+ # Step 1: Run pi-code with timeout
echo ""
- status "Phase: CODER"
- if ! "$SCRIPT_DIR/pi-code.sh" "$TASK_ID"; then
- error "pi-code failed"
+ status "Phase: CODER (timeout: $(format_duration $CODER_TIMEOUT))"
+ if ! timeout "$CODER_TIMEOUT" "$SCRIPT_DIR/pi-code.sh" "$TASK_ID"; then
+ EXIT_CODE=$?
+ if [ "$EXIT_CODE" -eq 124 ]; then
+ error "pi-code timed out after $(format_duration $CODER_TIMEOUT)"
+ else
+ error "pi-code failed with exit code $EXIT_CODE"
+ fi
exit 1
fi
- # Step 2: Run pi-review
+ # Check total timeout before reviewer
+ check_total_timeout
+
+ # Step 2: Run pi-review with timeout
echo ""
- status "Phase: REVIEWER"
+ status "Phase: REVIEWER (timeout: $(format_duration $REVIEWER_TIMEOUT))"
# Note: pi-review exits 0 even on REJECT (it handles it gracefully)
# We need to check the task status after to determine what happened
- "$SCRIPT_DIR/pi-review.sh" $REVIEW_ARGS "$TASK_ID" || true
+ if ! timeout "$REVIEWER_TIMEOUT" "$SCRIPT_DIR/pi-review.sh" $REVIEW_ARGS "$TASK_ID"; then
+ EXIT_CODE=$?
+ if [ "$EXIT_CODE" -eq 124 ]; then
+ warn "pi-review timed out after $(format_duration $REVIEWER_TIMEOUT)"
+ # Continue to check status - partial work may still be usable
+ fi
+ fi
# Step 3: Check task status
echo ""