#!/bin/sh
# Report PBS queue state: this user's whole queue, or one job's detail.
#
#   ./bin/aspire-status              # this user's jobs on both PBS servers
#   ./bin/aspire-status <job-id>     # full status of one job, live or finished
#
# Aspire runs two PBS servers, pbs101 and pbs102; qstat only talks to one
# unless told which via "@server", so a plain `qstat -u <user>` can silently
# miss jobs sitting on the other one. This always queries both.
#
# PBS also drops a job from the live qstat view soon after it finishes; once
# a job-id isn't found live on either server, this falls back to `qstat -x`
# (accounting history) on each, so "did it finish and how" keeps working
# after the fact.

set -eu

HOST="${ASPIRE_HOST:-aspire}"
BIN=$(cd -P "$(dirname "$0")" && pwd)
SERVERS="pbs101 pbs102"

"$BIN/aspire-connect"

if [ $# -eq 0 ]; then
    REMOTE_USER=$(ssh -o BatchMode=yes "$HOST" whoami)
    for server in $SERVERS; do
        echo "== $server =="
        ssh -o BatchMode=yes "$HOST" /opt/pbs/bin/qstat -u "$REMOTE_USER" "@$server" || true
    done
    exit 0
fi

JOB_ID="$1"

# A routed job (anything submitted to `ai`) exists on BOTH servers: pbs101 keeps a
# stale "moved" stub with job_state = M, no Exit_status and no resources_used, while
# pbs102 holds the real record. Taking the first server that answers therefore reports
# M forever and never sees the job finish.
#
# Worse, a single response can contain BOTH records under the same job id — pbs102's
# history returns the M stub and the real one together — so "does this mention M" is
# the wrong test. Ask instead whether a state other than M appears anywhere, and keep
# that response. Only fall back to an all-M response if that is genuinely all there is.
# `qstat -xf <id>` returns OTHER jobs alongside the one asked for -- verified: asking for
# 15087494 came back with 15086756 and 15087507 too, at exit 0. Printing the raw response
# therefore shows someone else's state and comment. Keep only the requested job's block.
filter_job() {
    awk -v id="$1" '
        /^Job Id:/ { want = ($3 == id || index($3, id ".") == 1) }
        want { print }
    '
}

report() {
    flag="$1"
    best=""
    fallback=""

    for server in $SERVERS; do
        raw=$(ssh -o BatchMode=yes "$HOST" /opt/pbs/bin/qstat "$flag" "$JOB_ID" "@$server" 2>/dev/null) || continue
        out=$(printf '%s\n' "$raw" | filter_job "$JOB_ID")
        [ -n "$out" ] || continue

        # Job states are single letters (Q R F H E B M S T W X); M means "moved to
        # the other server". Anything else is a real record.
        if echo "$out" | grep -qE '^[[:space:]]*job_state[[:space:]]*=[[:space:]]*[^M[:space:]][[:space:]]*$'; then
            best="$out"
        else
            [ -n "$fallback" ] || fallback="$out"
        fi
    done

    if [ -n "$best" ]; then
        printf '%s\n' "$best"
        return 0
    fi
    if [ -n "$fallback" ]; then
        echo "aspire-status: only a moved-job stub found; the job was routed to the" >&2
        echo "aspire-status: other server but has no record there yet." >&2
        printf '%s\n' "$fallback"
        return 0
    fi
    return 1
}

report -f && exit 0

echo "aspire-status: not in the live queue on pbs101 or pbs102; checking finished-job history..." >&2

report -xf && exit 0

echo "aspire-status: $JOB_ID not found live or in history on pbs101 or pbs102" >&2
exit 1
