#!/usr/bin/env bash
set -euo pipefail
LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TESTS_DIR="$(cd "$LIB_DIR/.." && pwd)"
SKILLS_DIR="$(cd "$LIB_DIR/../.." && pwd)"
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'
setup_colors() {
if [ -n "${NO_COLOR:-}" ]; then
disable_colors
elif [ -n "${FORCE_COLOR:-}" ]; then
enable_colors
elif [ -t 1 ]; then
enable_colors
else
disable_colors
fi
}
enable_colors() {
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
BOLD='\033[1m'
NC='\033[0m'
}
disable_colors() {
RED=''
GREEN=''
YELLOW=''
BLUE=''
CYAN=''
BOLD=''
NC=''
}
print_pass() {
echo -e " ${GREEN}[PASS]${NC} $*"
}
print_fail() {
echo -e " ${RED}[FAIL]${NC} $*"
}
print_skip() {
echo -e " ${YELLOW}[SKIP]${NC} $*"
}
print_info() {
echo -e " ${BLUE}[INFO]${NC} $*"
}
print_warn() {
echo -e " ${YELLOW}[WARN]${NC} $*"
}
print_error() {
echo -e " ${RED}[ERROR]${NC} $*"
}
print_section_header() {
local name="$1"
echo ""
echo -e "${BOLD}${CYAN}=== $name ===${NC}"
echo ""
}
print_compact_result() {
local output="$1"
local status="$2"
local duration="$3"
local show_all="${4:-false}"
local n_pass=0 n_fail=0 n_skip=0 n_warn=0
if [[ -n "$output" ]]; then
n_pass=$(echo "$output" | grep -cE '\[PASS\]' 2>/dev/null || echo 0)
n_fail=$(echo "$output" | grep -cE '\[FAIL\]' 2>/dev/null || echo 0)
n_skip=$(echo "$output" | grep -cE '\[SKIP\]' 2>/dev/null || echo 0)
n_warn=$(echo "$output" | grep -cE '\[WARN\]' 2>/dev/null || echo 0)
[[ "$n_pass" =~ ^[0-9]+$ ]] || n_pass=0
[[ "$n_fail" =~ ^[0-9]+$ ]] || n_fail=0
[[ "$n_skip" =~ ^[0-9]+$ ]] || n_skip=0
[[ "$n_warn" =~ ^[0-9]+$ ]] || n_warn=0
fi
local counts=""
[[ $n_pass -gt 0 ]] && counts+=" ${GREEN}${n_pass} passed${NC}"
[[ $n_fail -gt 0 ]] && counts+=" ${RED}${n_fail} failed${NC}"
[[ $n_skip -gt 0 ]] && counts+=" ${YELLOW}${n_skip} skipped${NC}"
[[ $n_warn -gt 0 ]] && counts+=" ${YELLOW}${n_warn} warnings${NC}"
case "$status" in
pass)
if [[ -z "$counts" ]]; then
echo -e " ${GREEN}[PASS]${NC} (${duration}s)"
else
echo -e " ${GREEN}[PASS]${NC} (${counts}, ${duration}s)"
fi
if [[ $n_warn -gt 0 ]]; then
echo ""
echo "$output" | grep -E '\[WARN\]' | head -20 | sed 's/^/ /'
echo ""
fi
;;
fail)
echo -e " ${RED}[FAIL]${NC} (${counts}, ${duration}s)"
if [[ $n_fail -gt 0 ]]; then
echo ""
echo -e " ${YELLOW}── Failures ──${NC}"
echo "$output" | grep -E '\[FAIL\]' | head -30 | sed 's/^/ /'
echo ""
fi
local n_err
n_err=$(echo "$output" | grep -cE '\[ERROR\]' 2>/dev/null || echo 0)
[[ "$n_err" =~ ^[0-9]+$ ]] || n_err=0
if [[ $n_err -gt 0 ]]; then
echo -e " ${RED}── Errors ──${NC}"
echo "$output" | grep -E '\[ERROR\]' | head -20 | sed 's/^/ /'
echo ""
fi
;;
esac
if $show_all; then
echo ""
echo -e " ${BLUE}── Full Output ──${NC}"
echo "$output" | sed 's/^/ /'
echo -e " ${BLUE}── End ──${NC}"
echo ""
fi
}
print_status_passed() {
echo -e "${GREEN}${BOLD}STATUS: PASSED${NC}"
}
print_status_failed() {
echo -e "${RED}${BOLD}STATUS: FAILED${NC}"
}
setup_colors
DEFAULT_TIMEOUT=60
DEFAULT_PLATFORM="opencode"
declare -g TEST_PASSED=0
declare -g TEST_FAILED=0
declare -g TEST_SKIPPED=0
declare -g TEST_START_TIME=0
is_platform_available() {
local platform="$1"
case "$platform" in
claude) command -v claude &> /dev/null ;;
opencode) command -v opencode &> /dev/null ;;
*) return 1 ;;
esac
}
get_platform_version() {
local platform="$1"
case "$platform" in
claude) claude --version 2>/dev/null || echo "unknown" ;;
opencode) opencode --version 2>/dev/null || echo "unknown" ;;
*) echo "unknown" ;;
esac
}
create_test_project() {
local prefix="${1:-cann-test}"
local test_dir=$(mktemp -d -t "${prefix}.XXXXXX")
echo "$test_dir"
}
cleanup_test_project() {
local test_dir="$1"
if [ -d "$test_dir" ]; then
rm -rf "$test_dir"
fi
}
run_claude() {
local prompt="$1"
local timeout="${2:-$DEFAULT_TIMEOUT}"
local allowed_tools="${3:-}"
local output_file=$(mktemp)
local cmd="claude -p \"$prompt\""
if [ -n "$allowed_tools" ]; then
cmd="$cmd --allowed-tools=$allowed_tools"
fi
if timeout "$timeout" bash -c "$cmd" > "$output_file" 2>&1; then
cat "$output_file"
rm -f "$output_file"
return 0
else
local exit_code=$?
cat "$output_file" >&2
rm -f "$output_file"
return $exit_code
fi
}
assert_order() {
local output="$1"
local pattern_a="$2"
local pattern_b="$3"
local test_name="${4:-test}"
local line_a=$(echo "$output" | grep -inE "$pattern_a" | head -1 | cut -d: -f1)
local line_b=$(echo "$output" | grep -inE "$pattern_b" | head -1 | cut -d: -f1)
if [ -z "$line_a" ]; then
print_fail "$test_name: pattern A not found: $pattern_a"
return 1
fi
if [ -z "$line_b" ]; then
print_fail "$test_name: pattern B not found: $pattern_b"
return 1
fi
if [ "$line_a" -lt "$line_b" ]; then
print_pass "$test_name (A at line $line_a, B at line $line_b)"
return 0
elif [ "$line_a" -eq "$line_b" ]; then
local the_line
the_line=$(echo "$output" | sed -n "${line_a}p")
local pos_a pos_b
pos_a=$(echo "$the_line" | grep -iobE "$pattern_a" | head -1 | cut -d: -f1)
pos_b=$(echo "$the_line" | grep -iobE "$pattern_b" | head -1 | cut -d: -f1)
if [ -n "$pos_a" ] && [ -n "$pos_b" ] && [ "$pos_a" -le "$pos_b" ]; then
print_pass "$test_name (both on line $line_a, A at pos $pos_a, B at pos $pos_b)"
return 0
elif [ -n "$pos_a" ] && [ -n "$pos_b" ]; then
print_fail "$test_name"
echo -e " ${YELLOW}Expected '$pattern_a' before '$pattern_b' on line $line_a${NC}"
echo " But found A at pos $pos_a, B at pos $pos_b"
return 1
else
print_pass "$test_name (both on line $line_a)"
return 0
fi
else
print_fail "$test_name"
echo -e " ${YELLOW}Expected '$pattern_a' before '$pattern_b'${NC}"
echo " But found A at line $line_a, B at line $line_b"
return 1
fi
}
assert_file_exists() {
local file="$1"
local test_name="${2:-file exists}"
if [ -f "$file" ]; then
print_pass "$test_name"
return 0
else
print_fail "$test_name"
echo " File not found: $file"
return 1
fi
}
check_file_links() {
local file="$1"
local file_type="$2"
local file_dir="$(dirname "$file")"
local item_name
if [ "$file_type" = "agent" ]; then
item_name=$(basename "$file" .md)
else
item_name=$(basename "$file_dir")
fi
local broken_links=()
local links
links=$(python3 - "$file" <<'PY'
import re, sys
path = sys.argv[1]
try:
text = open(path, encoding="utf-8", errors="replace").read()
except OSError:
sys.exit(0)
out = set()
for m in re.finditer(r'\]\((references/[^)\s]+|\./[^)\s]+)\)', text):
out.add(m.group(1))
for m in re.finditer(r'\{file:([^}\s]+)\}', text):
out.add(m.group(1))
for raw in out:
link = raw.split('#', 1)[0]
if link.startswith('./'):
link = link[2:]
if link:
print(link)
PY
)
while IFS= read -r link; do
[ -z "$link" ] && continue
if [ ! -e "$file_dir/$link" ]; then
broken_links+=("$link")
fi
done <<< "$links"
if [ ${#broken_links[@]} -gt 0 ]; then
print_fail "$file_type/$item_name: Broken links:"
echo -e " ${YELLOW}${broken_links[*]}${NC}"
return 1
else
print_pass "$file_type/$item_name: All links valid"
return 0
fi
}
run_behavior_test() {
local name="$1"
local prompt="$2"
local expected="$3"
local timeout="${4:-25}"
echo "Testing: $name"
if output=$(timeout "$timeout" claude -p "$prompt 用 1 行回答。" 2>&1); then
if echo "$output" | grep -qiE "$expected"; then
print_pass "Correct response"
pass_count=$((pass_count + 1))
else
print_fail "Incorrect response"
echo -e " ${YELLOW}Expected:${NC} $expected"
echo " Output: ${output:0:80}..."
fail_count=$((fail_count + 1))
fi
else
print_skip "Claude CLI timed out"
skip_count=$((skip_count + 1))
fi
echo ""
}
analyze_premature_actions() {
local session_file="$1"
local target_type="$2"
local target_name="$3"
if [ ! -f "$session_file" ]; then
echo " [SKIP] Session file not found"
return 0
fi
local passed=true
local tool_invoked=false
local first_tool_line=""
local search_pattern='"name":"Skill"'
[ "$target_type" = "agent" ] && search_pattern='"name":"Task"'
if grep -q "$search_pattern" "$session_file" 2>/dev/null; then
tool_invoked=true
first_tool_line=$(grep -n "$search_pattern" "$session_file" 2>/dev/null | head -1 | cut -d: -f1)
fi
if $tool_invoked; then
local target_pattern
if [ "$target_type" = "skill" ]; then
target_pattern='"skill":"([^"]*:)?'"${target_name}"'"'
else
target_pattern="\"subagent_type\":\"[^\"]*${target_name}[^\"]*\""
fi
if grep -qE "$target_pattern" "$session_file" 2>/dev/null; then
print_pass "Target $target_type '$target_name' was invoked"
else
print_info "Other ${target_type}s were invoked instead of '$target_name'"
if [ "$target_type" = "skill" ]; then
local triggered=$(grep -o '"skill":"[^"]*"' "$session_file" 2>/dev/null | sort -u | sed 's/^/ - /')
else
local triggered=$(grep -o '"subagent_type":"[^"]*"' "$session_file" 2>/dev/null | sort -u | sed 's/^/ - /')
fi
if [ -n "$triggered" ]; then
print_info "Triggered ${target_type}s:"
echo "$triggered"
fi
fi
else
if [ "$target_type" = "skill" ]; then
print_warn "No Skill tool was invoked in this session"
else
print_info "No explicit Task tool invocation (agent may be triggered implicitly)"
fi
fi
if $tool_invoked && [ -n "$first_tool_line" ]; then
local premature_tools=$(head -n "$first_tool_line" "$session_file" 2>/dev/null | \
grep '"type":"tool_use"' 2>/dev/null | \
grep -v '"name":"Skill"' 2>/dev/null | \
grep -v '"name":"Task"' 2>/dev/null | \
grep -v '"name":"TodoWrite"' 2>/dev/null | \
grep -v '"name":"Read"' 2>/dev/null | \
grep -v '"name":"Glob"' 2>/dev/null | \
grep -v '"name":"Grep"' 2>/dev/null | \
grep -E '"name":"(Write|Edit|Bash)"' 2>/dev/null || true)
if [ -n "$premature_tools" ]; then
print_fail "Premature actions detected BEFORE $target_type invocation:"
echo "$premature_tools" | head -5 | sed 's/^/ /'
passed=false
else
print_pass "No premature Write/Edit/Bash actions before $target_type"
fi
fi
local todo_before=0
if $tool_invoked && [ -n "$first_tool_line" ]; then
todo_before=$(head -n "$first_tool_line" "$session_file" 2>/dev/null | \
grep -c '"name":"TodoWrite"' 2>/dev/null | tr -cd '0-9' || true)
todo_before="${todo_before:-0}"
fi
if [ "$todo_before" -gt 0 ]; then
print_info "TodoWrite used $todo_before time(s) before $target_type (acceptable for planning)"
fi
local read_before=0
if $tool_invoked && [ -n "$first_tool_line" ]; then
read_before=$(head -n "$first_tool_line" "$session_file" 2>/dev/null | \
grep -c '"name":"Read"' 2>/dev/null | tr -cd '0-9' || true)
read_before="${read_before:-0}"
fi
if [ "$read_before" -gt 0 ]; then
print_info "Read used $read_before time(s) before $target_type (acceptable for context)"
fi
if $passed; then
return 0
else
return 1
fi
}
_is_inside_nested_git_repo() {
local file="$1"
local dir
dir=$(dirname "$file")
while [ "$dir" != "$SKILLS_DIR" ] && [ "$dir" != "/" ]; do
if [ -d "$dir/.git" ]; then
return 0
fi
dir=$(dirname "$dir")
done
return 1
}
get_all_skills_with_paths() {
local tmpfile
tmpfile=$(mktemp)
find "$SKILLS_DIR" \
\( -name "node_modules" -o -name ".git" -o -name "teams" \) -prune -o \
\( -path "*/skills/*/SKILL.md" -o -path "*/ops/*/SKILL.md" -o -path "*/graph/*/SKILL.md" -o -path "*/model/*/SKILL.md" \) \
-print 2>/dev/null > "$tmpfile" || true
while IFS= read -r f; do
[ -f "$f" ] || continue
if _is_inside_nested_git_repo "$f"; then
continue
fi
echo "$(basename "$(dirname "$f")"):$f"
done < "$tmpfile" | sort -u -t: -k1,1
rm -f "$tmpfile"
}
get_all_skills() {
get_all_skills_with_paths | cut -d: -f1
}
is_incremental_mode() {
[ -n "${INCREMENTAL_SKILLS:-}${INCREMENTAL_AGENTS:-}${INCREMENTAL_TEAMS:-}" ]
}
should_test_skill() {
local skill_name="$1"
if ! is_incremental_mode; then
return 0
fi
if [ -n "${INCREMENTAL_SKILLS:-}" ]; then
echo " $INCREMENTAL_SKILLS " | grep -q " $skill_name "
return $?
fi
return 1
}
should_test_agent() {
local agent_name="$1"
if ! is_incremental_mode; then
return 0
fi
if [ -n "${INCREMENTAL_AGENTS:-}" ]; then
echo " $INCREMENTAL_AGENTS " | grep -q " $agent_name "
return $?
fi
return 1
}
should_test_team() {
local team_name="$1"
if ! is_incremental_mode; then
return 0
fi
if [ -n "${INCREMENTAL_TEAMS:-}" ]; then
echo " $INCREMENTAL_TEAMS " | grep -q " $team_name "
return $?
fi
return 1
}
get_skills_to_test() {
local all_skills
all_skills=$(get_all_skills)
if ! is_incremental_mode || [ -z "${INCREMENTAL_SKILLS:-}" ]; then
echo "$all_skills"
return
fi
for skill in $all_skills; do
if should_test_skill "$skill"; then
echo "$skill"
fi
done
}
get_agents_to_test() {
local all_agents
all_agents=$(get_all_agents)
if ! is_incremental_mode || [ -z "${INCREMENTAL_AGENTS:-}" ]; then
echo "$all_agents"
return
fi
for agent in $all_agents; do
if should_test_agent "$agent"; then
echo "$agent"
fi
done
}
get_teams_to_test() {
local all_teams
all_teams=$(get_all_teams)
if ! is_incremental_mode || [ -z "${INCREMENTAL_TEAMS:-}" ]; then
echo "$all_teams"
return
fi
for team in $all_teams; do
if should_test_team "$team"; then
echo "$team"
fi
done
}
find_skill_file() {
local skill_name="$1"
local result
result=$(get_all_skills_with_paths | grep "^${skill_name}:" | head -1 | cut -d: -f2-)
if [ -n "$result" ]; then
echo "$result"
else
echo "$SKILLS_DIR/skills/$skill_name/SKILL.md"
fi
}
get_all_agents_with_paths() {
local tmpfile
tmpfile=$(mktemp)
find "$SKILLS_DIR" -path "*/agents/*.md" -not -name "AGENTS.md" \
-not -path "*/node_modules/*" -not -path "*/.git/*" 2>/dev/null >> "$tmpfile" || true
while IFS= read -r f; do
[ -f "$f" ] || continue
if _is_inside_nested_git_repo "$f"; then
continue
fi
local name
name=$(basename "$f" .md)
echo "${name}:${f}"
done < "$tmpfile" | sort -u -t: -k1,1
rm -f "$tmpfile"
}
get_all_agents() {
get_all_agents_with_paths | cut -d: -f1
}
find_agent_file() {
local agent_name="$1"
local result
result=$(get_all_agents_with_paths | grep "^${agent_name}:" | head -1 | cut -d: -f2-)
if [ -n "$result" ]; then
echo "$result"
else
echo "$SKILLS_DIR/agents/${agent_name}.md"
fi
}
get_all_teams_with_paths() {
local tmpfile
tmpfile=$(mktemp)
find "$SKILLS_DIR" \
\( -name ".opencode" -o -name ".claude" -o -name ".claude-plugin" \
-o -name "node_modules" -o -name ".git" \) -prune -o \
-path "*/teams/*/AGENTS.md" -print 2>/dev/null > "$tmpfile" || true
if [ -d "$SKILLS_DIR/plugins-official" ]; then
find "$SKILLS_DIR/plugins-official" -maxdepth 2 -name "AGENTS.md" \
-not -path "*/.opencode/*" -not -path "*/.claude/*" \
-not -path "*/.claude-plugin/*" 2>/dev/null >> "$tmpfile" || true
fi
while IFS= read -r f; do
[ -f "$f" ] && echo "$(basename "$(dirname "$f")"):$f"
done < "$tmpfile" | sort -u -t: -k1,1
rm -f "$tmpfile"
}
get_all_teams() {
get_all_teams_with_paths | cut -d: -f1
}
find_team_file() {
local team_name="$1"
local result
result=$(get_all_teams_with_paths | grep "^${team_name}:" | head -1 | cut -d: -f2-)
if [ -n "$result" ]; then
echo "$result"
else
echo "$SKILLS_DIR/teams/$team_name/AGENTS.md"
fi
}
SKILL_VALIDATOR="$LIB_DIR/skill_validator.py"
_parse_jsonl() {
python3 - <<'PYEOF' "$1"
import json, sys
with open(sys.argv[1]) as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
level = obj.get("level", "")
rule = obj.get("rule", "")
msg = obj.get("msg", "")
print(f"{level}\t{rule}\t{msg}")
except json.JSONDecodeError:
pass
PYEOF
}
_run_validator() {
local item_name="$1"
local subcmd="$2"
local file="$3"
shift 3
local tmp
tmp=$(mktemp)
if ! python3 "$SKILL_VALIDATOR" "$subcmd" "$file" "$@" >"$tmp" 2>&1; then
print_fail "$item_name: validator invocation failed"
cat "$tmp" >&2
rm -f "$tmp"
return 1
fi
local had_error=0 err_count=0 warn_count=0
local parsed
parsed=$(_parse_jsonl "$tmp")
while IFS=$'\t' read -r level rule msg; do
[ -z "$level" ] && continue
case "$level" in
error)
had_error=1
err_count=$((err_count + 1))
print_error "${rule}: ${msg}"
;;
warn)
warn_count=$((warn_count + 1))
echo -e " ${YELLOW}[WARN]${NC} ${item_name}: ${rule}: ${msg}"
;;
*)
echo " ${line}"
;;
esac
done <<< "$parsed"
rm -f "$tmp"
if [ $had_error -ne 0 ]; then
print_fail "${item_name}: ${err_count} error(s), ${warn_count} warning(s)"
return 1
fi
if [ $warn_count -gt 0 ]; then
print_pass "${item_name}: valid (${warn_count} warning(s))"
else
print_pass "${item_name}: valid"
fi
return 0
}
validate_skill_structure() {
local skill_file="$1"
local skill_name
skill_name=$(basename "$(dirname "$skill_file")")
_run_validator "$skill_name" validate-skill "$skill_file" --subset=structure
}
validate_skill_content() {
local skill_file="$1"
local skill_name
skill_name=$(basename "$(dirname "$skill_file")")
_run_validator "$skill_name" validate-skill "$skill_file" --subset=content
}
validate_agent_structure() {
local agent_file="$1"
local agent_name
agent_name=$(basename "$agent_file" .md)
local skill_paths
skill_paths=$(get_all_skills_with_paths | cut -d: -f2-)
_run_validator "$agent_name" validate-agent "$agent_file" --subset=structure $skill_paths
}
validate_agent_content() {
local agent_file="$1"
local agent_name
agent_name=$(basename "$agent_file" .md)
local skill_paths
skill_paths=$(get_all_skills_with_paths | cut -d: -f2-)
_run_validator "$agent_name" validate-agent "$agent_file" --subset=content $skill_paths
}
validate_team_structure() {
local team_file="$1"
local team_name
team_name=$(basename "$(dirname "$team_file")")
local team_dir
team_dir=$(dirname "$team_file")
local skill_paths
skill_paths=$(get_all_skills_with_paths | cut -d: -f2-)
local local_skills
local_skills=$(find "$team_dir" -name "SKILL.md" 2>/dev/null || true)
_run_validator "$team_name" validate-team "$team_file" --subset=structure $skill_paths $local_skills
}
validate_team_content() {
local team_file="$1"
local team_name
team_name=$(basename "$(dirname "$team_file")")
local team_dir
team_dir=$(dirname "$team_file")
local skill_paths
skill_paths=$(get_all_skills_with_paths | cut -d: -f2-)
local local_skills
local_skills=$(find "$team_dir" -name "SKILL.md" 2>/dev/null || true)
_run_validator "$team_name" validate-team "$team_file" --subset=content $skill_paths $local_skills
}
validate_global_uniqueness() {
local kind="$1"
local paths
case "$kind" in
skill) paths=$(get_all_skills_with_paths | cut -d: -f2-) ;;
agent) paths=$(get_all_agents_with_paths | cut -d: -f2-) ;;
team) paths=$(get_all_teams_with_paths | cut -d: -f2-) ;;
*) print_error "validate_global_uniqueness: unknown kind '$kind'"; return 1 ;;
esac
if [ -z "$paths" ]; then
return 0
fi
local tmp
tmp=$(mktemp)
python3 "$SKILL_VALIDATOR" check-uniqueness "$kind" $paths >"$tmp"
local had_error=0
local parsed
parsed=$(_parse_jsonl "$tmp")
while IFS=$'\t' read -r level rule msg; do
[ -z "$level" ] && continue
if [ "$level" = "error" ]; then
had_error=1
print_error "${rule}: ${msg}"
fi
done <<< "$parsed"
rm -f "$tmp"
if [ $had_error -ne 0 ]; then
print_fail "${kind}: uniqueness check failed"
return 1
fi
print_pass "${kind}: uniqueness OK"
return 0
}
find_recent_session() {
local minutes_old="${1:-30}"
local session_dir="$HOME/.claude/projects"
find "$session_dir" -name "*.jsonl" -type f -mmin -"$minutes_old" 2>/dev/null | sort -r | head -1
}
verify_skill_invoked() {
local session_file="$1"
local skill_name="$2"
if [ ! -f "$session_file" ]; then
print_fail "Session file not found: $session_file"
return 1
fi
local skill_pattern='"skill":"([^"]*:)?'"${skill_name}"'"'
if grep -q '"name":"Skill"' "$session_file" && grep -qE "$skill_pattern" "$session_file"; then
print_pass "Skill '$skill_name' was invoked"
return 0
else
print_fail "Skill '$skill_name' was NOT invoked"
return 1
fi
}
verify_agent_dispatched() {
local session_file="$1"
local agent_name="$2"
if [ ! -f "$session_file" ]; then
print_fail "Session file not found: $session_file"
return 1
fi
if grep -qE '"subagent_type":"'"$agent_name"'"|"name":"'"$agent_name"'"' "$session_file"; then
print_pass "Agent '$agent_name' was dispatched"
return 0
else
print_fail "Agent '$agent_name' was NOT dispatched"
return 1
fi
}
count_tool_invocations() {
local session_file="$1"
local tool_name="$2"
if [ ! -f "$session_file" ]; then
echo "0"
return
fi
local count
count=$(grep -c "\"name\":\"$tool_name\"" "$session_file" 2>/dev/null | tr -cd '0-9' || true)
echo "${count:-0}"
}
check_premature_action() {
local session_file="$1"
local skill_name="$2"
if [ ! -f "$session_file" ]; then
print_skip "Session file not found"
return 0
fi
local first_skill_line=$(grep -n '"name":"Skill"' "$session_file" 2>/dev/null | head -1 | cut -d: -f1 || true)
if [ -z "$first_skill_line" ]; then
print_warn "No Skill invocation found"
return 0
fi
local premature_tools=$(head -n "$first_skill_line" "$session_file" 2>/dev/null | \
grep '"type":"tool_use"' 2>/dev/null | \
grep -v '"name":"Skill"' 2>/dev/null | \
grep -v '"name":"TodoWrite"' 2>/dev/null | \
grep -v '"name":"TaskOutput"' 2>/dev/null || true)
if [ -n "$premature_tools" ]; then
print_warn "Tools invoked BEFORE Skill '$skill_name':"
echo "$premature_tools" | head -3 | sed 's/^/ /' || true
return 1
else
print_pass "No premature tool invocations detected"
return 0
fi
}
get_triggered_skills() {
local session_file="$1"
if [ ! -f "$session_file" ]; then
return
fi
grep -o '"skill":"[^"]*"' "$session_file" 2>/dev/null | sort -u | sed 's/"skill":"//;s/"$//' || true
}
analyze_tool_chain() {
local session_file="$1"
if [ ! -f "$session_file" ]; then
echo "No session file"
return 1
fi
echo "=== Tool Call Chain ==="
local line_num=0
local tool_count=0
local skill_invoked=false
local skill_line=0
while IFS= read -r line; do
((line_num++)) || true
if echo "$line" | grep -q '"name":"Skill"'; then
skill_invoked=true
skill_line=$line_num
local skill_name=$(echo "$line" | grep -o '"skill":"[^"]*"' | sed 's/"skill":"//;s/"$//' | head -1)
echo " [SKILL] Line $line_num: $skill_name"
((tool_count++)) || true
fi
if echo "$line" | grep -q '"type":"tool_use"'; then
local tool_name=$(echo "$line" | grep -o '"name":"[^"]*"' | head -1 | sed 's/"name":"//;s/"$//')
if [ -n "$tool_name" ] && [ "$tool_name" != "Skill" ]; then
local marker="TOOL"
if $skill_invoked && [ $line_num -lt $skill_line ]; then
marker="PREMATURE"
fi
echo " [$marker] Line $line_num: $tool_name"
((tool_count++)) || true
fi
fi
done < "$session_file"
echo ""
echo "Total tool calls: $tool_count"
}
analyze_cost_breakdown() {
local session_file="$1"
if [ ! -f "$session_file" ]; then
echo "[ERROR] Session file not found"
return 1
fi
if ! command -v jq &> /dev/null; then
echo "[ERROR] jq is required for cost analysis"
return 1
fi
echo "=== Cost Breakdown Analysis ==="
echo ""
local main_usage=$(jq -s '[.[] | select(.type == "result")] | last | .usage // {}' "$session_file" 2>/dev/null)
if [ -n "$main_usage" ] && [ "$main_usage" != "{}" ]; then
echo "Main Session:"
echo "$main_usage" | jq -r '" Input: \(.input_tokens // 0), Output: \(.output_tokens // 0), Cache Read: \(.cache_read_input_tokens // 0)"' 2>/dev/null || echo " (unable to parse)"
echo ""
fi
local subagent_count=$(jq -s '[.[] | select(.type == "user") | .toolUseResult? | select(. != null) | select(.agentId != null)] | length' "$session_file" 2>/dev/null || echo "0")
if [ "$subagent_count" -gt 0 ]; then
echo "Subagents ($subagent_count total):"
jq -s '[.[] | select(.type == "user") | .toolUseResult? | select(. != null) | select(.agentId != null)] |
.[] | {agentId, input: .usage.input_tokens // 0, output: .usage.output_tokens // 0, prompt: .prompt[0:50]}' \
"$session_file" 2>/dev/null | \
jq -r '" \(.agentId): Input=\(.input), Output=\(.output) - \(.prompt)..."'
echo ""
fi
local total_input=$(jq -s '[.[] | .usage.input_tokens // 0, .toolUseResult?.usage?.input_tokens // 0] | add' "$session_file" 2>/dev/null || echo "0")
local total_output=$(jq -s '[.[] | .usage.output_tokens // 0, .toolUseResult?.usage?.output_tokens // 0] | add' "$session_file" 2>/dev/null || echo "0")
local total_cache=$(jq -s '[.[] | .usage.cache_read_input_tokens // 0] | add' "$session_file" 2>/dev/null || echo "0")
echo "Totals:"
echo " Input tokens: $total_input"
echo " Output tokens: $total_output"
echo " Cache read tokens: $total_cache"
echo ""
}
VERSION_STATE_DIR="$TESTS_DIR/.version-state"
get_team_plugin_json() {
local team_name="$1"
local plugin_path="$SKILLS_DIR/plugins-official/$team_name/.claude-plugin/plugin.json"
if [ -f "$plugin_path" ]; then
echo "$plugin_path"
else
echo ""
fi
}
extract_plugin_version() {
local plugin_json="$1"
if [ -f "$plugin_json" ]; then
grep '"version"' "$plugin_json" | head -1 | sed 's/.*"version":[[:space:]]*"\([^"]*\)".*/\1/'
fi
}
_extract_plugin_json_array() {
local plugin_json="$1"
local field_name="$2"
if [ -f "$plugin_json" ]; then
local in_array=false
while IFS= read -r line; do
if echo "$line" | grep -q "\"${field_name}\""; then
in_array=true
continue
fi
if $in_array; then
if echo "$line" | grep -q '^\s*\]'; then
break
fi
local item=$(echo "$line" | sed 's/.*"\(\.[^"]*\)".*/\1/' | tr -d '[:space:]')
if [ -n "$item" ] && [ "$item" != "$line" ]; then
echo "$item"
fi
fi
done < "$plugin_json"
fi
}
extract_plugin_skills() {
_extract_plugin_json_array "$1" "skills"
}
extract_plugin_agents() {
_extract_plugin_json_array "$1" "agents"
}
validate_semver() {
local version="$1"
if echo "$version" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$'; then
return 0
else
return 1
fi
}
compute_file_hash() {
local file="$1"
if [ -f "$file" ]; then
if command -v sha256sum &>/dev/null; then
sha256sum "$file" 2>/dev/null | cut -c1-16 || echo "MISSING"
elif command -v shasum &>/dev/null; then
shasum -a 256 "$file" 2>/dev/null | cut -c1-16 || echo "MISSING"
else
echo "MISSING"
fi
else
echo "MISSING"
fi
}
recommend_version_bump() {
local current="$1"
local skill_changed="$2"
local agent_changed="$3"
if ! validate_semver "$current"; then
echo "INVALID"
return 1
fi
local major minor patch
IFS='.' read -r major minor patch <<< "$current"
if [ "$skill_changed" = "true" ] || [ "$agent_changed" = "true" ]; then
patch=$((patch + 1))
fi
echo "${major}.${minor}.${patch}"
}
semver_compare() {
local v1="$1"
local v2="$2"
local major1 minor1 patch1 major2 minor2 patch2
IFS='.' read -r major1 minor1 patch1 <<< "$v1"
IFS='.' read -r major2 minor2 patch2 <<< "$v2"
if [ "$major1" -gt "$major2" ] 2>/dev/null; then echo "1"; return; fi
if [ "$major1" -lt "$major2" ] 2>/dev/null; then echo "-1"; return; fi
if [ "$minor1" -gt "$minor2" ] 2>/dev/null; then echo "1"; return; fi
if [ "$minor1" -lt "$minor2" ] 2>/dev/null; then echo "-1"; return; fi
if [ "$patch1" -gt "$patch2" ] 2>/dev/null; then echo "1"; return; fi
if [ "$patch1" -lt "$patch2" ] 2>/dev/null; then echo "-1"; return; fi
echo "0"
}
extract_token_usage() {
local session_file="$1"
if [ ! -f "$session_file" ]; then
echo "{}"
return 1
fi
if command -v jq &> /dev/null; then
jq -s '[.[] | select(.type == "result")] | last | .usage // {}' "$session_file" 2>/dev/null || echo "{}"
else
echo "{}"
fi
}
init_test_tracking() {
TEST_PASSED=0
TEST_FAILED=0
TEST_SKIPPED=0
TEST_START_TIME=$(date +%s)
}
record_test() {
local result="$1"
local test_name="$2"
local duration="${3:-0}"
case "$result" in
pass|PASS) ((TEST_PASSED++)) ;;
fail|FAIL) ((TEST_FAILED++)) ;;
skip|SKIP) ((TEST_SKIPPED++)) ;;
esac
}
print_test_summary() {
local end_time=$(date +%s)
local duration=$((end_time - TEST_START_TIME))
echo ""
echo "========================================"
echo -e " ${BOLD}Test Results Summary${NC}"
echo "========================================"
echo ""
echo -e " ${GREEN}Passed:${NC} $TEST_PASSED"
echo -e " ${RED}Failed:${NC} $TEST_FAILED"
echo -e " ${YELLOW}Skipped:${NC} $TEST_SKIPPED"
echo " Duration: ${duration}s"
echo ""
if [ "$TEST_FAILED" -gt 0 ]; then
print_status_failed
return 1
else
print_status_passed
return 0
fi
}
output_test_json() {
local end_time=$(date +%s)
local duration=$((end_time - TEST_START_TIME))
cat <<EOF
{
"status": "$([ "$TEST_FAILED" -gt 0 ] && echo "failed" || echo "passed")",
"passed": $TEST_PASSED,
"failed": $TEST_FAILED,
"skipped": $TEST_SKIPPED,
"duration": $duration,
"timestamp": "$(date -Iseconds)"
}
EOF
}
print_section() {
local name="$1"
echo ""
echo "----------------------------------------"
echo " $name"
echo "----------------------------------------"
}
print_test_banner() {
local name="$1"
local description="${2:-}"
echo ""
echo "========================================"
echo " $name"
echo "========================================"
[ -n "$description" ] && echo "" && echo "$description"
echo ""
}
export -f is_platform_available
export -f get_platform_version
export -f create_test_project
export -f cleanup_test_project
export -f run_claude
export -f assert_order
export -f assert_file_exists
export -f check_file_links
export -f run_behavior_test
export -f analyze_premature_actions
export -f get_all_skills
export -f get_all_skills_with_paths
export -f find_skill_file
export -f get_all_agents
export -f get_all_agents_with_paths
export -f find_agent_file
export -f get_all_teams
export -f get_all_teams_with_paths
export -f find_team_file
export -f is_incremental_mode
export -f should_test_skill
export -f should_test_agent
export -f should_test_team
export -f get_skills_to_test
export -f get_agents_to_test
export -f get_teams_to_test
export -f validate_skill_structure
export -f validate_skill_content
export -f validate_agent_structure
export -f validate_agent_content
export -f validate_team_structure
export -f validate_team_content
export -f validate_global_uniqueness
export -f find_recent_session
export -f verify_skill_invoked
export -f verify_agent_dispatched
export -f count_tool_invocations
export -f check_premature_action
export -f get_triggered_skills
export -f analyze_tool_chain
export -f analyze_cost_breakdown
export -f extract_token_usage
export -f init_test_tracking
export -f record_test
export -f print_test_summary
export -f output_test_json
export -f print_section
export -f print_test_banner
export -f setup_colors
export -f enable_colors
export -f disable_colors
export -f print_compact_result
export -f print_pass
export -f print_fail
export -f print_skip
export -f print_info
export -f print_warn
export -f print_error
export -f print_section_header
export -f print_status_passed
export -f print_status_failed
export -f get_team_plugin_json
export -f extract_plugin_version
export -f extract_plugin_skills
export -f extract_plugin_agents
export -f validate_semver
export -f compute_file_hash
export -f recommend_version_bump
export -f semver_compare
export LIB_DIR TESTS_DIR SKILLS_DIR
if [ -z "${NO_COLOR:-}" ]; then
FORCE_COLOR=1
setup_colors
fi