49 lines
1.3 KiB
Bash
Executable File
49 lines
1.3 KiB
Bash
Executable File
#!/bin/sh
|
|
# Run every auto category and write out/report.md with the matrix.
|
|
# Env: QA_EMACS (see run.sh).
|
|
|
|
set -u
|
|
root=$(cd "$(dirname "$0")/.." && pwd)
|
|
runner=$root/runner
|
|
mkdir -p "$root/out"
|
|
report=$root/out/report.md
|
|
|
|
pass=0; fail=0; manual=0; error=0
|
|
|
|
{
|
|
echo "# emacs-gpu visual QA report"
|
|
echo
|
|
echo "- Date: $(date '+%Y-%m-%d %H:%M')"
|
|
echo "- Host: $(uname -sm)"
|
|
echo "- Emacs: ${QA_EMACS:-emacs}"
|
|
echo
|
|
echo "| Category | Result | Diff % | Threshold % |"
|
|
echo "|---|---|---|---|"
|
|
} > "$report"
|
|
|
|
grep -v '^#' "$runner/categories.conf" | while IFS='|' read -r dir mode settle thresh action; do
|
|
[ -n "$dir" ] || continue
|
|
sh "$runner/run.sh" "$dir"
|
|
code=$?
|
|
pct="-"
|
|
[ -f "$root/out/$dir/result.json" ] && \
|
|
pct=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("blob_pct","-"))' \
|
|
"$root/out/$dir/result.json" 2>/dev/null || echo "-")
|
|
case $code in
|
|
0) res="PASS" ;;
|
|
1) res="**FAIL**" ;;
|
|
2) res="manual" ; thresh="-" ;;
|
|
*) res="**ERROR**" ;;
|
|
esac
|
|
echo "| $dir | $res | $pct | $thresh |" >> "$report"
|
|
done
|
|
|
|
{
|
|
echo
|
|
echo "Failing categories have a \`result-triptych.png\` (vanilla / gpu / diff heatmap) under \`out/<category>/\`."
|
|
} >> "$report"
|
|
|
|
echo
|
|
echo "Report: $report"
|
|
cat "$report"
|