58 lines
1.9 KiB
Bash
Executable File
58 lines
1.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# Run one QA category: capture GPU and vanilla renders, diff them.
|
|
#
|
|
# Usage: runner/run.sh NN-category
|
|
# Env: QA_EMACS emacs binary (default: emacs). On macOS it MUST be the
|
|
# installed signed app binary, e.g.
|
|
# "/Applications/Emacs GPU.app/Contents/MacOS/Emacs"
|
|
# (Screen Recording permission belongs to the bundle).
|
|
#
|
|
# Exit: 0 PASS, 1 FAIL, 2 skipped (manual), 3 error.
|
|
|
|
set -u
|
|
|
|
cat_dir=$1
|
|
root=$(cd "$(dirname "$0")/.." && pwd)
|
|
runner=$root/runner
|
|
out=$root/out/$cat_dir
|
|
emacs_bin=${QA_EMACS:-emacs}
|
|
|
|
line=$(grep "^$cat_dir|" "$runner/categories.conf") || {
|
|
echo "unknown category: $cat_dir" >&2; exit 3; }
|
|
mode=$(echo "$line" | cut -d'|' -f2)
|
|
settle=$(echo "$line" | cut -d'|' -f3)
|
|
thresh=$(echo "$line" | cut -d'|' -f4)
|
|
action=$(echo "$line" | cut -d'|' -f5)
|
|
|
|
if [ "$mode" != "auto" ]; then
|
|
echo "$cat_dir: MANUAL (layer 2, see its README.org)"
|
|
exit 2
|
|
fi
|
|
|
|
mkdir -p "$out"
|
|
rm -f "$out"/gpu.png "$out"/vanilla.png "$out"/*.error
|
|
|
|
capture() { # side, extra env via caller
|
|
QA_OUT=$out QA_SIDE=$1 QA_SETTLE=$settle \
|
|
QA_ACTION="$([ "$action" = "-" ] && echo "" || echo "$action")" \
|
|
"$emacs_bin" -Q -l "$runner/capture-lib.el" -l "$root/$cat_dir/test.el" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
capture gpu || { echo "$cat_dir: ERROR capturing gpu side" >&2; exit 3; }
|
|
EMACS_GPU_DISABLE=1 capture vanilla \
|
|
|| { echo "$cat_dir: ERROR capturing vanilla side" >&2; exit 3; }
|
|
|
|
result=$(python3 "$runner/compare.py" "$out/vanilla.png" "$out/gpu.png" \
|
|
"$out/result" --threshold="$thresh")
|
|
status=$?
|
|
echo "$result" > "$out/result.json"
|
|
pct=$(echo "$result" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("blob_pct","?"))')
|
|
|
|
if [ $status -eq 0 ]; then
|
|
echo "$cat_dir: PASS (diff $pct% <= $thresh%)"
|
|
else
|
|
echo "$cat_dir: FAIL (diff $pct% > $thresh%) -> $out/result-triptych.png"
|
|
fi
|
|
exit $status
|