CI: don't kill JuiceFS mounts when restarting the TiDB playground

.github/scripts/command/dump_load.sh

@@ -3,17 +3,36 @@ source .github/scripts/common/common.sh
 
 [[ -z "$META" ]] && META=sqlite3
 source .github/scripts/start_meta_engine.sh
-start_meta_engine $META
 META_URL=$(get_meta_url $META)
 META_URL2=$(get_meta_url2 $META)
+# Drop mounts left behind by a previous step *before* the metadata engine is
+# replaced. start_meta_engine tears the old engine down, and a mount that
+# survives it keeps restarting against a wiped database, which wedges the next
+# umount and hangs the job (see #7458). The old engine is still up here, so the
+# sessions can be looked up and the mount processes reaped cleanly.
+umount_jfs /jfs "$META_URL"
+umount_jfs /jfs2 "$META_URL2"
+umount_jfs /jfs2 sqlite3://test2.db
+start_meta_engine $META
 [[ -z "$SEED" ]] && SEED=$(date +%s)
 HEARTBEAT_INTERVAL=2
 DIR_QUOTA_FLUSH_INTERVAL=4
 # [[ -z "$SEED" ]] && SEED=1711594639
 [[ -z "$BINARY" ]] && BINARY=false
 [[ -z "$FAST" ]] && FAST=false
 
-trap "echo random seed is $SEED" EXIT
+# Leave no mount behind for the next workflow step: it would outlive the
+# metadata engine that start_meta_engine replaces. Each umount runs in a
+# subshell so that a failing cleanup cannot mask the real exit status.
+cleanup_on_exit(){
+    local exit_code=$?
+    echo "random seed is $SEED"
+    ( umount_jfs /jfs2 "$META_URL2" ) || true
+    ( umount_jfs /jfs2 sqlite3://test2.db ) || true
+    ( umount_jfs /jfs "$META_URL" ) || true
+    exit $exit_code
+}
+trap cleanup_on_exit EXIT
 
 if ! docker ps | grep -q minio; then
     docker run -d -p 9000:9000 --name minio \
@@ -252,6 +271,7 @@ get_load_option(){
 
 prepare_test(){
     umount_jfs /jfs $META_URL
+    umount_jfs /jfs2 $META_URL2
     umount_jfs /jfs2 sqlite3://test2.db
     python3 .github/scripts/flush_meta.py $META_URL
     rm test2.db -rf 

.github/scripts/common/common.sh

@@ -54,34 +54,88 @@ prepare_test() {
     esac
 }
 
+is_mounted() {
+    local mp=$1
+    case "$PLATFORM" in
+        mac)    mount | grep -qF " on ${mp} " ;;
+        linux)  awk -v mp="$mp" '$2 == mp { found = 1 } END { exit !found }' /proc/self/mounts ;;
+        *)      return 1 ;;
+    esac
+}
+
+# `timeout` is GNU coreutils and is absent from a stock macOS runner; fall back
+# to gtimeout, then to a watchdog. perl's alarm+exec trick is deliberately not
+# used: the Go runtime installs its own SIGALRM handler and silently ignores the
+# signal, so it would never interrupt ./juicefs.
+run_with_timeout() {
+    local secs=$1
+    shift
+    if command -v timeout >/dev/null 2>&1; then
+        timeout "$secs" "$@"
+        return $?
+    fi
+    if command -v gtimeout >/dev/null 2>&1; then
+        gtimeout "$secs" "$@"
+        return $?
+    fi
+    "$@" &
+    local cmd_pid=$!
+    # stdout is redirected so the watchdog never holds the caller's pipe open.
+    ( sleep "$secs"; kill -TERM "$cmd_pid" && sleep 2 && kill -KILL "$cmd_pid"; ) >/dev/null 2>&1 &
+    local watchdog_pid=$!
+    local rc=0
+    wait "$cmd_pid" 2>/dev/null || rc=$?
+    kill "$watchdog_pid" 2>/dev/null
+    wait "$watchdog_pid" 2>/dev/null
+    return $rc
+}
+
 umount_jfs() {
     local mp=$1
     local meta_url=$2
     [[ -z "$mp" ]] && echo "mount point is empty" && exit 1
     [[ -z "$meta_url" ]] && echo "meta url is empty" && exit 1
 
     echo "umount_jfs $mp $meta_url"
-    [[ ! -f "$mp/.config" ]] && return
+    # Probe the mount table rather than a file inside the mount: when the mount
+    # process is gone but its FUSE connection is still alive, any syscall
+    # against $mp blocks forever and hangs the whole job (see #7458).
+    is_mounted "$mp" || return 0
 
-    ls -l "$mp/.config"
     local status_log="status.log"
-    ./juicefs status --log-level error "$meta_url" 2>/dev/null | grep -v '^\[xorm\]' | tee "$status_log"
+    local status_json=""
+    if ! status_json=$(run_with_timeout 60 ./juicefs status --log-level error "$meta_url" 2>/dev/null); then
+        echo "cannot get status from $meta_url, umount $mp anyway"
+    fi
+    printf '%s\n' "$status_json" | grep -v '^\[xorm\]' > "$status_log" || true
+    cat "$status_log"
 
     local pids
-    pids=$(jq --arg mp "$mp" '.Sessions[] | select(.MountPoint == $mp) | .ProcessID' "$status_log")
-    [[ -z "$pids" ]] && cat "$status_log" && echo "pid is empty" && return
+    pids=$(jq --arg mp "$mp" '.Sessions[] | select(.MountPoint == $mp) | .ProcessID' "$status_log" 2>/dev/null) || pids=""
+    if [[ -z "$pids" ]]; then
+        # $meta_url does not know this mount: it may have been mounted from
+        # another volume (/jfs2 is used for both $META_URL2 and test2.db) or the
+        # metadata may already be gone. Fall back to the process table, which,
+        # unlike the mount point itself, can never block.
+        pids=$(ps -eo pid=,args= 2>/dev/null | awk -v mp="$mp" \
+            '/juicefs/ { ismount = 0; hasmp = 0
+                 for (i = 2; i <= NF; i++) { if ($i == "mount") ismount = 1; if ($i == mp) hasmp = 1 }
+                 if (ismount && hasmp) print $1 }')
+    fi
 
-    echo "umount is $mp, pids are $pids"
+    echo "umount is $mp, pids are ${pids:-<none>}"
 
-    for pid in $pids; do
+    # There may be multiple mounts stacked on the same path (for example, ACL
+    # tests remount after changing the volume configuration). Unmount once per
+    # session, or at least once when session discovery failed.
+    local pid
+    for pid in ${pids:-0}; do
         case "$PLATFORM" in
             mac)
-                if mount | grep -q "$mp"; then
-                    diskutil unmount "$mp" || umount "$mp"
-                fi
+                diskutil unmount "$mp" || diskutil unmount force "$mp" || umount -f "$mp" || true
                 ;;
             linux)
-                umount -l "$mp"
+                umount -l "$mp" || true
                 ;;
         esac
     done
@@ -175,5 +229,5 @@ ensure_directory() {
 init_platform
 
 # Make functions available to subprocesses
-export -f cleanup_test_mounts prepare_test umount_jfs wait_mount_process_killed compare_md5sum wait_command_success ensure_directory
+export -f cleanup_test_mounts prepare_test is_mounted run_with_timeout umount_jfs wait_mount_process_killed compare_md5sum wait_command_success ensure_directory
 export PLATFORM META_URL

.github/scripts/start_meta_engine.sh

@@ -22,18 +22,55 @@ retry() {
     done
 }
 
+# Tell whether a pid belongs to a tiup playground: either its executable lives
+# under the tiup home, or its name matches a known playground component. Used to
+# keep kill_tiup_playground from signalling unrelated processes.
+is_tiup_component() {
+    local pid=$1 tiup_home=$2 exe comm
+    exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true)
+    if [ -n "$exe" ] && [ -n "$tiup_home" ]; then
+        case "$exe" in
+            "$tiup_home"/*) return 0 ;;
+        esac
+    fi
+    # /proc/<pid>/exe is unavailable on macOS and may be unreadable for a
+    # foreign owner; fall back to the process name, which ps truncates to 15
+    # characters, hence the prefix patterns.
+    comm=$(ps -p "$pid" -o comm= 2>/dev/null | tr -d '[:space:]')
+    comm=${comm##*/}
+    case "$comm" in
+        tiup*|playground|pd-server|tikv-server|tidb-server|tiflash*|tiproxy*|prometheus*|grafana*|ng-monitoring*) return 0 ;;
+    esac
+    return 1
+}
+
 # Reap a tiup playground and every process it spawned. Killing only the tiup
 # parent orphans its pd-server/tikv-server/tidb-server children, which keep
 # holding the cluster ports and poison the next retry attempt with
 # "mismatch cluster id" / "connection refused" errors. Clean up the data dir
 # and kill whoever still holds the given ports (by their actual PIDs).
+#
+# Two filters keep this from hitting innocent bystanders:
+#   * -sTCP:LISTEN, because `lsof -i tcp:<port>` also reports *clients* of that
+#     port. A running JuiceFS mount keeps connections to TiDB on 4000, so an
+#     unfiltered kill takes the mount down together with the server, leaving a
+#     wedged mount point behind (see #7458).
+#   * is_tiup_component, so that even a listener is only killed when it really
+#     belongs to the playground.
 kill_tiup_playground() {
     local tiup_bin=$1
     shift
-    local port pids
+    local tiup_home
+    tiup_home=$(dirname "$(dirname "$tiup_bin")")
+    local port pid
     for port in "$@"; do
-        pids=$(lsof -ti tcp:"$port" 2>/dev/null || true)
-        [ -n "$pids" ] && kill -9 $pids 2>/dev/null || true
+        for pid in $(lsof -ti tcp:"$port" -sTCP:LISTEN 2>/dev/null || true); do
+            if is_tiup_component "$pid" "$tiup_home"; then
+                kill -9 "$pid" 2>/dev/null || true
+            else
+                echo "kill_tiup_playground: keep pid $pid ($(ps -p "$pid" -o comm= 2>/dev/null | tr -d '[:space:]')) listening on $port, not a tiup component"
+            fi
+        done
     done
     "$tiup_bin" clean --all >/dev/null 2>&1 || true
     return 0
@@ -121,6 +158,13 @@ install_tikv(){
 }
 
 install_tidb(){
+    # Reuse a healthy playground instead of rebuilding it: tearing the cluster
+    # down on every workflow step wipes the metadata of volumes that earlier
+    # steps left mounted (see #7458). install_tikv has the same guard.
+    if lsof -i:4000 -sTCP:LISTEN && pgrep pd-server && timeout 10 mysql -h127.0.0.1 -P4000 -uroot -e "select version();"; then
+        echo "TiDB is already running and healthy"
+        return 0
+    fi
     user=$(whoami)
     echo user is $user
     if [[ "$user" == "root" ]]; then