~ salmonella-environment-setup (master) ac8cf03665f43e21a8b4294e30c79277ed32a920

commit ac8cf03665f43e21a8b4294e30c79277ed32a920
Author:     Mario Domenech Goulart <mario@parenteses.org>
AuthorDate: Sat Apr 17 20:49:26 2021 +0200
Commit:     Mario Domenech Goulart <mario@parenteses.org>
CommitDate: Sat Apr 17 20:49:29 2021 +0200

    run-salmonella.sh: proper locking to prevent multiple instances
    
    This change implements locking to prevent multiple simultaneous
    executions of run-salmonella.sh.
    
    Locks are created in /tmp.  THe lock is a directory (salmonella-lock)
    with a `pid' file which contains the process identifier of the running
    process.
    
    For debugging, `/tmp/salmonella-$$.log` files are created.  They
    contain information about locking events and are cleaned up upon
    termination of run-salmonella.sh.

diff --git a/run-salmonella.sh b/run-salmonella.sh
index 8a7aaeb..a3cf665 100755
--- a/run-salmonella.sh
+++ b/run-salmonella.sh
@@ -5,7 +5,23 @@ SRC_CONF_DIR=$ROOT_DIR/conf
 HOOKS_DIR=$ROOT_DIR/hooks
 LOCAL_CONF_DIR=$HOME/.salmonella-run-publish
 WORK_DIR=$HOME/salmonella/build
-LOG_DIR=$HOME/salmonella
+
+LOCK_DIR=/tmp/salmonella-lock
+LOCK_PID=/tmp/salmonella-lock/pid
+
+# Temporary log file.  It won't be published as part of job artifacts.
+# It is mostly to log events related to locking.
+TMP_LOG_FILE=/tmp/salmonella-$$.log
+
+
+cleanup() {
+    rm -rf "$LOCK_DIR"
+    rm -f "$TMP_LOG_FILE"
+}
+
+
+trap cleanup EXIT INT TERM
+
 
 usage() {
     cat <<EOF
@@ -71,22 +87,19 @@ SETTINGS_FILES="\
 run_hooks() {
     # Call hooks scripts, passing $OS, $ARCH, CHICKEN_4_PREFIX and
     # CHICKEN_5_PREFIX as arguments
-    local hooks_logfile
     local hook
     local hook_dir
 
-    hooks_logfile=${LOG_DIR}/hooks.log
-
-    mkdir -p "$LOG_DIR"
-    rm -f "$hooks_logfile"
+    mkdir -p "$WORK_DIR"
+    hooks_log_file=${WORK_DIR}/hooks.log
 
     for hook_dir in "${HOOKS_DIR}/common" "${HOOKS_DIR}/${OS}" "${HOOKS_DIR}/${OS}/${ARCH}"
     do
         if [ -d "$hook_dir" ]; then
             for hook in "${hook_dir}/"*; do
                 if [ -x "$hook" ]; then
-                    echo "=== Running hook $hook ===" >>"$hooks_logfile"
-                    "$hook" "$OS" "$ARCH" "$CHICKEN_4_PREFIX" "$CHICKEN_5_PREFIX" >>"$hooks_logfile" 2>&1
+                    echo "=== Running hook $hook ===" > "$hooks_log_file"
+                    "$hook" "$OS" "$ARCH" "$CHICKEN_4_PREFIX" "$CHICKEN_5_PREFIX" >>"$hooks_log_file" 2>&1
                 fi
             done
         fi
@@ -94,50 +107,35 @@ run_hooks() {
 }
 
 
-main() {
+run_salmonella() {
+    local confs
     confs=$@
 
-    # Check if all conf files exist
-    for conf in $confs; do
-        if [ ! -e "$SRC_CONF_DIR/${conf}.conf" ]; then
-            echo "No configuration file found for '$conf'.  Aborting." >&2
-            exit 1
-        fi
-    done
-
-    mkdir -p "$LOG_DIR" "$WORK_DIR"
-    local watchdog_logfile
-    watchdog_logfile="$LOG_DIR/watchdog.log"
+    mkdir -p "$WORK_DIR"
 
     # Run salmonella from $WORK_DIR
     cd "$WORK_DIR"
 
-    ### Prevent two salmonellas from running simultaneously
-    while ps ax | grep '[s]almonella-run-publish'; do
-        echo "[`date`] Salmonella still running" >> "$watchdog_logfile"
-        # OpenBSD's sleep doesn't suffixes like `h' (hours) or `m' (minutes)
-        sleep 3600
-    done
-
-    # Remove leftovers from previous executions
-    rm -rf salmonella-run-publish
-
-    # Create the directory for the log file
-    mkdir -p "$RUN_SALMONELLA_TMP_DIR"
-
-    # Load settings files
-    local settings_file
-    for settings_file in $SETTINGS_FILES; do
-        if [ -e "$settings_file" ]; then
-            info "Loading $settings_file"
-            . "$settings_file"
-        fi
-    done
-
     local salmonella_run_publish
     salmonella_run_publish="$CHICKEN_TESTS_PREFIX/bin/salmonella-run-publish"
 
     for conf in $confs; do
+
+        # Remove leftovers from previous executions.  Once
+        # `$RUN_SALMONELLA_TMP_DIR` is created it is safe to use
+        # `info'.
+        rm -rf "$RUN_SALMONELLA_TMP_DIR"
+        mkdir -p "$RUN_SALMONELLA_TMP_DIR"
+
+        # Load settings files
+        local settings_file
+        for settings_file in $SETTINGS_FILES; do
+            if [ -e "$settings_file" ]; then
+                info "Loading $settings_file"
+                . "$settings_file"
+            fi
+        done
+
         # The inline egg writes some stuff to the home dir
         rm -rf ~/.chicken-inline
 
@@ -157,10 +155,46 @@ main() {
             info "Loading $conf_path"
             "$salmonella_run_publish" "$conf_path" >>"$log_file" 2>&1
         fi
-        [ -d salmonella-run-publish ] && mv salmonella-run-publish "salmonella-$conf"
+        [ -d "$RUN_SALMONELLA_TMP_DIR" ] &&
+            mv "$RUN_SALMONELLA_TMP_DIR" "salmonella-$conf"
     done
 }
 
+
+main() {
+    # Check if all conf files exist
+    local conf
+    for conf in $@; do
+        if [ ! -e "$SRC_CONF_DIR/${conf}.conf" ]; then
+            echo "No configuration file found for '$conf'.  Aborting." >&2
+            exit 1
+        fi
+    done
+
+    while true; do
+        if mkdir "$LOCK_DIR" 2>/dev/null; then
+            echo $$ > "$LOCK_PID"
+            echo "[$(date)] $$ acquired lock" >> "$TMP_LOG_FILE"
+            run_salmonella "$@"
+            cleanup
+            exit
+        else
+            # Check if the lock is stale
+            local pid
+            pid=$(cat "$LOCK_PID")
+            if kill -0 "$pid" 2>/dev/null; then
+                echo "[$(date)] Waiting for $pid" >> "$TMP_LOG_FILE"
+                sleep 60
+            else
+                # Lock is stale.  Cleanup and continue.
+                echo "[$(date)] Cleaning up stale lock (pid=$pid)" >> "$TMP_LOG_FILE"
+                cleanup
+            fi
+        fi
+    done
+}
+
+
 if [ "$1" = "-h" ] || [ "$1" = "-help" ] || [ "$1" = "--help" ]; then
     usage
     exit 0