dougal-software/bin/runner.sh

#!/bin/bash

# Maximum runtime in seconds before killing an overdue instance (e.g., 10 minutes)
MAX_RUNTIME_SECONDS=$((15 * 60))

DOUGAL_ROOT=${DOUGAL_ROOT:-$(dirname "$0")/..}

BINDIR="$DOUGAL_ROOT/bin"
VARDIR=${VARDIR:-$DOUGAL_ROOT/var}
LOCKFILE=${LOCKFILE:-$VARDIR/runner.lock}

[ -f ~/.profile ] && . ~/.profile

DOUGAL_LOG_TAG="dougal.runner[$$]"

# Only send output to the logger if we have the appropriate
# configuration set.
if [[ -n "$DOUGAL_LOG_TAG" && -n "$DOUGAL_LOG_FACILITY" ]]; then
	function _logger () {
		logger $*
	}
else
	function _logger () {
		: # This is the Bash null command
	}
fi

function tstamp () {
	date -u +%Y-%m-%dT%H:%M:%SZ
}

function prefix () {
	printf "\033[30;1m$(tstamp)\t"
}

function print_log () {
	printf "$(prefix)\033[36m%s\033[0m\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.info" "$*"
}

function print_info () {
	printf "$(prefix)\033[0m%s\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.debug" "$*"
}

function print_warning () {
	printf "$(prefix)\033[33;1m%s\033[0m\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.warning" "$*"
}

function print_error () {
	printf "$(prefix)\033[31m%s\033[0m\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.error" "$*"
}

function run () {
	PROGNAME=${PROGNAME:-$(basename "$1")}

	STDOUTLOG="$VARDIR/$PROGNAME.out"
	STDERRLOG="$VARDIR/$PROGNAME.err"

	# What follows runs the command that we have been given (with any arguments passed)
	# and logs:
	# * stdout to $STDOUTLOG (a temporary file) and possibly to syslog, if enabled.
	# * stderr to $STDERRLOG (a temporary file) and possibly to syslog, if enabled.
	#
	# When logging to syslog, stdout goes as debug level and stderr as warning (not error)
	#
	# The temporary file is used in case the command fails, at which point we try to log
	# a warning in GitLab's alerts facility.

	$* \
		>  >(tee $STDOUTLOG |_logger -t "dougal.runner.$PROGNAME[$$]" -p "$DOUGAL_LOG_FACILITY.debug") \
		2> >(tee $STDERRLOG |_logger -t "dougal.runner.$PROGNAME[$$]" -p "$DOUGAL_LOG_FACILITY.warning") || {

		print_error "Failed: $PROGNAME"
		cat $STDOUTLOG
		cat $STDERRLOG

		print_warning "Sending alert (if configured)"
		TITLE="$PROGNAME failed"
# 		DESCRIPTION=""
		SERVICE="deferred_imports"

# Disable GitLab alerts. They're just not very practical
# 		$BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \
# 			-O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)"

		exit 2
	}

	unset PROGNAME
	rm $STDOUTLOG $STDERRLOG
}

function cleanup () {
	if [[ -f $LOCKFILE ]]; then
		rm "$LOCKFILE"
	fi
}

if [[ -f $LOCKFILE ]]; then
    PID=$(cat "$LOCKFILE")
    if kill -0 "$PID" 2>/dev/null; then  # Check if process is running
        # Get elapsed time in D-HH:MM:SS format and convert to seconds
        ELAPSED_STR=$(ps -p "$PID" -o etime= | tr -d '[:space:]')
        if [ -n "$ELAPSED_STR" ]; then
            # Convert D-HH:MM:SS to seconds
            ELAPSED_SECONDS=$(echo "$ELAPSED_STR" | awk -F'[-:]' '{
                seconds = 0
                if (NF == 4) { seconds += $1 * 86400 }  # Days
                if (NF >= 3) { seconds += $NF-2 * 3600 } # Hours
                if (NF >= 2) { seconds += $NF-1 * 60 }   # Minutes
                seconds += $NF                           # Seconds
                print seconds
            }')
            if [ "$ELAPSED_SECONDS" -gt "$MAX_RUNTIME_SECONDS" ]; then
                # Kill the overdue process (SIGTERM; use -9 for SIGKILL if needed)
                kill "$PID" 2>/dev/null
                print_warning $(printf "Killed overdue process (%d) that ran for %s (%d seconds)" "$PID" "$ELAPSED_STR" "$ELAPSED_SECONDS")
                rm "$LOCKFILE"
            else
                print_warning $(printf "Previous process is still running (%d) for %s (%d seconds)" "$PID" "$ELAPSED_STR" "$ELAPSED_SECONDS")
                exit 1
            fi
        else
            print_warning $(printf "Could not retrieve elapsed time for process (%d)" "$PID")
            exit 1
        fi
    else
        rm "$LOCKFILE"
        print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" "$PID")
    fi
fi

echo "$$" > "$LOCKFILE" || {
	print_error "Error creating lock file"
	exit 255
}
print_info "Start run"

print_log "Check if data is accessible"
$BINDIR/check_mounts_present.py || {
	print_warning "Import mounts not accessible. Inhibiting all tasks!"
	cleanup
	exit 253
}

print_log "Purge deleted files"
run $BINDIR/purge_deleted_files.py

print_log "Import survey configurations"
run $BINDIR/import_survey_config.py

print_log "Import preplots"
run $BINDIR/import_preplots.py

print_log "Import raw P1/11"
run $BINDIR/import_raw_p111.py

#print_log "Import raw P1/90"
#run $BINDIR/import_raw_p190.py

print_log "Import final P1/11"
run $BINDIR/import_final_p111.py

#print_log "Import final P1/90"
#run $BINDIR/import_final_p190.py

print_log "Import SmartSource data"
run $BINDIR/import_smsrc.py

print_log "Import map user layers"
run $BINDIR/import_map_layers.py

# if [[ -z "$RUNNER_NOEXPORT" ]]; then
# 	print_log "Export system data"
# 	run $BINDIR/system_exports.py
# fi

# if [[ -n "$RUNNER_IMPORT" ]]; then
# 	print_log "Import system data"
# 	run $BINDIR/system_imports.py
# fi

# print_log "Export QC data"
# run $BINDIR/human_exports_qc.py

# print_log "Export sequence data"
# run $BINDIR/human_exports_seis.py

print_log "Process ASAQC queue"
# Run insecure in test mode:
# export NODE_TLS_REJECT_UNAUTHORIZED=0
PROGNAME=asaqc_queue run $DOUGAL_ROOT/lib/www/server/queues/asaqc/index.js

print_log "Run database housekeeping actions"
run $BINDIR/housekeep_database.py

print_log "Run QCs"
PROGNAME=run_qc run $DOUGAL_ROOT/lib/www/server/lib/qc/index.js


rm "$LOCKFILE"
print_info "End run"