dougal-software/bin/runner.sh

#!/bin/bash


DOUGAL_ROOT=${DOUGAL_ROOT:-$(dirname "$0")/..}

BINDIR="$DOUGAL_ROOT/bin"
VARDIR=${VARDIR:-$DOUGAL_ROOT/var}
LOCKFILE=${LOCKFILE:-$VARDIR/runner.lock}

[ -f ~/.profile ] && . ~/.profile

DOUGAL_LOG_TAG="dougal.runner[$$]"

# Only send output to the logger if we have the appropriate
# configuration set.
if [[ -n "$DOUGAL_LOG_TAG" && -n "$DOUGAL_LOG_FACILITY" ]]; then
	function _logger () {
		logger $*
	}
else
	function _logger () {
		: # This is the Bash null command
	}
fi

function tstamp () {
	date -u +%Y-%m-%dT%H:%M:%SZ
}

function prefix () {
	printf "\033[30;1m$(tstamp)\t"
}

function print_log () {
	printf "$(prefix)\033[36m%s\033[0m\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.info" "$*"
}

function print_info () {
	printf "$(prefix)\033[0m%s\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.debug" "$*"
}

function print_warning () {
	printf "$(prefix)\033[33;1m%s\033[0m\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.warning" "$*"
}

function print_error () {
	printf "$(prefix)\033[31m%s\033[0m\n" "$*"
	_logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.error" "$*"
}

function run () {
	PROGNAME=${PROGNAME:-$(basename "$1")}

	STDOUTLOG="$VARDIR/$PROGNAME.out"
	STDERRLOG="$VARDIR/$PROGNAME.err"

	# What follows runs the command that we have been given (with any arguments passed)
	# and logs:
	# * stdout to $STDOUTLOG (a temporary file) and possibly to syslog, if enabled.
	# * stderr to $STDERRLOG (a temporary file) and possibly to syslog, if enabled.
	#
	# When logging to syslog, stdout goes as debug level and stderr as warning (not error)
	#
	# The temporary file is used in case the command fails, at which point we try to log
	# a warning in GitLab's alerts facility.

	$* \
		>  >(tee $STDOUTLOG |_logger -t "dougal.runner.$PROGNAME[$$]" -p "$DOUGAL_LOG_FACILITY.debug") \
		2> >(tee $STDERRLOG |_logger -t "dougal.runner.$PROGNAME[$$]" -p "$DOUGAL_LOG_FACILITY.warning") || {

		print_error "Failed: $PROGNAME"
		cat $STDOUTLOG
		cat $STDERRLOG

		print_warning "Sending alert (if configured)"
		TITLE="$PROGNAME failed"
# 		DESCRIPTION=""
		SERVICE="deferred_imports"

		$BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \
			-O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)"

		exit 2
	}

	unset PROGNAME
	rm $STDOUTLOG $STDERRLOG
}

function cleanup () {
	if [[ -f $LOCKFILE ]]; then
		rm "$LOCKFILE"
	fi
}

if [[ -f $LOCKFILE ]]; then
	PID=$(cat "$LOCKFILE")
	if pgrep -F "$LOCKFILE"; then
		print_warning $(printf "The previous process is still running (%d)" $PID)
		exit 1
	else
		rm "$LOCKFILE"
		print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" $PID)
	fi
fi

echo "$$" > "$LOCKFILE" || {
	print_error "Error creating lock file"
	exit 255
}
print_info "Start run"

print_log "Check if data is accessible"
$BINDIR/check_mounts_present.py || {
	print_warning "Import mounts not accessible. Inhibiting all tasks!"
	cleanup
	exit 253
}

print_log "Purge deleted files"
run $BINDIR/purge_deleted_files.py

print_log "Import survey configurations"
run $BINDIR/import_survey_config.py

print_log "Import preplots"
run $BINDIR/import_preplots.py

print_log "Import raw P1/11"
run $BINDIR/import_raw_p111.py

print_log "Import raw P1/90"
run $BINDIR/import_raw_p190.py

print_log "Import final P1/11"
run $BINDIR/import_final_p111.py

print_log "Import final P1/90"
run $BINDIR/import_final_p190.py

print_log "Import SmartSource data"
run $BINDIR/import_smsrc.py

# if [[ -z "$RUNNER_NOEXPORT" ]]; then
# 	print_log "Export system data"
# 	run $BINDIR/system_exports.py
# fi

# if [[ -n "$RUNNER_IMPORT" ]]; then
# 	print_log "Import system data"
# 	run $BINDIR/system_imports.py
# fi

# print_log "Export QC data"
# run $BINDIR/human_exports_qc.py

# print_log "Export sequence data"
# run $BINDIR/human_exports_seis.py

print_log "Process ASAQC queue"
# Run insecure in test mode:
# export NODE_TLS_REJECT_UNAUTHORIZED=0
PROGNAME=asaqc_queue run $DOUGAL_ROOT/lib/www/server/queues/asaqc/index.js

print_log "Run database housekeeping actions"
run $BINDIR/housekeep_database.py

print_log "Run QCs"
PROGNAME=run_qc run $DOUGAL_ROOT/lib/www/server/lib/qc/index.js


rm "$LOCKFILE"
print_info "End run"