#!/bin/bash DOUGAL_ROOT=${DOUGAL_ROOT:-$(dirname "$0")/..} BINDIR="$DOUGAL_ROOT/bin" VARDIR=${VARDIR:-$DOUGAL_ROOT/var} LOCKFILE=${LOCKFILE:-$VARDIR/runner.lock} [ -f ~/.profile ] && . ~/.profile DOUGAL_LOG_TAG="dougal.runner[$$]" # Only send output to the logger if we have the appropriate # configuration set. if [[ -n "$DOUGAL_LOG_TAG" && -n "$DOUGAL_LOG_FACILITY" ]]; then function _logger () { logger $* } else function _logger () { : # This is the Bash null command } fi function tstamp () { date -u +%Y-%m-%dT%H:%M:%SZ } function prefix () { printf "\033[30;1m$(tstamp)\t" } function print_log () { printf "$(prefix)\033[36m%s\033[0m\n" "$*" _logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.info" "$*" } function print_info () { printf "$(prefix)\033[0m%s\n" "$*" _logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.debug" "$*" } function print_warning () { printf "$(prefix)\033[33;1m%s\033[0m\n" "$*" _logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.warning" "$*" } function print_error () { printf "$(prefix)\033[31m%s\033[0m\n" "$*" _logger -t "$DOUGAL_LOG_TAG" -p "$DOUGAL_LOG_FACILITY.error" "$*" } function run () { PROGNAME=${PROGNAME:-$(basename "$1")} STDOUTLOG="$VARDIR/$PROGNAME.out" STDERRLOG="$VARDIR/$PROGNAME.err" # What follows runs the command that we have been given (with any arguments passed) # and logs: # * stdout to $STDOUTLOG (a temporary file) and possibly to syslog, if enabled. # * stderr to $STDERRLOG (a temporary file) and possibly to syslog, if enabled. # # When logging to syslog, stdout goes as debug level and stderr as warning (not error) # # The temporary file is used in case the command fails, at which point we try to log # a warning in GitLab's alerts facility. $* \ > >(tee $STDOUTLOG |_logger -t "dougal.runner.$PROGNAME[$$]" -p "$DOUGAL_LOG_FACILITY.debug") \ 2> >(tee $STDERRLOG |_logger -t "dougal.runner.$PROGNAME[$$]" -p "$DOUGAL_LOG_FACILITY.warning") || { print_error "Failed: $PROGNAME" cat $STDOUTLOG cat $STDERRLOG print_warning "Sending alert (if configured)" TITLE="$PROGNAME failed" # DESCRIPTION="" SERVICE="deferred_imports" $BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \ -O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)" exit 2 } unset PROGNAME rm $STDOUTLOG $STDERRLOG } function cleanup () { if [[ -f $LOCKFILE ]]; then rm "$LOCKFILE" fi } if [[ -f $LOCKFILE ]]; then PID=$(cat "$LOCKFILE") if pgrep -F "$LOCKFILE"; then print_warning $(printf "The previous process is still running (%d)" $PID) exit 1 else rm "$LOCKFILE" print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" $PID) fi fi echo "$$" > "$LOCKFILE" || { print_error "Error creating lock file" exit 255 } print_info "Start run" print_log "Check if data is accessible" $BINDIR/check_mounts_present.py || { print_warning "Import mounts not accessible. Inhibiting all tasks!" cleanup exit 253 } print_log "Purge deleted files" run $BINDIR/purge_deleted_files.py print_log "Import survey configurations" run $BINDIR/import_survey_config.py print_log "Import preplots" run $BINDIR/import_preplots.py print_log "Import raw P1/11" run $BINDIR/import_raw_p111.py #print_log "Import raw P1/90" #run $BINDIR/import_raw_p190.py print_log "Import final P1/11" run $BINDIR/import_final_p111.py #print_log "Import final P1/90" #run $BINDIR/import_final_p190.py print_log "Import SmartSource data" run $BINDIR/import_smsrc.py print_log "Import map user layers" run $BINDIR/import_map_layers.py # if [[ -z "$RUNNER_NOEXPORT" ]]; then # print_log "Export system data" # run $BINDIR/system_exports.py # fi # if [[ -n "$RUNNER_IMPORT" ]]; then # print_log "Import system data" # run $BINDIR/system_imports.py # fi # print_log "Export QC data" # run $BINDIR/human_exports_qc.py # print_log "Export sequence data" # run $BINDIR/human_exports_seis.py print_log "Process ASAQC queue" # Run insecure in test mode: # export NODE_TLS_REJECT_UNAUTHORIZED=0 PROGNAME=asaqc_queue run $DOUGAL_ROOT/lib/www/server/queues/asaqc/index.js print_log "Run database housekeeping actions" run $BINDIR/housekeep_database.py print_log "Run QCs" PROGNAME=run_qc run $DOUGAL_ROOT/lib/www/server/lib/qc/index.js rm "$LOCKFILE" print_info "End run"