From f486b4835e769ff866aa48cad9d2931a96033b9a Mon Sep 17 00:00:00 2001
From: "D. Berge" <dberge@aaltronav.eu>
Date: Tue, 25 Aug 2020 11:32:13 +0200
Subject: [PATCH] Instrument alerts for HTTP backend.

An alert is sent whenever an endpoint returns
an error other than an explicit failure (e.g.,
it won't send an alert if a middleware intentionally
returns a {status: XXX} object).
---
 lib/www/server/api/index.js      | 12 ++++++++
 lib/www/server/lib/alerts.js     | 52 ++++++++++++++++++++++++++++++++
 lib/www/server/package-lock.json |  5 +++
 lib/www/server/package.json      |  1 +
 4 files changed, 70 insertions(+)
 create mode 100644 lib/www/server/lib/alerts.js

diff --git a/lib/www/server/api/index.js b/lib/www/server/api/index.js
index 134b8b3..8b3ed03 100644
--- a/lib/www/server/api/index.js
+++ b/lib/www/server/api/index.js
@@ -3,6 +3,7 @@ const http = require('http');
 const express = require('express');
 const cookieParser = require('cookie-parser')
 
+const maybeSendAlert = require("../lib/alerts");
 const mw = require('./middleware');
 
 const app = express();
@@ -153,14 +154,25 @@ app.map({
 // Generic error handler. Stops stack dumps
 // being sent to clients.
 app.use(function (err, req, res, next) {
+	const title = `HTTP backend error at ${req.method} ${req.originalUrl}`;
+	const description = err.message;
+	const message = err.message;
+	const alert = {title, message, description, error: err};
+
 	console.log("Error:", err);
+
 	if (err instanceof Error && err.name != "UnauthorizedError") {
 		console.error(err.stack);
 		res.status(500).send('General internal error');
+		maybeSendAlert(alert);
 	} else if (typeof err === 'string') {
 		res.status(500).send({message: err});
+		maybeSendAlert(alert);
 	} else {
 		res.status(err.status || 500).send({message: err.message || (err.inner && err.inner.message) || "Internal error"});
+		if (!res.status) {
+			maybeSendAlert(alert);
+		}
 	}
 });
 
diff --git a/lib/www/server/lib/alerts.js b/lib/www/server/lib/alerts.js
new file mode 100644
index 0000000..c139ae4
--- /dev/null
+++ b/lib/www/server/lib/alerts.js
@@ -0,0 +1,52 @@
+const fetch = require('node-fetch');
+const cfg = require("./config");
+
+const defaultOptions = {
+	title: "HTTP backend failure",
+	description: null,
+	service: `backend (NODE_ENV=${process.env.NODE_ENV})`,
+	severity: "critical",
+	hosts: process.env.HOST
+};
+
+function send (options = {}) {
+
+	let url, authkey;
+	try {
+
+		url = cfg.global.webhooks.alert.url;
+		authkey = process.env.GITLAB_ALERTS_AUTHKEY || cfg.global.webhooks.alert.authkey;
+
+		if (!url) return;
+
+	} catch (err) {
+		if (!(err instanceof TypeError)) {
+			console.error("Alert failed with error", err);
+		}
+		return;
+	}
+
+	const opts = Object.assign({}, defaultOptions, options);
+	opts.hosts = defaultOptions.hosts; // Do not allow override
+	const body = JSON.stringify(opts);
+
+	fetch(url, {
+		method: "POST",
+		headers: {
+			"Authorization": "Bearer " + authkey,
+			"Content-Type": "application/json"
+		},
+		body
+	})
+	.then(res => {
+		if (res.ok && res.status == 200) {
+			return;
+		}
+		console.error("Alert sending failed with status", res.status, res.statusText);
+	})
+	.catch(err => {
+		console.error("Alert sending failed with error", err);
+	});
+}
+
+module.exports = send;
diff --git a/lib/www/server/package-lock.json b/lib/www/server/package-lock.json
index 4e8d77b..045c7fc 100644
--- a/lib/www/server/package-lock.json
+++ b/lib/www/server/package-lock.json
@@ -366,6 +366,11 @@
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
       "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
     },
+    "node-fetch": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",
+      "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA=="
+    },
     "on-finished": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
diff --git a/lib/www/server/package.json b/lib/www/server/package.json
index b979c99..4f9bb8d 100644
--- a/lib/www/server/package.json
+++ b/lib/www/server/package.json
@@ -13,6 +13,7 @@
     "express": "^4.17.1",
     "express-jwt": "^6.0.0",
     "jsonwebtoken": "^8.5.1",
+    "node-fetch": "^2.6.0",
     "pg": "^8.3.0",
     "yaml": "^2.0.0-0"
   }