Merge branch '287-the-project_summary-view-is-too-slow' into 'devel'

Resolve "The `project_summary` view is too slow"

Closes #287

See merge request wgp/dougal/software!55

lib/www/server/lib/db/project/summary/index.js

@@ -1,3 +1,4 @@
 module.exports = {
 	get: require('./get'),
+	refresh: require('./refresh')
 };

lib/www/server/lib/db/project/summary/refresh.js

@@ -0,0 +1,23 @@
+const { setSurvey } = require('../../connection');
+
+async function refresh (projectId, opts = {}) {
+	try {
+		const client = await setSurvey(projectId);
+
+		const text = `
+			REFRESH MATERIALIZED VIEW project_summary;
+		`;
+
+		const res = await client.query(text);
+		client.release();
+		return res.rows[0];
+	} catch (err) {
+		if (err.code == "42P01") {
+			throw { status: 404, message: "Not found" };
+		} else {
+			throw err;
+		}
+	}
+}
+
+module.exports = refresh;

lib/www/server/package.json

@@ -11,7 +11,7 @@
   "license": "UNLICENSED",
   "private": true,
   "config": {
-    "db_schema": "^0.4.2",
+    "db_schema": "^0.4.5",
     "api": "^0.4.0"
   },
   "engines": {

lib/www/server/periodic-tasks/index.js

@@ -4,10 +4,11 @@ const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debu
 function init () {
 	const iids = [];
 
-	function start () {
+	async function start () {
 		INFO("Initialising %d periodic tasks", tasks.length);
 		for (let t of tasks) {
-			const iid = setInterval(t.task, t.timeout);
+			const fn = t.init ? await t.init() : t.task;
+			const iid = setInterval(fn, t.timeout);
 			iids.push(iid);
 		}
 		return iids;

lib/www/server/periodic-tasks/tasks/index.js

@@ -1,4 +1,5 @@
 
 module.exports = [
-	require('./purge-notifications')
+	require('./purge-notifications'),
+	require('./refresh-project-summary')
 ];

lib/www/server/periodic-tasks/tasks/refresh-project-summary.js

@@ -0,0 +1,85 @@
+const db = require('../../lib/db');
+const { listen } = require('../../lib/db/notify');
+const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+const timeout = 30*1000;
+
+async function init () {
+	INFO("Setting up task");
+
+	// Full list of channels in
+	// ../lib/db/channels
+	const channels = [
+		"project",
+		"preplot_lines", "preplot_points",
+		"raw_lines", "raw_shots",
+		"final_lines", "final_shots"
+	];
+
+	const throttlePeriod = 10*1000;
+
+	const projects = {};
+
+	listen (channels, (data) => {
+		// Something important has changed,
+		// set the dirty flag for the relevant project
+		const pid = data.payload?.pid ?? data.payload?.new?.pid ?? data.payload?.old?.pid;
+		if (pid) {
+			if (!pid in projects) {
+				projects[pid] = {
+					lastRefreshed: 0
+				};
+			}
+			if (!projects[pid].needsRefresh) {
+				projects[pid].needsRefresh = true;
+				DEBUG("Setting up refresh flag for %s: %j", pid, projects[pid]);
+			}
+		}
+	});
+
+	const task = async () => {
+		for (pid in projects) {
+			const project = projects[pid];
+			if (project.needsRefresh) {
+				const now = Date.now();
+				const lastRefreshAge = now - project.lastRefreshed;
+				if (lastRefreshAge > throttlePeriod) {
+					// Do the actual refresh
+					try {
+						DEBUG("Refreshing", pid);
+						await db.project.summary.refresh(pid);
+					} catch (err) {
+						if (err.status == 404) {
+							DEBUG("Project %s not found. Removing from refresh list", pid);
+							delete projects[pid];
+						} else {
+							ERROR(err);
+						}
+					}
+					project.needsRefresh = false;
+					project.lastRefreshed = now;
+				}
+			}
+		}
+	};
+
+	// Let us populate the project list and do a first refresh on startup
+	for (const project of await db.project.get()) {
+		projects[project.pid] = {
+			lastRefreshed: 0,
+			needsRefresh: true
+		}
+	}
+	task(); // No need to await
+
+	return task;
+}
+
+async function cleanup () {
+}
+
+module.exports = {
+	init,
+	timeout,
+	cleanup
+};