Bump the required database schema version

It listens for events that might indicate that the project_summary
materialised view needs to be refreshed and schedules a refresh.

Refreshes are throttled to a maximum of one every throttlePeriod
milliseconds so that things don't get too crazy for instance when
importing a lot of data.

etc/db/upgrades/upgrade32-v0.4.5-materialise-view-project_summary.sql

@@ -0,0 +1,147 @@
+-- Turn project_summary into a materialised view
+--
+-- New schema version: 0.4.5
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- The project_summary view is quite a bottleneck. While it itself is
+-- not the real culprit (rather the underlying views are), this is one
+-- relatively cheap way of improving responsiveness from the client's
+-- point of view.
+-- We leave the details of how / when to refresh the view to the non-
+-- database code.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_survey_schema (schema_name text) AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+
+	DROP VIEW project_summary;
+
+	CREATE MATERIALIZED VIEW project_summary AS
+	WITH fls AS (
+		SELECT
+			avg((final_lines_summary.duration / ((final_lines_summary.num_points - 1))::double precision)) AS shooting_rate,
+			avg((final_lines_summary.length / date_part('epoch'::text, final_lines_summary.duration))) AS speed,
+			sum(final_lines_summary.duration) AS prod_duration,
+			sum(final_lines_summary.length) AS prod_distance
+		FROM final_lines_summary
+	), project AS (
+		SELECT
+			p.pid,
+			p.name,
+			p.schema
+		FROM public.projects p
+		WHERE (split_part(current_setting('search_path'::text), ','::text, 1) = p.schema)
+	)
+	SELECT
+		project.pid,
+		project.name,
+		project.schema,
+		( SELECT count(*) AS count
+			FROM preplot_lines
+			WHERE (preplot_lines.class = 'V'::bpchar)) AS lines,
+		ps.total,
+		ps.virgin,
+		ps.prime,
+		ps.other,
+		ps.ntba,
+		ps.remaining,
+		( SELECT to_json(fs.*) AS to_json
+			FROM final_shots fs
+			ORDER BY fs.tstamp
+			LIMIT 1) AS fsp,
+		( SELECT to_json(fs.*) AS to_json
+			FROM final_shots fs
+			ORDER BY fs.tstamp DESC
+			LIMIT 1) AS lsp,
+		( SELECT count(*) AS count
+			FROM raw_lines rl) AS seq_raw,
+		( SELECT count(*) AS count
+			FROM final_lines rl) AS seq_final,
+		fls.prod_duration,
+		fls.prod_distance,
+		fls.speed AS shooting_rate
+	FROM preplot_summary ps,
+		fls,
+		project;
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.4.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.4.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL pg_temp.upgrade_survey_schema(row.schema_name);
+	END LOOP;
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_survey_schema (schema_name text);
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.5"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.4.5"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/www/client/source/package.json

@@ -15,7 +15,8 @@
     "leaflet-arrowheads": "^1.2.2",
     "leaflet-realtime": "^2.2.0",
     "leaflet.markercluster": "^1.4.1",
-    "marked": "^2.0.3",
+    "marked": "^9.1.4",
+    "path-browserify": "^1.0.1",
     "plotly.js-dist": "^2.27.0",
     "suncalc": "^1.8.0",
     "typeface-roboto": "0.0.75",
@@ -27,10 +28,10 @@
   },
   "devDependencies": {
     "@babel/plugin-proposal-logical-assignment-operators": "^7.14.5",
-    "@vue/cli-plugin-babel": "~4.4.0",
-    "@vue/cli-plugin-router": "~4.4.0",
-    "@vue/cli-plugin-vuex": "~4.4.0",
-    "@vue/cli-service": "^4.5.13",
+    "@vue/cli-plugin-babel": "^5.0.8",
+    "@vue/cli-plugin-router": "^5.0.8",
+    "@vue/cli-plugin-vuex": "^5.0.8",
+    "@vue/cli-service": "^5.0.8",
     "sass": "~1.32",
     "sass-loader": "^8.0.0",
     "stylus": "^0.54.8",

lib/www/client/source/src/components/footer.vue

@@ -29,7 +29,7 @@
 <style>
 @font-face {
   font-family: "Bank Gothic Medium";
-  src: local("Bank Gothic Medium"), url("/fonts/bank-gothic-medium.woff");
+  src: local("Bank Gothic Medium"), url("/public/fonts/bank-gothic-medium.woff");
 }
 
 .brand {

lib/www/client/source/src/components/graph-arrays-ij-scatter.vue

@@ -1,5 +1,5 @@
 <template>
-<v-card style="min-height:400px;">
+<v-card style="min-height:400px;" outlined>
   <v-card-title class="headline">
     Array inline / crossline error
     <v-spacer></v-spacer>
@@ -35,7 +35,6 @@
 <style scoped>
 
 .graph-container {
-  background-color: red;
   width: 100%;
   height: 100%;
 }
@@ -95,6 +94,10 @@ export default {
     scatterplot () {
       this.plot();
       this.$emit("update:settings", {[`${this.$options.name}.scatterplot`]: this.scatterplot});
+    },
+
+    "$vuetify.theme.isDark" () {
+      this.plot();
     }
   },
 
@@ -175,6 +178,11 @@ export default {
 			title: "Shotpoint",
             anchor: "x1"
 		},
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 
@@ -233,6 +241,11 @@ export default {
 		xaxis: {
 			title: "Crossline (m)"
 		},
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 
@@ -306,6 +319,11 @@ export default {
           domain: [ 0.55, 1 ],
           anchor: 'x2'
         },
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 

lib/www/client/source/src/components/graph-guns-depth.vue

@@ -1,5 +1,5 @@
 <template>
-<v-card style="min-height:400px;">
+<v-card style="min-height:400px;" outlined>
   <v-card-title class="headline">
     Gun depth
     <v-spacer></v-spacer>
@@ -98,6 +98,10 @@ export default {
         this.plotViolin();
       }
       this.$emit("update:settings", {[`${this.$options.name}.violinplot`]: this.violinplot});
+    },
+
+    "$vuetify.theme.isDark" () {
+      this.plot();
     }
 
   },
@@ -196,6 +200,11 @@ export default {
           title: "Shotpoint",
           showspikes: true
         },
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 
@@ -232,6 +241,11 @@ export default {
             title: "Gun number",
             type: 'category'
           },
+          font: {
+            color: this.$vuetify.theme.isDark ? "#fff" : undefined
+          },
+          plot_bgcolor:"rgba(0,0,0,0)",
+          paper_bgcolor:"rgba(0,0,0,0)",
           meta: {
             point
           }
@@ -305,6 +319,11 @@ export default {
 		xaxis: {
 			title: "Gun number"
 		},
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 

lib/www/client/source/src/components/graph-guns-heatmap.vue

@@ -1,5 +1,5 @@
 <template>
-<v-card style="min-height:400px;">
+<v-card style="min-height:400px;" outlined>
   <v-card-title class="headline">
     Gun details
   </v-card-title>
@@ -76,6 +76,10 @@ export default {
       if (this.violinplot) {
         this.plotViolin();
       }
+    },
+
+    "$vuetify.theme.isDark" () {
+      this.plot();
     }
 
   },
@@ -332,6 +336,11 @@ export default {
           title: "Shotpoint",
           showspikes: true
         },
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
 
         meta: this.data.meta
       };

lib/www/client/source/src/components/graph-guns-pressure.vue

@@ -1,5 +1,5 @@
 <template>
-<v-card style="min-height:400px;">
+<v-card style="min-height:400px;" outlined>
   <v-card-title class="headline">
     Gun pressures
     <v-spacer></v-spacer>
@@ -98,6 +98,10 @@ export default {
         this.plotViolin();
       }
       this.$emit("update:settings", {[`${this.$options.name}.violinplot`]: this.violinplot});
+    },
+
+    "$vuetify.theme.isDark" () {
+      this.plot();
     }
 
   },
@@ -123,7 +127,7 @@ export default {
         const gunPressuresSorted = gunPressures.map(s => d3a.sort(s));
         const gunVolumes = guns.map(s => s.map(g => g[12]));
         const gunPressureWeights = gunVolumes.map( (s, sidx) => s.map( v => v/meta[sidx].volume ));
-        const gunsWeightedAvgPressure = gunPressures.map( (s, sidx) => 
+        const gunsWeightedAvgPressure = gunPressures.map( (s, sidx) =>
           d3a.sum(s.map( (pressure, gidx) => pressure * gunPressureWeights[sidx][gidx] )) / d3a.sum(gunPressureWeights[sidx])
         );
 
@@ -210,6 +214,11 @@ export default {
           title: "Shotpoint",
           showspikes: true
         },
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 
@@ -249,6 +258,11 @@ export default {
             title: "Gun number",
             type: 'category'
           },
+          font: {
+            color: this.$vuetify.theme.isDark ? "#fff" : undefined
+          },
+          plot_bgcolor:"rgba(0,0,0,0)",
+          paper_bgcolor:"rgba(0,0,0,0)",
           meta: {
             point
           }
@@ -322,6 +336,11 @@ export default {
 		xaxis: {
 			title: "Gun number"
 		},
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 

lib/www/client/source/src/components/graph-guns-timing.vue

@@ -1,5 +1,5 @@
 <template>
-<v-card style="min-height:400px;">
+<v-card style="min-height:400px;" outlined>
   <v-card-title class="headline">
     Gun timing
     <v-spacer></v-spacer>
@@ -98,6 +98,10 @@ export default {
         this.plotViolin();
       }
       this.$emit("update:settings", {[`${this.$options.name}.violinplot`]: this.violinplot});
+    },
+
+    "$vuetify.theme.isDark" () {
+      this.plot();
     }
 
   },
@@ -196,6 +200,11 @@ export default {
           title: "Shotpoint",
           showspikes: true
         },
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 
@@ -232,6 +241,11 @@ export default {
             title: "Gun number",
             type: 'category'
           },
+          font: {
+            color: this.$vuetify.theme.isDark ? "#fff" : undefined
+          },
+          plot_bgcolor:"rgba(0,0,0,0)",
+          paper_bgcolor:"rgba(0,0,0,0)",
           meta: {
             point
           }
@@ -305,6 +319,11 @@ export default {
 		xaxis: {
 			title: "Gun number"
 		},
+        font: {
+          color: this.$vuetify.theme.isDark ? "#fff" : undefined
+        },
+        plot_bgcolor:"rgba(0,0,0,0)",
+        paper_bgcolor:"rgba(0,0,0,0)",
         meta: this.data.meta
       };
 

lib/www/client/source/src/lib/markdown.js

@@ -1,11 +1,11 @@
-const marked = require('marked');
+const { marked, parseInline } = require('marked');
 
 function markdown (str) {
   return marked(String(str));
 }
 
 function markdownInline (str) {
-  return marked.parseInline(String(str));
+  return parseInline(String(str));
 }
 
 module.exports = { markdown, markdownInline };

lib/www/client/source/src/views/Log.vue

@@ -64,9 +64,13 @@
 
           <v-menu v-if="$route.params.sequence">
             <template v-slot:activator="{on, attrs}">
-              <v-btn class="ml-5" small v-on="on" v-bind="attrs">
-                <span class="d-none d-lg-inline">Download as…</span>
-                <v-icon right small>mdi-cloud-download</v-icon>
+              <v-btn class="ml-3" small v-on="on" v-bind="attrs">
+                <span class="d-none d-lg-inline">Download as…
+                  <v-icon right small>mdi-cloud-download</v-icon>
+                </span>
+                <span class="d-lg-none">
+                  <v-icon small>mdi-cloud-download</v-icon>
+                </span>
               </v-btn>
             </template>
 

lib/www/client/source/vue.config.js

@@ -5,6 +5,18 @@ module.exports = {
   ],
   devServer: {
     host: "0.0.0.0",
+    webSocketServer: {
+      // Not found in the Webpack docs. Reference:
+      // https://github.com/webpack/webpack-dev-server/blob/540c43852ea33f9cb18820e1cef05d5ddb86cc3e/lib/servers/WebsocketServer.js#L21
+      options: {
+        // Not found in the ws docs. Reference:
+        // https://github.com/websockets/ws/blob/d8dd4852b81982fc0a6d633673968dff90985000/lib/websocket-server.js#L68
+        path: "/webpack-dev-server"
+      }
+    },
+    client: {
+      webSocketURL: "auto://0.0.0.0:0/webpack-dev-server"
+    },
     proxy: {
       "^/api(/|$)": {
         target: "http://127.0.0.1:3000",
@@ -22,5 +34,20 @@ module.exports = {
         args[0].title = 'Dougal Web'
         return args
       })
+  },
+  configureWebpack: {
+    resolve: {
+      fallback: {
+        path: require.resolve("path-browserify")
+      }
+    },
+    module: {
+      rules: [
+        {
+          test: /\.(woff|woff2|eot|ttf|otf)$/i,
+          type: 'asset/resource'
+        },
+      ],
+    },
   }
 }

lib/www/server/api/middleware/etag/cache.js

@@ -30,11 +30,14 @@ function saveResponse (res) {
 	if (res?.headersSent) {
 		const etag = res.get("ETag");
 		if (etag && res.locals.saveEtag !== false) {
+			if (res.get("set-cookie")) {
+				// Do not save any responses containing cookies
+				return;
+			}
 			const cache = getCache(res);
 			const req = res.req;
 			console.log(`Saving ETag: ${req.method} ${req.url} → ${etag}`);
 			const headers = structuredClone(res.getHeaders());
-			delete headers["set-cookie"];
 			cache[req.url] = {etag, headers};
 		}
 	}

lib/www/server/lib/db/line/list.js

@@ -9,6 +9,7 @@ async function list (projectId, opts = {}) {
 	const offset = Math.abs((opts.page-1)*opts.itemsPerPage) || 0;
 	const limit = Math.abs(Number(opts.itemsPerPage)) || null;
 
+	// FIXME Move this into the database as a view or something
 	const text = `
 		WITH counts AS (
 			SELECT pls.*, COALESCE(ppc.virgin, 0) na, COALESCE(ppc.tba, 0) tba

lib/www/server/lib/db/project/summary/index.js

@@ -1,3 +1,4 @@
 module.exports = {
 	get: require('./get'),
+	refresh: require('./refresh')
 };

lib/www/server/lib/db/project/summary/refresh.js

@@ -0,0 +1,23 @@
+const { setSurvey } = require('../../connection');
+
+async function refresh (projectId, opts = {}) {
+	try {
+		const client = await setSurvey(projectId);
+
+		const text = `
+			REFRESH MATERIALIZED VIEW project_summary;
+		`;
+
+		const res = await client.query(text);
+		client.release();
+		return res.rows[0];
+	} catch (err) {
+		if (err.code == "42P01") {
+			throw { status: 404, message: "Not found" };
+		} else {
+			throw err;
+		}
+	}
+}
+
+module.exports = refresh;

lib/www/server/package.json

@@ -11,7 +11,7 @@
   "license": "UNLICENSED",
   "private": true,
   "config": {
-    "db_schema": "^0.4.2",
+    "db_schema": "^0.4.5",
     "api": "^0.4.0"
   },
   "engines": {

lib/www/server/periodic-tasks/index.js

@@ -4,10 +4,11 @@ const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debu
 function init () {
 	const iids = [];
 
-	function start () {
+	async function start () {
 		INFO("Initialising %d periodic tasks", tasks.length);
 		for (let t of tasks) {
-			const iid = setInterval(t.task, t.timeout);
+			const fn = t.init ? await t.init() : t.task;
+			const iid = setInterval(fn, t.timeout);
 			iids.push(iid);
 		}
 		return iids;

lib/www/server/periodic-tasks/tasks/index.js

@@ -1,4 +1,5 @@
 
 module.exports = [
-	require('./purge-notifications')
+	require('./purge-notifications'),
+	require('./refresh-project-summary')
 ];

lib/www/server/periodic-tasks/tasks/refresh-project-summary.js

@@ -0,0 +1,85 @@
+const db = require('../../lib/db');
+const { listen } = require('../../lib/db/notify');
+const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+const timeout = 30*1000;
+
+async function init () {
+	INFO("Setting up task");
+
+	// Full list of channels in
+	// ../lib/db/channels
+	const channels = [
+		"project",
+		"preplot_lines", "preplot_points",
+		"raw_lines", "raw_shots",
+		"final_lines", "final_shots"
+	];
+
+	const throttlePeriod = 10*1000;
+
+	const projects = {};
+
+	listen (channels, (data) => {
+		// Something important has changed,
+		// set the dirty flag for the relevant project
+		const pid = data.payload?.pid ?? data.payload?.new?.pid ?? data.payload?.old?.pid;
+		if (pid) {
+			if (!pid in projects) {
+				projects[pid] = {
+					lastRefreshed: 0
+				};
+			}
+			if (!projects[pid].needsRefresh) {
+				projects[pid].needsRefresh = true;
+				DEBUG("Setting up refresh flag for %s: %j", pid, projects[pid]);
+			}
+		}
+	});
+
+	const task = async () => {
+		for (pid in projects) {
+			const project = projects[pid];
+			if (project.needsRefresh) {
+				const now = Date.now();
+				const lastRefreshAge = now - project.lastRefreshed;
+				if (lastRefreshAge > throttlePeriod) {
+					// Do the actual refresh
+					try {
+						DEBUG("Refreshing", pid);
+						await db.project.summary.refresh(pid);
+					} catch (err) {
+						if (err.status == 404) {
+							DEBUG("Project %s not found. Removing from refresh list", pid);
+							delete projects[pid];
+						} else {
+							ERROR(err);
+						}
+					}
+					project.needsRefresh = false;
+					project.lastRefreshed = now;
+				}
+			}
+		}
+	};
+
+	// Let us populate the project list and do a first refresh on startup
+	for (const project of await db.project.get()) {
+		projects[project.pid] = {
+			lastRefreshed: 0,
+			needsRefresh: true
+		}
+	}
+	task(); // No need to await
+
+	return task;
+}
+
+async function cleanup () {
+}
+
+module.exports = {
+	init,
+	timeout,
+	cleanup
+};