Fix indentation

This should be run at regular intervals (via cron or so) to keep
the comparisons up to date.

It is not necessarily a good idea to run this as part of the
runner.sh script as it will delay other tasks trying to
update the active project every time.

Probably OK to put it on a cronjbo every 2‒24 hours. If two
copies are running concurrently that should not break anything
but it will increase the server load.

bin/update_comparisons.js

@@ -0,0 +1,60 @@
+#!/usr/bin/node
+
+const cmp = require('../lib/www/server/lib/comparisons');
+
+
+async function main () {
+	console.log("Retrieving project groups");
+	const groups = await cmp.groups();
+
+	if (!Object.keys(groups??{})?.length) {
+		console.log("No groups found");
+		return 0;
+	}
+
+	console.log(`Found ${groups.length} groups: ${Object.keys(groups).join(", ")}`);
+
+	for (const groupName of Object.keys(groups)) {
+		const projects = groups[groupName];
+
+		console.log(`Fetching saved comparisons for ${groupName}`);
+
+		const comparisons = await cmp.getGroup(groupName);
+
+		// Check if there are any projects that have been modified since last comparison
+		// or if there are any pairs that are no longer part of the group
+
+		const outdated = comparisons.filter( c => {
+			const baseline_tstamp = projects.find( p => p.pid === c.baseline_pid )?.tstamp;
+			const monitor_tstamp = projects.find( p => p.pid === c.monitor_pid )?.tstamp;
+			return (c.tstamp < baseline_tstamp) || (c.tstamp < monitor_tstamp) ||
+				baseline_tstamp == null || monitor_tstamp == null;
+		});
+
+		for (const comparison of outdated) {
+			console.log(`Removing stale comparison: ${comparison.baseline_pid} → ${comparison.monitor_pid}`);
+			await cmp.remove(comparison.baseline_pid, comparison.monitor_pid);
+		}
+
+		if (projects?.length < 2) {
+			console.log(`Group ${groupName} has less than two projects. No comparisons are possible`);
+			continue;
+		}
+
+		// Re-run the comparisons that are not in the database. They may
+		// be missing either beacause they were not there to start with
+		// or because we just removed them due to being stale
+
+		console.log(`Recalculating group ${groupName}`);
+		await cmp.saveGroup(groupName);
+	}
+
+	console.log("Comparisons update done");
+	return 0;
+}
+
+if (require.main === module) {
+	main();
+} else {
+	module.exports = main;
+}

etc/db/upgrades/upgrade44-v0.6.6-add-last_project_update-function.sql

@@ -0,0 +1,157 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.6
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This adds a last_project_update(pid) function. It takes a project ID
+-- and returns the last known timestamp from that project. Timestamps
+-- are derived from multiple sources:
+--
+-- - raw_shots table
+-- - final_shots table
+-- - events_log_full table
+-- - info table where key = 'qc'
+-- - files table, from the hashes (which contain the file's mtime)
+-- - project configuration, looking for an _updatedOn property
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.last_project_update(p_pid text)
+	RETURNS timestamp with time zone
+	LANGUAGE plpgsql
+	AS $function$
+	DECLARE
+		v_last_ts timestamptz := NULL;
+		v_current_ts timestamptz;
+		v_current_str text;
+		v_current_unix numeric;
+		v_sid_rec record;
+	BEGIN
+		-- From raw_shots, final_shots, info, and files
+		FOR v_sid_rec IN SELECT schema FROM public.projects WHERE pid = p_pid
+		LOOP
+			-- From raw_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.raw_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From final_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.final_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From info where key = 'qc'
+			EXECUTE 'SELECT value->>''updatedOn'' FROM ' || v_sid_rec.schema || '.info WHERE key = ''qc''' INTO v_current_str;
+			IF v_current_str IS NOT NULL THEN
+				v_current_ts := v_current_str::timestamptz;
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From files hash second part, only for valid colon-separated hashes
+			EXECUTE 'SELECT max( split_part(hash, '':'', 2)::numeric ) FROM ' || v_sid_rec.schema || '.files WHERE hash ~ ''^[0-9]+:[0-9]+\\.[0-9]+:[0-9]+\\.[0-9]+:[0-9a-f]+$''' INTO v_current_unix;
+			IF v_current_unix IS NOT NULL THEN
+				v_current_ts := to_timestamp(v_current_unix);
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From event_log_full
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.event_log_full' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+		END LOOP;
+
+		-- From projects.meta->_updatedOn
+		SELECT (meta->>'_updatedOn')::timestamptz FROM public.projects WHERE pid = p_pid INTO v_current_ts;
+		IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+			v_last_ts := v_current_ts;
+		END IF;
+
+		RETURN v_last_ts;
+	END;
+	$function$;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.6' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.6"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.6"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/www/client/source/src/components/groups/group-map.vue

@@ -118,6 +118,33 @@
               @click="zoomOut"
             >mdi-magnify-minus-outline</v-icon>
           </div>
+          <div>
+            <v-icon
+              class="my-1"
+              title="Tilt out"
+              @click="tiltOut"
+            >mdi-axis-x-rotate-counterclockwise</v-icon>
+          </div>
+          <div>
+            <v-icon
+              class="my-1"
+              title="Tilt in"
+              @click="tiltIn"
+            >mdi-axis-x-rotate-clockwise</v-icon>
+          </div>
+          <div>
+            <v-icon v-if="bearing==0"
+              class="my-1"
+              title="Bin up"
+              @click="setBearing('ζ')"
+            >mdi-view-grid-outline</v-icon>
+            <v-icon v-else
+              class="my-1"
+              title="North up"
+              :style="`transform: rotate(${-bearing}deg);`"
+              @click="setBearing(0)"
+            >mdi-navigation</v-icon>
+          </div>
           <div>
             <v-icon
               class="my-1"
@@ -363,6 +390,7 @@ export default {
         //maxZoom: 18,
         maxPitch: 89
       },
+      bearing: 0,
 
       heatmapValue: "total_error",
       isFullscreen: false,
@@ -407,6 +435,11 @@ export default {
   watch: {
 
     baseline () {
+      // We need the configuration of the baseline project so that
+      // the "bin up" orientation control will work.
+      if (this.baseline?.pid) {
+        this.$store.dispatch('getProject', this.baseline.pid);
+      }
       this.populateDataLayersAvailable();
     },
 
@@ -518,6 +551,41 @@ export default {
       }
     },
 
+    tiltIn () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch -= 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    tiltOut () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch += 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    setBearing (bearing) {
+      if (deck) {
+
+        if (bearing === 'ζ') {
+          bearing = this.$store.getters.projectConfiguration?.binning?.theta ?? 0;
+        }
+
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.bearing = (bearing + 360) % 360;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
     toggleFullscreen() {
       const mapElement = document.getElementById('map-container');
       if (!this.isFullscreen) {
@@ -745,7 +813,7 @@ export default {
 
 
     async initLayers (gl) {
-      deck.onViewStateChange = this.updateURL;
+      // Does nothing
     },
 
     setViewState () {
@@ -760,6 +828,10 @@ export default {
       }
     },
 
+    viewStateUpdated ({viewState}) {
+      this.bearing = viewState.bearing;
+    },
+
     checkWebGLSupport() {
       const canvas = document.createElement('canvas');
       const gl = canvas.getContext('webgl2') || canvas.getContext('webgl');
@@ -1182,7 +1254,8 @@ export default {
       layers: [],
       getTooltip: this.getTooltip,
       pickingRadius: 24,
-      onWebGLInitialized: this.initLayers
+      onWebGLInitialized: this.initLayers,
+      onViewStateChange: this.viewStateUpdated,
     });
     //console.log("deck = ", deck);
 
@@ -1194,6 +1267,10 @@ export default {
 
       this.layersPendingLoad.push(`${this.baseline.pid}-seqfl`);
       this.layersPendingLoad.push(`${this.baseline.pid}-seqfp`);
+
+      // We need the configuration of the baseline project so that
+      // the "bin up" orientation control will work.
+      this.$store.dispatch('getProject', this.baseline.pid);
     }
 
     if (this.monitors) {

lib/www/client/source/src/views/Map.vue

@@ -470,6 +470,33 @@
           @click="zoomOut"
           >mdi-magnify-minus-outline</v-icon>
       </div>
+      <div>
+        <v-icon
+          class="my-1"
+          title="Tilt out"
+          @click="tiltOut"
+          >mdi-axis-x-rotate-counterclockwise</v-icon>
+      </div>
+      <div>
+        <v-icon
+          class="my-1"
+          title="Tilt in"
+          @click="tiltIn"
+          >mdi-axis-x-rotate-clockwise</v-icon>
+      </div>
+      <div>
+        <v-icon v-if="bearing==0"
+          class="my-1"
+          title="Bin up"
+          @click="setBearing('ζ')"
+          >mdi-view-grid-outline</v-icon>
+        <v-icon v-else
+          class="my-1"
+          title="North up"
+          :style="`transform: rotate(${-bearing}deg);`"
+          @click="setBearing(0)"
+          >mdi-navigation</v-icon>
+      </div>
       <div>
         <v-icon
           class="my-1"
@@ -661,6 +688,7 @@ export default {
         //maxZoom: 18,
         maxPitch: 89
       },
+      bearing: 0,
 
       vesselPosition: null,
       vesselTrackLastRefresh: 0,
@@ -977,6 +1005,41 @@ export default {
       }
     },
 
+    tiltIn () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch -= 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    tiltOut () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch += 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    setBearing (bearing) {
+      if (deck) {
+
+        if (bearing === 'ζ') {
+          bearing = this.$store.getters.projectConfiguration?.binning?.theta ?? 0;
+        }
+
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.bearing = (bearing + 360) % 360;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
     toggleFullscreen() {
       const mapElement = document.getElementById('map-container');
       if (!this.isFullscreen) {
@@ -1368,7 +1431,7 @@ export default {
       //console.log("SHOULD BE INITIALISING LAYERS HERE", gl);
       this.decodeURL();
       this.decodeURLHash();
-      deck.onViewStateChange = this.updateURL;
+      //deck.onViewStateChange = this.viewStateUpdated;
     },
 
     setViewState () {
@@ -1383,6 +1446,11 @@ export default {
       }
     },
 
+    viewStateUpdated ({viewState}) {
+      this.bearing = viewState.bearing;
+      this.updateURL({viewState});
+    },
+
     updateURL ({viewState} = {}) {
       if (!viewState && deck?.viewManager) {
         viewState = deck.getViewports()[0];
@@ -1709,7 +1777,8 @@ export default {
       layers: [],
       getTooltip: this.getTooltip,
       pickingRadius: 24,
-      onWebGLInitialized: this.initLayers
+      onWebGLInitialized: this.initLayers,
+      onViewStateChange: this.viewStateUpdated,
     });
 
     // Get fullscreen state

lib/www/server/lib/comparisons/geometric-differences.js

@@ -36,8 +36,16 @@ async function fetchErrors (pid) {
 }
 
 
+async function groupTimestamps (groupName) {
+	const projects = await groups()?.[groupName];
+	if (projects?.length) {
+
+	}
+}
+
+
 async function groups () {
-	const projects = await db.project.get();
+	const projects = await db.project.get({timestamps: true});
 	const groupNames = [
 		...projects
 			.reduce( (acc, cur) => acc.add(...cur.groups), new Set() )
@@ -46,19 +54,6 @@ async function groups () {
 	return Object.fromEntries(groupNames.map( g => [g, projects.filter( p => p.groups.includes(g) )] ));
 }
 
-/*
-async function compare (baselineProjectID, monitorProjectID) {
-	console.log("Getting baseline", baselineProjectID);
-	const baselineData = await db.sequence.get(baselineProjectID);
-	console.log("Getting monitor", monitorProjectID);
-	const monitorData = await db.sequence.get(monitorProjectID);
-	console.log("Comparing");
-
-	const comparison = comparisonGeometricDifferences(baselineData, monitorData);
-	return comparison;
-}
-*/
-
 function geometric_differences (baseline, monitor) {
 
 	if (!baseline || !baseline.length) {
@@ -160,67 +155,6 @@ async function save (baselineProjectID, monitorProjectID, bundle, meta) {
 }
 
 
-async function saveSample (baselineProjectID, monitorProjectID, opts = {}) {
-	DEBUG("Not bothering to save samples. This feature will be removed.");
-}
-
-/*
-async function saveSample (baselineProjectID, monitorProjectID, opts = {}) {
-	let sample = opts.sample;
-	let populationStats = opts.populationStats;
-	let sampleStats = opts.sampleStats;
-
-	if (!sample?.length) {
-		const sampleSize = opts.sampleSize ?? 2000;
-		const record = await get(baselineProjectID, monitorProjectID);
-		let data;
-
-		if (record) {
-			data = record.data;
-		} else {
-			console.log("Full data not found in database");
-			data = asBundle(await compare(baselineProjectID, monitorProjectID));
-		}
-
-		sample = computeSample(data, opts);
-
-		if (!populationStats) {
-			populationStats = stats(data);
-		}
-	}
-
-	const bundle = asBundle(sample, {type: 0x0c});
-
-	if (!sampleStats) {
-		sampleStats = stats(bundle);
-	}
-
-	meta = {tstamp: (new Date()), populationStats, sampleStats, ...(opts.meta??{})};
-
-	const client = await pool.connect();
-
-	try {
-		const text = `
-			INSERT INTO comparisons.comparisons
-				(type, baseline_pid, monitor_pid, data, meta)
-			VALUES ('geometric_difference_sample', $1, $2, $3, $4)
-			ON CONFLICT (type, baseline_pid, monitor_pid)
-			DO UPDATE SET
-				data = EXCLUDED.data,
-				meta = EXCLUDED.meta;
-		`;
-
-		const values = [ baselineProjectID, monitorProjectID, Buffer.from(bundle), meta ];
-		const res = await client.query(text, values);
-		return res.rowCount;
-	} catch (err) {
-		console.error(err);
-	} finally {
-		client.release();
-	}
-}
-*/
-
 async function get (baselineProjectID, monitorProjectID, type = 'geometric_difference') {
 
 	const client = await pool.connect();
@@ -328,77 +262,6 @@ function stats (comparison) {
 }
 
 
-/** Compare two projects' errorfinal quantities.
- *
- * Assumes that the preplots are the same.
- * It is not a terribly efficient way of doing it, but considering
- * that this is, by and large only going to be done once every few
- * hours for an active prospect, and never for inactive ones, I
- * think and hope we can live with that.
- *
- * `baseline` and `monitor` are the result of calling
- * db.sequence.get(projectId) on each of the respective
- * projects.
- */
-/*
-function comparisonGeometricDifferences (baseline, monitor) {
-
-	if (!baseline || !baseline.length) {
-		throw new Error("No baseline data");
-	}
-
-	if (!monitor || !monitor.length) {
-		throw new Error("No monitor data");
-	}
-
-	const comparison = []; // An array of { line, point, εi, εj, δts }; line + point may be repeated
-
-	for (const bp of baseline) {
-		if (!bp.errorfinal) {
-			console.log(`No final data for baseline point L${bp.line} S${bp.sequence} P${bp.point}`);
-			continue;
-		}
-
-		const monitor_points = monitor.filter( mp => mp.line === bp.line && mp.point === bp.point );
-		for (const mp of monitor_points) {
-			if (!mp.errorfinal) {
-				console.log(`No final data for monitor point L${mp.line} S${mp.sequence} P${mp.point}`);
-				continue;
-			}
-
-			const line = bp.line;
-			const point = bp.point;
-			const baseSeq = bp.sequence;
-			const monSeq = mp.sequence;
-			const baseTStamp = bp.tstamp;
-			const monTStamp = mp.tstamp;
-			const δi = bp.errorfinal.coordinates[0] - mp.errorfinal.coordinates[0];
-			const δj = bp.errorfinal.coordinates[1] - mp.errorfinal.coordinates[1];
-
-			const obj = {line, point, baseSeq, monSeq, baseTStamp, monTStamp, δi, δj};
-			comparison.push(obj);
-			// console.log(obj);
-		}
-	}
-
-	return comparison.sort(sortFn);
-}
-
-function sortComparison (comparison) {
-	return comparison.sort( (a, b) => {
-		if (a.line == b.line) {
-			if (a.point == b.point) {
-				return a.baseTStamp - b.baseTStamp;
-			} else {
-				return a.point - b.point;
-			}
-		} else {
-			return a.line - b.line;
-		}
-	})
-}
-*/
-
 function sortFn (a, b) {
 	if (a.line == b.line) {
 		if (a.point == b.point) {
@@ -454,12 +317,7 @@ async function saveGroup (group, opts = {}) {
 					}
 				}
 
-				const isSaved = await save(baselineProjectID, monitorProjectID);
-				if (isSaved) {
-					await saveSample(baselineProjectID, monitorProjectID, opts.sampleOpts);
-				} else {
-					await remove(baselineProjectID, monitorProjectID);
-				}
+				await save(baselineProjectID, monitorProjectID);
 				DEBUG("Saved comparison between %s and %s", baselineProjectID, monitorProjectID);
 			} catch (err) {
 				console.error(err);
@@ -469,44 +327,6 @@ async function saveGroup (group, opts = {}) {
 	}
 }
 
-/*
-async function getGroup (groupName, opts = {}) {
-
-	const group = (await groups())?.[groupName]?.map( i => i.pid)?.sort();
-
-	if (!group?.length) return;
-
-	const client = await pool.connect();
-
-	try {
-
-		const text = `
-			-- SQL query goes here
-		`;
-
-		const values = combinations(group, 2);
-		const res = await client.query(text, values);
-		if (!res.rows.length) {
-			console.log("Comparison not found in database");
-			return;
-		}
-
-		if (opts.returnData) {
-			return res.rows.map( row => ({
-				data: DougalBinaryBundle.clone(row.data),
-				meta: row.meta
-			});
-		} else {
-			return res.rows.map( row => row.meta );
-		}
-	} catch (err) {
-		console.error(err);
-	} finally {
-		client.release();
-	}
-}
-*/
-
 
 async function getGroup (groupName, opts = {}) {
 
@@ -535,9 +355,6 @@ async function getGroup (groupName, opts = {}) {
 			ORDER BY baseline_pid, monitor_pid
 		`;
 
-		console.log(text);
-		console.log(flatValues);
-
 		const res = await client.query(text, flatValues);
 		if (!res.rows.length) {
 			console.log("Comparison not found in database");
@@ -567,12 +384,10 @@ module.exports = {
 	get,
 	save,
 	getSample,
-	saveSample,
 	saveGroup,
 	getGroup,
 	remove,
 	stats,
-	// comparisonGeometricDifferences,
 	asBundle,
 	fromBundle
 };

lib/www/server/lib/db/project/get.js

@@ -1,8 +1,14 @@
 const { setSurvey, pool } = require('../connection');
 
-async function get () {
+async function get (opts = {}) {
+
+	const select = opts.timestamps
+		? "last_project_update(pid) tstamp,"
+		: "";
+
 	const text = `
 		SELECT
+			${select}
 			pid,
 			name,
 			schema,

lib/www/server/package.json

@@ -16,7 +16,7 @@
       "api": "0.4.0"
     },
     "wanted": {
-      "db_schema": "^0.6.5"
+      "db_schema": "^0.6.6"
     }
   },
   "engines": {