Fix indentation

This should be run at regular intervals (via cron or so) to keep
the comparisons up to date.

It is not necessarily a good idea to run this as part of the
runner.sh script as it will delay other tasks trying to
update the active project every time.

Probably OK to put it on a cronjbo every 2‒24 hours. If two
copies are running concurrently that should not break anything
but it will increase the server load.

bin/update_comparisons.js

@@ -0,0 +1,60 @@
+#!/usr/bin/node
+
+const cmp = require('../lib/www/server/lib/comparisons');
+
+
+async function main () {
+	console.log("Retrieving project groups");
+	const groups = await cmp.groups();
+
+	if (!Object.keys(groups??{})?.length) {
+		console.log("No groups found");
+		return 0;
+	}
+
+	console.log(`Found ${groups.length} groups: ${Object.keys(groups).join(", ")}`);
+
+	for (const groupName of Object.keys(groups)) {
+		const projects = groups[groupName];
+
+		console.log(`Fetching saved comparisons for ${groupName}`);
+
+		const comparisons = await cmp.getGroup(groupName);
+
+		// Check if there are any projects that have been modified since last comparison
+		// or if there are any pairs that are no longer part of the group
+
+		const outdated = comparisons.filter( c => {
+			const baseline_tstamp = projects.find( p => p.pid === c.baseline_pid )?.tstamp;
+			const monitor_tstamp = projects.find( p => p.pid === c.monitor_pid )?.tstamp;
+			return (c.tstamp < baseline_tstamp) || (c.tstamp < monitor_tstamp) ||
+				baseline_tstamp == null || monitor_tstamp == null;
+		});
+
+		for (const comparison of outdated) {
+			console.log(`Removing stale comparison: ${comparison.baseline_pid} → ${comparison.monitor_pid}`);
+			await cmp.remove(comparison.baseline_pid, comparison.monitor_pid);
+		}
+
+		if (projects?.length < 2) {
+			console.log(`Group ${groupName} has less than two projects. No comparisons are possible`);
+			continue;
+		}
+
+		// Re-run the comparisons that are not in the database. They may
+		// be missing either beacause they were not there to start with
+		// or because we just removed them due to being stale
+
+		console.log(`Recalculating group ${groupName}`);
+		await cmp.saveGroup(groupName);
+	}
+
+	console.log("Comparisons update done");
+	return 0;
+}
+
+if (require.main === module) {
+	main();
+} else {
+	module.exports = main;
+}

etc/db/upgrades/upgrade41-v0.6.3-add-comparisons.sql

@@ -0,0 +1,109 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.3
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade creates a new schema called `comparisons`.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update adds a `comparisons` table to a `comparisons` schema.
+-- The `comparisons.comparisons` table holds 4D prospect comparison data.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE SCHEMA IF NOT EXISTS comparisons
+		AUTHORIZATION postgres;
+
+	COMMENT ON SCHEMA comparisons
+		IS 'Holds 4D comparison data and logic';
+
+	CREATE TABLE IF NOT EXISTS comparisons.comparisons
+	(
+		type text COLLATE pg_catalog."default" NOT NULL,
+		baseline_pid text COLLATE pg_catalog."default" NOT NULL,
+		monitor_pid text COLLATE pg_catalog."default" NOT NULL,
+		data bytea,
+		meta jsonb NOT NULL DEFAULT '{}'::jsonb,
+		CONSTRAINT comparisons_pkey PRIMARY KEY (baseline_pid, monitor_pid, type)
+	)
+
+	TABLESPACE pg_default;
+
+	ALTER TABLE IF EXISTS comparisons.comparisons
+		OWNER to postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.3"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.3"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade42-v0.6.4-notify-exclude-columns.sql

@@ -0,0 +1,169 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.4
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.notify()
+		RETURNS trigger
+		LANGUAGE 'plpgsql'
+		COST 100
+		VOLATILE NOT LEAKPROOF
+	AS $BODY$
+		DECLARE
+		channel text := TG_ARGV[0];
+		pid text;
+		payload text;
+		notification text;
+		payload_id integer;
+		old_json jsonb;
+		new_json jsonb;
+		excluded_col text;
+		i integer;
+		BEGIN
+
+		-- Fetch pid
+		SELECT projects.pid INTO pid FROM projects WHERE schema = TG_TABLE_SCHEMA;
+
+		-- Build old and new as jsonb, excluding specified columns if provided
+		IF OLD IS NOT NULL THEN
+			old_json := row_to_json(OLD)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				old_json := old_json - excluded_col;
+			END LOOP;
+		ELSE
+			old_json := NULL;
+		END IF;
+
+		IF NEW IS NOT NULL THEN
+			new_json := row_to_json(NEW)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				new_json := new_json - excluded_col;
+			END LOOP;
+		ELSE
+			new_json := NULL;
+		END IF;
+
+		-- Build payload
+		payload := json_build_object(
+			'tstamp', CURRENT_TIMESTAMP,
+			'operation', TG_OP,
+			'schema', TG_TABLE_SCHEMA,
+			'table', TG_TABLE_NAME,
+			'old', old_json,
+			'new', new_json,
+			'pid', pid
+		)::text;
+
+		-- Handle large payloads
+		IF octet_length(payload) < 1000 THEN
+			PERFORM pg_notify(channel, payload);
+		ELSE
+			-- Store large payload and notify with ID (as before)
+			INSERT INTO notify_payloads (payload) VALUES (payload) RETURNING id INTO payload_id;
+
+			notification := json_build_object(
+				'tstamp', CURRENT_TIMESTAMP,
+				'operation', TG_OP,
+				'schema', TG_TABLE_SCHEMA,
+				'table', TG_TABLE_NAME,
+				'pid', pid,
+				'payload_id', payload_id
+			)::text;
+
+			PERFORM pg_notify(channel, notification);
+			RAISE INFO 'Payload over limit';
+		END IF;
+		RETURN NULL;
+		END;
+	$BODY$;
+
+	ALTER FUNCTION public.notify()
+		OWNER TO postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.4"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.4"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade43-v0.6.5-add-comparisons-notification-channel.sql

@@ -0,0 +1,96 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.5
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE TRIGGER comparisons_tg
+		AFTER INSERT OR DELETE OR UPDATE
+		ON comparisons.comparisons
+		FOR EACH ROW
+		EXECUTE FUNCTION public.notify('comparisons', 'data');
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.5"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.5"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade44-v0.6.6-add-last_project_update-function.sql

@@ -0,0 +1,157 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.6
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This adds a last_project_update(pid) function. It takes a project ID
+-- and returns the last known timestamp from that project. Timestamps
+-- are derived from multiple sources:
+--
+-- - raw_shots table
+-- - final_shots table
+-- - events_log_full table
+-- - info table where key = 'qc'
+-- - files table, from the hashes (which contain the file's mtime)
+-- - project configuration, looking for an _updatedOn property
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.last_project_update(p_pid text)
+	RETURNS timestamp with time zone
+	LANGUAGE plpgsql
+	AS $function$
+	DECLARE
+		v_last_ts timestamptz := NULL;
+		v_current_ts timestamptz;
+		v_current_str text;
+		v_current_unix numeric;
+		v_sid_rec record;
+	BEGIN
+		-- From raw_shots, final_shots, info, and files
+		FOR v_sid_rec IN SELECT schema FROM public.projects WHERE pid = p_pid
+		LOOP
+			-- From raw_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.raw_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From final_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.final_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From info where key = 'qc'
+			EXECUTE 'SELECT value->>''updatedOn'' FROM ' || v_sid_rec.schema || '.info WHERE key = ''qc''' INTO v_current_str;
+			IF v_current_str IS NOT NULL THEN
+				v_current_ts := v_current_str::timestamptz;
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From files hash second part, only for valid colon-separated hashes
+			EXECUTE 'SELECT max( split_part(hash, '':'', 2)::numeric ) FROM ' || v_sid_rec.schema || '.files WHERE hash ~ ''^[0-9]+:[0-9]+\\.[0-9]+:[0-9]+\\.[0-9]+:[0-9a-f]+$''' INTO v_current_unix;
+			IF v_current_unix IS NOT NULL THEN
+				v_current_ts := to_timestamp(v_current_unix);
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From event_log_full
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.event_log_full' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+		END LOOP;
+
+		-- From projects.meta->_updatedOn
+		SELECT (meta->>'_updatedOn')::timestamptz FROM public.projects WHERE pid = p_pid INTO v_current_ts;
+		IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+			v_last_ts := v_current_ts;
+		END IF;
+
+		RETURN v_last_ts;
+	END;
+	$function$;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.6' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.6"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.6"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/modules/@dougal/binary/classes.js

@@ -503,6 +503,37 @@ class DougalBinaryBundle extends ArrayBuffer {
 		return ab;
 	}
 
+	get records () {
+		const data = [];
+		for (const record of this) {
+			data.push(record.slice(1));
+		}
+		return data;
+	}
+
+	[Symbol.iterator]() {
+		const chunks = this.chunks();
+		let chunkIndex = 0;
+		let chunkIterator = chunks.length > 0 ? chunks[0][Symbol.iterator]() : null;
+
+		return {
+			next() {
+				if (!chunkIterator) {
+					return { done: true };
+				}
+
+				let result = chunkIterator.next();
+				while (result.done && chunkIndex < chunks.length - 1) {
+					chunkIndex++;
+					chunkIterator = chunks[chunkIndex][Symbol.iterator]();
+					result = chunkIterator.next();
+				}
+
+				return result;
+			}
+		};
+	}
+
 }
 
 
@@ -707,6 +738,21 @@ class DougalBinaryChunkSequential extends ArrayBuffer {
 
 		return arr;
 	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
 }
 
 
@@ -901,6 +947,21 @@ class DougalBinaryChunkInterleaved extends ArrayBuffer {
 
 		return arr;
 	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
 }
 
 

lib/www/client/source/src/App.vue

@@ -85,7 +85,9 @@ export default {
     },
 
     handleProject (context, {payload}) {
-      this.refreshProjects();
+      if (payload?.table == "public") {
+        this.refreshProjects();
+      }
     },
 
     registerNotificationHandlers () {

lib/www/client/source/src/components/groups/group-comparison-summary.vue

@@ -0,0 +1,187 @@
+<template>
+  <v-card v-if="comparison" class="ma-1">
+    <v-card-title>Comparison Summary: Baseline {{ baseline.pid }} vs Monitor {{ monitor.pid }}</v-card-title>
+    <v-card-text>
+      <v-row>
+        <v-col cols="12" md="6">
+          <h3>Deviation Statistics</h3>
+          <v-simple-table dense>
+            <template v-slot:default>
+              <thead>
+                <tr>
+                  <th>Metric</th>
+                  <th>I (m)</th>
+                  <th>J (m)</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td>Mean (μ)</td>
+                  <td>{{ comparison['μ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['μ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>Std Dev (σ)</td>
+                  <td>{{ comparison['σ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['σ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>RMS</td>
+                  <td>{{ comparison.rms[0].toFixed(3) }}</td>
+                  <td>{{ comparison.rms[1].toFixed(3) }}</td>
+                </tr>
+              </tbody>
+            </template>
+          </v-simple-table>
+
+          <h3 class="mt-4">Error distribution</h3>
+          <ul>
+            <li title="Relative to I-axis positive direction">Primary Direction: {{ (comparison.primaryDirection * 180 / Math.PI).toFixed(2) }}°</li>
+            <li>Anisotropy: {{ comparison.anisotropy.toFixed(2) }}</li>
+            <li title="Length of the semi-major axis of the error ellipse">Semi-Major Axis: {{ semiMajorAxis.toFixed(2) }} m</li>
+            <li title="Length of the semi-minor axis of the error ellipse">Semi-Minor Axis: {{ semiMinorAxis.toFixed(2) }} m</li>
+            <li title="Area of the error ellipse">Error Ellipse Area: {{ ellipseArea.toFixed(2) }} m²</li>
+          </ul>
+
+          <h3 class="mt-4">Counts</h3>
+          <ul>
+            <li title="Unique line / point pairs found in both projects">Common Points: {{ comparison.common }}</li>
+            <li title="Total number of points compared, including reshoots, infills, etc.">Comparison Length: {{ comparison.length }}</li>
+            <li title="Number of points in the baseline project">Baseline Points: {{ comparison.baselineLength }} (Unique: {{ comparison.baselineUniqueLength }})</li>
+            <li title="Number of points in the monitor project">Monitor Points: {{ comparison.monitorLength }} (Unique: {{ comparison.monitorUniqueLength }})</li>
+          </ul>
+
+          <p class="mt-3" title="Date and time when the comparison was last performed">Computation timestamp: {{ new Date(comparison.tstamp).toLocaleString() }}</p>
+        </v-col>
+
+        <v-col cols="12" md="6">
+          <h3>Error Ellipse</h3>
+          <svg width="300" height="300" style="border: 1px solid #ccc;">
+            <g :transform="`translate(150, 150) scale(${ellipseScale})`">
+              <line x1="0" y1="-150" x2="0" y2="150" stroke="lightgray" stroke-dasharray="5,5"/>
+              <line x1="-150" y1="0" x2="150" y2="0" stroke="lightgray" stroke-dasharray="5,5"/>
+              <ellipse
+                :rx="Math.sqrt(comparison.eigenvalues[0])"
+                :ry="Math.sqrt(comparison.eigenvalues[1])"
+                :transform="`rotate(${ellipseAngle})`"
+                fill="none"
+                stroke="blue"
+                stroke-width="2"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[0]) * Math.cos(ellipseRad)"
+                :y2="Math.sqrt(comparison.eigenvalues[0]) * Math.sin(ellipseRad)"
+                stroke="red"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[1]) * Math.cos(ellipseRad + Math.PI / 2)"
+                :y2="Math.sqrt(comparison.eigenvalues[1]) * Math.sin(ellipseRad + Math.PI / 2)"
+                stroke="green"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+            </g>
+          </svg>
+          <p class="text-caption">Ellipse scaled for visibility (factor: {{ ellipseScale.toFixed(1) }}). Axes represent sqrt(eigenvalues).</p>
+        </v-col>
+      </v-row>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: "DougalGroupComparisonSummary",
+
+  props: {
+    baseline: { type: Object, required: true },
+    monitor: { type: Object, required: true },
+    comparison: { type: Object, required: true }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+
+    ellipseAngle () {
+      if (!this.comparison) return 0;
+      const ev = this.comparison.eigenvectors[0];
+      return Math.atan2(ev[1], ev[0]) * 180 / Math.PI;
+    },
+
+    ellipseRad () {
+      return this.ellipseAngle * Math.PI / 180;
+    },
+
+    ellipseRx () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[0]) * this.ellipseScale;
+    },
+
+    ellipseRy () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[1]) * this.ellipseScale;
+    },
+
+    ellipseScale () {
+      if (!this.comparison) return 1;
+      const maxSigma = Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+      const maxMu = Math.max(
+        Math.abs(this.comparison['μ'][0]),
+        Math.abs(this.comparison['μ'][1])
+      );
+      //const maxExtent = maxMu + 3 * maxSigma;
+      const maxExtent = 20;
+      return 100 / maxExtent; // Adjust scale to fit within ~200 pixels diameter
+    },
+
+    ellipseArea () {
+      if (!this.comparison) return 0;
+      const a = Math.sqrt(this.comparison.eigenvalues[0]);
+      const b = Math.sqrt(this.comparison.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+
+    semiMajorAxis () {
+      if (!this.comparison) return 0;
+      return Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    semiMinorAxis () {
+      if (!this.comparison) return 0;
+      return Math.min(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    meanX () {
+      return this.comparison ? this.comparison['μ'][0] : 0;
+    },
+
+    meanY () {
+      return this.comparison ? this.comparison['μ'][1] : 0;
+    },
+
+    ellipseViewBox () {
+      return '-150 -150 300 300';
+    },
+
+  }
+}
+</script>

lib/www/client/source/src/components/groups/group-repeatability-summary.vue

@@ -0,0 +1,118 @@
+<template>
+  <v-card class="ma-1">
+    <v-card-title>Group Repeatability Summary</v-card-title>
+    <v-card-text>
+      <p>Error ellipse area for each baseline-monitor pair. Lower values indicate better repeatability. Colors range from green (best) to red (worst).</p>
+      <v-simple-table dense>
+        <thead>
+          <tr>
+            <th>Baseline \ Monitor</th>
+            <th v-for="project in projects" :key="project.pid">{{ project.pid }}</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr v-for="(baselineProject, rowIndex) in projects" :key="baselineProject.pid">
+            <td>{{ baselineProject.pid }}</td>
+            <td v-for="(monitorProject, colIndex) in projects" :key="monitorProject.pid">
+              <v-tooltip v-if="colIndex > rowIndex" top>
+                <template v-slot:activator="{ on, attrs }">
+                  <div
+                    :style="{ backgroundColor: getEllipseAreaColor(baselineProject.pid, monitorProject.pid), color: 'white', textAlign: 'center', padding: '4px' }"
+                    v-bind="attrs"
+                    v-on="on"
+                    @click="emitInput(baselineProject, monitorProject)"
+                  >
+                    {{ formatEllipseArea(baselineProject.pid, monitorProject.pid) }}
+                  </div>
+                </template>
+                <span v-if="getComp(baselineProject.pid, monitorProject.pid)">
+                  <div>σ_i: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][0].toFixed(2) }} m</div>
+                  <div>σ_j: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][1].toFixed(2) }} m</div>
+                  <div>Anisotropy: {{ getComp(baselineProject.pid, monitorProject.pid).meta.anisotropy.toFixed(0) }}</div>
+                  <div>Ellipse Area: {{ getEllipseArea(baselineProject.pid, monitorProject.pid).toFixed(2) }} m²</div>
+                  <div>Primary Direction: {{ formatPrimaryDirection(getComp(baselineProject.pid, monitorProject.pid)) }}°</div>
+                </span>
+              </v-tooltip>
+            </td>
+          </tr>
+        </tbody>
+      </v-simple-table>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: 'DougalGroupRepeatabilitySummary',
+
+  props: {
+    comparisons: {
+      type: Array,
+      required: true
+    },
+    projects: {
+      type: Array,
+      required: true
+    }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+    compMap () {
+      return new Map(this.comparisons.map(c => [`${c.baseline_pid}-${c.monitor_pid}`, c]));
+    },
+    minEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.min(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    },
+    maxEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.max(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    }
+  },
+  methods: {
+    getComp (basePid, monPid) {
+      return this.compMap.get(`${basePid}-${monPid}`);
+    },
+    getEllipseArea (basePid, monPid) {
+      const comp = this.getComp(basePid, monPid);
+      if (!comp) return null;
+      const a = Math.sqrt(comp.meta.eigenvalues[0]);
+      const b = Math.sqrt(comp.meta.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+    formatEllipseArea (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      return val !== null ? val.toFixed(1) : '';
+    },
+    getEllipseAreaColor (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      if (val === null) return '';
+      const ratio = (val - this.minEllipseArea) / (this.maxEllipseArea - this.minEllipseArea);
+      const hue = (1 - ratio) * 120;
+      return `hsl(${hue}, 70%, 70%)`;
+    },
+    formatPrimaryDirection (comp) {
+      if (!comp) return '';
+      return (comp.meta.primaryDirection * 180 / Math.PI).toFixed(1);
+    },
+    emitInput (baselineProject, monitorProject) {
+      if (this.getComp(baselineProject.pid, monitorProject.pid)) {
+        this.$emit('input', baselineProject, monitorProject);
+      }
+    }
+  }
+}
+</script>

lib/www/client/source/src/lib/deck.gl/DougalBinaryLoader.js

@@ -84,8 +84,12 @@ const DougalBinaryLoader = {
       for (let k = 0; k < values.length; k++) {
         values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? BigUint64Array : Float32Array)(totalCount);
       }
+    } else if (udv == 4) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? Uint16Array : Float32Array)(totalCount);
+      }
     } else {
-      throw new Error(`Invalid udv: Expected 0, 1, or 2; found ${udv}`);
+      throw new Error(`Invalid udv: Expected 0, 1, 2, or 4; found ${udv}`);
     }
 
     let offset = 0;
@@ -110,7 +114,7 @@ const DougalBinaryLoader = {
       offset += chunk.jCount;
     }
 
-    console.log(`Parsed ${totalCount} points, ${values.length} value arrays`);
+    console.log(`Parsed ${totalCount} points, ${values.length} value arrays, udv = ${udv}`);
 
     const attributes = {
       getPosition: {

lib/www/client/source/src/lib/durations.js

@@ -0,0 +1,47 @@
+
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+
+export {
+  duration_to_ms,
+  ms_to_duration,
+  normalise_duration,
+  add_durations
+}

lib/www/client/source/src/router/index.js

@@ -20,6 +20,9 @@ import ProjectSettings from '../views/ProjectSettings.vue'
 import Users from '../views/Users.vue'
 import DougalAppBarExtensionProject from '../components/app-bar-extension-project'
 import DougalAppBarExtensionProjectList from '../components/app-bar-extension-project-list'
+import GroupList from '../views/GroupList.vue'
+import Group from '../views/Group.vue'
+
 
 Vue.use(VueRouter)
 
@@ -196,7 +199,57 @@ Vue.use(VueRouter)
         component: ProjectSettings
       }
     ]
-  }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups",
+    redirect: "/groups/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/",
+    component: GroupList,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups", disabled: true }
+      ],
+      appBarExtension: {
+        // component: DougalAppBarExtensionProjectList
+      }
+    }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group",
+    redirect: "/groups/:group/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group/",
+    name: "Group",
+    component: Group,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups" },
+        { text: (ctx) => ctx.$route.params.group }
+        /*
+        {
+          text: (ctx) => ctx.$store.state.project.projectName || "…",
+          href: (ctx) => `/projects/${ctx.$store.state.project.projectId || ctx.$route.params.project || ""}/`,
+          title: (ctx) => Object.entries(ctx.$store.getters.projectConfiguration?.organisations ?? {}).map( ([org, ops]) => `* ${org}: ${Object.entries(ops).filter( ([k, v]) => v ).map( ([k, v]) => k ).join(", ")}`).join("\n"),
+          organisations: (ctx) => ctx.$store.getters.projectConfiguration?.organisations ?? {}
+        }
+        */
+      ],
+      /*
+      appBarExtension: {
+        component: DougalAppBarExtensionGroup
+      }
+      */
+    },
+    children: [
+    ]
+  },
 ]
 
 const router = new VueRouter({

lib/www/client/source/src/store/modules/project/actions.js

@@ -1,5 +1,10 @@
 
 async function getProject ({commit, dispatch}, projectId) {
+  if (projectId == null) {
+    console.log(`Skipping call to getProject${projectId})`);
+    return;
+  }
+
   const init = {
     headers: {
       cache: "reload",

lib/www/client/source/src/store/modules/projects/actions.js

@@ -1,13 +1,27 @@
+import * as d3a from 'd3-array';
+import { duration_to_ms, ms_to_duration, normalise_duration, add_durations } from '@/lib/durations';
 
 /** Fetch projects from server
  */
 async function refreshProjects ({commit, dispatch, state, rootState}) {
 
+  async function getSummary (project) {
+      const url = `/project/${project.pid}/summary`;
+      const init = {};
+      const summary = await dispatch('api', [url, init, null, {silent:true}]);
+      if (summary) {
+        return {...project, ...summary};
+      } else {
+        return project;
+      }
+  }
+
   if (state.loading) {
     commit('abortProjectsLoading');
   }
 
   commit('setProjectsLoading');
+  const tstamp = new Date();
   const pid = rootState.project.projectId;
   const url = `/project`;
   const init = {
@@ -17,10 +31,25 @@ async function refreshProjects ({commit, dispatch, state, rootState}) {
   const res = await dispatch('api', [url, init, null, {silent:true}]);
 
   if (res) {
-    commit('setProjects', res);
-    commit('setProjectsTimestamp');
+
+    let projects;
+
+    if (res.some( project => project.pid == null )) {
+      console.warn("At least one project found with no PID!");
+      projects = res.filter( project => project.pid != null );
+    } else {
+      projects = res;
+    }
+
+    commit('setProjects', projects); // First without summaries
+    commit('setProjectsTimestamp', tstamp);
+
+    projects = await Promise.all(projects.map( getSummary ));
+
+    commit('setProjects', projects); // Then with summaries
   }
   commit('clearProjectsLoading');
+  dispatch('prepareGroups');
 }
 
 /** Return a subset of projects from state.projects
@@ -118,4 +147,83 @@ async function getProjects ({commit, dispatch, state}, [{pid, name, schema, grou
   return {projects: filteredProjects, count};
 }
 
-export default { refreshProjects, getProjects };
+
+
+async function prepareGroups ({commit, dispatch, state, rootState}) {
+  const groups = {};
+
+  for (const project of state.projects) {
+
+    if (!project.prod_distance) {
+      // This project has no production data (either not started yet
+      // or production data has not been imported) so we skip it.
+      continue;
+    }
+
+    if (!project.prod_duration.days) {
+      project.prod_duration = normalise_duration(project.prod_duration);
+    }
+
+    for (const name of project.groups) {
+      if (!(name in groups)) {
+        groups[name] = {
+          group: name,
+          num_projects: 0,
+          lines: 0,
+          points: 0,
+          sequences: 0,
+          // Shots:
+          prime: 0,
+          other: 0,
+          ntba: 0,
+          prod_duration: {
+            days: 0,
+            hours: 0,
+            minutes: 0,
+            seconds: 0,
+            milliseconds: 0
+          },
+          prod_distance: 0,
+          shooting_rate: [],
+          projects: []
+        };
+      }
+      const group = groups[name];
+
+      group.num_projects++;
+      group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+      group.points = Math.max(group.points, project.total); // Idem
+      group.sequences += project.seq_final;
+      group.prime += project.prime;
+      group.other += project.other;
+      //group.ntba += project.ntba;
+      group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+      group.prod_distance += project.prod_distance;
+      group.shooting_rate.push(project.shooting_rate);
+      group.projects.push(project);
+    }
+  }
+
+  const grouplist = [];
+  for (const group of Object.values(groups)) {
+    group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+    group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+    delete group.shooting_rate;
+
+    grouplist.push(group);
+  }
+
+  commit('setGroups', grouplist);
+
+}
+
+async function getGroups({commit, dispatch, state, rootState}) {
+  if (!state.groups.length) {
+    await dispatch('refreshProjects');
+  }
+
+  return state.groups;
+}
+
+
+export default { refreshProjects, getProjects, prepareGroups, getGroups };

lib/www/client/source/src/store/modules/projects/getters.js

@@ -3,7 +3,7 @@ function projects (state) {
   return state.projects;
 }
 
-function projectGroups (state) {
+function projectGroupNames (state) {
   return [...new Set(state.projects.map(i => i.groups).flat())].sort();
 }
 
@@ -15,4 +15,8 @@ function projectsLoading (state) {
   return !!state.loading;
 }
 
-export default { projects, projectGroups, projectCount, projectsLoading };
+function groups (state) {
+  return state.groups;
+}
+
+export default { projects, projectGroupNames, projectCount, projectsLoading, groups };

lib/www/client/source/src/store/modules/projects/mutations.js

@@ -39,10 +39,15 @@ function abortProjectsLoading (state) {
   state.loading = null;
 }
 
+function setGroups (state, groups) {
+  state.groups = Object.freeze(groups);
+}
+
 export default {
   setProjects,
   setProjectsLoading,
   clearProjectsLoading,
   setProjectsTimestamp,
-  setProjectsETag
+  setProjectsETag,
+  setGroups
 };

lib/www/client/source/src/store/modules/projects/state.js

@@ -1,5 +1,6 @@
 const state = () => ({
   projects: Object.freeze([]),
+  groups: Object.freeze([]),
   loading: null,
   timestamp: null,
   etag: null,

lib/www/client/source/src/views/Group.vue

@@ -0,0 +1,339 @@
+<template>
+    <dougal-group-map v-if="mapView"
+      :baseline="baseline"
+      :monitor="monitor"
+      :monitors="monitors"
+      @input="mapView=$event"
+    ></dougal-group-map>
+  <v-container fluid fill-height class="ma-0 pa-0" v-else>
+
+    <v-overlay :value="loading && !comparisons.length" absolute>
+      <v-progress-circular
+        indeterminate
+        size="64"
+      ></v-progress-circular>
+    </v-overlay>
+
+    <v-overlay :value="!loading && !groupFound" absolute opacity="0.8">
+      <v-row justify="center">
+        <v-alert
+          type="error"
+        >
+          Group not found
+        </v-alert>
+      </v-row>
+      <v-row justify="center">
+        <v-btn color="primary" @click="refreshProjects">Retry</v-btn>
+      </v-row>
+    </v-overlay>
+
+    <v-row no-gutters align="stretch" class="fill-height">
+      <v-col cols="12" v-if="groupFound">
+
+        <v-data-table class="ma-1"
+          :headers="projectHeaders"
+          :items="projects"
+          dense
+        >
+
+          <template v-slot:item.baseline="{item, value, index}">
+            <v-simple-checkbox v-if="index+1 < projects.length"
+              color="primary"
+              :value="baseline === item"
+              @input="setBaseline(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.monitor="{item, value, index}">
+            <v-simple-checkbox v-if="index > 0 && !(index <= baselineIndex)"
+              color="primary"
+              :value="monitor === item"
+              @input="setMonitor(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.pid="{item, value}">
+            <v-chip
+              label
+              small
+              outlined
+              :href="`/projects/${item.pid}`"
+              :color="!item.archived ? 'primary' : ''"
+            >{{ value }}</v-chip>
+          </template>
+
+          <template v-slot:item.fsp="{item, value}">
+            <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.lsp="{item, value}">
+            <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.prod_duration="{item, value}">
+            <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+              {{ value.days }} d
+            </span>
+            <span v-else>
+              {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+            </span>
+          </template>
+
+          <template v-slot:item.prod_distance="{item, value}">
+            {{ (value/1000).toFixed(1) }} km
+          </template>
+
+          <template v-slot:footer.prepend>
+
+            <v-btn v-if="baseline && !mapView"
+              text
+              color="primary"
+              title="Switch to map view"
+              @click="mapView = true"
+            >View map</v-btn>
+
+            <v-btn v-if="comparison"
+              text
+              color="primary"
+              title="Back to summary"
+              @click="clearComparison"
+            >Back</v-btn>
+          </template>
+
+        </v-data-table>
+
+<!-- BEGIN TEST -->
+
+    <dougal-group-comparison-summary v-if="comparison"
+      :baseline="baseline"
+      :monitor="monitor"
+      :comparison="comparison"
+    ></dougal-group-comparison-summary>
+
+    <dougal-group-repeatability-summary v-else-if="comparisons.length"
+      :comparisons="comparisons"
+      :projects="projects"
+      @input="setComparison"
+    ></dougal-group-repeatability-summary>
+
+<!-- END   TEST -->
+
+
+      </v-col>
+      <v-col cols="12" v-else>
+        <v-card>
+          <v-card-text>
+            Group does not exist.
+          </v-card-text>
+        </v-card>
+      </v-col>
+    </v-row>
+  </v-container>
+</template>
+
+<script>
+import { mapActions, mapGetters } from 'vuex'
+import AccessMixin from '@/mixins/access';
+import DougalGroupRepeatabilitySummary from '@/components/groups/group-repeatability-summary.vue';
+import DougalGroupComparisonSummary from '@/components/groups/group-comparison-summary';
+import DougalGroupMap from '@/components/groups/group-map';
+
+export default {
+  name: 'Group',
+
+  mixins: [
+    AccessMixin
+  ],
+
+  components: {
+    DougalGroupRepeatabilitySummary,
+    DougalGroupComparisonSummary,
+    DougalGroupMap
+  },
+
+  data () {
+    return {
+      projectHeaders: [
+        {
+          value: "baseline",
+          text: "Baseline"
+        },
+        {
+          value: "monitor",
+          text: "Monitor"
+        },
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      mapView: false,
+
+      baseline: null,
+      monitor: null,
+      comparisons: []
+    }
+  },
+
+  computed: {
+
+    groupName () {
+      return this.$route.params.group;
+    },
+
+    group () {
+      return this.groups.find( i => i.group === this.groupName );
+    },
+
+    groupFound () {
+      return !!(this.loading || this.group);
+    },
+
+    projects () {
+      return this.group?.projects.toSorted((a, b) => a.pid.localeCompare(b.pid));
+    },
+
+    baselineIndex () {
+      return this.projects.indexOf(this.baseline);
+    },
+
+    monitors () {
+      if (this.baseline && this.comparisons) {
+        return this.comparisons
+          .filter( i => i.baseline_pid == this.baseline.pid )
+          .map( i => this.projects.find( p => p.pid == i.monitor_pid ));
+      } else {
+        return null;
+      }
+    },
+
+    comparison () {
+      return this.comparisons.find( row =>
+        row.baseline_pid == this.baseline?.pid && row.monitor_pid == this.monitor?.pid
+      )?.meta;
+    },
+
+    ...mapGetters(["loading", "groups"])
+  },
+
+  methods: {
+
+    setBaseline (project) {
+      if (project === this.baseline) {
+        this.baseline = null;
+      } else {
+        this.baseline = project;
+        if (this.monitor) {
+          if (this.projects.indexOf(this.monitor) <= this.projects.indexOf(this.baseline)) {
+            this.monitor = null;
+          }
+        }
+      }
+    },
+
+    setMonitor (project) {
+      if (project === this.monitor) {
+        this.monitor = null;
+      } else {
+        this.monitor = project;
+      }
+    },
+
+    clearComparison () {
+      this.baseline = null;
+      this.monitor = null;
+    },
+
+    setComparison (baseline, monitor) {
+      this.clearComparison();
+      this.setBaseline(baseline);
+      this.setMonitor(monitor);
+    },
+
+    async getComparisons () {
+      const url = `/comparison/group/${this.$route.params.group}`;
+      this.comparisons = await this.api([url]);
+    },
+
+    // TODO Should this go in a Vuex action rather?
+    async refreshComparisons () {
+      await this.getGroups();
+      if (this.groupFound) {
+        await this.getComparisons();
+      }
+    },
+
+    /*
+    async getComparison () {
+      if (this.baseline && this.monitor) {
+        const url = `/comparison/group/${this.$route.params.group}/baseline/${this.baseline.pid}/monitor/${this.monitor.pid}`;
+        const comparison = await this.api([url]);
+        if (comparison) {
+          this.comparison = comparison;
+        }
+      }
+    },
+    */
+
+    handleComparisons (context, {payload}) {
+      this.refreshComparisons();
+    },
+
+    registerNotificationHandlers (action = "registerHandler") {
+
+      this.$store.dispatch(action, {
+        table: 'comparisons',
+        handler: this.handleComparisons
+      });
+
+    },
+
+    unregisterNotificationHandlers () {
+      return this.registerNotificationHandlers("unregisterHandler");
+    },
+
+
+    ...mapActions(["api", "getGroups", "refreshProjects"])
+  },
+
+  async mounted () {
+    this.registerNotificationHandlers();
+    this.refreshComparisons()
+  },
+
+  beforeDestroy () {
+    this.unregisterNotificationHandlers();
+  }
+
+
+}
+</script>

lib/www/client/source/src/views/GroupList.vue

@@ -0,0 +1,396 @@
+<template>
+  <v-container fluid>
+
+    <v-data-table
+    :headers="headers"
+    :items="displayItems"
+    item-key="group"
+    :options.sync="options"
+    :expanded.sync="expanded"
+    show-expand
+    :loading="loading"
+    >
+
+      <template v-slot:item.group="{item, value}">
+        <v-chip
+          label
+          small
+          :href="`./${value}`"
+        >{{ value }}</v-chip>
+      </template>
+
+      <template v-slot:item.shots_total="{item, value}">
+        <div>{{ item.prime + item.other }}</div>
+        <v-progress-linear
+          background-color="secondary"
+          color="primary"
+          :value="item.prime/(item.prime+item.other)*100"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:item.prime="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.other="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.prod_duration="{item, value}">
+        <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+          {{ value.days }} d
+        </span>
+        <span v-else>
+          {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+        </span>
+      </template>
+
+      <template v-slot:item.prod_distance="{item, value}">
+        {{ (value/1000).toFixed(1) }} km
+      </template>
+
+      <template v-slot:item.shooting_rate_mean="{item, value}">
+        {{ (value).toFixed(2) }} s ±{{ (item.shooting_rate_sd).toFixed(3) }} s
+      </template>
+
+      <template v-slot:item.shots_per_point="{item, value}">
+        <div>
+          {{ ((item.prime + item.other)/item.points).toFixed(1) }}
+          ({{ ((((item.prime + item.other)/item.points) / item.num_projects)*100).toFixed(2) }}%)
+        </div>
+        <v-progress-linear
+          :value="((((item.prime + item.other)/item.points) / item.num_projects)*100)"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:expanded-item="{ headers, item }">
+        <td :colspan="headers.length">
+          <v-data-table class="ma-1"
+            :headers="projectHeaders"
+            :items="item.projects"
+            dense
+            hide-default-footer
+          >
+
+            <template v-slot:item.pid="{item, value}">
+              <a :href="`/projects/${value}`" title="Go to project">{{ value }}</a>
+            </template>
+
+            <template v-slot:item.fsp="{item, value}">
+              <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.lsp="{item, value}">
+              <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.prod_duration="{item, value}">
+              <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+                {{ value.days }} d
+              </span>
+              <span v-else>
+                {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+              </span>
+            </template>
+
+            <template v-slot:item.prod_distance="{item, value}">
+              {{ (value/1000).toFixed(1) }} km
+            </template>
+
+          </v-data-table>
+        </td>
+      </template>
+
+    </v-data-table>
+
+  </v-container>
+</template>
+
+<style>
+td p:last-of-type {
+  margin-bottom: 0;
+}
+</style>
+
+<script>
+import { mapActions, mapGetters } from 'vuex';
+import AccessMixin from '@/mixins/access';
+
+
+// FIXME send to lib/utils or so
+/*
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+*/
+
+export default {
+  name: "GroupList",
+
+  components: {
+  },
+
+  mixins: [
+    AccessMixin
+  ],
+
+  data () {
+    return {
+      headers: [
+        {
+          value: "group",
+          text: "Group name"
+        },
+        {
+          value: "num_projects",
+          text: "Number of campaigns"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "points",
+          text: "Preplot points"
+        },
+        {
+          value: "sequences",
+          text: "Total sequences"
+        },
+        {
+          value: "shots_total",
+          text: "Total shots"
+        },
+        {
+          value: "prime",
+          text: "Total prime"
+        },
+        {
+          value: "other",
+          text: "Total reshoot + infill"
+        },
+        /*
+        {
+          value: "ntba",
+          text: "Total NTBA"
+        },
+        */
+        {
+          value: "prod_duration",
+          text: "Total duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Total distance"
+        },
+        {
+          value: "shooting_rate_mean",
+          text: "Shooting rate (mean)"
+        },
+        {
+          value: "shots_per_point",
+          text: "Shots per point"
+        },
+      ],
+      items: [],
+      expanded: [],
+      options: { sortBy: ["group"], sortDesc: [false] },
+
+      projectHeaders: [
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      // Context menu stuff
+      contextMenuShow: false,
+      contextMenuX: 0,
+      contextMenuY: 0,
+      contextMenuItem: null,
+
+      /*
+      // FIXME Eventually need to move this into Vuex
+      groups: []
+      */
+    }
+  },
+
+  computed: {
+    displayItems () {
+      return this.items.filter(i => i.prod_distance);
+    },
+
+    ...mapGetters(['loading', 'groups'])
+  },
+
+  methods: {
+
+    /*
+    async prepareGroups () {
+      //const groups = await this.api(["/prospects"]);
+      //console.log("groups", groups);
+      const groups = {};
+
+      for (const project of this.projects) {
+
+        if (!project.prod_distance) {
+          // This project has no production data (either not started yet
+          // or production data has not been imported) so we skip it.
+          continue;
+        }
+
+        if (!project.prod_duration.days) {
+          project.prod_duration = normalise_duration(project.prod_duration);
+        }
+
+        for (const name of project.groups) {
+          if (!(name in groups)) {
+            groups[name] = {
+              group: name,
+              num_projects: 0,
+              lines: 0,
+              points: 0,
+              sequences: 0,
+              // Shots:
+              prime: 0,
+              other: 0,
+              ntba: 0,
+              prod_duration: {
+                days: 0,
+                hours: 0,
+                minutes: 0,
+                seconds: 0,
+                milliseconds: 0
+              },
+              prod_distance: 0,
+              shooting_rate: [],
+              projects: []
+            };
+          }
+          const group = groups[name];
+
+          group.num_projects++;
+          group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+          group.points = Math.max(group.points, project.total); // Idem
+          group.sequences += project.seq_final;
+          group.prime += project.prime;
+          group.other += project.other;
+          //group.ntba += project.ntba;
+          group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+          group.prod_distance += project.prod_distance;
+          group.shooting_rate.push(project.shooting_rate);
+          group.projects.push(project);
+        }
+      }
+
+      this.groups = [];
+      for (const group of Object.values(groups)) {
+        group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+        group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+        delete group.shooting_rate;
+
+        this.groups.push(group);
+      }
+
+    },
+    */
+
+    async list () {
+      this.items = [...this.groups];
+    },
+
+    async load () {
+      await this.refreshProjects();
+      //await this.prepareGroups();
+      await this.list();
+    },
+
+    registerNotificationHandlers () {
+      this.$store.dispatch('registerHandler', {
+        table: 'project`',
+
+        handler: (context, message) => {
+          if (message.payload?.table == "public") {
+            this.load();
+          }
+        }
+      });
+    },
+
+    ...mapActions(["api", "showSnack", "refreshProjects"])
+  },
+
+  mounted () {
+    this.registerNotificationHandlers();
+    this.load();
+  }
+}
+
+</script>

lib/www/client/source/src/views/Map.vue

@@ -470,6 +470,33 @@
           @click="zoomOut"
           >mdi-magnify-minus-outline</v-icon>
       </div>
+      <div>
+        <v-icon
+          class="my-1"
+          title="Tilt out"
+          @click="tiltOut"
+          >mdi-axis-x-rotate-counterclockwise</v-icon>
+      </div>
+      <div>
+        <v-icon
+          class="my-1"
+          title="Tilt in"
+          @click="tiltIn"
+          >mdi-axis-x-rotate-clockwise</v-icon>
+      </div>
+      <div>
+        <v-icon v-if="bearing==0"
+          class="my-1"
+          title="Bin up"
+          @click="setBearing('ζ')"
+          >mdi-view-grid-outline</v-icon>
+        <v-icon v-else
+          class="my-1"
+          title="North up"
+          :style="`transform: rotate(${-bearing}deg);`"
+          @click="setBearing(0)"
+          >mdi-navigation</v-icon>
+      </div>
       <div>
         <v-icon
           class="my-1"
@@ -661,6 +688,7 @@ export default {
         //maxZoom: 18,
         maxPitch: 89
       },
+      bearing: 0,
 
       vesselPosition: null,
       vesselTrackLastRefresh: 0,
@@ -977,6 +1005,41 @@ export default {
       }
     },
 
+    tiltIn () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch -= 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    tiltOut () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch += 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    setBearing (bearing) {
+      if (deck) {
+
+        if (bearing === 'ζ') {
+          bearing = this.$store.getters.projectConfiguration?.binning?.theta ?? 0;
+        }
+
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.bearing = (bearing + 360) % 360;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
     toggleFullscreen() {
       const mapElement = document.getElementById('map-container');
       if (!this.isFullscreen) {
@@ -1368,7 +1431,7 @@ export default {
       //console.log("SHOULD BE INITIALISING LAYERS HERE", gl);
       this.decodeURL();
       this.decodeURLHash();
-      deck.onViewStateChange = this.updateURL;
+      //deck.onViewStateChange = this.viewStateUpdated;
     },
 
     setViewState () {
@@ -1383,6 +1446,11 @@ export default {
       }
     },
 
+    viewStateUpdated ({viewState}) {
+      this.bearing = viewState.bearing;
+      this.updateURL({viewState});
+    },
+
     updateURL ({viewState} = {}) {
       if (!viewState && deck?.viewManager) {
         viewState = deck.getViewports()[0];
@@ -1709,7 +1777,8 @@ export default {
       layers: [],
       getTooltip: this.getTooltip,
       pickingRadius: 24,
-      onWebGLInitialized: this.initLayers
+      onWebGLInitialized: this.initLayers,
+      onViewStateChange: this.viewStateUpdated,
     });
 
     // Get fullscreen state

lib/www/client/source/src/views/ProjectList.vue

@@ -27,6 +27,8 @@
         <v-chip v-for="group in value"
           label
           small
+          :title="`View repeatability data for ${group}`"
+          :href="`/groups/${group}`"
         >{{ group }}</v-chip>
       </template>
 
@@ -187,19 +189,20 @@ export default {
     ...mapGetters(['loading', 'projects'])
   },
 
+  watch: {
+
+    async projects () {
+      await this.load();
+    }
+
+  },
+
   methods: {
 
     async list () {
       this.items = [...this.projects];
     },
 
-    async summary (item) {
-      const details = await this.api([`/project/${item.pid}/summary`]);
-      if (details) {
-        return Object.assign({}, details, item);
-      }
-    },
-
     title (item) {
       if (item.organisations) {
         return "Access:\n" + Object.entries(item.organisations).map( org =>
@@ -210,30 +213,22 @@ export default {
     },
 
     async load () {
-      await this.refreshProjects();
+      if (!this.projects.length) {
+        this.refreshProjects();
+      }
       await this.list();
-      const promises = [];
-      for (const key in this.items) {
-        const item = this.items[key];
-        const promise = this.summary(item)
-        .then( expanded => {
-          if (expanded) {
-            this.$set(this.items, key, expanded);
-          }
-        });
-        promises.push(promise);
+    },
+
+    handlerLoad (context, {payload}) {
+      if (payload?.table == "public") {
+        this.load();
       }
     },
 
     registerNotificationHandlers () {
       this.$store.dispatch('registerHandler', {
         table: 'project`',
-
-        handler: (context, message) => {
-          if (message.payload?.table == "public") {
-            this.load();
-          }
-        }
+        handler: this.handlerLoad
       });
     },
 

lib/www/server/api/index.js

@@ -357,6 +357,26 @@ app.map({
 			delete: [ mw.auth.operations, mw.auth.access.write, mw.info.delete ]
 		}
 	},
+
+	/*
+	 * 4D comparisons
+	 */
+
+	// FIXME no authentication yet!
+
+	'/comparison/group': {
+		get: [ mw.etag.noSave, mw.comparisons.groups.list ],
+		'/:group': {
+			get: [ mw.etag.noSave, mw.comparisons.groups.get ],
+		},
+	},
+
+
+
+	/*
+	 * Other endpoints
+	 */
+
 	'/queue/outgoing/': {
 		'asaqc': {
 			get: [ mw.etag.noSave, mw.queue.asaqc.get ],

lib/www/server/api/middleware/comparisons/groups/get.js

@@ -0,0 +1,20 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		// const data = await comparisons.groups()
+		// if (data?.[req.params.group]) {
+		const data = await comparisons.getGroup(req.params.group);
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(404).send({message: "Group does not exist"});
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/groups/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+	list: require('./list'),
+	get: require('./get'),
+}

lib/www/server/api/middleware/comparisons/groups/list.js

@@ -0,0 +1,18 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const data = await comparisons.groups()
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(204).end();
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	groups: require('./groups')
+}

lib/www/server/api/middleware/index.js

@@ -23,4 +23,5 @@ module.exports = {
 	version: require('./version'),
 	admin: require('./admin'),
 	compress: require('./compress'),
+	comparisons: require('./comparisons'),
 };

lib/www/server/lib/binary/bundle.js

@@ -38,7 +38,7 @@ function bundle (json, opts = {}) {
 
 		return encode.sequential(json, el => el.sailline, el => el.point, deltas, values, type)
 
-	} if (type == 1) {
+	} else if (type == 1) {
 		/* Preplot information – source line points
 		 *
 		 * elem 0: Float32Array Longitude
@@ -273,6 +273,113 @@ function bundle (json, opts = {}) {
 		});
 
 		return encode.sequential(json, el => el.sequence, el => el.point, deltas, values, type)
+	} else if (type == 4) {
+		/* Bare final positions
+		 *
+		 * Δelem    0: Sequence no. (Uint16Array, Uint8Array)
+		 * elem   0‒1: Float32Array, Float32Array – Final positions (x, y)
+		 *
+		 */
+
+		deltas.push({
+			key: el => el[2],
+			baseType: Uint16Array,
+			incrType: Int8Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[4],
+			type: Float32Array
+		});
+
+		return encode.sequential(json, el => el[0], el => el[1], deltas, values, type)
+	} else if (type == 0xa) {
+		/* 4D comparison data:
+		 *
+		 * elem0: i differences
+		 * elem1: j differences
+		 *
+		 * Note that line/point may not be unique.
+		 *
+		 */
+
+		/*
+		deltas.push({
+			key: el => el.baseTStamp,
+			baseType: BigUint64Array,
+			incrType: Int32Array
+		});
+
+		deltas.push({
+			key: el => el.monTStamp,
+			baseType: BigUint64Array,
+			incrType: Int32Array
+		})
+		*/
+
+		values.push({
+			key: el => el[2],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		/*
+		values.push({
+			key: el => el.baseSeq,
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el.monSeq,
+			type: Uint16Array
+		});
+		*/
+
+		return encode.sequential(json, el => el[0], el => el[1], deltas, values, type)
+	} else if (type == 0xc) {
+		/* 4D comparison data (reduced sample)
+		 *
+		 * Input is comparison records, i.e.:
+		 * [ [ line, point, δi, δj ], … ]
+		 *
+		 * elem0: line
+		 * elem1: point
+		 * elem2: δi
+		 * elem3: δj
+		 *
+		 * Note that the chunk's `i` and `j` values are not used
+		 */
+
+		values.push({
+			key: el => el[0],
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el[1],
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el[2],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		return encode.sequential(json, el => 0, el => 0, deltas, values, type)
 	}
 }
 

lib/www/server/lib/comparisons/geometric-differences.js

@@ -0,0 +1,393 @@
+const d3a = require('d3-array');
+const { DougalBinaryBundle } = require('@dougal/binary');
+const { pool, setSurvey } = require('../db/connection');
+const db = require('../db');
+const { bundle } = require('../binary/bundle');
+const setops = require('../utils/setops');
+const { ijRMS, combinations, computeSample } = require('./utils');
+const { computePCA } = require('./pca');
+const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+
+async function fetchErrors (pid) {
+
+	const client = await setSurvey(pid);
+
+	try {
+		const text = `
+			SELECT
+				fs.line, fs.point,
+				ij_error(fs.line::double precision, fs.point::double precision, fs.geometry)::json AS errorfinal
+			FROM
+				final_shots fs
+			ORDER BY fs.line, fs.point;
+		`;
+
+		const res = await client.query(text);
+
+		return res.rows.map( row =>
+			[row.line, row.point, row.errorfinal.coordinates[0], row.errorfinal.coordinates[1]]
+		);
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function groupTimestamps (groupName) {
+	const projects = await groups()?.[groupName];
+	if (projects?.length) {
+
+	}
+}
+
+
+async function groups () {
+	const projects = await db.project.get({timestamps: true});
+	const groupNames = [
+		...projects
+			.reduce( (acc, cur) => acc.add(...cur.groups), new Set() )
+	].filter( i => !!i );
+
+	return Object.fromEntries(groupNames.map( g => [g, projects.filter( p => p.groups.includes(g) )] ));
+}
+
+function geometric_differences (baseline, monitor) {
+
+	if (!baseline || !baseline.length) {
+		throw new Error("No baseline data");
+	}
+
+	if (!monitor || !monitor.length) {
+		throw new Error("No monitor data");
+	}
+
+	const comparison = []; // An array of { line, point, εi, εj }; line + point may be repeated
+
+	for (const bp of baseline) {
+		const monitor_points = monitor.filter( mp => mp[0] === bp[0] && mp[1] === bp[1] );
+
+		if (!monitor_points.length) {
+			// console.log(`No match for L${bp[0]} P${bp[1]}`);
+			continue;
+		}
+
+		for (const mp of monitor_points) {
+			const εi = mp[2] - bp[2], εj = mp[3] - bp[3];
+			comparison.push([bp[0], bp[1], εi, εj]);
+		}
+
+	}
+
+	return comparison;
+}
+
+async function compare (baselineProjectID, monitorProjectID, infoObj) {
+	console.log("Getting baseline", baselineProjectID);
+	const baselineData = await fetchErrors(baselineProjectID);
+	console.log("Getting monitor", monitorProjectID);
+	const monitorData = await fetchErrors(monitorProjectID);
+	console.log("Comparing");
+
+	const comparison = geometric_differences(baselineData, monitorData);
+
+	if (infoObj instanceof Object) {
+		const baselineIJ = baselineData.map(i => i.slice(0,2));
+		const monitorIJ = monitorData.map(i => i.slice(0,2));
+
+		infoObj.compared = comparison.length;
+		infoObj.baselineLength = baselineData.length;
+		infoObj.monitorLength = monitorData.length;
+		infoObj.baselineUniqueLength = setops.unique(baselineIJ).length;
+		infoObj.monitorUniqueLength = setops.unique(monitorIJ).length;
+		infoObj.common = setops.intersection(baselineIJ, monitorIJ).length;
+	}
+
+	return comparison;
+}
+
+
+async function save (baselineProjectID, monitorProjectID, bundle, meta) {
+	const info = {};
+	if (!bundle) {
+		const comparison = await compare(baselineProjectID, monitorProjectID, info);
+		if (comparison.length) {
+			bundle = asBundle(comparison);
+		} else {
+			console.warn(`No matching points between ${baselineProjectID} and ${monitorProjectID}`);
+			return;
+		}
+	} else if (!(bundle instanceof DougalBinaryBundle)) {
+		throw new Error("Illegal data: `bundle` must of null or of type DougalBinaryBundle");
+	}
+
+	if (!bundle.byteLength) {
+		console.warn(`Empty comparison results between ${baselineProjectID} and ${monitorProjectID}. Refusing to store`);
+		return;
+	}
+
+	meta = {tstamp: (new Date()), ...info, ...stats(bundle), ...meta};
+
+	console.log("Storing in database");
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			INSERT INTO comparisons.comparisons
+				(type, baseline_pid, monitor_pid, data, meta)
+			VALUES ('geometric_difference', $1, $2, $3, $4)
+			ON CONFLICT (type, baseline_pid, monitor_pid)
+			DO UPDATE SET
+				data = EXCLUDED.data,
+				meta = EXCLUDED.meta;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, Buffer.from(bundle), meta ];
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function get (baselineProjectID, monitorProjectID, type = 'geometric_difference') {
+
+	const client = await pool.connect();
+
+	try {
+
+		const text = `
+			SELECT data, meta
+			FROM comparisons.comparisons
+			WHERE type = $3 AND baseline_pid = $1 AND monitor_pid = $2;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, type ];
+		const res = await client.query(text, values);
+		if (!res.rows.length) {
+			console.log("Comparison not found in database");
+			return;
+		}
+
+		const { data, meta } = res.rows[0];
+		return {
+			data: DougalBinaryBundle.clone(data),
+			meta
+		};
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function getSample (baselineProjectID, monitorProjectID) {
+	return await get(baselineProjectID, monitorProjectID, 'geometric_difference_sample');
+}
+
+
+async function remove (baselineProjectID, monitorProjectID) {
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			DELETE
+			FROM comparisons.comparisons
+			WHERE
+				(type = 'geometric_difference' OR type = 'geometric_difference_sample')
+				AND baseline_pid = $1
+				AND monitor_pid = $2;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID ];
+
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+function stats (comparison) {
+	let i, j, δi, δj;
+
+	if (comparison instanceof DougalBinaryBundle) {
+		console.log("Computing stats");
+		const udv = comparison.chunks()[0]?.udv;
+
+		if (!udv) {
+			console.error("Could not determine udv from first chunk");
+			console.log(comparison.chunks());
+			return;
+		}
+
+		let records;
+
+		if (udv == 0xa) {
+			records = comparison.records;
+
+			// Transpose the records
+			[ i, j, δi, δj ] = Array.from({ length: 4 }, (_, i) => records.map(row => row[i]));
+		} else if (udv == 0xc) {
+			records = comparison.records;
+			let _;
+			[ _, _, i, j, δi, δj ] = Array.from({ length: 6 }, (_, i) => records.map(row => row[i]));
+		} else {
+			throw new Error(`Unrecognised DougalBinaryBundle User Defined Value: ${udv}`);
+		}
+
+		return {
+			length: records.length,
+			μ: [ d3a.mean(δi), d3a.mean(δj) ],
+			σ: [ d3a.deviation(δi), d3a.deviation(δj) ],
+			//rms: ijRMS(δi, δj),
+			...computePCA(records)
+		}
+	} else if (Array.isArray(comparison)) {
+		if (Array.isArray(comparison[0])) {
+			return stats(asBundle(comparison, {type: 0xc}));
+		} else {
+			// Assume object
+			return stats(asBundle(comparison));
+		}
+	}
+}
+
+
+function sortFn (a, b) {
+	if (a.line == b.line) {
+		if (a.point == b.point) {
+			return a.baseTStamp - b.baseTStamp;
+		} else {
+			return a.point - b.point;
+		}
+	} else {
+		return a.line - b.line;
+	}
+}
+
+function asBundle (comparison, opts = {type: 0x0a}) {
+	return DougalBinaryBundle.clone(bundle(comparison, opts));
+}
+
+function fromBundle (bundle) {
+	if (!(bundle instanceof DougalBinaryBundle)) {
+		bundle = DougalBinaryBundle.clone(bundle);
+	}
+
+	const json = [];
+	for (const record of bundle) {
+		record.shift();
+		json.push(record);
+	}
+	return json;
+}
+
+async function saveGroup (group, opts = {}) {
+	if (group == null) {
+		// Save everything
+		const g = await groups();
+		for (const group of Object.values(g)) {
+			await saveGroup(group)
+		}
+	} if (typeof group === "string") {
+		// This is a group name
+		const g = await groups();
+		group = groups[g];
+	}
+
+	if (Array.isArray(group)) {
+		const pids = group.map( i => i.pid ).sort();
+
+		for (const [ baselineProjectID, monitorProjectID ] of combinations(pids, 2)) {
+			try {
+				if (!opts.overwrite) {
+					const exists = await get(baselineProjectID, monitorProjectID);
+					if (exists) {
+						DEBUG("Not overwriting existing comparison between %s and %s. Skipping", baselineProjectID, monitorProjectID);
+						continue;
+					}
+				}
+
+				await save(baselineProjectID, monitorProjectID);
+				DEBUG("Saved comparison between %s and %s", baselineProjectID, monitorProjectID);
+			} catch (err) {
+				console.error(err);
+				ERROR("Error saving comparison between %s and %s", baselineProjectID, monitorProjectID);
+			}
+		}
+	}
+}
+
+
+async function getGroup (groupName, opts = {}) {
+
+	const group = (await groups())?.[groupName]?.map( i => i.pid)?.sort();
+
+	if (!group?.length) return;
+
+	const client = await pool.connect();
+
+	try {
+
+		const pairs = combinations(group, 2);
+		const flatValues = pairs.flat();
+		const placeholders = [];
+		for (let i = 0; i < pairs.length; i++) {
+			placeholders.push(`($${i * 2 + 1}, $${i * 2 + 2})`);
+		}
+		const inClause = placeholders.join(',');
+		const selectFields = opts.returnData ? 'data, meta' : 'meta';
+
+		const text = `
+			SELECT baseline_pid, monitor_pid, ${selectFields}
+			FROM comparisons.comparisons
+			WHERE type = 'geometric_difference'
+			AND (baseline_pid, monitor_pid) IN (VALUES ${inClause})
+			ORDER BY baseline_pid, monitor_pid
+		`;
+
+		const res = await client.query(text, flatValues);
+		if (!res.rows.length) {
+			console.log("Comparison not found in database");
+			return;
+		}
+
+		if (opts.returnData) {
+			return res.rows.map( row => ({
+				...row,
+				data: DougalBinaryBundle.clone(row.data),
+			}));
+		} else {
+			return res.rows;
+		}
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+module.exports = {
+	groups,
+	fetchErrors,
+	compare,
+	computeSample,
+	get,
+	save,
+	getSample,
+	saveGroup,
+	getGroup,
+	remove,
+	stats,
+	asBundle,
+	fromBundle
+};

lib/www/server/lib/comparisons/index.js

@@ -0,0 +1,4 @@
+
+module.exports = {
+	...require('./geometric-differences')
+}

lib/www/server/lib/comparisons/pca.js

@@ -0,0 +1,83 @@
+const math = require('mathjs');
+
+/**
+ * Compute PCA (eigenvectors and eigenvalues) for deviation data to assess geometric repeatability.
+ * @param {Array<Array<number>>} deviations - Array of [point, line, i_deviation, j_deviation]
+ * @returns {Object} - { eigenvalues, eigenvectors, rms, anisotropy, primaryDirection }
+ */
+function computePCA(deviations) {
+	// Extract i_deviation and j_deviation
+	const deviationMatrix = deviations.map(row => [row[2], row[3]]);
+
+	// Convert to mathjs matrix
+	const D = math.matrix(deviationMatrix);
+
+	// Compute mean for centering (1 x 2 matrix)
+	const mean = math.mean(D, 0);
+
+	// Manually repeat-mean to match D's shape (n x 2)
+	const n = deviationMatrix.length;
+	const meanArr = mean.toArray();
+	const meanRepeated = math.matrix(
+		Array(n).fill().map(() => [meanArr[0], meanArr[1]])
+	);
+
+	// Center the data
+	const centered = math.subtract(D, meanRepeated);
+
+	// Compute covariance matrix: (1/(n-1)) * (D_centered^T * D_centered)
+	const covMatrix = math.multiply(
+		math.multiply(1 / (n - 1), math.transpose(centered)),
+		centered
+	);
+
+	// Perform eigen decomposition
+	const result = math.eigs(covMatrix);
+	let eigenvalues = result.values;
+	const evObjs = result.eigenvectors;
+
+	// Convert eigenvalues to array if it's a matrix
+	eigenvalues = Array.isArray(eigenvalues) ? eigenvalues : eigenvalues.toArray();
+
+	// Create pairs and convert vector to array if necessary
+	const pairs = eigenvalues.map((val, i) => {
+		let vec = evObjs[i].vector;
+		if (vec.toArray) vec = vec.toArray();
+		return { val, vec };
+	});
+
+	// Sort by descending eigenvalues
+	pairs.sort((a, b) => b.val - a.val);
+
+	// Sorted eigenvalues
+	const sortedEigenvalues = pairs.map(p => p.val);
+
+	// Build eigenvector matrix: rows as components, columns as eigenvectors
+	const dimension = pairs[0].vec.length; // e.g., 2
+	const evecRows = [];
+	for (let comp = 0; comp < dimension; comp++) {
+		evecRows.push(pairs.map(p => p.vec[comp]));
+	}
+	const sortedEigenvectors = math.matrix(evecRows);
+
+	// Compute RMS errors along principal axes
+	const rms = sortedEigenvalues.map(val => Math.sqrt(Math.max(val, 0)));
+
+	// Compute anisotropy (ratio of major to minor axis variance)
+	const anisotropy = sortedEigenvalues[0] / (sortedEigenvalues[1] || 1); // Avoid division by zero
+
+	// Primary direction (angle in degrees of major eigenvector)
+	const primaryVector = sortedEigenvectors.subset(math.index([0, 1], 0)).toArray();
+	const primaryDirection = Math.atan2(primaryVector[1], primaryVector[0]) * 180 / Math.PI;
+
+	return {
+		eigenvalues: sortedEigenvalues,
+		eigenvectors: sortedEigenvectors.toArray(),
+		rms: rms, // RMS errors along major/minor axes
+		anisotropy: anisotropy, // Ratio of variances
+		primaryDirection: primaryDirection // Angle of major axis (degrees)
+	};
+}
+
+
+module.exports = { computePCA };

lib/www/server/lib/comparisons/utils.js

@@ -0,0 +1,310 @@
+const d3 = require('d3-array');
+
+// Function to calculate the root mean square (RMS) of position deviations
+// This computes the RMS of the Euclidean distances: sqrt( (1/n) * sum(δi² + δj²) )
+// Assumes deviations are already centered (mean deviation ~0); if normalization by std dev or range is needed, adjust accordingly
+function ijRMS(δi, δj) {
+	if (!δi.length || !δj.length) return 0;
+
+	if (δi.length != δj.length) {
+		console.warn(`δi and δj have different lengths!`);
+	}
+
+	let sumSquares = 0;
+	const n = Math.min(δi.length, δj.length);
+
+	for (let i=0; i < n; i++) {
+		sumSquares += (δi[i] * δi[i]) + (δj[i] * δj[i]);
+	}
+
+	const meanSquare = sumSquares / n;
+	const rms = Math.sqrt(meanSquare);
+
+	return rms;
+}
+
+
+/**
+ * Performs stratified sampling on an array of [line, point, δi, δj] data points.
+ * Groups by line and samples proportionally to preserve shape and spread.
+ *
+ * @param {Array<Array<number>>} data - Input data: [[line, point, δi, δj], ...]
+ * @param {number} sampleSize - Target number of samples (e.g., 2000)
+ * @returns {Array<Array<number>>} Sampled data in same format
+ */
+function old_stratifiedSample(data, sampleSize) {
+	if (!Array.isArray(data) || data.length === 0) return [];
+	if (!Number.isInteger(sampleSize) || sampleSize <= 0) {
+		throw new Error('sampleSize must be a positive integer');
+	}
+
+	// Group data by line (first element)
+	const grouped = d3.group(data, d => d[0]);
+	const totalSize = data.length;
+	const sampled = [];
+
+	// Ensure sampleSize doesn't exceed data size
+	const effectiveSampleSize = Math.min(sampleSize, totalSize);
+
+	// Iterate over each line group
+	for (const [line, group] of grouped) {
+		// Calculate proportional sample size for this group
+		const groupSize = group.length;
+		const groupSampleSize = Math.max(1, Math.round((groupSize / totalSize) * effectiveSampleSize));
+
+		// Shuffle group and take first N elements
+		const shuffled = d3.shuffle([...group]);
+		sampled.push(...shuffled.slice(0, groupSampleSize));
+	}
+
+	// If sampled size is slightly off due to rounding, adjust
+	if (sampled.length > effectiveSampleSize) {
+		return d3.shuffle(sampled).slice(0, effectiveSampleSize);
+	} else if (sampled.length < effectiveSampleSize) {
+		// Pad with random samples from entire dataset if needed
+		const remaining = effectiveSampleSize - sampled.length;
+		const additional = d3.shuffle(data.filter(d => !sampled.includes(d))).slice(0, remaining);
+		sampled.push(...additional);
+	}
+
+	return sampled;
+}
+
+/**
+ * Performs stratified sampling on an array of [line, point, δi, δj] data points.
+ * Stratifies by line and δi quantiles to preserve shape and spread, with outlier control.
+ *
+ * @param {Array<Array<number>>} data - Input data: [[line, point, δi, δj], ...]
+ * @param {number} sampleSize - Target number of samples (e.g., 2000)
+ * @param {number} [binsPerLine=10] - Number of δi quantile bins per line
+ * @returns {Array<Array<number>>} Sampled data in same format
+ */
+function stratifiedSample(data, sampleSize, binsPerLine = 10) {
+	if (!Array.isArray(data) || data.length === 0) return [];
+	if (!Number.isInteger(sampleSize) || sampleSize <= 0) {
+		throw new Error('sampleSize must be a positive integer');
+	}
+	if (!Number.isInteger(binsPerLine) || binsPerLine <= 0) {
+		throw new Error('binsPerLine must be a positive integer');
+	}
+
+	const totalSize = data.length;
+	const effectiveSampleSize = Math.min(sampleSize, totalSize);
+	const sampled = [];
+
+	// Group by line
+	const groupedByLine = d3.group(data, d => d[0]);
+
+	// Compute population stats for validation
+	const populationStats = computeStats(data);
+
+	// Iterate over each line
+	for (const [line, group] of groupedByLine) {
+		const groupSize = group.length;
+		const lineSampleSize = Math.max(1, Math.round((groupSize / totalSize) * effectiveSampleSize));
+
+		// Create quantile-based bins for δi
+		const δiValues = group.map(d => d[2]).sort(d3.ascending);
+		const quantiles = d3.range(0, binsPerLine + 1).map(i => d3.quantile(δiValues, i / binsPerLine));
+		const binnedData = group.map(d => {
+			const δi = d[2];
+			let binIndex = 0;
+			for (let i = 0; i < binsPerLine; i++) {
+				if (δi >= quantiles[i] && δi < quantiles[i + 1]) {
+					binIndex = i;
+					break;
+				}
+			}
+			return { data: d, bin: binIndex };
+		});
+		const groupedByBin = d3.group(binnedData, d => d.bin);
+
+		// Allocate samples across bins, inversely weighted by density to control outliers
+		const binSampleSizes = new Map();
+		let remainingLineSamples = lineSampleSize;
+		const binCounts = Array(binsPerLine).fill(0);
+		for (const [bin, binGroup] of groupedByBin) {
+			binCounts[bin] = binGroup.length;
+		}
+		const maxBinCount = d3.max(binCounts);
+		for (const [bin, binGroup] of groupedByBin) {
+			const binSize = binGroup.length;
+			// Inverse weighting: smaller bins (outliers) get fewer samples
+			const weight = binSize > 0 ? Math.max(0.1, 1 - (binSize / maxBinCount) * 0.5) : 1;
+			const binSampleSize = Math.max(1, Math.round(lineSampleSize * (binSize / groupSize) * weight));
+			binSampleSizes.set(bin, Math.min(binSampleSize, binSize));
+			remainingLineSamples -= binSampleSizes.get(bin);
+		}
+
+		// Distribute remaining samples
+		if (remainingLineSamples > 0) {
+			const nonEmptyBins = Array.from(groupedByBin.keys());
+			for (let i = 0; i < remainingLineSamples && nonEmptyBins.length > 0; i++) {
+				const bin = nonEmptyBins[i % nonEmptyBins.length];
+				binSampleSizes.set(bin, binSampleSizes.get(bin) + 1);
+			}
+		}
+
+		// Sample from each bin
+		for (const [bin, binGroup] of groupedByBin) {
+			const samples = d3.shuffle([...binGroup]).slice(0, binSampleSizes.get(bin)).map(s => s.data);
+			sampled.push(...samples);
+		}
+	}
+
+	// Adjust sample size
+	let finalSample = sampled;
+	if (sampled.length > effectiveSampleSize) {
+		finalSample = d3.shuffle(sampled).slice(0, effectiveSampleSize);
+	} else if (sampled.length < effectiveSampleSize) {
+		const remaining = effectiveSampleSize - sampled.length;
+		const additional = d3.shuffle(data.filter(d => !sampled.includes(d))).slice(0, remaining);
+		finalSample = [...sampled, ...additional];
+	}
+
+	// Validate and adjust if stats are off
+	const sampleStats = computeStats(finalSample);
+	const statTolerance = { μ: 0.1, σ: 0.2 }; // Allowable relative deviation
+	const needsAdjustment =
+		Math.abs(sampleStats.μ[0] - populationStats.μ[0]) / populationStats.μ[0] > statTolerance.μ ||
+		Math.abs(sampleStats.μ[1] - populationStats.μ[1]) / populationStats.μ[1] > statTolerance.μ ||
+		Math.abs(sampleStats.σ[0] - populationStats.σ[0]) / populationStats.σ[0] > statTolerance.σ ||
+		Math.abs(sampleStats.σ[1] - populationStats.σ[1]) / populationStats.σ[1] > statTolerance.σ;
+
+	if (needsAdjustment) {
+		// Add points from underrepresented regions
+		const δiSample = finalSample.map(d => d[2]);
+		const δiPopulation = data.map(d => d[2]);
+		const quantiles = d3.range(0, binsPerLine + 1).map(i => d3.quantile(δiPopulation, i / binsPerLine));
+		const sampleBins = d3.histogram().domain(d3.extent(δiPopulation)).thresholds(quantiles)(δiSample);
+		const populationBins = d3.histogram().domain(d3.extent(δiPopulation)).thresholds(quantiles)(δiPopulation);
+		const underSampledBins = sampleBins
+			.map((b, i) => ({ bin: i, diff: populationBins[i].length / totalSize - b.length / finalSample.length }))
+			.filter(b => b.diff > 0.1); // Significant under-sampling
+
+		if (underSampledBins.length > 0) {
+			const additionalSamples = [];
+			for (const { bin } of underSampledBins) {
+				const binData = data.filter(d => d[2] >= quantiles[bin] && d[2] < quantiles[bin + 1] && !finalSample.includes(d));
+				const needed = Math.round((underSampledBins[0].diff * effectiveSampleSize) / 2);
+				additionalSamples.push(...d3.shuffle(binData).slice(0, needed));
+			}
+			finalSample = d3.shuffle([...finalSample, ...additionalSamples]).slice(0, effectiveSampleSize);
+		}
+	}
+
+	return finalSample;
+}
+
+function decimate (data, decimationCount = 20) {
+	return data.filter( (row, index) => (index % decimationCount) == 0 );
+}
+
+function computeSample (data, opts = {}) {
+	const DEFAULT_SAMPLE_SIZE = 2000;
+	let sample;
+
+	if (opts.decimate === true) {
+		if (opts.sampleSize > 0) {
+			sample = decimate(data.records, Math.floor(data.records.length / opts.sampleSize));
+		} else {
+			sample = decimate(data.records, Math.floor(data.records.length / DEFAULT_SAMPLE_SIZE));
+		}
+	} else if (opts.decimate > 0) {
+		sample = decimate(data.records, opts.decimate);
+	} else if (opts.sampleSize) {
+		sample = stratifiedSample(data.records, opt.sampleSize);
+	} else {
+		sample = stratifiedSample(data.records, DEFAULT_SAMPLE_SIZE);
+	}
+
+	return sample;
+}
+
+// Optional: Utility to compute stats for validation
+function computeStats(data) {
+	const δi = data.map(d => d[2]);
+	const δj = data.map(d => d[3]);
+	const rms = Math.sqrt(d3.mean(data, d => d[2] ** 2 + d[3] ** 2));
+	return {
+		l: data.length,
+		μ: [d3.mean(δi), d3.mean(δj)],
+		σ: [d3.deviation(δi), d3.deviation(δj)],
+		rms
+	};
+}
+
+function centre (data) {
+	const stats = computeStats(data);
+
+	return data.map( row => [row[0], row[1], row[2]-stats.μ[0], row[3]-stats.μ[1]] )
+}
+
+function outliers (data, sd=1.96) {
+	const stats = computeStats(data);
+
+	function fn ([l, p, i, j]) {
+		return (i - stats.μ[0]) > stats.σ[0]*sd ||
+			(j - stats.μ[1]) > stats.σ[1]*sd;
+	}
+
+	return data.filter(fn)
+}
+
+function inliers (data, sd=1.96) {
+	const stats = computeStats(data);
+
+	function fn ([l, p, i, j]) {
+		return (i - stats.μ[0]) <= stats.σ[0]*sd &&
+			(j - stats.μ[1]) <= stats.σ[1]*sd;
+	}
+
+	return data.filter(fn)
+}
+
+function difference (a, b) {
+	const obj = Array.isArray(a) ? [] : {};
+	for (const k in a) {
+		const v0 = a[k];
+		const v1 = b[k]
+		if (v0 instanceof Object && v1 instanceof Object) {
+			obj[k] = difference (v0, v1);
+		} else if (!isNaN(Number(v0)) && !isNaN(Number(v1))) {
+			obj[k] = v1 - v0;
+		}
+	}
+	return obj;
+}
+
+function combinations (a, n) {
+	const results = [];
+
+	function combine(current, start) {
+		if (current.length === n) {
+			results.push([...current]);
+			return;
+		}
+		for (let i = start; i < a.length; i++) {
+			current.push(a[i]);
+			combine(current, i + 1);
+			current.pop();
+		}
+	}
+
+	combine([], 0);
+	return results;
+}
+
+module.exports = {
+	combinations,
+	centre,
+	ijRMS,
+	computeStats,
+	computeSample,
+	stratifiedSample,
+	old_stratifiedSample,
+	decimate,
+	difference,
+	outliers,
+	inliers
+}

lib/www/server/lib/db/channels.js

@@ -10,5 +10,6 @@ module.exports = [
 	"planned_lines",
 	"raw_lines", "raw_shots",
 	"final_lines", "final_shots", "info",
-	"queue_items"
+	"queue_items",
+	"comparisons",
 ];

lib/www/server/lib/db/project/get.js

@@ -1,8 +1,14 @@
 const { setSurvey, pool } = require('../connection');
 
-async function get () {
+async function get (opts = {}) {
+
+	const select = opts.timestamps
+		? "last_project_update(pid) tstamp,"
+		: "";
+
 	const text = `
 		SELECT
+			${select}
 			pid,
 			name,
 			schema,

lib/www/server/lib/db/sequence/get.js

@@ -15,11 +15,50 @@ async function getSummary (projectId, sequence, opts = {}) {
 	return res.rows[0];
 }
 
+async function getPoints (projectId, sequence, opts = {}) {
+
+	const offset = Math.abs(opts.offset) || Math.abs((opts.page-1)*opts.itemsPerPage) || 0;
+	const limit = Math.abs(opts.limit) || Math.abs(Number(opts.itemsPerPage)) || null;
+
+	const client = await setSurvey(projectId);
+
+	const restriction = sequence
+		? "sequence = $3"
+		: "TRUE OR $3";
+
+	const text = `
+		SELECT line, point, sequence, st_x(ST_Transform(geometry, 4326)) longitude, st_y(ST_Transform(geometry, 4326)) latitude
+		FROM final_shots
+		WHERE ${restriction}
+		ORDER BY sequence, point
+		OFFSET $1
+		LIMIT $2;
+	`;
+
+	try {
+		const res = await client.query({text, values: [offset, limit, sequence], rowMode: 'array'});
+		return res.rows;
+	} catch (err) {
+		console.error(err);
+		// throw { status: 500, message: "Internal error" };
+	} finally {
+		client.release();
+	}
+
+}
 
 async function get (projectId, sequence, opts = {}) {
 	if (opts.summary) {
 		return await getSummary(projectId, sequence, opts);
 	}
+	if (opts.type == 4) {
+		// The user is request that we send just the bare details:
+		// sequence, sailline, line, longitude, latitude.
+		//
+		// This will probably be a binary data request (though doesn't
+		// need to).
+		return await getPoints(projectId, sequence, opts);
+	}
 
 	const client = await setSurvey(projectId);
 

lib/www/server/lib/utils/setops.js

@@ -0,0 +1,52 @@
+
+function unique(arr) {
+	const set = new Set(arr.map(JSON.stringify));
+	return Array.from(set).map(JSON.parse);
+}
+
+function duplicates(arr) {
+	const seen = new Set();
+	const dups = new Set();
+	for (const item of arr.map(JSON.stringify)) {
+		if (seen.has(item)) {
+			dups.add(item);
+		} else {
+			seen.add(item);
+		}
+	}
+	return Array.from(dups).map(JSON.parse);
+}
+
+function union(arr1, arr2) {
+	const set = new Set([...arr1, ...arr2].map(JSON.stringify));
+	return Array.from(set).map(JSON.parse);
+}
+
+
+function intersection(arr1, arr2) {
+	const set2 = new Set(arr2.map(JSON.stringify));
+	return arr1.filter(item => set2.has(JSON.stringify(item)));
+}
+
+function difference(arr1, arr2) {
+	const set2 = new Set(arr2.map(JSON.stringify));
+	return arr1.filter(item => !set2.has(JSON.stringify(item)));
+}
+
+function symmetricDifference(arr1, arr2) {
+	const set1 = new Set(arr1.map(JSON.stringify));
+	const set2 = new Set(arr2.map(JSON.stringify));
+	return [
+		...arr1.filter(item => !set2.has(JSON.stringify(item))),
+		...arr2.filter(item => !set1.has(JSON.stringify(item)))
+	];
+}
+
+module.exports = {
+	unique,
+	duplicates,
+	union,
+	intersection,
+	difference,
+	symmetricDifference
+}

lib/www/server/package.json

@@ -16,7 +16,7 @@
       "api": "0.4.0"
     },
     "wanted": {
-      "db_schema": "^0.6.0"
+      "db_schema": "^0.6.6"
     }
   },
   "engines": {
@@ -44,6 +44,7 @@
     "jsonwebtoken": "^9.0.2",
     "leaflet-headless": "git+https://git@gitlab.com/aaltronav/contrib/leaflet-headless.git#devel",
     "marked": "^4.0.12",
+    "mathjs": "^14.6.0",
     "node-fetch": "^2.6.1",
     "nunjucks": "^3.2.3",
     "path-to-regexp": "^6.2.1",

package-lock.json

@@ -1678,17 +1678,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "lib/www/client/source/node_modules/@babel/runtime": {
-      "version": "7.23.2",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "regenerator-runtime": "^0.14.0"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      }
-    },
     "lib/www/client/source/node_modules/@babel/template": {
       "version": "7.27.2",
       "dev": true,
@@ -7524,11 +7513,6 @@
         "node": ">=4"
       }
     },
-    "lib/www/client/source/node_modules/regenerator-runtime": {
-      "version": "0.14.0",
-      "dev": true,
-      "license": "MIT"
-    },
     "lib/www/client/source/node_modules/regenerator-transform": {
       "version": "0.15.2",
       "dev": true,
@@ -9374,6 +9358,7 @@
         "jsonwebtoken": "^9.0.2",
         "leaflet-headless": "git+https://git@gitlab.com/aaltronav/contrib/leaflet-headless.git#devel",
         "marked": "^4.0.12",
+        "mathjs": "^14.6.0",
         "node-fetch": "^2.6.1",
         "nunjucks": "^3.2.3",
         "path-to-regexp": "^6.2.1",
@@ -10527,17 +10512,6 @@
         "node": ">=6.0.0"
       }
     },
-    "lib/www/server/node_modules/redoc-cli/node_modules/@babel/runtime": {
-      "version": "7.16.7",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "regenerator-runtime": "^0.13.4"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      }
-    },
     "lib/www/server/node_modules/redoc-cli/node_modules/@babel/template": {
       "version": "7.12.13",
       "dev": true,
@@ -12532,11 +12506,6 @@
         "url": "https://github.com/Mermade/oas-kit?sponsor=1"
       }
     },
-    "lib/www/server/node_modules/redoc-cli/node_modules/regenerator-runtime": {
-      "version": "0.13.9",
-      "dev": true,
-      "license": "MIT"
-    },
     "lib/www/server/node_modules/redoc-cli/node_modules/require-directory": {
       "version": "2.1.1",
       "dev": true,
@@ -13297,6 +13266,15 @@
         "node": ">=0.4"
       }
     },
+    "node_modules/@babel/runtime": {
+      "version": "7.28.3",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.3.tgz",
+      "integrity": "sha512-9uIQ10o0WGdpP6GDhXcdOJPJuDgFtIDtN/9+ArJQ2NAfAmiuhTQdzkaTGR33v43GYS2UrSA0eX2pPPHoFVvpxA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
     "node_modules/@deck.gl/aggregation-layers": {
       "version": "9.1.13",
       "resolved": "https://registry.npmjs.org/@deck.gl/aggregation-layers/-/aggregation-layers-9.1.13.tgz",
@@ -14389,6 +14367,19 @@
         "node": ">= 10"
       }
     },
+    "node_modules/complex.js": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.2.tgz",
+      "integrity": "sha512-qtx7HRhPGSCBtGiST4/WGHuW+zeaND/6Ld+db6PbrulIB1i2Ev/2UPiqcmpQNPSyfBKraC0EOvOKCB5dGZKt3g==",
+      "license": "MIT",
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/rawify"
+      }
+    },
     "node_modules/compressible": {
       "version": "2.0.18",
       "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
@@ -15089,6 +15080,12 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/escape-latex": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz",
+      "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==",
+      "license": "MIT"
+    },
     "node_modules/escodegen": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz",
@@ -15205,6 +15202,19 @@
         "node": ">= 6"
       }
     },
+    "node_modules/fraction.js": {
+      "version": "5.3.1",
+      "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.1.tgz",
+      "integrity": "sha512-PhqCuhSKIGbbkJ+cojHv47eEWClU71FIOhiUsYdZYTwhIzCeIN8rXeEjserTvPat5JLJChumn8chHz64WkZgTw==",
+      "license": "MIT",
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/rawify"
+      }
+    },
     "node_modules/fs-minipass": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -15635,6 +15645,12 @@
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
       "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g=="
     },
+    "node_modules/javascript-natural-sort": {
+      "version": "0.7.1",
+      "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz",
+      "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==",
+      "license": "MIT"
+    },
     "node_modules/jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
@@ -15848,6 +15864,29 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/mathjs": {
+      "version": "14.6.0",
+      "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.6.0.tgz",
+      "integrity": "sha512-5vI2BLB5GKQmiSK9BH6hVkZ+GgqpdnOgEfmHl7mqVmdQObLynr63KueyYYLCQMzj66q69mV2XZZGQqqxeftQbA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@babel/runtime": "^7.26.10",
+        "complex.js": "^2.2.5",
+        "decimal.js": "^10.4.3",
+        "escape-latex": "^1.2.0",
+        "fraction.js": "^5.2.1",
+        "javascript-natural-sort": "^0.7.1",
+        "seedrandom": "^3.0.5",
+        "tiny-emitter": "^2.1.0",
+        "typed-function": "^4.2.1"
+      },
+      "bin": {
+        "mathjs": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/md5": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
@@ -16343,6 +16382,12 @@
         "node": ">=10"
       }
     },
+    "node_modules/seedrandom": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+      "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==",
+      "license": "MIT"
+    },
     "node_modules/semver": {
       "version": "7.7.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
@@ -16606,6 +16651,12 @@
         "texture-compressor": "bin/texture-compressor.js"
       }
     },
+    "node_modules/tiny-emitter": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz",
+      "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==",
+      "license": "MIT"
+    },
     "node_modules/toidentifier": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -16667,6 +16718,15 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/typed-function": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.2.1.tgz",
+      "integrity": "sha512-EGjWssW7Tsk4DGfE+5yluuljS1OGYWiI1J6e8puZz9nTMM51Oug8CD5Zo4gWMsOhq5BI+1bF+rWTm4Vbj3ivRA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/undici-types": {
       "version": "7.8.0",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",