Change CPU load scaling

Resolve "Add 4D comparisons"

Closes #334

See merge request wgp/dougal/software!60

bin/runner.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# Maximum runtime in seconds before killing an overdue instance (e.g., 10 minutes)
+MAX_RUNTIME_SECONDS=$((15 * 60))
 
 DOUGAL_ROOT=${DOUGAL_ROOT:-$(dirname "$0")/..}
 
@@ -80,8 +82,9 @@ function run () {
 # 		DESCRIPTION=""
 		SERVICE="deferred_imports"
 
-		$BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \
-			-O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)"
+# Disable GitLab alerts. They're just not very practical
+# 		$BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \
+# 			-O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)"
 
 		exit 2
 	}
@@ -97,14 +100,37 @@ function cleanup () {
 }
 
 if [[ -f $LOCKFILE ]]; then
-	PID=$(cat "$LOCKFILE")
-	if pgrep -F "$LOCKFILE"; then
-		print_warning $(printf "The previous process is still running (%d)" $PID)
-		exit 1
-	else
-		rm "$LOCKFILE"
-		print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" $PID)
-	fi
+    PID=$(cat "$LOCKFILE")
+    if kill -0 "$PID" 2>/dev/null; then  # Check if process is running
+        # Get elapsed time in D-HH:MM:SS format and convert to seconds
+        ELAPSED_STR=$(ps -p "$PID" -o etime= | tr -d '[:space:]')
+        if [ -n "$ELAPSED_STR" ]; then
+            # Convert D-HH:MM:SS to seconds
+            ELAPSED_SECONDS=$(echo "$ELAPSED_STR" | awk -F'[-:]' '{
+                seconds = 0
+                if (NF == 4) { seconds += $1 * 86400 }  # Days
+                if (NF >= 3) { seconds += $NF-2 * 3600 } # Hours
+                if (NF >= 2) { seconds += $NF-1 * 60 }   # Minutes
+                seconds += $NF                           # Seconds
+                print seconds
+            }')
+            if [ "$ELAPSED_SECONDS" -gt "$MAX_RUNTIME_SECONDS" ]; then
+                # Kill the overdue process (SIGTERM; use -9 for SIGKILL if needed)
+                kill "$PID" 2>/dev/null
+                print_warning $(printf "Killed overdue process (%d) that ran for %s (%d seconds)" "$PID" "$ELAPSED_STR" "$ELAPSED_SECONDS")
+                rm "$LOCKFILE"
+            else
+                print_warning $(printf "Previous process is still running (%d) for %s (%d seconds)" "$PID" "$ELAPSED_STR" "$ELAPSED_SECONDS")
+                exit 1
+            fi
+        else
+            print_warning $(printf "Could not retrieve elapsed time for process (%d)" "$PID")
+            exit 1
+        fi
+    else
+        rm "$LOCKFILE"
+        print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" "$PID")
+    fi
 fi
 
 echo "$$" > "$LOCKFILE" || {

bin/update_comparisons.js

@@ -0,0 +1,89 @@
+#!/usr/bin/node
+
+const cmp = require('../lib/www/server/lib/comparisons');
+
+async function purgeComparisons () {
+	const groups = await cmp.groups();
+	const comparisons = await cmp.getGroup();
+
+	const pids = new Set(Object.values(groups).flat().map( p => p.pid ));
+	const comparison_pids = new Set(comparisons.map( c => [ c.baseline_pid, c.monitor_pid ] ).flat());
+
+	for (const pid of comparison_pids) {
+		if (!pids.has(pid)) {
+			console.log(`${pid} no longer par of a group. Deleting comparisons`);
+
+			staleComps = comparisons.filter( c => c.baseline_pid == pid || c.monitor_pid == pid );
+			for (c of staleComps) {
+				console.log(`Deleting comparison ${c.baseline_pid} → ${c.monitor_pid}`);
+				await cmp.remove(c.baseline_pid, c.monitor_pid);
+			}
+		}
+	}
+}
+
+
+async function main () {
+
+	console.log("Looking for unreferenced comparisons to purge");
+	await purgeComparisons();
+
+	console.log("Retrieving project groups");
+	const groups = await cmp.groups();
+
+	if (!Object.keys(groups??{})?.length) {
+		console.log("No groups found");
+		return 0;
+	}
+
+	console.log(`Found ${Object.keys(groups)?.length} groups: ${Object.keys(groups).join(", ")}`);
+
+	for (const groupName of Object.keys(groups)) {
+		const projects = groups[groupName];
+
+		console.log(`Fetching saved comparisons for ${groupName}`);
+
+		const comparisons = await cmp.getGroup(groupName);
+
+		if (!comparisons || !comparisons.length) {
+			console.log(`No comparisons found for ${groupName}`);
+			continue;
+		}
+
+		// Check if there are any projects that have been modified since last comparison
+		// or if there are any pairs that are no longer part of the group
+
+		const outdated = comparisons.filter( c => {
+			const baseline_tstamp = projects.find( p => p.pid === c.baseline_pid )?.tstamp;
+			const monitor_tstamp = projects.find( p => p.pid === c.monitor_pid )?.tstamp;
+			return (c.tstamp < baseline_tstamp) || (c.tstamp < monitor_tstamp) ||
+				baseline_tstamp == null || monitor_tstamp == null;
+		});
+
+		for (const comparison of outdated) {
+			console.log(`Removing stale comparison: ${comparison.baseline_pid} → ${comparison.monitor_pid}`);
+			await cmp.remove(comparison.baseline_pid, comparison.monitor_pid);
+		}
+
+		if (projects?.length < 2) {
+			console.log(`Group ${groupName} has less than two projects. No comparisons are possible`);
+			continue;
+		}
+
+		// Re-run the comparisons that are not in the database. They may
+		// be missing either beacause they were not there to start with
+		// or because we just removed them due to being stale
+
+		console.log(`Recalculating group ${groupName}`);
+		await cmp.saveGroup(groupName);
+	}
+
+	console.log("Comparisons update done");
+	return 0;
+}
+
+if (require.main === module) {
+	main();
+} else {
+	module.exports = main;
+}

etc/db/upgrades/upgrade41-v0.6.3-add-comparisons.sql

@@ -0,0 +1,109 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.3
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade creates a new schema called `comparisons`.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update adds a `comparisons` table to a `comparisons` schema.
+-- The `comparisons.comparisons` table holds 4D prospect comparison data.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE SCHEMA IF NOT EXISTS comparisons
+		AUTHORIZATION postgres;
+
+	COMMENT ON SCHEMA comparisons
+		IS 'Holds 4D comparison data and logic';
+
+	CREATE TABLE IF NOT EXISTS comparisons.comparisons
+	(
+		type text COLLATE pg_catalog."default" NOT NULL,
+		baseline_pid text COLLATE pg_catalog."default" NOT NULL,
+		monitor_pid text COLLATE pg_catalog."default" NOT NULL,
+		data bytea,
+		meta jsonb NOT NULL DEFAULT '{}'::jsonb,
+		CONSTRAINT comparisons_pkey PRIMARY KEY (baseline_pid, monitor_pid, type)
+	)
+
+	TABLESPACE pg_default;
+
+	ALTER TABLE IF EXISTS comparisons.comparisons
+		OWNER to postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.3"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.3"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade42-v0.6.4-notify-exclude-columns.sql

@@ -0,0 +1,169 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.4
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.notify()
+		RETURNS trigger
+		LANGUAGE 'plpgsql'
+		COST 100
+		VOLATILE NOT LEAKPROOF
+	AS $BODY$
+		DECLARE
+		channel text := TG_ARGV[0];
+		pid text;
+		payload text;
+		notification text;
+		payload_id integer;
+		old_json jsonb;
+		new_json jsonb;
+		excluded_col text;
+		i integer;
+		BEGIN
+
+		-- Fetch pid
+		SELECT projects.pid INTO pid FROM projects WHERE schema = TG_TABLE_SCHEMA;
+
+		-- Build old and new as jsonb, excluding specified columns if provided
+		IF OLD IS NOT NULL THEN
+			old_json := row_to_json(OLD)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				old_json := old_json - excluded_col;
+			END LOOP;
+		ELSE
+			old_json := NULL;
+		END IF;
+
+		IF NEW IS NOT NULL THEN
+			new_json := row_to_json(NEW)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				new_json := new_json - excluded_col;
+			END LOOP;
+		ELSE
+			new_json := NULL;
+		END IF;
+
+		-- Build payload
+		payload := json_build_object(
+			'tstamp', CURRENT_TIMESTAMP,
+			'operation', TG_OP,
+			'schema', TG_TABLE_SCHEMA,
+			'table', TG_TABLE_NAME,
+			'old', old_json,
+			'new', new_json,
+			'pid', pid
+		)::text;
+
+		-- Handle large payloads
+		IF octet_length(payload) < 1000 THEN
+			PERFORM pg_notify(channel, payload);
+		ELSE
+			-- Store large payload and notify with ID (as before)
+			INSERT INTO notify_payloads (payload) VALUES (payload) RETURNING id INTO payload_id;
+
+			notification := json_build_object(
+				'tstamp', CURRENT_TIMESTAMP,
+				'operation', TG_OP,
+				'schema', TG_TABLE_SCHEMA,
+				'table', TG_TABLE_NAME,
+				'pid', pid,
+				'payload_id', payload_id
+			)::text;
+
+			PERFORM pg_notify(channel, notification);
+			RAISE INFO 'Payload over limit';
+		END IF;
+		RETURN NULL;
+		END;
+	$BODY$;
+
+	ALTER FUNCTION public.notify()
+		OWNER TO postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.4"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.4"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade43-v0.6.5-add-comparisons-notification-channel.sql

@@ -0,0 +1,96 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.5
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE TRIGGER comparisons_tg
+		AFTER INSERT OR DELETE OR UPDATE
+		ON comparisons.comparisons
+		FOR EACH ROW
+		EXECUTE FUNCTION public.notify('comparisons', 'data');
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.5"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.5"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade44-v0.6.6-add-last_project_update-function.sql

@@ -0,0 +1,157 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.6
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This adds a last_project_update(pid) function. It takes a project ID
+-- and returns the last known timestamp from that project. Timestamps
+-- are derived from multiple sources:
+--
+-- - raw_shots table
+-- - final_shots table
+-- - events_log_full table
+-- - info table where key = 'qc'
+-- - files table, from the hashes (which contain the file's mtime)
+-- - project configuration, looking for an _updatedOn property
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.last_project_update(p_pid text)
+	RETURNS timestamp with time zone
+	LANGUAGE plpgsql
+	AS $function$
+	DECLARE
+		v_last_ts timestamptz := NULL;
+		v_current_ts timestamptz;
+		v_current_str text;
+		v_current_unix numeric;
+		v_sid_rec record;
+	BEGIN
+		-- From raw_shots, final_shots, info, and files
+		FOR v_sid_rec IN SELECT schema FROM public.projects WHERE pid = p_pid
+		LOOP
+			-- From raw_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.raw_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From final_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.final_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From info where key = 'qc'
+			EXECUTE 'SELECT value->>''updatedOn'' FROM ' || v_sid_rec.schema || '.info WHERE key = ''qc''' INTO v_current_str;
+			IF v_current_str IS NOT NULL THEN
+				v_current_ts := v_current_str::timestamptz;
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From files hash second part, only for valid colon-separated hashes
+			EXECUTE 'SELECT max( split_part(hash, '':'', 2)::numeric ) FROM ' || v_sid_rec.schema || '.files WHERE hash ~ ''^[0-9]+:[0-9]+\\.[0-9]+:[0-9]+\\.[0-9]+:[0-9a-f]+$''' INTO v_current_unix;
+			IF v_current_unix IS NOT NULL THEN
+				v_current_ts := to_timestamp(v_current_unix);
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From event_log_full
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.event_log_full' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+		END LOOP;
+
+		-- From projects.meta->_updatedOn
+		SELECT (meta->>'_updatedOn')::timestamptz FROM public.projects WHERE pid = p_pid INTO v_current_ts;
+		IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+			v_last_ts := v_current_ts;
+		END IF;
+
+		RETURN v_last_ts;
+	END;
+	$function$;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.6' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.6"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.6"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/modules/@dougal/binary/classes.js

@@ -503,6 +503,37 @@ class DougalBinaryBundle extends ArrayBuffer {
 		return ab;
 	}
 
+	get records () {
+		const data = [];
+		for (const record of this) {
+			data.push(record.slice(1));
+		}
+		return data;
+	}
+
+	[Symbol.iterator]() {
+		const chunks = this.chunks();
+		let chunkIndex = 0;
+		let chunkIterator = chunks.length > 0 ? chunks[0][Symbol.iterator]() : null;
+
+		return {
+			next() {
+				if (!chunkIterator) {
+					return { done: true };
+				}
+
+				let result = chunkIterator.next();
+				while (result.done && chunkIndex < chunks.length - 1) {
+					chunkIndex++;
+					chunkIterator = chunks[chunkIndex][Symbol.iterator]();
+					result = chunkIterator.next();
+				}
+
+				return result;
+			}
+		};
+	}
+
 }
 
 
@@ -693,7 +724,7 @@ class DougalBinaryChunkSequential extends ArrayBuffer {
 	getRecord (index) {
 		if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
 
-		const arr = [thid.udv, this.i, this.j0 + index * this.Δj];
+		const arr = [this.udv, this.i, this.j0 + index * this.Δj];
 
 		for (let m = 0; m < this.ΔelemCount; m++) {
 			const values = this.Δelem(m);
@@ -707,6 +738,21 @@ class DougalBinaryChunkSequential extends ArrayBuffer {
 
 		return arr;
 	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
 }
 
 
@@ -901,6 +947,21 @@ class DougalBinaryChunkInterleaved extends ArrayBuffer {
 
 		return arr;
 	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
 }
 
 

lib/www/client/source/package.json

@@ -9,10 +9,12 @@
   "dependencies": {
     "@deck.gl/aggregation-layers": "^9.1.13",
     "@deck.gl/geo-layers": "^9.1.13",
+    "@deck.gl/mesh-layers": "^9.1.14",
     "@dougal/binary": "file:../../../modules/@dougal/binary",
     "@dougal/concurrency": "file:../../../modules/@dougal/concurrency",
     "@dougal/organisations": "file:../../../modules/@dougal/organisations",
     "@dougal/user": "file:../../../modules/@dougal/user",
+    "@loaders.gl/obj": "^4.3.4",
     "@mdi/font": "^7.2.96",
     "buffer": "^6.0.3",
     "core-js": "^3.6.5",

lib/www/client/source/src/App.vue

@@ -85,25 +85,21 @@ export default {
     },
 
     handleProject (context, {payload}) {
-      this.refreshProjects();
+      if (payload?.table == "public") {
+        this.refreshProjects();
+      }
     },
 
     registerNotificationHandlers () {
 
       this.$store.dispatch('registerHandler', {
         table: '.jwt',
-
-        handler: (context, message) => {
-          this.handleJWT(context, message);
-        }
+        handler: this.handleJWT
       });
 
       this.$store.dispatch('registerHandler', {
         table: 'project',
-
-        handler: (context, message) => {
-          this.handleProject(context, message);
-        }
+        handler: this.handleProject
       });
 
     },

lib/www/client/source/src/components/footer.vue

@@ -10,7 +10,10 @@
 	<v-spacer></v-spacer>
 
 	<template v-if="isFrontendRemote">
-      <v-icon v-if="serverConnected" class="mr-6" title="Connected to server via gateway">mdi-cloud-outline</v-icon>
+      <template v-if="serverConnected">
+        <v-icon v-if="isGatewayReliable" class="mr-6" title="Connected to server via gateway">mdi-cloud-outline</v-icon>
+        <v-icon v-else class="mr-6" color="orange" title="Gateway connection is unreliable. Expect outages.">mdi-cloud-off</v-icon>
+      </template>
       <v-icon v-else class="mr-6" color="red" :title="`Server connection lost: the gateway cannot reach the remote server.\nWe will reconnect automatically when the link with the remote server is restored.`">mdi-cloud-off</v-icon>
 	</template>
 	<template v-else>
@@ -57,6 +60,13 @@ export default {
     DougalNotificationsControl
   },
 
+  data () {
+    return {
+      lastGatewayErrorTimestamp: 0,
+      gatewayErrorSilencePeriod: 60000,
+    }
+  },
+
   computed: {
     year () {
       const date = new Date();
@@ -65,8 +75,24 @@ export default {
 
     ...mapState({
       serverConnected: state => state.notify.serverConnected,
-      isFrontendRemote: state => state.api.serverInfo?.["remote-frontend"] ?? false
+      isFrontendRemote: state => state.api.serverInfo?.["remote-frontend"] ?? false,
+      isGatewayReliable: state => state.api.isGatewayReliable
     })
+  },
+
+  watch: {
+
+    isGatewayReliable (val) {
+      if (val === false) {
+        const elapsed = Date.now() - this.lastGatewayErrorTimestamp;
+        const lastGatewayErrorTimestamp = Date.now();
+        if (elapsed > this.gatewayErrorSilencePeriod) {
+          this.$root.showSnack("Gateway error", "warning");
+        }
+      }
+    }
+
   }
+
 };
 </script>

lib/www/client/source/src/components/graph-arrays-ij-scatter.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Array inline / crossline error
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="scatterplot" label="Scatterplot"></v-switch>
     <v-switch class="ml-4" v-model="histogram" label="Histogram"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -57,8 +59,8 @@ export default {
       graph: [],
       busy: false,
       resizeObserver: null,
-      scatterplot: false,
-      histogram: false
+      scatterplot: true,
+      histogram: true
     };
   },
 

lib/www/client/source/src/components/graph-guns-depth.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Gun depth
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="shotpoint" label="Shotpoint"></v-switch>
     <v-switch class="ml-4" v-model="violinplot" label="Violin plot"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -59,7 +61,7 @@ export default {
       busy: false,
       resizeObserver: null,
       shotpoint: true,
-      violinplot: false
+      violinplot: true
     };
   },
 

lib/www/client/source/src/components/graph-guns-pressure.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Gun pressures
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="shotpoint" label="Shotpoint"></v-switch>
     <v-switch class="ml-4" v-model="violinplot" label="Violin plot"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -59,7 +61,7 @@ export default {
       busy: false,
       resizeObserver: null,
       shotpoint: true,
-      violinplot: false
+      violinplot: true
     };
   },
 

lib/www/client/source/src/components/graph-guns-timing.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Gun timing
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="shotpoint" label="Shotpoint"></v-switch>
     <v-switch class="ml-4" v-model="violinplot" label="Violin plot"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -59,7 +61,7 @@ export default {
       busy: false,
       resizeObserver: null,
       shotpoint: true,
-      violinplot: false
+      violinplot: true
     };
   },
 

lib/www/client/source/src/components/graphs/project/sequence/inline-crossline.vue

@@ -39,7 +39,8 @@ export default {
         default:
           return {
             editable: false,
-            displaylogo: false
+            displaylogo: false,
+            responsive: true
           };
       }
     },
@@ -48,7 +49,8 @@ export default {
       const base = {
         font: {
           color: this.$vuetify.theme.isDark ? "#fff" : undefined
-        }
+        },
+        autosize: true
       };
 
       switch (this.facet) {
@@ -274,18 +276,25 @@ export default {
     replot () {
       if (this.plotted) {
         const ref = this.$refs.graph;
-        Plotly.relayout(ref, {
-          width: ref.clientWidth,
-          height: ref.clientHeight
-        });
+        if (ref && ref.clientWidth > 0 && ref.clientHeight > 0) {
+          Plotly.relayout(ref, {
+            width: ref.clientWidth,
+            height: ref.clientHeight
+          });
+        }
       }
     }
 
   },
 
   mounted () {
-    this.resizeObserver = new ResizeObserver(this.replot)
-    this.resizeObserver.observe(this.$refs.graph);
+    this.$nextTick( () => {
+      if (this.items?.length) {
+        this.plot();
+      }
+      this.resizeObserver = new ResizeObserver(this.replot)
+      this.resizeObserver.observe(this.$refs.graph);
+    });
   },
 
   beforeDestroy () {

lib/www/client/source/src/components/graphs/project/sequence/shotpoint-timing.vue

@@ -36,7 +36,8 @@ export default {
     config () {
       return {
         editable: false,
-        displaylogo: false
+        displaylogo: false,
+        responsive: true
       };
     },
 
@@ -53,7 +54,8 @@ export default {
           title: "Time (s)"
         },
         plot_bgcolor:"rgba(0,0,0,0)",
-        paper_bgcolor:"rgba(0,0,0,0)"
+        paper_bgcolor:"rgba(0,0,0,0)",
+        autosize: true
       };
     },
 
@@ -154,10 +156,12 @@ export default {
     replot () {
       if (this.plotted) {
         const ref = this.$refs.graph;
-        Plotly.relayout(ref, {
-          width: ref.clientWidth,
-          height: ref.clientHeight
-        });
+        if (ref && ref.clientWidth > 0 && ref.clientHeight > 0) {
+          Plotly.relayout(ref, {
+            width: ref.clientWidth,
+            height: ref.clientHeight
+          });
+        }
       }
     },
 
@@ -190,8 +194,13 @@ export default {
   },
 
   mounted () {
-    this.resizeObserver = new ResizeObserver(this.replot)
-    this.resizeObserver.observe(this.$refs.graph);
+    this.$nextTick( () => {
+      if (this.items?.length) {
+        this.plot();
+      }
+      this.resizeObserver = new ResizeObserver(this.replot)
+      this.resizeObserver.observe(this.$refs.graph);
+    });
   },
 
   beforeDestroy () {

lib/www/client/source/src/components/groups/group-comparison-summary.vue

@@ -0,0 +1,187 @@
+<template>
+  <v-card v-if="comparison" class="ma-1">
+    <v-card-title>Comparison Summary: Baseline {{ baseline.pid }} vs Monitor {{ monitor.pid }}</v-card-title>
+    <v-card-text>
+      <v-row>
+        <v-col cols="12" md="6">
+          <h3>Deviation Statistics</h3>
+          <v-simple-table dense>
+            <template v-slot:default>
+              <thead>
+                <tr>
+                  <th>Metric</th>
+                  <th>I (m)</th>
+                  <th>J (m)</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td>Mean (μ)</td>
+                  <td>{{ comparison['μ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['μ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>Std Dev (σ)</td>
+                  <td>{{ comparison['σ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['σ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>RMS</td>
+                  <td>{{ comparison.rms[0].toFixed(3) }}</td>
+                  <td>{{ comparison.rms[1].toFixed(3) }}</td>
+                </tr>
+              </tbody>
+            </template>
+          </v-simple-table>
+
+          <h3 class="mt-4">Error distribution</h3>
+          <ul>
+            <li title="Relative to I-axis positive direction">Primary Direction: {{ (comparison.primaryDirection * 180 / Math.PI).toFixed(2) }}°</li>
+            <li>Anisotropy: {{ comparison.anisotropy.toFixed(2) }}</li>
+            <li title="Length of the semi-major axis of the error ellipse">Semi-Major Axis: {{ semiMajorAxis.toFixed(2) }} m</li>
+            <li title="Length of the semi-minor axis of the error ellipse">Semi-Minor Axis: {{ semiMinorAxis.toFixed(2) }} m</li>
+            <li title="Area of the error ellipse">Error Ellipse Area: {{ ellipseArea.toFixed(2) }} m²</li>
+          </ul>
+
+          <h3 class="mt-4">Counts</h3>
+          <ul>
+            <li title="Unique line / point pairs found in both projects">Common Points: {{ comparison.common }}</li>
+            <li title="Total number of points compared, including reshoots, infills, etc.">Comparison Length: {{ comparison.length }}</li>
+            <li title="Number of points in the baseline project">Baseline Points: {{ comparison.baselineLength }} (Unique: {{ comparison.baselineUniqueLength }})</li>
+            <li title="Number of points in the monitor project">Monitor Points: {{ comparison.monitorLength }} (Unique: {{ comparison.monitorUniqueLength }})</li>
+          </ul>
+
+          <p class="mt-3" title="Date and time when the comparison was last performed">Computation timestamp: {{ new Date(comparison.tstamp).toLocaleString() }}</p>
+        </v-col>
+
+        <v-col cols="12" md="6">
+          <h3>Error Ellipse</h3>
+          <svg width="300" height="300" style="border: 1px solid #ccc;">
+            <g :transform="`translate(150, 150) scale(${ellipseScale})`">
+              <line x1="0" y1="-150" x2="0" y2="150" stroke="lightgray" stroke-dasharray="5,5"/>
+              <line x1="-150" y1="0" x2="150" y2="0" stroke="lightgray" stroke-dasharray="5,5"/>
+              <ellipse
+                :rx="Math.sqrt(comparison.eigenvalues[0])"
+                :ry="Math.sqrt(comparison.eigenvalues[1])"
+                :transform="`rotate(${ellipseAngle})`"
+                fill="none"
+                stroke="blue"
+                stroke-width="2"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[0]) * Math.cos(ellipseRad)"
+                :y2="Math.sqrt(comparison.eigenvalues[0]) * Math.sin(ellipseRad)"
+                stroke="red"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[1]) * Math.cos(ellipseRad + Math.PI / 2)"
+                :y2="Math.sqrt(comparison.eigenvalues[1]) * Math.sin(ellipseRad + Math.PI / 2)"
+                stroke="green"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+            </g>
+          </svg>
+          <p class="text-caption">Ellipse scaled for visibility (factor: {{ ellipseScale.toFixed(1) }}). Axes represent sqrt(eigenvalues).</p>
+        </v-col>
+      </v-row>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: "DougalGroupComparisonSummary",
+
+  props: {
+    baseline: { type: Object, required: true },
+    monitor: { type: Object, required: true },
+    comparison: { type: Object, required: true }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+
+    ellipseAngle () {
+      if (!this.comparison) return 0;
+      const ev = this.comparison.eigenvectors[0];
+      return Math.atan2(ev[1], ev[0]) * 180 / Math.PI;
+    },
+
+    ellipseRad () {
+      return this.ellipseAngle * Math.PI / 180;
+    },
+
+    ellipseRx () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[0]) * this.ellipseScale;
+    },
+
+    ellipseRy () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[1]) * this.ellipseScale;
+    },
+
+    ellipseScale () {
+      if (!this.comparison) return 1;
+      const maxSigma = Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+      const maxMu = Math.max(
+        Math.abs(this.comparison['μ'][0]),
+        Math.abs(this.comparison['μ'][1])
+      );
+      //const maxExtent = maxMu + 3 * maxSigma;
+      const maxExtent = 20;
+      return 100 / maxExtent; // Adjust scale to fit within ~200 pixels diameter
+    },
+
+    ellipseArea () {
+      if (!this.comparison) return 0;
+      const a = Math.sqrt(this.comparison.eigenvalues[0]);
+      const b = Math.sqrt(this.comparison.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+
+    semiMajorAxis () {
+      if (!this.comparison) return 0;
+      return Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    semiMinorAxis () {
+      if (!this.comparison) return 0;
+      return Math.min(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    meanX () {
+      return this.comparison ? this.comparison['μ'][0] : 0;
+    },
+
+    meanY () {
+      return this.comparison ? this.comparison['μ'][1] : 0;
+    },
+
+    ellipseViewBox () {
+      return '-150 -150 300 300';
+    },
+
+  }
+}
+</script>

lib/www/client/source/src/components/groups/group-repeatability-summary.vue

@@ -0,0 +1,118 @@
+<template>
+  <v-card class="ma-1">
+    <v-card-title>Group Repeatability Summary</v-card-title>
+    <v-card-text>
+      <p>Error ellipse area for each baseline-monitor pair. Lower values indicate better repeatability. Colors range from green (best) to red (worst).</p>
+      <v-simple-table dense>
+        <thead>
+          <tr>
+            <th>Baseline \ Monitor</th>
+            <th v-for="project in projects" :key="project.pid">{{ project.pid }}</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr v-for="(baselineProject, rowIndex) in projects" :key="baselineProject.pid">
+            <td>{{ baselineProject.pid }}</td>
+            <td v-for="(monitorProject, colIndex) in projects" :key="monitorProject.pid">
+              <v-tooltip v-if="colIndex > rowIndex" top>
+                <template v-slot:activator="{ on, attrs }">
+                  <div
+                    :style="{ backgroundColor: getEllipseAreaColor(baselineProject.pid, monitorProject.pid), color: 'white', textAlign: 'center', padding: '4px' }"
+                    v-bind="attrs"
+                    v-on="on"
+                    @click="emitInput(baselineProject, monitorProject)"
+                  >
+                    {{ formatEllipseArea(baselineProject.pid, monitorProject.pid) }}
+                  </div>
+                </template>
+                <span v-if="getComp(baselineProject.pid, monitorProject.pid)">
+                  <div>σ_i: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][0].toFixed(2) }} m</div>
+                  <div>σ_j: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][1].toFixed(2) }} m</div>
+                  <div>Anisotropy: {{ getComp(baselineProject.pid, monitorProject.pid).meta.anisotropy.toFixed(0) }}</div>
+                  <div>Ellipse Area: {{ getEllipseArea(baselineProject.pid, monitorProject.pid).toFixed(2) }} m²</div>
+                  <div>Primary Direction: {{ formatPrimaryDirection(getComp(baselineProject.pid, monitorProject.pid)) }}°</div>
+                </span>
+              </v-tooltip>
+            </td>
+          </tr>
+        </tbody>
+      </v-simple-table>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: 'DougalGroupRepeatabilitySummary',
+
+  props: {
+    comparisons: {
+      type: Array,
+      required: true
+    },
+    projects: {
+      type: Array,
+      required: true
+    }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+    compMap () {
+      return new Map(this.comparisons.map(c => [`${c.baseline_pid}-${c.monitor_pid}`, c]));
+    },
+    minEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.min(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    },
+    maxEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.max(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    }
+  },
+  methods: {
+    getComp (basePid, monPid) {
+      return this.compMap.get(`${basePid}-${monPid}`);
+    },
+    getEllipseArea (basePid, monPid) {
+      const comp = this.getComp(basePid, monPid);
+      if (!comp) return null;
+      const a = Math.sqrt(comp.meta.eigenvalues[0]);
+      const b = Math.sqrt(comp.meta.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+    formatEllipseArea (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      return val !== null ? val.toFixed(1) : '';
+    },
+    getEllipseAreaColor (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      if (val === null) return '';
+      const ratio = (val - this.minEllipseArea) / (this.maxEllipseArea - this.minEllipseArea);
+      const hue = (1 - ratio) * 120;
+      return `hsl(${hue}, 70%, 70%)`;
+    },
+    formatPrimaryDirection (comp) {
+      if (!comp) return '';
+      return (comp.meta.primaryDirection * 180 / Math.PI).toFixed(1);
+    },
+    emitInput (baselineProject, monitorProject) {
+      if (this.getComp(baselineProject.pid, monitorProject.pid)) {
+        this.$emit('input', baselineProject, monitorProject);
+      }
+    }
+  }
+}
+</script>

lib/www/client/source/src/components/help-dialog.vue

@@ -2,6 +2,7 @@
 <v-dialog
   v-model="dialog"
   max-width="500"
+  scrollable
   style="z-index:2020;"
 >
   <template v-slot:activator="{ on, attrs }">
@@ -44,11 +45,23 @@
         </v-card-title>
 
         <v-card-text>
-          <pre>{{ versionHistory }}</pre>
+          <v-carousel v-model="releaseShown"
+            :continuous="false"
+            :cycle="false"
+            :show-arrows="true"
+            :hide-delimiters="true"
+          >
+            <v-carousel-item v-for="release in releaseHistory">
+              <pre>{{release}}</pre>
+            </v-carousel-item>
+          </v-carousel>
         </v-card-text>
 
 
       </v-window-item>
+      <v-window-item value="serverinfo">
+        <dougal-server-status :status="serverStatus"></dougal-server-status>
+      </v-window-item>
     </v-window>
 
     <v-divider></v-divider>
@@ -60,8 +73,7 @@
         text
         :href="`mailto:${email}?Subject=Question`"
       >
-        <v-icon class="d-lg-none">mdi-help-circle</v-icon>
-        <span class="d-none d-lg-inline">Ask a question</span>
+        <v-icon title="Ask a question">mdi-help-circle</v-icon>
       </v-btn>
 
       <v-btn
@@ -69,8 +81,7 @@
         text
         href="mailto:dougal-support@aaltronav.eu?Subject=Bug report"
       >
-        <v-icon class="d-lg-none">mdi-bug</v-icon>
-        <span class="d-none d-lg-inline">Report a bug</span>
+        <v-icon title="Report a bug">mdi-bug</v-icon>
       </v-btn>
 
       <!---
@@ -84,16 +95,36 @@
       </v-btn>
       --->
 
+      <v-btn
+        color="info"
+        text
+        title="View support info"
+        :input-value="page == 'support'"
+        @click="page = 'support'"
+      >
+        <v-icon>mdi-account-question</v-icon>
+      </v-btn>
+
       <v-btn v-if="versionHistory"
         color="info"
         text
-        :title="page == 'support' ? 'View release notes' : 'View support info'"
+        title="View release notes"
         :input-value="page == 'changelog'"
-        @click="page = page == 'support' ? 'changelog' : 'support'"
+        @click="page = 'changelog'"
       >
         <v-icon>mdi-history</v-icon>
       </v-btn>
 
+      <v-btn v-if="serverStatus"
+        color="info"
+        text
+        title="View server status"
+        :input-value="page == 'serverinfo'"
+        @click="page = 'serverinfo'"
+      >
+        <v-icon>mdi-server-network</v-icon>
+      </v-btn>
+
       <v-spacer></v-spacer>
 
 
@@ -115,43 +146,110 @@
 
 <script>
 import { mapActions, mapGetters } from 'vuex';
+import DougalServerStatus from './server-status';
 
 export default {
   name: 'DougalHelpDialog',
 
+  components: {
+    DougalServerStatus
+  },
+
   data () {
     return {
       dialog: false,
       email: "dougal-support@aaltronav.eu",
       feed: btoa(encodeURIComponent("https://gitlab.com/wgp/dougal/software.atom?feed_token=XSPpvsYEny8YmH75Nz5W")),
+      serverStatus: null,
       clientVersion: process.env.DOUGAL_FRONTEND_VERSION ?? "(unknown)",
       serverVersion: null,
       versionHistory: null,
-      page: "support"
+      releaseHistory: [],
+      releaseShown: null,
+      page: "support",
+
+      lastUpdate: 0,
+      updateInterval: 12000,
+      refreshTimer: null
     };
   },
 
+  computed: {
+    sinceUpdate () {
+      return this.lastUpdate
+        ? (Date.now() - this.lastUpdate)
+        : +Infinity;
+    }
+  },
+
+  watch: {
+    dialog(newVal) {
+      if (newVal) {
+        this.startAutoRefresh();
+      } else {
+        this.stopAutoRefresh();
+      }
+    },
+    page(newVal) {
+      if (newVal === 'serverinfo' && this.dialog) {
+        this.getServerStatus(); // Immediate update when switching to serverinfo
+        this.startAutoRefresh();
+      } else {
+        this.stopAutoRefresh();
+      }
+    }
+  },
+
   methods: {
     async getServerVersion () {
       if (!this.serverVersion) {
         const version = await this.api(['/version', {}, null, {silent:true}]);
         this.serverVersion = version?.tag ?? "(unknown)";
+        if (version) this.lastUpdate = Date.now();
       }
       if (!this.versionHistory) {
-        const history = await this.api(['/version/history?count=1', {}, null, {silent:true}]);
+        const history = await this.api(['/version/history?count=6', {}, null, {silent:true}]);
+        this.releaseHistory = history;
         this.versionHistory = history?.[this.serverVersion.replace(/-.*$/, "")] ?? null;
       }
     },
 
+    async getServerStatus () {
+      const status = await this.api(['/diagnostics', {}, null, {silent: true}]);
+      if (status) {
+        this.serverStatus = status;
+        this.lastUpdate = Date.now();
+      }
+    },
+
+    startAutoRefresh() {
+      if (this.refreshTimer) return; // Prevent multiple timers
+      this.refreshTimer = setInterval(() => {
+        if (this.dialog && this.page === 'serverinfo') {
+          this.getServerStatus();
+          // Optionally refresh server version if needed
+          // this.getServerVersion();
+        }
+      }, this.updateInterval);
+    },
+
+    stopAutoRefresh() {
+      if (this.refreshTimer) {
+        clearInterval(this.refreshTimer);
+        this.refreshTimer = null;
+      }
+    },
+
     ...mapActions(["api"])
   },
 
   async mounted () {
     this.getServerVersion();
+    this.getServerStatus();
   },
 
-  async beforeUpdate () {
-    this.getServerVersion();
+  beforeDestroy() {
+    this.stopAutoRefresh(); // Clean up timer on component destruction
   }
 
 };

lib/www/client/source/src/components/line-status.vue

@@ -1,8 +1,5 @@
 <template>
-<div class="line-status" v-if="sequences.length == 0">
-  <slot name="empty"></slot>
-</div>
-<div class="line-status" v-else-if="sequenceHref || plannedSequenceHref || pendingReshootHref">
+<div class="line-status" v-if="sequenceHref || plannedSequenceHref || pendingReshootHref">
   <router-link v-for="sequence in sequences" :key="sequence.sequence" v-if="sequenceHref"
     class="sequence"
     :class="sequence.status"
@@ -26,7 +23,7 @@
   >
   </router-link>
 </div>
-<div class="line-status" v-else>
+<div class="line-status" v-else-if="sequences.length || plannedSequences.length || Object.keys(pendingReshoots).length">
   <div v-for="sequence in sequences" :key="sequence.sequence"
     class="sequence"
     :class="sequence.status"
@@ -47,6 +44,9 @@
   >
   </div>
 </div>
+<div class="line-status" v-else>
+  <slot name="empty"></slot>
+</div>
 </template>
 
 <style lang="stylus" scoped>

lib/www/client/source/src/components/server-status.vue

@@ -0,0 +1,213 @@
+<template>
+  <v-card max-width="800" max-height="600" class="mx-auto" style="overflow-y: auto;">
+    <v-card-title class="headline">
+      Server status – {{ status.hostname }}
+    </v-card-title>
+    <v-card-text>
+      <v-expansion-panels accordion>
+        <!-- System Info -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>System Info</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-row>
+              <v-col cols="6">
+                <strong>Uptime:</strong> {{ formatUptime(status.uptime) }}
+              </v-col>
+              <v-col cols="6">
+                <strong>Load:</strong> {{ status.loadavg[0].toFixed(2) }} / {{ status.loadavg[1].toFixed(2) }} / {{ status.loadavg[2].toFixed(2) }}
+                <v-progress-linear
+                  :value="loadAvgPercent"
+                  :color="getLoadAvgColor(status.loadavg[0])"
+                  height="6"
+                  rounded
+                ></v-progress-linear>
+                <div class="text-caption">
+                  1-min Load: {{ status.loadavg[0].toFixed(2) }} ({{ loadAvgPercent.toFixed(1) }}% of max)
+                </div>
+              </v-col>
+            </v-row>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Memory -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Memory</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-progress-linear
+              :value="memoryUsedPercent"
+              :color="getProgressColor(memoryUsedPercent)"
+              height="10"
+              rounded
+            ></v-progress-linear>
+            <div class="text-caption mt-2">
+              Used: {{ formatBytes(status.memory.total - status.memory.free) }} / Total: {{ formatBytes(status.memory.total) }} ({{ memoryUsedPercent.toFixed(1) }}%)
+            </div>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- CPUs -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>CPUs ({{ status.cpus.length }} cores)</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-row dense>
+              <v-col v-for="(cpu, index) in status.cpus" :key="index" cols="12" sm="6">
+                <v-card outlined class="pa-2">
+                  <div class="text-caption">Core {{ index + 1 }}: {{ cpu.model }} @ {{ cpu.speed }} MHz</div>
+                  <v-progress-linear
+                    :value="cpuUsagePercent(cpu)"
+                    :color="getProgressColor(cpuUsagePercent(cpu))"
+                    height="8"
+                    rounded
+                  ></v-progress-linear>
+                  <div class="text-caption">
+                    Usage: {{ cpuUsagePercent(cpu).toFixed(1) }}% (Idle: {{ cpuIdlePercent(cpu).toFixed(1) }}%)
+                  </div>
+                </v-card>
+              </v-col>
+            </v-row>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Network Interfaces -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Network Interfaces</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-list dense>
+              <v-list-item v-for="(iface, name) in status.networkInterfaces" :key="name">
+                <v-list-item-content>
+                  <v-list-item-title>{{ name }}</v-list-item-title>
+                  <v-list-item-subtitle v-for="(addr, idx) in iface" :key="idx">
+                    {{ addr.family }}: {{ addr.address }} (Netmask: {{ addr.netmask }})
+                  </v-list-item-subtitle>
+                </v-list-item-content>
+              </v-list-item>
+            </v-list>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Storage -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Storage</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <!-- Root -->
+            <div class="mb-4">
+              <strong>Root (/):</strong>
+              <v-progress-linear
+                :value="status.storage.root.usedPercent"
+                :color="getProgressColor(status.storage.root.usedPercent)"
+                height="10"
+                rounded
+              ></v-progress-linear>
+              <div class="text-caption">
+                Used: {{ formatBytes(status.storage.root.used) }} / Total: {{ formatBytes(status.storage.root.total) }} ({{ status.storage.root.usedPercent.toFixed(1) }}%)
+              </div>
+            </div>
+            <!-- Data subfolders -->
+            <div>
+              <strong>Data:</strong>
+              <v-expansion-panels flat>
+                <v-expansion-panel v-for="(folder, name) in status.storage.data" :key="name">
+                  <v-expansion-panel-header disable-icon-rotate>{{ name }}</v-expansion-panel-header>
+                  <v-expansion-panel-content>
+                    <v-progress-linear
+                      :value="folder.usedPercent"
+                      :color="getProgressColor(folder.usedPercent)"
+                      height="10"
+                      rounded
+                    ></v-progress-linear>
+                    <div class="text-caption">
+                      Used: {{ formatBytes(folder.used) }} / Total: {{ formatBytes(folder.total) }} ({{ folder.usedPercent.toFixed(1) }}%)
+                    </div>
+                  </v-expansion-panel-content>
+                </v-expansion-panel>
+              </v-expansion-panels>
+            </div>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Database -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Database</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <div class="mb-2">
+              <strong>Total Size:</strong> {{ formatBytes(status.database.size) }}
+            </div>
+            <v-list dense>
+              <v-list-item v-for="(project, name) in status.database.projects" :key="name">
+                <v-list-item-content>
+                  <v-list-item-title>{{ name }}</v-list-item-title>
+                  <v-progress-linear
+                    :value="project.percent"
+                    :color="getProgressColor(project.percent)"
+                    height="8"
+                    rounded
+                  ></v-progress-linear>
+                  <v-list-item-subtitle>
+                    Size: {{ formatBytes(project.size) }} ({{ project.percent.toFixed(2) }}%)
+                  </v-list-item-subtitle>
+                </v-list-item-content>
+              </v-list-item>
+            </v-list>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+      </v-expansion-panels>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: "DougalServerStatus",
+  props: {
+    status: {
+      type: Object,
+      required: true
+    }
+  },
+  computed: {
+    memoryUsedPercent() {
+      return ((this.status.memory.total - this.status.memory.free) / this.status.memory.total) * 100;
+    },
+    loadAvgPercent() {
+      const maxLoad = this.status.cpus.length * 2; // Assume 4x cores as max for scaling
+      return Math.min((this.status.loadavg[0] / maxLoad) * 100, 100); // Cap at 100%
+    }
+  },
+  methods: {
+    getProgressColor(value) {
+      if (value >= 80) return 'error'; // Red for 80–100%
+      if (value >= 60) return 'warning'; // Yellow for 60–80%
+      return 'success'; // Green for 0–60%
+    },
+    getLoadAvgColor(load) {
+      const coreCount = this.status.cpus.length;
+      if (load >= coreCount * 2) return 'error'; // Red for load ≥ 2x cores
+      if (load >= coreCount) return 'warning'; // Yellow for load ≥ 1x cores but < 2x
+      return 'success'; // Green for load < 1x cores
+    },
+    formatBytes(bytes) {
+      if (bytes === 0) return '0 Bytes';
+      const k = 1024;
+      const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'];
+      const i = Math.floor(Math.log(bytes) / Math.log(k));
+      return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+    },
+    formatUptime(seconds) {
+      const days = Math.floor(seconds / 86400);
+      seconds %= 86400;
+      const hours = Math.floor(seconds / 3600);
+      seconds %= 3600;
+      const minutes = Math.floor(seconds / 60);
+      return `${days}d ${hours}h ${minutes}m`;
+    },
+    cpuUsagePercent(cpu) {
+      const total = cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.idle + cpu.times.irq;
+      return ((total - cpu.times.idle) / total) * 100;
+    },
+    cpuIdlePercent(cpu) {
+      const total = cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.idle + cpu.times.irq;
+      return (cpu.times.idle / total) * 100;
+    }
+  }
+};
+</script>

lib/www/client/source/src/lib/deck.gl/DougalBinaryLoader.js

@@ -84,8 +84,12 @@ const DougalBinaryLoader = {
       for (let k = 0; k < values.length; k++) {
         values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? BigUint64Array : Float32Array)(totalCount);
       }
+    } else if (udv == 4) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? Uint16Array : Float32Array)(totalCount);
+      }
     } else {
-      throw new Error(`Invalid udv: Expected 0, 1, or 2; found ${udv}`);
+      throw new Error(`Invalid udv: Expected 0, 1, 2, or 4; found ${udv}`);
     }
 
     let offset = 0;
@@ -110,7 +114,7 @@ const DougalBinaryLoader = {
       offset += chunk.jCount;
     }
 
-    console.log(`Parsed ${totalCount} points, ${values.length} value arrays`);
+    console.log(`Parsed ${totalCount} points, ${values.length} value arrays, udv = ${udv}`);
 
     const attributes = {
       getPosition: {

lib/www/client/source/src/lib/durations.js

@@ -0,0 +1,47 @@
+
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+
+export {
+  duration_to_ms,
+  ms_to_duration,
+  normalise_duration,
+  add_durations
+}

lib/www/client/source/src/main.js

@@ -62,9 +62,7 @@ new Vue({
 
     showSnack(text, colour = "primary") {
       console.log("showSnack", text, colour);
-      this.snackColour = colour;
-      this.snackText = text;
-      this.snack = true;
+      this.$store.dispatch("showSnack", [text, colour]);
     },
 
     sendJwt () {

lib/www/client/source/src/router/index.js

@@ -20,6 +20,9 @@ import ProjectSettings from '../views/ProjectSettings.vue'
 import Users from '../views/Users.vue'
 import DougalAppBarExtensionProject from '../components/app-bar-extension-project'
 import DougalAppBarExtensionProjectList from '../components/app-bar-extension-project-list'
+import GroupList from '../views/GroupList.vue'
+import Group from '../views/Group.vue'
+
 
 Vue.use(VueRouter)
 
@@ -155,6 +158,7 @@ Vue.use(VueRouter)
         component: SequenceList
       },
       {
+        name: "shotlog",
         path: "sequences/:sequence",
         component: SequenceSummary
       },
@@ -196,7 +200,57 @@ Vue.use(VueRouter)
         component: ProjectSettings
       }
     ]
-  }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups",
+    redirect: "/groups/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/",
+    component: GroupList,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups", disabled: true }
+      ],
+      appBarExtension: {
+        // component: DougalAppBarExtensionProjectList
+      }
+    }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group",
+    redirect: "/groups/:group/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group/",
+    name: "Group",
+    component: Group,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups" },
+        { text: (ctx) => ctx.$route.params.group }
+        /*
+        {
+          text: (ctx) => ctx.$store.state.project.projectName || "…",
+          href: (ctx) => `/projects/${ctx.$store.state.project.projectId || ctx.$route.params.project || ""}/`,
+          title: (ctx) => Object.entries(ctx.$store.getters.projectConfiguration?.organisations ?? {}).map( ([org, ops]) => `* ${org}: ${Object.entries(ops).filter( ([k, v]) => v ).map( ([k, v]) => k ).join(", ")}`).join("\n"),
+          organisations: (ctx) => ctx.$store.getters.projectConfiguration?.organisations ?? {}
+        }
+        */
+      ],
+      /*
+      appBarExtension: {
+        component: DougalAppBarExtensionGroup
+      }
+      */
+    },
+    children: [
+    ]
+  },
 ]
 
 const router = new VueRouter({

lib/www/client/source/src/store/modules/api/actions.js

@@ -71,7 +71,7 @@ async function api ({state, getters, commit, dispatch}, [resource, init = {}, cb
       res = await limiter.enqueue(async () => await fetch(url, init));
     }
 
-    if (cache && !isCached) {
+    if (cache && !isCached && res.ok) { // Only cache successful responses
       cache.put(url, res.clone());
     }
 
@@ -95,6 +95,12 @@ async function api ({state, getters, commit, dispatch}, [resource, init = {}, cb
           return [key, value];
         });
         state.serverInfo = entries.length ? Object.fromEntries(entries) : {};
+
+        if (state.serverInfo["remote-frontend"]) {
+          state.isGatewayReliable = ![ 502, 503, 504 ].includes(res.status);
+        } else {
+          state.isGatewayReliable = null;
+        }
     }
 
     if (res.ok) {

lib/www/client/source/src/store/modules/api/state.js

@@ -2,7 +2,8 @@ const state = () => ({
   apiUrl: "/api",
   requestsCount: 0,
   maxConcurrent: 15,
-  serverInfo: {} // Contents of the last received X-Dougal-Server HTTP header
+  serverInfo: {}, // Contents of the last received X-Dougal-Server HTTP header
+  isGatewayReliable: null, // True if we start seeing HTTP 502‒504 responses
 });
 
 export default state;

lib/www/client/source/src/store/modules/event/actions.js

@@ -36,7 +36,7 @@ async function refreshEvents ({commit, dispatch, state, rootState}, [modifiedAft
 
 /** Return a subset of events from state.events
  */
-async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text, label}]) {
+async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text, label, excludeLabels}]) {
   let filteredEvents = [...state.events];
 
   if (sortBy) {
@@ -114,6 +114,10 @@ async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date
     filteredEvents = filteredEvents.filter( event => event.labels?.includes(label) );
   }
 
+  if (excludeLabels) {
+    filteredEvents = filteredEvents.filter( event => !excludeLabels?.some( label => event.labels?.includes(label) ) );
+  }
+
   const count = filteredEvents.length;
 
   if (itemsPerPage && itemsPerPage > 0) {

lib/www/client/source/src/store/modules/notify/actions.js

@@ -80,4 +80,4 @@ function processServerEvent({ commit, dispatch, state, rootState }, message) {
   state.debouncedRunners[table](message);
 }
 
-export default { registerHandler, processServerEvent };
+export default { registerHandler, unregisterHandler, processServerEvent };

lib/www/client/source/src/store/modules/notify/mutations.js

@@ -30,4 +30,10 @@ function UNREGISTER_HANDLER(state, { table, handler }) {
 }
 
 
-export default { setServerEvent, clearServerEvent, setServerConnectionState, REGISTER_HANDLER };
+export default {
+  setServerEvent,
+  clearServerEvent,
+  setServerConnectionState,
+  REGISTER_HANDLER,
+  UNREGISTER_HANDLER
+};

lib/www/client/source/src/store/modules/project/actions.js

@@ -1,5 +1,10 @@
 
 async function getProject ({commit, dispatch}, projectId) {
+  if (projectId == null) {
+    console.log(`Skipping call to getProject${projectId})`);
+    return;
+  }
+
   const init = {
     headers: {
       cache: "reload",

lib/www/client/source/src/store/modules/projects/actions.js

@@ -1,13 +1,27 @@
+import * as d3a from 'd3-array';
+import { duration_to_ms, ms_to_duration, normalise_duration, add_durations } from '@/lib/durations';
 
 /** Fetch projects from server
  */
 async function refreshProjects ({commit, dispatch, state, rootState}) {
 
+  async function getSummary (project) {
+      const url = `/project/${project.pid}/summary`;
+      const init = {};
+      const summary = await dispatch('api', [url, init, null, {silent:true}]);
+      if (summary) {
+        return {...project, ...summary};
+      } else {
+        return project;
+      }
+  }
+
   if (state.loading) {
     commit('abortProjectsLoading');
   }
 
   commit('setProjectsLoading');
+  const tstamp = new Date();
   const pid = rootState.project.projectId;
   const url = `/project`;
   const init = {
@@ -17,10 +31,25 @@ async function refreshProjects ({commit, dispatch, state, rootState}) {
   const res = await dispatch('api', [url, init, null, {silent:true}]);
 
   if (res) {
-    commit('setProjects', res);
-    commit('setProjectsTimestamp');
+
+    let projects;
+
+    if (res.some( project => project.pid == null )) {
+      console.warn("At least one project found with no PID!");
+      projects = res.filter( project => project.pid != null );
+    } else {
+      projects = res;
+    }
+
+    commit('setProjects', projects); // First without summaries
+    commit('setProjectsTimestamp', tstamp);
+
+    projects = await Promise.all(projects.map( getSummary ));
+
+    commit('setProjects', projects); // Then with summaries
   }
   commit('clearProjectsLoading');
+  dispatch('prepareGroups');
 }
 
 /** Return a subset of projects from state.projects
@@ -118,4 +147,83 @@ async function getProjects ({commit, dispatch, state}, [{pid, name, schema, grou
   return {projects: filteredProjects, count};
 }
 
-export default { refreshProjects, getProjects };
+
+
+async function prepareGroups ({commit, dispatch, state, rootState}) {
+  const groups = {};
+
+  for (const project of state.projects) {
+
+    if (!project.prod_distance) {
+      // This project has no production data (either not started yet
+      // or production data has not been imported) so we skip it.
+      continue;
+    }
+
+    if (!project.prod_duration.days) {
+      project.prod_duration = normalise_duration(project.prod_duration);
+    }
+
+    for (const name of project.groups) {
+      if (!(name in groups)) {
+        groups[name] = {
+          group: name,
+          num_projects: 0,
+          lines: 0,
+          points: 0,
+          sequences: 0,
+          // Shots:
+          prime: 0,
+          other: 0,
+          ntba: 0,
+          prod_duration: {
+            days: 0,
+            hours: 0,
+            minutes: 0,
+            seconds: 0,
+            milliseconds: 0
+          },
+          prod_distance: 0,
+          shooting_rate: [],
+          projects: []
+        };
+      }
+      const group = groups[name];
+
+      group.num_projects++;
+      group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+      group.points = Math.max(group.points, project.total); // Idem
+      group.sequences += project.seq_final;
+      group.prime += project.prime;
+      group.other += project.other;
+      //group.ntba += project.ntba;
+      group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+      group.prod_distance += project.prod_distance;
+      group.shooting_rate.push(project.shooting_rate);
+      group.projects.push(project);
+    }
+  }
+
+  const grouplist = [];
+  for (const group of Object.values(groups)) {
+    group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+    group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+    delete group.shooting_rate;
+
+    grouplist.push(group);
+  }
+
+  commit('setGroups', grouplist);
+
+}
+
+async function getGroups({commit, dispatch, state, rootState}) {
+  if (!state.groups.length) {
+    await dispatch('refreshProjects');
+  }
+
+  return state.groups;
+}
+
+
+export default { refreshProjects, getProjects, prepareGroups, getGroups };

lib/www/client/source/src/store/modules/projects/getters.js

@@ -3,7 +3,7 @@ function projects (state) {
   return state.projects;
 }
 
-function projectGroups (state) {
+function projectGroupNames (state) {
   return [...new Set(state.projects.map(i => i.groups).flat())].sort();
 }
 
@@ -15,4 +15,8 @@ function projectsLoading (state) {
   return !!state.loading;
 }
 
-export default { projects, projectGroups, projectCount, projectsLoading };
+function groups (state) {
+  return state.groups;
+}
+
+export default { projects, projectGroupNames, projectCount, projectsLoading, groups };

lib/www/client/source/src/store/modules/projects/mutations.js

@@ -39,10 +39,15 @@ function abortProjectsLoading (state) {
   state.loading = null;
 }
 
+function setGroups (state, groups) {
+  state.groups = Object.freeze(groups);
+}
+
 export default {
   setProjects,
   setProjectsLoading,
   clearProjectsLoading,
   setProjectsTimestamp,
-  setProjectsETag
+  setProjectsETag,
+  setGroups
 };

lib/www/client/source/src/store/modules/projects/state.js

@@ -1,5 +1,6 @@
 const state = () => ({
   projects: Object.freeze([]),
+  groups: Object.freeze([]),
   loading: null,
   timestamp: null,
   etag: null,

lib/www/client/source/src/store/modules/user/actions.js

@@ -25,23 +25,10 @@ async function login ({ commit, dispatch }, loginRequest) {
 async function logout ({ commit, dispatch }) {
   commit('setToken', null);
   commit('setUser', null);
-  commit('setCookie', {value: null});
   await dispatch('api', ["/logout"]);
   commit('setPreferences', {});
 }
 
-function setCookie(context, {name, value, expiry, path}) {
-  if (!name) name = "JWT";
-  if (!path) path = "/";
-  if (!value) value = "";
-
-  if (expiry) {
-    document.cookie = `${name}=${value}; expiry=${(new Date(expiry)).toUTCString()}; path=${path}`;
-  } else {
-    document.cookie = `${name}=${value}; path=${path}`;
-  }
-}
-
 function setCredentials({ state, commit, getters, dispatch, rootState }, { force, token, response } = {}) {
   try {
     let tokenValue = token;
@@ -59,17 +46,7 @@ function setCredentials({ state, commit, getters, dispatch, rootState }, { force
       const decoded = jwt_decode(tokenValue);
       commit('setToken', tokenValue);
       commit('setUser', decoded ? new User(decoded, rootState.api.api) : null);
-
-      if (tokenValue && decoded) {
-        if (decoded?.exp) {
-          dispatch('setCookie', {value: tokenValue, expiry: decoded.exp*1000});
-        } else {
-          dispatch('setCookie', {value: tokenValue});
-        }
-      } else {
-        // Clear the cookie
-        dispatch('setCookie', {value: "", expiry: 0});
-      }
+      commit('setCookie', {name: "JWT", value: tokenValue, expires: (decoded.exp??0)*1000});
 
       console.log('Credentials refreshed at', new Date().toISOString());
     } else {
@@ -80,6 +57,7 @@ function setCredentials({ state, commit, getters, dispatch, rootState }, { force
     if (err.name === 'InvalidTokenError') {
       commit('setToken', null);
       commit('setUser', null);
+      commit('clearCookie', "JWT")
     }
   }
   dispatch('loadUserPreferences');
@@ -114,7 +92,6 @@ async function loadUserPreferences({ state, commit }) {
 export default {
   login,
   logout,
-  setCookie,
   setCredentials,
   saveUserPreference,
   loadUserPreferences

lib/www/client/source/src/store/modules/user/getters.js

@@ -7,12 +7,6 @@ function jwt (state) {
   return state.token;
 }
 
-function cookie (state) {
-  if (state.token) {
-    return "JWT="+token;
-  }
-}
-
 function preferences (state) {
   return state.preferences;
 }

lib/www/client/source/src/store/modules/user/mutations.js

@@ -16,4 +16,18 @@ function setPreferences (state, preferences) {
   state.preferences = preferences;
 }
 
-export default { setToken, setUser, setPreferences };
+function setCookie (state, opts = {}) {
+  const name = opts.name ?? "JWT";
+  const value = opts.value ?? "";
+  const expires = opts.expires ? (new Date(opts.expires)) : (new Date(0));
+  const path = opts.path ?? "/";
+  const sameSite = opts.sameSite ?? "Lax";
+
+  document.cookie = `${name}=${value};path=${path};SameSite=${sameSite};expires=${expires.toUTCString()}`;
+}
+
+function clearCookie (state, name) {
+  setCookie(state, {name});
+}
+
+export default { setToken, setUser, setPreferences, setCookie, clearCookie };

lib/www/client/source/src/views/Group.vue

@@ -0,0 +1,339 @@
+<template>
+    <dougal-group-map v-if="mapView"
+      :baseline="baseline"
+      :monitor="monitor"
+      :monitors="monitors"
+      @input="mapView=$event"
+    ></dougal-group-map>
+  <v-container fluid fill-height class="ma-0 pa-0" v-else>
+
+    <v-overlay :value="loading && !comparisons.length" absolute>
+      <v-progress-circular
+        indeterminate
+        size="64"
+      ></v-progress-circular>
+    </v-overlay>
+
+    <v-overlay :value="!loading && !groupFound" absolute opacity="0.8">
+      <v-row justify="center">
+        <v-alert
+          type="error"
+        >
+          Group not found
+        </v-alert>
+      </v-row>
+      <v-row justify="center">
+        <v-btn color="primary" @click="refreshProjects">Retry</v-btn>
+      </v-row>
+    </v-overlay>
+
+    <v-row no-gutters align="stretch" class="fill-height">
+      <v-col cols="12" v-if="groupFound">
+
+        <v-data-table class="ma-1"
+          :headers="projectHeaders"
+          :items="projects"
+          dense
+        >
+
+          <template v-slot:item.baseline="{item, value, index}">
+            <v-simple-checkbox v-if="index+1 < projects.length"
+              color="primary"
+              :value="baseline === item"
+              @input="setBaseline(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.monitor="{item, value, index}">
+            <v-simple-checkbox v-if="index > 0 && !(index <= baselineIndex)"
+              color="primary"
+              :value="monitor === item"
+              @input="setMonitor(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.pid="{item, value}">
+            <v-chip
+              label
+              small
+              outlined
+              :href="`/projects/${item.pid}`"
+              :color="!item.archived ? 'primary' : ''"
+            >{{ value }}</v-chip>
+          </template>
+
+          <template v-slot:item.fsp="{item, value}">
+            <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.lsp="{item, value}">
+            <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.prod_duration="{item, value}">
+            <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+              {{ value.days }} d
+            </span>
+            <span v-else>
+              {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+            </span>
+          </template>
+
+          <template v-slot:item.prod_distance="{item, value}">
+            {{ (value/1000).toFixed(1) }} km
+          </template>
+
+          <template v-slot:footer.prepend>
+
+            <v-btn v-if="baseline && !mapView"
+              text
+              color="primary"
+              title="Switch to map view"
+              @click="mapView = true"
+            >View map</v-btn>
+
+            <v-btn v-if="comparison"
+              text
+              color="primary"
+              title="Back to summary"
+              @click="clearComparison"
+            >Back</v-btn>
+          </template>
+
+        </v-data-table>
+
+<!-- BEGIN TEST -->
+
+    <dougal-group-comparison-summary v-if="comparison"
+      :baseline="baseline"
+      :monitor="monitor"
+      :comparison="comparison"
+    ></dougal-group-comparison-summary>
+
+    <dougal-group-repeatability-summary v-else-if="comparisons.length"
+      :comparisons="comparisons"
+      :projects="projects"
+      @input="setComparison"
+    ></dougal-group-repeatability-summary>
+
+<!-- END   TEST -->
+
+
+      </v-col>
+      <v-col cols="12" v-else>
+        <v-card>
+          <v-card-text>
+            Group does not exist.
+          </v-card-text>
+        </v-card>
+      </v-col>
+    </v-row>
+  </v-container>
+</template>
+
+<script>
+import { mapActions, mapGetters } from 'vuex'
+import AccessMixin from '@/mixins/access';
+import DougalGroupRepeatabilitySummary from '@/components/groups/group-repeatability-summary.vue';
+import DougalGroupComparisonSummary from '@/components/groups/group-comparison-summary';
+import DougalGroupMap from '@/components/groups/group-map';
+
+export default {
+  name: 'Group',
+
+  mixins: [
+    AccessMixin
+  ],
+
+  components: {
+    DougalGroupRepeatabilitySummary,
+    DougalGroupComparisonSummary,
+    DougalGroupMap
+  },
+
+  data () {
+    return {
+      projectHeaders: [
+        {
+          value: "baseline",
+          text: "Baseline"
+        },
+        {
+          value: "monitor",
+          text: "Monitor"
+        },
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      mapView: false,
+
+      baseline: null,
+      monitor: null,
+      comparisons: []
+    }
+  },
+
+  computed: {
+
+    groupName () {
+      return this.$route.params.group;
+    },
+
+    group () {
+      return this.groups.find( i => i.group === this.groupName );
+    },
+
+    groupFound () {
+      return !!(this.loading || this.group);
+    },
+
+    projects () {
+      return this.group?.projects.toSorted((a, b) => a.pid.localeCompare(b.pid));
+    },
+
+    baselineIndex () {
+      return this.projects.indexOf(this.baseline);
+    },
+
+    monitors () {
+      if (this.baseline && this.comparisons) {
+        return this.comparisons
+          .filter( i => i.baseline_pid == this.baseline.pid )
+          .map( i => this.projects.find( p => p.pid == i.monitor_pid ));
+      } else {
+        return null;
+      }
+    },
+
+    comparison () {
+      return this.comparisons.find( row =>
+        row.baseline_pid == this.baseline?.pid && row.monitor_pid == this.monitor?.pid
+      )?.meta;
+    },
+
+    ...mapGetters(["loading", "groups"])
+  },
+
+  methods: {
+
+    setBaseline (project) {
+      if (project === this.baseline) {
+        this.baseline = null;
+      } else {
+        this.baseline = project;
+        if (this.monitor) {
+          if (this.projects.indexOf(this.monitor) <= this.projects.indexOf(this.baseline)) {
+            this.monitor = null;
+          }
+        }
+      }
+    },
+
+    setMonitor (project) {
+      if (project === this.monitor) {
+        this.monitor = null;
+      } else {
+        this.monitor = project;
+      }
+    },
+
+    clearComparison () {
+      this.baseline = null;
+      this.monitor = null;
+    },
+
+    setComparison (baseline, monitor) {
+      this.clearComparison();
+      this.setBaseline(baseline);
+      this.setMonitor(monitor);
+    },
+
+    async getComparisons () {
+      const url = `/comparison/group/${this.$route.params.group}`;
+      this.comparisons = await this.api([url]);
+    },
+
+    // TODO Should this go in a Vuex action rather?
+    async refreshComparisons () {
+      await this.getGroups();
+      if (this.groupFound) {
+        await this.getComparisons();
+      }
+    },
+
+    /*
+    async getComparison () {
+      if (this.baseline && this.monitor) {
+        const url = `/comparison/group/${this.$route.params.group}/baseline/${this.baseline.pid}/monitor/${this.monitor.pid}`;
+        const comparison = await this.api([url]);
+        if (comparison) {
+          this.comparison = comparison;
+        }
+      }
+    },
+    */
+
+    handleComparisons (context, {payload}) {
+      this.refreshComparisons();
+    },
+
+    registerNotificationHandlers (action = "registerHandler") {
+
+      this.$store.dispatch(action, {
+        table: 'comparisons',
+        handler: this.handleComparisons
+      });
+
+    },
+
+    unregisterNotificationHandlers () {
+      return this.registerNotificationHandlers("unregisterHandler");
+    },
+
+
+    ...mapActions(["api", "getGroups", "refreshProjects"])
+  },
+
+  async mounted () {
+    this.registerNotificationHandlers();
+    this.refreshComparisons()
+  },
+
+  beforeDestroy () {
+    this.unregisterNotificationHandlers();
+  }
+
+
+}
+</script>

lib/www/client/source/src/views/GroupList.vue

@@ -0,0 +1,396 @@
+<template>
+  <v-container fluid>
+
+    <v-data-table
+    :headers="headers"
+    :items="displayItems"
+    item-key="group"
+    :options.sync="options"
+    :expanded.sync="expanded"
+    show-expand
+    :loading="loading"
+    >
+
+      <template v-slot:item.group="{item, value}">
+        <v-chip
+          label
+          small
+          :href="`./${value}`"
+        >{{ value }}</v-chip>
+      </template>
+
+      <template v-slot:item.shots_total="{item, value}">
+        <div>{{ item.prime + item.other }}</div>
+        <v-progress-linear
+          background-color="secondary"
+          color="primary"
+          :value="item.prime/(item.prime+item.other)*100"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:item.prime="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.other="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.prod_duration="{item, value}">
+        <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+          {{ value.days }} d
+        </span>
+        <span v-else>
+          {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+        </span>
+      </template>
+
+      <template v-slot:item.prod_distance="{item, value}">
+        {{ (value/1000).toFixed(1) }} km
+      </template>
+
+      <template v-slot:item.shooting_rate_mean="{item, value}">
+        {{ (value).toFixed(2) }} s ±{{ (item.shooting_rate_sd).toFixed(3) }} s
+      </template>
+
+      <template v-slot:item.shots_per_point="{item, value}">
+        <div>
+          {{ ((item.prime + item.other)/item.points).toFixed(1) }}
+          ({{ ((((item.prime + item.other)/item.points) / item.num_projects)*100).toFixed(2) }}%)
+        </div>
+        <v-progress-linear
+          :value="((((item.prime + item.other)/item.points) / item.num_projects)*100)"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:expanded-item="{ headers, item }">
+        <td :colspan="headers.length">
+          <v-data-table class="ma-1"
+            :headers="projectHeaders"
+            :items="item.projects"
+            dense
+            hide-default-footer
+          >
+
+            <template v-slot:item.pid="{item, value}">
+              <a :href="`/projects/${value}`" title="Go to project">{{ value }}</a>
+            </template>
+
+            <template v-slot:item.fsp="{item, value}">
+              <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.lsp="{item, value}">
+              <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.prod_duration="{item, value}">
+              <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+                {{ value.days }} d
+              </span>
+              <span v-else>
+                {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+              </span>
+            </template>
+
+            <template v-slot:item.prod_distance="{item, value}">
+              {{ (value/1000).toFixed(1) }} km
+            </template>
+
+          </v-data-table>
+        </td>
+      </template>
+
+    </v-data-table>
+
+  </v-container>
+</template>
+
+<style>
+td p:last-of-type {
+  margin-bottom: 0;
+}
+</style>
+
+<script>
+import { mapActions, mapGetters } from 'vuex';
+import AccessMixin from '@/mixins/access';
+
+
+// FIXME send to lib/utils or so
+/*
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+*/
+
+export default {
+  name: "GroupList",
+
+  components: {
+  },
+
+  mixins: [
+    AccessMixin
+  ],
+
+  data () {
+    return {
+      headers: [
+        {
+          value: "group",
+          text: "Group name"
+        },
+        {
+          value: "num_projects",
+          text: "Number of campaigns"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "points",
+          text: "Preplot points"
+        },
+        {
+          value: "sequences",
+          text: "Total sequences"
+        },
+        {
+          value: "shots_total",
+          text: "Total shots"
+        },
+        {
+          value: "prime",
+          text: "Total prime"
+        },
+        {
+          value: "other",
+          text: "Total reshoot + infill"
+        },
+        /*
+        {
+          value: "ntba",
+          text: "Total NTBA"
+        },
+        */
+        {
+          value: "prod_duration",
+          text: "Total duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Total distance"
+        },
+        {
+          value: "shooting_rate_mean",
+          text: "Shooting rate (mean)"
+        },
+        {
+          value: "shots_per_point",
+          text: "Shots per point"
+        },
+      ],
+      items: [],
+      expanded: [],
+      options: { sortBy: ["group"], sortDesc: [false] },
+
+      projectHeaders: [
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      // Context menu stuff
+      contextMenuShow: false,
+      contextMenuX: 0,
+      contextMenuY: 0,
+      contextMenuItem: null,
+
+      /*
+      // FIXME Eventually need to move this into Vuex
+      groups: []
+      */
+    }
+  },
+
+  computed: {
+    displayItems () {
+      return this.items.filter(i => i.prod_distance);
+    },
+
+    ...mapGetters(['loading', 'groups'])
+  },
+
+  methods: {
+
+    /*
+    async prepareGroups () {
+      //const groups = await this.api(["/prospects"]);
+      //console.log("groups", groups);
+      const groups = {};
+
+      for (const project of this.projects) {
+
+        if (!project.prod_distance) {
+          // This project has no production data (either not started yet
+          // or production data has not been imported) so we skip it.
+          continue;
+        }
+
+        if (!project.prod_duration.days) {
+          project.prod_duration = normalise_duration(project.prod_duration);
+        }
+
+        for (const name of project.groups) {
+          if (!(name in groups)) {
+            groups[name] = {
+              group: name,
+              num_projects: 0,
+              lines: 0,
+              points: 0,
+              sequences: 0,
+              // Shots:
+              prime: 0,
+              other: 0,
+              ntba: 0,
+              prod_duration: {
+                days: 0,
+                hours: 0,
+                minutes: 0,
+                seconds: 0,
+                milliseconds: 0
+              },
+              prod_distance: 0,
+              shooting_rate: [],
+              projects: []
+            };
+          }
+          const group = groups[name];
+
+          group.num_projects++;
+          group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+          group.points = Math.max(group.points, project.total); // Idem
+          group.sequences += project.seq_final;
+          group.prime += project.prime;
+          group.other += project.other;
+          //group.ntba += project.ntba;
+          group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+          group.prod_distance += project.prod_distance;
+          group.shooting_rate.push(project.shooting_rate);
+          group.projects.push(project);
+        }
+      }
+
+      this.groups = [];
+      for (const group of Object.values(groups)) {
+        group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+        group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+        delete group.shooting_rate;
+
+        this.groups.push(group);
+      }
+
+    },
+    */
+
+    async list () {
+      this.items = [...this.groups];
+    },
+
+    async load () {
+      await this.refreshProjects();
+      //await this.prepareGroups();
+      await this.list();
+    },
+
+    registerNotificationHandlers () {
+      this.$store.dispatch('registerHandler', {
+        table: 'project`',
+
+        handler: (context, message) => {
+          if (message.payload?.table == "public") {
+            this.load();
+          }
+        }
+      });
+    },
+
+    ...mapActions(["api", "showSnack", "refreshProjects"])
+  },
+
+  mounted () {
+    this.registerNotificationHandlers();
+    this.load();
+  }
+}
+
+</script>

lib/www/client/source/src/views/Log.vue

@@ -5,6 +5,22 @@
       <v-card-title>
         <v-toolbar flat>
           <v-toolbar-title>
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[sequences.length-1]||{}).sequence }}"
+                title="Go to the first sequence"
+              >
+                <v-icon dense>mdi-chevron-double-left</v-icon>
+              </v-btn>
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[sequenceIndex+1]||{}).sequence }}"
+                title="Go to the previous sequence"
+              >
+                <v-icon dense>mdi-chevron-left</v-icon>
+              </v-btn>
+            </template>
             <span class="d-none d-lg-inline">
               {{
                 $route.params.sequence
@@ -31,18 +47,38 @@
                     : ""
               }}
             </span>
+
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex==0"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[sequenceIndex-1]||{}).sequence }}"
+                title="Go to the next sequence"
+              >
+                <v-icon dense>mdi-chevron-right</v-icon>
+              </v-btn>
+              <v-btn icon small class="mr-1"
+                :disabled="sequenceIndex==0"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[0]||{}).sequence }}"
+                title="Go to the last sequence"
+              >
+                <v-icon dense>mdi-chevron-double-right</v-icon>
+              </v-btn>
+            </template>
+
+            <a v-if="$route.params.sequence"
+              class="mr-3"
+              :href="`/projects/${$route.params.project}/sequences/${$route.params.sequence}`"
+              title="View the shotlog for this sequence"
+            >
+              <v-icon
+                right
+                color="teal"
+              >mdi-format-list-numbered</v-icon>
+            </a>
+
           </v-toolbar-title>
 
-          <a v-if="$route.params.sequence"
-            class="mr-3"
-            :href="`/projects/${$route.params.project}/sequences/${$route.params.sequence}`"
-            title="View the shotlog for this sequence"
-          >
-            <v-icon
-              right
-              color="teal"
-            >mdi-format-list-numbered</v-icon>
-          </a>
+
 
             <dougal-event-edit v-if="$parent.writeaccess()"
               v-model="eventDialog"
@@ -325,6 +361,12 @@
                   @click="labelSearch=label"
                 >{{label}}</v-chip>
               </span>
+              <v-icon v-if="entry.meta.auto || entry.meta.author"
+                x-small
+                left
+                color="primary"
+                :title="entry.meta.author?`Automatic event by ${entry.meta.author}`:'Automatic event'"
+              >mdi-robot</v-icon>
               <dougal-event-edit-history v-if="entry.has_edits && $parent.writeaccess()"
                 :id="entry.id"
                 :disabled="eventsLoading"
@@ -488,17 +530,6 @@ export default {
     rows () {
       const rows = {};
       this.items
-      .filter(i => {
-        return !this.$route.params.sequence || (this.$route.params.sequence == i.sequence)
-      })
-      .filter(i => {
-        for (const label of this.filterableLabels) {
-          if (!this.shownLabels.includes(label) && i.labels.includes(label)) {
-            return false;
-          }
-        }
-        return true;
-      })
       .forEach(i => {
         const key = (i.sequence && i.point) ? (i.sequence+"@"+i.point) : i.tstamp;
         if (!rows[key]) {
@@ -529,6 +560,10 @@ export default {
       .sort( (a, b) => b[1]-a[1] );
     },
 
+    filteredLabels () {
+      return this.filterableLabels.filter( label => !this.shownLabels.includes(label) );
+    },
+
     presetRemarks () {
       return this.projectConfiguration?.events?.presetRemarks ?? [];
     },
@@ -541,7 +576,17 @@ export default {
       }
     },
 
-    ...mapGetters(['user', 'eventsLoading', 'online', 'sequence', 'line', 'point', 'position', 'timestamp', 'lineName', 'events', 'labels', 'userLabels', 'projectConfiguration']),
+    sequenceIndex () {
+      if ("sequence" in this.$route.params) {
+        const index = this.sequences.findIndex( i => i.sequence == this.$route.params.sequence );
+        if (index != -1) {
+          return index;
+        }
+      }
+      // return undefined
+    },
+
+    ...mapGetters(['user', 'eventsLoading', 'online', 'sequence', 'sequences', 'line', 'point', 'position', 'timestamp', 'lineName', 'events', 'labels', 'userLabels', 'projectConfiguration']),
     ...mapState({projectSchema: state => state.project.projectSchema})
 
   },
@@ -549,6 +594,7 @@ export default {
   watch: {
     options: {
       async handler () {
+        this.savePrefs(),
         await this.fetchEvents();
       },
       deep: true
@@ -567,12 +613,19 @@ export default {
     },
 
     filter (newVal, oldVal) {
+      this.savePrefs();
       if (newVal?.toLowerCase() != oldVal?.toLowerCase()) {
         this.fetchEvents();
       }
     },
 
     labelSearch () {
+      this.savePrefs();
+      this.fetchEvents();
+    },
+
+    filteredLabels () {
+      this.savePrefs()
       this.fetchEvents();
     },
 
@@ -581,7 +634,7 @@ export default {
     },
 
     user (newVal, oldVal) {
-      this.itemsPerPage = Number(localStorage.getItem(`dougal/prefs/${this.user?.name}/${this.$route.params.project}/${this.$options.name}/items-per-page`)) || 25;
+      this.loadPrefs();
     }
 
   },
@@ -632,8 +685,10 @@ export default {
 
     async fetchEvents (opts = {}) {
       const options = {
+        sequence: this.$route.params.sequence,
         text: this.filter,
         label: this.labelSearch,
+        excludeLabels: this.filteredLabels,
         ...this.options
       };
       const res = await this.getEvents([this.$route.params.project, options]);
@@ -737,6 +792,13 @@ export default {
         if (event.id) {
           const id = event.id;
           delete event.id;
+
+          // If this is an edit, ensure that it is *either*
+          // a timestamp event or a sequence + point one.
+          if (event.sequence && event.point && event.tstamp) {
+            delete event.tstamp;
+          }
+
           this.putEvent(id, event, callback); // No await
         } else {
           this.postEvent(event, callback); // No await
@@ -864,10 +926,36 @@ export default {
       */
     },
 
+    getPrefsKey () {
+      return `dougal/prefs/${this.user?.name}/${this.$route.params.project}/Log/v1`;
+    },
+
+    savePrefs () {
+      const prefs = {
+        shownLabels: this.shownLabels,
+        labelSearch: this.labelSearch,
+        filter: this.filter,
+        options: this.options
+      };
+      localStorage.setItem(this.getPrefsKey(), JSON.stringify(prefs));
+    },
+
+    loadPrefs () {
+      const stored = localStorage.getItem(this.getPrefsKey());
+      if (stored) {
+        const prefs = JSON.parse(stored);
+        if (prefs.shownLabels !== undefined) this.shownLabels = prefs.shownLabels;
+        if (prefs.labelSearch !== undefined) this.labelSearch = prefs.labelSearch;
+        if (prefs.filter !== undefined) this.filter = prefs.filter;
+        if (prefs.options !== undefined) this.options = prefs.options;
+      }
+    },
+
     ...mapActions(["api", "showSnack", "refreshEvents", "getEvents"])
   },
 
   async mounted () {
+    this.loadPrefs();
     this.fetchEvents();
 
     window.addEventListener('keyup', this.handleKeyboardEvent);

lib/www/client/source/src/views/Map.vue

@@ -31,7 +31,47 @@
 
           <span>Vessel track</span>
           <label title="Show points"><v-icon small left class="mx-0">mdi-vector-point</v-icon> <input type="checkbox" value="navp" v-model="layerSelection"/></label>
+
+          <!--
           <label title="Show lines" disabled><v-icon small left class="mx-0">mdi-vector-line</v-icon> <input type="checkbox" value="navl" v-model="layerSelection"/></label>
+          -->
+
+          <div>
+            <v-menu bottom offset-y class="pb-1">
+              <template v-slot:activator="{ on, attrs }">
+                <v-icon style="margin-right: 3px;" small v-bind="attrs" v-on="on" :title="`Show lines.\nCurrently selected period: ${vesselTrackPeriodSettings[vesselTrackPeriod].title}. Click to change`">mdi-vector-line</v-icon>
+              </template>
+              <v-list nav dense>
+                <v-list-item @click="vesselTrackPeriod = 'hour'">
+                  <v-list-item-content>
+                    <v-list-item-title>Last hour</v-list-item-title>
+                  </v-list-item-content>
+                </v-list-item>
+                <v-list-item @click="vesselTrackPeriod = 'hour6'">
+                  <v-list-item-content>
+                    <v-list-item-title>Last 6 hours</v-list-item-title>
+                  </v-list-item-content>
+                </v-list-item>
+                <v-list-item @click="vesselTrackPeriod = 'hour12'">
+                  <v-list-item-content>
+                    <v-list-item-title>Last 12 hours</v-list-item-title>
+                  </v-list-item-content>
+                </v-list-item>
+                <v-list-item @click="vesselTrackPeriod = 'day'">
+                  <v-list-item-content>
+                    <v-list-item-title>Last 24 hours</v-list-item-title>
+                  </v-list-item-content>
+                </v-list-item>
+                <v-list-item @click="vesselTrackPeriod = 'week'">
+                  <v-list-item-content>
+                    <v-list-item-title>Last week</v-list-item-title>
+                  </v-list-item-content>
+                </v-list-item>
+              </v-list>
+            </v-menu>
+          <input type="checkbox" value="navl" v-model="layerSelection"/>
+         </div>
+
           <label><!-- No heatmap available --></label>
 
           <span>Sail lines</span>
@@ -359,6 +399,7 @@
         </v-select>
         END QC data -->
 
+<!--
         <hr class="my-2"/>
 
         <div title="Not yet implemented">
@@ -371,6 +412,7 @@
             Map settings
           </v-btn>
         </div>
+-->
       </div>
     </div>
 
@@ -428,6 +470,33 @@
           @click="zoomOut"
           >mdi-magnify-minus-outline</v-icon>
       </div>
+      <div>
+        <v-icon
+          class="my-1"
+          title="Tilt out"
+          @click="tiltOut"
+          >mdi-axis-x-rotate-counterclockwise</v-icon>
+      </div>
+      <div>
+        <v-icon
+          class="my-1"
+          title="Tilt in"
+          @click="tiltIn"
+          >mdi-axis-x-rotate-clockwise</v-icon>
+      </div>
+      <div>
+        <v-icon v-if="bearing==0"
+          class="my-1"
+          title="Bin up"
+          @click="setBearing('ζ')"
+          >mdi-view-grid-outline</v-icon>
+        <v-icon v-else
+          class="my-1"
+          title="North up"
+          :style="`transform: rotate(${-bearing}deg);`"
+          @click="setBearing(0)"
+          >mdi-navigation</v-icon>
+      </div>
       <div>
         <v-icon
           class="my-1"
@@ -619,7 +688,69 @@ export default {
         //maxZoom: 18,
         maxPitch: 89
       },
+      bearing: 0,
 
+      vesselPosition: null,
+      vesselTrackLastRefresh: 0,
+      vesselTrackRefreshInterval: 12, // seconds
+      vesselTrackIntervalID: null,
+      vesselTrackPeriod: "hour",
+      vesselTrackPeriodSettings: {
+        hour: {
+          title: "1 hour",
+          offset: 3600 * 1000,
+          decimation: 1,
+          refreshInterval: 18,
+        },
+        hour6: {
+          title: "6 hours",
+          offset: 6 * 3600 * 1000,
+          decimation: 1,
+          refreshInterval: 18,
+        },
+        hour12: {
+          title: "12 hours",
+          offset: 12 * 3600 * 1000,
+          decimation: 1,
+          refreshInterval: 18,
+        },
+        day: {
+          title: "24 hours",
+          offset: 24 * 3600 * 1000,
+          decimation: 12,
+          refreshInterval: 18,
+        },
+        week: {
+          title: "7 days",
+          offset: 7 * 24 * 3600 * 1000,
+          decimation: 60,
+          refreshInterval: 60,
+        },
+        week2: {
+          title: "14 days",
+          offset: 14 * 24 * 3600 * 1000,
+          decimation: 60,
+          refreshInterval: 90,
+        },
+        month: {
+          title: "30 days",
+          offset: 30 * 24 * 3600 * 1000,
+          decimation: 90,
+          refreshInterval: 120,
+        },
+        quarter: {
+          title: "90 days",
+          offset: 90 * 24 * 3600 * 1000,
+          decimation: 180,
+          refreshInterval: 300,
+        },
+        year: {
+          title: "1 year",
+          offset: 365 * 24 * 3600 * 1000,
+          decimation: 1200,
+          refreshInterval: 1800,
+        },
+      },
       heatmapValue: "total_error",
       isFullscreen: false,
       crosshairsPositions: [],
@@ -761,6 +892,14 @@ export default {
       deep: true
     },
 
+    vesselTrackPeriod () {
+      this.updateVesselIntervalTimer();
+    },
+
+    vesselTrackLastRefresh () {
+      this.render();
+    },
+
     lines () {
       // Refresh map on change of preplot data
       this.render();
@@ -866,6 +1005,41 @@ export default {
       }
     },
 
+    tiltIn () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch -= 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    tiltOut () {
+      if (deck) {
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.pitch += 10;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
+    setBearing (bearing) {
+      if (deck) {
+
+        if (bearing === 'ζ') {
+          bearing = this.$store.getters.projectConfiguration?.binning?.theta ?? 0;
+        }
+
+        const viewState = deck.getViewports()[0];
+        const initialViewState = {...this.viewStateDefaults, ...viewState};
+        initialViewState.bearing = (bearing + 360) % 360;
+        initialViewState.transitionDuration = 300;
+        deck.setProps({initialViewState});
+      }
+    },
+
     toggleFullscreen() {
       const mapElement = document.getElementById('map-container');
       if (!this.isFullscreen) {
@@ -1062,6 +1236,12 @@ export default {
       return [[λ0 - mλ, φ0 - mφ], [λ1 + mλ, φ1 + mφ]];
     },
 
+    // Returns the current second, as an integer.
+    // Used for triggering Deck.gl URL refreshes
+    currentSecond () {
+      return Math.floor(Date.now()/1000);
+    },
+
     async getSequenceData (sequenceNumbers, types = [2, 3]) {
 
       //const types = [2, 3]; // Bundle types: 2 → raw/gun data; 3 ‒ final data. See bundles.js
@@ -1251,7 +1431,7 @@ export default {
       //console.log("SHOULD BE INITIALISING LAYERS HERE", gl);
       this.decodeURL();
       this.decodeURLHash();
-      deck.onViewStateChange = this.updateURL;
+      //deck.onViewStateChange = this.viewStateUpdated;
     },
 
     setViewState () {
@@ -1266,6 +1446,11 @@ export default {
       }
     },
 
+    viewStateUpdated ({viewState}) {
+      this.bearing = viewState.bearing;
+      this.updateURL({viewState});
+    },
+
     updateURL ({viewState} = {}) {
       if (!viewState && deck?.viewManager) {
         viewState = deck.getViewports()[0];
@@ -1407,6 +1592,19 @@ export default {
       return arr.buffer;
     },
 
+    updateVesselIntervalTimer (refreshInterval) {
+      this.vesselTrackRefreshInterval = refreshInterval ??
+        this.vesselTrackPeriodSettings[this.vesselTrackPeriod]?.refreshInterval ?? 0;
+
+      this.vesselTrackIntervalID = clearInterval(this.vesselTrackIntervalID);
+      if (this.vesselTrackRefreshInterval) {
+        this.vesselTrackLastRefresh = this.currentSecond();
+        this.vesselTrackIntervalID = setInterval( () => {
+          this.vesselTrackLastRefresh = this.currentSecond();
+        }, this.vesselTrackRefreshInterval * 1000);
+      }
+    },
+
     async handleSequences (context, {payload}) {
       if (payload.pid != this.$route.params.project) {
         console.warn(`${this.$route.params.project} ignoring notification for ${payload.pid}`);
@@ -1438,18 +1636,46 @@ export default {
       }
     },
 
+    handleVesselPosition (context, {payload}) {
+      if (payload.new?.geometry?.coordinates) {
+        const now = Date.now();
+        const lastRefresh = this.vesselPosition?._lastRefresh;
+
+        // Limits refreshes to once every five seconds max
+        if (lastRefresh && (now-lastRefresh) < 5000) return;
+
+        this.vesselPosition = {
+          ...payload.new.meta,
+          tstamp: payload.new.tstamp,
+          _lastRefresh: now
+        };
+        if (this.vesselPosition.lineStatus == "offline") {
+          this.vesselPosition.x = this.vesselPosition.longitude ?? payload.new.geometry.coordinates[0];
+          this.vesselPosition.y = this.vesselPosition.latitude ?? payload.new.geometry.coordinates[1];
+        } else {
+          this.vesselPosition.x = this.vesselPosition.longitudeMaster
+            ?? payload.new.geometry.coordinates[0];
+          this.vesselPosition.y = this.vesselPosition.latitudeMaster
+            ?? payload.new.geometry.coordinates[1];
+        }
+        this.render();
+      }
+    },
+
     registerNotificationHandlers (action = "registerHandler") {
 
       ["raw_lines", "raw_shots", "final_lines", "final_shots"].forEach( table => {
         this.$store.dispatch(action, {
           table,
-
-          handler: (context, message) => {
-            this.handleSequences(context, message);
-          }
+          handler: this.handleSequences
         })
       });
 
+      this.$store.dispatch(action, {
+        table: 'realtime',
+        handler: this.handleVesselPosition
+      });
+
     },
 
     unregisterNotificationHandlers () {
@@ -1473,6 +1699,8 @@ export default {
       console.log("TODO: Should switch to legacy map view");
     }
 
+    this.updateVesselIntervalTimer();
+
     this.layersAvailable.osm = this.osmLayer;
 
     this.layersAvailable.sea = this.openSeaMapLayer;
@@ -1549,7 +1777,8 @@ export default {
       layers: [],
       getTooltip: this.getTooltip,
       pickingRadius: 24,
-      onWebGLInitialized: this.initLayers
+      onWebGLInitialized: this.initLayers,
+      onViewStateChange: this.viewStateUpdated,
     });
 
     // Get fullscreen state
@@ -1572,6 +1801,7 @@ export default {
 
   beforeDestroy () {
     this.unregisterNotificationHandlers();
+    this.vesselTrackIntervalID = this.clearInterval(this.vesselTrackIntervalID);
   }
 
 }

lib/www/client/source/src/views/MapLayersMixin.vue

@@ -5,8 +5,9 @@
 import { Deck, WebMercatorViewport, FlyToInterpolator, CompositeLayer } from '@deck.gl/core';
 import { GeoJsonLayer, LineLayer, PathLayer, BitmapLayer, ScatterplotLayer, ColumnLayer, IconLayer } from '@deck.gl/layers';
 import {HeatmapLayer} from '@deck.gl/aggregation-layers';
-import { TileLayer, MVTLayer } from '@deck.gl/geo-layers';
-
+import { TileLayer, MVTLayer, TripsLayer } from '@deck.gl/geo-layers';
+import { SimpleMeshLayer } from '@deck.gl/mesh-layers';
+import { OBJLoader } from '@loaders.gl/obj';
 
 //import { json } from 'd3-fetch';
 import * as d3a from 'd3-array';
@@ -18,7 +19,6 @@ import DougalBinaryLoader from '@/lib/deck.gl/DougalBinaryLoader';
 
 import { colors } from 'vuetify/lib'
 
-
 function hexToArray (hex, defaultValue = [ 0xc0, 0xc0, 0xc0, 0xff ]) {
 
   if (typeof hex != "string" || hex.length < 6) {
@@ -121,6 +121,21 @@ export default {
       };
     },
 
+    loadOptions (options = {}) {
+      return {
+        loadOptions: {
+          fetch: {
+            method: 'GET',
+            headers: {
+              'Authorization': `Bearer ${this.$store.getters.jwt}`,
+            }
+          },
+          ...options
+        },
+      };
+
+    },
+
 
     osmLayer (options = {}) {
       return new TileLayer({
@@ -241,45 +256,99 @@ export default {
     },
 
     vesselTrackPointsLayer (options = {}) {
-      return new ScatterplotLayer({
+
+      if (!this.vesselPosition) return;
+
+      return new SimpleMeshLayer({
         id: 'navp',
-        data: `/api/navdata?limit=10000`,
-        getPosition: (d) => ([d.longitude, d.latitude]),
-        getRadius: d => (d.speed),
-        radiusScale: 1,
-        lineWidthMinPixels: 2,
-        getFillColor: d => d.guns
-          ? d.lineStatus == "online"
-            ? [0xaa, 0x00, 0xff] // Online
-            : [0xd5, 0x00, 0xf9] // Soft start or guns otherwise active
-          : [0xea, 0x80, 0xfc],  // Offline, guns inactive
-        getLineColor: [127, 65, 90],
-        getColor: [ 255, 0, 0 ],
-        getPointRadius: 12,
-        radiusUnits: "pixels",
-        pointRadiusMinPixels: 4,
-        stroked: false,
-        filled: true,
+        data: [ this.vesselPosition ],
+        //getColor: [ 255, 48, 0 ],
+        getColor: [ 174, 1, 174 ],
+        getOrientation: d => [0, (270 - (d.heading ?? d.cmg ?? d.bearing ?? d.lineBearing ?? 0)) % 360 , 0],
+        getPosition: d => [ d.x, d.y ],
+        mesh: `/assets/boat0.obj`,
+        sizeScale: 0.1,
+        loaders: [OBJLoader],
         pickable: true,
         ...options
-      })
+      });
+
     },
 
     vesselTrackLinesLayer (options = {}) {
-      return new LineLayer({ // TODO Change to TrackLayer
+
+      const cfg = this.vesselTrackPeriodSettings[this.vesselTrackPeriod];
+
+      let ts1 = new Date(this.vesselTrackLastRefresh*1000);
+      let ts0 = new Date(ts1.valueOf() - cfg.offset);
+      let di = cfg.decimation;
+      let l = 10000;
+
+      const breakLimit = (di ? di*20 : 5 * 60) * 1000;
+
+      let trailLength = (ts1 - ts0) / 1000;
+
+      return new TripsLayer({
         id: 'navl',
-        data: `/api/navdata?v=${Date.now()}`, // NOTE Not too sure about this
-        lineWidthMinPixels: 2,
-        getLineColor: (d) => d.properties.ntba ? [240, 248, 255, 200] : [85, 170, 255, 200],
-        getSourcePosition: (obj, i) => i.index < i.data?.length ? [i.data[i.index]?.longitude, i.data[i.index]?.latitude] : null,
-        getTargetPosition: (obj, i) => i.index < i.data?.length ? [i.data[i.index+1]?.longitude, i.data[i.index+1]?.latitude] : null,
-        getLineWidth: 3,
-        getPointRadius: 2,
-        radiusUnits: "pixels",
-        pointRadiusMinPixels: 2,
+        data: `/api/vessel/track/?di=${di}&l=${l}&project=&ts0=${ts0.toISOString()}&ts1=${ts1.toISOString()}`,
+        ...this.loadOptions({
+          fetch: {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${this.$store.getters.jwt}`,
+            }
+          }
+        }),
+        dataTransform: (data) => {
+            if (data.length >= l) {
+              console.warn(`Vessel track data may be truncated! Limit: ${l}`);
+            }
+
+            const paths = [];
+            let prevTstamp;
+            paths.push({path: [], timestamps: [], num: 0, ts0: +Infinity, ts1: -Infinity});
+            for (const el of data) {
+              const tstamp = new Date(el.tstamp).valueOf();
+              const curPath = () => paths[paths.length-1];
+              if (prevTstamp && Math.abs(tstamp - prevTstamp) > breakLimit) {
+                // Start a new path
+                console.log(`Breaking path on interval ${Math.abs(tstamp - prevTstamp)} > ${breakLimit}`);
+                paths.push({path: [], timestamps: [], num: paths.length, ts0: +Infinity, ts1: -Infinity});
+              }
+
+              if (tstamp < curPath().ts0) {
+                curPath().ts0 = tstamp;
+              }
+              if (tstamp > curPath().ts1) {
+                curPath().ts1 = tstamp;
+              }
+
+              curPath().path.push([el.x, el.y]);
+              curPath().timestamps.push(tstamp/1000);
+              prevTstamp = tstamp;
+            }
+
+            paths.forEach (path => {
+              path.nums = paths.length;
+              path.ts0 = new Date(path.ts0);
+              path.ts1 = new Date(path.ts1);
+            });
+
+            return paths;
+        },
+        getPath: d => d.path,
+        getTimestamps: d => d.timestamps,
+        currentTime: ts1.valueOf() / 1000,
+        trailLength,
+        widthUnits: "meters",
+        widthMinPixels: 4,
+        getWidth: 10,
+        getColor: [ 174, 1, 126, 200 ],
+        stroked: true,
         pickable: true,
         ...options
-      })
+      });
+
     },
 
     eventsLogLayer (options = {}) {
@@ -308,6 +377,7 @@ export default {
       return new DougalEventsLayer({
         id: 'log',
         data: `/api/project/${this.$route.params.project}/event?mime=application/geo%2Bjson`,
+        ...this.loadOptions(),
         lineWidthMinPixels: 2,
         getPosition: d => d.geometry.coordinates,
         jitter: 0.00015,
@@ -332,6 +402,7 @@ export default {
       return new GeoJsonLayer({
         id: 'psll',
         data: `/api/project/${this.$route.params.project}/gis/preplot/line?class=V&v=${this.lineTStamp?.valueOf()}`,
+        ...this.loadOptions(),
         lineWidthMinPixels: 1,
         getLineColor: (d) => d.properties.ntba ? [240, 248, 255, 200] : [85, 170, 255, 200],
         getLineWidth: 1,
@@ -347,6 +418,7 @@ export default {
       return new GeoJsonLayer({
         id: 'ppll',
         data: `/api/project/${this.$route.params.project}/gis/preplot/line?v=${this.lineTStamp?.valueOf()}`,
+        ...this.loadOptions(),
         lineWidthMinPixels: 1,
         getLineColor: (d) => d.properties.ntba ? [240, 248, 255, 200] : [85, 170, 255, 200],
         getLineWidth: 1,
@@ -393,6 +465,7 @@ export default {
       return new GeoJsonLayer({
         id: 'seqrl',
         data: `/api/project/${this.$route.params.project}/gis/raw/line?v=${this.sequenceTStamp?.valueOf()}`,
+        ...this.loadOptions(),
         lineWidthMinPixels: 1,
         getLineColor: (d) => d.properties.ntbp ? [0xe6, 0x51, 0x00, 200] : [0xff, 0x98, 0x00, 200],
         getLineWidth: 1,
@@ -408,6 +481,7 @@ export default {
       return new GeoJsonLayer({
         id: 'seqfl',
         data: `/api/project/${this.$route.params.project}/gis/final/line?v=${this.sequenceTStamp?.valueOf()}`,
+        ...this.loadOptions(),
         lineWidthMinPixels: 1,
         getLineColor: (d) => d.properties.pending ? [0xa7, 0xff, 0xab, 200] : [0x00, 0x96, 0x88, 200],
         getLineWidth: 1,
@@ -424,12 +498,15 @@ export default {
         id: 'pslp',
         data: `/api/project/${this.$route.params.project}/line/sail?v=${this.lineTStamp?.valueOf()}`, // API endpoint returning binary data
         loaders: [DougalBinaryLoader],
-        loadOptions: {
+        ...this.loadOptions({
           fetch: {
             method: 'GET',
-            headers: { Accept: 'application/vnd.aaltronav.dougal+octet-stream' }
+            headers: {
+              Authorization: `Bearer ${this.$store.getters.jwt}`,
+              Accept: 'application/vnd.aaltronav.dougal+octet-stream'
+            }
           }
-        },
+        }),
         getRadius: 2,
         getFillColor: (d, {data, index}) => data.attributes.value2.value[index] ? [240, 248, 255, 200] : [85, 170, 255, 200],
         //getFillColor: [0, 120, 220, 200],
@@ -443,12 +520,15 @@ export default {
         id: 'pplp',
         data: `/api/project/${this.$route.params.project}/line/source?v=${this.lineTStamp?.valueOf()}`, // API endpoint returning binary data
         loaders: [DougalBinaryLoader],
-        loadOptions: {
+        ...this.loadOptions({
           fetch: {
             method: 'GET',
-            headers: { Accept: 'application/vnd.aaltronav.dougal+octet-stream' }
+            headers: {
+              Authorization: `Bearer ${this.$store.getters.jwt}`,
+              Accept: 'application/vnd.aaltronav.dougal+octet-stream'
+            }
           }
-        },
+        }),
         getRadius: 2,
         getFillColor: (d, {data, index}) => data.attributes.value2.value[index] ? [240, 248, 255, 200] : [85, 170, 255, 200],
         //getFillColor: [0, 120, 220, 200],

lib/www/client/source/src/views/MapTooltipsMixin.vue

@@ -1,4 +1,5 @@
 <script>
+import * as d3a from 'd3-array';
 
 export default {
   name: "MapTooltipsMixin",
@@ -28,6 +29,8 @@ export default {
         return this.sequenceLinesTooltip(args);
       } else if (args?.layer?.id == "navp") {
         return this.vesselTrackPointsTooltip(args);
+      } else if (args?.layer?.id == "navl") {
+        return this.vesselTrackLinesTooltip(args);
       }
     },
 
@@ -235,7 +238,20 @@ export default {
       }
     },
 
+    vesselTrackLinesTooltip (args) {
+      const p = args.object;
 
+      console.log("track lines tooltip", p);
+
+      if (p) {
+
+        let html = `Segment ${p.num+1} / ${p.nums}<br/>\n`
+          html += `${p.ts0.toISOString()}<br/>\n`
+          html += `${p.ts1.toISOString()}<br/>\n`;
+
+        return {html, style: this.tooltipDefaultStyle};
+      }
+    },
 
   }
 

lib/www/client/source/src/views/ProjectList.vue

@@ -27,6 +27,8 @@
         <v-chip v-for="group in value"
           label
           small
+          :title="`View repeatability data for ${group}`"
+          :href="`/groups/${group}`"
         >{{ group }}</v-chip>
       </template>
 
@@ -187,19 +189,20 @@ export default {
     ...mapGetters(['loading', 'projects'])
   },
 
+  watch: {
+
+    async projects () {
+      await this.load();
+    }
+
+  },
+
   methods: {
 
     async list () {
       this.items = [...this.projects];
     },
 
-    async summary (item) {
-      const details = await this.api([`/project/${item.pid}/summary`]);
-      if (details) {
-        return Object.assign({}, details, item);
-      }
-    },
-
     title (item) {
       if (item.organisations) {
         return "Access:\n" + Object.entries(item.organisations).map( org =>
@@ -210,30 +213,22 @@ export default {
     },
 
     async load () {
-      await this.refreshProjects();
+      if (!this.projects.length) {
+        this.refreshProjects();
+      }
       await this.list();
-      const promises = [];
-      for (const key in this.items) {
-        const item = this.items[key];
-        const promise = this.summary(item)
-        .then( expanded => {
-          if (expanded) {
-            this.$set(this.items, key, expanded);
-          }
-        });
-        promises.push(promise);
+    },
+
+    handlerLoad (context, {payload}) {
+      if (payload?.table == "public") {
+        this.load();
       }
     },
 
     registerNotificationHandlers () {
       this.$store.dispatch('registerHandler', {
         table: 'project`',
-
-        handler: (context, message) => {
-          if (message.payload?.table == "public") {
-            this.load();
-          }
-        }
+        handler: this.handlerLoad
       });
     },
 

lib/www/client/source/src/views/SequenceSummary.vue

@@ -6,8 +6,42 @@
         <v-progress-linear indeterminate v-if="loading"></v-progress-linear>
         <v-toolbar flat>
           <v-toolbar-title>
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'shotlog', params: { sequence: (sequences[sequences.length-1]||{}).sequence }}"
+                title="Go to the first sequence"
+              >
+                <v-icon dense>mdi-chevron-double-left</v-icon>
+              </v-btn>
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'shotlog', params: { sequence: (sequences[sequenceIndex+1]||{}).sequence }}"
+                title="Go to the previous sequence"
+              >
+                <v-icon dense>mdi-chevron-left</v-icon>
+              </v-btn>
+            </template>
+
             Sequence {{sequenceNumber}}
             <small :class="statusColour" v-if="sequence">({{sequence.status}})</small>
+
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex==0"
+                :to="{name: 'shotlog', params: { sequence: (sequences[sequenceIndex-1]||{}).sequence }}"
+                title="Go to the next sequence"
+              >
+                <v-icon dense>mdi-chevron-right</v-icon>
+              </v-btn>
+              <v-btn icon small class="mr-1"
+                :disabled="sequenceIndex==0"
+                :to="{name: 'shotlog', params: { sequence: (sequences[0]||{}).sequence }}"
+                title="Go to the last sequence"
+              >
+                <v-icon dense>mdi-chevron-double-right</v-icon>
+              </v-btn>
+            </template>
           </v-toolbar-title>
 
           <a v-if="$route.params.sequence"
@@ -352,6 +386,16 @@ export default {
       return this.sequences.find(i => i.sequence == this.sequenceNumber);
     },
 
+    sequenceIndex () {
+      if ("sequence" in this.$route.params) {
+        const index = this.sequences.findIndex( i => i.sequence == this.$route.params.sequence );
+        if (index != -1) {
+          return index;
+        }
+      }
+      // return undefined
+    },
+
     remarks () {
       return this.sequence?.remarks || "Nil.";
     },

lib/www/server/api/index.js

@@ -69,6 +69,7 @@ const allMeta = (key, value) => {
 	return { all: [ meta(key, value) ] };
 };
 
+
 //
 // NOTICE These routes do not require authentication
 //
@@ -103,6 +104,9 @@ app.use(mw.auth.access.user);
 // Don't process the request if the data hasn't changed
 app.use(mw.etag.ifNoneMatch);
 
+// Use compression across the board
+app.use(mw.compress);
+
 // We must be authenticated before we can access these
 app.map({
 	'/project': {
@@ -117,10 +121,12 @@ app.map({
 		get: [ mw.auth.access.read, mw.project.summary.get ],
 	},
 	'/project/:project/configuration': {
-		get: [ mw.project.configuration.get ],	// Get project configuration
 		patch: [ mw.auth.access.edit, mw.project.configuration.patch ],	// Modify project configuration
 		put: [ mw.auth.access.edit, mw.project.configuration.put ],	// Overwrite configuration
 	},
+	'/project/:project/configuration/:path(*)?': {
+		get: [ mw.auth.access.read, mw.configuration.get ],
+	},
 
 	/*
 	 * GIS endpoints
@@ -219,16 +225,28 @@ app.map({
 		'changes/:since': {
 			get: [ mw.auth.access.read, mw.event.changes ]
 		},
-		// TODO Rename -/:sequence → sequence/:sequence
+		// NOTE: old alias for /sequence/:sequence
 		'-/:sequence/': { // NOTE: We need to avoid conflict with the next endpoint ☹
 			get: [ mw.auth.access.read, mw.event.sequence.get ],
 		},
-		':id/': {
+		'sequence/:sequence/': {
+			get: [ mw.auth.access.read, mw.event.sequence.get ],
+		},
+		':id(\\d+)/': {
 			get: [ mw.auth.access.read, mw.event.get ],
 			put: [ mw.auth.access.write, mw.event.put ],
 			patch: [ mw.auth.access.write, mw.event.patch ],
 			delete: [mw.auth.access.write,  mw.event.delete ]
 		},
+		'import': {
+			put: [ mw.auth.access.write, mw.event.import.csv, mw.event.import.put ],
+			post: [ mw.auth.access.write, mw.event.import.csv, mw.event.import.put ],
+			'/:filename': {
+				put: [ mw.auth.access.read, mw.event.import.csv, mw.event.import.put ],
+				post: [ mw.auth.access.write, mw.event.import.csv, mw.event.import.put ],
+				delete: [ mw.auth.access.write, mw.event.import.delete ]
+			},
+		},
 	},
 
 	/*
@@ -268,10 +286,6 @@ app.map({
 	'/project/:project/label/': {
 		get: [ mw.auth.access.read, mw.label.list ],
 // 		post: [ mw.label.post ],
-	},
-	'/project/:project/configuration/:path(*)?': {
-		get: [ mw.auth.access.read, mw.configuration.get ],
-// 		post: [ mw.auth.access.admin, mw.label.post ],
 	},
 	'/project/:project/info/:path(*)': {
 		get: [ mw.auth.operations, mw.auth.access.read, mw.info.get ],
@@ -311,6 +325,30 @@ app.map({
 			get: [ mw.etag.noSave, mw.gis.navdata.get ]
 		}
 	},
+	'/vessel/track': {
+		get: [ /*mw.etag.noSave,*/ mw.vessel.track.get ], // JSON array
+		'/line': {
+			get: [ // GeoJSON Feature: type = LineString
+				//mw.etag.noSave,
+				(req, res, next) => { req.query.geojson = 'LineString'; next(); },
+				mw.vessel.track.get
+			]
+		},
+		'/point': {
+			get: [ // GeoJSON FeatureCollection: feature types = Point
+				//mw.etag.noSave,
+				(req, res, next) => { req.query.geojson = 'Point'; next(); },
+				mw.vessel.track.get
+			]
+		},
+		'/points': {
+			get: [ // JSON array of (Feature: type = Point)
+				mw.etag.noSave,
+				(req, res, next) => { req.query.geojson = true; next(); },
+				mw.vessel.track.get
+			],
+		},
+	},
 	'/info/': {
 		':path(*)': {
 			get: [ mw.auth.operations, mw.info.get ],
@@ -319,6 +357,26 @@ app.map({
 			delete: [ mw.auth.operations, mw.auth.access.write, mw.info.delete ]
 		}
 	},
+
+	/*
+	 * 4D comparisons
+	 */
+
+	// FIXME no authentication yet!
+
+	'/comparison/group': {
+		get: [ mw.etag.noSave, mw.comparisons.groups.list ],
+		'/:group': {
+			get: [ mw.etag.noSave, mw.comparisons.groups.get ],
+		},
+	},
+
+
+
+	/*
+	 * Other endpoints
+	 */
+
 	'/queue/outgoing/': {
 		'asaqc': {
 			get: [ mw.etag.noSave, mw.queue.asaqc.get ],

lib/www/server/api/middleware/auth/access.js

@@ -1,6 +1,7 @@
 const { projectOrganisations, vesselOrganisations/*, orgAccess */} = require('../../../lib/db/project/organisations');
 const ServerUser = require('../../../lib/db/user/User');
 const { Organisations } = require('@dougal/organisations');
+const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
 /** Second-order function.
  * Returns a middleware that checks if the user has access to
@@ -14,11 +15,7 @@ function operation (operation) {
 		if (req.params.project) {
 			const projectOrgs = new Organisations(await projectOrganisations(req.params.project));
 			const availableOrgs = projectOrgs.accessToOperation(operation).filter(user.organisations);
-			console.log("Operation:      ", operation);
-			console.log("User:           ", user.name);
-			console.log("User orgs:      ", user.organisations);
-			console.log("Project orgs:   ", projectOrgs);
-			console.log("Available orgs: ", availableOrgs);
+			DEBUG(`operation = ${operation}, user = ${user?.name}, user orgs = %j, project orgs = %j, availableOrgs = %j`, user.organisations.toJSON(), projectOrgs.toJSON(), availableOrgs.toJSON());
 			if (availableOrgs.length > 0) {
 				next();
 				return;
@@ -26,16 +23,13 @@ function operation (operation) {
 		} else {
 			const vesselOrgs = new Organisations(await vesselOrganisations());
 			const availableOrgs = vesselOrgs.accessToOperation(operation).filter(user.organisations);
-			console.log("Operation:      ", operation);
-			console.log("User:           ", user.name);
-			console.log("User orgs:      ", user.organisations);
-			console.log("Vessel orgs:    ", vesselOrgs);
-			console.log("Available orgs: ", availableOrgs);
+			DEBUG(`operation = ${operation}, user = ${user?.name}, user orgs = %j, vessel orgs = %j, availableOrgs = %j`, user.organisations.toJSON(), vesselOrgs.toJSON(), availableOrgs.toJSON());
 			if (availableOrgs.length > 0) {
 				next();
 				return;
 			}
 		}
+		DEBUG(`Access denied to operation ${operation}.`);
 		next({status: 403, message: "Access denied"});
 	}
 }

lib/www/server/api/middleware/auth/authentify.js

@@ -1,41 +1,123 @@
 const dns = require('dns');
-const { Netmask } = require('netmask');
+const ipaddr = require('ipaddr.js');
+const { isIPv6, isIPv4 } = require('net');
 const cfg = require('../../../lib/config');
 const jwt = require('../../../lib/jwt');
 const user = require('../../../lib/db/user');
 const ServerUser = require('../../../lib/db/user/User');
+const { ERROR, WARNING, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
-async function authorisedIP (req, res) {
-	const validIPs = await user.ip({active: true}); // Get all active IP logins
-	validIPs.forEach( i => i.$block = new Netmask(i.ip) );
-	validIPs.sort( (a, b) => b.$block.bitmask - a.$block.netmask ); // More specific IPs have precedence
-	for (const ip of validIPs) {
-		const block = ip.$block;
-		if (block.contains(req.ip)) {
-			const payload = {
-				...ip,
-				ip: req.ip,
-				autologin: true
-			};
-			delete payload.$block;
-			delete payload.hash;
-			delete payload.active;
-			jwt.issue(payload, req, res);
-			return true;
+function parseIP(ip) {
+	if (!ip || typeof ip !== 'string') {
+		WARNING('Invalid IP input:', ip);
+		return null;
+	}
+	// Handle comma-separated X-Forwarded-For (e.g., "87.90.254.127,")
+	const cleanIp = ip.split(',')[0].trim();
+	if (!cleanIp) {
+		WARNING('Empty IP after parsing:', ip);
+		return null;
+	}
+	// Convert IPv6-mapped IPv4 (e.g., ::ffff:127.0.0.1 -> 127.0.0.1)
+	if (cleanIp.startsWith('::ffff:') && isIPv4(cleanIp.split('::ffff:')[1])) {
+		return cleanIp.split('::ffff:')[1];
+	}
+	return cleanIp;
+}
+
+function normalizeCIDR(range) {
+	if (!range || typeof range !== 'string') {
+		WARNING('Invalid CIDR range:', range);
+		return null;
+	}
+	// If no /prefix, assume /32 for IPv4 or /128 for IPv6
+	if (!range.includes('/')) {
+		try {
+			const parsed = ipaddr.parse(range);
+			const prefix = parsed.kind() === 'ipv4' ? 32 : 128;
+			return `${range}/${prefix}`;
+		} catch (err) {
+			WARNING(`Failed to parse bare IP ${range}:`, err.message);
+			return null;
+		}
+	}
+	return range;
+}
+
+async function authorisedIP(req, res) {
+	const ip = parseIP(req.ip || req.headers['x-forwarded-for'] || req.headers['x-real-ip']);
+	DEBUG('authorisedIP:', { ip, headers: req.headers }); // Debug
+	if (!ip) {
+		WARNING('No valid IP provided:', { ip, headers: req.headers });
+		return false;
+	}
+
+	let addr;
+	try {
+		addr = ipaddr.parse(ip);
+	} catch (err) {
+		WARNING('Invalid IP:', ip, err.message);
+		return false;
+	}
+
+	const validIPs = await user.ip({ active: true }); // Get active IP logins
+	// Attach parsed CIDR to each IP entry
+	validIPs.forEach(i => {
+		const normalized = normalizeCIDR(i.ip);
+		if (!normalized) {
+			i.$range = null;
+			return;
+		}
+		try {
+			const [rangeAddr, prefix] = ipaddr.parseCIDR(normalized);
+			i.$range = { addr: rangeAddr, prefix };
+		} catch (err) {
+			WARNING(`Invalid CIDR range ${i.ip}:`, err.message);
+			i.$range = null; // Skip invalid ranges
+		}
+	});
+	// Filter out invalid ranges and sort by specificity (descending prefix length)
+	const validRanges = validIPs.filter(i => i.$range).sort((a, b) => b.$range.prefix - a.$range.prefix);
+
+	for (const ipEntry of validRanges) {
+		const { addr: rangeAddr, prefix } = ipEntry.$range;
+		try {
+			if (addr.match(rangeAddr, prefix)) {
+				const payload = {
+					...ipEntry,
+					ip,
+					autologin: true
+				};
+				delete payload.$range;
+				delete payload.hash;
+				delete payload.active;
+				jwt.issue(payload, req, res);
+				return true;
+			}
+		} catch (err) {
+			WARNING(`Error checking range ${ipEntry.ip}:`, err.message);
+			continue;
 		}
 	}
 	return false;
 }
 
-async function authorisedHost (req, res) {
-	const validHosts = await user.host({active: true}); // Get all active host logins
+async function authorisedHost(req, res) {
+	const ip = parseIP(req.ip || req.headers['x-forwarded-for'] || req.headers['x-real-ip']);
+	DEBUG('authorisedHost:', { ip, headers: req.headers }); // Debug
+	if (!ip) {
+		WARNING('No valid IP for host check:', { ip, headers: req.headers });
+		return false;
+	}
+
+	const validHosts = await user.host({ active: true });
 	for (const key in validHosts) {
 		try {
-			const ip = await dns.promises.resolve(key);
-			if (ip == req.ip) {
+			const resolvedIPs = await dns.promises.resolve(key);
+			if (resolvedIPs.includes(ip)) {
 				const payload = {
 					...validHosts[key],
-					ip: req.ip,
+					ip,
 					autologin: true
 				};
 				delete payload.$block;
@@ -45,49 +127,28 @@ async function authorisedHost (req, res) {
 				return true;
 			}
 		} catch (err) {
-			if (err.code != "ENODATA") {
-				console.error(err);
+			if (err.code !== 'ENODATA') {
+				ERROR(`DNS error for host ${key}:`, err);
 			}
 		}
 	}
 	return false;
 }
 
-// TODO: Check client TLS certificates
-// Probably will do this via Nginx with
-// ssl_verify_client optional;
-// and then putting either of the
-// $ssl_client_s_dn or $ssl_client_escaped_cert
-// variables into an HTTP header for Node
-// to check (naturally, it must be ensured
-// that a user cannot just insert the header
-// in a request).
-
-
-async function auth (req, res, next) {
-
+async function auth(req, res, next) {
 	if (res.headersSent) {
-		// Nothing to do, this request must have been
-		// handled already by another middleware.
-		return;
+		return; // Handled by another middleware
 	}
 
-	// Check for a valid JWT (already decoded by a previous
-	// middleware).
+	// Check for valid JWT
 	if (req.user) {
-		if (!req.user.autologin) {
-			// If this is not an automatic login, check if the token is in the
-			// second half of its lifetime. If so, reissue a new one, valid for
-			// another cfg.jwt.options.expiresIn seconds.
-			if (req.user.exp) {
-				const ttl = req.user.exp - Date.now()/1000;
-				if (ttl < cfg.jwt.options.expiresIn/2) {
-					const credentials = await ServerUser.fromSQL(null, req.user.id);
-					if (credentials) {
-						// Refresh token
-						payload = Object.assign({}, credentials.toJSON());
-						jwt.issue(Object.assign({}, credentials.toJSON()), req, res);
-					}
+		if (!req.user.autologin && req.user.exp) {
+			const ttl = req.user.exp - Date.now() / 1000;
+			if (ttl < cfg.jwt.options.expiresIn / 2) {
+				const credentials = await ServerUser.fromSQL(null, req.user.id);
+				if (credentials) {
+					const payload = Object.assign({}, credentials.toJSON());
+					jwt.issue(payload, req, res);
 				}
 			}
 		}
@@ -95,19 +156,27 @@ async function auth (req, res, next) {
 		return;
 	}
 
-	// Check if the IP is known to us
+	// Check IP and host
 	if (await authorisedIP(req, res)) {
 		next();
 		return;
 	}
-
-	// Check if the hostname is known to us
 	if (await authorisedHost(req, res)) {
 		next();
 		return;
 	}
 
-	next({status: 401, message: "Not authorised"});
+	// If *all* else fails, check if the user came with a cookie
+	// (see https://gitlab.com/wgp/dougal/software/-/issues/335)
+	if (req.cookies.JWT) {
+		const token = req.cookies.JWT;
+		delete req.cookies.JWT;
+		DEBUG("falling back to cookie-based authentication");
+		req.user = await jwt.checkValidCredentials({jwt: token});
+		return await auth(req, res, next);
+	}
+
+	next({ status: 401, message: 'Not authorised' });
 }
 
 module.exports = auth;

lib/www/server/api/middleware/auth/jwt.js

@@ -5,8 +5,6 @@ const cfg = require("../../../lib/config").jwt;
 const getToken = function (req) {
 	if (req.headers.authorization && req.headers.authorization.split(' ')[0] == 'Bearer') {
 		return req.headers.authorization.split(' ')[1];
-	} else if (req.cookies.JWT) {
-		return req.cookies.JWT;
 	}
 	return null;
 }

lib/www/server/api/middleware/comparisons/groups/get.js

@@ -0,0 +1,20 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		// const data = await comparisons.groups()
+		// if (data?.[req.params.group]) {
+		const data = await comparisons.getGroup(req.params.group);
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(404).send({message: "Group does not exist"});
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/groups/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+	list: require('./list'),
+	get: require('./get'),
+}

lib/www/server/api/middleware/comparisons/groups/list.js

@@ -0,0 +1,18 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const data = await comparisons.groups()
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(204).end();
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	groups: require('./groups')
+}

lib/www/server/api/middleware/compress/index.js

@@ -0,0 +1,18 @@
+const compression = require('compression');
+
+const compress = compression({
+	level: 6, // Balance speed vs. ratio (1-9)
+	threshold: 512, // Compress only if response >512 bytes to avoid overhead on small bundles
+	filter: (req, res) => { // Ensure bundles are compressed
+		const accept = req.get("Accept");
+		if (accept.startsWith("application/vnd.aaltronav.dougal+octet-stream")) return true;
+		if (accept.includes("json")) return true;
+		if (accept.startsWith("text/")) return true;
+		if (accept.startsWith("model/obj")) return true;
+
+		// fallback to standard filter function
+		return compression.filter(req, res)
+	}
+});
+
+module.exports = compress;

lib/www/server/api/middleware/etag/if-none-match.js

@@ -23,9 +23,9 @@ function ifNoneMatch (req, res, next) {
 	if (cached) {
 		DEBUG("ETag match. Returning cached response (ETag: %s, If-None-Match: %s) for %s %s",
 			  cached.etag, req.get("If-None-Match"), req.method, req.url);
-		setHeaders(res, cached.headers);
 		if (req.method == "GET" || req.method == "HEAD") {
-			res.status(304).send();
+			setHeaders(res, cached.headers);
+			res.status(304).end();
 			// No next()
 		} else if (!isIdempotentMethod(req.method)) {
 			res.status(412).send();

lib/www/server/api/middleware/etag/watch.js

@@ -66,8 +66,18 @@ const rels = [
 
 function invalidateCache (data, cache) {
 	return new Promise((resolve, reject) => {
+		if (!data) {
+			ERROR("invalidateCache called with no data");
+			return;
+		}
+
+		if (!data.payload) {
+			ERROR("invalidateCache called without a payload; channel = %s", data.channel);
+			return;
+		}
+
 		const channel = data.channel;
-		const project = data.payload.pid ?? data.payload?.new?.pid ?? data.payload?.old?.pid;
+		const project = data.payload?.pid ?? data.payload?.new?.pid ?? data.payload?.old?.pid;
 		const operation = data.payload.operation;
 		const table = data.payload.table;
 		const fields = { channel, project, operation, table };

lib/www/server/api/middleware/event/import/csv.js

@@ -0,0 +1,146 @@
+const Busboy = require('busboy');
+const { parse } = require('csv-parse/sync');
+
+async function middleware(req, res, next) {
+	const contentType = req.headers['content-type'] || '';
+	let csvText = null;
+	let filename = null;
+
+	if (req.params.filename && contentType.startsWith('text/csv')) {
+		csvText = typeof req.body === 'string' ? req.body : req.body.toString('utf8');
+		filename = req.params.filename;
+		processCsv();
+	} else if (contentType.startsWith('multipart/form-data')) {
+		const busboy = Busboy({ headers: req.headers });
+		let found = false;
+		busboy.on('file', (name, file, info) => {
+			if (found) {
+				file.resume();
+				return;
+			}
+			if (info.mimeType === 'text/csv') {
+				found = true;
+				filename = info.filename || 'unnamed.csv';
+				csvText = '';
+				file.setEncoding('utf8');
+				file.on('data', (data) => { csvText += data; });
+				file.on('end', () => {});
+			} else {
+				file.resume();
+			}
+		});
+		busboy.on('field', () => {}); // Ignore fields
+		busboy.on('finish', () => {
+			if (!found) {
+				return next();
+			}
+			processCsv();
+		});
+		req.pipe(busboy);
+		return;
+	} else {
+		return next();
+	}
+
+	function processCsv() {
+		let records;
+		try {
+			records = parse(csvText, {
+				relax_quotes: true,
+				quote: '"',
+				escape: '"',
+				skip_empty_lines: true,
+				trim: true
+			});
+		} catch (e) {
+			return res.status(400).json({ error: 'Invalid CSV' });
+		}
+		if (!records.length) {
+			return res.status(400).json({ error: 'Empty CSV' });
+		}
+		const headers = records[0].map(h => h.toLowerCase().trim());
+		const rows = records.slice(1);
+		let lastDate = null;
+		let lastTime = null;
+		const currentDate = new Date().toISOString().slice(0, 10);
+		const currentTime = new Date().toISOString().slice(11, 19);
+		const events = [];
+		for (let row of rows) {
+			let object = { labels: [] };
+			for (let k = 0; k < headers.length; k++) {
+				let key = headers[k];
+				let val = row[k] ? row[k].trim() : '';
+				if (!key) continue;
+				if (['remarks', 'event', 'comment', 'comments', 'text'].includes(key)) {
+					object.remarks = val;
+				} else if (key === 'label') {
+					if (val) object.labels.push(val);
+				} else if (key === 'labels') {
+					if (val) object.labels.push(...val.split(';').map(l => l.trim()).filter(l => l));
+				} else if (key === 'sequence' || key === 'seq') {
+					if (val) object.sequence = Number(val);
+				} else if (['point', 'shot', 'shotpoint'].includes(key)) {
+					if (val) object.point = Number(val);
+				} else if (key === 'date') {
+					object.date = val;
+				} else if (key === 'time') {
+					object.time = val;
+				} else if (key === 'timestamp') {
+					object.timestamp = val;
+				} else if (key === 'latitude') {
+					object.latitude = parseFloat(val);
+				} else if (key === 'longitude') {
+					object.longitude = parseFloat(val);
+				}
+			}
+			if (!object.remarks) continue;
+			let useSeqPoint = Number.isFinite(object.sequence) && Number.isFinite(object.point);
+			let tstamp = null;
+			if (!useSeqPoint) {
+				if (object.timestamp) {
+					tstamp = new Date(object.timestamp);
+				}
+				if (!tstamp || isNaN(tstamp.getTime())) {
+					let dateStr = object.date || lastDate || currentDate;
+					let timeStr = object.time || lastTime || currentTime;
+					if (timeStr.length === 5) timeStr += ':00';
+					let full = `${dateStr}T${timeStr}.000Z`;
+					tstamp = new Date(full);
+					if (isNaN(tstamp.getTime())) continue;
+				}
+				if (object.date) lastDate = object.date;
+				if (object.time) lastTime = object.time;
+			}
+			let event = {
+				remarks: object.remarks,
+				labels: object.labels,
+				meta: {
+					author: "*CSVImport*",
+					"*CSVImport*": {
+						filename,
+						tstamp: new Date().toISOString()
+					}
+				}
+			};
+			if (!isNaN(object.latitude) && !isNaN(object.longitude)) {
+				event.meta.geometry = {
+					type: "Point",
+					coordinates: [object.longitude, object.latitude]
+				};
+			}
+			if (useSeqPoint) {
+				event.sequence = object.sequence;
+				event.point = object.point;
+			} else if (tstamp) {
+				event.tstamp = tstamp.toISOString();
+			} else {
+				continue;
+			}
+			events.push(event);
+		}
+		req.body = events;
+		next();
+	}
+}
+
+module.exports = middleware;

lib/www/server/api/middleware/event/import/delete.js

@@ -0,0 +1,18 @@
+
+const { event } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		if (req.params.project && req.params.filename) {
+			await event.unimport(req.params.project, req.params.filename, req.query);
+			res.status(204).end();
+		} else {
+			res.status(400).send({message: "Malformed request"});
+		}
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/event/import/index.js

@@ -0,0 +1,6 @@
+
+module.exports = {
+	csv: require('./csv'),
+	put: require('./put'),
+	delete: require('./delete'),
+}

lib/www/server/api/middleware/event/import/put.js

@@ -0,0 +1,16 @@
+
+const { event } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const payload = req.body;
+
+		await event.import(req.params.project, payload, req.query);
+		res.status(200).send(payload);
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/event/index.js

@@ -7,5 +7,6 @@ module.exports = {
 	put: require('./put'),
 	patch: require('./patch'),
 	delete: require('./delete'),
-	changes: require('./changes')
+	changes: require('./changes'),
+	import: require('./import'),
 }

lib/www/server/api/middleware/index.js

@@ -11,6 +11,7 @@ module.exports = {
 	gis: require('./gis'),
 	label: require('./label'),
 	navdata: require('./navdata'),
+	vessel: require('./vessel'),
 	queue: require('./queue'),
 	qc: require('./qc'),
 	configuration: require('./configuration'),
@@ -20,5 +21,7 @@ module.exports = {
 	rss: require('./rss'),
 	etag: require('./etag'),
 	version: require('./version'),
-	admin: require('./admin')
+	admin: require('./admin'),
+	compress: require('./compress'),
+	comparisons: require('./comparisons'),
 };

lib/www/server/api/middleware/project/configuration/get.js

@@ -10,7 +10,6 @@ function json (req, res, next) {
 	} else {
 		res.status(404).send({message: "Not found"});
 	}
-	next();
 }
 
 function yaml (req, res, next) {
@@ -19,7 +18,6 @@ function yaml (req, res, next) {
 	} else {
 		res.status(404).send({message: "Not found"});
 	}
-	next();
 }
 
 function csv (req, res, next) {
@@ -33,7 +31,6 @@ function csv (req, res, next) {
 	} else {
 		res.status(404).send({message: "Not found"});
 	}
-	next();
 }
 
 module.exports = async function (req, res, next) {
@@ -53,9 +50,10 @@ module.exports = async function (req, res, next) {
 			await handlers[mimetype](req, res, next);
 		} else {
 			res.status(406).send();
-			next();
 		}
+		next();
 	} catch (err) {
+		console.error(err);
 		next(err);
 	}
 }

lib/www/server/api/middleware/sequence/get/binary.js

@@ -16,7 +16,6 @@ module.exports = async function (req, res, next) {
 
 		if (json.length) {
 			const data = bundle(json, {type});
-			console.log("bundle", data);
 			res.status(200).send(Buffer.from(data));
 		} else {
 			res.status(404).send();

lib/www/server/api/middleware/user/login.js

@@ -8,7 +8,6 @@ async function login (req, res, next) {
 		if (payload) {
 			const token = jwt.issue(payload, req, res);
 			res.set("X-JWT", token);
-			res.set("Set-Cookie", `JWT=${token}`); // For good measure
 			res.status(200).send({token});
 			next();
 			return;

lib/www/server/api/middleware/user/logout.js

@@ -1,6 +1,5 @@
 
 async function logout (req, res, next) {
-	res.clearCookie("JWT");
 	res.status(204).send();
 	next();
 }

lib/www/server/api/middleware/vessel/index.js

@@ -0,0 +1,4 @@
+
+module.exports = {
+	track: require('./track'),
+}

lib/www/server/api/middleware/vessel/track/get.js

@@ -0,0 +1,10 @@
+const { gis } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+	try {
+		res.status(200).send(await gis.vesseltrack.get(req.query));
+		next();
+	} catch (err) {
+		next(err);
+	}
+}

lib/www/server/api/middleware/vessel/track/index.js

@@ -0,0 +1,4 @@
+
+module.exports = {
+	get: require('./get'),
+}

lib/www/server/events/handlers/detect-project-configuration-change.js

@@ -1,4 +1,3 @@
-const project = require('../../lib/db/project');
 const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
 class DetectProjectConfigurationChange {
@@ -10,7 +9,7 @@ class DetectProjectConfigurationChange {
 
 		// Grab project configurations.
 		// NOTE that this will run asynchronously
-		this.run({channel: "project"}, ctx);
+		//this.run({channel: "project"}, ctx);
 	}
 
 	async run (data, ctx) {
@@ -28,13 +27,13 @@ class DetectProjectConfigurationChange {
 		try {
 			DEBUG("Project configuration change detected")
 
-			const projects = await project.get();
-			project.organisations.setCache(projects);
+			const projects = await ctx.db.project.get();
+			ctx.db.project.organisations.setCache(projects);
 
 			const _ctx_data = {};
 			for (let pid of projects.map(i => i.pid)) {
 				DEBUG("Retrieving configuration for", pid);
-				const cfg = await project.configuration.get(pid);
+				const cfg = await ctx.db.project.configuration.get(pid);
 				if (cfg?.archived === true) {
 					DEBUG(pid, "is archived. Ignoring");
 					continue;

lib/www/server/events/handlers/detect-soft-start.js

@@ -1,5 +1,3 @@
-const { schema2pid } = require('../../lib/db/connection');
-const { event } = require('../../lib/db');
 const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
 class DetectSoftStart {
@@ -33,14 +31,19 @@ class DetectSoftStart {
 			const prev = this.prev?.payload?.new?.meta;
 			// DEBUG("%j", prev);
 			// DEBUG("%j", cur);
-			DEBUG("cur.num_guns: %d\ncur.num_active: %d\nprv.num_active: %d\ntest passed: %j", cur.num_guns, cur.num_active, prev.num_active, cur.num_active >= 1 && !prev.num_active && cur.num_active < cur.num_guns);
+			if (cur.lineStatus == "online" || prev.lineStatus == "online") {
+				DEBUG("lineStatus is online, assuming not in a soft start situation");
+				return;
+			}
+
+			DEBUG("cur.num_guns: %d\ncur.num_active: %d\nprv.num_active: %d\ncur.num_nofire: %d\nprev.num_nofire: %d", cur.num_guns, cur.num_active, prev.num_active, cur.num_nofire, prev.num_nofire);
 
 
 			if (cur.num_active >= 1 && !prev.num_active && cur.num_active < cur.num_guns) {
 				INFO("Soft start detected @", cur.tstamp);
 
 				// FIXME Shouldn't need to use schema2pid as pid already present in payload.
-				const projectId = await schema2pid(cur._schema ?? prev._schema);
+				const projectId = await ctx.schema2pid(cur._schema ?? prev._schema);
 
 				// TODO: Try and grab the corresponding comment from the configuration?
 				const payload = {
@@ -50,12 +53,16 @@ class DetectSoftStart {
 					meta: {auto: true, author: `*${this.constructor.name}*`}
 				};
 				DEBUG("Posting event", projectId, payload);
-				await event.post(projectId, payload);
+				if (ctx.dryRun) {
+					DEBUG(`DRY RUN: await ctx.db.event.post(${projectId}, ${payload});`);
+				} else {
+					await ctx.db.event.post(projectId, payload);
+				}
 
-			} else if (cur.num_active == cur.num_guns && prev.num_active < cur.num_active) {
+			} else if ((cur.num_active == cur.num_guns || (prev.num_nofire > 0 && cur.num_nofire == 0)) && prev.num_active < cur.num_active) {
 				INFO("Full volume detected @", cur.tstamp);
 
-				const projectId = await schema2pid(cur._schema ?? prev._schema);
+				const projectId = await ctx.schema2pid(cur._schema ?? prev._schema);
 
 				// TODO: Try and grab the corresponding comment from the configuration?
 				const payload = {
@@ -65,7 +72,11 @@ class DetectSoftStart {
 					meta: {auto: true, author: `*${this.constructor.name}*`}
 				};
 				DEBUG("Posting event", projectId, payload);
-				await event.post(projectId, payload);
+				if (ctx.dryRun) {
+					DEBUG(`DRY RUN: await ctx.db.event.post(${projectId}, ${payload});`);
+				} else {
+					await ctx.db.event.post(projectId, payload);
+				}
 			}
 
 		} catch (err) {

lib/www/server/events/handlers/detect-soleol.js

@@ -1,5 +1,3 @@
-const { schema2pid } = require('../../lib/db/connection');
-const { event } = require('../../lib/db');
 const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
 class DetectSOLEOL {
@@ -43,7 +41,7 @@ class DetectSOLEOL {
 
 				// We must use schema2pid because the pid may not have been
 				// populated for this event.
-				const projectId = await schema2pid(cur._schema ?? prev._schema);
+				const projectId = await ctx.schema2pid(cur._schema ?? prev._schema);
 				const labels = ["FSP", "FGSP"];
 				const remarks = `SEQ ${cur._sequence}, SOL ${cur.lineName}, BSP: ${(cur.speed*3.6/1.852).toFixed(1)} kt, Water depth: ${Number(cur.waterDepth).toFixed(0)} m.`;
 				const payload = {
@@ -55,24 +53,32 @@ class DetectSOLEOL {
 					meta: {auto: true, author: `*${this.constructor.name}*`}
 				}
 				INFO("Posting event", projectId, payload);
-				await event.post(projectId, payload);
+				if (ctx.dryRun) {
+					DEBUG(`DRY RUN: await ctx.db.event.post(${projectId}, ${payload});`);
+				} else {
+					await ctx.db.event.post(projectId, payload);
+				}
 			} else if (prev.lineName == cur.lineName && prev._sequence == cur._sequence &&
 				prev.lineStatus == "online" && cur.lineStatus != "online" && sequence) {
 				INFO("Transition to OFFLINE detected");
 
-				const projectId = await schema2pid(prev._schema ?? cur._schema);
+				const projectId = await ctx.schema2pid(prev._schema ?? cur._schema);
 				const labels = ["LSP", "LGSP"];
-				const remarks = `SEQ ${cur._sequence}, EOL ${cur.lineName}, BSP: ${(cur.speed*3.6/1.852).toFixed(1)} kt, Water depth: ${Number(cur.waterDepth).toFixed(0)} m.`;
+				const remarks = `SEQ ${prev._sequence}, EOL ${prev.lineName}, BSP: ${(prev.speed*3.6/1.852).toFixed(1)} kt, Water depth: ${Number(prev.waterDepth).toFixed(0)} m.`;
 				const payload = {
 					type: "sequence",
 					sequence,
-					point: cur._point,
+					point: prev._point,
 					remarks,
 					labels,
 					meta: {auto: true, author: `*${this.constructor.name}*`}
 				}
 				INFO("Posting event", projectId, payload);
-				await event.post(projectId, payload);
+				if (ctx.dryRun) {
+					DEBUG(`DRY RUN: await ctx.db.event.post(${projectId}, ${payload});`);
+				} else {
+					await ctx.db.event.post(projectId, payload);
+				}
 			}
 
 		} catch (err) {

lib/www/server/events/handlers/index.js

@@ -8,37 +8,6 @@ const Handlers = [
 	require('./detect-fdsp')
 ];
 
-function init (ctx) {
-
-	const instances = Handlers.map(Handler => new Handler(ctx));
-
-	function prepare (data, ctx) {
-		const promises = [];
-		for (let instance of instances) {
-			const promise = new Promise(async (resolve, reject) => {
-				try {
-					DEBUG("Run", instance.author);
-					const result = await instance.run(data, ctx);
-					DEBUG("%s result: %O", instance.author, result);
-					resolve(result);
-				} catch (err) {
-					ERROR("%s error:\n%O", instance.author, err);
-					reject(err);
-				}
-			});
-			promises.push(promise);
-		}
-		return promises;
-	}
-
-	function despatch (data, ctx) {
-		return Promise.allSettled(prepare(data, ctx));
-	}
-
-	return { instances, prepare, despatch };
-}
-
 module.exports = {
 	Handlers,
-	init
 };

lib/www/server/events/handlers/report-line-change-time.js

@@ -1,6 +1,3 @@
-const { event, project } = require('../../lib/db');
-const { withinValidity } = require('../../lib/utils/ranges');
-const unique = require('../../lib/utils/unique');
 const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
 class ReportLineChangeTime {
@@ -44,7 +41,7 @@ class ReportLineChangeTime {
 
 			async function getLineChangeTime (data, forward = false) {
 				if (forward) {
-					const ospEvents = await event.list(projectId, {label: "FGSP"});
+					const ospEvents = await ctx.db.event.list(projectId, {label: "FGSP"});
 					// DEBUG("ospEvents", ospEvents);
 					const osp = ospEvents.filter(i => i.tstamp > data.tstamp).pop();
 					DEBUG("fsp", osp);
@@ -55,7 +52,7 @@ class ReportLineChangeTime {
 						return { lineChangeTime: osp.tstamp - data.tstamp, osp };
 					}
 				} else {
-					const ospEvents = await event.list(projectId, {label: "LGSP"});
+					const ospEvents = await ctx.db.event.list(projectId, {label: "LGSP"});
 					// DEBUG("ospEvents", ospEvents);
 					const osp = ospEvents.filter(i => i.tstamp < data.tstamp).shift();
 					DEBUG("lsp", osp);
@@ -96,16 +93,20 @@ class ReportLineChangeTime {
 					const opts = {jpq};
 
 					if (Array.isArray(seq)) {
-						opts.sequences = unique(seq).filter(i => !!i);
+						opts.sequences = ctx.unique(seq).filter(i => !!i);
 					} else {
 						opts.sequence = seq;
 					}
 
-					const staleEvents = await event.list(projectId, opts);
+					const staleEvents = await ctx.db.event.list(projectId, opts);
 					DEBUG(staleEvents.length ?? 0, "events to delete");
 					for (let staleEvent of staleEvents) {
 						DEBUG(`Deleting event id ${staleEvent.id} (seq = ${staleEvent.sequence}, point = ${staleEvent.point})`);
-						await event.del(projectId, staleEvent.id);
+						if (ctx.dryRun) {
+							DEBUG(`await ctx.db.event.del(${projectId}, ${staleEvent.id});`);
+						} else {
+							await ctx.db.event.del(projectId, staleEvent.id);
+						}
 					}
 				}
 			}
@@ -180,7 +181,11 @@ class ReportLineChangeTime {
 
 			const maybePostEvent = async (projectId, payload) => {
 				DEBUG("Posting event", projectId, payload);
-				await event.post(projectId, payload);
+				if (ctx.dryRun) {
+					DEBUG(`await ctx.db.event.post(${projectId}, ${payload});`);
+				} else {
+					await ctx.db.event.post(projectId, payload);
+				}
 			}
 
 
@@ -192,7 +197,7 @@ class ReportLineChangeTime {
 				const data = n;
 				DEBUG("INSERT seen: will add lct events related to ", data.id);
 
-				if (withinValidity(data.validity)) {
+				if (ctx.withinValidity(data.validity)) {
 					DEBUG("Event within validity period", data.validity, new Date());
 
 					data.tstamp = new Date(data.tstamp);

lib/www/server/events/index.js

@@ -1,29 +1,101 @@
+const nodeAsync = require('async'); // npm install async
 const { listen } = require('../lib/db/notify');
+const db = require('../lib/db'); // Adjust paths; include all needed DB utils
+const { schema2pid } = require('../lib/db/connection');
+const unique = require('../lib/utils/unique'); // If needed by handlers
+const withinValidity = require('../lib/utils/ranges').withinValidity; // If needed
+const { ALERT, ERROR, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+// List of handler classes (add more as needed)
+const handlerClasses = require('./handlers').Handlers;
+
+// Channels to listen to (hardcoded for simplicity; could scan handlers for mentions)
 const channels = require('../lib/db/channels');
-const handlers = require('./handlers');
-const { ActionsQueue } = require('../lib/queue');
-const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
-function start () {
+// Queue config: Process one at a time for order; max retries=3
+const eventQueue = nodeAsync.queue(async (task, callback) => {
+	const { data, ctx } = task;
+	DEBUG(`Processing event on channel ${data.channel} with timestamp ${data._received ?? 'unknown'}`);
 
-	const queue = new ActionsQueue();
-	const ctx = {}; // Context object
+	for (const handler of ctx.handlers) {
+		try {
+			await handler.run(data, ctx);
+		} catch (err) {
+			ERROR(`Error in handler ${handler.constructor.name}:`, err);
+			// Retry logic: Could add task.retries++, re-enqueue if < max
+		}
+	}
 
-	const { prepare, despatch } = handlers.init(ctx);
+	if (typeof callback === 'function') {
+		// async v3.2.6+ does not use callsbacks with AsyncFunctions, but anyway
+		callback();
+	}
+}, 1); // Concurrency=1 for strict order
 
-	listen(channels, function (data) {
-		DEBUG("Incoming data", data);
+eventQueue.error((err, task) => {
+	ALERT(`Queue error processing task:`, err, task);
+});
 
-		// We don't bother awaiting
-		queue.enqueue(() => despatch(data, ctx));
-		DEBUG("Queue size", queue.length());
+// Main setup function (call from server init)
+async function setupEventHandlers(projectsConfig) {
+	// Shared context
+	const ctx = {
+		dryRun: Boolean(process.env.DOUGAL_HANDLERS_DRY_RUN) ?? false, // If true, don't commit changes
+		projects: { configuration: projectsConfig }, // From user config
+		handlers: handlerClasses.map(Cls => new Cls()), // Instances
+		// DB utils (add more as needed)
+		db,
+		schema2pid,
+		unique,
+		withinValidity
+		// Add other utils, e.g., ctx.logger = DEBUG;
+	};
+
+	// Optional: Replay recent events on startup to rebuild state
+	// await replayRecentEvents(ctx);
+
+	// Setup listener
+	const subscriber = await listen(channels, (rawData) => {
+		const data = {
+			...rawData,
+			enqueuedAt: new Date() // For monitoring
+		};
+		eventQueue.push({ data, ctx });
 	});
 
-	INFO("Events manager started");
+	DEBUG('Event handler system initialized with channels:', channels);
+	if (ctx.dryRun) {
+		DEBUG('DRY RUNNING');
+	}
+
+	// Return for cleanup if needed
+	return {
+		close: () => {
+			subscriber.events.removeAllListeners();
+			subscriber.close();
+			eventQueue.kill();
+		}
+	};
 }
 
-module.exports = { start }
+// Optional: Replay last N events to rebuild handler state (e.g., this.prev)
+// async function replayRecentEvents(ctx) {
+// 	try {
+// 		// Example: Fetch last 10 realtime events, sorted by tstamp
+// 		const recentRealtime = await event.listAllProjects({ channel: 'realtime', limit: 10, sort: 'tstamp DESC' });
+// 		// Assume event.listAllProjects is a custom DB method; implement if needed
+//
+// 		// Enqueue in original order (reverse sort)
+// 		recentRealtime.reverse().forEach((evt) => {
+// 			const data = { channel: 'realtime', payload: { new: evt } };
+// 			eventQueue.push({ data, ctx });
+// 		});
+//
+// 		// Similarly for 'event' channel if needed
+// 		DEBUG('Replayed recent events for state rebuild');
+// 	} catch (err) {
+// 		ERROR('Error replaying events:', err);
+// 	}
+// }
 
-if (require.main === module) {
-	start();
-}
+module.exports = { setupEventHandlers };

lib/www/server/index.js

@@ -2,18 +2,37 @@
 
 const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
+async function getProjectConfigurations (opts = {}) {
+	const includeArchived = {includeArchived: false, ...opts};
+	let projectConfigurations = {};
+	try {
+		const db = require('./lib/db');
+		const pids = (await db.project.get())
+			.filter(i => includeArchived || !i.archived)
+			.map(i => i.pid);
+		for (const pid of pids) {
+			DEBUG(`Reading project configuration for ${pid}`);
+			const cfg = await db.project.configuration.get(pid);
+			projectConfigurations[pid] = cfg;
+		}
+	} catch (err) {
+		ERROR("Failed to get project configurations");
+		ERROR(err);
+	}
+	return projectConfigurations;
+}
+
 async function main () {
 	// Check that we're running against the correct database version
 	const version = require('./lib/version');
 	INFO("Running version", await version.describe());
 	version.compatible()
-	.then( (versions) => {
+	.then( async (versions) => {
 		try {
 			const api = require('./api');
 			const ws = require('./ws');
 			const periodicTasks = require('./periodic-tasks').init();
-
-			const { fork } = require('child_process');
+			const { setupEventHandlers } = require('./events');
 
 			const port = process.env.HTTP_PORT || 3000;
 			const host = process.env.HTTP_HOST || "127.0.0.1";
@@ -25,33 +44,31 @@ async function main () {
 
 			periodicTasks.start();
 
-			const eventManagerPath = [__dirname, "events"].join("/");
-			const eventManager = fork(eventManagerPath, /*{ stdio: 'ignore' }*/);
+			const projectConfigurations = await getProjectConfigurations();
+			const handlerSystem = await setupEventHandlers(projectConfigurations);
 
 			process.on("SIGINT", async () => {
 				DEBUG("Interrupted (SIGINT)");
-				eventManager.kill()
+				handlerSystem.close();
 				await periodicTasks.cleanup();
 				process.exit(0);
 			})
 
 			process.on("SIGHUP", async () => {
 				DEBUG("Stopping (SIGHUP)");
-				eventManager.kill()
+				handlerSystem.close();
 				await periodicTasks.cleanup();
 				process.exit(0);
 			})
 
 			process.on('beforeExit', async () => {
 				DEBUG("Preparing to exit");
-				eventManager.kill()
+				handlerSystem.close();
 				await periodicTasks.cleanup();
 			});
 
 			process.on('exit', async () => {
 				DEBUG("Exiting");
-				// eventManager.kill()
-				// periodicTasks.cleanup();
 			});
 		} catch (err) {
 			ERROR(err);

lib/www/server/lib/binary/bundle.js

@@ -8,8 +8,6 @@ function bundle (json, opts = {}) {
 	const deltas = [];
 	const values = [];
 
-	// console.log("JSON LENGTH", json.length);
-	// console.log("OPTS", geometries, payload);
 
 	if (type == 0) {
 		/* Preplot information – sail line points
@@ -40,7 +38,7 @@ function bundle (json, opts = {}) {
 
 		return encode.sequential(json, el => el.sailline, el => el.point, deltas, values, type)
 
-	} if (type == 1) {
+	} else if (type == 1) {
 		/* Preplot information – source line points
 		 *
 		 * elem 0: Float32Array Longitude
@@ -74,7 +72,6 @@ function bundle (json, opts = {}) {
 			type: Uint16Array
 		});
 
-		console.log("JSON", json[0]);
 		return encode.sequential(json, el => el.line, el => el.point, deltas, values, type)
 
 	} else if (type == 2) {
@@ -222,9 +219,6 @@ function bundle (json, opts = {}) {
 			type: Uint8Array
 		});
 
-		console.log("DELTAS", deltas);
-		console.log("VALUES", values);
-
 		return encode.sequential(json, el => el.sequence, el => el.point, deltas, values, type)
 	} else if (type == 3) {
 		/* Final positions and raw vs final errors:
@@ -279,6 +273,113 @@ function bundle (json, opts = {}) {
 		});
 
 		return encode.sequential(json, el => el.sequence, el => el.point, deltas, values, type)
+	} else if (type == 4) {
+		/* Bare final positions
+		 *
+		 * Δelem    0: Sequence no. (Uint16Array, Uint8Array)
+		 * elem   0‒1: Float32Array, Float32Array – Final positions (x, y)
+		 *
+		 */
+
+		deltas.push({
+			key: el => el[2],
+			baseType: Uint16Array,
+			incrType: Int8Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[4],
+			type: Float32Array
+		});
+
+		return encode.sequential(json, el => el[0], el => el[1], deltas, values, type)
+	} else if (type == 0xa) {
+		/* 4D comparison data:
+		 *
+		 * elem0: i differences
+		 * elem1: j differences
+		 *
+		 * Note that line/point may not be unique.
+		 *
+		 */
+
+		/*
+		deltas.push({
+			key: el => el.baseTStamp,
+			baseType: BigUint64Array,
+			incrType: Int32Array
+		});
+
+		deltas.push({
+			key: el => el.monTStamp,
+			baseType: BigUint64Array,
+			incrType: Int32Array
+		})
+		*/
+
+		values.push({
+			key: el => el[2],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		/*
+		values.push({
+			key: el => el.baseSeq,
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el.monSeq,
+			type: Uint16Array
+		});
+		*/
+
+		return encode.sequential(json, el => el[0], el => el[1], deltas, values, type)
+	} else if (type == 0xc) {
+		/* 4D comparison data (reduced sample)
+		 *
+		 * Input is comparison records, i.e.:
+		 * [ [ line, point, δi, δj ], … ]
+		 *
+		 * elem0: line
+		 * elem1: point
+		 * elem2: δi
+		 * elem3: δj
+		 *
+		 * Note that the chunk's `i` and `j` values are not used
+		 */
+
+		values.push({
+			key: el => el[0],
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el[1],
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el[2],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		return encode.sequential(json, el => 0, el => 0, deltas, values, type)
 	}
 }
 

lib/www/server/lib/comparisons/geometric-differences.js

@@ -0,0 +1,429 @@
+const d3a = require('d3-array');
+const { DougalBinaryBundle } = require('@dougal/binary');
+const { pool, setSurvey } = require('../db/connection');
+const db = require('../db');
+const { bundle } = require('../binary/bundle');
+const setops = require('../utils/setops');
+const { ijRMS, combinations, computeSample } = require('./utils');
+const { computePCA } = require('./pca');
+const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+
+async function fetchErrors (pid) {
+
+	const client = await setSurvey(pid);
+
+	try {
+		const text = `
+			SELECT
+				fs.line, fs.point,
+				ij_error(fs.line::double precision, fs.point::double precision, fs.geometry)::json AS errorfinal
+			FROM
+				final_shots fs
+			ORDER BY fs.line, fs.point;
+		`;
+
+		const res = await client.query(text);
+
+		return res.rows.map( row =>
+			[row.line, row.point, row.errorfinal.coordinates[0], row.errorfinal.coordinates[1]]
+		);
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function groupTimestamps (groupName) {
+	const projects = await groups()?.[groupName];
+	if (projects?.length) {
+
+	}
+}
+
+
+async function groups () {
+	const projects = await db.project.get({timestamps: true});
+	const groupNames = [
+		...projects
+			.reduce( (acc, cur) => acc.add(...cur.groups), new Set() )
+	].filter( i => !!i );
+
+	return Object.fromEntries(groupNames.map( g => [g, projects.filter( p => p.groups.includes(g) )] ));
+}
+
+function geometric_differences (baseline, monitor) {
+
+	if (!baseline || !baseline.length) {
+		throw new Error("No baseline data");
+	}
+
+	if (!monitor || !monitor.length) {
+		throw new Error("No monitor data");
+	}
+
+	const comparison = []; // An array of { line, point, εi, εj }; line + point may be repeated
+
+	for (const bp of baseline) {
+		const monitor_points = monitor.filter( mp => mp[0] === bp[0] && mp[1] === bp[1] );
+
+		if (!monitor_points.length) {
+			// console.log(`No match for L${bp[0]} P${bp[1]}`);
+			continue;
+		}
+
+		for (const mp of monitor_points) {
+			const εi = mp[2] - bp[2], εj = mp[3] - bp[3];
+			comparison.push([bp[0], bp[1], εi, εj]);
+		}
+
+	}
+
+	return comparison;
+}
+
+async function compare (baselineProjectID, monitorProjectID, infoObj) {
+	console.log("Getting baseline", baselineProjectID);
+	const baselineData = await fetchErrors(baselineProjectID);
+	console.log("Getting monitor", monitorProjectID);
+	const monitorData = await fetchErrors(monitorProjectID);
+	console.log("Comparing");
+
+	const comparison = geometric_differences(baselineData, monitorData);
+
+	if (infoObj instanceof Object) {
+		const baselineIJ = baselineData.map(i => i.slice(0,2));
+		const monitorIJ = monitorData.map(i => i.slice(0,2));
+
+		infoObj.compared = comparison.length;
+		infoObj.baselineLength = baselineData.length;
+		infoObj.monitorLength = monitorData.length;
+		infoObj.baselineUniqueLength = setops.unique(baselineIJ).length;
+		infoObj.monitorUniqueLength = setops.unique(monitorIJ).length;
+		infoObj.common = setops.intersection(baselineIJ, monitorIJ).length;
+	}
+
+	return comparison;
+}
+
+
+async function save (baselineProjectID, monitorProjectID, bundle, meta) {
+	const info = {};
+	if (!bundle) {
+		const comparison = await compare(baselineProjectID, monitorProjectID, info);
+		if (comparison.length) {
+			bundle = asBundle(comparison);
+		} else {
+			console.warn(`No matching points between ${baselineProjectID} and ${monitorProjectID}`);
+			return;
+		}
+	} else if (!(bundle instanceof DougalBinaryBundle)) {
+		throw new Error("Illegal data: `bundle` must of null or of type DougalBinaryBundle");
+	}
+
+	if (!bundle.byteLength) {
+		console.warn(`Empty comparison results between ${baselineProjectID} and ${monitorProjectID}. Refusing to store`);
+		return;
+	}
+
+	meta = {tstamp: (new Date()), ...info, ...stats(bundle), ...meta};
+
+	console.log("Storing in database");
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			INSERT INTO comparisons.comparisons
+				(type, baseline_pid, monitor_pid, data, meta)
+			VALUES ('geometric_difference', $1, $2, $3, $4)
+			ON CONFLICT (type, baseline_pid, monitor_pid)
+			DO UPDATE SET
+				data = EXCLUDED.data,
+				meta = EXCLUDED.meta;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, Buffer.from(bundle), meta ];
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function get (baselineProjectID, monitorProjectID, type = 'geometric_difference') {
+
+	const client = await pool.connect();
+
+	try {
+
+		const text = `
+			SELECT data, meta
+			FROM comparisons.comparisons
+			WHERE type = $3 AND baseline_pid = $1 AND monitor_pid = $2;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, type ];
+		const res = await client.query(text, values);
+		if (!res.rows.length) {
+			console.log("Comparison not found in database");
+			return;
+		}
+
+		const { data, meta } = res.rows[0];
+		return {
+			data: DougalBinaryBundle.clone(data),
+			meta
+		};
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function getSample (baselineProjectID, monitorProjectID) {
+	return await get(baselineProjectID, monitorProjectID, 'geometric_difference_sample');
+}
+
+
+async function remove (baselineProjectID, monitorProjectID) {
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			DELETE
+			FROM comparisons.comparisons
+			WHERE
+				(type = 'geometric_difference' OR type = 'geometric_difference_sample')
+				AND baseline_pid = $1
+				AND monitor_pid = $2;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID ];
+
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+function stats (comparison) {
+	let i, j, δi, δj;
+
+	if (comparison instanceof DougalBinaryBundle) {
+		console.log("Computing stats");
+		const udv = comparison.chunks()[0]?.udv;
+
+		if (!udv) {
+			console.error("Could not determine udv from first chunk");
+			console.log(comparison.chunks());
+			return;
+		}
+
+		let records;
+
+		if (udv == 0xa) {
+			records = comparison.records;
+
+			// Transpose the records
+			[ i, j, δi, δj ] = Array.from({ length: 4 }, (_, i) => records.map(row => row[i]));
+		} else if (udv == 0xc) {
+			records = comparison.records;
+			let _;
+			[ _, _, i, j, δi, δj ] = Array.from({ length: 6 }, (_, i) => records.map(row => row[i]));
+		} else {
+			throw new Error(`Unrecognised DougalBinaryBundle User Defined Value: ${udv}`);
+		}
+
+		return {
+			length: records.length,
+			μ: [ d3a.mean(δi), d3a.mean(δj) ],
+			σ: [ d3a.deviation(δi), d3a.deviation(δj) ],
+			//rms: ijRMS(δi, δj),
+			...computePCA(records)
+		}
+	} else if (Array.isArray(comparison)) {
+		if (Array.isArray(comparison[0])) {
+			return stats(asBundle(comparison, {type: 0xc}));
+		} else {
+			// Assume object
+			return stats(asBundle(comparison));
+		}
+	}
+}
+
+
+function sortFn (a, b) {
+	if (a.line == b.line) {
+		if (a.point == b.point) {
+			return a.baseTStamp - b.baseTStamp;
+		} else {
+			return a.point - b.point;
+		}
+	} else {
+		return a.line - b.line;
+	}
+}
+
+function asBundle (comparison, opts = {type: 0x0a}) {
+	return DougalBinaryBundle.clone(bundle(comparison, opts));
+}
+
+function fromBundle (bundle) {
+	if (!(bundle instanceof DougalBinaryBundle)) {
+		bundle = DougalBinaryBundle.clone(bundle);
+	}
+
+	const json = [];
+	for (const record of bundle) {
+		record.shift();
+		json.push(record);
+	}
+	return json;
+}
+
+async function saveGroup (group, opts = {}) {
+	if (group == null) {
+		// Save everything
+		const g = await groups();
+		for (const group of Object.values(g)) {
+			await saveGroup(group)
+		}
+	} if (typeof group === "string") {
+		// This is a group name
+		const g = await groups();
+		group = groups[g];
+	}
+
+	if (Array.isArray(group)) {
+		const pids = group.map( i => i.pid ).sort();
+
+		for (const [ baselineProjectID, monitorProjectID ] of combinations(pids, 2)) {
+			try {
+				if (!opts.overwrite) {
+					const exists = await get(baselineProjectID, monitorProjectID);
+					if (exists) {
+						DEBUG("Not overwriting existing comparison between %s and %s. Skipping", baselineProjectID, monitorProjectID);
+						continue;
+					}
+				}
+
+				await save(baselineProjectID, monitorProjectID);
+				DEBUG("Saved comparison between %s and %s", baselineProjectID, monitorProjectID);
+			} catch (err) {
+				console.error(err);
+				ERROR("Error saving comparison between %s and %s", baselineProjectID, monitorProjectID);
+			}
+		}
+	}
+}
+
+
+async function getGroup (groupName, opts = {}) {
+
+	const client = await pool.connect();
+
+	try {
+
+		if (groupName) {
+
+			const group = (await groups())?.[groupName]?.map( i => i.pid)?.sort();
+
+			if (!group?.length || group?.length < 2) return;
+
+
+			const pairs = combinations(group, 2);
+			const flatValues = pairs.flat();
+			const placeholders = [];
+			for (let i = 0; i < pairs.length; i++) {
+				placeholders.push(`($${i * 2 + 1}, $${i * 2 + 2})`);
+			}
+			const inClause = placeholders.join(',');
+			const selectFields = opts.returnData ? 'data, meta' : 'meta';
+
+			const text = `
+				SELECT baseline_pid, monitor_pid, ${selectFields}
+				FROM comparisons.comparisons
+				WHERE type = 'geometric_difference'
+				AND (baseline_pid, monitor_pid) IN (VALUES ${inClause})
+				ORDER BY baseline_pid, monitor_pid
+			`;
+
+			if (!placeholders) {
+				console.log("No pairs found in group");
+				return [];
+			}
+
+			const res = await client.query(text, flatValues);
+			if (!res.rows.length) {
+				console.log("Comparison not found in database");
+				return;
+			}
+
+			if (opts.returnData) {
+				return res.rows.map( row => ({
+					...row,
+					data: DougalBinaryBundle.clone(row.data),
+				}));
+			} else {
+				return res.rows;
+			}
+
+		} else {
+
+			const selectFields = opts.returnData ? 'data, meta' : 'meta';
+
+			const text = `
+				SELECT baseline_pid, monitor_pid, ${selectFields}
+				FROM comparisons.comparisons
+				WHERE type = 'geometric_difference'
+				ORDER BY baseline_pid, monitor_pid
+			`;
+
+			const res = await client.query(text);
+			if (!res.rows.length) {
+				console.log("Comparison not found in database");
+				return;
+			}
+
+			if (opts.returnData) {
+				return res.rows.map( row => ({
+					...row,
+					data: DougalBinaryBundle.clone(row.data),
+				}));
+			} else {
+				return res.rows;
+			}
+
+		}
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+module.exports = {
+	groups,
+	fetchErrors,
+	compare,
+	computeSample,
+	get,
+	save,
+	getSample,
+	saveGroup,
+	getGroup,
+	remove,
+	stats,
+	asBundle,
+	fromBundle
+};

lib/www/server/lib/comparisons/index.js

@@ -0,0 +1,4 @@
+
+module.exports = {
+	...require('./geometric-differences')
+}

lib/www/server/lib/comparisons/pca.js

@@ -0,0 +1,83 @@
+const math = require('mathjs');
+
+/**
+ * Compute PCA (eigenvectors and eigenvalues) for deviation data to assess geometric repeatability.
+ * @param {Array<Array<number>>} deviations - Array of [point, line, i_deviation, j_deviation]
+ * @returns {Object} - { eigenvalues, eigenvectors, rms, anisotropy, primaryDirection }
+ */
+function computePCA(deviations) {
+	// Extract i_deviation and j_deviation
+	const deviationMatrix = deviations.map(row => [row[2], row[3]]);
+
+	// Convert to mathjs matrix
+	const D = math.matrix(deviationMatrix);
+
+	// Compute mean for centering (1 x 2 matrix)
+	const mean = math.mean(D, 0);
+
+	// Manually repeat-mean to match D's shape (n x 2)
+	const n = deviationMatrix.length;
+	const meanArr = mean.toArray();
+	const meanRepeated = math.matrix(
+		Array(n).fill().map(() => [meanArr[0], meanArr[1]])
+	);
+
+	// Center the data
+	const centered = math.subtract(D, meanRepeated);
+
+	// Compute covariance matrix: (1/(n-1)) * (D_centered^T * D_centered)
+	const covMatrix = math.multiply(
+		math.multiply(1 / (n - 1), math.transpose(centered)),
+		centered
+	);
+
+	// Perform eigen decomposition
+	const result = math.eigs(covMatrix);
+	let eigenvalues = result.values;
+	const evObjs = result.eigenvectors;
+
+	// Convert eigenvalues to array if it's a matrix
+	eigenvalues = Array.isArray(eigenvalues) ? eigenvalues : eigenvalues.toArray();
+
+	// Create pairs and convert vector to array if necessary
+	const pairs = eigenvalues.map((val, i) => {
+		let vec = evObjs[i].vector;
+		if (vec.toArray) vec = vec.toArray();
+		return { val, vec };
+	});
+
+	// Sort by descending eigenvalues
+	pairs.sort((a, b) => b.val - a.val);
+
+	// Sorted eigenvalues
+	const sortedEigenvalues = pairs.map(p => p.val);
+
+	// Build eigenvector matrix: rows as components, columns as eigenvectors
+	const dimension = pairs[0].vec.length; // e.g., 2
+	const evecRows = [];
+	for (let comp = 0; comp < dimension; comp++) {
+		evecRows.push(pairs.map(p => p.vec[comp]));
+	}
+	const sortedEigenvectors = math.matrix(evecRows);
+
+	// Compute RMS errors along principal axes
+	const rms = sortedEigenvalues.map(val => Math.sqrt(Math.max(val, 0)));
+
+	// Compute anisotropy (ratio of major to minor axis variance)
+	const anisotropy = sortedEigenvalues[0] / (sortedEigenvalues[1] || 1); // Avoid division by zero
+
+	// Primary direction (angle in degrees of major eigenvector)
+	const primaryVector = sortedEigenvectors.subset(math.index([0, 1], 0)).toArray();
+	const primaryDirection = Math.atan2(primaryVector[1], primaryVector[0]) * 180 / Math.PI;
+
+	return {
+		eigenvalues: sortedEigenvalues,
+		eigenvectors: sortedEigenvectors.toArray(),
+		rms: rms, // RMS errors along major/minor axes
+		anisotropy: anisotropy, // Ratio of variances
+		primaryDirection: primaryDirection // Angle of major axis (degrees)
+	};
+}
+
+
+module.exports = { computePCA };

lib/www/server/lib/comparisons/utils.js

@@ -0,0 +1,310 @@
+const d3 = require('d3-array');
+
+// Function to calculate the root mean square (RMS) of position deviations
+// This computes the RMS of the Euclidean distances: sqrt( (1/n) * sum(δi² + δj²) )
+// Assumes deviations are already centered (mean deviation ~0); if normalization by std dev or range is needed, adjust accordingly
+function ijRMS(δi, δj) {
+	if (!δi.length || !δj.length) return 0;
+
+	if (δi.length != δj.length) {
+		console.warn(`δi and δj have different lengths!`);
+	}
+
+	let sumSquares = 0;
+	const n = Math.min(δi.length, δj.length);
+
+	for (let i=0; i < n; i++) {
+		sumSquares += (δi[i] * δi[i]) + (δj[i] * δj[i]);
+	}
+
+	const meanSquare = sumSquares / n;
+	const rms = Math.sqrt(meanSquare);
+
+	return rms;
+}
+
+
+/**
+ * Performs stratified sampling on an array of [line, point, δi, δj] data points.
+ * Groups by line and samples proportionally to preserve shape and spread.
+ *
+ * @param {Array<Array<number>>} data - Input data: [[line, point, δi, δj], ...]
+ * @param {number} sampleSize - Target number of samples (e.g., 2000)
+ * @returns {Array<Array<number>>} Sampled data in same format
+ */
+function old_stratifiedSample(data, sampleSize) {
+	if (!Array.isArray(data) || data.length === 0) return [];
+	if (!Number.isInteger(sampleSize) || sampleSize <= 0) {
+		throw new Error('sampleSize must be a positive integer');
+	}
+
+	// Group data by line (first element)
+	const grouped = d3.group(data, d => d[0]);
+	const totalSize = data.length;
+	const sampled = [];
+
+	// Ensure sampleSize doesn't exceed data size
+	const effectiveSampleSize = Math.min(sampleSize, totalSize);
+
+	// Iterate over each line group
+	for (const [line, group] of grouped) {
+		// Calculate proportional sample size for this group
+		const groupSize = group.length;
+		const groupSampleSize = Math.max(1, Math.round((groupSize / totalSize) * effectiveSampleSize));
+
+		// Shuffle group and take first N elements
+		const shuffled = d3.shuffle([...group]);
+		sampled.push(...shuffled.slice(0, groupSampleSize));
+	}
+
+	// If sampled size is slightly off due to rounding, adjust
+	if (sampled.length > effectiveSampleSize) {
+		return d3.shuffle(sampled).slice(0, effectiveSampleSize);
+	} else if (sampled.length < effectiveSampleSize) {
+		// Pad with random samples from entire dataset if needed
+		const remaining = effectiveSampleSize - sampled.length;
+		const additional = d3.shuffle(data.filter(d => !sampled.includes(d))).slice(0, remaining);
+		sampled.push(...additional);
+	}
+
+	return sampled;
+}
+
+/**
+ * Performs stratified sampling on an array of [line, point, δi, δj] data points.
+ * Stratifies by line and δi quantiles to preserve shape and spread, with outlier control.
+ *
+ * @param {Array<Array<number>>} data - Input data: [[line, point, δi, δj], ...]
+ * @param {number} sampleSize - Target number of samples (e.g., 2000)
+ * @param {number} [binsPerLine=10] - Number of δi quantile bins per line
+ * @returns {Array<Array<number>>} Sampled data in same format
+ */
+function stratifiedSample(data, sampleSize, binsPerLine = 10) {
+	if (!Array.isArray(data) || data.length === 0) return [];
+	if (!Number.isInteger(sampleSize) || sampleSize <= 0) {
+		throw new Error('sampleSize must be a positive integer');
+	}
+	if (!Number.isInteger(binsPerLine) || binsPerLine <= 0) {
+		throw new Error('binsPerLine must be a positive integer');
+	}
+
+	const totalSize = data.length;
+	const effectiveSampleSize = Math.min(sampleSize, totalSize);
+	const sampled = [];
+
+	// Group by line
+	const groupedByLine = d3.group(data, d => d[0]);
+
+	// Compute population stats for validation
+	const populationStats = computeStats(data);
+
+	// Iterate over each line
+	for (const [line, group] of groupedByLine) {
+		const groupSize = group.length;
+		const lineSampleSize = Math.max(1, Math.round((groupSize / totalSize) * effectiveSampleSize));
+
+		// Create quantile-based bins for δi
+		const δiValues = group.map(d => d[2]).sort(d3.ascending);
+		const quantiles = d3.range(0, binsPerLine + 1).map(i => d3.quantile(δiValues, i / binsPerLine));
+		const binnedData = group.map(d => {
+			const δi = d[2];
+			let binIndex = 0;
+			for (let i = 0; i < binsPerLine; i++) {
+				if (δi >= quantiles[i] && δi < quantiles[i + 1]) {
+					binIndex = i;
+					break;
+				}
+			}
+			return { data: d, bin: binIndex };
+		});
+		const groupedByBin = d3.group(binnedData, d => d.bin);
+
+		// Allocate samples across bins, inversely weighted by density to control outliers
+		const binSampleSizes = new Map();
+		let remainingLineSamples = lineSampleSize;
+		const binCounts = Array(binsPerLine).fill(0);
+		for (const [bin, binGroup] of groupedByBin) {
+			binCounts[bin] = binGroup.length;
+		}
+		const maxBinCount = d3.max(binCounts);
+		for (const [bin, binGroup] of groupedByBin) {
+			const binSize = binGroup.length;
+			// Inverse weighting: smaller bins (outliers) get fewer samples
+			const weight = binSize > 0 ? Math.max(0.1, 1 - (binSize / maxBinCount) * 0.5) : 1;
+			const binSampleSize = Math.max(1, Math.round(lineSampleSize * (binSize / groupSize) * weight));
+			binSampleSizes.set(bin, Math.min(binSampleSize, binSize));
+			remainingLineSamples -= binSampleSizes.get(bin);
+		}
+
+		// Distribute remaining samples
+		if (remainingLineSamples > 0) {
+			const nonEmptyBins = Array.from(groupedByBin.keys());
+			for (let i = 0; i < remainingLineSamples && nonEmptyBins.length > 0; i++) {
+				const bin = nonEmptyBins[i % nonEmptyBins.length];
+				binSampleSizes.set(bin, binSampleSizes.get(bin) + 1);
+			}
+		}
+
+		// Sample from each bin
+		for (const [bin, binGroup] of groupedByBin) {
+			const samples = d3.shuffle([...binGroup]).slice(0, binSampleSizes.get(bin)).map(s => s.data);
+			sampled.push(...samples);
+		}
+	}
+
+	// Adjust sample size
+	let finalSample = sampled;
+	if (sampled.length > effectiveSampleSize) {
+		finalSample = d3.shuffle(sampled).slice(0, effectiveSampleSize);
+	} else if (sampled.length < effectiveSampleSize) {
+		const remaining = effectiveSampleSize - sampled.length;
+		const additional = d3.shuffle(data.filter(d => !sampled.includes(d))).slice(0, remaining);
+		finalSample = [...sampled, ...additional];
+	}
+
+	// Validate and adjust if stats are off
+	const sampleStats = computeStats(finalSample);
+	const statTolerance = { μ: 0.1, σ: 0.2 }; // Allowable relative deviation
+	const needsAdjustment =
+		Math.abs(sampleStats.μ[0] - populationStats.μ[0]) / populationStats.μ[0] > statTolerance.μ ||
+		Math.abs(sampleStats.μ[1] - populationStats.μ[1]) / populationStats.μ[1] > statTolerance.μ ||
+		Math.abs(sampleStats.σ[0] - populationStats.σ[0]) / populationStats.σ[0] > statTolerance.σ ||
+		Math.abs(sampleStats.σ[1] - populationStats.σ[1]) / populationStats.σ[1] > statTolerance.σ;
+
+	if (needsAdjustment) {
+		// Add points from underrepresented regions
+		const δiSample = finalSample.map(d => d[2]);
+		const δiPopulation = data.map(d => d[2]);
+		const quantiles = d3.range(0, binsPerLine + 1).map(i => d3.quantile(δiPopulation, i / binsPerLine));
+		const sampleBins = d3.histogram().domain(d3.extent(δiPopulation)).thresholds(quantiles)(δiSample);
+		const populationBins = d3.histogram().domain(d3.extent(δiPopulation)).thresholds(quantiles)(δiPopulation);
+		const underSampledBins = sampleBins
+			.map((b, i) => ({ bin: i, diff: populationBins[i].length / totalSize - b.length / finalSample.length }))
+			.filter(b => b.diff > 0.1); // Significant under-sampling
+
+		if (underSampledBins.length > 0) {
+			const additionalSamples = [];
+			for (const { bin } of underSampledBins) {
+				const binData = data.filter(d => d[2] >= quantiles[bin] && d[2] < quantiles[bin + 1] && !finalSample.includes(d));
+				const needed = Math.round((underSampledBins[0].diff * effectiveSampleSize) / 2);
+				additionalSamples.push(...d3.shuffle(binData).slice(0, needed));
+			}
+			finalSample = d3.shuffle([...finalSample, ...additionalSamples]).slice(0, effectiveSampleSize);
+		}
+	}
+
+	return finalSample;
+}
+
+function decimate (data, decimationCount = 20) {
+	return data.filter( (row, index) => (index % decimationCount) == 0 );
+}
+
+function computeSample (data, opts = {}) {
+	const DEFAULT_SAMPLE_SIZE = 2000;
+	let sample;
+
+	if (opts.decimate === true) {
+		if (opts.sampleSize > 0) {
+			sample = decimate(data.records, Math.floor(data.records.length / opts.sampleSize));
+		} else {
+			sample = decimate(data.records, Math.floor(data.records.length / DEFAULT_SAMPLE_SIZE));
+		}
+	} else if (opts.decimate > 0) {
+		sample = decimate(data.records, opts.decimate);
+	} else if (opts.sampleSize) {
+		sample = stratifiedSample(data.records, opt.sampleSize);
+	} else {
+		sample = stratifiedSample(data.records, DEFAULT_SAMPLE_SIZE);
+	}
+
+	return sample;
+}
+
+// Optional: Utility to compute stats for validation
+function computeStats(data) {
+	const δi = data.map(d => d[2]);
+	const δj = data.map(d => d[3]);
+	const rms = Math.sqrt(d3.mean(data, d => d[2] ** 2 + d[3] ** 2));
+	return {
+		l: data.length,
+		μ: [d3.mean(δi), d3.mean(δj)],
+		σ: [d3.deviation(δi), d3.deviation(δj)],
+		rms
+	};
+}
+
+function centre (data) {
+	const stats = computeStats(data);
+
+	return data.map( row => [row[0], row[1], row[2]-stats.μ[0], row[3]-stats.μ[1]] )
+}
+
+function outliers (data, sd=1.96) {
+	const stats = computeStats(data);
+
+	function fn ([l, p, i, j]) {
+		return (i - stats.μ[0]) > stats.σ[0]*sd ||
+			(j - stats.μ[1]) > stats.σ[1]*sd;
+	}
+
+	return data.filter(fn)
+}
+
+function inliers (data, sd=1.96) {
+	const stats = computeStats(data);
+
+	function fn ([l, p, i, j]) {
+		return (i - stats.μ[0]) <= stats.σ[0]*sd &&
+			(j - stats.μ[1]) <= stats.σ[1]*sd;
+	}
+
+	return data.filter(fn)
+}
+
+function difference (a, b) {
+	const obj = Array.isArray(a) ? [] : {};
+	for (const k in a) {
+		const v0 = a[k];
+		const v1 = b[k]
+		if (v0 instanceof Object && v1 instanceof Object) {
+			obj[k] = difference (v0, v1);
+		} else if (!isNaN(Number(v0)) && !isNaN(Number(v1))) {
+			obj[k] = v1 - v0;
+		}
+	}
+	return obj;
+}
+
+function combinations (a, n) {
+	const results = [];
+
+	function combine(current, start) {
+		if (current.length === n) {
+			results.push([...current]);
+			return;
+		}
+		for (let i = start; i < a.length; i++) {
+			current.push(a[i]);
+			combine(current, i + 1);
+			current.pop();
+		}
+	}
+
+	combine([], 0);
+	return results;
+}
+
+module.exports = {
+	combinations,
+	centre,
+	ijRMS,
+	computeStats,
+	computeSample,
+	stratifiedSample,
+	old_stratifiedSample,
+	decimate,
+	difference,
+	outliers,
+	inliers
+}

lib/www/server/lib/db/channels.js

@@ -10,5 +10,6 @@ module.exports = [
 	"planned_lines",
 	"raw_lines", "raw_shots",
 	"final_lines", "final_shots", "info",
-	"queue_items"
+	"queue_items",
+	"comparisons",
 ];

lib/www/server/lib/db/event/import.js

@@ -0,0 +1,105 @@
+const { DEBUG, ERROR } = require('DOUGAL_ROOT/debug')(__filename);
+const { setSurvey, transaction } = require('../connection');
+
+
+/** Remove a previous import from the database.
+ *
+ * ATTENTION!
+ *
+ * This will not just mark the events as deleted but actually
+ * remove them.
+ */
+async function bulk_unimport (projectId, filename, opts = {}) {
+
+	const client = opts.client ?? await setSurvey(projectId);
+	try {
+
+		const text = `
+			DELETE
+			FROM event_log
+			WHERE meta ? 'author'
+				AND meta->(meta->>'author')->>'filename' = $1;
+		`;
+		const values = [ filename ];
+
+		DEBUG("Removing all event data imported from filename '%s'", filename);
+		await client.query(text, values);
+	} catch (err) {
+		err.origin = __filename;
+		throw err;
+	} finally {
+		if (client !== opts.client) client.release();
+	}
+
+	return;
+}
+
+
+async function bulk_import (projectId, payload, opts = {}) {
+
+	const client = opts.client ?? await setSurvey(projectId);
+	try {
+
+		if (!payload.length) {
+			DEBUG("Called with no rows to be imported. Returning");
+			return [];
+		}
+
+		const filename = payload[0].meta[payload[0].meta.author].filename;
+
+		// Delete previous data from this file
+		await transaction.begin(client);
+		await bulk_unimport(projectId, filename, {client});
+
+
+		// Prepare arrays for each column
+		const tstamps = [];
+		const sequences = [];
+		const points = [];
+		const remarks = [];
+		const labels = [];
+		const metas = [];
+
+		for (const event of payload) {
+			tstamps.push(event.tstamp ? new Date(event.tstamp) : null);
+			sequences.push(Number.isInteger(event.sequence) ? event.sequence : null);
+			points.push(Number.isInteger(event.point) ? event.point : null);
+			remarks.push(event.remarks || '');
+			labels.push(Array.isArray(event.labels) && event.labels.length
+				? `{${event.labels.map(l => `"${l.replace(/"/g, '""')}"`).join(',')}}`
+				: '{}'
+			);
+			metas.push(event.meta ? JSON.stringify(event.meta) : '{}');
+		}
+
+
+		const text = `
+			INSERT INTO event_log (tstamp, sequence, point, remarks, labels, meta)
+				SELECT
+					UNNEST($1::TIMESTAMP[]) AS tstamp,
+					UNNEST($2::INTEGER[]) AS sequence,
+					UNNEST($3::INTEGER[]) AS point,
+					replace_placeholders(UNNEST($4::TEXT[]), UNNEST($1::TIMESTAMP[]), UNNEST($2::INTEGER[]), UNNEST($3::INTEGER[])) AS remarks,
+					UNNEST($5::TEXT[])::TEXT[] AS labels,
+					UNNEST($6::JSONB[]) AS meta
+				RETURNING id;
+		`;
+		const values = [ tstamps, sequences, points, remarks, labels, metas ];
+
+		DEBUG("Importing %d rows from filename '%s'", payload.length, filename);
+		const res = await client.query(text, values);
+
+		transaction.commit(client);
+
+		return res.rows.map(row => row.id);
+	} catch (err) {
+		err.origin = __filename;
+		throw err;
+	} finally {
+		if (client !== opts.client) client.release();
+	}
+
+	return;
+}
+
+module.exports = { import: bulk_import, unimport: bulk_unimport };

lib/www/server/lib/db/event/index.js

@@ -6,5 +6,7 @@ module.exports = {
 	put: require('./put'),
 	patch: require('./patch'),
 	del: require('./delete'),
-	changes: require('./changes')
+	changes: require('./changes'),
+	import: require('./import').import,
+	unimport: require('./import').unimport,
 }

lib/www/server/lib/db/event/unimport.js

@@ -0,0 +1,37 @@
+
+const { DEBUG, ERROR } = require('DOUGAL_ROOT/debug')(__filename);
+const { setSurvey, transaction } = require('../connection');
+
+/** Remove a previous import from the database.
+ *
+ * ATTENTION!
+ *
+ * This will not just mark the events as deleted but actually
+ * remove them.
+ */
+async function unimport (projectId, filename, opts = {}) {
+
+	const client = await setSurvey(projectId);
+	try {
+
+		const text = `
+			DELETE
+			FROM event_log
+			WHERE meta ? 'author'
+				AND meta->(meta->'author')->>'filename' = $1;
+		`;
+		const values = [ filename ];
+
+		DEBUG("Removing all event data imported from filename '%s'", filename);
+		await client.query(text, values);
+	} catch (err) {
+		err.origin = __filename;
+		throw err;
+	} finally {
+		client.release();
+	}
+
+	return;
+}
+
+module.exports = post;

lib/www/server/lib/db/gis/index.js

@@ -1,6 +1,7 @@
 
 module.exports = {
 	project: require('./project'),
-	navdata: require('./navdata')
+	navdata: require('./navdata'),
+	vesseltrack: require('./vesseltrack'),
 // 	line: require('./line')
 };

lib/www/server/lib/db/gis/navdata/get.js

@@ -7,7 +7,10 @@ async function lines (options = {}) {
 	const client = await pool.connect();
 
 	const text = `
-		SELECT ST_AsGeoJSON(ST_MakeLine(geometry)) geojson
+		SELECT json_build_object(
+			'type', 'Feature',
+			'geometry', ST_AsGeoJSON(ST_MakeLine(geometry))::json
+		) geojson
 		FROM (
 			SELECT geometry
 			FROM real_time_inputs

lib/www/server/lib/db/gis/vesseltrack/get.js

@@ -0,0 +1,191 @@
+const { pool } = require('../../connection');
+const { project } = require('../../utils');
+
+async function get(options = {}) {
+	/*
+	 * ts0: earliest timestamp (default: NOW - 7 days)
+	 * ts1: latest timestamp (if null, assume NOW)
+	 * di: decimation interval (return every di-th record, if null: no decimation)
+	 * l: limit (return no more than l records, default: 1000, max: 1,000,000)
+	 * geojson: 'LineString' (GeoJSON LineString Feature), 'Point' (GeoJSON FeatureCollection),
+	 *          truthy (array of Point features), falsy (array of {x, y, tstamp, meta})
+	 */
+	let { l, di, ts1, ts0, geojson, projection } = {
+		l: 1000,
+		di: null,
+		ts1: null,
+		ts0: null,
+		geojson: false,
+		...options
+	};
+
+	// Input validation and sanitization
+	l = Math.max(1, Math.min(parseInt(l) || 1000, 1000000)); // Enforce 1 <= l <= 1,000,000
+	di = di != null ? Math.max(1, parseInt(di)) : null; // Ensure di is positive integer or null
+	ts0 = ts0 ? new Date(ts0).toISOString() : new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString(); // Default: 7 days ago
+	ts1 = ts1 ? new Date(ts1).toISOString() : null; // Convert to ISO string or null
+	geojson = geojson === 'LineString' || geojson === 'Point' ? geojson : !!geojson; // Normalize geojson
+
+	const client = await pool.connect();
+
+	// Build the WHERE clause and values array dynamically
+	let whereClauses = [];
+	let values = [];
+	let paramIndex = 1;
+
+	if (ts0) {
+		whereClauses.push(`tstamp >= $${paramIndex++}`);
+		values.push(ts0);
+	}
+	if (ts1) {
+		whereClauses.push(`tstamp <= $${paramIndex++}`);
+		values.push(ts1);
+	}
+
+	// Add limit to values
+	values.push(l);
+	const limitClause = `LIMIT $${paramIndex++}`;
+
+	// Base query with conditional geometry selection
+	let queryText = `
+		SELECT
+			tstamp,
+			CASE
+				WHEN meta->>'lineStatus' = 'offline' THEN
+					ARRAY[COALESCE((meta->>'longitude')::float, ST_X(geometry)), COALESCE((meta->>'latitude')::float, ST_Y(geometry))]
+				ELSE
+					ARRAY[
+						COALESCE(
+							(meta->>'longitudeMaster')::float,
+							ST_X(geometry)
+						),
+						COALESCE(
+							(meta->>'latitudeMaster')::float,
+							ST_Y(geometry)
+						)
+					]
+			END AS coordinates,
+			meta::json AS meta
+		FROM public.real_time_inputs
+		${whereClauses.length ? 'WHERE ' + whereClauses.join(' AND ') : ''}
+		ORDER BY tstamp DESC
+		${limitClause}
+	`;
+
+	// If decimation is requested, wrap the query in a subquery with ROW_NUMBER
+	if (di != null && di > 1) {
+		values.push(di);
+		queryText = `
+			SELECT tstamp, coordinates, meta
+			FROM (
+				SELECT
+					tstamp,
+					CASE
+						WHEN meta->>'lineStatus' = 'offline' THEN
+							ARRAY[COALESCE((meta->>'longitude')::float, ST_X(geometry)), COALESCE((meta->>'latitude')::float, ST_Y(geometry))]
+						ELSE
+							ARRAY[
+								COALESCE(
+									(meta->>'longitudeMaster')::float,
+									ST_X(geometry)
+								),
+								COALESCE(
+									(meta->>'latitudeMaster')::float,
+									ST_Y(geometry)
+								)
+							]
+					END AS coordinates,
+					meta::json AS meta,
+					ROW_NUMBER() OVER (ORDER BY tstamp DESC) AS rn
+				FROM public.real_time_inputs
+				${whereClauses.length ? 'WHERE ' + whereClauses.join(' AND ') : ''}
+			) sub
+			WHERE rn % $${paramIndex} = 0
+			ORDER BY tstamp DESC
+			${limitClause}
+		`;
+	}
+
+	try {
+		const res = await client.query(queryText, values);
+		if (!res.rows?.length) {
+			throw { status: 204 }; // No Content
+		}
+
+		// Process rows: Convert tstamp to Date and extract coordinates
+		let processed = res.rows.map(row => ({
+			tstamp: new Date(row.tstamp),
+			x: row.coordinates[0], // Longitude
+			y: row.coordinates[1], // Latitude
+			meta: projection != null
+			? projection == ""
+				? undefined
+				: project([row.meta], projection)[0] : row.meta
+		}));
+
+		// Handle geojson output formats
+		if (geojson === 'LineString') {
+			// Compute line length (haversine formula in JavaScript for simplicity)
+			let length = 0;
+			for (let i = 1; i < processed.length; i++) {
+				const p1 = processed[i - 1];
+				const p2 = processed[i];
+				const R = 6371e3; // Earth's radius in meters
+				const φ1 = p1.y * Math.PI / 180;
+				const φ2 = p2.y * Math.PI / 180;
+				const Δφ = (p2.y - p1.y) * Math.PI / 180;
+				const Δλ = (p2.x - p1.x) * Math.PI / 180;
+				const a = Math.sin(Δφ / 2) ** 2 +
+						 Math.cos(φ1) * Math.cos(φ2) * Math.sin(Δλ / 2) ** 2;
+				const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+				length += R * c;
+			}
+
+			return {
+				type: 'Feature',
+				geometry: {
+					type: 'LineString',
+					coordinates: processed.map(p => [p.x, p.y])
+				},
+				properties: {
+					ts0: processed[processed.length - 1].tstamp.toISOString(),
+					ts1: processed[0].tstamp.toISOString(),
+					distance: length
+				}
+			};
+		} else if (geojson === 'Point') {
+			return {
+				type: 'FeatureCollection',
+				features: processed.map(p => ({
+					type: 'Feature',
+					geometry: {
+						type: 'Point',
+						coordinates: [p.x, p.y]
+					},
+					properties: {
+						...p.meta,
+						tstamp: p.tstamp.toISOString()
+					}
+				}))
+			};
+		} else if (geojson) {
+			return processed.map(p => ({
+				type: 'Feature',
+				geometry: {
+					type: 'Point',
+					coordinates: [p.x, p.y]
+				},
+				properties: {
+					...p.meta,
+					tstamp: p.tstamp.toISOString()
+				}
+			}));
+		} else {
+			return processed;
+		}
+	} finally {
+		client.release();
+	}
+}
+
+module.exports = get;

lib/www/server/lib/db/gis/vesseltrack/index.js

@@ -0,0 +1,5 @@
+
+module.exports = {
+	get: require('./get')
+};
+

lib/www/server/lib/db/project/clone.js

@@ -0,0 +1,94 @@
+const { exec } = require('child_process');
+const util = require('util');
+const fs = require('fs');
+const execPromise = util.promisify(exec);
+const { setSurvey, pool } = require('../connection');
+
+async function createProject(pid, name, src_srid, dst_srid, src_schema, dst_schema) {
+  const client = await pool.connect();
+  try {
+    await client.query('BEGIN');
+
+    // Determine default src_schema and src_srid
+    let src_schema_val;
+    let src_srid_val;
+    const res = await client.query(`
+      SELECT schema, meta->>'epsg' as epsg
+      FROM public.projects
+      ORDER BY CAST(SUBSTRING(schema FROM 8) AS INTEGER) DESC
+      LIMIT 1
+    `);
+    if (res.rows.length === 0) {
+      src_schema_val = 'survey_0';
+      src_srid_val = 23031;
+    } else {
+      src_schema_val = res.rows[0].schema;
+      src_srid_val = parseInt(res.rows[0].epsg, 10);
+    }
+
+    // Apply parameters or defaults
+    src_schema = src_schema || src_schema_val;
+    src_srid = src_srid ?? src_srid_val;
+    dst_srid = dst_srid ?? src_srid;
+    if (dst_schema === undefined) {
+      const srcNum = parseInt(src_schema.replace('survey_', ''), 10);
+      dst_schema = `survey_${srcNum + 1}`;
+    }
+
+    // Dump the source schema structure
+    const pgDumpCmd = `PGPASSWORD=${pool.options.password} pg_dump --schema-only --schema=${src_schema} --host=${pool.options.host} --port=${pool.options.port} --username=${pool.options.user} ${pool.options.database}`;
+    const { stdout: sqlDump } = await execPromise(pgDumpCmd);
+    //fs.writeFileSync('sqlDump.sql', sqlDump);
+    //console.log('Saved original SQL to sqlDump.sql');
+
+    // Modify the dump to use the destination schema and update SRID
+    const escapedSrcSchema = src_schema.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+    let modifiedSql = sqlDump
+      .replace(new RegExp(`CREATE SCHEMA ${escapedSrcSchema};`, 'gi'), `CREATE SCHEMA ${dst_schema};`)
+      .replace(new RegExp(`ALTER SCHEMA ${escapedSrcSchema} OWNER TO`, 'gi'), `ALTER SCHEMA ${dst_schema} OWNER TO`)
+      .replace(/SELECT pg_catalog\.set_config\('search_path',\s*'', false\);/, `SELECT pg_catalog.set_config('search_path', '${dst_schema}, public', false);`)
+      .replace(new RegExp(`${escapedSrcSchema}\\.`, 'g'), `${dst_schema}.`);
+
+    // Replace SRID in the SQL dump if src_srid !== dst_srid
+    if (src_srid !== dst_srid) {
+      // Replace SRID in geometry column definitions (e.g., geometry(Point, 23031))
+      modifiedSql = modifiedSql.replace(
+        new RegExp(`geometry\\((\\w+),\\s*${src_srid}\\s*\\)`, 'g'),
+        `geometry($1, ${dst_srid})`
+      );
+      // Replace SRID in AddGeometryColumn calls (if used in the dump)
+      modifiedSql = modifiedSql.replace(
+        new RegExp(`AddGeometryColumn\\((['"]?)${escapedSrcSchema}\\1,\\s*(['"]\\w+['"]),\\s*(['"]\\w+['"]),\\s*${src_srid},`, 'g'),
+        `AddGeometryColumn($1${dst_schema}$1, $2, $3, ${dst_srid},`
+      );
+      console.log(`Replaced SRID ${src_srid} with ${dst_srid} in SQL dump`);
+    }
+
+    //fs.writeFileSync('modifiedSql.sql', modifiedSql);
+    //console.log('Saved modified SQL to modifiedSql.sql');
+
+    // Execute the modified SQL to create the cloned schema
+    await client.query(modifiedSql);
+    console.log('Applied modified SQL successfully');
+
+    // Insert the new project into public.projects
+    const meta = { epsg: dst_srid.toString() }; // Ensure string for JSONB
+    await client.query(`
+      INSERT INTO public.projects (pid, name, schema, meta)
+      VALUES ($1, $2, $3, $4)
+    `, [pid, name, dst_schema, meta]);
+    console.log(`Inserted project ${pid} into public.projects with schema ${dst_schema}`);
+
+    await client.query('COMMIT');
+    console.log('Transaction committed successfully');
+  } catch (error) {
+    await client.query('ROLLBACK');
+    console.error('Transaction rolled back due to error:', error);
+    throw error;
+  } finally {
+    client.release();
+    console.log('Database client released');
+  }
+}
+
+module.exports = createProject;

lib/www/server/lib/db/project/get.js

@@ -1,8 +1,14 @@
 const { setSurvey, pool } = require('../connection');
 
-async function get () {
+async function get (opts = {}) {
+
+	const select = opts.timestamps
+		? "last_project_update(pid) tstamp,"
+		: "";
+
 	const text = `
 		SELECT
+			${select}
 			pid,
 			name,
 			schema,

lib/www/server/lib/db/sequence/get.js

@@ -15,11 +15,50 @@ async function getSummary (projectId, sequence, opts = {}) {
 	return res.rows[0];
 }
 
+async function getPoints (projectId, sequence, opts = {}) {
+
+	const offset = Math.abs(opts.offset) || Math.abs((opts.page-1)*opts.itemsPerPage) || 0;
+	const limit = Math.abs(opts.limit) || Math.abs(Number(opts.itemsPerPage)) || null;
+
+	const client = await setSurvey(projectId);
+
+	const restriction = sequence
+		? "sequence = $3"
+		: "TRUE OR $3";
+
+	const text = `
+		SELECT line, point, sequence, st_x(ST_Transform(geometry, 4326)) longitude, st_y(ST_Transform(geometry, 4326)) latitude
+		FROM final_shots
+		WHERE ${restriction}
+		ORDER BY sequence, point
+		OFFSET $1
+		LIMIT $2;
+	`;
+
+	try {
+		const res = await client.query({text, values: [offset, limit, sequence], rowMode: 'array'});
+		return res.rows;
+	} catch (err) {
+		console.error(err);
+		// throw { status: 500, message: "Internal error" };
+	} finally {
+		client.release();
+	}
+
+}
 
 async function get (projectId, sequence, opts = {}) {
 	if (opts.summary) {
 		return await getSummary(projectId, sequence, opts);
 	}
+	if (opts.type == 4) {
+		// The user is request that we send just the bare details:
+		// sequence, sailline, line, longitude, latitude.
+		//
+		// This will probably be a binary data request (though doesn't
+		// need to).
+		return await getPoints(projectId, sequence, opts);
+	}
 
 	const client = await setSurvey(projectId);
 

lib/www/server/lib/jwt.js

@@ -46,10 +46,6 @@ function issue (payload, req, res) {
 		if (token) {
 			res.set("X-JWT", token);
 			const expiry = payload.exp ? (new Date(payload.exp*1000)).toUTCString() : null;
-			const cookie = expiry
-				? `JWT=${token}; path=/; SameSite=lax; expires=${expiry}`
-				: `JWT=${token}; path=/; SameSite=lax`
-			res.set("Set-Cookie", cookie); // For good measure
 		}
 	}
 

lib/www/server/lib/queue/actions-queue.js

@@ -1,52 +0,0 @@
-const Queue = require('./queue');
-
-// Inspired by:
-// https://stackoverflow.com/questions/53540348/js-async-await-tasks-queue#53540586
-
-class ActionsQueue extends Queue {
-
-	constructor (items = []) {
-		super(items);
-
-		this.pending = false;
-	}
-
-	enqueue (action) {
-		return new Promise ((resolve, reject) => {
-			super.enqueue({ action, resolve, reject });
-			this.dequeue();
-		});
-	}
-
-	async dequeue () {
-
-		if (this.pending) {
-			return false;
-		}
-
-		const item = super.dequeue();
-
-		if (!item) {
-			return false;
-		}
-
-		try {
-
-			this.pending = true;
-
-			const result = await item.action(this);
-
-			this.pending = false;
-			item.resolve(result);
-		} catch (err) {
-			this.pending = false;
-			item.reject(err);
-		} finally {
-			this.dequeue();
-		}
-
-	}
-
-}
-
-module.exports = ActionsQueue;

lib/www/server/lib/queue/index.js

@@ -1,6 +0,0 @@
-
-module.exports = {
-	Queue: require('./queue'),
-	ActionsQueue: require('./actions-queue')
-};
-

lib/www/server/lib/queue/queue.js

@@ -1,22 +0,0 @@
-
-class Queue {
-
-	constructor (items = []) {
-		this.items = items;
-	}
-
-	enqueue (item) {
-		this.items.push(item);
-	}
-
-	dequeue () {
-		return this.items.shift();
-	}
-
-	length () {
-		return this.items.length;
-	}
-
-}
-
-module.exports = Queue;

lib/www/server/lib/selenium/index.js

@@ -1,52 +1,110 @@
-// TODO Append location to PATH
-
 const path = require('path');
 const fs = require('fs');
-const {Builder, By, Key, until} = require('selenium-webdriver');
-const firefox   = require('selenium-webdriver/firefox');
+const { Builder, By, Key, until } = require('selenium-webdriver');
+const firefox = require('selenium-webdriver/firefox');
+const { execSync } = require('child_process');
 
 const geckodriverPath = path.resolve(__dirname, "geckodriver");
 
-// We launch a browser instance and then start an activity timer.
-// We shut down the browser after a period of inactivity, to
-// save memory.
+// State to prevent race conditions
 let driver = null;
 let timer = null;
+let isShuttingDown = false;
 
-function resetTimer () {
-	clearTimeout(timer);
-	timer = setTimeout(shutdown, 120000); // Yup, hardcoded to two minutes. For now anyway
+// Verify GeckoDriver exists
+if (!fs.existsSync(geckodriverPath)) {
+	throw new Error(`GeckoDriver not found at ${geckodriverPath}`);
 }
 
-async function launch () {
+function resetTimer() {
+	clearTimeout(timer);
+	timer = setTimeout(shutdown, 120000); // 2 minutes inactivity timeout
+}
+
+async function launch() {
+	if (isShuttingDown) {
+		console.log("Shutdown in progress, waiting...");
+		await new Promise(resolve => setTimeout(resolve, 1000));
+		return launch(); // Retry after delay
+	}
 	resetTimer();
 	if (!driver) {
 		console.log("Launching Firefox");
 		const options = new firefox.Options();
+		// Explicitly set headless mode and optimize for server
+		options.addArguments('--headless', '--no-sandbox', '--disable-gpu');
+		// Limit content processes to reduce resource usage
+		options.setPreference('dom.ipc.processCount', 1);
+
+		const service = new firefox.ServiceBuilder(geckodriverPath);
 		driver = await new Builder()
 			.forBrowser('firefox')
-			.setFirefoxService(new firefox.ServiceBuilder(geckodriverPath))
-			.setFirefoxOptions(options.headless())
+			.setFirefoxService(service)
+			.setFirefoxOptions(options)
 			.build();
 	}
 }
 
-async function shutdown () {
-	if (driver) {
+async function shutdown() {
+	if (driver && !isShuttingDown) {
+		isShuttingDown = true;
 		console.log("Shutting down Firefox");
-		// This is an attempt at avoiding a race condition if someone
-		// makes a call and resets the timer while the shutdown is in
-		// progress.
-		const d = driver;
-		driver = null;
-		await d.quit();
+		try {
+			const d = driver;
+			driver = null;
+			await d.quit();
+			// Explicitly stop the service
+			const service = d.service;
+			if (service) {
+				service.stop();
+			}
+			console.log("Firefox shutdown complete");
+		} catch (error) {
+			console.error("Error during shutdown:", error);
+			// Forcefully kill lingering processes (Linux/Unix)
+			try {
+				execSync('pkill -u $USER firefox || true');
+				execSync('pkill -u $USER geckodriver || true');
+				console.log("Terminated lingering Firefox/GeckoDriver processes");
+			} catch (killError) {
+				console.error("Error killing processes:", killError);
+			}
+		} finally {
+			isShuttingDown = false;
+		}
 	}
 }
 
-async function url2pdf (url) {
+async function url2pdf(url) {
 	await launch();
-	await driver.get(url);
-	return await driver.printPage({width: 21.0, height: 29.7});
+	try {
+		console.log(`Navigating to ${url}`);
+		await driver.get(url);
+		// Add delay to stabilize Marionette communication
+		await driver.sleep(3000);
+		const pdf = await driver.printPage({ width: 21.0, height: 29.7 });
+		resetTimer(); // Reset timer after successful operation
+		return pdf;
+	} catch (error) {
+		console.error("Error in url2pdf:", error);
+		await shutdown(); // Force shutdown on error
+		throw error;
+	}
 }
 
+// Periodically clean up orphaned processes (every 5 minutes)
+setInterval(() => {
+	try {
+		const firefoxCount = execSync('pgrep -c firefox || echo 0').toString().trim();
+		if (parseInt(firefoxCount) > 0 && !driver) {
+			console.log(`Found ${firefoxCount} orphaned Firefox processes, cleaning up...`);
+			execSync('pkill -u $USER firefox || true');
+			execSync('pkill -u $USER geckodriver || true');
+			console.log("Cleaned up orphaned processes");
+		}
+	} catch (error) {
+		console.error("Error checking orphaned processes:", error);
+	}
+}, 300000);
+
 module.exports = { url2pdf };