Merge branch '334-add-4d-comparisons' into 'devel'

Resolve "Add 4D comparisons"

Closes #334

See merge request wgp/dougal/software!60

bin/fwr.py

@@ -23,6 +23,7 @@ transform = {
 }
 
 def parse_line (line, fields, fixed = None):
+	# print("parse_line", line, fields, fixed)
 	data = dict()
 
 	if fixed:
@@ -51,6 +52,7 @@ def parse_line (line, fields, fixed = None):
 
 		data[key] = value
 
+	# print("parse_line data =", data)
 	return data
 
 

bin/import_final_p111.py

@@ -73,6 +73,12 @@ if __name__ == '__main__':
 
 		lineNameInfo = final_p111.get("lineNameInfo")
 		pattern = final_p111.get("pattern")
+		if not lineNameInfo:
+			if not pattern:
+				print("ERROR! Missing final.p111.lineNameInfo in project configuration. Cannot import final P111")
+				raise Exception("Missing final.p111.lineNameInfo")
+			else:
+				print("WARNING! No `lineNameInfo` in project configuration (final.p111). You should add it to the settings.")
 		rx = None
 		if pattern and pattern.get("regex"):
 			rx = re.compile(pattern["regex"])
@@ -114,27 +120,27 @@ if __name__ == '__main__':
 							file_info = dict(zip(pattern["captures"], match.groups()))
 							file_info["meta"] = {}
 
-						if lineNameInfo:
-							basename = os.path.basename(physical_filepath)
-							fields = lineNameInfo.get("fields", {})
-							fixed = lineNameInfo.get("fixed")
-							try:
-								parsed_line = fwr.parse_line(basename, fields, fixed)
-							except ValueError as err:
-								parsed_line = "Line format error: " + str(err)
-							if type(parsed_line) == str:
-								print(parsed_line, file=sys.stderr)
-								print("This file will be ignored!")
-								continue
+							if lineNameInfo:
+								basename = os.path.basename(physical_filepath)
+								fields = lineNameInfo.get("fields", {})
+								fixed = lineNameInfo.get("fixed")
+								try:
+									parsed_line = fwr.parse_line(basename, fields, fixed)
+								except ValueError as err:
+									parsed_line = "Line format error: " + str(err)
+								if type(parsed_line) == str:
+									print(parsed_line, file=sys.stderr)
+									print("This file will be ignored!")
+									continue
 
-							file_info = {}
-							file_info["sequence"] = parsed_line["sequence"]
-							file_info["line"] = parsed_line["line"]
-							del(parsed_line["sequence"])
-							del(parsed_line["line"])
-							file_info["meta"] = {
-								"fileInfo": parsed_line
-							}
+								file_info = {}
+								file_info["sequence"] = parsed_line["sequence"]
+								file_info["line"] = parsed_line["line"]
+								del(parsed_line["sequence"])
+								del(parsed_line["line"])
+								file_info["meta"] = {
+									"fileInfo": parsed_line
+								}
 
 						if pending:
 							print("Skipping / removing final file because marked as PENDING", logical_filepath)

bin/import_raw_p111.py

@@ -41,6 +41,12 @@ if __name__ == '__main__':
 
 		lineNameInfo = raw_p111.get("lineNameInfo")
 		pattern = raw_p111.get("pattern")
+		if not lineNameInfo:
+			if not pattern:
+				print("ERROR! Missing raw.p111.lineNameInfo in project configuration. Cannot import raw P111")
+				raise Exception("Missing raw.p111.lineNameInfo")
+			else:
+				print("WARNING! No `lineNameInfo` in project configuration (raw.p111). You should add it to the settings.")
 		rx = None
 		if pattern and pattern.get("regex"):
 			rx = re.compile(pattern["regex"])
@@ -96,14 +102,15 @@ if __name__ == '__main__':
 								print("This file will be ignored!")
 								continue
 
-						file_info = {}
-						file_info["sequence"] = parsed_line["sequence"]
-						file_info["line"] = parsed_line["line"]
-						del(parsed_line["sequence"])
-						del(parsed_line["line"])
-						file_info["meta"] = {
-							"fileInfo": parsed_line
-						}
+							file_info = {}
+							file_info["sequence"] = parsed_line["sequence"]
+							file_info["line"] = parsed_line["line"]
+							del(parsed_line["sequence"])
+							del(parsed_line["line"])
+							file_info["meta"] = {
+								"fileInfo": parsed_line
+							}
+
 						p111_data = p111.from_file(physical_filepath)
 
 						print("Saving")

bin/runner.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# Maximum runtime in seconds before killing an overdue instance (e.g., 10 minutes)
+MAX_RUNTIME_SECONDS=$((15 * 60))
 
 DOUGAL_ROOT=${DOUGAL_ROOT:-$(dirname "$0")/..}
 
@@ -80,8 +82,9 @@ function run () {
 # 		DESCRIPTION=""
 		SERVICE="deferred_imports"
 
-		$BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \
-			-O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)"
+# Disable GitLab alerts. They're just not very practical
+# 		$BINDIR/send_alert.py -t "$TITLE" -s "$SERVICE" -l "critical" \
+# 			-O "$(cat $STDOUTLOG)" -E "$(cat $STDERRLOG)"
 
 		exit 2
 	}
@@ -97,14 +100,37 @@ function cleanup () {
 }
 
 if [[ -f $LOCKFILE ]]; then
-	PID=$(cat "$LOCKFILE")
-	if pgrep -F "$LOCKFILE"; then
-		print_warning $(printf "The previous process is still running (%d)" $PID)
-		exit 1
-	else
-		rm "$LOCKFILE"
-		print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" $PID)
-	fi
+    PID=$(cat "$LOCKFILE")
+    if kill -0 "$PID" 2>/dev/null; then  # Check if process is running
+        # Get elapsed time in D-HH:MM:SS format and convert to seconds
+        ELAPSED_STR=$(ps -p "$PID" -o etime= | tr -d '[:space:]')
+        if [ -n "$ELAPSED_STR" ]; then
+            # Convert D-HH:MM:SS to seconds
+            ELAPSED_SECONDS=$(echo "$ELAPSED_STR" | awk -F'[-:]' '{
+                seconds = 0
+                if (NF == 4) { seconds += $1 * 86400 }  # Days
+                if (NF >= 3) { seconds += $NF-2 * 3600 } # Hours
+                if (NF >= 2) { seconds += $NF-1 * 60 }   # Minutes
+                seconds += $NF                           # Seconds
+                print seconds
+            }')
+            if [ "$ELAPSED_SECONDS" -gt "$MAX_RUNTIME_SECONDS" ]; then
+                # Kill the overdue process (SIGTERM; use -9 for SIGKILL if needed)
+                kill "$PID" 2>/dev/null
+                print_warning $(printf "Killed overdue process (%d) that ran for %s (%d seconds)" "$PID" "$ELAPSED_STR" "$ELAPSED_SECONDS")
+                rm "$LOCKFILE"
+            else
+                print_warning $(printf "Previous process is still running (%d) for %s (%d seconds)" "$PID" "$ELAPSED_STR" "$ELAPSED_SECONDS")
+                exit 1
+            fi
+        else
+            print_warning $(printf "Could not retrieve elapsed time for process (%d)" "$PID")
+            exit 1
+        fi
+    else
+        rm "$LOCKFILE"
+        print_warning $(printf "Previous process (%d) not found. Must have died unexpectedly" "$PID")
+    fi
 fi
 
 echo "$$" > "$LOCKFILE" || {

bin/update_comparisons.js

@@ -0,0 +1,89 @@
+#!/usr/bin/node
+
+const cmp = require('../lib/www/server/lib/comparisons');
+
+async function purgeComparisons () {
+	const groups = await cmp.groups();
+	const comparisons = await cmp.getGroup();
+
+	const pids = new Set(Object.values(groups).flat().map( p => p.pid ));
+	const comparison_pids = new Set(comparisons.map( c => [ c.baseline_pid, c.monitor_pid ] ).flat());
+
+	for (const pid of comparison_pids) {
+		if (!pids.has(pid)) {
+			console.log(`${pid} no longer par of a group. Deleting comparisons`);
+
+			staleComps = comparisons.filter( c => c.baseline_pid == pid || c.monitor_pid == pid );
+			for (c of staleComps) {
+				console.log(`Deleting comparison ${c.baseline_pid} → ${c.monitor_pid}`);
+				await cmp.remove(c.baseline_pid, c.monitor_pid);
+			}
+		}
+	}
+}
+
+
+async function main () {
+
+	console.log("Looking for unreferenced comparisons to purge");
+	await purgeComparisons();
+
+	console.log("Retrieving project groups");
+	const groups = await cmp.groups();
+
+	if (!Object.keys(groups??{})?.length) {
+		console.log("No groups found");
+		return 0;
+	}
+
+	console.log(`Found ${Object.keys(groups)?.length} groups: ${Object.keys(groups).join(", ")}`);
+
+	for (const groupName of Object.keys(groups)) {
+		const projects = groups[groupName];
+
+		console.log(`Fetching saved comparisons for ${groupName}`);
+
+		const comparisons = await cmp.getGroup(groupName);
+
+		if (!comparisons || !comparisons.length) {
+			console.log(`No comparisons found for ${groupName}`);
+			continue;
+		}
+
+		// Check if there are any projects that have been modified since last comparison
+		// or if there are any pairs that are no longer part of the group
+
+		const outdated = comparisons.filter( c => {
+			const baseline_tstamp = projects.find( p => p.pid === c.baseline_pid )?.tstamp;
+			const monitor_tstamp = projects.find( p => p.pid === c.monitor_pid )?.tstamp;
+			return (c.tstamp < baseline_tstamp) || (c.tstamp < monitor_tstamp) ||
+				baseline_tstamp == null || monitor_tstamp == null;
+		});
+
+		for (const comparison of outdated) {
+			console.log(`Removing stale comparison: ${comparison.baseline_pid} → ${comparison.monitor_pid}`);
+			await cmp.remove(comparison.baseline_pid, comparison.monitor_pid);
+		}
+
+		if (projects?.length < 2) {
+			console.log(`Group ${groupName} has less than two projects. No comparisons are possible`);
+			continue;
+		}
+
+		// Re-run the comparisons that are not in the database. They may
+		// be missing either beacause they were not there to start with
+		// or because we just removed them due to being stale
+
+		console.log(`Recalculating group ${groupName}`);
+		await cmp.saveGroup(groupName);
+	}
+
+	console.log("Comparisons update done");
+	return 0;
+}
+
+if (require.main === module) {
+	main();
+} else {
+	module.exports = main;
+}

etc/db/upgrades/upgrade38-v0.6.0-add-keystore-table.sql

@@ -38,6 +38,7 @@ CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
 BEGIN
 
 	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
 
 	CREATE TABLE IF NOT EXISTS keystore (
 		type TEXT NOT NULL,     -- A class of data to be stored
@@ -79,7 +80,7 @@ BEGIN
 			USING MESSAGE='Patch already applied';
 	END IF;
 
-	IF current_db_version != '0.5.3' THEN
+	IF current_db_version != '0.5.4' THEN
 		RAISE EXCEPTION
 			USING MESSAGE='Invalid database version: ' || current_db_version,
 				HINT='Ensure all previous patches have been applied.';

etc/db/upgrades/upgrade39-v0.6.1-add-default-user.sql

@@ -41,6 +41,7 @@ CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
 BEGIN
 
 	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
 
 	INSERT INTO keystore (type, key, data)
 	VALUES ('user', '6f1e7159-4ca0-4ae4-ab4e-89078166cc10', '

etc/db/upgrades/upgrade40-v0.6.2-add-default-project-organisations.sql

@@ -0,0 +1,106 @@
+-- Fix final_lines_summary view
+--
+-- New schema version: 0.6.2
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade only affects the `public` schema.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update adds an "organisations" section to the configuration,
+-- with a default configured organisation of "WGP" with full access.
+-- This is so that projects can be made accessible after migrating
+-- to the new permissions architecture.
+--
+-- In addition, projects with an id starting with "eq" are assumed to
+-- be Equinor projects, and an additional organisation is added with
+-- read-only access. This is intended for clients, which should be
+-- assigned to the "Equinor organisation".
+--
+-- Finally, we assign the vessel to the "WGP" organisation (full access)
+-- so that we can actually use administrative endpoints.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- Add "organisations" section to configurations, if not already present
+	UPDATE projects
+	SET
+		meta = jsonb_set(meta, '{organisations}', '{"WGP": {"read": true, "write": true, "edit": true}}'::jsonb, true)
+	WHERE meta->'organisations' IS NULL;
+
+	-- Add (or overwrite!) "organisations.Equinor" giving read-only access (can be changed later via API)
+	UPDATE projects
+	SET
+		meta = jsonb_set(meta, '{organisations, Equinor}', '{"read": true, "write": false, "edit": false}'::jsonb, true)
+	WHERE pid LIKE 'eq%';
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.1' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.2"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.2"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade41-v0.6.3-add-comparisons.sql

@@ -0,0 +1,109 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.3
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade creates a new schema called `comparisons`.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update adds a `comparisons` table to a `comparisons` schema.
+-- The `comparisons.comparisons` table holds 4D prospect comparison data.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE SCHEMA IF NOT EXISTS comparisons
+		AUTHORIZATION postgres;
+
+	COMMENT ON SCHEMA comparisons
+		IS 'Holds 4D comparison data and logic';
+
+	CREATE TABLE IF NOT EXISTS comparisons.comparisons
+	(
+		type text COLLATE pg_catalog."default" NOT NULL,
+		baseline_pid text COLLATE pg_catalog."default" NOT NULL,
+		monitor_pid text COLLATE pg_catalog."default" NOT NULL,
+		data bytea,
+		meta jsonb NOT NULL DEFAULT '{}'::jsonb,
+		CONSTRAINT comparisons_pkey PRIMARY KEY (baseline_pid, monitor_pid, type)
+	)
+
+	TABLESPACE pg_default;
+
+	ALTER TABLE IF EXISTS comparisons.comparisons
+		OWNER to postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.3"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.3"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade42-v0.6.4-notify-exclude-columns.sql

@@ -0,0 +1,169 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.4
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.notify()
+		RETURNS trigger
+		LANGUAGE 'plpgsql'
+		COST 100
+		VOLATILE NOT LEAKPROOF
+	AS $BODY$
+		DECLARE
+		channel text := TG_ARGV[0];
+		pid text;
+		payload text;
+		notification text;
+		payload_id integer;
+		old_json jsonb;
+		new_json jsonb;
+		excluded_col text;
+		i integer;
+		BEGIN
+
+		-- Fetch pid
+		SELECT projects.pid INTO pid FROM projects WHERE schema = TG_TABLE_SCHEMA;
+
+		-- Build old and new as jsonb, excluding specified columns if provided
+		IF OLD IS NOT NULL THEN
+			old_json := row_to_json(OLD)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				old_json := old_json - excluded_col;
+			END LOOP;
+		ELSE
+			old_json := NULL;
+		END IF;
+
+		IF NEW IS NOT NULL THEN
+			new_json := row_to_json(NEW)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				new_json := new_json - excluded_col;
+			END LOOP;
+		ELSE
+			new_json := NULL;
+		END IF;
+
+		-- Build payload
+		payload := json_build_object(
+			'tstamp', CURRENT_TIMESTAMP,
+			'operation', TG_OP,
+			'schema', TG_TABLE_SCHEMA,
+			'table', TG_TABLE_NAME,
+			'old', old_json,
+			'new', new_json,
+			'pid', pid
+		)::text;
+
+		-- Handle large payloads
+		IF octet_length(payload) < 1000 THEN
+			PERFORM pg_notify(channel, payload);
+		ELSE
+			-- Store large payload and notify with ID (as before)
+			INSERT INTO notify_payloads (payload) VALUES (payload) RETURNING id INTO payload_id;
+
+			notification := json_build_object(
+				'tstamp', CURRENT_TIMESTAMP,
+				'operation', TG_OP,
+				'schema', TG_TABLE_SCHEMA,
+				'table', TG_TABLE_NAME,
+				'pid', pid,
+				'payload_id', payload_id
+			)::text;
+
+			PERFORM pg_notify(channel, notification);
+			RAISE INFO 'Payload over limit';
+		END IF;
+		RETURN NULL;
+		END;
+	$BODY$;
+
+	ALTER FUNCTION public.notify()
+		OWNER TO postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.4"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.4"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade43-v0.6.5-add-comparisons-notification-channel.sql

@@ -0,0 +1,96 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.5
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE TRIGGER comparisons_tg
+		AFTER INSERT OR DELETE OR UPDATE
+		ON comparisons.comparisons
+		FOR EACH ROW
+		EXECUTE FUNCTION public.notify('comparisons', 'data');
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.5"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.5"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade44-v0.6.6-add-last_project_update-function.sql

@@ -0,0 +1,157 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.6
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This adds a last_project_update(pid) function. It takes a project ID
+-- and returns the last known timestamp from that project. Timestamps
+-- are derived from multiple sources:
+--
+-- - raw_shots table
+-- - final_shots table
+-- - events_log_full table
+-- - info table where key = 'qc'
+-- - files table, from the hashes (which contain the file's mtime)
+-- - project configuration, looking for an _updatedOn property
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.last_project_update(p_pid text)
+	RETURNS timestamp with time zone
+	LANGUAGE plpgsql
+	AS $function$
+	DECLARE
+		v_last_ts timestamptz := NULL;
+		v_current_ts timestamptz;
+		v_current_str text;
+		v_current_unix numeric;
+		v_sid_rec record;
+	BEGIN
+		-- From raw_shots, final_shots, info, and files
+		FOR v_sid_rec IN SELECT schema FROM public.projects WHERE pid = p_pid
+		LOOP
+			-- From raw_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.raw_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From final_shots
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.final_shots' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+
+			-- From info where key = 'qc'
+			EXECUTE 'SELECT value->>''updatedOn'' FROM ' || v_sid_rec.schema || '.info WHERE key = ''qc''' INTO v_current_str;
+			IF v_current_str IS NOT NULL THEN
+				v_current_ts := v_current_str::timestamptz;
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From files hash second part, only for valid colon-separated hashes
+			EXECUTE 'SELECT max( split_part(hash, '':'', 2)::numeric ) FROM ' || v_sid_rec.schema || '.files WHERE hash ~ ''^[0-9]+:[0-9]+\\.[0-9]+:[0-9]+\\.[0-9]+:[0-9a-f]+$''' INTO v_current_unix;
+			IF v_current_unix IS NOT NULL THEN
+				v_current_ts := to_timestamp(v_current_unix);
+				IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+					v_last_ts := v_current_ts;
+				END IF;
+			END IF;
+
+			-- From event_log_full
+			EXECUTE 'SELECT max(tstamp) FROM ' || v_sid_rec.schema || '.event_log_full' INTO v_current_ts;
+			IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+				v_last_ts := v_current_ts;
+			END IF;
+		END LOOP;
+
+		-- From projects.meta->_updatedOn
+		SELECT (meta->>'_updatedOn')::timestamptz FROM public.projects WHERE pid = p_pid INTO v_current_ts;
+		IF v_current_ts > v_last_ts OR v_last_ts IS NULL THEN
+			v_last_ts := v_current_ts;
+		END IF;
+
+		RETURN v_last_ts;
+	END;
+	$function$;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.6' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.6"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.6"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/modules/@dougal/binary/classes.js

@@ -0,0 +1,968 @@
+const codeToType = {
+	0: Int8Array,
+	1: Uint8Array,
+	2: Int16Array,
+	3: Uint16Array,
+	4: Int32Array,
+	5: Uint32Array,
+	7: Float32Array,
+	8: Float64Array,
+	9: BigInt64Array,
+	10: BigUint64Array
+};
+
+const typeToBytes = {
+	Int8Array: 1,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 2,
+	Int32Array: 4,
+	Uint32Array: 4,
+	Float32Array: 4,
+	Float64Array: 8,
+	BigInt64Array: 8,
+	BigUint64Array: 8
+};
+
+function readTypedValue(view, offset, type) {
+	switch (type) {
+		case Int8Array: return view.getInt8(offset);
+		case Uint8Array: return view.getUint8(offset);
+		case Int16Array: return view.getInt16(offset, true);
+		case Uint16Array: return view.getUint16(offset, true);
+		case Int32Array: return view.getInt32(offset, true);
+		case Uint32Array: return view.getUint32(offset, true);
+		case Float32Array: return view.getFloat32(offset, true);
+		case Float64Array: return view.getFloat64(offset, true);
+		case BigInt64Array: return view.getBigInt64(offset, true);
+		case BigUint64Array: return view.getBigUint64(offset, true);
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+function writeTypedValue(view, offset, value, type) {
+	switch (type) {
+		case Int8Array: view.setInt8(offset, value); break;
+		case Uint8Array: view.setUint8(offset, value); break;
+		case Int16Array: view.setInt16(offset, value, true); break;
+		case Uint16Array: view.setUint16(offset, value, true); break;
+		case Int32Array: view.setInt32(offset, value, true); break;
+		case Uint32Array: view.setUint32(offset, value, true); break;
+		case Float32Array: view.setFloat32(offset, value, true); break;
+		case Float64Array: view.setFloat64(offset, value, true); break;
+		case BigInt64Array: view.setBigInt64(offset, BigInt(value), true); break;
+		case BigUint64Array: view.setBigUint64(offset, BigInt(value), true); break;
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+class DougalBinaryBundle extends ArrayBuffer {
+
+	static HEADER_LENGTH = 4; // Length of a bundle header
+
+	/** Clone an existing ByteArray into a DougalBinaryBundle
+	 */
+	static clone (buffer) {
+		const clone = new DougalBinaryBundle(buffer.byteLength);
+		const uint8Array = new Uint8Array(buffer);
+		const uint8ArrayClone = new Uint8Array(clone);
+		uint8ArrayClone.set(uint8Array);
+		return clone;
+	}
+
+	constructor (length, options) {
+		super (length, options);
+	}
+
+	/** Get the count of bundles in this ByteArray.
+	 *
+	 * Stops at the first non-bundle looking offset
+	 */
+	get bundleCount () {
+		let count = 0;
+		let currentBundleOffset = 0;
+		const view = new DataView(this);
+
+		while (currentBundleOffset < this.byteLength) {
+
+			const currentBundleHeader = view.getUint32(currentBundleOffset, true);
+			if ((currentBundleHeader & 0xff) !== 0x1c) {
+				// This is not a bundle
+				return count;
+			}
+			let currentBundleLength =	currentBundleHeader >>> 8;
+
+			currentBundleOffset += currentBundleLength + DougalBinaryBundle.HEADER_LENGTH;
+			count++;
+		}
+
+		return count;
+	}
+
+
+	/** Get the number of chunks in the bundles of this ByteArray
+	 */
+	get chunkCount () {
+		let count = 0;
+		let bundleOffset = 0;
+		const view = new DataView(this);
+
+		while (bundleOffset < this.byteLength) {
+			const header = view.getUint32(bundleOffset, true);
+			if ((header & 0xFF) !== 0x1C) break;
+			const length = header >>> 8;
+			if (bundleOffset + 4 + length > this.byteLength) break;
+
+			let chunkOffset = bundleOffset + 4; // relative to buffer start
+
+			while (chunkOffset < bundleOffset + 4 + length) {
+				const chunkType = view.getUint8(chunkOffset);
+				if (chunkType !== 0x11 && chunkType !== 0x12) break;
+
+				const cCount = view.getUint16(chunkOffset + 2, true);
+				const ΔelemC = view.getUint8(chunkOffset + 10);
+				const elemC = view.getUint8(chunkOffset + 11);
+
+				let localOffset = 12; // header size
+
+				localOffset += ΔelemC + elemC; // preface
+
+				// initial values
+				for (let k = 0; k < ΔelemC; k++) {
+					const typeByte = view.getUint8(chunkOffset + 12 + k);
+					const baseCode = typeByte & 0xF;
+					const baseType = codeToType[baseCode];
+					if (!baseType) throw new Error('Invalid base type code');
+					localOffset += typeToBytes[baseType.name];
+				}
+
+				// pad after initial
+				while (localOffset % 4 !== 0) localOffset++;
+
+				if (chunkType === 0x11) { // Sequential
+					// record data: Δelems incrs
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						localOffset += cCount * typeToBytes[incrType.name];
+					}
+
+					// elems
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						localOffset += cCount * typeToBytes[type.name];
+					}
+				} else { // Interleaved
+					// Compute exact stride for interleaved record data
+					let ΔelemStride = 0;
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						ΔelemStride += typeToBytes[incrType.name];
+					}
+					let elemStride = 0;
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						elemStride += typeToBytes[type.name];
+					}
+					const recordStride = ΔelemStride + elemStride;
+					localOffset += cCount * recordStride;
+				}
+
+				// pad after record
+				while (localOffset % 4 !== 0) localOffset++;
+
+				chunkOffset += localOffset;
+				count++;
+			}
+
+			bundleOffset += 4 + length;
+		}
+
+		return count;
+	}
+
+	/** Return an array of DougalBinaryChunkSequential or DougalBinaryChunkInterleaved instances
+	 */
+	chunks () {
+		const chunks = [];
+		let bundleOffset = 0;
+		const view = new DataView(this);
+
+		while (bundleOffset < this.byteLength) {
+			const header = view.getUint32(bundleOffset, true);
+			if ((header & 0xFF) !== 0x1C) break;
+			const length = header >>> 8;
+			if (bundleOffset + 4 + length > this.byteLength) break;
+
+			let chunkOffset = bundleOffset + 4;
+
+			while (chunkOffset < bundleOffset + 4 + length) {
+				const chunkType = view.getUint8(chunkOffset);
+				if (chunkType !== 0x11 && chunkType !== 0x12) break;
+
+				const cCount = view.getUint16(chunkOffset + 2, true);
+				const ΔelemC = view.getUint8(chunkOffset + 10);
+				const elemC = view.getUint8(chunkOffset + 11);
+
+				let localOffset = 12;
+
+				localOffset += ΔelemC + elemC;
+
+				// initial values
+				for (let k = 0; k < ΔelemC; k++) {
+					const typeByte = view.getUint8(chunkOffset + 12 + k);
+					const baseCode = typeByte & 0xF;
+					const baseType = codeToType[baseCode];
+					if (!baseType) throw new Error('Invalid base type code');
+					localOffset += typeToBytes[baseType.name];
+				}
+
+				// pad after initial
+				while (localOffset % 4 !== 0) localOffset++;
+
+				if (chunkType === 0x11) { // Sequential
+					// record data: Δelems incrs
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						localOffset += cCount * typeToBytes[incrType.name];
+					}
+
+					// elems
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						localOffset += cCount * typeToBytes[type.name];
+					}
+				} else { // Interleaved
+					// Compute exact stride for interleaved record data
+					let ΔelemStride = 0;
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						ΔelemStride += typeToBytes[incrType.name];
+					}
+					let elemStride = 0;
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						elemStride += typeToBytes[type.name];
+					}
+					const recordStride = ΔelemStride + elemStride;
+					localOffset += cCount * recordStride;
+				}
+
+				// pad after record
+				while (localOffset % 4 !== 0) localOffset++;
+
+				switch (chunkType) {
+					case 0x11:
+						chunks.push(new DougalBinaryChunkSequential(this, chunkOffset, localOffset));
+						break;
+					case 0x12:
+						chunks.push(new DougalBinaryChunkInterleaved(this, chunkOffset, localOffset));
+						break;
+					default:
+						throw new Error('Invalid chunk type');
+				}
+
+				chunkOffset += localOffset;
+			}
+
+			bundleOffset += 4 + length;
+		}
+
+		return chunks;
+	}
+
+	/** Return a ByteArray containing all data from all
+	 * chunks including reconstructed i, j and incremental
+	 * values as follows:
+	 *
+	 * <i_0> <i_1> … <i_x> // i values (constant)
+	 * <j_0> <j_1> … <j_x> // j values (j0 + Δj*i)
+	 * <Δelem_0_0> <Δelem_0_1> … <Δelem_0_x> // reconstructed Δelem0 (uses baseType)
+	 * <Δelem_1_0> <Δelem_1_1> … <Δelem_1_x> // reconstructed Δelem1
+	 * …
+	 * <Δelem_y_0> <Δelem_y_1> … <Δelem_y_x> // reconstructed Δelem1
+	 * <elem_0_0> <elem_0_1> … <elem_0_x> // First elem
+	 * <elem_1_0> <elem_1_1> … <elem_1_x> // Second elem
+	 * …
+	 * <elem_z_0> <elem_z_1> … <elem_z_x> // Last elem
+	 *
+	 * It does not matter whether the underlying chunks are
+	 * sequential or interleaved. This function will transform
+	 * as necessary.
+	 *
+	 */
+	getDataSequentially () {
+		const chunks = this.chunks();
+		if (chunks.length === 0) return new ArrayBuffer(0);
+
+		const firstChunk = chunks[0];
+		const ΔelemC = firstChunk.ΔelemCount;
+		const elemC = firstChunk.elemCount;
+
+		// Check consistency across chunks
+		for (const chunk of chunks) {
+			if (chunk.ΔelemCount !== ΔelemC || chunk.elemCount !== elemC) {
+				throw new Error('Inconsistent chunk structures');
+			}
+		}
+
+		// Get types from first chunk
+		const view = new DataView(firstChunk);
+		const ΔelemBaseTypes = [];
+		for (let k = 0; k < ΔelemC; k++) {
+			const typeByte = view.getUint8(12 + k);
+			const baseCode = typeByte & 0xF;
+			const baseType = codeToType[baseCode];
+			if (!baseType) throw new Error('Invalid base type code');
+			ΔelemBaseTypes.push(baseType);
+		}
+		const elemTypes = [];
+		for (let k = 0; k < elemC; k++) {
+			const typeCode = view.getUint8(12 + ΔelemC + k);
+			const type = codeToType[typeCode];
+			if (!type) throw new Error('Invalid elem type code');
+			elemTypes.push(type);
+		}
+
+		// Compute total records
+		const totalN = chunks.reduce((sum, c) => sum + c.jCount, 0);
+
+		// Compute sizes
+		const size_i = totalN * 2; // Uint16 for i
+		const size_j = totalN * 4; // Int32 for j
+		let size_Δelems = 0;
+		for (const t of ΔelemBaseTypes) {
+			size_Δelems += totalN * typeToBytes[t.name];
+		}
+		let size_elems = 0;
+		for (const t of elemTypes) {
+			size_elems += totalN * typeToBytes[t.name];
+		}
+		const totalSize = size_i + size_j + size_Δelems + size_elems;
+
+		const ab = new ArrayBuffer(totalSize);
+		const dv = new DataView(ab);
+
+		// Write i's
+		let off = 0;
+		for (const chunk of chunks) {
+			const i = chunk.i;
+			for (let idx = 0; idx < chunk.jCount; idx++) {
+				dv.setUint16(off, i, true);
+				off += 2;
+			}
+		}
+
+		// Write j's
+		off = size_i;
+		for (const chunk of chunks) {
+			const j0 = chunk.j0;
+			const Δj = chunk.Δj;
+			for (let idx = 0; idx < chunk.jCount; idx++) {
+				const j = j0 + idx * Δj;
+				dv.setInt32(off, j, true);
+				off += 4;
+			}
+		}
+
+		// Write Δelems
+		off = size_i + size_j;
+		for (let m = 0; m < ΔelemC; m++) {
+			const type = ΔelemBaseTypes[m];
+			const bytes = typeToBytes[type.name];
+			for (const chunk of chunks) {
+				const arr = chunk.Δelem(m);
+				for (let idx = 0; idx < chunk.jCount; idx++) {
+					writeTypedValue(dv, off, arr[idx], type);
+					off += bytes;
+				}
+			}
+		}
+
+		// Write elems
+		for (let m = 0; m < elemC; m++) {
+			const type = elemTypes[m];
+			const bytes = typeToBytes[type.name];
+			for (const chunk of chunks) {
+				const arr = chunk.elem(m);
+				for (let idx = 0; idx < chunk.jCount; idx++) {
+					writeTypedValue(dv, off, arr[idx], type);
+					off += bytes;
+				}
+			}
+		}
+
+		return ab;
+	}
+
+	/** Return a ByteArray containing all data from all
+	 * chunks including reconstructed i, j and incremental
+	 * values, interleaved as follows:
+	 *
+	 * <i_0> <j_0> <Δelem_0_0> <Δelem_1_0> … <Δelem_y_0> <elem_0_0> <elem_1_0> … <elem_z_0>
+	 * <i_1> <j_1> <Δelem_0_1> <Δelem_1_1> … <Δelem_y_1> <elem_0_1> <elem_1_1> … <elem_z_1>
+	 * <i_x> <j_x> <Δelem_0_x> <Δelem_1_x> … <Δelem_y_x> <elem_0_x> <elem_1_x> … <elem_z_x>
+	 *
+	 * It does not matter whether the underlying chunks are
+	 * sequential or interleaved. This function will transform
+	 * as necessary.
+	 *
+	 */
+	getDataInterleaved () {
+		const chunks = this.chunks();
+		if (chunks.length === 0) return new ArrayBuffer(0);
+
+		const firstChunk = chunks[0];
+		const ΔelemC = firstChunk.ΔelemCount;
+		const elemC = firstChunk.elemCount;
+
+		// Check consistency across chunks
+		for (const chunk of chunks) {
+			if (chunk.ΔelemCount !== ΔelemC || chunk.elemCount !== elemC) {
+				throw new Error('Inconsistent chunk structures');
+			}
+		}
+
+		// Get types from first chunk
+		const view = new DataView(firstChunk);
+		const ΔelemBaseTypes = [];
+		for (let k = 0; k < ΔelemC; k++) {
+			const typeByte = view.getUint8(12 + k);
+			const baseCode = typeByte & 0xF;
+			const baseType = codeToType[baseCode];
+			if (!baseType) throw new Error('Invalid base type code');
+			ΔelemBaseTypes.push(baseType);
+		}
+		const elemTypes = [];
+		for (let k = 0; k < elemC; k++) {
+			const typeCode = view.getUint8(12 + ΔelemC + k);
+			const type = codeToType[typeCode];
+			if (!type) throw new Error('Invalid elem type code');
+			elemTypes.push(type);
+		}
+
+		// Compute total records
+		const totalN = chunks.reduce((sum, c) => sum + c.jCount, 0);
+
+		// Compute record size
+		const recordSize = 2 + 4 + // i (Uint16) + j (Int32)
+			ΔelemBaseTypes.reduce((sum, t) => sum + typeToBytes[t.name], 0) +
+			elemTypes.reduce((sum, t) => sum + typeToBytes[t.name], 0);
+		const totalSize = totalN * recordSize;
+
+		const ab = new ArrayBuffer(totalSize);
+		const dv = new DataView(ab);
+
+		let off = 0;
+		for (const chunk of chunks) {
+			const i = chunk.i;
+			const j0 = chunk.j0;
+			const Δj = chunk.Δj;
+			for (let idx = 0; idx < chunk.jCount; idx++) {
+				dv.setUint16(off, i, true);
+				off += 2;
+				const j = j0 + idx * Δj;
+				dv.setInt32(off, j, true);
+				off += 4;
+				for (let m = 0; m < ΔelemC; m++) {
+					const type = ΔelemBaseTypes[m];
+					const bytes = typeToBytes[type.name];
+					const arr = chunk.Δelem(m);
+					writeTypedValue(dv, off, arr[idx], type);
+					off += bytes;
+				}
+				for (let m = 0; m < elemC; m++) {
+					const type = elemTypes[m];
+					const bytes = typeToBytes[type.name];
+					const arr = chunk.elem(m);
+					writeTypedValue(dv, off, arr[idx], type);
+					off += bytes;
+				}
+			}
+		}
+
+		return ab;
+	}
+
+	get records () {
+		const data = [];
+		for (const record of this) {
+			data.push(record.slice(1));
+		}
+		return data;
+	}
+
+	[Symbol.iterator]() {
+		const chunks = this.chunks();
+		let chunkIndex = 0;
+		let chunkIterator = chunks.length > 0 ? chunks[0][Symbol.iterator]() : null;
+
+		return {
+			next() {
+				if (!chunkIterator) {
+					return { done: true };
+				}
+
+				let result = chunkIterator.next();
+				while (result.done && chunkIndex < chunks.length - 1) {
+					chunkIndex++;
+					chunkIterator = chunks[chunkIndex][Symbol.iterator]();
+					result = chunkIterator.next();
+				}
+
+				return result;
+			}
+		};
+	}
+
+}
+
+
+class DougalBinaryChunkSequential extends ArrayBuffer {
+
+	constructor (buffer, offset, length) {
+		super(length);
+		new Uint8Array(this).set(new Uint8Array(buffer, offset, length));
+		this._ΔelemCaches = new Array(this.ΔelemCount);
+		this._elemCaches = new Array(this.elemCount);
+		this._ΔelemBlockOffsets = null;
+		this._elemBlockOffsets = null;
+		this._recordOffset = null;
+	}
+
+	_getRecordOffset() {
+		if (this._recordOffset !== null) return this._recordOffset;
+		const view = new DataView(this);
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		let recordOffset = 12 + ΔelemC + elemC;
+		for (let k = 0; k < ΔelemC; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			recordOffset += typeToBytes[bt.name];
+		}
+		while (recordOffset % 4 !== 0) recordOffset++;
+		this._recordOffset = recordOffset;
+		return recordOffset;
+	}
+
+	_initBlockOffsets() {
+		if (this._ΔelemBlockOffsets !== null) return;
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		const recordOffset = this._getRecordOffset();
+
+		this._ΔelemBlockOffsets = [];
+		let o = recordOffset;
+		for (let k = 0; k < ΔelemC; k++) {
+			this._ΔelemBlockOffsets[k] = o;
+			const tb = view.getUint8(12 + k);
+			const ic = tb >> 4;
+			const it = codeToType[ic];
+			o += count * typeToBytes[it.name];
+		}
+
+		this._elemBlockOffsets = [];
+		for (let k = 0; k < elemC; k++) {
+			this._elemBlockOffsets[k] = o;
+			const tc = view.getUint8(12 + ΔelemC + k);
+			const t = codeToType[tc];
+			o += count * typeToBytes[t.name];
+		}
+	}
+
+	/** Return the user-defined value
+	 */
+	get udv () {
+		return new DataView(this).getUint8(1);
+	}
+
+	/** Return the number of j elements in this chunk
+	 */
+	get jCount () {
+		return new DataView(this).getUint16(2, true);
+	}
+
+	/** Return the i value in this chunk
+	 */
+	get i () {
+		return new DataView(this).getUint16(4, true);
+	}
+
+	/** Return the j0 value in this chunk
+	 */
+	get j0 () {
+		return new DataView(this).getUint16(6, true);
+	}
+
+	/** Return the Δj value in this chunk
+	 */
+	get Δj () {
+		return new DataView(this).getInt16(8, true);
+	}
+
+	/** Return the Δelem_count value in this chunk
+	 */
+	get ΔelemCount () {
+		return new DataView(this).getUint8(10);
+	}
+
+	/** Return the elem_count value in this chunk
+	 */
+	get elemCount () {
+		return new DataView(this).getUint8(11);
+	}
+
+	/** Return a TypedArray (e.g., Uint16Array, …) for the n-th Δelem in the chunk
+	 */
+	Δelem (n) {
+		if (this._ΔelemCaches[n]) return this._ΔelemCaches[n];
+
+		if (n < 0 || n >= this.ΔelemCount) throw new Error(`Invalid Δelem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+
+		const typeByte = view.getUint8(12 + n);
+		const baseCode = typeByte & 0xF;
+		const incrCode = typeByte >> 4;
+		const baseType = codeToType[baseCode];
+		const incrType = codeToType[incrCode];
+		if (!baseType || !incrType) throw new Error('Invalid type codes for Δelem');
+
+		// Find offset for initial value of this Δelem
+		let initialOffset = 12 + ΔelemC + this.elemCount;
+		for (let k = 0; k < n; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			initialOffset += typeToBytes[bt.name];
+		}
+
+		let current = readTypedValue(view, initialOffset, baseType);
+
+		// Advance to start of record data (after all initials and pad)
+		const recordOffset = this._getRecordOffset();
+
+		// Find offset for deltas of this Δelem (skip previous Δelems' delta blocks)
+		this._initBlockOffsets();
+		const deltaOffset = this._ΔelemBlockOffsets[n];
+
+		// Reconstruct the array
+		const arr = new baseType(count);
+		const isBigInt = baseType === BigInt64Array || baseType === BigUint64Array;
+		arr[0] = current;
+		for (let idx = 1; idx < count; idx++) {
+			let delta = readTypedValue(view, deltaOffset + idx * typeToBytes[incrType.name], incrType);
+			if (isBigInt) {
+				delta = BigInt(delta);
+				current += delta;
+			} else {
+				current += delta;
+			}
+			arr[idx] = current;
+		}
+
+		this._ΔelemCaches[n] = arr;
+		return arr;
+	}
+
+	/** Return a TypedArray (e.g., Uint16Array, …) for the n-th elem in the chunk
+	 */
+	elem (n) {
+		if (this._elemCaches[n]) return this._elemCaches[n];
+
+		if (n < 0 || n >= this.elemCount) throw new Error(`Invalid elem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		const typeCode = view.getUint8(12 + ΔelemC + n);
+		const type = codeToType[typeCode];
+		if (!type) throw new Error('Invalid type code for elem');
+
+		// Find offset for this elem's data block
+		this._initBlockOffsets();
+		const elemOffset = this._elemBlockOffsets[n];
+
+		// Create and populate the array
+		const arr = new type(count);
+		const bytes = typeToBytes[type.name];
+		for (let idx = 0; idx < count; idx++) {
+			arr[idx] = readTypedValue(view, elemOffset + idx * bytes, type);
+		}
+
+		this._elemCaches[n] = arr;
+		return arr;
+	}
+
+	getRecord (index) {
+		if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
+
+		const arr = [this.udv, this.i, this.j0 + index * this.Δj];
+
+		for (let m = 0; m < this.ΔelemCount; m++) {
+			const values = this.Δelem(m);
+			arr.push(values[index]);
+		}
+
+		for (let m = 0; m < this.elemCount; m++) {
+			const values = this.elem(m);
+			arr.push(values[index]);
+		}
+
+		return arr;
+	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
+}
+
+
+class DougalBinaryChunkInterleaved extends ArrayBuffer {
+	constructor(buffer, offset, length) {
+		super(length);
+		new Uint8Array(this).set(new Uint8Array(buffer, offset, length));
+		this._incrStrides = [];
+		this._elemStrides = [];
+		this._incrOffsets = [];
+		this._elemOffsets = [];
+		this._recordStride = 0;
+		this._recordOffset = null;
+		this._initStrides();
+		this._ΔelemCaches = new Array(this.ΔelemCount);
+		this._elemCaches = new Array(this.elemCount);
+	}
+
+	_getRecordOffset() {
+		if (this._recordOffset !== null) return this._recordOffset;
+		const view = new DataView(this);
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		let recordOffset = 12 + ΔelemC + elemC;
+		for (let k = 0; k < ΔelemC; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			recordOffset += typeToBytes[bt.name];
+		}
+		while (recordOffset % 4 !== 0) recordOffset++;
+		this._recordOffset = recordOffset;
+		return recordOffset;
+	}
+
+	_initStrides() {
+		const view = new DataView(this);
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		// Compute incr strides and offsets
+		let incrOffset = 0;
+		for (let k = 0; k < ΔelemC; k++) {
+			const typeByte = view.getUint8(12 + k);
+			const incrCode = typeByte >> 4;
+			const incrType = codeToType[incrCode];
+			if (!incrType) throw new Error('Invalid incr type code');
+			this._incrOffsets.push(incrOffset);
+			const bytes = typeToBytes[incrType.name];
+			this._incrStrides.push(bytes);
+			incrOffset += bytes;
+			this._recordStride += bytes;
+		}
+
+		// Compute elem strides and offsets
+		let elemOffset = incrOffset;
+		for (let k = 0; k < elemC; k++) {
+			const typeCode = view.getUint8(12 + ΔelemC + k);
+			const type = codeToType[typeCode];
+			if (!type) throw new Error('Invalid elem type code');
+			this._elemOffsets.push(elemOffset);
+			const bytes = typeToBytes[type.name];
+			this._elemStrides.push(bytes);
+			elemOffset += bytes;
+			this._recordStride += bytes;
+		}
+	}
+
+	get udv() {
+		return new DataView(this).getUint8(1);
+	}
+
+	get jCount() {
+		return new DataView(this).getUint16(2, true);
+	}
+
+	get i() {
+		return new DataView(this).getUint16(4, true);
+	}
+
+	get j0() {
+		return new DataView(this).getUint16(6, true);
+	}
+
+	get Δj() {
+		return new DataView(this).getInt16(8, true);
+	}
+
+	get ΔelemCount() {
+		return new DataView(this).getUint8(10);
+	}
+
+	get elemCount() {
+		return new DataView(this).getUint8(11);
+	}
+
+	Δelem(n) {
+		if (this._ΔelemCaches[n]) return this._ΔelemCaches[n];
+
+		if (n < 0 || n >= this.ΔelemCount) throw new Error(`Invalid Δelem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+
+		const typeByte = view.getUint8(12 + n);
+		const baseCode = typeByte & 0xF;
+		const incrCode = typeByte >> 4;
+		const baseType = codeToType[baseCode];
+		const incrType = codeToType[incrCode];
+		if (!baseType || !incrType) throw new Error('Invalid type codes for Δelem');
+
+		// Find offset for initial value of this Δelem
+		let initialOffset = 12 + ΔelemC + this.elemCount;
+		for (let k = 0; k < n; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			initialOffset += typeToBytes[bt.name];
+		}
+
+		let current = readTypedValue(view, initialOffset, baseType);
+
+		// Find offset to start of record data
+		const recordOffset = this._getRecordOffset();
+
+		// Use precomputed offset for this Δelem
+		const deltaOffset = recordOffset + this._incrOffsets[n];
+
+		// Reconstruct the array
+		const arr = new baseType(count);
+		const isBigInt = baseType === BigInt64Array || baseType === BigUint64Array;
+		arr[0] = current;
+		for (let idx = 1; idx < count; idx++) {
+			let delta = readTypedValue(view, deltaOffset + idx * this._recordStride, incrType);
+			if (isBigInt) {
+				delta = BigInt(delta);
+				current += delta;
+			} else {
+				current += delta;
+			}
+			arr[idx] = current;
+		}
+
+		this._ΔelemCaches[n] = arr;
+		return arr;
+	}
+
+	elem(n) {
+		if (this._elemCaches[n]) return this._elemCaches[n];
+
+		if (n < 0 || n >= this.elemCount) throw new Error(`Invalid elem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+
+		const typeCode = view.getUint8(12 + ΔelemC + n);
+		const type = codeToType[typeCode];
+		if (!type) throw new Error('Invalid type code for elem');
+
+		// Find offset to start of record data
+		const recordOffset = this._getRecordOffset();
+
+		// Use precomputed offset for this elem (relative to start of record data)
+		const elemOffset = recordOffset + this._elemOffsets[n];
+
+		// Create and populate the array
+		const arr = new type(count);
+		const bytes = typeToBytes[type.name];
+		for (let idx = 0; idx < count; idx++) {
+			arr[idx] = readTypedValue(view, elemOffset + idx * this._recordStride, type);
+		}
+
+		this._elemCaches[n] = arr;
+		return arr;
+	}
+
+	getRecord (index) {
+		if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
+
+		const arr = [this.udv, this.i, this.j0 + index * this.Δj];
+
+		for (let m = 0; m < this.ΔelemCount; m++) {
+			const values = this.Δelem(m);
+			arr.push(values[index]);
+		}
+
+		for (let m = 0; m < this.elemCount; m++) {
+			const values = this.elem(m);
+			arr.push(values[index]);
+		}
+
+		return arr;
+	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
+}
+
+
+module.exports = { DougalBinaryBundle, DougalBinaryChunkSequential, DougalBinaryChunkInterleaved }

lib/modules/@dougal/binary/decode.js

@@ -0,0 +1,327 @@
+const codeToType = {
+	0: Int8Array,
+	1: Uint8Array,
+	2: Int16Array,
+	3: Uint16Array,
+	4: Int32Array,
+	5: Uint32Array,
+	7: Float32Array,
+	8: Float64Array,
+	9: BigInt64Array,
+	10: BigUint64Array
+};
+
+const typeToBytes = {
+	Int8Array: 1,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 2,
+	Int32Array: 4,
+	Uint32Array: 4,
+	Float32Array: 4,
+	Float64Array: 8,
+	BigInt64Array: 8,
+	BigUint64Array: 8
+};
+
+function sequential(binary) {
+	if (!(binary instanceof Uint8Array) || binary.length < 4) {
+		throw new Error('Invalid binary input');
+	}
+
+	const view = new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
+	let offset = 0;
+
+	// Initialize result (assuming single i value for simplicity; extend for multiple i values if needed)
+	const result = { i: null, j: [], Δelems: [], elems: [] };
+
+	// Process bundles
+	while (offset < binary.length) {
+		// Read bundle header
+		if (offset + 4 > binary.length) throw new Error('Incomplete bundle header');
+
+		const bundleHeader = view.getUint32(offset, true);
+		if ((bundleHeader & 0xFF) !== 0x1C) throw new Error('Invalid bundle marker');
+		const bundleLength = bundleHeader >> 8;
+		offset += 4;
+		const bundleEnd = offset + bundleLength;
+
+		if (bundleEnd > binary.length) throw new Error('Bundle length exceeds input size');
+
+		// Process chunks in bundle
+		while (offset < bundleEnd) {
+			// Read chunk header
+			if (offset + 12 > bundleEnd) throw new Error('Incomplete chunk header');
+			const chunkType = view.getUint8(offset);
+			if (chunkType !== 0x11) throw new Error(`Unsupported chunk type: ${chunkType}`);
+			offset += 1; // Skip udv
+			offset += 1;
+			const count = view.getUint16(offset, true); offset += 2;
+			if (count > 65535) throw new Error('Chunk count exceeds 65535');
+			const iValue = view.getUint16(offset, true); offset += 2;
+			const j0 = view.getUint16(offset, true); offset += 2;
+			const Δj = view.getInt16(offset, true); offset += 2;
+			const ΔelemCount = view.getUint8(offset++); // Δelem_count
+			const elemCount = view.getUint8(offset++); // elem_count
+
+			// Set i value (assuming all chunks share the same i)
+			if (result.i === null) result.i = iValue;
+			else if (result.i !== iValue) throw new Error('Multiple i values not supported');
+
+			// Read preface (element types)
+			const ΔelemTypes = [];
+			for (let i = 0; i < ΔelemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete Δelem types');
+				const typeByte = view.getUint8(offset++);
+				const baseCode = typeByte & 0x0F;
+				const incrCode = typeByte >> 4;
+				if (!codeToType[baseCode] || !codeToType[incrCode]) {
+					throw new Error(`Invalid type code in Δelem: ${typeByte}`);
+				}
+				ΔelemTypes.push({ baseType: codeToType[baseCode], incrType: codeToType[incrCode] });
+			}
+			const elemTypes = [];
+			for (let i = 0; i < elemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete elem types');
+				const typeCode = view.getUint8(offset++);
+				if (!codeToType[typeCode]) throw new Error(`Invalid type code in elem: ${typeCode}`);
+				elemTypes.push(codeToType[typeCode]);
+			}
+
+			// Initialize Δelems and elems arrays if first chunk
+			if (!result.Δelems.length && ΔelemCount > 0) {
+				result.Δelems = Array(ΔelemCount).fill().map(() => []);
+			}
+			if (!result.elems.length && elemCount > 0) {
+				result.elems = Array(elemCount).fill().map(() => []);
+			}
+
+			// Read initial values for Δelems
+			const initialValues = [];
+			for (const { baseType } of ΔelemTypes) {
+				if (offset + typeToBytes[baseType.name] > bundleEnd) {
+					throw new Error('Incomplete initial values');
+				}
+				initialValues.push(readTypedValue(view, offset, baseType));
+				offset += typeToBytes[baseType.name];
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after initial values');
+				offset++;
+			}
+
+			// Reconstruct j values
+			for (let idx = 0; idx < count; idx++) {
+				result.j.push(j0 + idx * Δj);
+			}
+
+			// Read record data (non-interleaved)
+			for (let i = 0; i < ΔelemCount; i++) {
+				let current = initialValues[i];
+				const values = result.Δelems[i];
+				const incrType = ΔelemTypes[i].incrType;
+				const isBigInt = typeof current === 'bigint';
+				for (let idx = 0; idx < count; idx++) {
+					if (offset + typeToBytes[incrType.name] > bundleEnd) {
+						throw new Error('Incomplete Δelem data');
+					}
+					let delta = readTypedValue(view, offset, incrType);
+					if (idx === 0) {
+						values.push(isBigInt ? Number(current) : current);
+					} else {
+						if (isBigInt) {
+							delta = BigInt(delta);
+							current += delta;
+							values.push(Number(current));
+						} else {
+							current += delta;
+							values.push(current);
+						}
+					}
+					offset += typeToBytes[incrType.name];
+				}
+			}
+			for (let i = 0; i < elemCount; i++) {
+				const values = result.elems[i];
+				const type = elemTypes[i];
+				const isBigInt = type === BigInt64Array || type === BigUint64Array;
+				for (let idx = 0; idx < count; idx++) {
+					if (offset + typeToBytes[type.name] > bundleEnd) {
+						throw new Error('Incomplete elem data');
+					}
+					let value = readTypedValue(view, offset, type);
+					values.push(isBigInt ? Number(value) : value);
+					offset += typeToBytes[type.name];
+				}
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after record data');
+				offset++;
+			}
+		}
+	}
+
+	return result;
+}
+
+
+function interleaved(binary) {
+	if (!(binary instanceof Uint8Array) || binary.length < 4) {
+		throw new Error('Invalid binary input');
+	}
+
+	const view = new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
+	let offset = 0;
+
+	// Initialize result (assuming single i value for simplicity; extend for multiple i values if needed)
+	const result = { i: null, j: [], Δelems: [], elems: [] };
+
+	// Process bundles
+	while (offset < binary.length) {
+		// Read bundle header
+		if (offset + 4 > binary.length) throw new Error('Incomplete bundle header');
+
+		const bundleHeader = view.getUint32(offset, true);
+		if ((bundleHeader & 0xFF) !== 0x1C) throw new Error('Invalid bundle marker');
+		const bundleLength = bundleHeader >> 8;
+		offset += 4;
+		const bundleEnd = offset + bundleLength;
+
+		if (bundleEnd > binary.length) throw new Error('Bundle length exceeds input size');
+
+		// Process chunks in bundle
+		while (offset < bundleEnd) {
+			// Read chunk header
+			if (offset + 12 > bundleEnd) throw new Error('Incomplete chunk header');
+			const chunkType = view.getUint8(offset);
+			if (chunkType !== 0x12) throw new Error(`Unsupported chunk type: ${chunkType}`);
+			offset += 1; // Skip udv
+			offset += 1;
+			const count = view.getUint16(offset, true); offset += 2;
+			if (count > 65535) throw new Error('Chunk count exceeds 65535');
+			const iValue = view.getUint16(offset, true); offset += 2;
+			const j0 = view.getUint16(offset, true); offset += 2;
+			const Δj = view.getInt16(offset, true); offset += 2;
+			const ΔelemCount = view.getUint8(offset++); // Δelem_count
+			const elemCount = view.getUint8(offset++); // elem_count
+
+			// Set i value (assuming all chunks share the same i)
+			if (result.i === null) result.i = iValue;
+			else if (result.i !== iValue) throw new Error('Multiple i values not supported');
+
+			// Read preface (element types)
+			const ΔelemTypes = [];
+			for (let i = 0; i < ΔelemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete Δelem types');
+				const typeByte = view.getUint8(offset++);
+				const baseCode = typeByte & 0x0F;
+				const incrCode = typeByte >> 4;
+				if (!codeToType[baseCode] || !codeToType[incrCode]) {
+					throw new Error(`Invalid type code in Δelem: ${typeByte}`);
+				}
+				ΔelemTypes.push({ baseType: codeToType[baseCode], incrType: codeToType[incrCode] });
+			}
+			const elemTypes = [];
+			for (let i = 0; i < elemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete elem types');
+				const typeCode = view.getUint8(offset++);
+				if (!codeToType[typeCode]) throw new Error(`Invalid type code in elem: ${typeCode}`);
+				elemTypes.push(codeToType[typeCode]);
+			}
+
+			// Initialize Δelems and elems arrays if first chunk
+			if (!result.Δelems.length && ΔelemCount > 0) {
+				result.Δelems = Array(ΔelemCount).fill().map(() => []);
+			}
+			if (!result.elems.length && elemCount > 0) {
+				result.elems = Array(elemCount).fill().map(() => []);
+			}
+
+			// Read initial values for Δelems
+			const initialValues = [];
+			for (const { baseType } of ΔelemTypes) {
+				if (offset + typeToBytes[baseType.name] > bundleEnd) {
+					throw new Error('Incomplete initial values');
+				}
+				initialValues.push(readTypedValue(view, offset, baseType));
+				offset += typeToBytes[baseType.name];
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after initial values');
+				offset++;
+			}
+
+			// Reconstruct j values
+			for (let idx = 0; idx < count; idx++) {
+				result.j.push(j0 + idx * Δj);
+			}
+
+			// Read interleaved record data
+			for (let idx = 0; idx < count; idx++) {
+				// Read Δelems
+				for (let i = 0; i < ΔelemCount; i++) {
+					const values = result.Δelems[i];
+					const incrType = ΔelemTypes[i].incrType;
+					const isBigInt = typeof initialValues[i] === 'bigint';
+					if (offset + typeToBytes[incrType.name] > bundleEnd) {
+						throw new Error('Incomplete Δelem data');
+					}
+					let delta = readTypedValue(view, offset, incrType);
+					offset += typeToBytes[incrType.name];
+					if (idx === 0) {
+						values.push(isBigInt ? Number(initialValues[i]) : initialValues[i]);
+					} else {
+						if (isBigInt) {
+							delta = BigInt(delta);
+							initialValues[i] += delta;
+							values.push(Number(initialValues[i]));
+						} else {
+							initialValues[i] += delta;
+							values.push(initialValues[i]);
+						}
+					}
+				}
+				// Read elems
+				for (let i = 0; i < elemCount; i++) {
+					const values = result.elems[i];
+					const type = elemTypes[i];
+					const isBigInt = type === BigInt64Array || type === BigUint64Array;
+					if (offset + typeToBytes[type.name] > bundleEnd) {
+						throw new Error('Incomplete elem data');
+					}
+					let value = readTypedValue(view, offset, type);
+					values.push(isBigInt ? Number(value) : value);
+					offset += typeToBytes[type.name];
+				}
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after record data');
+				offset++;
+			}
+		}
+	}
+
+	return result;
+}
+
+function readTypedValue(view, offset, type) {
+	switch (type) {
+		case Int8Array: return view.getInt8(offset);
+		case Uint8Array: return view.getUint8(offset);
+		case Int16Array: return view.getInt16(offset, true);
+		case Uint16Array: return view.getUint16(offset, true);
+		case Int32Array: return view.getInt32(offset, true);
+		case Uint32Array: return view.getUint32(offset, true);
+		case Float32Array: return view.getFloat32(offset, true);
+		case Float64Array: return view.getFloat64(offset, true);
+		case BigInt64Array: return view.getBigInt64(offset, true);
+		case BigUint64Array: return view.getBigUint64(offset, true);
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+module.exports = { sequential, interleaved };

lib/modules/@dougal/binary/encode.js

@@ -0,0 +1,380 @@
+const typeToCode = {
+	Int8Array: 0,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 3,
+	Int32Array: 4,
+	Uint32Array: 5,
+	Float32Array: 7, // Float16 not natively supported in JS, use Float32
+	Float64Array: 8,
+	BigInt64Array: 9,
+	BigUint64Array: 10
+};
+
+const typeToBytes = {
+	Int8Array: 1,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 2,
+	Int32Array: 4,
+	Uint32Array: 4,
+	Float32Array: 4,
+	Float64Array: 8,
+	BigInt64Array: 8,
+	BigUint64Array: 8
+};
+
+function sequential(json, iGetter, jGetter, Δelems = [], elems = [], udv = 0) {
+	if (!Array.isArray(json) || !json.length) return new Uint8Array(0);
+	if (typeof iGetter !== 'function' || typeof jGetter !== 'function') throw new Error('i and j must be getter functions');
+	Δelems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`Δelems[${idx}].key must be a getter function`);
+	});
+	elems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`elems[${idx}].key must be a getter function`);
+	});
+
+	// Group records by i value
+	const groups = new Map();
+	for (const record of json) {
+		const iValue = iGetter(record);
+		if (iValue == null) throw new Error('Missing i value from getter');
+		if (!groups.has(iValue)) groups.set(iValue, []);
+		groups.get(iValue).push(record);
+	}
+
+	const maxBundleSize = 0xFFFFFF; // Max bundle length (24 bits)
+	const buffers = [];
+
+	// Process each group (i value)
+	for (const [iValue, records] of groups) {
+		// Sort records by j to ensure consistent order
+		records.sort((a, b) => jGetter(a) - jGetter(b));
+		const jValues = records.map(jGetter);
+		if (jValues.some(v => v == null)) throw new Error('Missing j value from getter');
+
+		// Split records into chunks based on Δj continuity
+		const chunks = [];
+		let currentChunk = [records[0]];
+		let currentJ0 = jValues[0];
+		let currentΔj = records.length > 1 ? jValues[1] - jValues[0] : 0;
+
+		for (let idx = 1; idx < records.length; idx++) {
+			const chunkIndex = chunks.reduce((sum, c) => sum + c.records.length, 0);
+			const expectedJ = currentJ0 + (idx - chunkIndex) * currentΔj;
+			if (jValues[idx] !== expectedJ || idx - chunkIndex >= 65536) {
+				chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+				currentChunk = [records[idx]];
+				currentJ0 = jValues[idx];
+				currentΔj = idx + 1 < records.length ? jValues[idx + 1] - jValues[idx] : 0;
+			} else {
+				currentChunk.push(records[idx]);
+			}
+		}
+		if (currentChunk.length > 0) {
+			chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+		}
+
+		// Calculate total size for all chunks in this group by simulating offsets
+		const chunkSizes = chunks.map(({ records: chunkRecords }) => {
+			if (chunkRecords.length > 65535) throw new Error(`Chunk size exceeds 65535 for i=${iValue}`);
+			let simulatedOffset = 0; // Relative to chunk start
+			simulatedOffset += 12; // Header
+			simulatedOffset += Δelems.length + elems.length; // Preface
+			simulatedOffset += Δelems.reduce((sum, e) => sum + typeToBytes[e.baseType.name], 0); // Initial values
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after initial
+			simulatedOffset += chunkRecords.length * (
+				Δelems.reduce((sum, e) => sum + typeToBytes[e.incrType.name], 0) +
+				elems.reduce((sum, e) => sum + typeToBytes[e.type.name], 0)
+			); // Record data
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after record
+			return simulatedOffset;
+		});
+		const totalChunkSize = chunkSizes.reduce((sum, size) => sum + size, 0);
+
+		// Start a new bundle if needed
+		const lastBundle = buffers[buffers.length - 1];
+		if (!lastBundle || lastBundle.offset + totalChunkSize > maxBundleSize) {
+			buffers.push({ offset: 4, buffer: null, view: null });
+		}
+
+		// Initialize DataView for current bundle
+		const currentBundle = buffers[buffers.length - 1];
+		if (!currentBundle.buffer) {
+			const requiredSize = totalChunkSize + 4;
+			currentBundle.buffer = new ArrayBuffer(requiredSize);
+			currentBundle.view = new DataView(currentBundle.buffer);
+		}
+
+		// Process each chunk
+		for (const { records: chunkRecords, j0, Δj } of chunks) {
+			const chunkSize = chunkSizes.shift();
+
+			// Ensure buffer is large enough
+			if (currentBundle.offset + chunkSize > currentBundle.buffer.byteLength) {
+				const newSize = currentBundle.offset + chunkSize;
+				const newBuffer = new ArrayBuffer(newSize);
+				new Uint8Array(newBuffer).set(new Uint8Array(currentBundle.buffer));
+				currentBundle.buffer = newBuffer;
+				currentBundle.view = new DataView(newBuffer);
+			}
+
+			// Write chunk header
+			let offset = currentBundle.offset;
+			currentBundle.view.setUint8(offset++, 0x11); // Chunk type
+			currentBundle.view.setUint8(offset++, udv); // udv
+			currentBundle.view.setUint16(offset, chunkRecords.length, true); offset += 2; // count
+			currentBundle.view.setUint16(offset, iValue, true); offset += 2; // i
+			currentBundle.view.setUint16(offset, j0, true); offset += 2; // j0
+			currentBundle.view.setInt16(offset, Δj, true); offset += 2; // Δj
+			currentBundle.view.setUint8(offset++, Δelems.length); // Δelem_count
+			currentBundle.view.setUint8(offset++, elems.length); // elem_count
+
+			// Write chunk preface (element types)
+			for (const elem of Δelems) {
+				const baseCode = typeToCode[elem.baseType.name];
+				const incrCode = typeToCode[elem.incrType.name];
+				currentBundle.view.setUint8(offset++, (incrCode << 4) | baseCode);
+			}
+			for (const elem of elems) {
+				currentBundle.view.setUint8(offset++, typeToCode[elem.type.name]);
+			}
+
+			// Write initial values for Δelems
+			for (const elem of Δelems) {
+				const value = elem.key(chunkRecords[0]);
+				if (value == null) throw new Error('Missing Δelem value from getter');
+				writeTypedValue(currentBundle.view, offset, value, elem.baseType);
+				offset += typeToBytes[elem.baseType.name];
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Write record data (non-interleaved)
+			for (const elem of Δelems) {
+				let prev = elem.key(chunkRecords[0]);
+				for (let idx = 0; idx < chunkRecords.length; idx++) {
+					const value = idx === 0 ? 0 : elem.key(chunkRecords[idx]) - prev;
+					writeTypedValue(currentBundle.view, offset, value, elem.incrType);
+					offset += typeToBytes[elem.incrType.name];
+					prev = elem.key(chunkRecords[idx]);
+				}
+			}
+			for (const elem of elems) {
+				for (const record of chunkRecords) {
+					const value = elem.key(record);
+					if (value == null) throw new Error('Missing elem value from getter');
+					writeTypedValue(currentBundle.view, offset, value, elem.type);
+					offset += typeToBytes[elem.type.name];
+				}
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Update bundle offset
+			currentBundle.offset = offset;
+		}
+
+		// Update bundle header
+		currentBundle.view.setUint32(0, 0x1C | ((currentBundle.offset - 4) << 8), true);
+	}
+
+	// Combine buffers into final Uint8Array
+	const finalLength = buffers.reduce((sum, b) => sum + b.offset, 0);
+	const result = new Uint8Array(finalLength);
+	let offset = 0;
+	for (const { buffer, offset: bundleOffset } of buffers) {
+		result.set(new Uint8Array(buffer, 0, bundleOffset), offset);
+		offset += bundleOffset;
+	}
+
+	return result;
+}
+
+
+function interleaved(json, iGetter, jGetter, Δelems = [], elems = [], udv = 0) {
+	if (!Array.isArray(json) || !json.length) return new Uint8Array(0);
+	if (typeof iGetter !== 'function' || typeof jGetter !== 'function') throw new Error('i and j must be getter functions');
+	Δelems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`Δelems[${idx}].key must be a getter function`);
+	});
+	elems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`elems[${idx}].key must be a getter function`);
+	});
+
+	// Group records by i value
+	const groups = new Map();
+	for (const record of json) {
+		const iValue = iGetter(record);
+		if (iValue == null) throw new Error('Missing i value from getter');
+		if (!groups.has(iValue)) groups.set(iValue, []);
+		groups.get(iValue).push(record);
+	}
+
+	const maxBundleSize = 0xFFFFFF; // Max bundle length (24 bits)
+	const buffers = [];
+
+	// Process each group (i value)
+	for (const [iValue, records] of groups) {
+		// Sort records by j to ensure consistent order
+		records.sort((a, b) => jGetter(a) - jGetter(b));
+		const jValues = records.map(jGetter);
+		if (jValues.some(v => v == null)) throw new Error('Missing j value from getter');
+
+		// Split records into chunks based on Δj continuity
+		const chunks = [];
+		let currentChunk = [records[0]];
+		let currentJ0 = jValues[0];
+		let currentΔj = records.length > 1 ? jValues[1] - jValues[0] : 0;
+
+		for (let idx = 1; idx < records.length; idx++) {
+			const chunkIndex = chunks.reduce((sum, c) => sum + c.records.length, 0);
+			const expectedJ = currentJ0 + (idx - chunkIndex) * currentΔj;
+			if (jValues[idx] !== expectedJ || idx - chunkIndex >= 65536) {
+				chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+				currentChunk = [records[idx]];
+				currentJ0 = jValues[idx];
+				currentΔj = idx + 1 < records.length ? jValues[idx + 1] - jValues[idx] : 0;
+			} else {
+				currentChunk.push(records[idx]);
+			}
+		}
+		if (currentChunk.length > 0) {
+			chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+		}
+
+		// Calculate total size for all chunks in this group by simulating offsets
+		const chunkSizes = chunks.map(({ records: chunkRecords }) => {
+			if (chunkRecords.length > 65535) throw new Error(`Chunk size exceeds 65535 for i=${iValue}`);
+			let simulatedOffset = 0; // Relative to chunk start
+			simulatedOffset += 12; // Header
+			simulatedOffset += Δelems.length + elems.length; // Preface
+			simulatedOffset += Δelems.reduce((sum, e) => sum + typeToBytes[e.baseType.name], 0); // Initial values
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after initial
+			simulatedOffset += chunkRecords.length * (
+				Δelems.reduce((sum, e) => sum + typeToBytes[e.incrType.name], 0) +
+				elems.reduce((sum, e) => sum + typeToBytes[e.type.name], 0)
+			); // Interleaved record data
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after record
+			return simulatedOffset;
+		});
+		const totalChunkSize = chunkSizes.reduce((sum, size) => sum + size, 0);
+
+		// Start a new bundle if needed
+		const lastBundle = buffers[buffers.length - 1];
+		if (!lastBundle || lastBundle.offset + totalChunkSize > maxBundleSize) {
+			buffers.push({ offset: 4, buffer: null, view: null });
+		}
+
+		// Initialize DataView for current bundle
+		const currentBundle = buffers[buffers.length - 1];
+		if (!currentBundle.buffer) {
+			const requiredSize = totalChunkSize + 4;
+			currentBundle.buffer = new ArrayBuffer(requiredSize);
+			currentBundle.view = new DataView(currentBundle.buffer);
+		}
+
+		// Process each chunk
+		for (const { records: chunkRecords, j0, Δj } of chunks) {
+			const chunkSize = chunkSizes.shift();
+
+			// Ensure buffer is large enough
+			if (currentBundle.offset + chunkSize > currentBundle.buffer.byteLength) {
+				const newSize = currentBundle.offset + chunkSize;
+				const newBuffer = new ArrayBuffer(newSize);
+				new Uint8Array(newBuffer).set(new Uint8Array(currentBundle.buffer));
+				currentBundle.buffer = newBuffer;
+				currentBundle.view = new DataView(newBuffer);
+			}
+
+			// Write chunk header
+			let offset = currentBundle.offset;
+			currentBundle.view.setUint8(offset++, 0x12); // Chunk type
+			currentBundle.view.setUint8(offset++, udv); // udv
+			currentBundle.view.setUint16(offset, chunkRecords.length, true); offset += 2; // count
+			currentBundle.view.setUint16(offset, iValue, true); offset += 2; // i
+			currentBundle.view.setUint16(offset, j0, true); offset += 2; // j0
+			currentBundle.view.setInt16(offset, Δj, true); offset += 2; // Δj
+			currentBundle.view.setUint8(offset++, Δelems.length); // Δelem_count
+			currentBundle.view.setUint8(offset++, elems.length); // elem_count
+
+			// Write chunk preface (element types)
+			for (const elem of Δelems) {
+				const baseCode = typeToCode[elem.baseType.name];
+				const incrCode = typeToCode[elem.incrType.name];
+				currentBundle.view.setUint8(offset++, (incrCode << 4) | baseCode);
+			}
+			for (const elem of elems) {
+				currentBundle.view.setUint8(offset++, typeToCode[elem.type.name]);
+			}
+
+			// Write initial values for Δelems
+			for (const elem of Δelems) {
+				const value = elem.key(chunkRecords[0]);
+				if (value == null) throw new Error('Missing Δelem value from getter');
+				writeTypedValue(currentBundle.view, offset, value, elem.baseType);
+				offset += typeToBytes[elem.baseType.name];
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Write interleaved record data
+			const prevValues = Δelems.map(elem => elem.key(chunkRecords[0]));
+			for (let idx = 0; idx < chunkRecords.length; idx++) {
+				// Write Δelems increments
+				for (let i = 0; i < Δelems.length; i++) {
+					const elem = Δelems[i];
+					const value = idx === 0 ? 0 : elem.key(chunkRecords[idx]) - prevValues[i];
+					writeTypedValue(currentBundle.view, offset, value, elem.incrType);
+					offset += typeToBytes[elem.incrType.name];
+					prevValues[i] = elem.key(chunkRecords[idx]);
+				}
+				// Write elems
+				for (const elem of elems) {
+					const value = elem.key(chunkRecords[idx]);
+					if (value == null) throw new Error('Missing elem value from getter');
+					writeTypedValue(currentBundle.view, offset, value, elem.type);
+					offset += typeToBytes[elem.type.name];
+				}
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Update bundle offset
+			currentBundle.offset = offset;
+		}
+
+		// Update bundle header
+		currentBundle.view.setUint32(0, 0x1C | ((currentBundle.offset - 4) << 8), true);
+	}
+
+	// Combine buffers into final Uint8Array
+	const finalLength = buffers.reduce((sum, b) => sum + b.offset, 0);
+	const result = new Uint8Array(finalLength);
+	let offset = 0;
+	for (const { buffer, offset: bundleOffset } of buffers) {
+		result.set(new Uint8Array(buffer, 0, bundleOffset), offset);
+		offset += bundleOffset;
+	}
+
+	return result;
+}
+
+
+function writeTypedValue(view, offset, value, type) {
+	switch (type) {
+		case Int8Array: view.setInt8(offset, value); break;
+		case Uint8Array: view.setUint8(offset, value); break;
+		case Int16Array: view.setInt16(offset, value, true); break;
+		case Uint16Array: view.setUint16(offset, value, true); break;
+		case Int32Array: view.setInt32(offset, value, true); break;
+		case Uint32Array: view.setUint32(offset, value, true); break;
+		case Float32Array: view.setFloat32(offset, value, true); break;
+		case Float64Array: view.setFloat64(offset, value, true); break;
+		case BigInt64Array: view.setBigInt64(offset, BigInt(value), true); break;
+		case BigUint64Array: view.setBigUint64(offset, BigInt(value), true); break;
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+module.exports = { sequential, interleaved };

lib/modules/@dougal/binary/index.js

@@ -0,0 +1,139 @@
+
+/** Binary encoder
+ *
+ * This module encodes scalar data from a grid-like source
+ * into a packed binary format for bandwidth efficiency and
+ * speed of access.
+ *
+ * Data are indexed by i & j values, with "i" being constant
+ * (e.g., a sequence or line number) and "j" expected to change
+ * by a constant, linear amount (e.g., point numbers). All data
+ * from consecutive "j" values will be encoded as a single array
+ * (or series of arrays if multiple values are encoded).
+ * If there is a jump in the "j" progression, a new "chunk" will
+ * be started with a new array (or series of arrays).
+ *
+ * Multiple values may be encoded per (i, j) pair, using any of
+ * the types supported by JavaScript's TypedArray except for
+ * Float16 and Uint8Clamped. Each variable can be encoded with
+ * a different size.
+ *
+ * Values may be encoded directly or as deltas from an initial
+ * value. The latter is particularly efficient when dealing with
+ * monotonically incrementing data, such as timestamps.
+ *
+ * The conceptual packet format for sequentially encoded data
+ * looks like this:
+ *
+ * <msg-type> <count: x> <i> <j0> <Δj>
+ *
+ * <Δelement_count: y>
+ * <element_count: z>
+ *
+ * <Δelement_1_type_base> … <Δelement_y_type_base>
+ * <Δelement_1_type_incr> … <Δelement_y_type_incr>
+ * <elem_1_type> … <elem_z_type>
+ *
+ * <Δelement_1_first> … <Δelement_z_first>
+ *
+ * <Δelem_1_0> … <Δelem_1_x>
+ * …
+ * <Δelem_y_0> … <Δelem_y_x>
+ * <elem_1_0> … <elem_1_x>
+ * …
+ * <elem_z_0> … <elem_z_x>
+ *
+ *
+ * The conceptual packet format for interleaved encoded data
+ * looks like this:
+ *
+ *
+ * <msg-type> <count: x> <i> <j0> <Δj>
+ *
+ * <Δelement_count: y>
+ * <element_count: z>
+ *
+ * <Δelement_1_type_base> … <Δelement_y_type_base>
+ * <Δelement_1_type_incr> … <Δelement_y_type_incr>
+ * <elem_1_type> … <elem_z_type>
+ *
+ * <Δelement_1_first> … <Δelement_y_first>
+ *
+ * <Δelem_1_0> <Δelem_2_0> … <Δelem_y_0> <elem_1_0> <elem_2_0> … <elem_z_0>
+ * <Δelem_1_1> <Δelem_2_1> … <Δelem_y_1> <elem_1_1> <elem_2_1> … <elem_z_1>
+ * …
+ * <Δelem_1_x> <Δelem_2_x> … <Δelem_y_x> <elem_1_x> <elem_2_x> … <elem_z_x>
+ *
+ *
+ * Usage example:
+ *
+ * json = [
+ *   {
+ *     sequence: 7,
+ *     sailline: 5354,
+ *     line: 5356,
+ *     point: 1068,
+ *     tstamp: 1695448704372,
+ *     objrefraw: 3,
+ *     objreffinal: 4
+ *   },
+ *   {
+ *     sequence: 7,
+ *     sailline: 5354,
+ *     line: 5352,
+ *     point: 1070,
+ *     tstamp: 1695448693612,
+ *     objrefraw: 2,
+ *     objreffinal: 3
+ *   },
+ *   {
+ *     sequence: 7,
+ *     sailline: 5354,
+ *     line: 5356,
+ *     point: 1072,
+ *     tstamp: 1695448684624,
+ *     objrefraw: 3,
+ *     objreffinal: 4
+ *   }
+ * ];
+ *
+ * deltas = [
+ *   { key: el => el.tstamp, baseType: BigUint64Array, incrType: Int16Array }
+ * ];
+ *
+ * elems = [
+ *   { key: el => el.objrefraw, type: Uint8Array },
+ *   { key: el => el.objreffinal, type: Uint8Array }
+ * ];
+ *
+ * i = el => el.sequence;
+ *
+ * j = el => el.point;
+ *
+ * bundle = encode(json, i, j, deltas, elems);
+ *
+ * // bundle:
+ *
+ * Uint8Array(40) [
+ *    36,   0,   0,  28, 17, 0,  3, 0, 7,   0,
+ *    44,   4,   2,   0,  1, 2, 42, 1, 1, 116,
+ *    37, 158, 192, 138,  1, 0,  0, 0, 0,   0,
+ *   248, 213, 228, 220,  3, 2,  3, 4, 3,   4
+ * ]
+ *
+ * decode(bundle);
+ *
+ * {
+ *   i: 7,
+ *   j: [ 1068, 1070, 1072 ],
+ *   'Δelems': [ [ 1695448704372, 1695448693612, 1695448684624 ] ],
+ *   elems: [ [ 3, 2, 3 ], [ 4, 3, 4 ] ]
+ * }
+ *
+ */
+
+module.exports = {
+	encode: {...require('./encode')},
+	decode: {...require('./decode')},
+	...require('./classes')
+};

lib/modules/@dougal/binary/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@dougal/binary",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": ""
+}

lib/modules/@dougal/concurrency/index.js

@@ -0,0 +1,25 @@
+class ConcurrencyLimiter {
+
+  constructor(maxConcurrent) {
+    this.maxConcurrent = maxConcurrent;
+    this.active = 0;
+    this.queue = [];
+  }
+
+  async enqueue(task) {
+    if (this.active >= this.maxConcurrent) {
+      await new Promise(resolve => this.queue.push(resolve));
+    }
+    this.active++;
+    try {
+      return await task();
+    } finally {
+      this.active--;
+      if (this.queue.length > 0) {
+        this.queue.shift()();
+      }
+    }
+  }
+}
+
+module.exports = ConcurrencyLimiter;

lib/modules/@dougal/concurrency/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@dougal/concurrency",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": ""
+}

lib/modules/@dougal/user/package-lock.json

@@ -1,24 +0,0 @@
-{
-  "name": "@dougal/user",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "@dougal/user",
-      "version": "1.0.0",
-      "license": "ISC",
-      "dependencies": {
-        "@dougal/organisations": "file:../organisations"
-      }
-    },
-    "../organisations": {
-      "version": "1.0.0",
-      "license": "ISC"
-    },
-    "node_modules/@dougal/organisations": {
-      "resolved": "../organisations",
-      "link": true
-    }
-  }
-}

lib/www/client/source/package.json

@@ -7,8 +7,14 @@
     "build": "vue-cli-service build"
   },
   "dependencies": {
+    "@deck.gl/aggregation-layers": "^9.1.13",
+    "@deck.gl/geo-layers": "^9.1.13",
+    "@deck.gl/mesh-layers": "^9.1.14",
+    "@dougal/binary": "file:../../../modules/@dougal/binary",
+    "@dougal/concurrency": "file:../../../modules/@dougal/concurrency",
     "@dougal/organisations": "file:../../../modules/@dougal/organisations",
     "@dougal/user": "file:../../../modules/@dougal/user",
+    "@loaders.gl/obj": "^4.3.4",
     "@mdi/font": "^7.2.96",
     "buffer": "^6.0.3",
     "core-js": "^3.6.5",
@@ -19,6 +25,7 @@
     "leaflet-arrowheads": "^1.2.2",
     "leaflet-realtime": "^2.2.0",
     "leaflet.markercluster": "^1.4.1",
+    "lodash.debounce": "^4.0.8",
     "marked": "^9.1.4",
     "path-browserify": "^1.0.1",
     "plotly.js-dist": "^2.27.0",

lib/www/client/source/src/App.vue

@@ -9,7 +9,7 @@
       :color="snackColour"
       :timeout="6000"
     >
-      {{ snackText }}
+      <div v-html="snackText"></div>
       <template v-slot:action="{ attrs }">
         <v-btn
           text
@@ -52,9 +52,8 @@ export default {
   }),
 
   computed: {
-    snackText () { return this.$store.state.snack.snackText },
+    snackText () { return this.$root.markdownInline(this.$store.state.snack.snackText) },
     snackColour () { return this.$store.state.snack.snackColour },
-    ...mapGetters(["serverEvent"])
   },
 
   watch: {
@@ -77,24 +76,41 @@ export default {
         this.$store.commit('setSnackText', "");
       }
     },
-
-    async serverEvent (event) {
-      if (event.channel == "project" && event.payload?.schema == "public") {
-        // Projects changed in some way or another
-        await this.refreshProjects();
-      } else if (event.channel == ".jwt" && event.payload?.token) {
-        await this.setCredentials({token: event.payload?.token});
-      }
-    }
   },
 
   methods: {
+
+    handleJWT (context, {payload}) {
+      this.setCredentials({token: payload.token});
+    },
+
+    handleProject (context, {payload}) {
+      if (payload?.table == "public") {
+        this.refreshProjects();
+      }
+    },
+
+    registerNotificationHandlers () {
+
+      this.$store.dispatch('registerHandler', {
+        table: '.jwt',
+        handler: this.handleJWT
+      });
+
+      this.$store.dispatch('registerHandler', {
+        table: 'project',
+        handler: this.handleProject
+      });
+
+    },
+
     ...mapActions(["setCredentials", "refreshProjects"])
   },
 
   async mounted () {
     // Local Storage values are always strings
     this.$vuetify.theme.dark = localStorage.getItem("darkTheme") == "true";
+    this.registerNotificationHandlers();
     await this.setCredentials();
     this.refreshProjects();
   }

lib/www/client/source/src/components/footer.vue

@@ -9,8 +9,17 @@
 
 	<v-spacer></v-spacer>
 
-	<v-icon v-if="serverConnected" class="mr-6" small title="Connected to server">mdi-lan-connect</v-icon>
-	<v-icon v-else class="mr-6" small color="red" title="Server connection lost (we'll reconnect automatically when the server comes back)">mdi-lan-disconnect</v-icon>
+	<template v-if="isFrontendRemote">
+      <template v-if="serverConnected">
+        <v-icon v-if="isGatewayReliable" class="mr-6" title="Connected to server via gateway">mdi-cloud-outline</v-icon>
+        <v-icon v-else class="mr-6" color="orange" title="Gateway connection is unreliable. Expect outages.">mdi-cloud-off</v-icon>
+      </template>
+      <v-icon v-else class="mr-6" color="red" :title="`Server connection lost: the gateway cannot reach the remote server.\nWe will reconnect automatically when the link with the remote server is restored.`">mdi-cloud-off</v-icon>
+	</template>
+	<template v-else>
+      <v-icon v-if="serverConnected" class="mr-6" small title="Connected to server">mdi-lan-connect</v-icon>
+      <v-icon v-else class="mr-6" small color="red" :title="`Server connection lost.\nWe will reconnect automatically when the server comes back.`">mdi-lan-disconnect</v-icon>
+	</template>
 
 	<dougal-notifications-control class="mr-6"></dougal-notifications-control>
 
@@ -51,13 +60,39 @@ export default {
     DougalNotificationsControl
   },
 
+  data () {
+    return {
+      lastGatewayErrorTimestamp: 0,
+      gatewayErrorSilencePeriod: 60000,
+    }
+  },
+
   computed: {
     year () {
       const date = new Date();
       return date.getUTCFullYear();
     },
 
-    ...mapState({serverConnected: state => state.notify.serverConnected})
+    ...mapState({
+      serverConnected: state => state.notify.serverConnected,
+      isFrontendRemote: state => state.api.serverInfo?.["remote-frontend"] ?? false,
+      isGatewayReliable: state => state.api.isGatewayReliable
+    })
+  },
+
+  watch: {
+
+    isGatewayReliable (val) {
+      if (val === false) {
+        const elapsed = Date.now() - this.lastGatewayErrorTimestamp;
+        const lastGatewayErrorTimestamp = Date.now();
+        if (elapsed > this.gatewayErrorSilencePeriod) {
+          this.$root.showSnack("Gateway error", "warning");
+        }
+      }
+    }
+
   }
+
 };
 </script>

lib/www/client/source/src/components/graph-arrays-ij-scatter.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Array inline / crossline error
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="scatterplot" label="Scatterplot"></v-switch>
     <v-switch class="ml-4" v-model="histogram" label="Histogram"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -57,8 +59,8 @@ export default {
       graph: [],
       busy: false,
       resizeObserver: null,
-      scatterplot: false,
-      histogram: false
+      scatterplot: true,
+      histogram: true
     };
   },
 

lib/www/client/source/src/components/graph-guns-depth.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Gun depth
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="shotpoint" label="Shotpoint"></v-switch>
     <v-switch class="ml-4" v-model="violinplot" label="Violin plot"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -59,7 +61,7 @@ export default {
       busy: false,
       resizeObserver: null,
       shotpoint: true,
-      violinplot: false
+      violinplot: true
     };
   },
 

lib/www/client/source/src/components/graph-guns-pressure.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Gun pressures
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="shotpoint" label="Shotpoint"></v-switch>
     <v-switch class="ml-4" v-model="violinplot" label="Violin plot"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -59,7 +61,7 @@ export default {
       busy: false,
       resizeObserver: null,
       shotpoint: true,
-      violinplot: false
+      violinplot: true
     };
   },
 

lib/www/client/source/src/components/graph-guns-timing.vue

@@ -3,8 +3,10 @@
   <v-card-title class="headline">
     Gun timing
     <v-spacer></v-spacer>
+    <!--
     <v-switch v-model="shotpoint" label="Shotpoint"></v-switch>
     <v-switch class="ml-4" v-model="violinplot" label="Violin plot"></v-switch>
+    -->
   </v-card-title>
 
   <v-container fluid fill-height>
@@ -59,7 +61,7 @@ export default {
       busy: false,
       resizeObserver: null,
       shotpoint: true,
-      violinplot: false
+      violinplot: true
     };
   },
 

lib/www/client/source/src/components/graphs/project/sequence/inline-crossline.vue

@@ -39,7 +39,8 @@ export default {
         default:
           return {
             editable: false,
-            displaylogo: false
+            displaylogo: false,
+            responsive: true
           };
       }
     },
@@ -48,7 +49,8 @@ export default {
       const base = {
         font: {
           color: this.$vuetify.theme.isDark ? "#fff" : undefined
-        }
+        },
+        autosize: true
       };
 
       switch (this.facet) {
@@ -274,18 +276,25 @@ export default {
     replot () {
       if (this.plotted) {
         const ref = this.$refs.graph;
-        Plotly.relayout(ref, {
-          width: ref.clientWidth,
-          height: ref.clientHeight
-        });
+        if (ref && ref.clientWidth > 0 && ref.clientHeight > 0) {
+          Plotly.relayout(ref, {
+            width: ref.clientWidth,
+            height: ref.clientHeight
+          });
+        }
       }
     }
 
   },
 
   mounted () {
-    this.resizeObserver = new ResizeObserver(this.replot)
-    this.resizeObserver.observe(this.$refs.graph);
+    this.$nextTick( () => {
+      if (this.items?.length) {
+        this.plot();
+      }
+      this.resizeObserver = new ResizeObserver(this.replot)
+      this.resizeObserver.observe(this.$refs.graph);
+    });
   },
 
   beforeDestroy () {

lib/www/client/source/src/components/graphs/project/sequence/shotpoint-timing.vue

@@ -36,7 +36,8 @@ export default {
     config () {
       return {
         editable: false,
-        displaylogo: false
+        displaylogo: false,
+        responsive: true
       };
     },
 
@@ -53,7 +54,8 @@ export default {
           title: "Time (s)"
         },
         plot_bgcolor:"rgba(0,0,0,0)",
-        paper_bgcolor:"rgba(0,0,0,0)"
+        paper_bgcolor:"rgba(0,0,0,0)",
+        autosize: true
       };
     },
 
@@ -154,10 +156,12 @@ export default {
     replot () {
       if (this.plotted) {
         const ref = this.$refs.graph;
-        Plotly.relayout(ref, {
-          width: ref.clientWidth,
-          height: ref.clientHeight
-        });
+        if (ref && ref.clientWidth > 0 && ref.clientHeight > 0) {
+          Plotly.relayout(ref, {
+            width: ref.clientWidth,
+            height: ref.clientHeight
+          });
+        }
       }
     },
 
@@ -190,8 +194,13 @@ export default {
   },
 
   mounted () {
-    this.resizeObserver = new ResizeObserver(this.replot)
-    this.resizeObserver.observe(this.$refs.graph);
+    this.$nextTick( () => {
+      if (this.items?.length) {
+        this.plot();
+      }
+      this.resizeObserver = new ResizeObserver(this.replot)
+      this.resizeObserver.observe(this.$refs.graph);
+    });
   },
 
   beforeDestroy () {

lib/www/client/source/src/components/groups/group-comparison-summary.vue

@@ -0,0 +1,187 @@
+<template>
+  <v-card v-if="comparison" class="ma-1">
+    <v-card-title>Comparison Summary: Baseline {{ baseline.pid }} vs Monitor {{ monitor.pid }}</v-card-title>
+    <v-card-text>
+      <v-row>
+        <v-col cols="12" md="6">
+          <h3>Deviation Statistics</h3>
+          <v-simple-table dense>
+            <template v-slot:default>
+              <thead>
+                <tr>
+                  <th>Metric</th>
+                  <th>I (m)</th>
+                  <th>J (m)</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td>Mean (μ)</td>
+                  <td>{{ comparison['μ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['μ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>Std Dev (σ)</td>
+                  <td>{{ comparison['σ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['σ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>RMS</td>
+                  <td>{{ comparison.rms[0].toFixed(3) }}</td>
+                  <td>{{ comparison.rms[1].toFixed(3) }}</td>
+                </tr>
+              </tbody>
+            </template>
+          </v-simple-table>
+
+          <h3 class="mt-4">Error distribution</h3>
+          <ul>
+            <li title="Relative to I-axis positive direction">Primary Direction: {{ (comparison.primaryDirection * 180 / Math.PI).toFixed(2) }}°</li>
+            <li>Anisotropy: {{ comparison.anisotropy.toFixed(2) }}</li>
+            <li title="Length of the semi-major axis of the error ellipse">Semi-Major Axis: {{ semiMajorAxis.toFixed(2) }} m</li>
+            <li title="Length of the semi-minor axis of the error ellipse">Semi-Minor Axis: {{ semiMinorAxis.toFixed(2) }} m</li>
+            <li title="Area of the error ellipse">Error Ellipse Area: {{ ellipseArea.toFixed(2) }} m²</li>
+          </ul>
+
+          <h3 class="mt-4">Counts</h3>
+          <ul>
+            <li title="Unique line / point pairs found in both projects">Common Points: {{ comparison.common }}</li>
+            <li title="Total number of points compared, including reshoots, infills, etc.">Comparison Length: {{ comparison.length }}</li>
+            <li title="Number of points in the baseline project">Baseline Points: {{ comparison.baselineLength }} (Unique: {{ comparison.baselineUniqueLength }})</li>
+            <li title="Number of points in the monitor project">Monitor Points: {{ comparison.monitorLength }} (Unique: {{ comparison.monitorUniqueLength }})</li>
+          </ul>
+
+          <p class="mt-3" title="Date and time when the comparison was last performed">Computation timestamp: {{ new Date(comparison.tstamp).toLocaleString() }}</p>
+        </v-col>
+
+        <v-col cols="12" md="6">
+          <h3>Error Ellipse</h3>
+          <svg width="300" height="300" style="border: 1px solid #ccc;">
+            <g :transform="`translate(150, 150) scale(${ellipseScale})`">
+              <line x1="0" y1="-150" x2="0" y2="150" stroke="lightgray" stroke-dasharray="5,5"/>
+              <line x1="-150" y1="0" x2="150" y2="0" stroke="lightgray" stroke-dasharray="5,5"/>
+              <ellipse
+                :rx="Math.sqrt(comparison.eigenvalues[0])"
+                :ry="Math.sqrt(comparison.eigenvalues[1])"
+                :transform="`rotate(${ellipseAngle})`"
+                fill="none"
+                stroke="blue"
+                stroke-width="2"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[0]) * Math.cos(ellipseRad)"
+                :y2="Math.sqrt(comparison.eigenvalues[0]) * Math.sin(ellipseRad)"
+                stroke="red"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[1]) * Math.cos(ellipseRad + Math.PI / 2)"
+                :y2="Math.sqrt(comparison.eigenvalues[1]) * Math.sin(ellipseRad + Math.PI / 2)"
+                stroke="green"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+            </g>
+          </svg>
+          <p class="text-caption">Ellipse scaled for visibility (factor: {{ ellipseScale.toFixed(1) }}). Axes represent sqrt(eigenvalues).</p>
+        </v-col>
+      </v-row>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: "DougalGroupComparisonSummary",
+
+  props: {
+    baseline: { type: Object, required: true },
+    monitor: { type: Object, required: true },
+    comparison: { type: Object, required: true }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+
+    ellipseAngle () {
+      if (!this.comparison) return 0;
+      const ev = this.comparison.eigenvectors[0];
+      return Math.atan2(ev[1], ev[0]) * 180 / Math.PI;
+    },
+
+    ellipseRad () {
+      return this.ellipseAngle * Math.PI / 180;
+    },
+
+    ellipseRx () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[0]) * this.ellipseScale;
+    },
+
+    ellipseRy () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[1]) * this.ellipseScale;
+    },
+
+    ellipseScale () {
+      if (!this.comparison) return 1;
+      const maxSigma = Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+      const maxMu = Math.max(
+        Math.abs(this.comparison['μ'][0]),
+        Math.abs(this.comparison['μ'][1])
+      );
+      //const maxExtent = maxMu + 3 * maxSigma;
+      const maxExtent = 20;
+      return 100 / maxExtent; // Adjust scale to fit within ~200 pixels diameter
+    },
+
+    ellipseArea () {
+      if (!this.comparison) return 0;
+      const a = Math.sqrt(this.comparison.eigenvalues[0]);
+      const b = Math.sqrt(this.comparison.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+
+    semiMajorAxis () {
+      if (!this.comparison) return 0;
+      return Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    semiMinorAxis () {
+      if (!this.comparison) return 0;
+      return Math.min(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    meanX () {
+      return this.comparison ? this.comparison['μ'][0] : 0;
+    },
+
+    meanY () {
+      return this.comparison ? this.comparison['μ'][1] : 0;
+    },
+
+    ellipseViewBox () {
+      return '-150 -150 300 300';
+    },
+
+  }
+}
+</script>

lib/www/client/source/src/components/groups/group-repeatability-summary.vue

@@ -0,0 +1,118 @@
+<template>
+  <v-card class="ma-1">
+    <v-card-title>Group Repeatability Summary</v-card-title>
+    <v-card-text>
+      <p>Error ellipse area for each baseline-monitor pair. Lower values indicate better repeatability. Colors range from green (best) to red (worst).</p>
+      <v-simple-table dense>
+        <thead>
+          <tr>
+            <th>Baseline \ Monitor</th>
+            <th v-for="project in projects" :key="project.pid">{{ project.pid }}</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr v-for="(baselineProject, rowIndex) in projects" :key="baselineProject.pid">
+            <td>{{ baselineProject.pid }}</td>
+            <td v-for="(monitorProject, colIndex) in projects" :key="monitorProject.pid">
+              <v-tooltip v-if="colIndex > rowIndex" top>
+                <template v-slot:activator="{ on, attrs }">
+                  <div
+                    :style="{ backgroundColor: getEllipseAreaColor(baselineProject.pid, monitorProject.pid), color: 'white', textAlign: 'center', padding: '4px' }"
+                    v-bind="attrs"
+                    v-on="on"
+                    @click="emitInput(baselineProject, monitorProject)"
+                  >
+                    {{ formatEllipseArea(baselineProject.pid, monitorProject.pid) }}
+                  </div>
+                </template>
+                <span v-if="getComp(baselineProject.pid, monitorProject.pid)">
+                  <div>σ_i: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][0].toFixed(2) }} m</div>
+                  <div>σ_j: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][1].toFixed(2) }} m</div>
+                  <div>Anisotropy: {{ getComp(baselineProject.pid, monitorProject.pid).meta.anisotropy.toFixed(0) }}</div>
+                  <div>Ellipse Area: {{ getEllipseArea(baselineProject.pid, monitorProject.pid).toFixed(2) }} m²</div>
+                  <div>Primary Direction: {{ formatPrimaryDirection(getComp(baselineProject.pid, monitorProject.pid)) }}°</div>
+                </span>
+              </v-tooltip>
+            </td>
+          </tr>
+        </tbody>
+      </v-simple-table>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: 'DougalGroupRepeatabilitySummary',
+
+  props: {
+    comparisons: {
+      type: Array,
+      required: true
+    },
+    projects: {
+      type: Array,
+      required: true
+    }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+    compMap () {
+      return new Map(this.comparisons.map(c => [`${c.baseline_pid}-${c.monitor_pid}`, c]));
+    },
+    minEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.min(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    },
+    maxEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.max(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    }
+  },
+  methods: {
+    getComp (basePid, monPid) {
+      return this.compMap.get(`${basePid}-${monPid}`);
+    },
+    getEllipseArea (basePid, monPid) {
+      const comp = this.getComp(basePid, monPid);
+      if (!comp) return null;
+      const a = Math.sqrt(comp.meta.eigenvalues[0]);
+      const b = Math.sqrt(comp.meta.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+    formatEllipseArea (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      return val !== null ? val.toFixed(1) : '';
+    },
+    getEllipseAreaColor (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      if (val === null) return '';
+      const ratio = (val - this.minEllipseArea) / (this.maxEllipseArea - this.minEllipseArea);
+      const hue = (1 - ratio) * 120;
+      return `hsl(${hue}, 70%, 70%)`;
+    },
+    formatPrimaryDirection (comp) {
+      if (!comp) return '';
+      return (comp.meta.primaryDirection * 180 / Math.PI).toFixed(1);
+    },
+    emitInput (baselineProject, monitorProject) {
+      if (this.getComp(baselineProject.pid, monitorProject.pid)) {
+        this.$emit('input', baselineProject, monitorProject);
+      }
+    }
+  }
+}
+</script>

lib/www/client/source/src/components/help-dialog.vue

@@ -2,6 +2,7 @@
 <v-dialog
   v-model="dialog"
   max-width="500"
+  scrollable
   style="z-index:2020;"
 >
   <template v-slot:activator="{ on, attrs }">
@@ -14,15 +15,54 @@
   </template>
 
   <v-card>
-    <v-card-title class="headline">
-      Dougal user support
-    </v-card-title>
+    <v-window v-model="page">
+      <v-window-item value="support">
+        <v-card-title class="headline">
+          Dougal user support
+        </v-card-title>
 
-    <v-card-text>
-      <p>You can get help or report a problem by sending an email to <a :href="`mailto:${email}`">{{email}}</a>. Please include as much information as possible about your problem or question—screenshots are often a good idea, and data files may also be attached.</p>
+        <v-card-text>
+          <p>You can get help or report a problem by sending an email to <a :href="`mailto:${email}`">{{email}}</a>. Please include as much information as possible about your problem or question—screenshots are often a good idea, and data files may also be attached.</p>
 
-      <p>When you write to the above address a ticket will be automatically created in the project's issue tracking system.</p>
-    </v-card-text>
+          <p>When you write to the above address a ticket will be automatically created in the project's issue tracking system.</p>
+
+          <v-alert dense type="info" border="left" outlined>
+            <div class="text-body-2">
+            You are using Dougal version:
+            <ul>
+              <li><code>{{clientVersion}}</code> (client)</li>
+              <li><code>{{serverVersion}}</code> (server)</li>
+            </ul>
+            </div>
+          </v-alert>
+
+        </v-card-text>
+      </v-window-item>
+
+      <v-window-item value="changelog">
+        <v-card-title class="headline">
+          Dougal release notes
+        </v-card-title>
+
+        <v-card-text>
+          <v-carousel v-model="releaseShown"
+            :continuous="false"
+            :cycle="false"
+            :show-arrows="true"
+            :hide-delimiters="true"
+          >
+            <v-carousel-item v-for="release in releaseHistory">
+              <pre>{{release}}</pre>
+            </v-carousel-item>
+          </v-carousel>
+        </v-card-text>
+
+
+      </v-window-item>
+      <v-window-item value="serverinfo">
+        <dougal-server-status :status="serverStatus"></dougal-server-status>
+      </v-window-item>
+    </v-window>
 
     <v-divider></v-divider>
 
@@ -33,8 +73,7 @@
         text
         :href="`mailto:${email}?Subject=Question`"
       >
-        <v-icon class="d-lg-none">mdi-help-circle</v-icon>
-        <span class="d-none d-lg-inline">Ask a question</span>
+        <v-icon title="Ask a question">mdi-help-circle</v-icon>
       </v-btn>
 
       <v-btn
@@ -42,10 +81,10 @@
         text
         href="mailto:dougal-support@aaltronav.eu?Subject=Bug report"
       >
-        <v-icon class="d-lg-none">mdi-bug</v-icon>
-        <span class="d-none d-lg-inline">Report a bug</span>
+        <v-icon title="Report a bug">mdi-bug</v-icon>
       </v-btn>
 
+      <!---
       <v-btn
         color="info"
         text
@@ -54,6 +93,37 @@
       >
         <v-icon>mdi-rss</v-icon>
       </v-btn>
+      --->
+
+      <v-btn
+        color="info"
+        text
+        title="View support info"
+        :input-value="page == 'support'"
+        @click="page = 'support'"
+      >
+        <v-icon>mdi-account-question</v-icon>
+      </v-btn>
+
+      <v-btn v-if="versionHistory"
+        color="info"
+        text
+        title="View release notes"
+        :input-value="page == 'changelog'"
+        @click="page = 'changelog'"
+      >
+        <v-icon>mdi-history</v-icon>
+      </v-btn>
+
+      <v-btn v-if="serverStatus"
+        color="info"
+        text
+        title="View server status"
+        :input-value="page == 'serverinfo'"
+        @click="page = 'serverinfo'"
+      >
+        <v-icon>mdi-server-network</v-icon>
+      </v-btn>
 
       <v-spacer></v-spacer>
 
@@ -75,15 +145,111 @@
 </template>
 
 <script>
+import { mapActions, mapGetters } from 'vuex';
+import DougalServerStatus from './server-status';
+
 export default {
   name: 'DougalHelpDialog',
 
+  components: {
+    DougalServerStatus
+  },
+
   data () {
     return {
       dialog: false,
       email: "dougal-support@aaltronav.eu",
-      feed: btoa(encodeURIComponent("https://gitlab.com/wgp/dougal/software.atom?feed_token=XSPpvsYEny8YmH75Nz5W"))
+      feed: btoa(encodeURIComponent("https://gitlab.com/wgp/dougal/software.atom?feed_token=XSPpvsYEny8YmH75Nz5W")),
+      serverStatus: null,
+      clientVersion: process.env.DOUGAL_FRONTEND_VERSION ?? "(unknown)",
+      serverVersion: null,
+      versionHistory: null,
+      releaseHistory: [],
+      releaseShown: null,
+      page: "support",
+
+      lastUpdate: 0,
+      updateInterval: 12000,
+      refreshTimer: null
     };
+  },
+
+  computed: {
+    sinceUpdate () {
+      return this.lastUpdate
+        ? (Date.now() - this.lastUpdate)
+        : +Infinity;
+    }
+  },
+
+  watch: {
+    dialog(newVal) {
+      if (newVal) {
+        this.startAutoRefresh();
+      } else {
+        this.stopAutoRefresh();
+      }
+    },
+    page(newVal) {
+      if (newVal === 'serverinfo' && this.dialog) {
+        this.getServerStatus(); // Immediate update when switching to serverinfo
+        this.startAutoRefresh();
+      } else {
+        this.stopAutoRefresh();
+      }
+    }
+  },
+
+  methods: {
+    async getServerVersion () {
+      if (!this.serverVersion) {
+        const version = await this.api(['/version', {}, null, {silent:true}]);
+        this.serverVersion = version?.tag ?? "(unknown)";
+        if (version) this.lastUpdate = Date.now();
+      }
+      if (!this.versionHistory) {
+        const history = await this.api(['/version/history?count=6', {}, null, {silent:true}]);
+        this.releaseHistory = history;
+        this.versionHistory = history?.[this.serverVersion.replace(/-.*$/, "")] ?? null;
+      }
+    },
+
+    async getServerStatus () {
+      const status = await this.api(['/diagnostics', {}, null, {silent: true}]);
+      if (status) {
+        this.serverStatus = status;
+        this.lastUpdate = Date.now();
+      }
+    },
+
+    startAutoRefresh() {
+      if (this.refreshTimer) return; // Prevent multiple timers
+      this.refreshTimer = setInterval(() => {
+        if (this.dialog && this.page === 'serverinfo') {
+          this.getServerStatus();
+          // Optionally refresh server version if needed
+          // this.getServerVersion();
+        }
+      }, this.updateInterval);
+    },
+
+    stopAutoRefresh() {
+      if (this.refreshTimer) {
+        clearInterval(this.refreshTimer);
+        this.refreshTimer = null;
+      }
+    },
+
+    ...mapActions(["api"])
+  },
+
+  async mounted () {
+    this.getServerVersion();
+    this.getServerStatus();
+  },
+
+  beforeDestroy() {
+    this.stopAutoRefresh(); // Clean up timer on component destruction
   }
 
 };

lib/www/client/source/src/components/line-status.vue

@@ -1,8 +1,5 @@
 <template>
-<div class="line-status" v-if="sequences.length == 0">
-  <slot name="empty"></slot>
-</div>
-<div class="line-status" v-else-if="sequenceHref || plannedSequenceHref || pendingReshootHref">
+<div class="line-status" v-if="sequenceHref || plannedSequenceHref || pendingReshootHref">
   <router-link v-for="sequence in sequences" :key="sequence.sequence" v-if="sequenceHref"
     class="sequence"
     :class="sequence.status"
@@ -26,7 +23,7 @@
   >
   </router-link>
 </div>
-<div class="line-status" v-else>
+<div class="line-status" v-else-if="sequences.length || plannedSequences.length || Object.keys(pendingReshoots).length">
   <div v-for="sequence in sequences" :key="sequence.sequence"
     class="sequence"
     :class="sequence.status"
@@ -47,6 +44,9 @@
   >
   </div>
 </div>
+<div class="line-status" v-else>
+  <slot name="empty"></slot>
+</div>
 </template>
 
 <style lang="stylus" scoped>

lib/www/client/source/src/components/project-settings/file-matching-parameters.vue

@@ -4,15 +4,15 @@
     <v-card-subtitle v-text="subtitle"></v-card-subtitle>
     <v-card-text>
       <v-tabs v-model="tab">
-        <v-tab>Paths</v-tab>
-        <v-tab>Globs</v-tab>
-        <v-tab v-if="pattern">Pattern</v-tab>
-        <v-tab v-if="lineNameInfo">Line info</v-tab>
+        <v-tab tab-value="paths">Paths</v-tab>
+        <v-tab tab-value="globs">Globs</v-tab>
+        <v-tab tab-value="pattern" v-if="pattern">Pattern</v-tab>
+        <v-tab tab-value="lineNameInfo" v-if="lineNameInfo">Line info</v-tab>
       </v-tabs>
 
       <v-tabs-items v-model="tab">
 
-        <v-tab-item>
+        <v-tab-item value="paths">
           <v-card flat>
             <v-card-subtitle>
               A list of directories which are searched for matching files.
@@ -56,7 +56,7 @@
           </v-card>
         </v-tab-item>
 
-        <v-tab-item>
+        <v-tab-item value="globs">
           <v-card flat>
             <v-card-subtitle>
               A list of <a href="https://en.wikipedia.org/wiki/Glob_(programming)" target="_blank">glob patterns</a> expanding to match the files of interest. Note that Linux is case-sensitive.
@@ -93,7 +93,7 @@
           </v-card>
         </v-tab-item>
 
-        <v-tab-item v-if="pattern">
+        <v-tab-item value="pattern" v-if="pattern">
           <v-card flat>
             <v-card-subtitle>
               Regular expression that describes the file format definition. Used to capture information such as line and sequence number, etc.
@@ -153,7 +153,7 @@
           </v-card>
         </v-tab-item>
 
-        <v-tab-item v-if="lineNameInfo">
+        <v-tab-item value="lineNameInfo">
           <v-card flat>
             <v-card-subtitle>
               Line information that will be extracted from file names
@@ -165,14 +165,14 @@
                   label="Example file name"
                   hint="Enter the name of a representative file to make it easier to visualise your configuration"
                   persistent-hint
-                  v-model="lineNameInfo.example"
+                  v-model="lineNameInfo_.example"
                 ></v-text-field>
 
                 <dougal-fixed-string-decoder
                   :multiline="true"
-                  :text="lineNameInfo.example"
-                  :fixed.sync="lineNameInfo.fixed"
-                  :fields.sync="lineNameInfo.fields"
+                  :text="lineNameInfo_.example"
+                  :fixed.sync="lineNameInfo_.fixed"
+                  :fields.sync="lineNameInfo_.fields"
                 ></dougal-fixed-string-decoder>
 
               </v-form>
@@ -195,6 +195,23 @@
         @click="reset"
       >Reset</v-btn>
       -->
+        <v-btn
+          v-if="tab=='lineNameInfo'"
+          :disabled="!validLineNameInfo"
+          @click="copyLineNameInfo"
+          title="Copy this definition into the clipboard. It can then be pasted into other sections or configurations."
+        >
+          <v-icon left>mdi-content-copy</v-icon>
+          Copy
+        </v-btn>
+        <v-btn
+          v-if="tab=='lineNameInfo'"
+          @click="pasteLineNameInfo"
+          title="Paste a line info definition copied from elsewhere"
+        >
+          <v-icon left>mdi-content-paste</v-icon>
+          Paste
+        </v-btn>
       <v-spacer></v-spacer>
       <v-btn
         color="secondary"
@@ -253,6 +270,9 @@ export default {
   },
 
   computed: {
+    validLineNameInfo () {
+      return typeof this.lineNameInfo == 'object';
+    },
   },
 
   watch: {
@@ -285,6 +305,28 @@ export default {
 
   methods: {
 
+    async copyLineNameInfo () {
+      await navigator.clipboard.writeText(JSON.stringify(this.lineNameInfo, null, 4));
+      this.showSnack(["Line name information copied to clipboard", "primary"]);
+    },
+
+    async pasteLineNameInfo () {
+      const text = await navigator.clipboard.readText();
+      try {
+        const data = JSON.parse(text);
+        if (["fixed", "fields", "example"].every( key => key in data )) {
+          this.$emit("update:lineNameInfo", data);
+          this.showSnack(["Line name information pasted from clipboard", "primary"]);
+        } else {
+          this.showSnack(["Clipboard contents are not valid line name information", "error"]);
+        }
+      } catch (err) {
+        if (err instanceof SyntaxError) {
+          this.showSnack(["Clipboard contents are not valid line name information", "error"]);
+        }
+      }
+    },
+
     reset () {
       this.globs_ = this.globs;
       this.paths_ = this.paths;
@@ -302,6 +344,8 @@ export default {
       this.$emit('close');
     },
 
+    ...mapActions(["showSnack"])
+
   },
 
   mounted () {

lib/www/client/source/src/components/project-settings/preplots-preplot.vue

@@ -519,17 +519,18 @@ export default {
   methods: {
 
     async getHead () {
-      console.log("getHead", this.value?.path);
       if (this.value?.path) {
         const url = `/files/${this.value.path}`;
         const init = {
-          text: true,
           headers: {
             "Range": `bytes=0-${this.sampleSize}`
           }
         };
-        const head = await this.api([url, init]);
-        return head?.substring(0, head.lastIndexOf("\n")) || "";
+        const opts = {format: "text"};
+        const head = await this.api([url, init, null, opts]);
+        return typeof head === "string"
+          ? head?.substring(0, head.lastIndexOf("\n")) || ""
+          : this.head ?? "";
       }
       return "";
     },

lib/www/client/source/src/components/server-status.vue

@@ -0,0 +1,213 @@
+<template>
+  <v-card max-width="800" max-height="600" class="mx-auto" style="overflow-y: auto;">
+    <v-card-title class="headline">
+      Server status – {{ status.hostname }}
+    </v-card-title>
+    <v-card-text>
+      <v-expansion-panels accordion>
+        <!-- System Info -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>System Info</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-row>
+              <v-col cols="6">
+                <strong>Uptime:</strong> {{ formatUptime(status.uptime) }}
+              </v-col>
+              <v-col cols="6">
+                <strong>Load:</strong> {{ status.loadavg[0].toFixed(2) }} / {{ status.loadavg[1].toFixed(2) }} / {{ status.loadavg[2].toFixed(2) }}
+                <v-progress-linear
+                  :value="loadAvgPercent"
+                  :color="getLoadAvgColor(status.loadavg[0])"
+                  height="6"
+                  rounded
+                ></v-progress-linear>
+                <div class="text-caption">
+                  1-min Load: {{ status.loadavg[0].toFixed(2) }} ({{ loadAvgPercent.toFixed(1) }}% of max)
+                </div>
+              </v-col>
+            </v-row>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Memory -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Memory</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-progress-linear
+              :value="memoryUsedPercent"
+              :color="getProgressColor(memoryUsedPercent)"
+              height="10"
+              rounded
+            ></v-progress-linear>
+            <div class="text-caption mt-2">
+              Used: {{ formatBytes(status.memory.total - status.memory.free) }} / Total: {{ formatBytes(status.memory.total) }} ({{ memoryUsedPercent.toFixed(1) }}%)
+            </div>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- CPUs -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>CPUs ({{ status.cpus.length }} cores)</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-row dense>
+              <v-col v-for="(cpu, index) in status.cpus" :key="index" cols="12" sm="6">
+                <v-card outlined class="pa-2">
+                  <div class="text-caption">Core {{ index + 1 }}: {{ cpu.model }} @ {{ cpu.speed }} MHz</div>
+                  <v-progress-linear
+                    :value="cpuUsagePercent(cpu)"
+                    :color="getProgressColor(cpuUsagePercent(cpu))"
+                    height="8"
+                    rounded
+                  ></v-progress-linear>
+                  <div class="text-caption">
+                    Usage: {{ cpuUsagePercent(cpu).toFixed(1) }}% (Idle: {{ cpuIdlePercent(cpu).toFixed(1) }}%)
+                  </div>
+                </v-card>
+              </v-col>
+            </v-row>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Network Interfaces -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Network Interfaces</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <v-list dense>
+              <v-list-item v-for="(iface, name) in status.networkInterfaces" :key="name">
+                <v-list-item-content>
+                  <v-list-item-title>{{ name }}</v-list-item-title>
+                  <v-list-item-subtitle v-for="(addr, idx) in iface" :key="idx">
+                    {{ addr.family }}: {{ addr.address }} (Netmask: {{ addr.netmask }})
+                  </v-list-item-subtitle>
+                </v-list-item-content>
+              </v-list-item>
+            </v-list>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Storage -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Storage</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <!-- Root -->
+            <div class="mb-4">
+              <strong>Root (/):</strong>
+              <v-progress-linear
+                :value="status.storage.root.usedPercent"
+                :color="getProgressColor(status.storage.root.usedPercent)"
+                height="10"
+                rounded
+              ></v-progress-linear>
+              <div class="text-caption">
+                Used: {{ formatBytes(status.storage.root.used) }} / Total: {{ formatBytes(status.storage.root.total) }} ({{ status.storage.root.usedPercent.toFixed(1) }}%)
+              </div>
+            </div>
+            <!-- Data subfolders -->
+            <div>
+              <strong>Data:</strong>
+              <v-expansion-panels flat>
+                <v-expansion-panel v-for="(folder, name) in status.storage.data" :key="name">
+                  <v-expansion-panel-header disable-icon-rotate>{{ name }}</v-expansion-panel-header>
+                  <v-expansion-panel-content>
+                    <v-progress-linear
+                      :value="folder.usedPercent"
+                      :color="getProgressColor(folder.usedPercent)"
+                      height="10"
+                      rounded
+                    ></v-progress-linear>
+                    <div class="text-caption">
+                      Used: {{ formatBytes(folder.used) }} / Total: {{ formatBytes(folder.total) }} ({{ folder.usedPercent.toFixed(1) }}%)
+                    </div>
+                  </v-expansion-panel-content>
+                </v-expansion-panel>
+              </v-expansion-panels>
+            </div>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+
+        <!-- Database -->
+        <v-expansion-panel>
+          <v-expansion-panel-header>Database</v-expansion-panel-header>
+          <v-expansion-panel-content>
+            <div class="mb-2">
+              <strong>Total Size:</strong> {{ formatBytes(status.database.size) }}
+            </div>
+            <v-list dense>
+              <v-list-item v-for="(project, name) in status.database.projects" :key="name">
+                <v-list-item-content>
+                  <v-list-item-title>{{ name }}</v-list-item-title>
+                  <v-progress-linear
+                    :value="project.percent"
+                    :color="getProgressColor(project.percent)"
+                    height="8"
+                    rounded
+                  ></v-progress-linear>
+                  <v-list-item-subtitle>
+                    Size: {{ formatBytes(project.size) }} ({{ project.percent.toFixed(2) }}%)
+                  </v-list-item-subtitle>
+                </v-list-item-content>
+              </v-list-item>
+            </v-list>
+          </v-expansion-panel-content>
+        </v-expansion-panel>
+      </v-expansion-panels>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: "DougalServerStatus",
+  props: {
+    status: {
+      type: Object,
+      required: true
+    }
+  },
+  computed: {
+    memoryUsedPercent() {
+      return ((this.status.memory.total - this.status.memory.free) / this.status.memory.total) * 100;
+    },
+    loadAvgPercent() {
+      const maxLoad = this.status.cpus.length * 4; // Assume 4x cores as max for scaling
+      return Math.min((this.status.loadavg[0] / maxLoad) * 100, 100); // Cap at 100%
+    }
+  },
+  methods: {
+    getProgressColor(value) {
+      if (value >= 80) return 'error'; // Red for 80–100%
+      if (value >= 60) return 'warning'; // Yellow for 60–80%
+      return 'success'; // Green for 0–60%
+    },
+    getLoadAvgColor(load) {
+      const coreCount = this.status.cpus.length;
+      if (load >= coreCount * 2) return 'error'; // Red for load ≥ 2x cores
+      if (load >= coreCount) return 'warning'; // Yellow for load ≥ 1x cores but < 2x
+      return 'success'; // Green for load < 1x cores
+    },
+    formatBytes(bytes) {
+      if (bytes === 0) return '0 Bytes';
+      const k = 1024;
+      const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'];
+      const i = Math.floor(Math.log(bytes) / Math.log(k));
+      return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+    },
+    formatUptime(seconds) {
+      const days = Math.floor(seconds / 86400);
+      seconds %= 86400;
+      const hours = Math.floor(seconds / 3600);
+      seconds %= 3600;
+      const minutes = Math.floor(seconds / 60);
+      return `${days}d ${hours}h ${minutes}m`;
+    },
+    cpuUsagePercent(cpu) {
+      const total = cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.idle + cpu.times.irq;
+      return ((total - cpu.times.idle) / total) * 100;
+    },
+    cpuIdlePercent(cpu) {
+      const total = cpu.times.user + cpu.times.nice + cpu.times.sys + cpu.times.idle + cpu.times.irq;
+      return (cpu.times.idle / total) * 100;
+    }
+  }
+};
+</script>

lib/www/client/source/src/lib/binary

@@ -0,0 +1 @@
+../../../../server/lib/binary

lib/www/client/source/src/lib/deck.gl/DougalBinaryLoader.js

@@ -0,0 +1,150 @@
+// src/lib/deck.gl/DougalBinaryLoader.js
+import { LoaderObject } from '@loaders.gl/core';
+import { DougalBinaryBundle } from '@dougal/binary';
+
+async function cachedFetch(url, init, opts = {}) {
+    let res; // The response
+    let cache; // Potentially, a Cache API cache name
+    let isCached;
+
+    if (opts?.cache === true) {
+      opts.cache = { name: "dougal" };
+    } else if (typeof opts?.cache === "string") {
+      opts.cache = { name: opts.cache };
+    } else if (opts?.cache) {
+      if (!(opts.cache instanceof Object)) {
+        opts.cache = { name: "dougal" }
+      } else if (!(opts.cache.name)) {
+        opts.cache.name = "dougal";
+      }
+    }
+
+    if (opts?.cache && window.cache) {
+      cache = await caches.open(opts.cache.name);
+      res = await cache.match(url);
+      isCached = !!res;
+    }
+
+    if (!res) {
+      res = await fetch(url, init);
+    }
+
+    if (cache && !isCached) {
+      cache.put(url, res.clone());
+    }
+
+    return res;
+}
+
+const DougalBinaryLoader = {
+  name: 'DougalBinaryBundle Loader',
+  extensions: ['dbb'],
+  mimeTypes: ['application/vnd.aaltronav.dougal+octet-stream'],
+  parse: async (input, options) => {
+    let arrayBuffer;
+    if (typeof input === 'string') {
+      // Input is URL, fetch with caching
+      const response = await cachedFetch(input, options?.fetch, options);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch: ${response.statusText}`);
+      }
+      arrayBuffer = await response.arrayBuffer();
+    } else if (input instanceof ArrayBuffer) {
+      arrayBuffer = input;
+    } else {
+      throw new Error('Invalid input: Expected URL string or ArrayBuffer');
+    }
+
+    const bundle = DougalBinaryBundle.clone(arrayBuffer);
+
+    // Calculate total points
+    const totalCount = bundle.chunks().reduce((acc, chunk) => acc + chunk.jCount, 0);
+
+    // Prepare positions (Float32Array: [lon1, lat1, lon2, lat2, ...])
+    const positions = new Float32Array(totalCount * 2);
+
+    // Extract udv (assume constant across chunks)
+    const udv = bundle.chunks()[0].udv;
+
+    // Prepare values as an array of TypedArrays
+    const ΔelemCount = bundle.chunks()[0].ΔelemCount;
+    const elemCount = bundle.chunks()[0].elemCount;
+    const values = new Array(ΔelemCount + elemCount + 2);
+
+    // Initialize values arrays with correct types
+    if (udv == 0) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : Uint8Array)(totalCount);
+      }
+    } else if (udv == 1) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? Uint8Array : Uint16Array)(totalCount);
+      }
+    } else if (udv == 2) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? BigUint64Array : Float32Array)(totalCount);
+      }
+    } else if (udv == 4) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? Uint16Array : Float32Array)(totalCount);
+      }
+    } else {
+      throw new Error(`Invalid udv: Expected 0, 1, 2, or 4; found ${udv}`);
+    }
+
+    let offset = 0;
+    for (const chunk of bundle.chunks()) {
+      const λarray = chunk.elem(0);
+      const φarray = chunk.elem(1);
+      for (let i = 0; i < λarray.length; i++) {
+        positions[offset * 2 + i * 2] = λarray[i];
+        positions[offset * 2 + i * 2 + 1] = φarray[i];
+      }
+
+      values[0].set(new Uint16Array(chunk.jCount).fill(chunk.i), offset);
+      values[1].set(Uint32Array.from({ length: chunk.jCount }, (_, i) => chunk.j0 + i * chunk.Δj), offset);
+
+      for (let j = 0; j < ΔelemCount; j++) {
+        values[2 + j].set(chunk.Δelem(j), offset);
+      }
+      for (let j = 2; j < elemCount; j++) {
+        values[2 + ΔelemCount + j - 2].set(chunk.elem(j), offset);
+      }
+
+      offset += chunk.jCount;
+    }
+
+    console.log(`Parsed ${totalCount} points, ${values.length} value arrays, udv = ${udv}`);
+
+    const attributes = {
+      getPosition: {
+        value: positions,
+        type: 'float32',
+        size: 2
+      },
+      udv
+    };
+
+    values.forEach((valArray, k) => {
+      let value = valArray;
+      if (valArray instanceof BigUint64Array) {
+        value = Float64Array.from(valArray, v => Number(v));
+      }
+      attributes[`value${k}`] = {
+        value,
+        type: value instanceof Float64Array ? 'float64' :
+              value instanceof Uint16Array ? 'uint16' :
+              value instanceof Uint32Array ? 'uint32' : 'float32',
+        size: 1
+      };
+    });
+
+    return {
+      length: totalCount,
+      attributes
+    };
+  },
+  options: {} // Optional: Add custom options if needed
+};
+
+export default DougalBinaryLoader;

lib/www/client/source/src/lib/deck.gl/DougalEventsLayer.js

@@ -0,0 +1,144 @@
+// Ref.: https://deck.gl/docs/developer-guide/custom-layers/composite-layers
+import { CompositeLayer } from '@deck.gl/core';
+import { GeoJsonLayer, ColumnLayer } from '@deck.gl/layers';
+
+class DougalEventsLayer extends CompositeLayer {
+  static layerName = "DougalEventsLayer";
+
+  static defaultProps = {
+    columnsZoom: 11, // Threshold zoom level for switching layers
+    jitter: 0, // Add a small amount of jitter so that columns do not overlap.
+    // GeoJsonLayer props
+    getLineColor: [127, 65, 90],
+    getFillColor: [127, 65, 90],
+    getPointRadius: 2,
+    radiusUnits: "pixels",
+    pointRadiusMinPixels: 2,
+    lineWidthMinPixels: 2,
+    // ColumnLayer props
+    getPosition: { type: 'accessor', value: d => d.geometry.coordinates },
+    getElevation: { type: 'accessor', value: d => Math.min(Math.max(d.properties.remarks?.length || 10, 10), 200) },
+    diskResolution: 20,
+    radius: 5,
+    radiusUnits: "pixels",
+    radiusScale: 1,
+    elevationScale: 1,
+    filled: true,
+    stroked: false,
+    extruded: true,
+    wireframe: false,
+    material: true,
+    getFillColor: [255, 0, 0, 200],
+    getLineColor: [255, 0, 0, 200],
+    getLineWidth: 2,
+    pickable: true
+  }
+
+  constructor(props) {
+    super(props);
+    this.uid = "el-" + Math.random().toString().slice(2);
+    // Initialize state with current zoom
+    this.state = {
+      zoom: this.context?.viewport?.zoom || 0
+    };
+  }
+
+  shouldUpdateState({ changeFlags }) {
+    // Always update if viewport changed (including zoom)
+    if (changeFlags.viewportChanged) {
+      return true;
+    }
+    return super.shouldUpdateState({ changeFlags });
+  }
+
+  updateState({ props, oldProps, context, changeFlags }) {
+    // Check if zoom has changed
+    const newZoom = context.viewport?.zoom || 0;
+    if (newZoom !== this.state.zoom) {
+      this.setState({ zoom: newZoom });
+      this.setNeedsRedraw(); // Trigger re-render of sublayers
+      console.log(`Zoom changed to ${newZoom}, triggering redraw`);
+    }
+  }
+
+  getPickingInfo({ info, mode, sourceLayer }) {
+    if (info.index >= 0) {
+      info.object = {
+        ...info.object // Merge default picking info (GeoJSON feature or ColumnLayer object)
+      };
+      if (sourceLayer) {
+        info.object.type = sourceLayer.constructor.layerName;
+      }
+      //console.log(`Picked ${info.object.type}, index ${info.index}`);
+    }
+    return info;
+  }
+
+  renderLayers() {
+    const { zoom } = this.state;
+    const sublayers = [];
+
+    if (zoom >= this.props.columnsZoom) {
+      // Render ColumnLayer at high zoom
+      const data = Array.isArray(this.props.data) ? this.props.data : this.props.data.features || [];
+
+      const positionFn = this.props.jitter
+        ? (d, info) => {
+          let pos;
+          if (typeof this.props.getPosition == 'function') {
+            pos = this.props.getPosition(d, info);
+          } else {
+            pos = this.props.getPosition;
+          }
+            return pos.map( i => i + (Math.random() - 0.5) * this.props.jitter )
+        }
+        : this.props.getPosition;
+
+      sublayers.push(
+        new ColumnLayer(this.getSubLayerProps({
+          id: `${this.uid}-column`,
+          data,
+          visible: this.props.visible,
+          getPosition: positionFn,
+          getElevation: this.props.getElevation,
+          diskResolution: this.props.diskResolution,
+          radius: this.props.radius,
+          radiusUnits: this.props.radiusUnits,
+          radiusScale: this.props.radiusScale,
+          elevationScale: this.props.elevationScale,
+          filled: this.props.filled,
+          stroked: this.props.stroked,
+          extruded: this.props.extruded,
+          wireframe: this.props.wireframe,
+          material: this.props.material,
+          getFillColor: this.props.getFillColor,
+          getLineColor: this.props.getLineColor,
+          getLineWidth: this.props.getLineWidth,
+          pickable: this.props.pickable
+        }))
+      );
+    } else {
+      // Render GeoJsonLayer at low zoom
+      sublayers.push(
+        new GeoJsonLayer(this.getSubLayerProps({
+          id: `${this.uid}-geojson`,
+          data: this.props.data,
+          visible: this.props.visible,
+          getLineColor: this.props.getLineColor,
+          getFillColor: this.props.getFillColor,
+          getPointRadius: this.props.getPointRadius,
+          radiusUnits: this.props.radiusUnits,
+          pointRadiusMinPixels: this.props.pointRadiusMinPixels,
+          lineWidthMinPixels: this.props.lineWidthMinPixels,
+          pickable: this.props.pickable
+        }))
+      );
+    }
+
+    console.log(`Rendering ${sublayers.length} sublayer(s) at zoom ${zoom}`);
+
+    return sublayers;
+  }
+}
+
+export default DougalEventsLayer;

lib/www/client/source/src/lib/deck.gl/DougalSequenceLayer.js

@@ -0,0 +1,108 @@
+// Ref.: https://deck.gl/docs/developer-guide/custom-layers/layer-lifecycle
+import { ScatterplotLayer } from '@deck.gl/layers';
+
+class DougalSequenceLayer extends ScatterplotLayer {
+  static layerName = "DougalSequenceLayer";
+
+  static defaultProps = {
+    ...ScatterplotLayer.defaultProps,
+    valueIndex: 0,
+    radiusUnits: "pixels",
+    radiusScale: 1,
+    lineWidthUnits: "pixels",
+    lineWidthScale: 1,
+    stroked: false,
+    filled: true,
+    radiusMinPixels: 1,
+    radiusMaxPixels: 50,
+    lineWidthMinPixels: 1,
+    lineWidthMaxPixels: 50,
+    getPosition: { type: 'accessor', value: d => d.positions },
+    getRadius: 5,
+    getFillColor: [255, 0, 0, 200],
+    getLineColor: [255, 0, 0, 200],
+    getLineWidth: 2,
+    pickable: true
+  }
+
+  constructor(props) {
+    super(props);
+  }
+
+  initializeState(context) {
+    super.initializeState(context);
+  }
+
+  getPickingInfo({ info, mode }) {
+    const index = info.index;
+    if (index >= 0) {
+      const d = this.props.data.attributes;
+      if (d) {
+        if (d.udv == 0) {
+          info.object = {
+            udv: d.udv,
+            i: d.value0.value[index],
+            j: d.value1.value[index],
+            ntba: d.value2.value[index] & 0x01,
+            sailline_ntba: d.value2.value[index] & 0x02
+          };
+        } else if (d.udv == 1) {
+          info.object = {
+            udv: d.udv,
+            i: d.value0.value[index],
+            j: d.value1.value[index],
+            sailline: d.value3.value[index],
+            ntba: d.value2.value[index] & 0x01 ? true : false,
+            sailline_ntba: d.value2.value[index] & 0x02 ? true : false
+          };
+        } else if (d.udv == 2) {
+          info.object = {
+            udv: d.udv,
+            i: d.value0.value[index],
+            j: d.value1.value[index],
+            ts: Number(d.value2.value[index]),
+            εi: d.value3.value[index] / 100,
+            εj: d.value4.value[index] / 100,
+            delta_μ: d.value5.value[index] / 10,
+            delta_σ: d.value6.value[index] / 10,
+            delta_R: d.value7.value[index] / 10,
+            press_μ: d.value8.value[index],
+            press_σ: d.value9.value[index],
+            press_R: d.value10.value[index],
+            depth_μ: d.value11.value[index] / 10,
+            depth_σ: d.value12.value[index] / 10,
+            depth_R: d.value13.value[index] / 10,
+            fill_μ: d.value14.value[index],
+            fill_σ: d.value15.value[index],
+            fill_R: d.value16.value[index],
+            delay_μ: d.value17.value[index] / 10,
+            delay_σ: d.value18.value[index] / 10,
+            delay_R: d.value19.value[index] / 10,
+            nofire: d.value20.value[index] >> 4,
+            autofire: d.value20.value[index] & 0xf
+          };
+        } else if (d.udv == 3) {
+          info.object = {
+            udv: d.udv,
+            i: d.value0.value[index],
+            j: d.value1.value[index],
+            ts: Number(d.value2.value[index]),
+            εi: d.value3.value[index] / 100,
+            εj: d.value4.value[index] / 100,
+            co_i: d.value5.value[index] / 100,
+            co_j: d.value6.value[index] / 100,
+          }
+        } else {
+          console.warn(`Unknown udv value ${d.udv}. No picking info`);
+          info.object = {};
+        }
+        console.log(`Picked sequence ${info.object.i}, point ${info.object.j}, udv ${info.object.udv}`);
+      } else {
+        console.log(`No data found index = ${index}`);
+      }
+    }
+    return info;
+  }
+}
+
+export default DougalSequenceLayer;

lib/www/client/source/src/lib/deck.gl/index.js

@@ -0,0 +1,8 @@
+
+import DougalSequenceLayer from './DougalSequenceLayer'
+import DougalEventsLayer from './DougalEventsLayer'
+
+export {
+  DougalSequenceLayer,
+  DougalEventsLayer
+};

lib/www/client/source/src/lib/durations.js

@@ -0,0 +1,47 @@
+
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+
+export {
+  duration_to_ms,
+  ms_to_duration,
+  normalise_duration,
+  add_durations
+}

lib/www/client/source/src/main.js

@@ -4,7 +4,7 @@ import router from './router'
 import store from './store'
 import vuetify from './plugins/vuetify'
 import vueDebounce from 'vue-debounce'
-import { mapMutations } from 'vuex';
+import { mapMutations, mapActions } from 'vuex';
 import { markdown, markdownInline } from './lib/markdown';
 import { geometryAsString } from './lib/utils';
 import { mapGetters } from 'vuex';
@@ -46,6 +46,12 @@ new Vue({
 
   methods: {
 
+    async sleep (ms = 0) {
+      return await new Promise( (resolve) => {
+        setTimeout( resolve, ms );
+      });
+    },
+
     markdown (value) {
       return markdown(value);
     },
@@ -56,12 +62,17 @@ new Vue({
 
     showSnack(text, colour = "primary") {
       console.log("showSnack", text, colour);
-      this.snackColour = colour;
-      this.snackText = text;
-      this.snack = true;
+      this.$store.dispatch("showSnack", [text, colour]);
+    },
+
+    sendJwt () {
+      if (this.jwt) {
+        this.ws.send(JSON.stringify({ jwt: this.jwt }));
+      }
     },
 
     initWs () {
+
       if (this.ws) {
         console.log("WebSocket initWs already called");
         return;
@@ -71,11 +82,12 @@ new Vue({
 
       this.ws.addEventListener("message", (ev) => {
         const msg = JSON.parse(ev.data);
-        this.setServerEvent(msg);
+        this.processServerEvent(msg);
       });
 
       this.ws.addEventListener("open", (ev) => {
         console.log("WebSocket connection open", ev);
+        this.sendJwt()
         this.setServerConnectionState(true);
       });
 
@@ -101,14 +113,13 @@ new Vue({
       }
 
       this.wsCredentialsCheckTimer = setInterval( () => {
-        this.ws.send(JSON.stringify({
-          jwt: this.jwt
-        }));
+        this.sendJwt();
       }, this.wsCredentialsCheckInterval);
 
     },
 
-    ...mapMutations(['setServerEvent', 'setServerConnectionState'])
+    ...mapMutations(['setServerConnectionState']),
+    ...mapActions(['processServerEvent'])
 
   },
 

lib/www/client/source/src/router/index.js

@@ -20,6 +20,9 @@ import ProjectSettings from '../views/ProjectSettings.vue'
 import Users from '../views/Users.vue'
 import DougalAppBarExtensionProject from '../components/app-bar-extension-project'
 import DougalAppBarExtensionProjectList from '../components/app-bar-extension-project-list'
+import GroupList from '../views/GroupList.vue'
+import Group from '../views/Group.vue'
+
 
 Vue.use(VueRouter)
 
@@ -155,6 +158,7 @@ Vue.use(VueRouter)
         component: SequenceList
       },
       {
+        name: "shotlog",
         path: "sequences/:sequence",
         component: SequenceSummary
       },
@@ -196,7 +200,57 @@ Vue.use(VueRouter)
         component: ProjectSettings
       }
     ]
-  }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups",
+    redirect: "/groups/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/",
+    component: GroupList,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups", disabled: true }
+      ],
+      appBarExtension: {
+        // component: DougalAppBarExtensionProjectList
+      }
+    }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group",
+    redirect: "/groups/:group/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group/",
+    name: "Group",
+    component: Group,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups" },
+        { text: (ctx) => ctx.$route.params.group }
+        /*
+        {
+          text: (ctx) => ctx.$store.state.project.projectName || "…",
+          href: (ctx) => `/projects/${ctx.$store.state.project.projectId || ctx.$route.params.project || ""}/`,
+          title: (ctx) => Object.entries(ctx.$store.getters.projectConfiguration?.organisations ?? {}).map( ([org, ops]) => `* ${org}: ${Object.entries(ops).filter( ([k, v]) => v ).map( ([k, v]) => k ).join(", ")}`).join("\n"),
+          organisations: (ctx) => ctx.$store.getters.projectConfiguration?.organisations ?? {}
+        }
+        */
+      ],
+      /*
+      appBarExtension: {
+        component: DougalAppBarExtensionGroup
+      }
+      */
+    },
+    children: [
+    ]
+  },
 ]
 
 const router = new VueRouter({

lib/www/client/source/src/store/modules/api/actions.js

@@ -1,5 +1,27 @@
+const ConcurrencyLimiter = require('@dougal/concurrency');
 
+/** Make an API request
+ *
+ * @a resource {String} is the target URL
+ * @a init {Object} are the Fetch options
+ * @a cb {Function} is a callback function: (res, err) => {}
+ * @a opts {Object} are other optional parameters:
+ *    opts.silent {Boolean} controls whether snack messages are shown on failure
+ *    opts.cache {Object} controls whether Cache API is used
+ *    opts.cache.name {String} is the name of the cache to use. Defaults to "dougal"
+ *
+ * If Cache API is used, this function looks for a matching request in the cache
+ * first, and returns it if found. If not found, it makes the request over the API
+ * and then stores it in the cache.
+ *
+ * `opts.cache` may also be `true` (defaults to using the "dougal" cache),
+ * a cache name (equivalent to {name: "…"}) or even an empty object (equivalent
+ * to `true`).
+ */
 async function api ({state, getters, commit, dispatch}, [resource, init = {}, cb, opts = {}]) {
+
+  const limiter = api.limiter || (api.limiter = new ConcurrencyLimiter(state.maxConcurrent));
+
   try {
     commit("queueRequest");
     if (init && init.hasOwnProperty("body")) {
@@ -15,22 +37,89 @@ async function api ({state, getters, commit, dispatch}, [resource, init = {}, cb
     }
     // We also send Authorization: Bearer …
     if (getters.jwt) {
+      init.credentials = "include";
       init.headers["Authorization"] = "Bearer "+getters.jwt;
     }
     if (typeof init.body != "string") {
       init.body = JSON.stringify(init.body);
     }
     const url = /^https?:\/\//i.test(resource) ? resource : (state.apiUrl + resource);
-    const res = await fetch(url, init);
-    if (typeof cb === 'function') {
-      await cb(null, res);
+
+    let res; // The response
+    let cache; // Potentially, a Cache API cache name
+    let isCached;
+
+    if (opts?.cache === true) {
+      opts.cache = { name: "dougal" };
+    } else if (typeof opts?.cache === "string") {
+      opts.cache = { name: opts.cache };
+    } else if (opts?.cache) {
+      if (!(opts.cache instanceof Object)) {
+        opts.cache = { name: "dougal" }
+      } else if (!(opts.cache.name)) {
+        opts.cache.name = "dougal";
+      }
     }
+
+    if (opts?.cache && window.caches) {
+      cache = await caches.open(opts.cache.name);
+      res = await cache.match(url);
+      isCached = !!res;
+    }
+
+    if (!res) {
+      res = await limiter.enqueue(async () => await fetch(url, init));
+    }
+
+    if (cache && !isCached && res.ok) { // Only cache successful responses
+      cache.put(url, res.clone());
+    }
+
+    if (typeof cb === 'function') {
+      await cb(null, res.clone());
+    }
+
+    if (res.headers.has("x-dougal-server")) {
+      const header = res.headers.get("x-dougal-server")
+      const entries = header
+        .split(";")
+        .map(part => part.trim())
+        .filter(part => part.length > 0)
+        .map(part => {
+          const idx = part.indexOf('=');
+          if (idx === -1) {
+            return [part, true];
+          }
+          const key = part.slice(0, idx).trim();
+          const value = part.slice(idx + 1).trim();
+          return [key, value];
+        });
+        state.serverInfo = entries.length ? Object.fromEntries(entries) : {};
+
+        if (state.serverInfo["remote-frontend"]) {
+          state.isGatewayReliable = ![ 502, 503, 504 ].includes(res.status);
+        } else {
+          state.isGatewayReliable = null;
+        }
+    }
+
     if (res.ok) {
 
-      await dispatch('setCredentials');
+      if (!isCached) {
+        if (res.headers.has("x-jwt")) {
+          await dispatch('setCredentials', { response: res });
+        }
+      }
 
       try {
-        return init.text ? (await res.text()) : (await res.json());
+        if (!res.bodyUsed) { // It may have been consumed by a callback
+          const validFormats = [ "arrayBuffer", "blob", "formData", "json", "text" ];
+          if (opts.format && validFormats.includes(opts.format)) {
+            return await res[opts.format]();
+          } else {
+            return await res.json();
+          }
+        }
       } catch (err) {
         if (err instanceof SyntaxError) {
           if (Number(res.headers.get("Content-Length")) === 0) {

lib/www/client/source/src/store/modules/api/state.js

@@ -1,6 +1,9 @@
 const state = () => ({
   apiUrl: "/api",
-  requestsCount: 0
+  requestsCount: 0,
+  maxConcurrent: 15,
+  serverInfo: {}, // Contents of the last received X-Dougal-Server HTTP header
+  isGatewayReliable: null, // True if we start seeing HTTP 502‒504 responses
 });
 
 export default state;

lib/www/client/source/src/store/modules/event/actions.js

@@ -17,6 +17,7 @@ async function refreshEvents ({commit, dispatch, state, rootState}, [modifiedAft
     ? `/project/${pid}/event/changes/${(new Date(modifiedAfter)).toISOString()}?unique=t`
     : `/project/${pid}/event`;
   const init = {
+    cache: "reload",
     signal: state.loading.signal
   };
   const res = await dispatch('api', [url, init]);
@@ -35,7 +36,7 @@ async function refreshEvents ({commit, dispatch, state, rootState}, [modifiedAft
 
 /** Return a subset of events from state.events
  */
-async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text, label}]) {
+async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text, label, excludeLabels}]) {
   let filteredEvents = [...state.events];
 
   if (sortBy) {
@@ -113,6 +114,10 @@ async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date
     filteredEvents = filteredEvents.filter( event => event.labels?.includes(label) );
   }
 
+  if (excludeLabels) {
+    filteredEvents = filteredEvents.filter( event => !excludeLabels?.some( label => event.labels?.includes(label) ) );
+  }
+
   const count = filteredEvents.length;
 
   if (itemsPerPage && itemsPerPage > 0) {

lib/www/client/source/src/store/modules/label/actions.js

@@ -11,6 +11,7 @@ async function refreshLabels ({commit, dispatch, state, rootState}) {
   const pid = rootState.project.projectId;
   const url = `/project/${pid}/label`;
   const init = {
+    cache: "reload",
     signal: state.loading.signal
   };
   const res = await dispatch('api', [url, init]);

lib/www/client/source/src/store/modules/line/actions.js

@@ -11,6 +11,7 @@ async function refreshLines ({commit, dispatch, state, rootState}) {
   const pid = rootState.project.projectId;
   const url = `/project/${pid}/line`;
   const init = {
+    cache: "reload",
     signal: state.loading.signal
   };
   const res = await dispatch('api', [url, init]);

lib/www/client/source/src/store/modules/notify/actions.js

@@ -0,0 +1,83 @@
+import debounce from 'lodash/debounce';
+
+function registerHandler({ commit }, { table, handler }) {
+  commit('REGISTER_HANDLER', { table, handler });
+}
+
+function unregisterHandler({ commit }, { table, handler }) {
+  commit('UNREGISTER_HANDLER', { table, handler });
+}
+
+function processServerEvent({ commit, dispatch, state, rootState }, message) {
+  //console.log("processServerEvent", message);
+  // Error handling for invalid messages
+  if (!message) {
+    console.error("processServerEvent called without arguments");
+    return;
+  }
+
+  if (!message.channel) {
+    console.error("processServerEvent message missing channel");
+    return;
+  }
+
+  if (!message.payload) {
+    console.error("processServerEvent message missing payload");
+    return;
+  }
+
+  if (message.payload.operation == "INSERT") {
+    if (message.payload.new == null) {
+      console.error("Expected payload.new to be non-null");
+      return;
+    }
+  } else if (message.payload.operation == "UPDATE") {
+    if (message.payload.old == null || message.payload.new == null) {
+      console.error("Expected payload.old and paylaod.new to be non-null");
+      return;
+    }
+  } else if (message.payload.operation == "DELETE") {
+    if (message.payload.old == null) {
+      console.error("Expected payload.old to be non-null");
+      return;
+    }
+  } else {
+    console.warn(`Unrecognised operation: ${message.payload.operation}`);
+  }
+
+  const table = message.channel; // or message.payload?.table;
+  //console.log("table=", table);
+  if (!table || !state.handlers[table] || state.handlers[table].length === 0) {
+    return;
+  }
+
+  // Create a debounced runner per table if not exists
+  if (!state.debouncedRunners) {
+    state.debouncedRunners = {}; // Not reactive needed? Or use Vue.set
+  }
+  if (!state.debouncedRunners[table]) {
+    const config = {
+      wait: 300, // min silence in ms
+      maxWait: 1000, // max wait before force run, adjustable
+      trailing: true,
+      leading: false
+    };
+    state.debouncedRunners[table] = debounce((lastMessage) => {
+      const context = { commit, dispatch, state: rootState, rootState }; // Approximate action context
+      state.handlers[table].forEach(handler => {
+        try {
+          //console.log("Trying handler:", handler);
+          handler(context, lastMessage);
+        } catch (e) {
+          console.error(`Error in handler for table ${table}:`, e);
+        }
+      });
+    }, config.wait, { maxWait: config.maxWait });
+  }
+
+  // Call the debounced function with the current message
+  // Debounce will use the last call's argument if multiple
+  state.debouncedRunners[table](message);
+}
+
+export default { registerHandler, unregisterHandler, processServerEvent };

lib/www/client/source/src/store/modules/notify/mutations.js

@@ -11,4 +11,29 @@ function setServerConnectionState (state, isConnected) {
   state.serverConnected = !!isConnected;
 }
 
-export default { setServerEvent, clearServerEvent, setServerConnectionState };
+function REGISTER_HANDLER(state, { table, handler }) {
+  if (!state.handlers[table]) {
+    state.handlers[table] = [];
+  }
+  if (!state.handlers[table].includes(handler)) {
+    state.handlers[table].push(handler);
+  }
+}
+
+function UNREGISTER_HANDLER(state, { table, handler }) {
+  if (state.handlers[table]) {
+    const handlerIndex = state.handlers[table].findIndex(el => el === handler);
+    if (handlerIndex != -1) {
+      state.handlers[table].splice(handlerIndex, 1);
+    }
+  }
+}
+
+
+export default {
+  setServerEvent,
+  clearServerEvent,
+  setServerConnectionState,
+  REGISTER_HANDLER,
+  UNREGISTER_HANDLER
+};

lib/www/client/source/src/store/modules/notify/state.js

@@ -1,6 +1,7 @@
 const state = () => ({
   serverEvent: null,
-  serverConnected: false
+  serverConnected: false,
+  handlers: {}, // table: array of functions (each fn receives { commit, dispatch, state, rootState, message })
 });
 
 export default state;

lib/www/client/source/src/store/modules/plan/actions.js

@@ -11,6 +11,7 @@ async function refreshPlan ({commit, dispatch, state, rootState}) {
   const pid = rootState.project.projectId;
   const url = `/project/${pid}/plan`;
   const init = {
+    cache: "reload",
     signal: state.loading.signal
   };
   const res = await dispatch('api', [url, init]);

lib/www/client/source/src/store/modules/plan/mutations.js

@@ -1,18 +1,47 @@
 
 
 function transform (item) {
-  item.ts0 = new Date(item.ts0);
-  item.ts1 = new Date(item.ts1);
-  return item;
+  const newItem = {...item}
+  newItem.ts0 = new Date(newItem.ts0);
+  newItem.ts1 = new Date(newItem.ts1);
+  return newItem;
 }
 
 // ATTENTION: This relies on the new planner endpoint
 // as per issue #281.
 
+function setRemarks (state, remarks) {
+  state.remarks = remarks;
+}
+
+function setSequence (state, sequence) {
+  state.sequences.push(Object.freeze(transform(sequence)));
+}
+
+function deleteSequence (state, sequence) {
+  const seq = transform(sequence)
+  const idx = state.sequences?.findIndex( s => Object.keys(seq).every( k => JSON.stringify(s[k]) == JSON.stringify(seq[k]) ));
+  if (idx != -1) {
+    state.sequences.splice(idx, 1)
+  }
+}
+
+function replaceSequence (state, [oldSequence, newSequence]) {
+  console.log("replaceSequence", oldSequence, newSequence);
+  const seq = transform(oldSequence)
+  const idx = state.sequences?.findIndex( s => Object.keys(seq).every( k => JSON.stringify(s[k]) == JSON.stringify(seq[k]) ));
+  console.log("idx", idx);
+  if (idx != -1) {
+    state.sequences.splice(idx, 1, transform(newSequence))
+    console.log("spliced in");
+  }
+}
+
 function setPlan (state, plan) {
   // We don't need or want the planned sequences array to be reactive
-  state.sequences = Object.freeze(plan.sequences.map(transform));
-  state.remarks = plan.remarks;
+  state.sequences = [];
+  plan.sequences.forEach( sequence => setSequence(state, sequence) );
+  setRemarks(state, plan.remarks);
 }
 
 function setPlanLoading (state, abortController = new AbortController()) {
@@ -51,6 +80,10 @@ function abortPlanLoading (state) {
 }
 
 export default {
+  setRemarks,
+  setSequence,
+  deleteSequence,
+  replaceSequence,
   setPlan,
   setPlanLoading,
   clearPlanLoading,

lib/www/client/source/src/store/modules/plan/state.js

@@ -1,5 +1,5 @@
 const state = () => ({
-  sequences: Object.freeze([]),
+  sequences: [],
   remarks: null,
   loading: null,
   timestamp: null,

lib/www/client/source/src/store/modules/project/actions.js

@@ -1,7 +1,13 @@
 
 async function getProject ({commit, dispatch}, projectId) {
+  if (projectId == null) {
+    console.log(`Skipping call to getProject${projectId})`);
+    return;
+  }
+
   const init = {
     headers: {
+      cache: "reload",
       "If-None-Match": "" // Ensure we get a fresh response
     }
   };

lib/www/client/source/src/store/modules/projects/actions.js

@@ -1,25 +1,55 @@
+import * as d3a from 'd3-array';
+import { duration_to_ms, ms_to_duration, normalise_duration, add_durations } from '@/lib/durations';
 
 /** Fetch projects from server
  */
 async function refreshProjects ({commit, dispatch, state, rootState}) {
 
+  async function getSummary (project) {
+      const url = `/project/${project.pid}/summary`;
+      const init = {};
+      const summary = await dispatch('api', [url, init, null, {silent:true}]);
+      if (summary) {
+        return {...project, ...summary};
+      } else {
+        return project;
+      }
+  }
+
   if (state.loading) {
     commit('abortProjectsLoading');
   }
 
   commit('setProjectsLoading');
+  const tstamp = new Date();
   const pid = rootState.project.projectId;
   const url = `/project`;
   const init = {
+    cache: "reload",
     signal: state.loading.signal
   };
   const res = await dispatch('api', [url, init, null, {silent:true}]);
 
   if (res) {
-    commit('setProjects', res);
-    commit('setProjectsTimestamp');
+
+    let projects;
+
+    if (res.some( project => project.pid == null )) {
+      console.warn("At least one project found with no PID!");
+      projects = res.filter( project => project.pid != null );
+    } else {
+      projects = res;
+    }
+
+    commit('setProjects', projects); // First without summaries
+    commit('setProjectsTimestamp', tstamp);
+
+    projects = await Promise.all(projects.map( getSummary ));
+
+    commit('setProjects', projects); // Then with summaries
   }
   commit('clearProjectsLoading');
+  dispatch('prepareGroups');
 }
 
 /** Return a subset of projects from state.projects
@@ -117,4 +147,83 @@ async function getProjects ({commit, dispatch, state}, [{pid, name, schema, grou
   return {projects: filteredProjects, count};
 }
 
-export default { refreshProjects, getProjects };
+
+
+async function prepareGroups ({commit, dispatch, state, rootState}) {
+  const groups = {};
+
+  for (const project of state.projects) {
+
+    if (!project.prod_distance) {
+      // This project has no production data (either not started yet
+      // or production data has not been imported) so we skip it.
+      continue;
+    }
+
+    if (!project.prod_duration.days) {
+      project.prod_duration = normalise_duration(project.prod_duration);
+    }
+
+    for (const name of project.groups) {
+      if (!(name in groups)) {
+        groups[name] = {
+          group: name,
+          num_projects: 0,
+          lines: 0,
+          points: 0,
+          sequences: 0,
+          // Shots:
+          prime: 0,
+          other: 0,
+          ntba: 0,
+          prod_duration: {
+            days: 0,
+            hours: 0,
+            minutes: 0,
+            seconds: 0,
+            milliseconds: 0
+          },
+          prod_distance: 0,
+          shooting_rate: [],
+          projects: []
+        };
+      }
+      const group = groups[name];
+
+      group.num_projects++;
+      group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+      group.points = Math.max(group.points, project.total); // Idem
+      group.sequences += project.seq_final;
+      group.prime += project.prime;
+      group.other += project.other;
+      //group.ntba += project.ntba;
+      group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+      group.prod_distance += project.prod_distance;
+      group.shooting_rate.push(project.shooting_rate);
+      group.projects.push(project);
+    }
+  }
+
+  const grouplist = [];
+  for (const group of Object.values(groups)) {
+    group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+    group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+    delete group.shooting_rate;
+
+    grouplist.push(group);
+  }
+
+  commit('setGroups', grouplist);
+
+}
+
+async function getGroups({commit, dispatch, state, rootState}) {
+  if (!state.groups.length) {
+    await dispatch('refreshProjects');
+  }
+
+  return state.groups;
+}
+
+
+export default { refreshProjects, getProjects, prepareGroups, getGroups };

lib/www/client/source/src/store/modules/projects/getters.js

@@ -3,7 +3,7 @@ function projects (state) {
   return state.projects;
 }
 
-function projectGroups (state) {
+function projectGroupNames (state) {
   return [...new Set(state.projects.map(i => i.groups).flat())].sort();
 }
 
@@ -15,4 +15,8 @@ function projectsLoading (state) {
   return !!state.loading;
 }
 
-export default { projects, projectGroups, projectCount, projectsLoading };
+function groups (state) {
+  return state.groups;
+}
+
+export default { projects, projectGroupNames, projectCount, projectsLoading, groups };

lib/www/client/source/src/store/modules/projects/mutations.js

@@ -39,10 +39,15 @@ function abortProjectsLoading (state) {
   state.loading = null;
 }
 
+function setGroups (state, groups) {
+  state.groups = Object.freeze(groups);
+}
+
 export default {
   setProjects,
   setProjectsLoading,
   clearProjectsLoading,
   setProjectsTimestamp,
-  setProjectsETag
+  setProjectsETag,
+  setGroups
 };

lib/www/client/source/src/store/modules/projects/state.js

@@ -1,5 +1,6 @@
 const state = () => ({
   projects: Object.freeze([]),
+  groups: Object.freeze([]),
   loading: null,
   timestamp: null,
   etag: null,

lib/www/client/source/src/store/modules/sequence/actions.js

@@ -11,6 +11,7 @@ async function refreshSequences ({commit, dispatch, state, rootState}) {
   const pid = rootState.project.projectId;
   const url = `/project/${pid}/sequence?files=true`;
   const init = {
+    cache: "reload",
     signal: state.loading.signal
   };
   const res = await dispatch('api', [url, init]);

lib/www/client/source/src/store/modules/user/actions.js

@@ -1,7 +1,7 @@
 import jwt_decode from 'jwt-decode';
 import { User } from '@/lib/user';
 
-async function login ({commit, dispatch}, loginRequest) {
+async function login ({ commit, dispatch }, loginRequest) {
   const url = "/login";
   const init = {
     method: "POST",
@@ -9,93 +9,86 @@ async function login ({commit, dispatch}, loginRequest) {
       "Content-Type": "application/json"
     },
     body: loginRequest
+  };
+
+  const callback = async (err, res) => {
+    if (!err && res) {
+      const { token } = (await res.json());
+      await dispatch('setCredentials', {token});
+    }
   }
-  const res = await dispatch('api', [url, init]);
-  if (res && res.ok) {
-    await dispatch('setCredentials', {force: true});
-    await dispatch('loadUserPreferences');
-  }
+
+  await dispatch('api', [url, init, callback]);
+  await dispatch('loadUserPreferences');
 }
 
-async function logout ({commit, dispatch}) {
-  commit('setCookie', null);
+async function logout ({ commit, dispatch }) {
+  commit('setToken', null);
   commit('setUser', null);
-  // Should delete JWT cookie
   await dispatch('api', ["/logout"]);
-
-  // Clear preferences
   commit('setPreferences', {});
 }
 
-function browserCookie (state) {
-  return document.cookie.split(/; */).find(i => /^JWT=.+/.test(i));
-}
+function setCredentials({ state, commit, getters, dispatch, rootState }, { force, token, response } = {}) {
+  try {
+    let tokenValue = token;
 
-function cookieChanged (cookie) {
-  return browserCookie != cookie;
-}
+    if (!tokenValue && response?.headers?.get('x-jwt')) {
+      tokenValue = response.headers.get('x-jwt');
+    }
 
-function setCredentials ({state, commit, getters, dispatch, rootState}, {force, token} = {}) {
-  if (token || force || cookieChanged(state.cookie)) {
-    try {
-      const cookie = browserCookie();
-      const decoded = (token ?? cookie) ? jwt_decode(token ?? cookie.split("=")[1]) : null;
-      commit('setCookie', (cookie ?? (token && ("JWT="+token))) || undefined);
+    if (!tokenValue) {
+      console.log('No JWT found in token or response');
+      return;
+    }
+
+    if (force || tokenValue !== getters.jwt) {
+      const decoded = jwt_decode(tokenValue);
+      commit('setToken', tokenValue);
       commit('setUser', decoded ? new User(decoded, rootState.api.api) : null);
-    } catch (err) {
-      if (err.name == "InvalidTokenError") {
-        console.warn("Failed to decode", browserCookie());
-      } else {
-        console.error("setCredentials", err);
-      }
+      commit('setCookie', {name: "JWT", value: tokenValue, expires: (decoded.exp??0)*1000});
+
+      console.log('Credentials refreshed at', new Date().toISOString());
+    } else {
+      console.log('JWT unchanged, skipping update');
+    }
+  } catch (err) {
+    console.error('setCredentials error:', err.message, 'token:', token, 'response:', response?.headers?.get('x-jwt'));
+    if (err.name === 'InvalidTokenError') {
+      commit('setToken', null);
+      commit('setUser', null);
+      commit('clearCookie', "JWT")
     }
   }
   dispatch('loadUserPreferences');
 }
 
-/**
- * Save user preferences to localStorage and store.
- *
- * User preferences are identified by a key that gets
- * prefixed with the user ID. The value can
- * be anything that JSON.stringify can parse.
- */
-function saveUserPreference ({state, commit}, [key, value]) {
+function saveUserPreference({ state, commit }, [key, value]) {
   const k = `${state.user?.id}.${key}`;
-
   if (value !== undefined) {
     localStorage.setItem(k, JSON.stringify(value));
-
-    const preferences = state.preferences;
-    preferences[key] = value;
+    const preferences = { ...state.preferences, [key]: value };
     commit('setPreferences', preferences);
   } else {
     localStorage.removeItem(k);
-
-    const preferences = state.preferences;
+    const preferences = { ...state.preferences };
     delete preferences[key];
     commit('setPreferences', preferences);
   }
 }
 
-async function loadUserPreferences ({state, commit}) {
-  // Get all keys which are of interest to us
+async function loadUserPreferences({ state, commit }) {
   const prefix = `${state.user?.id}`;
-  const keys = Object.keys(localStorage).filter( k => k.startsWith(prefix) );
-
-  // Build the preferences object
+  const keys = Object.keys(localStorage).filter(k => k.startsWith(prefix));
   const preferences = {};
-  keys.map(str => {
+  keys.forEach(str => {
     const value = JSON.parse(localStorage.getItem(str));
     const key = str.split(".").slice(2).join(".");
     preferences[key] = value;
   });
-
-  // Commit it
   commit('setPreferences', preferences);
 }
 
-
 export default {
   login,
   logout,

lib/www/client/source/src/store/modules/user/getters.js

@@ -4,9 +4,7 @@ function user (state) {
 }
 
 function jwt (state) {
-  if (state.cookie?.startsWith("JWT=")) {
-    return state.cookie.substring(4);
-  }
+  return state.token;
 }
 
 function preferences (state) {

lib/www/client/source/src/store/modules/user/mutations.js

@@ -1,6 +1,11 @@
 
-function setCookie (state, cookie) {
-  state.cookie = cookie;
+function setToken (state, token) {
+  state.token = token;
+  if (token) {
+    localStorage?.setItem("jwt", token);
+  } else {
+    localStorage?.removeItem("jwt");
+  }
 }
 
 function setUser (state, user) {
@@ -11,4 +16,18 @@ function setPreferences (state, preferences) {
   state.preferences = preferences;
 }
 
-export default { setCookie, setUser, setPreferences };
+function setCookie (state, opts = {}) {
+  const name = opts.name ?? "JWT";
+  const value = opts.value ?? "";
+  const expires = opts.expires ? (new Date(opts.expires)) : (new Date(0));
+  const path = opts.path ?? "/";
+  const sameSite = opts.sameSite ?? "Lax";
+
+  document.cookie = `${name}=${value};path=${path};SameSite=${sameSite};expires=${expires.toUTCString()}`;
+}
+
+function clearCookie (state, name) {
+  setCookie(state, {name});
+}
+
+export default { setToken, setUser, setPreferences, setCookie, clearCookie };

lib/www/client/source/src/store/modules/user/state.js

@@ -1,5 +1,5 @@
 const state = () => ({
-  cookie: null,
+  token: localStorage?.getItem("jwt") ?? null,
   user: null,
   preferences: {}
 });

lib/www/client/source/src/views/Feed.vue

@@ -91,7 +91,7 @@ export default {
     },
 
     async refresh () {
-      const text = await this.api([`/rss/?remote=${atob(this.$route.params.source)}`, {text:true}]);
+      const text = await this.api([`/rss/?remote=${atob(this.$route.params.source)}`, {format:"text"}]);
       try {
         this.feed = this.parse(text);
       } catch (err) {

lib/www/client/source/src/views/Group.vue

@@ -0,0 +1,339 @@
+<template>
+    <dougal-group-map v-if="mapView"
+      :baseline="baseline"
+      :monitor="monitor"
+      :monitors="monitors"
+      @input="mapView=$event"
+    ></dougal-group-map>
+  <v-container fluid fill-height class="ma-0 pa-0" v-else>
+
+    <v-overlay :value="loading && !comparisons.length" absolute>
+      <v-progress-circular
+        indeterminate
+        size="64"
+      ></v-progress-circular>
+    </v-overlay>
+
+    <v-overlay :value="!loading && !groupFound" absolute opacity="0.8">
+      <v-row justify="center">
+        <v-alert
+          type="error"
+        >
+          Group not found
+        </v-alert>
+      </v-row>
+      <v-row justify="center">
+        <v-btn color="primary" @click="refreshProjects">Retry</v-btn>
+      </v-row>
+    </v-overlay>
+
+    <v-row no-gutters align="stretch" class="fill-height">
+      <v-col cols="12" v-if="groupFound">
+
+        <v-data-table class="ma-1"
+          :headers="projectHeaders"
+          :items="projects"
+          dense
+        >
+
+          <template v-slot:item.baseline="{item, value, index}">
+            <v-simple-checkbox v-if="index+1 < projects.length"
+              color="primary"
+              :value="baseline === item"
+              @input="setBaseline(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.monitor="{item, value, index}">
+            <v-simple-checkbox v-if="index > 0 && !(index <= baselineIndex)"
+              color="primary"
+              :value="monitor === item"
+              @input="setMonitor(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.pid="{item, value}">
+            <v-chip
+              label
+              small
+              outlined
+              :href="`/projects/${item.pid}`"
+              :color="!item.archived ? 'primary' : ''"
+            >{{ value }}</v-chip>
+          </template>
+
+          <template v-slot:item.fsp="{item, value}">
+            <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.lsp="{item, value}">
+            <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.prod_duration="{item, value}">
+            <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+              {{ value.days }} d
+            </span>
+            <span v-else>
+              {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+            </span>
+          </template>
+
+          <template v-slot:item.prod_distance="{item, value}">
+            {{ (value/1000).toFixed(1) }} km
+          </template>
+
+          <template v-slot:footer.prepend>
+
+            <v-btn v-if="baseline && !mapView"
+              text
+              color="primary"
+              title="Switch to map view"
+              @click="mapView = true"
+            >View map</v-btn>
+
+            <v-btn v-if="comparison"
+              text
+              color="primary"
+              title="Back to summary"
+              @click="clearComparison"
+            >Back</v-btn>
+          </template>
+
+        </v-data-table>
+
+<!-- BEGIN TEST -->
+
+    <dougal-group-comparison-summary v-if="comparison"
+      :baseline="baseline"
+      :monitor="monitor"
+      :comparison="comparison"
+    ></dougal-group-comparison-summary>
+
+    <dougal-group-repeatability-summary v-else-if="comparisons.length"
+      :comparisons="comparisons"
+      :projects="projects"
+      @input="setComparison"
+    ></dougal-group-repeatability-summary>
+
+<!-- END   TEST -->
+
+
+      </v-col>
+      <v-col cols="12" v-else>
+        <v-card>
+          <v-card-text>
+            Group does not exist.
+          </v-card-text>
+        </v-card>
+      </v-col>
+    </v-row>
+  </v-container>
+</template>
+
+<script>
+import { mapActions, mapGetters } from 'vuex'
+import AccessMixin from '@/mixins/access';
+import DougalGroupRepeatabilitySummary from '@/components/groups/group-repeatability-summary.vue';
+import DougalGroupComparisonSummary from '@/components/groups/group-comparison-summary';
+import DougalGroupMap from '@/components/groups/group-map';
+
+export default {
+  name: 'Group',
+
+  mixins: [
+    AccessMixin
+  ],
+
+  components: {
+    DougalGroupRepeatabilitySummary,
+    DougalGroupComparisonSummary,
+    DougalGroupMap
+  },
+
+  data () {
+    return {
+      projectHeaders: [
+        {
+          value: "baseline",
+          text: "Baseline"
+        },
+        {
+          value: "monitor",
+          text: "Monitor"
+        },
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      mapView: false,
+
+      baseline: null,
+      monitor: null,
+      comparisons: []
+    }
+  },
+
+  computed: {
+
+    groupName () {
+      return this.$route.params.group;
+    },
+
+    group () {
+      return this.groups.find( i => i.group === this.groupName );
+    },
+
+    groupFound () {
+      return !!(this.loading || this.group);
+    },
+
+    projects () {
+      return this.group?.projects.toSorted((a, b) => a.pid.localeCompare(b.pid));
+    },
+
+    baselineIndex () {
+      return this.projects.indexOf(this.baseline);
+    },
+
+    monitors () {
+      if (this.baseline && this.comparisons) {
+        return this.comparisons
+          .filter( i => i.baseline_pid == this.baseline.pid )
+          .map( i => this.projects.find( p => p.pid == i.monitor_pid ));
+      } else {
+        return null;
+      }
+    },
+
+    comparison () {
+      return this.comparisons.find( row =>
+        row.baseline_pid == this.baseline?.pid && row.monitor_pid == this.monitor?.pid
+      )?.meta;
+    },
+
+    ...mapGetters(["loading", "groups"])
+  },
+
+  methods: {
+
+    setBaseline (project) {
+      if (project === this.baseline) {
+        this.baseline = null;
+      } else {
+        this.baseline = project;
+        if (this.monitor) {
+          if (this.projects.indexOf(this.monitor) <= this.projects.indexOf(this.baseline)) {
+            this.monitor = null;
+          }
+        }
+      }
+    },
+
+    setMonitor (project) {
+      if (project === this.monitor) {
+        this.monitor = null;
+      } else {
+        this.monitor = project;
+      }
+    },
+
+    clearComparison () {
+      this.baseline = null;
+      this.monitor = null;
+    },
+
+    setComparison (baseline, monitor) {
+      this.clearComparison();
+      this.setBaseline(baseline);
+      this.setMonitor(monitor);
+    },
+
+    async getComparisons () {
+      const url = `/comparison/group/${this.$route.params.group}`;
+      this.comparisons = await this.api([url]);
+    },
+
+    // TODO Should this go in a Vuex action rather?
+    async refreshComparisons () {
+      await this.getGroups();
+      if (this.groupFound) {
+        await this.getComparisons();
+      }
+    },
+
+    /*
+    async getComparison () {
+      if (this.baseline && this.monitor) {
+        const url = `/comparison/group/${this.$route.params.group}/baseline/${this.baseline.pid}/monitor/${this.monitor.pid}`;
+        const comparison = await this.api([url]);
+        if (comparison) {
+          this.comparison = comparison;
+        }
+      }
+    },
+    */
+
+    handleComparisons (context, {payload}) {
+      this.refreshComparisons();
+    },
+
+    registerNotificationHandlers (action = "registerHandler") {
+
+      this.$store.dispatch(action, {
+        table: 'comparisons',
+        handler: this.handleComparisons
+      });
+
+    },
+
+    unregisterNotificationHandlers () {
+      return this.registerNotificationHandlers("unregisterHandler");
+    },
+
+
+    ...mapActions(["api", "getGroups", "refreshProjects"])
+  },
+
+  async mounted () {
+    this.registerNotificationHandlers();
+    this.refreshComparisons()
+  },
+
+  beforeDestroy () {
+    this.unregisterNotificationHandlers();
+  }
+
+
+}
+</script>

lib/www/client/source/src/views/GroupList.vue

@@ -0,0 +1,396 @@
+<template>
+  <v-container fluid>
+
+    <v-data-table
+    :headers="headers"
+    :items="displayItems"
+    item-key="group"
+    :options.sync="options"
+    :expanded.sync="expanded"
+    show-expand
+    :loading="loading"
+    >
+
+      <template v-slot:item.group="{item, value}">
+        <v-chip
+          label
+          small
+          :href="`./${value}`"
+        >{{ value }}</v-chip>
+      </template>
+
+      <template v-slot:item.shots_total="{item, value}">
+        <div>{{ item.prime + item.other }}</div>
+        <v-progress-linear
+          background-color="secondary"
+          color="primary"
+          :value="item.prime/(item.prime+item.other)*100"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:item.prime="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.other="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.prod_duration="{item, value}">
+        <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+          {{ value.days }} d
+        </span>
+        <span v-else>
+          {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+        </span>
+      </template>
+
+      <template v-slot:item.prod_distance="{item, value}">
+        {{ (value/1000).toFixed(1) }} km
+      </template>
+
+      <template v-slot:item.shooting_rate_mean="{item, value}">
+        {{ (value).toFixed(2) }} s ±{{ (item.shooting_rate_sd).toFixed(3) }} s
+      </template>
+
+      <template v-slot:item.shots_per_point="{item, value}">
+        <div>
+          {{ ((item.prime + item.other)/item.points).toFixed(1) }}
+          ({{ ((((item.prime + item.other)/item.points) / item.num_projects)*100).toFixed(2) }}%)
+        </div>
+        <v-progress-linear
+          :value="((((item.prime + item.other)/item.points) / item.num_projects)*100)"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:expanded-item="{ headers, item }">
+        <td :colspan="headers.length">
+          <v-data-table class="ma-1"
+            :headers="projectHeaders"
+            :items="item.projects"
+            dense
+            hide-default-footer
+          >
+
+            <template v-slot:item.pid="{item, value}">
+              <a :href="`/projects/${value}`" title="Go to project">{{ value }}</a>
+            </template>
+
+            <template v-slot:item.fsp="{item, value}">
+              <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.lsp="{item, value}">
+              <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.prod_duration="{item, value}">
+              <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+                {{ value.days }} d
+              </span>
+              <span v-else>
+                {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+              </span>
+            </template>
+
+            <template v-slot:item.prod_distance="{item, value}">
+              {{ (value/1000).toFixed(1) }} km
+            </template>
+
+          </v-data-table>
+        </td>
+      </template>
+
+    </v-data-table>
+
+  </v-container>
+</template>
+
+<style>
+td p:last-of-type {
+  margin-bottom: 0;
+}
+</style>
+
+<script>
+import { mapActions, mapGetters } from 'vuex';
+import AccessMixin from '@/mixins/access';
+
+
+// FIXME send to lib/utils or so
+/*
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+*/
+
+export default {
+  name: "GroupList",
+
+  components: {
+  },
+
+  mixins: [
+    AccessMixin
+  ],
+
+  data () {
+    return {
+      headers: [
+        {
+          value: "group",
+          text: "Group name"
+        },
+        {
+          value: "num_projects",
+          text: "Number of campaigns"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "points",
+          text: "Preplot points"
+        },
+        {
+          value: "sequences",
+          text: "Total sequences"
+        },
+        {
+          value: "shots_total",
+          text: "Total shots"
+        },
+        {
+          value: "prime",
+          text: "Total prime"
+        },
+        {
+          value: "other",
+          text: "Total reshoot + infill"
+        },
+        /*
+        {
+          value: "ntba",
+          text: "Total NTBA"
+        },
+        */
+        {
+          value: "prod_duration",
+          text: "Total duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Total distance"
+        },
+        {
+          value: "shooting_rate_mean",
+          text: "Shooting rate (mean)"
+        },
+        {
+          value: "shots_per_point",
+          text: "Shots per point"
+        },
+      ],
+      items: [],
+      expanded: [],
+      options: { sortBy: ["group"], sortDesc: [false] },
+
+      projectHeaders: [
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      // Context menu stuff
+      contextMenuShow: false,
+      contextMenuX: 0,
+      contextMenuY: 0,
+      contextMenuItem: null,
+
+      /*
+      // FIXME Eventually need to move this into Vuex
+      groups: []
+      */
+    }
+  },
+
+  computed: {
+    displayItems () {
+      return this.items.filter(i => i.prod_distance);
+    },
+
+    ...mapGetters(['loading', 'groups'])
+  },
+
+  methods: {
+
+    /*
+    async prepareGroups () {
+      //const groups = await this.api(["/prospects"]);
+      //console.log("groups", groups);
+      const groups = {};
+
+      for (const project of this.projects) {
+
+        if (!project.prod_distance) {
+          // This project has no production data (either not started yet
+          // or production data has not been imported) so we skip it.
+          continue;
+        }
+
+        if (!project.prod_duration.days) {
+          project.prod_duration = normalise_duration(project.prod_duration);
+        }
+
+        for (const name of project.groups) {
+          if (!(name in groups)) {
+            groups[name] = {
+              group: name,
+              num_projects: 0,
+              lines: 0,
+              points: 0,
+              sequences: 0,
+              // Shots:
+              prime: 0,
+              other: 0,
+              ntba: 0,
+              prod_duration: {
+                days: 0,
+                hours: 0,
+                minutes: 0,
+                seconds: 0,
+                milliseconds: 0
+              },
+              prod_distance: 0,
+              shooting_rate: [],
+              projects: []
+            };
+          }
+          const group = groups[name];
+
+          group.num_projects++;
+          group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+          group.points = Math.max(group.points, project.total); // Idem
+          group.sequences += project.seq_final;
+          group.prime += project.prime;
+          group.other += project.other;
+          //group.ntba += project.ntba;
+          group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+          group.prod_distance += project.prod_distance;
+          group.shooting_rate.push(project.shooting_rate);
+          group.projects.push(project);
+        }
+      }
+
+      this.groups = [];
+      for (const group of Object.values(groups)) {
+        group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+        group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+        delete group.shooting_rate;
+
+        this.groups.push(group);
+      }
+
+    },
+    */
+
+    async list () {
+      this.items = [...this.groups];
+    },
+
+    async load () {
+      await this.refreshProjects();
+      //await this.prepareGroups();
+      await this.list();
+    },
+
+    registerNotificationHandlers () {
+      this.$store.dispatch('registerHandler', {
+        table: 'project`',
+
+        handler: (context, message) => {
+          if (message.payload?.table == "public") {
+            this.load();
+          }
+        }
+      });
+    },
+
+    ...mapActions(["api", "showSnack", "refreshProjects"])
+  },
+
+  mounted () {
+    this.registerNotificationHandlers();
+    this.load();
+  }
+}
+
+</script>

lib/www/client/source/src/views/Log.vue

@@ -5,6 +5,22 @@
       <v-card-title>
         <v-toolbar flat>
           <v-toolbar-title>
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[sequences.length-1]||{}).sequence }}"
+                title="Go to the first sequence"
+              >
+                <v-icon dense>mdi-chevron-double-left</v-icon>
+              </v-btn>
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[sequenceIndex+1]||{}).sequence }}"
+                title="Go to the previous sequence"
+              >
+                <v-icon dense>mdi-chevron-left</v-icon>
+              </v-btn>
+            </template>
             <span class="d-none d-lg-inline">
               {{
                 $route.params.sequence
@@ -31,18 +47,38 @@
                     : ""
               }}
             </span>
+
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex==0"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[sequenceIndex-1]||{}).sequence }}"
+                title="Go to the next sequence"
+              >
+                <v-icon dense>mdi-chevron-right</v-icon>
+              </v-btn>
+              <v-btn icon small class="mr-1"
+                :disabled="sequenceIndex==0"
+                :to="{name: 'logBySequence', params: { sequence: (sequences[0]||{}).sequence }}"
+                title="Go to the last sequence"
+              >
+                <v-icon dense>mdi-chevron-double-right</v-icon>
+              </v-btn>
+            </template>
+
+            <a v-if="$route.params.sequence"
+              class="mr-3"
+              :href="`/projects/${$route.params.project}/sequences/${$route.params.sequence}`"
+              title="View the shotlog for this sequence"
+            >
+              <v-icon
+                right
+                color="teal"
+              >mdi-format-list-numbered</v-icon>
+            </a>
+
           </v-toolbar-title>
 
-          <a v-if="$route.params.sequence"
-            class="mr-3"
-            :href="`/projects/${$route.params.project}/sequences/${$route.params.sequence}`"
-            title="View the shotlog for this sequence"
-          >
-            <v-icon
-              right
-              color="teal"
-            >mdi-format-list-numbered</v-icon>
-          </a>
+
 
             <dougal-event-edit v-if="$parent.writeaccess()"
               v-model="eventDialog"
@@ -325,6 +361,12 @@
                   @click="labelSearch=label"
                 >{{label}}</v-chip>
               </span>
+              <v-icon v-if="entry.meta.auto || entry.meta.author"
+                x-small
+                left
+                color="primary"
+                :title="entry.meta.author?`Automatic event by ${entry.meta.author}`:'Automatic event'"
+              >mdi-robot</v-icon>
               <dougal-event-edit-history v-if="entry.has_edits && $parent.writeaccess()"
                 :id="entry.id"
                 :disabled="eventsLoading"
@@ -488,17 +530,6 @@ export default {
     rows () {
       const rows = {};
       this.items
-      .filter(i => {
-        return !this.$route.params.sequence || (this.$route.params.sequence == i.sequence)
-      })
-      .filter(i => {
-        for (const label of this.filterableLabels) {
-          if (!this.shownLabels.includes(label) && i.labels.includes(label)) {
-            return false;
-          }
-        }
-        return true;
-      })
       .forEach(i => {
         const key = (i.sequence && i.point) ? (i.sequence+"@"+i.point) : i.tstamp;
         if (!rows[key]) {
@@ -529,6 +560,10 @@ export default {
       .sort( (a, b) => b[1]-a[1] );
     },
 
+    filteredLabels () {
+      return this.filterableLabels.filter( label => !this.shownLabels.includes(label) );
+    },
+
     presetRemarks () {
       return this.projectConfiguration?.events?.presetRemarks ?? [];
     },
@@ -541,7 +576,17 @@ export default {
       }
     },
 
-    ...mapGetters(['user', 'eventsLoading', 'online', 'sequence', 'line', 'point', 'position', 'timestamp', 'lineName', 'events', 'labels', 'userLabels', 'projectConfiguration']),
+    sequenceIndex () {
+      if ("sequence" in this.$route.params) {
+        const index = this.sequences.findIndex( i => i.sequence == this.$route.params.sequence );
+        if (index != -1) {
+          return index;
+        }
+      }
+      // return undefined
+    },
+
+    ...mapGetters(['user', 'eventsLoading', 'online', 'sequence', 'sequences', 'line', 'point', 'position', 'timestamp', 'lineName', 'events', 'labels', 'userLabels', 'projectConfiguration']),
     ...mapState({projectSchema: state => state.project.projectSchema})
 
   },
@@ -549,6 +594,7 @@ export default {
   watch: {
     options: {
       async handler () {
+        this.savePrefs(),
         await this.fetchEvents();
       },
       deep: true
@@ -567,12 +613,19 @@ export default {
     },
 
     filter (newVal, oldVal) {
+      this.savePrefs();
       if (newVal?.toLowerCase() != oldVal?.toLowerCase()) {
         this.fetchEvents();
       }
     },
 
     labelSearch () {
+      this.savePrefs();
+      this.fetchEvents();
+    },
+
+    filteredLabels () {
+      this.savePrefs()
       this.fetchEvents();
     },
 
@@ -581,7 +634,7 @@ export default {
     },
 
     user (newVal, oldVal) {
-      this.itemsPerPage = Number(localStorage.getItem(`dougal/prefs/${this.user?.name}/${this.$route.params.project}/${this.$options.name}/items-per-page`)) || 25;
+      this.loadPrefs();
     }
 
   },
@@ -632,8 +685,10 @@ export default {
 
     async fetchEvents (opts = {}) {
       const options = {
+        sequence: this.$route.params.sequence,
         text: this.filter,
         label: this.labelSearch,
+        excludeLabels: this.filteredLabels,
         ...this.options
       };
       const res = await this.getEvents([this.$route.params.project, options]);
@@ -737,6 +792,13 @@ export default {
         if (event.id) {
           const id = event.id;
           delete event.id;
+
+          // If this is an edit, ensure that it is *either*
+          // a timestamp event or a sequence + point one.
+          if (event.sequence && event.point && event.tstamp) {
+            delete event.tstamp;
+          }
+
           this.putEvent(id, event, callback); // No await
         } else {
           this.postEvent(event, callback); // No await
@@ -829,7 +891,7 @@ export default {
     viewOnMap(item) {
       if (item?.meta && item.meta?.geometry?.type == "Point") {
         const [ lon, lat ] = item.meta.geometry.coordinates;
-        return `map#15/${lon.toFixed(6)}/${lat.toFixed(6)}`;
+        return `map#z15x${lon.toFixed(6)}y${lat.toFixed(6)}::${lon.toFixed(6)},${lat.toFixed(6)}`;
       } else if (item?.items) {
         return this.viewOnMap(item.items[0]);
       }
@@ -864,10 +926,36 @@ export default {
       */
     },
 
+    getPrefsKey () {
+      return `dougal/prefs/${this.user?.name}/${this.$route.params.project}/Log/v1`;
+    },
+
+    savePrefs () {
+      const prefs = {
+        shownLabels: this.shownLabels,
+        labelSearch: this.labelSearch,
+        filter: this.filter,
+        options: this.options
+      };
+      localStorage.setItem(this.getPrefsKey(), JSON.stringify(prefs));
+    },
+
+    loadPrefs () {
+      const stored = localStorage.getItem(this.getPrefsKey());
+      if (stored) {
+        const prefs = JSON.parse(stored);
+        if (prefs.shownLabels !== undefined) this.shownLabels = prefs.shownLabels;
+        if (prefs.labelSearch !== undefined) this.labelSearch = prefs.labelSearch;
+        if (prefs.filter !== undefined) this.filter = prefs.filter;
+        if (prefs.options !== undefined) this.options = prefs.options;
+      }
+    },
+
     ...mapActions(["api", "showSnack", "refreshEvents", "getEvents"])
   },
 
   async mounted () {
+    this.loadPrefs();
     this.fetchEvents();
 
     window.addEventListener('keyup', this.handleKeyboardEvent);

lib/www/client/source/src/views/Login.vue

@@ -81,6 +81,13 @@ export default {
       await this.logout();
       await this.login(this.credentials);
 
+      if (this.user) {
+        console.log("Login successful");
+        // Should trigger auto-refresh over ws as well as authenticating the
+        // user over ws.
+        this.$root.sendJwt();
+      }
+
       if (this.user && !this.user.autologin) {
         this.$router.replace("/");
       } else {

lib/www/client/source/src/views/MapLayersMixin.vue

@@ -0,0 +1,761 @@
+<script>
+// Important info about performance:
+// https://deck.gl/docs/developer-guide/performance#supply-attributes-directly
+
+import { Deck, WebMercatorViewport, FlyToInterpolator, CompositeLayer } from '@deck.gl/core';
+import { GeoJsonLayer, LineLayer, PathLayer, BitmapLayer, ScatterplotLayer, ColumnLayer, IconLayer } from '@deck.gl/layers';
+import {HeatmapLayer} from '@deck.gl/aggregation-layers';
+import { TileLayer, MVTLayer, TripsLayer } from '@deck.gl/geo-layers';
+import { SimpleMeshLayer } from '@deck.gl/mesh-layers';
+import { OBJLoader } from '@loaders.gl/obj';
+
+//import { json } from 'd3-fetch';
+import * as d3a from 'd3-array';
+
+import { DougalBinaryBundle, DougalBinaryChunkSequential, DougalBinaryChunkInterleaved } from '@dougal/binary';
+import { DougalShotLayer } from '@/lib/deck.gl';
+import { DougalSequenceLayer, DougalEventsLayer } from '@/lib/deck.gl';
+import DougalBinaryLoader from '@/lib/deck.gl/DougalBinaryLoader';
+
+import { colors } from 'vuetify/lib'
+
+function hexToArray (hex, defaultValue = [ 0xc0, 0xc0, 0xc0, 0xff ]) {
+
+  if (typeof hex != "string" || hex.length < 6) {
+    return defaultValue;
+  }
+
+  if (hex[0] == "#") {
+    hex = hex.slice(1); // remove the '#' character
+  }
+
+  return [
+    parseInt(hex.slice(0, 2), 16),
+    parseInt(hex.slice(2, 4), 16),
+    parseInt(hex.slice(4, 6), 16),
+    hex.length > 6 ? parseInt(hex.slice(6, 8), 16) : 255
+  ];
+}
+
+function namedColourToArray (name) {
+  const parts = name.split(/\s+/).map( (s, i) =>
+    i
+      ? s.replace("-", "")
+      : s.replace(/-([a-z])/g, (match, group1) => group1.toUpperCase())
+  );
+  parts[0]
+  if (parts.length == 1) parts[1] = "base";
+  const hex = parts.reduce((acc, key) => acc[key], colors);
+  return hexToArray(hex);
+}
+
+export default {
+  name: "MapLayersMixin",
+
+  data () {
+
+    return {
+
+    COLOUR_SCALE_1: [
+      // negative
+      [65, 182, 196],
+      [127, 205, 187],
+      [199, 233, 180],
+      [237, 248, 177],
+
+      // positive
+      [255, 255, 204],
+      [255, 237, 160],
+      [254, 217, 118],
+      [254, 178, 76],
+      [253, 141, 60],
+      [252, 78, 42],
+      [227, 26, 28],
+      [189, 0, 38],
+      [128, 0, 38]
+    ]
+
+    };
+
+  },
+
+  methods: {
+
+    normalisedColourScale(v, scale = this.COLOUR_SCALE_1, min = 0, max = 1) {
+      const range = max-min;
+      const i = Math.min(scale.length, Math.max(Math.round((v-min) / range * scale.length), 0));
+      //console.log(`v=${v}, scale.length=${scale.length}, min=${min}, max=${max}, i=${i}, → ${scale[i]}`);
+      return scale[i];
+    },
+
+
+    makeDataFromBinary ( {positions, values, udv} ) {
+      const totalCount = positions.length / 2;
+
+      const attributes = {
+        getPosition: {
+          value: positions,
+          type: 'float32',
+          size: 2
+        },
+        udv
+      };
+
+      values.forEach((valArray, k) => {
+        let value = valArray;
+        if (valArray instanceof BigUint64Array) {
+          value = Float64Array.from(valArray, v => Number(v));
+        }
+        attributes[`value${k}`] = {
+          value,
+          type: value instanceof Float64Array ? 'float64' :
+                value instanceof Uint16Array ? 'uint16' :
+                value instanceof Uint32Array ? 'uint32' : 'float32',
+          size: 1
+        };
+      });
+
+      return {
+        length: totalCount,
+        attributes
+      };
+    },
+
+    loadOptions (options = {}) {
+      return {
+        loadOptions: {
+          fetch: {
+            method: 'GET',
+            headers: {
+              'Authorization': `Bearer ${this.$store.getters.jwt}`,
+            }
+          },
+          ...options
+        },
+      };
+
+    },
+
+
+    osmLayer (options = {}) {
+      return new TileLayer({
+        id: "osm",
+        // https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Tile_servers
+        data: 'https://c.tile.openstreetmap.org/{z}/{x}/{y}.png',
+
+        minZoom: 0,
+        maxZoom: 19,
+        tileSize: 256,
+
+        renderSubLayers: props => {
+          const {
+            bbox: {west, south, east, north}
+          } = props.tile;
+
+          return new BitmapLayer(props, {
+            data: null,
+            image: props.data,
+            bounds: [west, south, east, north]
+          });
+        },
+        ...options
+      })
+    },
+
+
+    // OSM tiles layer. Handy to make water transparent
+    // but not super reliable yet
+
+    osmVectorLayer (options = {}) {
+      return new MVTLayer({
+        id: 'osm',
+        data: 'https://vector.openstreetmap.org/shortbread_v1/{z}/{x}/{y}.mvt',
+        minZoom: 0,
+        maxZoom: 14,
+        getFillColor: feature => {
+          const layer = feature.properties.layerName;
+          //console.log("layer =", layer, feature.properties.kind);
+          switch (layer) {
+            case "ocean":
+              return [0, 0, 0, 0];
+            case "land":
+              return [ 0x54, 0x6E, 0x7A, 255 ];
+            default:
+              return [ 240, 240, 240, 255 ];
+          }
+        },
+        getLineColor: feature => {
+          if (feature.properties.layer === 'water') {
+            return [0, 0, 0, 0]; // No outline for water
+          }
+          return [192, 192, 192, 255]; // Default line color for roads, etc.
+        },
+        getLineWidth: feature => {
+          if (feature.properties.highway) {
+            return feature.properties.highway === 'motorway' ? 6 : 3; // Example road widths
+          }
+          return 1;
+        },
+        stroked: true,
+        filled: true,
+        pickable: true
+      });
+    },
+
+
+    openSeaMapLayer (options = {}) {
+      return new TileLayer({
+        id: "sea",
+        data: 'https://tiles.openseamap.org/seamark/{z}/{x}/{y}.png',
+
+        minZoom: 0,
+        maxZoom: 19,
+        tileSize: 256,
+
+        renderSubLayers: props => {
+          const {
+            bbox: {west, south, east, north}
+          } = props.tile;
+
+          return new BitmapLayer(props, {
+            data: null,
+            image: props.data,
+            bounds: [west, south, east, north]
+          });
+        },
+        ...options
+      })
+    },
+
+
+    // Norwegian nautical charts
+    // As of 2025, not available for some weird reason
+    nauLayer (options = {}) {
+      return new TileLayer({
+        id: "nau",
+        // https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Tile_servers
+        data: 'https://opencache.statkart.no/gatekeeper/gk/gk.open_gmaps?layers=sjokartraster&zoom={z}&x={x}&y={y}',
+
+        minZoom: 0,
+        maxZoom: 19,
+        tileSize: 256,
+
+        renderSubLayers: props => {
+          const {
+            bbox: {west, south, east, north}
+          } = props.tile;
+
+          return new BitmapLayer(props, {
+            data: null,
+            image: props.data,
+            bounds: [west, south, east, north]
+          });
+        },
+        ...options
+      })
+    },
+
+    vesselTrackPointsLayer (options = {}) {
+
+      if (!this.vesselPosition) return;
+
+      return new SimpleMeshLayer({
+        id: 'navp',
+        data: [ this.vesselPosition ],
+        //getColor: [ 255, 48, 0 ],
+        getColor: [ 174, 1, 174 ],
+        getOrientation: d => [0, (270 - (d.heading ?? d.cmg ?? d.bearing ?? d.lineBearing ?? 0)) % 360 , 0],
+        getPosition: d => [ d.x, d.y ],
+        mesh: `/assets/boat0.obj`,
+        sizeScale: 0.1,
+        loaders: [OBJLoader],
+        pickable: true,
+        ...options
+      });
+
+    },
+
+    vesselTrackLinesLayer (options = {}) {
+
+      const cfg = this.vesselTrackPeriodSettings[this.vesselTrackPeriod];
+
+      let ts1 = new Date(this.vesselTrackLastRefresh*1000);
+      let ts0 = new Date(ts1.valueOf() - cfg.offset);
+      let di = cfg.decimation;
+      let l = 10000;
+
+      const breakLimit = (di ? di*20 : 5 * 60) * 1000;
+
+      let trailLength = (ts1 - ts0) / 1000;
+
+      return new TripsLayer({
+        id: 'navl',
+        data: `/api/vessel/track/?di=${di}&l=${l}&project=&ts0=${ts0.toISOString()}&ts1=${ts1.toISOString()}`,
+        ...this.loadOptions({
+          fetch: {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${this.$store.getters.jwt}`,
+            }
+          }
+        }),
+        dataTransform: (data) => {
+            if (data.length >= l) {
+              console.warn(`Vessel track data may be truncated! Limit: ${l}`);
+            }
+
+            const paths = [];
+            let prevTstamp;
+            paths.push({path: [], timestamps: [], num: 0, ts0: +Infinity, ts1: -Infinity});
+            for (const el of data) {
+              const tstamp = new Date(el.tstamp).valueOf();
+              const curPath = () => paths[paths.length-1];
+              if (prevTstamp && Math.abs(tstamp - prevTstamp) > breakLimit) {
+                // Start a new path
+                console.log(`Breaking path on interval ${Math.abs(tstamp - prevTstamp)} > ${breakLimit}`);
+                paths.push({path: [], timestamps: [], num: paths.length, ts0: +Infinity, ts1: -Infinity});
+              }
+
+              if (tstamp < curPath().ts0) {
+                curPath().ts0 = tstamp;
+              }
+              if (tstamp > curPath().ts1) {
+                curPath().ts1 = tstamp;
+              }
+
+              curPath().path.push([el.x, el.y]);
+              curPath().timestamps.push(tstamp/1000);
+              prevTstamp = tstamp;
+            }
+
+            paths.forEach (path => {
+              path.nums = paths.length;
+              path.ts0 = new Date(path.ts0);
+              path.ts1 = new Date(path.ts1);
+            });
+
+            return paths;
+        },
+        getPath: d => d.path,
+        getTimestamps: d => d.timestamps,
+        currentTime: ts1.valueOf() / 1000,
+        trailLength,
+        widthUnits: "meters",
+        widthMinPixels: 4,
+        getWidth: 10,
+        getColor: [ 174, 1, 126, 200 ],
+        stroked: true,
+        pickable: true,
+        ...options
+      });
+
+    },
+
+    eventsLogLayer (options = {}) {
+
+      const labelColour = (d, i, t, c = [127, 65, 90]) => {
+        const label = d?.properties?.labels?.[0];
+        const colour = this.labels[label]?.view?.colour ?? "#cococo";
+
+        if (colour) {
+          if (colour[0] == "#") {
+            c = hexToArray(colour);
+          } else {
+            c = namedColourToArray(colour);
+          }
+        } else {
+          //return [127, 65, 90];
+        }
+
+        if (t != null) {
+          c[3] = t;
+        }
+
+        return c;
+      };
+
+      return new DougalEventsLayer({
+        id: 'log',
+        data: `/api/project/${this.$route.params.project}/event?mime=application/geo%2Bjson`,
+        ...this.loadOptions(),
+        lineWidthMinPixels: 2,
+        getPosition: d => d.geometry.coordinates,
+        jitter: 0.00015,
+        getElevation: d => Math.min(Math.max(d.properties.remarks?.length || 10, 10), 200),
+        getFillColor: (d, i) => labelColour(d, i, 200),
+        getLineColor: (d, i) => labelColour(d, i, 200),
+        radius: 0.001,
+        radiusScale: 1,
+        // This just won't work with radiusUnits = "pixels".
+        // See: https://grok.com/share/c2hhcmQtMw%3D%3D_16578be4-20fd-4000-a765-f082503d0495
+        radiusUnits: "pixels",
+        radiusMinPixels: 1.5,
+        radiusMaxPixels: 2.5,
+
+        pickable: true,
+        ...options
+      })
+
+    },
+
+    preplotSaillinesLinesLayer (options = {}) {
+      return new GeoJsonLayer({
+        id: 'psll',
+        data: `/api/project/${this.$route.params.project}/gis/preplot/line?class=V&v=${this.lineTStamp?.valueOf()}`,
+        ...this.loadOptions(),
+        lineWidthMinPixels: 1,
+        getLineColor: (d) => d.properties.ntba ? [240, 248, 255, 200] : [85, 170, 255, 200],
+        getLineWidth: 1,
+        getPointRadius: 2,
+        radiusUnits: "pixels",
+        pointRadiusMinPixels: 2,
+        pickable: true,
+        ...options
+      })
+    },
+
+    preplotLinesLayer (options = {}) {
+      return new GeoJsonLayer({
+        id: 'ppll',
+        data: `/api/project/${this.$route.params.project}/gis/preplot/line?v=${this.lineTStamp?.valueOf()}`,
+        ...this.loadOptions(),
+        lineWidthMinPixels: 1,
+        getLineColor: (d) => d.properties.ntba ? [240, 248, 255, 200] : [85, 170, 255, 200],
+        getLineWidth: 1,
+        getPointRadius: 2,
+        radiusUnits: "pixels",
+        pointRadiusMinPixels: 2,
+        pickable: true,
+        ...options
+      })
+    },
+
+    plannedLinesLinesLayer (options = {}) {
+      return new PathLayer({
+        id: 'planl',
+        data: [...this.plannedSequences], // Create new array to trigger Deck.gl update
+        dataTransform: (sequences) => {
+          // Raise the data 10 m above ground so that it's visible over heatmaps, etc.
+          return sequences.map( seq => ({
+            ...seq,
+            geometry: {
+              ...seq.geometry,
+              coordinates: seq.geometry.coordinates.map( pos => [...pos, 10] )
+            }
+          }))
+        },
+        getPath: d => d.geometry.coordinates,
+        //getSourcePosition: d => d.geometry.coordinates[0],
+        //getTargetPosition: d => d.geometry.coordinates[1],
+        widthUnits: "meters",
+        widthMinPixels: 4,
+        getWidth: 25,
+        //getLineWidth: 10,
+        getColor: (d) => {
+          const k = (d?.azimuth??0)/360*255;
+          return [ k, 128, k, 200 ];
+        },
+        stroked: true,
+        pickable: true,
+        ...options
+      });
+    },
+
+    rawSequencesLinesLayer (options = {}) {
+      return new GeoJsonLayer({
+        id: 'seqrl',
+        data: `/api/project/${this.$route.params.project}/gis/raw/line?v=${this.sequenceTStamp?.valueOf()}`,
+        ...this.loadOptions(),
+        lineWidthMinPixels: 1,
+        getLineColor: (d) => d.properties.ntbp ? [0xe6, 0x51, 0x00, 200] : [0xff, 0x98, 0x00, 200],
+        getLineWidth: 1,
+        getPointRadius: 2,
+        radiusUnits: "pixels",
+        pointRadiusMinPixels: 2,
+        pickable: true,
+        ...options
+      })
+    },
+
+    finalSequencesLinesLayer (options = {}) {
+      return new GeoJsonLayer({
+        id: 'seqfl',
+        data: `/api/project/${this.$route.params.project}/gis/final/line?v=${this.sequenceTStamp?.valueOf()}`,
+        ...this.loadOptions(),
+        lineWidthMinPixels: 1,
+        getLineColor: (d) => d.properties.pending ? [0xa7, 0xff, 0xab, 200] : [0x00, 0x96, 0x88, 200],
+        getLineWidth: 1,
+        getPointRadius: 2,
+        radiusUnits: "pixels",
+        pointRadiusMinPixels: 2,
+        pickable: true,
+        ...options
+      })
+    },
+
+    preplotSaillinesPointLayer (options = {}) {
+      return new DougalSequenceLayer({
+        id: 'pslp',
+        data: `/api/project/${this.$route.params.project}/line/sail?v=${this.lineTStamp?.valueOf()}`, // API endpoint returning binary data
+        loaders: [DougalBinaryLoader],
+        ...this.loadOptions({
+          fetch: {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${this.$store.getters.jwt}`,
+              Accept: 'application/vnd.aaltronav.dougal+octet-stream'
+            }
+          }
+        }),
+        getRadius: 2,
+        getFillColor: (d, {data, index}) => data.attributes.value2.value[index] ? [240, 248, 255, 200] : [85, 170, 255, 200],
+        //getFillColor: [0, 120, 220, 200],
+        pickable: true,
+        ...options
+      });
+    },
+
+    preplotPointsLayer (options = {}) {
+      return new DougalSequenceLayer({
+        id: 'pplp',
+        data: `/api/project/${this.$route.params.project}/line/source?v=${this.lineTStamp?.valueOf()}`, // API endpoint returning binary data
+        loaders: [DougalBinaryLoader],
+        ...this.loadOptions({
+          fetch: {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${this.$store.getters.jwt}`,
+              Accept: 'application/vnd.aaltronav.dougal+octet-stream'
+            }
+          }
+        }),
+        getRadius: 2,
+        getFillColor: (d, {data, index}) => data.attributes.value2.value[index] ? [240, 248, 255, 200] : [85, 170, 255, 200],
+        //getFillColor: [0, 120, 220, 200],
+        pickable: true,
+        ...options
+      });
+    },
+
+    plannedLinesPointsLayer (options = {}) {
+    },
+
+    rawSequencesPointsLayer (options = {}) {
+
+      return new DougalSequenceLayer({
+        id: 'seqrp',
+        data: this.makeDataFromBinary(this.sequenceBinaryData),
+        getRadius: 2,
+        getFillColor: [0, 120, 220, 200],
+        pickable: true,
+        ...options
+      });
+    },
+
+    finalSequencesPointsLayer (options = {}) {
+
+      return new DougalSequenceLayer({
+        id: 'seqfp',
+        data: this.makeDataFromBinary(this.sequenceBinaryDataFinal),
+        getRadius: 2,
+        getFillColor: [220, 120, 0, 200],
+        pickable: true,
+        ...options
+      });
+    },
+
+    heatmapLayer(options = {}) {
+      const { positions, values } = this.heatmapValue?.startsWith("co_")
+        ? this.sequenceBinaryDataFinal
+        : this.sequenceBinaryData;
+
+      if (!positions?.length || !values?.length) {
+        console.warn('No valid data for heatmapLayer');
+
+        return new HeatmapLayer({
+          id: 'seqrh',
+          data: [],
+          ...options
+        });
+      }
+
+
+      let weights, offset = 0, scaler = 1;
+      let colorDomain = null;
+      let aggregation = "MEAN";
+      let transform = (v) => v;
+
+      switch (this.heatmapValue) {
+        case "total_error":
+          weights = Float32Array.from(values[3], (ei, i) => {
+            const ej = values[4][i];
+            return Math.sqrt(ei * ei + ej * ej) / 100; // Euclidean distance in meters
+          });
+          colorDomain = [2, 20]; // scale: 1 (already divided by 100 above)
+          break;
+        case "delta_i":
+          weights = values[3];
+          scaler = 0.1;
+          colorDomain = [100, 1200]; // scale: 100 (1 ‒ 12 m)
+          break;
+        case "delta_j":
+          weights = values[4];
+          scaler = 0.1;
+          colorDomain = [10, 80]; // scale: 100 (0.1 ‒ 0.8 m)
+          break;
+
+        case "co_total_error":
+          weights = Float32Array.from(values[3], (ei, i) => {
+            const ej = values[4][i];
+            return Math.sqrt(ei * ei + ej * ej) / 100; // Euclidean distance in meters
+          });
+          colorDomain = [10, 150]; // Scale: 100 (0.1 ‒ 1 m)
+          break;
+        case "co_delta_i":
+          weights = values[5];
+          scaler = 0.1;
+          colorDomain = [10, 150];
+          break;
+        case "co_delta_j":
+          weights = values[6];
+          scaler = 0.1;
+          colorDomain = [0.2, 2];
+          break;
+
+        case "delta_μ":
+          weights = values[5];
+          scaler = 0.1;
+          break;
+        case "delta_σ":
+          weights = values[6];
+          scaler = 0.1;
+          colorDomain = [ 0.1, 1.5 ];
+          break;
+        case "delta_R":
+          weights = values[7];
+          scaler = 0.1;
+          colorDomain = [ 0.5, 1.0 ];
+          break;
+        case "press_μ":
+          weights = values[8];
+          offset = -2000;
+          colorDomain = [ 5, 50 ];
+          break;
+        case "press_σ":
+          weights = values[9];
+          colorDomain = [ 1, 19 ];
+          break;
+        case "press_R":
+          weights = values[10];
+          colorDomain = [ 3, 50 ];
+          break;
+        case "depth_μ":
+          weights = values[11];
+          offset = -6;
+          scaler = 0.1;
+          colorDomain = [ 0.1, 1 ];
+          break;
+        case "depth_σ":
+          weights = values[12];
+          scaler = 0.1;
+          break;
+        case "depth_R":
+          weights = values[13];
+          scaler = 0.1;
+          break;
+        case "fill_μ":
+          weights = values[14];
+          colorDomain = [ 300, 1000 ];
+          break;
+        case "fill_σ":
+          weights = values[15];
+          offset = -250;
+          colorDomain = [ 0, 250 ];
+          break;
+        case "fill_R":
+          weights = values[16];
+          offset = -500;
+          colorDomain = [ 0, 500 ];
+          break;
+        case "delay_μ":
+          weights = values[17];
+          offset = -150;
+          colorDomain = [ 1.5, 25 ];
+          //transform = (v) => {console.log("τ(v)", v); return v;};
+          break;
+        case "delay_σ":
+          weights = values[18];
+          break;
+        case "delay_R":
+          weights = values[19];
+          break;
+        case "no_fire":
+          weights = values[20];
+          transform = v => v >> 4;
+          aggregation = "SUM";
+          colorDomain = [ 0.1, 1.5 ];
+          break;
+        case "autofire":
+          weights = values[20];
+          transform = v => v & 0xf;
+          aggregation = "SUM";
+          colorDomain = [ 0.5, 1.5 ];
+          break;
+        case "misfire":
+          weights = values[20];
+          aggregation = "SUM";
+          colorDomain = [ 0.5, 1.5 ];
+          break;
+      }
+
+
+      const stats = {
+        min: d3a.min(weights),
+        mode: d3a.mode(weights),
+        mean: d3a.mean(weights),
+        max: d3a.max(weights),
+        sd: d3a.deviation(weights),
+      };
+      const sr0 = [ stats.mean - 2.1*stats.sd, stats.mean + 2.1*stats.sd ];
+      const sr1 = [ stats.mode - 2.1*stats.sd, stats.mode + 2.1*stats.sd ];
+
+      /*
+      console.log('Positions sample:', positions.slice(0, 10));
+      console.log('Weights sample:', weights.slice(0, 10));
+      console.log("Mode:", this.heatmapValue);
+      console.log('Weight stats:', stats);
+      console.log("Suggested ranges");
+      console.log(sr0);
+      console.log(sr1);
+      console.log("Actual ranges");
+      console.log(colorDomain);
+      */
+
+      return new HeatmapLayer({
+        id: 'seqrh',
+        data: {
+          length: weights.length,
+          positions,
+          weights
+          /*
+          attributes: {
+            getPosition: { value: positions, type: 'float32', size: 2 },
+            getWeight: { value: weights, type: 'float32', size: 1 }
+          }
+          */
+        },
+        getPosition: (d, {index, data}) => [ data.positions[index*2], data.positions[index*2+1] ],
+        getWeight: (d, {index, data}) => transform(Math.abs(data.weights[index] * scaler + offset)),
+        colorDomain,
+        radiusPixels: 25,
+        aggregation,
+        pickable: false,
+        ...options
+      });
+    },
+
+  },
+
+}
+
+</script>

lib/www/client/source/src/views/MapTooltipsMixin.vue

@@ -0,0 +1,259 @@
+<script>
+import * as d3a from 'd3-array';
+
+export default {
+  name: "MapTooltipsMixin",
+
+  data () {
+    return {
+      tooltipDefaultStyle: { "max-width": "50ex"}
+    }
+  },
+
+  methods: {
+
+    getTooltip (args) {
+      if (args?.layer?.constructor?.tooltip) {
+        return args.layer.constructor.tooltip(args);
+      } else if (args?.layer?.id == "seqrp" || args?.layer?.id == "seqfp") {
+        return this.sequencePointsTooltip(args);
+      } else if (args?.layer?.id == "log") {
+        return this.eventLogTooltip(args);
+      } else if (args?.layer?.id == "pplp" || args?.layer?.id == "pslp") {
+        return this.preplotPointsTooltip(args);
+      } else if (args?.layer?.id == "ppll" || args?.layer?.id == "psll") {
+        return this.preplotLinesTooltip(args);
+      } else if (args?.layer?.id == "planl") {
+        return this.plannedLinesTooltip(args);
+      } else if (args?.layer?.id == "seqrl" || args?.layer?.id == "seqfl") {
+        return this.sequenceLinesTooltip(args);
+      } else if (args?.layer?.id == "navp") {
+        return this.vesselTrackPointsTooltip(args);
+      } else if (args?.layer?.id == "navl") {
+        return this.vesselTrackLinesTooltip(args);
+      }
+    },
+
+    preplotPointsTooltip (args) {
+      const p = args?.object;
+      if (p) {
+        let html = "Preplot<br/>\n";
+
+        if ("sailline" in p) {
+          // If there is a "sailline" attribute, this is actually a source line
+          // "i" is the source line number, "sailline" the sail line number.
+          html += `S${String(p.i).padStart(4, "0")}<br/>\n`;
+          html += `V${String(p.sailline).padStart(4, "0")}<br/>\n`;
+        } else {
+          html += `V${String(p.i).padStart(4, "0")}<br/>\n`;
+        }
+        html += `P${String(p.j).padStart(4, "0")}<br/>\n`;
+
+        if (p.sailline_ntba) {
+          html += `<b>Line <abbr title="Not to be acquired">NTBA</abbr></b><br/>\n`;
+        }
+
+        if (p.ntba) {
+          html += `<b>Point <abbr title="Not to be acquired">NTBA</abbr></b><br/>\n`;
+        }
+
+        return {html, style: this.tooltipDefaultStyle};
+      }
+    },
+
+    preplotLinesTooltip (args) {
+        const p = args?.object?.properties;
+        if (p) {
+          const lineType = args.layer.id == "psll" ? "Sailline" : "Source line";
+          const direction = p.incr ? "▲" : "▼";
+          let html = "";
+
+          html += `L${String(p.line).padStart(4, "0")} ${direction}<br/>\n`;
+          html += `${lineType}<br/>\n`;
+
+          if (p.ntba) {
+            html += "<b>Not to be acquired (NTBA)</b><br/>\n";
+          }
+
+          if (p.remarks) {
+            html += `<hr/>\n${this.$root.markdown(p.remarks)}\n`;
+          }
+
+          return {html, style: this.tooltipDefaultStyle};
+        }
+    },
+
+    sequenceLinesTooltip (args) {
+        let type;
+        switch(args.layer.id) {
+          case "seqrl":
+            type = "Raw";
+            break;
+          case "seqfl":
+            type = "Final";
+            break;
+        }
+
+        const p = args?.object?.properties;
+        if (p) {
+          let html = `Sequence ${p.sequence} (${type})<br/>\n`;
+          html += `Line <b>${p.line}</b><br/>\n`;
+          html += `${p.num_points} points (${p.missing_shots ? (p.missing_shots + " missing") : "None missing"})<br/>\n`;
+          html+= `${(p.length??0).toFixed(0)} m ${(p.azimuth??0).toFixed(1)}°<br/>\n`;
+          html += `${p.duration}<br/>\n`;
+          html += `<b>${p.fsp}</b> @ ${(new Date(p.ts0))?.toISOString()}<br/>\n`;
+          html += `<b>${p.lsp}</b> @ ${(new Date(p.ts1))?.toISOString()}<br/>\n`;
+          if (p.ntbp) {
+            html += "<b>Not to be processed</b><br/>\n";
+          } else if (p.pending) {
+            html += "<b>Pending</b><br/>\n";
+          }
+          html += "<hr/><br/>\n";
+          html += this.$root.markdown(p.remarks);
+
+          return {html, style: this.tooltipDefaultStyle};
+        }
+    },
+
+    sequencePointsTooltip (args) {
+
+      const p = args?.object;
+
+      if (p) {
+
+        let html = "";
+
+        if (p.udv == 2) { // FIXME Must change this to something more meaningful
+          // This is a shot info record:
+
+          html += `S${p.i.toString().padStart(3, "0")} ${p.j}</br>\n`;
+          html += `<small>${new Date(Number(p.ts)).toISOString()}</small><br/>\n`;
+          html += `Δj: ${p.εj.toFixed(2)} m / Δi: ${p.εi.toFixed(2)} m<br/>\n`;
+          html += `<hr/>\n`;
+          if (p.nofire) {
+            html += `<b>No fire: ${p.nofire} guns</b><br/>\n`;
+          }
+          if (p.autofire) {
+            html += `<b>Autofire: ${p.autofire} guns</b><br/>\n`;
+          }
+          html += `P: ${p.press_μ} psi ±${p.press_σ} psi (R=${p.press_R})<br/>\n`;
+          html += `Δ: ${p.delta_μ} ms ±${p.delta_σ} ms (R=${p.delta_R})<br/>\n`;
+          html += `D: ${p.depth_μ} m ±${p.depth_σ} m (R=${p.depth_R})<br/>\n`;
+          html += `F: ${p.fill_μ} ms ±${p.fill_σ} ms (R=${p.fill_R})<br/>\n`;
+          html += `W: ${p.delay_μ} ms ±${p.delay_σ} ms (R=${p.delay_R})<br/>\n`;
+
+          return {html, style: this.tooltipDefaultStyle};
+        } else if (p.udv == 3) {
+          // A final points record
+
+          html += `S${p.i.toString().padStart(3, "0")} ${p.j} (final)</br>\n`;
+          html += `<small>${new Date(Number(p.ts)).toISOString()}</small><br/>\n`;
+          html += `Δj: ${p.εj.toFixed(2)} m / Δi: ${p.εi.toFixed(2)} m (final−preplot)<br/>\n`;
+          html += `δj: ${p.co_j.toFixed(2)} m / δi: ${p.co_i.toFixed(2)} m (final−raw)<br/>\n`;
+
+          return {html, style: this.tooltipDefaultStyle};
+        }
+      }
+      console.log("no tooltip");
+
+    },
+
+    eventLogTooltip (args) {
+      const p = args?.object?.properties;
+      if (p) {
+        let html = "";
+        if (p.sequence && p.point) {
+          html += `S${p.sequence.toString().padStart(3, "0")} ${p.point}</br>\n`;
+        }
+        html += `<small>${p.tstamp}</small><br/>\n`;
+        html += `<span>${p.remarks}</span>`;
+        if (p.labels?.length) {
+          html += `</br>\n<small><i>${p.labels.join(", ")}</i></small>`;
+        }
+
+        return {html, style: this.tooltipDefaultStyle};
+      }
+    },
+
+    plannedLinesTooltip (args) {
+      const p = args?.object;
+      if (p) {
+        const duration = `${(p.duration?.hours??0).toString().padStart(2, "0")}:${(p.duration?.minutes??0).toString().padStart(2, "0")}`;
+        const Δt = ((new Date(p.ts1)).valueOf() - (new Date(p.ts0)).valueOf()) / 1000; // seconds
+        const speed = (p.length / Δt) * 3.6 / 1.852; // knots
+
+        let html = `Planned sequence <b>${p.sequence}</b></br>\n` +
+          `Line <b>${p.line}</b> – ${p.name}</br>\n` +
+          `${p.num_points} Points</br>\n` +
+          `${p.length.toFixed(0)} m ${p.azimuth.toFixed(2)}°</br>\n` +
+          `${duration} @ ${speed.toFixed(1)} kt</br>\n` +
+          `<b>${p.fsp}</b> @ ${(new Date(p.ts0))?.toISOString()}</br>\n` +
+          `<b>${p.lsp}</b> @ ${(new Date(p.ts1))?.toISOString()}`;
+
+          if (p.remarks) {
+            html += `</br>\n<hr/>${this.$root.markdown(p.remarks)}`;
+          }
+
+        return {html, style: this.tooltipDefaultStyle};
+      }
+    },
+
+    vesselTrackPointsTooltip (args) {
+      const p = args.object;
+
+      if (p) {
+        let html = `${p.vesselName}<br/>\n`
+          + `${p.tstamp}<br/>\n`
+          + `BSP ${(p.speed??0).toFixed(1)} kt CMG ${(p.cmg??0).toFixed(1).padStart(5, "0")}° HDG ${(p.bearing??0).toFixed(1).padStart(5, "0")}° DPT ${(p.waterDepth??0).toFixed(1)} m<br/>\n`
+          + `${p.lineStatus}<br/>\n`;
+
+        if (p.guns) {
+          console.log(p);
+          const pressure = p.guns.map( i => i[11] ); // 11 is gun pressure
+          const μpress = d3a.mean(pressure);
+          const σpress = d3a.deviation(pressure);
+
+          if (p.lineStatus && p.lineStatus != "offline") {
+            html += `${p.lineName}<br/>\n`
+              + `S: ${p._sequence} L: ${p._line} P: ${p.shot}<br/>`
+              + `Source ${p.src_number} `
+              + ((p.trg_mode && p.trg_mode != "external") ? `<b>${p.trg_mode.toUpperCase()} TRIGGER</b> ` : "")
+              + `<small>FSID ${p.fsid}</small> <small>mask ${p.mask}</small><br/>\n`
+              + `Δ ${(p.avg_delta??0).toFixed(3)} ms ±${(p.std_delta??0).toFixed(3)} ms<br/>\n`
+              + `${(μpress??0).toFixed(0)} psi ±${(σpress??0).toFixed(0)} psi / ${(p.volume??0).toFixed(0)} in³<br/>\n`
+              + `along ${(p.inline??0).toFixed(1)} m / across ${(p.crossline??0).toFixed(1)} m<br/>\n`;
+          } else {
+            // Soft start?
+            html +=
+                `Source ${p.src_number} `
+              + ((p.trg_mode && p.trg_mode != "external") ? `<b>${p.trg_mode.toUpperCase()} TRIGGER</b> ` : "")
+              + `<small>mask ${p.mask}</small><br/>\n`
+              + `Δ ${(p.avg_delta??0).toFixed(3)} ms ±${(p.std_delta??0).toFixed(3)} ms<br/>\n`
+              + `${(p.manifold??0).toFixed(0)} psi / ${(p.volume??0).toFixed(0)} in³<br/>\n`
+              + `along ${(p.inline??0).toFixed(1)} m / across ${(p.crossline??0).toFixed(1)} m<br/>\n`;
+          }
+        }
+
+        return {html, style: this.tooltipDefaultStyle};
+      }
+    },
+
+    vesselTrackLinesTooltip (args) {
+      const p = args.object;
+
+      console.log("track lines tooltip", p);
+
+      if (p) {
+
+        let html = `Segment ${p.num+1} / ${p.nums}<br/>\n`
+          html += `${p.ts0.toISOString()}<br/>\n`
+          html += `${p.ts1.toISOString()}<br/>\n`;
+
+        return {html, style: this.tooltipDefaultStyle};
+      }
+    },
+
+  }
+
+}
+</script>

lib/www/client/source/src/views/Project.vue

@@ -43,48 +43,114 @@ export default {
       return this.loading || this.projectId;
     },
 
-    ...mapGetters(["loading", "projectId", "projectSchema", "serverEvent"])
-  },
-
-  watch: {
-    async serverEvent (event) {
-      if (event.channel == "project" && event.payload?.operation == "DELETE" && event.payload?.schema == "public") {
-        // Project potentially deleted
-        await this.getProject(this.$route.params.project);
-      } else if (event.payload?.schema == this.projectSchema) {
-        if (event.channel == "event") {
-          this.refreshEvents();
-        } else if (event.channel == "planned_lines") {
-          this.refreshPlan();
-        } else if (["raw_lines", "raw_shots", "final_lines", "final_shots"].includes(event.channel)) {
-          this.refreshSequences();
-        } else if (["preplot_lines", "preplot_points"].includes(event.channel)) {
-          this.refreshLines();
-        } else if (event.channel == "info") {
-          if ((event.payload?.new ?? event.payload?.old)?.key == "plan") {
-            this.refreshPlan();
-          }
-        } else if (event.channel == "project") {
-          this.getProject(this.$route.params.project);
-        }
-      }
-    }
+    ...mapGetters(["loading", "projectId", "projectSchema"])
   },
 
   methods: {
+
+    handleEvents (context, {payload}) {
+      if (payload.pid != this.projectId) {
+        console.warn(`${this.projectId} ignoring notification for ${payload.pid}`);
+        return;
+      }
+
+      this.refreshEvents();
+    },
+
+    handleLines (context, {payload}) {
+      if (payload.pid != this.projectId) {
+        console.warn(`${this.projectId} ignoring notification for ${payload.pid}`);
+        return;
+      }
+
+      this.refreshLines();
+    },
+
+    handlePlannedLines (context, {payload}) {
+      if (payload.pid != this.projectId) {
+        console.warn(`${this.projectId} ignoring notification for ${payload.pid}`);
+        return;
+      }
+
+      this.refreshPlan();
+    },
+
+    handleSequences (context, {payload}) {
+      if (payload.pid != this.projectId) {
+        console.warn(`${this.projectId} ignoring notification for ${payload.pid}`);
+        return;
+      }
+
+      console.log("handleSequences");
+      this.refreshSequences();
+    },
+
+    registerNotificationHandlers (action = "registerHandler") {
+
+
+      this.$store.dispatch(action, {
+        table: 'event',
+
+        handler: (context, message) => {
+          this.handleEvents(context, message);
+        }
+      });
+
+      ["preplot_lines", "preplot_points"].forEach( table => {
+        this.$store.dispatch(action, {
+          table,
+
+          handler: (context, message) => {
+            this.handleLines(context, message);
+          }
+        })
+      });
+
+
+      this.$store.dispatch(action, {
+        table: 'planned_lines',
+
+        handler: (context, message) => {
+          this.handlePlannedLines(context, message);
+        }
+      });
+
+      ["raw_lines", "raw_shots", "final_lines", "final_shots"].forEach( table => {
+        this.$store.dispatch(action, {
+          table,
+
+          handler: (context, message) => {
+            this.handleSequences(context, message);
+          }
+        })
+      });
+
+    },
+
+    unregisterNotificationHandlers () {
+      return this.registerNotificationHandlers("unregisterHandler");
+    },
+
     ...mapActions(["getProject", "refreshLines", "refreshSequences", "refreshEvents", "refreshLabels", "refreshPlan"])
   },
 
   async mounted () {
     await this.getProject(this.$route.params.project);
     if (this.projectFound) {
+      this.registerNotificationHandlers();
+
       this.refreshLines();
       this.refreshSequences();
       this.refreshEvents();
       this.refreshLabels();
       this.refreshPlan();
     }
+  },
+
+  beforeDestroy () {
+    this.unregisterNotificationHandlers();
   }
 
+
 }
 </script>

lib/www/client/source/src/views/ProjectList.vue

@@ -27,6 +27,8 @@
         <v-chip v-for="group in value"
           label
           small
+          :title="`View repeatability data for ${group}`"
+          :href="`/groups/${group}`"
         >{{ group }}</v-chip>
       </template>
 
@@ -156,7 +158,7 @@ export default {
         },
       ],
       items: [],
-      options: {},
+      options: { sortBy: ["pid"], sortDesc: [true] },
 
       // Whether or not to show archived projects
       showArchived: true,
@@ -184,17 +186,15 @@ export default {
         : this.items.filter(i => !i.archived);
     },
 
-    ...mapGetters(['loading', 'serverEvent', 'projects'])
+    ...mapGetters(['loading', 'projects'])
   },
 
   watch: {
-    async serverEvent (event) {
-      if (event.channel == "project" && event.payload?.schema == "public") {
-        if (event.payload?.operation == "DELETE" || event.payload?.operation == "INSERT") {
-          await this.load();
-        }
-      }
+
+    async projects () {
+      await this.load();
     }
+
   },
 
   methods: {
@@ -203,13 +203,6 @@ export default {
       this.items = [...this.projects];
     },
 
-    async summary (item) {
-      const details = await this.api([`/project/${item.pid}/summary`]);
-      if (details) {
-        return Object.assign({}, details, item);
-      }
-    },
-
     title (item) {
       if (item.organisations) {
         return "Access:\n" + Object.entries(item.organisations).map( org =>
@@ -220,18 +213,23 @@ export default {
     },
 
     async load () {
-      await this.list();
-      const promises = [];
-      for (const key in this.items) {
-        const item = this.items[key];
-        const promise = this.summary(item)
-        .then( expanded => {
-          if (expanded) {
-            this.$set(this.items, key, expanded);
-          }
-        });
-        promises.push(promise);
+      if (!this.projects.length) {
+        this.refreshProjects();
       }
+      await this.list();
+    },
+
+    handlerLoad (context, {payload}) {
+      if (payload?.table == "public") {
+        this.load();
+      }
+    },
+
+    registerNotificationHandlers () {
+      this.$store.dispatch('registerHandler', {
+        table: 'project`',
+        handler: this.handlerLoad
+      });
     },
 
     contextMenu (e, {item}) {
@@ -372,10 +370,11 @@ export default {
 
     },
 
-    ...mapActions(["api", "showSnack"])
+    ...mapActions(["api", "showSnack", "refreshProjects"])
   },
 
   mounted () {
+    this.registerNotificationHandlers();
     this.load();
   }
 }

lib/www/client/source/src/views/SequenceList.vue

@@ -822,7 +822,7 @@ export default {
 
     async getNumLines () {
       const projectInfo = await this.api([`/project/${this.$route.params.project}`]);
-      this.num_rows = projectInfo.sequences;
+      this.num_rows = projectInfo?.sequences ?? 0;
     },
 
     async fetchSequences (opts = {}) {

lib/www/client/source/src/views/SequenceSummary.vue

@@ -6,8 +6,42 @@
         <v-progress-linear indeterminate v-if="loading"></v-progress-linear>
         <v-toolbar flat>
           <v-toolbar-title>
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'shotlog', params: { sequence: (sequences[sequences.length-1]||{}).sequence }}"
+                title="Go to the first sequence"
+              >
+                <v-icon dense>mdi-chevron-double-left</v-icon>
+              </v-btn>
+              <v-btn icon small
+                :disabled="sequenceIndex >= (sequences.length - 1)"
+                :to="{name: 'shotlog', params: { sequence: (sequences[sequenceIndex+1]||{}).sequence }}"
+                title="Go to the previous sequence"
+              >
+                <v-icon dense>mdi-chevron-left</v-icon>
+              </v-btn>
+            </template>
+
             Sequence {{sequenceNumber}}
             <small :class="statusColour" v-if="sequence">({{sequence.status}})</small>
+
+            <template v-if="$route.params.sequence">
+              <v-btn icon small
+                :disabled="sequenceIndex==0"
+                :to="{name: 'shotlog', params: { sequence: (sequences[sequenceIndex-1]||{}).sequence }}"
+                title="Go to the next sequence"
+              >
+                <v-icon dense>mdi-chevron-right</v-icon>
+              </v-btn>
+              <v-btn icon small class="mr-1"
+                :disabled="sequenceIndex==0"
+                :to="{name: 'shotlog', params: { sequence: (sequences[0]||{}).sequence }}"
+                title="Go to the last sequence"
+              >
+                <v-icon dense>mdi-chevron-double-right</v-icon>
+              </v-btn>
+            </template>
           </v-toolbar-title>
 
           <a v-if="$route.params.sequence"
@@ -129,7 +163,7 @@
 
               <template v-slot:item.position="{item}">
                 <a v-if="position(item).latitude"
-                  :href="`/projects/${$route.params.project}/map#15/${position(item).longitude}/${position(item).latitude}`"
+                  :href="`/projects/${$route.params.project}/map#z15x${position(item).longitude}y${position(item).latitude}:+seqrp;+psll:${position(item).longitude},${position(item).latitude}:S${String(item.sequence).padStart(3, '0')} ${item.point}`"
                   title="View on map"
                   :target="`/projects/${$route.params.project}/map`"
                   @click.stop
@@ -352,6 +386,16 @@ export default {
       return this.sequences.find(i => i.sequence == this.sequenceNumber);
     },
 
+    sequenceIndex () {
+      if ("sequence" in this.$route.params) {
+        const index = this.sequences.findIndex( i => i.sequence == this.$route.params.sequence );
+        if (index != -1) {
+          return index;
+        }
+      }
+      // return undefined
+    },
+
     remarks () {
       return this.sequence?.remarks || "Nil.";
     },

lib/www/client/source/vue.config.js

@@ -1,4 +1,7 @@
 const webpack = require('webpack');
+const { execSync } = require('child_process');
+
+const gitDescribe = execSync('git describe --tags --always', { encoding: 'utf8' }).trim();
 
 module.exports = {
   "transpileDependencies": [
@@ -21,10 +24,10 @@ module.exports = {
     },
     proxy: {
       "^/api(/|$)": {
-        target: "http://127.0.0.1:3000",
+        target: process.env.DOUGAL_API_URL || "http://127.0.0.1:3000",
       },
       "^/ws(/|$)": {
-        target: "ws://127.0.0.1:3000",
+        target: process.env.DOUGAL_API_URL?.replace(/^http(s?):/i, "ws$1:") || "ws://127.0.0.1:3000",
         ws: true
       }
     }
@@ -48,7 +51,11 @@ module.exports = {
       // https://github.com/webpack/changelog-v5/issues/10
       new webpack.ProvidePlugin({
         Buffer: ['buffer', 'Buffer'],
-      })
+      }),
+
+      new webpack.DefinePlugin({
+        'process.env.DOUGAL_FRONTEND_VERSION': JSON.stringify(gitDescribe),
+      }),
     ],
     module: {
       rules: [

lib/www/server/api/index.js

@@ -69,6 +69,7 @@ const allMeta = (key, value) => {
 	return { all: [ meta(key, value) ] };
 };
 
+
 //
 // NOTICE These routes do not require authentication
 //
@@ -83,7 +84,10 @@ app.map({
 		post: [ mw.user.logout ]
 	},
 	'/version': {
-		get: [ mw.auth.operations, mw.version.get ]
+		get: [ mw.auth.operations, mw.version.get ],
+		'/history': {
+			get: [ /*mw.auth.user,*/ mw.version.history.get ],
+		}
 	},
 	'/': {
 		get: [ mw.openapi.get ]
@@ -100,6 +104,9 @@ app.use(mw.auth.access.user);
 // Don't process the request if the data hasn't changed
 app.use(mw.etag.ifNoneMatch);
 
+// Use compression across the board
+app.use(mw.compress);
+
 // We must be authenticated before we can access these
 app.map({
 	'/project': {
@@ -114,10 +121,12 @@ app.map({
 		get: [ mw.auth.access.read, mw.project.summary.get ],
 	},
 	'/project/:project/configuration': {
-		get: [ mw.project.configuration.get ],	// Get project configuration
 		patch: [ mw.auth.access.edit, mw.project.configuration.patch ],	// Modify project configuration
 		put: [ mw.auth.access.edit, mw.project.configuration.put ],	// Overwrite configuration
 	},
+	'/project/:project/configuration/:path(*)?': {
+		get: [ mw.auth.access.read, mw.configuration.get ],
+	},
 
 	/*
 	 * GIS endpoints
@@ -152,10 +161,13 @@ app.map({
 	'/project/:project/line/': {
 		get: [ mw.auth.access.read, mw.line.list ],
 	},
-	'/project/:project/line/:line': {
-// 		get: [ mw.auth.access.read, mw.line.get ],
+	'/project/:project/line/:line(\\d+)': {
+		get: [ mw.auth.access.read, mw.line.get ],
 		patch: [ mw.auth.access.write, mw.line.patch ],
 	},
+	'/project/:project/line/:class(sail|source)': {
+		get: [ mw.auth.access.read, mw.line.get ],
+	},
 
 	/*
 	 * Sequence endpoints
@@ -213,16 +225,28 @@ app.map({
 		'changes/:since': {
 			get: [ mw.auth.access.read, mw.event.changes ]
 		},
-		// TODO Rename -/:sequence → sequence/:sequence
+		// NOTE: old alias for /sequence/:sequence
 		'-/:sequence/': { // NOTE: We need to avoid conflict with the next endpoint ☹
 			get: [ mw.auth.access.read, mw.event.sequence.get ],
 		},
-		':id/': {
+		'sequence/:sequence/': {
+			get: [ mw.auth.access.read, mw.event.sequence.get ],
+		},
+		':id(\\d+)/': {
 			get: [ mw.auth.access.read, mw.event.get ],
 			put: [ mw.auth.access.write, mw.event.put ],
 			patch: [ mw.auth.access.write, mw.event.patch ],
 			delete: [mw.auth.access.write,  mw.event.delete ]
 		},
+		'import': {
+			put: [ mw.auth.access.write, mw.event.import.csv, mw.event.import.put ],
+			post: [ mw.auth.access.write, mw.event.import.csv, mw.event.import.put ],
+			'/:filename': {
+				put: [ mw.auth.access.read, mw.event.import.csv, mw.event.import.put ],
+				post: [ mw.auth.access.write, mw.event.import.csv, mw.event.import.put ],
+				delete: [ mw.auth.access.write, mw.event.import.delete ]
+			},
+		},
 	},
 
 	/*
@@ -262,10 +286,6 @@ app.map({
 	'/project/:project/label/': {
 		get: [ mw.auth.access.read, mw.label.list ],
 // 		post: [ mw.label.post ],
-	},
-	'/project/:project/configuration/:path(*)?': {
-		get: [ mw.auth.access.read, mw.configuration.get ],
-// 		post: [ mw.auth.access.admin, mw.label.post ],
 	},
 	'/project/:project/info/:path(*)': {
 		get: [ mw.auth.operations, mw.auth.access.read, mw.info.get ],
@@ -294,10 +314,10 @@ app.map({
 // // 		delete: [ mw.permissions.delete ]
 // 	},
 	'/project/:project/files/:path(*)': {
-		get: [ mw.auth.access.write, mw.files.get ]
+		get: [ mw.files.get ]
 	},
 	'/files/?:path(*)': {
-		get: [ mw.auth.access.write, mw.etag.noSave, mw.files.get ]
+		get: [ mw.etag.noSave, mw.files.get ]
 	},
 	'/navdata/': { // TODO These endpoints should probably need read access auth
 		get: [ mw.etag.noSave, mw.navdata.get ],
@@ -305,6 +325,30 @@ app.map({
 			get: [ mw.etag.noSave, mw.gis.navdata.get ]
 		}
 	},
+	'/vessel/track': {
+		get: [ /*mw.etag.noSave,*/ mw.vessel.track.get ], // JSON array
+		'/line': {
+			get: [ // GeoJSON Feature: type = LineString
+				//mw.etag.noSave,
+				(req, res, next) => { req.query.geojson = 'LineString'; next(); },
+				mw.vessel.track.get
+			]
+		},
+		'/point': {
+			get: [ // GeoJSON FeatureCollection: feature types = Point
+				//mw.etag.noSave,
+				(req, res, next) => { req.query.geojson = 'Point'; next(); },
+				mw.vessel.track.get
+			]
+		},
+		'/points': {
+			get: [ // JSON array of (Feature: type = Point)
+				mw.etag.noSave,
+				(req, res, next) => { req.query.geojson = true; next(); },
+				mw.vessel.track.get
+			],
+		},
+	},
 	'/info/': {
 		':path(*)': {
 			get: [ mw.auth.operations, mw.info.get ],
@@ -313,6 +357,26 @@ app.map({
 			delete: [ mw.auth.operations, mw.auth.access.write, mw.info.delete ]
 		}
 	},
+
+	/*
+	 * 4D comparisons
+	 */
+
+	// FIXME no authentication yet!
+
+	'/comparison/group': {
+		get: [ mw.etag.noSave, mw.comparisons.groups.list ],
+		'/:group': {
+			get: [ mw.etag.noSave, mw.comparisons.groups.get ],
+		},
+	},
+
+
+
+	/*
+	 * Other endpoints
+	 */
+
 	'/queue/outgoing/': {
 		'asaqc': {
 			get: [ mw.etag.noSave, mw.queue.asaqc.get ],

lib/www/server/api/middleware/auth/access.js

@@ -1,6 +1,7 @@
 const { projectOrganisations, vesselOrganisations/*, orgAccess */} = require('../../../lib/db/project/organisations');
 const ServerUser = require('../../../lib/db/user/User');
 const { Organisations } = require('@dougal/organisations');
+const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
 /** Second-order function.
  * Returns a middleware that checks if the user has access to
@@ -14,11 +15,7 @@ function operation (operation) {
 		if (req.params.project) {
 			const projectOrgs = new Organisations(await projectOrganisations(req.params.project));
 			const availableOrgs = projectOrgs.accessToOperation(operation).filter(user.organisations);
-			console.log("Operation:      ", operation);
-			console.log("User:           ", user.name);
-			console.log("User orgs:      ", user.organisations);
-			console.log("Project orgs:   ", projectOrgs);
-			console.log("Available orgs: ", availableOrgs);
+			DEBUG(`operation = ${operation}, user = ${user?.name}, user orgs = %j, project orgs = %j, availableOrgs = %j`, user.organisations.toJSON(), projectOrgs.toJSON(), availableOrgs.toJSON());
 			if (availableOrgs.length > 0) {
 				next();
 				return;
@@ -26,16 +23,13 @@ function operation (operation) {
 		} else {
 			const vesselOrgs = new Organisations(await vesselOrganisations());
 			const availableOrgs = vesselOrgs.accessToOperation(operation).filter(user.organisations);
-			console.log("Operation:      ", operation);
-			console.log("User:           ", user.name);
-			console.log("User orgs:      ", user.organisations);
-			console.log("Vessel orgs:    ", vesselOrgs);
-			console.log("Available orgs: ", availableOrgs);
+			DEBUG(`operation = ${operation}, user = ${user?.name}, user orgs = %j, vessel orgs = %j, availableOrgs = %j`, user.organisations.toJSON(), vesselOrgs.toJSON(), availableOrgs.toJSON());
 			if (availableOrgs.length > 0) {
 				next();
 				return;
 			}
 		}
+		DEBUG(`Access denied to operation ${operation}.`);
 		next({status: 403, message: "Access denied"});
 	}
 }
@@ -67,7 +61,7 @@ async function user (req, res, next) {
 async function admin (req, res, next) {
 	if (req.user) {
 		const user = new ServerUser(req.user);
-		if (user.operations.accessToOperation("edit").length > 0) {
+		if (user.organisations.accessToOperation("edit").length > 0) {
 			next();
 			return;
 		}

lib/www/server/api/middleware/auth/authentify.js

@@ -1,41 +1,123 @@
 const dns = require('dns');
-const { Netmask } = require('netmask');
+const ipaddr = require('ipaddr.js');
+const { isIPv6, isIPv4 } = require('net');
 const cfg = require('../../../lib/config');
 const jwt = require('../../../lib/jwt');
 const user = require('../../../lib/db/user');
 const ServerUser = require('../../../lib/db/user/User');
+const { ERROR, WARNING, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 
-async function authorisedIP (req, res) {
-	const validIPs = await user.ip({active: true}); // Get all active IP logins
-	validIPs.forEach( i => i.$block = new Netmask(i.ip) );
-	validIPs.sort( (a, b) => b.$block.bitmask - a.$block.netmask ); // More specific IPs have precedence
-	for (const ip of validIPs) {
-		const block = ip.$block;
-		if (block.contains(req.ip)) {
-			const payload = {
-				...ip,
-				ip: req.ip,
-				autologin: true
-			};
-			delete payload.$block;
-			delete payload.hash;
-			delete payload.active;
-			jwt.issue(payload, req, res);
-			return true;
+function parseIP(ip) {
+	if (!ip || typeof ip !== 'string') {
+		WARNING('Invalid IP input:', ip);
+		return null;
+	}
+	// Handle comma-separated X-Forwarded-For (e.g., "87.90.254.127,")
+	const cleanIp = ip.split(',')[0].trim();
+	if (!cleanIp) {
+		WARNING('Empty IP after parsing:', ip);
+		return null;
+	}
+	// Convert IPv6-mapped IPv4 (e.g., ::ffff:127.0.0.1 -> 127.0.0.1)
+	if (cleanIp.startsWith('::ffff:') && isIPv4(cleanIp.split('::ffff:')[1])) {
+		return cleanIp.split('::ffff:')[1];
+	}
+	return cleanIp;
+}
+
+function normalizeCIDR(range) {
+	if (!range || typeof range !== 'string') {
+		WARNING('Invalid CIDR range:', range);
+		return null;
+	}
+	// If no /prefix, assume /32 for IPv4 or /128 for IPv6
+	if (!range.includes('/')) {
+		try {
+			const parsed = ipaddr.parse(range);
+			const prefix = parsed.kind() === 'ipv4' ? 32 : 128;
+			return `${range}/${prefix}`;
+		} catch (err) {
+			WARNING(`Failed to parse bare IP ${range}:`, err.message);
+			return null;
+		}
+	}
+	return range;
+}
+
+async function authorisedIP(req, res) {
+	const ip = parseIP(req.ip || req.headers['x-forwarded-for'] || req.headers['x-real-ip']);
+	DEBUG('authorisedIP:', { ip, headers: req.headers }); // Debug
+	if (!ip) {
+		WARNING('No valid IP provided:', { ip, headers: req.headers });
+		return false;
+	}
+
+	let addr;
+	try {
+		addr = ipaddr.parse(ip);
+	} catch (err) {
+		WARNING('Invalid IP:', ip, err.message);
+		return false;
+	}
+
+	const validIPs = await user.ip({ active: true }); // Get active IP logins
+	// Attach parsed CIDR to each IP entry
+	validIPs.forEach(i => {
+		const normalized = normalizeCIDR(i.ip);
+		if (!normalized) {
+			i.$range = null;
+			return;
+		}
+		try {
+			const [rangeAddr, prefix] = ipaddr.parseCIDR(normalized);
+			i.$range = { addr: rangeAddr, prefix };
+		} catch (err) {
+			WARNING(`Invalid CIDR range ${i.ip}:`, err.message);
+			i.$range = null; // Skip invalid ranges
+		}
+	});
+	// Filter out invalid ranges and sort by specificity (descending prefix length)
+	const validRanges = validIPs.filter(i => i.$range).sort((a, b) => b.$range.prefix - a.$range.prefix);
+
+	for (const ipEntry of validRanges) {
+		const { addr: rangeAddr, prefix } = ipEntry.$range;
+		try {
+			if (addr.match(rangeAddr, prefix)) {
+				const payload = {
+					...ipEntry,
+					ip,
+					autologin: true
+				};
+				delete payload.$range;
+				delete payload.hash;
+				delete payload.active;
+				jwt.issue(payload, req, res);
+				return true;
+			}
+		} catch (err) {
+			WARNING(`Error checking range ${ipEntry.ip}:`, err.message);
+			continue;
 		}
 	}
 	return false;
 }
 
-async function authorisedHost (req, res) {
-	const validHosts = await user.host({active: true}); // Get all active host logins
+async function authorisedHost(req, res) {
+	const ip = parseIP(req.ip || req.headers['x-forwarded-for'] || req.headers['x-real-ip']);
+	DEBUG('authorisedHost:', { ip, headers: req.headers }); // Debug
+	if (!ip) {
+		WARNING('No valid IP for host check:', { ip, headers: req.headers });
+		return false;
+	}
+
+	const validHosts = await user.host({ active: true });
 	for (const key in validHosts) {
 		try {
-			const ip = await dns.promises.resolve(key);
-			if (ip == req.ip) {
+			const resolvedIPs = await dns.promises.resolve(key);
+			if (resolvedIPs.includes(ip)) {
 				const payload = {
 					...validHosts[key],
-					ip: req.ip,
+					ip,
 					autologin: true
 				};
 				delete payload.$block;
@@ -45,49 +127,28 @@ async function authorisedHost (req, res) {
 				return true;
 			}
 		} catch (err) {
-			if (err.code != "ENODATA") {
-				console.error(err);
+			if (err.code !== 'ENODATA') {
+				ERROR(`DNS error for host ${key}:`, err);
 			}
 		}
 	}
 	return false;
 }
 
-// TODO: Check client TLS certificates
-// Probably will do this via Nginx with
-// ssl_verify_client optional;
-// and then putting either of the
-// $ssl_client_s_dn or $ssl_client_escaped_cert
-// variables into an HTTP header for Node
-// to check (naturally, it must be ensured
-// that a user cannot just insert the header
-// in a request).
-
-
-async function auth (req, res, next) {
-
+async function auth(req, res, next) {
 	if (res.headersSent) {
-		// Nothing to do, this request must have been
-		// handled already by another middleware.
-		return;
+		return; // Handled by another middleware
 	}
 
-	// Check for a valid JWT (already decoded by a previous
-	// middleware).
+	// Check for valid JWT
 	if (req.user) {
-		if (!req.user.autologin) {
-			// If this is not an automatic login, check if the token is in the
-			// second half of its lifetime. If so, reissue a new one, valid for
-			// another cfg.jwt.options.expiresIn seconds.
-			if (req.user.exp) {
-				const ttl = req.user.exp - Date.now()/1000;
-				if (ttl < cfg.jwt.options.expiresIn/2) {
-					const credentials = await ServerUser.fromSQL(null, req.user.id);
-					if (credentials) {
-						// Refresh token
-						payload = Object.assign({}, credentials.toJSON());
-						jwt.issue(Object.assign({}, credentials.toJSON()), req, res);
-					}
+		if (!req.user.autologin && req.user.exp) {
+			const ttl = req.user.exp - Date.now() / 1000;
+			if (ttl < cfg.jwt.options.expiresIn / 2) {
+				const credentials = await ServerUser.fromSQL(null, req.user.id);
+				if (credentials) {
+					const payload = Object.assign({}, credentials.toJSON());
+					jwt.issue(payload, req, res);
 				}
 			}
 		}
@@ -95,19 +156,27 @@ async function auth (req, res, next) {
 		return;
 	}
 
-	// Check if the IP is known to us
+	// Check IP and host
 	if (await authorisedIP(req, res)) {
 		next();
 		return;
 	}
-
-	// Check if the hostname is known to us
 	if (await authorisedHost(req, res)) {
 		next();
 		return;
 	}
 
-	next({status: 401, message: "Not authorised"});
+	// If *all* else fails, check if the user came with a cookie
+	// (see https://gitlab.com/wgp/dougal/software/-/issues/335)
+	if (req.cookies.JWT) {
+		const token = req.cookies.JWT;
+		delete req.cookies.JWT;
+		DEBUG("falling back to cookie-based authentication");
+		req.user = await jwt.checkValidCredentials({jwt: token});
+		return await auth(req, res, next);
+	}
+
+	next({ status: 401, message: 'Not authorised' });
 }
 
 module.exports = auth;

lib/www/server/api/middleware/auth/jwt.js

@@ -5,18 +5,31 @@ const cfg = require("../../../lib/config").jwt;
 const getToken = function (req) {
 	if (req.headers.authorization && req.headers.authorization.split(' ')[0] == 'Bearer') {
 		return req.headers.authorization.split(' ')[1];
-	} else if (req.cookies.JWT) {
-		return req.cookies.JWT;
 	}
 	return null;
 }
 
+const onExpired = async function (req, err) {
+	// If it's not too badly expired, let it through
+	// and hope that a new token will be issued soon.
+	const elapsed = new Date() - err.inner.expiredAt;
+	// TODO: Add proper logging
+	// console.log("Expiry details (elapsed, gracePeriod)", elapsed, cfg.gracePeriod*1000);
+	if (elapsed < cfg.gracePeriod*1000) {
+		// console.log("JWT within grace period");
+		return;
+	}
+
+	throw err;
+}
+
 const options = {
 	secret: cfg.secret,
 	credentialsRequired: false,
 	algorithms: ['HS256'],
 	requestProperty: "user",
-	getToken
+	getToken,
+	onExpired
 };
 
 module.exports = expressJWT(options);

lib/www/server/api/middleware/comparisons/groups/get.js

@@ -0,0 +1,20 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		// const data = await comparisons.groups()
+		// if (data?.[req.params.group]) {
+		const data = await comparisons.getGroup(req.params.group);
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(404).send({message: "Group does not exist"});
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/groups/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+	list: require('./list'),
+	get: require('./get'),
+}

lib/www/server/api/middleware/comparisons/groups/list.js

@@ -0,0 +1,18 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const data = await comparisons.groups()
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(204).end();
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	groups: require('./groups')
+}

lib/www/server/api/middleware/compress/index.js

@@ -0,0 +1,18 @@
+const compression = require('compression');
+
+const compress = compression({
+	level: 6, // Balance speed vs. ratio (1-9)
+	threshold: 512, // Compress only if response >512 bytes to avoid overhead on small bundles
+	filter: (req, res) => { // Ensure bundles are compressed
+		const accept = req.get("Accept");
+		if (accept.startsWith("application/vnd.aaltronav.dougal+octet-stream")) return true;
+		if (accept.includes("json")) return true;
+		if (accept.startsWith("text/")) return true;
+		if (accept.startsWith("model/obj")) return true;
+
+		// fallback to standard filter function
+		return compression.filter(req, res)
+	}
+});
+
+module.exports = compress;

lib/www/server/api/middleware/etag/if-none-match.js

@@ -23,9 +23,9 @@ function ifNoneMatch (req, res, next) {
 	if (cached) {
 		DEBUG("ETag match. Returning cached response (ETag: %s, If-None-Match: %s) for %s %s",
 			  cached.etag, req.get("If-None-Match"), req.method, req.url);
-		setHeaders(res, cached.headers);
 		if (req.method == "GET" || req.method == "HEAD") {
-			res.status(304).send();
+			setHeaders(res, cached.headers);
+			res.status(304).end();
 			// No next()
 		} else if (!isIdempotentMethod(req.method)) {
 			res.status(412).send();

lib/www/server/api/middleware/etag/watch.js

@@ -66,8 +66,18 @@ const rels = [
 
 function invalidateCache (data, cache) {
 	return new Promise((resolve, reject) => {
+		if (!data) {
+			ERROR("invalidateCache called with no data");
+			return;
+		}
+
+		if (!data.payload) {
+			ERROR("invalidateCache called without a payload; channel = %s", data.channel);
+			return;
+		}
+
 		const channel = data.channel;
-		const project = data.payload.pid ?? data.payload?.new?.pid ?? data.payload?.old?.pid;
+		const project = data.payload?.pid ?? data.payload?.new?.pid ?? data.payload?.old?.pid;
 		const operation = data.payload.operation;
 		const table = data.payload.table;
 		const fields = { channel, project, operation, table };

lib/www/server/api/middleware/event/import/csv.js

@@ -0,0 +1,146 @@
+const Busboy = require('busboy');
+const { parse } = require('csv-parse/sync');
+
+async function middleware(req, res, next) {
+	const contentType = req.headers['content-type'] || '';
+	let csvText = null;
+	let filename = null;
+
+	if (req.params.filename && contentType.startsWith('text/csv')) {
+		csvText = typeof req.body === 'string' ? req.body : req.body.toString('utf8');
+		filename = req.params.filename;
+		processCsv();
+	} else if (contentType.startsWith('multipart/form-data')) {
+		const busboy = Busboy({ headers: req.headers });
+		let found = false;
+		busboy.on('file', (name, file, info) => {
+			if (found) {
+				file.resume();
+				return;
+			}
+			if (info.mimeType === 'text/csv') {
+				found = true;
+				filename = info.filename || 'unnamed.csv';
+				csvText = '';
+				file.setEncoding('utf8');
+				file.on('data', (data) => { csvText += data; });
+				file.on('end', () => {});
+			} else {
+				file.resume();
+			}
+		});
+		busboy.on('field', () => {}); // Ignore fields
+		busboy.on('finish', () => {
+			if (!found) {
+				return next();
+			}
+			processCsv();
+		});
+		req.pipe(busboy);
+		return;
+	} else {
+		return next();
+	}
+
+	function processCsv() {
+		let records;
+		try {
+			records = parse(csvText, {
+				relax_quotes: true,
+				quote: '"',
+				escape: '"',
+				skip_empty_lines: true,
+				trim: true
+			});
+		} catch (e) {
+			return res.status(400).json({ error: 'Invalid CSV' });
+		}
+		if (!records.length) {
+			return res.status(400).json({ error: 'Empty CSV' });
+		}
+		const headers = records[0].map(h => h.toLowerCase().trim());
+		const rows = records.slice(1);
+		let lastDate = null;
+		let lastTime = null;
+		const currentDate = new Date().toISOString().slice(0, 10);
+		const currentTime = new Date().toISOString().slice(11, 19);
+		const events = [];
+		for (let row of rows) {
+			let object = { labels: [] };
+			for (let k = 0; k < headers.length; k++) {
+				let key = headers[k];
+				let val = row[k] ? row[k].trim() : '';
+				if (!key) continue;
+				if (['remarks', 'event', 'comment', 'comments', 'text'].includes(key)) {
+					object.remarks = val;
+				} else if (key === 'label') {
+					if (val) object.labels.push(val);
+				} else if (key === 'labels') {
+					if (val) object.labels.push(...val.split(';').map(l => l.trim()).filter(l => l));
+				} else if (key === 'sequence' || key === 'seq') {
+					if (val) object.sequence = Number(val);
+				} else if (['point', 'shot', 'shotpoint'].includes(key)) {
+					if (val) object.point = Number(val);
+				} else if (key === 'date') {
+					object.date = val;
+				} else if (key === 'time') {
+					object.time = val;
+				} else if (key === 'timestamp') {
+					object.timestamp = val;
+				} else if (key === 'latitude') {
+					object.latitude = parseFloat(val);
+				} else if (key === 'longitude') {
+					object.longitude = parseFloat(val);
+				}
+			}
+			if (!object.remarks) continue;
+			let useSeqPoint = Number.isFinite(object.sequence) && Number.isFinite(object.point);
+			let tstamp = null;
+			if (!useSeqPoint) {
+				if (object.timestamp) {
+					tstamp = new Date(object.timestamp);
+				}
+				if (!tstamp || isNaN(tstamp.getTime())) {
+					let dateStr = object.date || lastDate || currentDate;
+					let timeStr = object.time || lastTime || currentTime;
+					if (timeStr.length === 5) timeStr += ':00';
+					let full = `${dateStr}T${timeStr}.000Z`;
+					tstamp = new Date(full);
+					if (isNaN(tstamp.getTime())) continue;
+				}
+				if (object.date) lastDate = object.date;
+				if (object.time) lastTime = object.time;
+			}
+			let event = {
+				remarks: object.remarks,
+				labels: object.labels,
+				meta: {
+					author: "*CSVImport*",
+					"*CSVImport*": {
+						filename,
+						tstamp: new Date().toISOString()
+					}
+				}
+			};
+			if (!isNaN(object.latitude) && !isNaN(object.longitude)) {
+				event.meta.geometry = {
+					type: "Point",
+					coordinates: [object.longitude, object.latitude]
+				};
+			}
+			if (useSeqPoint) {
+				event.sequence = object.sequence;
+				event.point = object.point;
+			} else if (tstamp) {
+				event.tstamp = tstamp.toISOString();
+			} else {
+				continue;
+			}
+			events.push(event);
+		}
+		req.body = events;
+		next();
+	}
+}
+
+module.exports = middleware;

lib/www/server/api/middleware/event/import/delete.js

@@ -0,0 +1,18 @@
+
+const { event } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		if (req.params.project && req.params.filename) {
+			await event.unimport(req.params.project, req.params.filename, req.query);
+			res.status(204).end();
+		} else {
+			res.status(400).send({message: "Malformed request"});
+		}
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/event/import/index.js

@@ -0,0 +1,6 @@
+
+module.exports = {
+	csv: require('./csv'),
+	put: require('./put'),
+	delete: require('./delete'),
+}

lib/www/server/api/middleware/event/import/put.js

@@ -0,0 +1,16 @@
+
+const { event } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const payload = req.body;
+
+		await event.import(req.params.project, payload, req.query);
+		res.status(200).send(payload);
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/event/index.js

@@ -7,5 +7,6 @@ module.exports = {
 	put: require('./put'),
 	patch: require('./patch'),
 	delete: require('./delete'),
-	changes: require('./changes')
+	changes: require('./changes'),
+	import: require('./import'),
 }

lib/www/server/api/middleware/gis/project/bbox.js

@@ -4,6 +4,7 @@ const { gis } = require('../../../../lib/db');
 module.exports = async function (req, res, next) {
 
 	try {
+		res.set("Content-Type", "application/geo+json");
 		res.status(200).send(await gis.project.bbox(req.params.project));
 		next();
 	} catch (err) {

lib/www/server/api/middleware/index.js

@@ -11,6 +11,7 @@ module.exports = {
 	gis: require('./gis'),
 	label: require('./label'),
 	navdata: require('./navdata'),
+	vessel: require('./vessel'),
 	queue: require('./queue'),
 	qc: require('./qc'),
 	configuration: require('./configuration'),
@@ -20,5 +21,7 @@ module.exports = {
 	rss: require('./rss'),
 	etag: require('./etag'),
 	version: require('./version'),
-	admin: require('./admin')
+	admin: require('./admin'),
+	compress: require('./compress'),
+	comparisons: require('./comparisons'),
 };