Merge branch '334-add-4d-comparisons' into 'devel'

Draft: Resolve "Add 4D comparisons"

Closes #334

See merge request wgp/dougal/software!60

etc/db/upgrades/upgrade41-v0.6.3-add-comparisons.sql

@@ -0,0 +1,109 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.3
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade creates a new schema called `comparisons`.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update adds a `comparisons` table to a `comparisons` schema.
+-- The `comparisons.comparisons` table holds 4D prospect comparison data.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE SCHEMA IF NOT EXISTS comparisons
+		AUTHORIZATION postgres;
+
+	COMMENT ON SCHEMA comparisons
+		IS 'Holds 4D comparison data and logic';
+
+	CREATE TABLE IF NOT EXISTS comparisons.comparisons
+	(
+		type text COLLATE pg_catalog."default" NOT NULL,
+		baseline_pid text COLLATE pg_catalog."default" NOT NULL,
+		monitor_pid text COLLATE pg_catalog."default" NOT NULL,
+		data bytea,
+		meta jsonb NOT NULL DEFAULT '{}'::jsonb,
+		CONSTRAINT comparisons_pkey PRIMARY KEY (baseline_pid, monitor_pid, type)
+	)
+
+	TABLESPACE pg_default;
+
+	ALTER TABLE IF EXISTS comparisons.comparisons
+		OWNER to postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.3"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.3"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade42-v0.6.4-notify-exclude-columns.sql

@@ -0,0 +1,169 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.4
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE FUNCTION public.notify()
+		RETURNS trigger
+		LANGUAGE 'plpgsql'
+		COST 100
+		VOLATILE NOT LEAKPROOF
+	AS $BODY$
+		DECLARE
+		channel text := TG_ARGV[0];
+		pid text;
+		payload text;
+		notification text;
+		payload_id integer;
+		old_json jsonb;
+		new_json jsonb;
+		excluded_col text;
+		i integer;
+		BEGIN
+
+		-- Fetch pid
+		SELECT projects.pid INTO pid FROM projects WHERE schema = TG_TABLE_SCHEMA;
+
+		-- Build old and new as jsonb, excluding specified columns if provided
+		IF OLD IS NOT NULL THEN
+			old_json := row_to_json(OLD)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				old_json := old_json - excluded_col;
+			END LOOP;
+		ELSE
+			old_json := NULL;
+		END IF;
+
+		IF NEW IS NOT NULL THEN
+			new_json := row_to_json(NEW)::jsonb;
+			FOR i IN 1 .. TG_NARGS - 1 LOOP
+				excluded_col := TG_ARGV[i];
+				new_json := new_json - excluded_col;
+			END LOOP;
+		ELSE
+			new_json := NULL;
+		END IF;
+
+		-- Build payload
+		payload := json_build_object(
+			'tstamp', CURRENT_TIMESTAMP,
+			'operation', TG_OP,
+			'schema', TG_TABLE_SCHEMA,
+			'table', TG_TABLE_NAME,
+			'old', old_json,
+			'new', new_json,
+			'pid', pid
+		)::text;
+
+		-- Handle large payloads
+		IF octet_length(payload) < 1000 THEN
+			PERFORM pg_notify(channel, payload);
+		ELSE
+			-- Store large payload and notify with ID (as before)
+			INSERT INTO notify_payloads (payload) VALUES (payload) RETURNING id INTO payload_id;
+
+			notification := json_build_object(
+				'tstamp', CURRENT_TIMESTAMP,
+				'operation', TG_OP,
+				'schema', TG_TABLE_SCHEMA,
+				'table', TG_TABLE_NAME,
+				'pid', pid,
+				'payload_id', payload_id
+			)::text;
+
+			PERFORM pg_notify(channel, notification);
+			RAISE INFO 'Payload over limit';
+		END IF;
+		RETURN NULL;
+		END;
+	$BODY$;
+
+	ALTER FUNCTION public.notify()
+		OWNER TO postgres;
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.4"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.4"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade43-v0.6.5-add-comparisons-notification-channel.sql

@@ -0,0 +1,96 @@
+-- Add procedure to decimate old nav data
+--
+-- New schema version: 0.6.5
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This update modifies notify() to accept, as optional arguments, the
+-- names of columns that are to be *excluded* from the notification.
+-- It is intended for tables with large columns which are however of
+-- no particular interest in a notification.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_database () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', 'public';
+	SET search_path TO public;
+
+	-- BEGIN
+
+	CREATE OR REPLACE TRIGGER comparisons_tg
+		AFTER INSERT OR DELETE OR UPDATE
+		ON comparisons.comparisons
+		FOR EACH ROW
+		EXECUTE FUNCTION public.notify('comparisons', 'data');
+
+	-- END
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.6.5' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.6.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	CALL pg_temp.upgrade_database();
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_database ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.6.5"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.6.5"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/modules/@dougal/binary/classes.js

@@ -503,6 +503,37 @@ class DougalBinaryBundle extends ArrayBuffer {
 		return ab;
 	}
 
+	get records () {
+		const data = [];
+		for (const record of this) {
+			data.push(record.slice(1));
+		}
+		return data;
+	}
+
+	[Symbol.iterator]() {
+		const chunks = this.chunks();
+		let chunkIndex = 0;
+		let chunkIterator = chunks.length > 0 ? chunks[0][Symbol.iterator]() : null;
+
+		return {
+			next() {
+				if (!chunkIterator) {
+					return { done: true };
+				}
+
+				let result = chunkIterator.next();
+				while (result.done && chunkIndex < chunks.length - 1) {
+					chunkIndex++;
+					chunkIterator = chunks[chunkIndex][Symbol.iterator]();
+					result = chunkIterator.next();
+				}
+
+				return result;
+			}
+		};
+	}
+
 }
 
 
@@ -707,6 +738,21 @@ class DougalBinaryChunkSequential extends ArrayBuffer {
 
 		return arr;
 	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
 }
 
 
@@ -901,6 +947,21 @@ class DougalBinaryChunkInterleaved extends ArrayBuffer {
 
 		return arr;
 	}
+
+	[Symbol.iterator]() {
+		let index = 0;
+		const chunk = this;
+		return {
+			next() {
+				if (index < chunk.jCount) {
+					return { value: chunk.getRecord(index++), done: false };
+				} else {
+					return { done: true };
+				}
+			}
+		};
+	}
+
 }
 
 

lib/www/client/source/src/App.vue

@@ -85,7 +85,9 @@ export default {
     },
 
     handleProject (context, {payload}) {
-      this.refreshProjects();
+      if (payload?.table == "public") {
+        this.refreshProjects();
+      }
     },
 
     registerNotificationHandlers () {

lib/www/client/source/src/components/groups/group-comparison-summary.vue

@@ -0,0 +1,187 @@
+<template>
+  <v-card v-if="comparison" class="ma-1">
+    <v-card-title>Comparison Summary: Baseline {{ baseline.pid }} vs Monitor {{ monitor.pid }}</v-card-title>
+    <v-card-text>
+      <v-row>
+        <v-col cols="12" md="6">
+          <h3>Deviation Statistics</h3>
+          <v-simple-table dense>
+            <template v-slot:default>
+              <thead>
+                <tr>
+                  <th>Metric</th>
+                  <th>I (m)</th>
+                  <th>J (m)</th>
+                </tr>
+              </thead>
+              <tbody>
+                <tr>
+                  <td>Mean (μ)</td>
+                  <td>{{ comparison['μ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['μ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>Std Dev (σ)</td>
+                  <td>{{ comparison['σ'][0].toFixed(3) }}</td>
+                  <td>{{ comparison['σ'][1].toFixed(3) }}</td>
+                </tr>
+                <tr>
+                  <td>RMS</td>
+                  <td>{{ comparison.rms[0].toFixed(3) }}</td>
+                  <td>{{ comparison.rms[1].toFixed(3) }}</td>
+                </tr>
+              </tbody>
+            </template>
+          </v-simple-table>
+
+          <h3 class="mt-4">Error distribution</h3>
+          <ul>
+            <li title="Relative to I-axis positive direction">Primary Direction: {{ (comparison.primaryDirection * 180 / Math.PI).toFixed(2) }}°</li>
+            <li>Anisotropy: {{ comparison.anisotropy.toFixed(2) }}</li>
+            <li title="Length of the semi-major axis of the error ellipse">Semi-Major Axis: {{ semiMajorAxis.toFixed(2) }} m</li>
+            <li title="Length of the semi-minor axis of the error ellipse">Semi-Minor Axis: {{ semiMinorAxis.toFixed(2) }} m</li>
+            <li title="Area of the error ellipse">Error Ellipse Area: {{ ellipseArea.toFixed(2) }} m²</li>
+          </ul>
+
+          <h3 class="mt-4">Counts</h3>
+          <ul>
+            <li title="Unique line / point pairs found in both projects">Common Points: {{ comparison.common }}</li>
+            <li title="Total number of points compared, including reshoots, infills, etc.">Comparison Length: {{ comparison.length }}</li>
+            <li title="Number of points in the baseline project">Baseline Points: {{ comparison.baselineLength }} (Unique: {{ comparison.baselineUniqueLength }})</li>
+            <li title="Number of points in the monitor project">Monitor Points: {{ comparison.monitorLength }} (Unique: {{ comparison.monitorUniqueLength }})</li>
+          </ul>
+
+          <p class="mt-3" title="Date and time when the comparison was last performed">Computation timestamp: {{ new Date(comparison.tstamp).toLocaleString() }}</p>
+        </v-col>
+
+        <v-col cols="12" md="6">
+          <h3>Error Ellipse</h3>
+          <svg width="300" height="300" style="border: 1px solid #ccc;">
+            <g :transform="`translate(150, 150) scale(${ellipseScale})`">
+              <line x1="0" y1="-150" x2="0" y2="150" stroke="lightgray" stroke-dasharray="5,5"/>
+              <line x1="-150" y1="0" x2="150" y2="0" stroke="lightgray" stroke-dasharray="5,5"/>
+              <ellipse
+                :rx="Math.sqrt(comparison.eigenvalues[0])"
+                :ry="Math.sqrt(comparison.eigenvalues[1])"
+                :transform="`rotate(${ellipseAngle})`"
+                fill="none"
+                stroke="blue"
+                stroke-width="2"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[0]) * Math.cos(ellipseRad)"
+                :y2="Math.sqrt(comparison.eigenvalues[0]) * Math.sin(ellipseRad)"
+                stroke="red"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+              <line
+                :x1="0"
+                :y1="0"
+                :x2="Math.sqrt(comparison.eigenvalues[1]) * Math.cos(ellipseRad + Math.PI / 2)"
+                :y2="Math.sqrt(comparison.eigenvalues[1]) * Math.sin(ellipseRad + Math.PI / 2)"
+                stroke="green"
+                stroke-width="2"
+                arrow-end="classic-wide-long"
+              />
+            </g>
+          </svg>
+          <p class="text-caption">Ellipse scaled for visibility (factor: {{ ellipseScale.toFixed(1) }}). Axes represent sqrt(eigenvalues).</p>
+        </v-col>
+      </v-row>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: "DougalGroupComparisonSummary",
+
+  props: {
+    baseline: { type: Object, required: true },
+    monitor: { type: Object, required: true },
+    comparison: { type: Object, required: true }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+
+    ellipseAngle () {
+      if (!this.comparison) return 0;
+      const ev = this.comparison.eigenvectors[0];
+      return Math.atan2(ev[1], ev[0]) * 180 / Math.PI;
+    },
+
+    ellipseRad () {
+      return this.ellipseAngle * Math.PI / 180;
+    },
+
+    ellipseRx () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[0]) * this.ellipseScale;
+    },
+
+    ellipseRy () {
+      if (!this.comparison) return 0;
+      return Math.sqrt(this.comparison.eigenvalues[1]) * this.ellipseScale;
+    },
+
+    ellipseScale () {
+      if (!this.comparison) return 1;
+      const maxSigma = Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+      const maxMu = Math.max(
+        Math.abs(this.comparison['μ'][0]),
+        Math.abs(this.comparison['μ'][1])
+      );
+      //const maxExtent = maxMu + 3 * maxSigma;
+      const maxExtent = 20;
+      return 100 / maxExtent; // Adjust scale to fit within ~200 pixels diameter
+    },
+
+    ellipseArea () {
+      if (!this.comparison) return 0;
+      const a = Math.sqrt(this.comparison.eigenvalues[0]);
+      const b = Math.sqrt(this.comparison.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+
+    semiMajorAxis () {
+      if (!this.comparison) return 0;
+      return Math.max(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    semiMinorAxis () {
+      if (!this.comparison) return 0;
+      return Math.min(
+        Math.sqrt(this.comparison.eigenvalues[0]),
+        Math.sqrt(this.comparison.eigenvalues[1])
+      );
+    },
+
+    meanX () {
+      return this.comparison ? this.comparison['μ'][0] : 0;
+    },
+
+    meanY () {
+      return this.comparison ? this.comparison['μ'][1] : 0;
+    },
+
+    ellipseViewBox () {
+      return '-150 -150 300 300';
+    },
+
+  }
+}
+</script>

lib/www/client/source/src/components/groups/group-repeatability-summary.vue

@@ -0,0 +1,118 @@
+<template>
+  <v-card class="ma-1">
+    <v-card-title>Group Repeatability Summary</v-card-title>
+    <v-card-text>
+      <p>Error ellipse area for each baseline-monitor pair. Lower values indicate better repeatability. Colors range from green (best) to red (worst).</p>
+      <v-simple-table dense>
+        <thead>
+          <tr>
+            <th>Baseline \ Monitor</th>
+            <th v-for="project in projects" :key="project.pid">{{ project.pid }}</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr v-for="(baselineProject, rowIndex) in projects" :key="baselineProject.pid">
+            <td>{{ baselineProject.pid }}</td>
+            <td v-for="(monitorProject, colIndex) in projects" :key="monitorProject.pid">
+              <v-tooltip v-if="colIndex > rowIndex" top>
+                <template v-slot:activator="{ on, attrs }">
+                  <div
+                    :style="{ backgroundColor: getEllipseAreaColor(baselineProject.pid, monitorProject.pid), color: 'white', textAlign: 'center', padding: '4px' }"
+                    v-bind="attrs"
+                    v-on="on"
+                    @click="emitInput(baselineProject, monitorProject)"
+                  >
+                    {{ formatEllipseArea(baselineProject.pid, monitorProject.pid) }}
+                  </div>
+                </template>
+                <span v-if="getComp(baselineProject.pid, monitorProject.pid)">
+                  <div>σ_i: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][0].toFixed(2) }} m</div>
+                  <div>σ_j: {{ getComp(baselineProject.pid, monitorProject.pid).meta['σ'][1].toFixed(2) }} m</div>
+                  <div>Anisotropy: {{ getComp(baselineProject.pid, monitorProject.pid).meta.anisotropy.toFixed(0) }}</div>
+                  <div>Ellipse Area: {{ getEllipseArea(baselineProject.pid, monitorProject.pid).toFixed(2) }} m²</div>
+                  <div>Primary Direction: {{ formatPrimaryDirection(getComp(baselineProject.pid, monitorProject.pid)) }}°</div>
+                </span>
+              </v-tooltip>
+            </td>
+          </tr>
+        </tbody>
+      </v-simple-table>
+    </v-card-text>
+  </v-card>
+</template>
+
+<script>
+export default {
+  name: 'DougalGroupRepeatabilitySummary',
+
+  props: {
+    comparisons: {
+      type: Array,
+      required: true
+    },
+    projects: {
+      type: Array,
+      required: true
+    }
+  },
+
+  data () {
+    return {
+    };
+  },
+
+  computed: {
+    compMap () {
+      return new Map(this.comparisons.map(c => [`${c.baseline_pid}-${c.monitor_pid}`, c]));
+    },
+    minEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.min(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    },
+    maxEllipseArea () {
+      if (!this.comparisons.length) return 0;
+      return Math.max(...this.comparisons.map(c => {
+        const a = Math.sqrt(c.meta.eigenvalues[0]);
+        const b = Math.sqrt(c.meta.eigenvalues[1]);
+        return Math.PI * a * b;
+      }));
+    }
+  },
+  methods: {
+    getComp (basePid, monPid) {
+      return this.compMap.get(`${basePid}-${monPid}`);
+    },
+    getEllipseArea (basePid, monPid) {
+      const comp = this.getComp(basePid, monPid);
+      if (!comp) return null;
+      const a = Math.sqrt(comp.meta.eigenvalues[0]);
+      const b = Math.sqrt(comp.meta.eigenvalues[1]);
+      return Math.PI * a * b;
+    },
+    formatEllipseArea (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      return val !== null ? val.toFixed(1) : '';
+    },
+    getEllipseAreaColor (basePid, monPid) {
+      const val = this.getEllipseArea(basePid, monPid);
+      if (val === null) return '';
+      const ratio = (val - this.minEllipseArea) / (this.maxEllipseArea - this.minEllipseArea);
+      const hue = (1 - ratio) * 120;
+      return `hsl(${hue}, 70%, 70%)`;
+    },
+    formatPrimaryDirection (comp) {
+      if (!comp) return '';
+      return (comp.meta.primaryDirection * 180 / Math.PI).toFixed(1);
+    },
+    emitInput (baselineProject, monitorProject) {
+      if (this.getComp(baselineProject.pid, monitorProject.pid)) {
+        this.$emit('input', baselineProject, monitorProject);
+      }
+    }
+  }
+}
+</script>

lib/www/client/source/src/lib/deck.gl/DougalBinaryLoader.js

@@ -84,8 +84,12 @@ const DougalBinaryLoader = {
       for (let k = 0; k < values.length; k++) {
         values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? BigUint64Array : Float32Array)(totalCount);
       }
+    } else if (udv == 4) {
+      for (let k = 0; k < values.length; k++) {
+        values[k] = new (k === 0 ? Uint16Array : k === 1 ? Uint32Array : k === 2 ? Uint16Array : Float32Array)(totalCount);
+      }
     } else {
-      throw new Error(`Invalid udv: Expected 0, 1, or 2; found ${udv}`);
+      throw new Error(`Invalid udv: Expected 0, 1, 2, or 4; found ${udv}`);
     }
 
     let offset = 0;
@@ -110,7 +114,7 @@ const DougalBinaryLoader = {
       offset += chunk.jCount;
     }
 
-    console.log(`Parsed ${totalCount} points, ${values.length} value arrays`);
+    console.log(`Parsed ${totalCount} points, ${values.length} value arrays, udv = ${udv}`);
 
     const attributes = {
       getPosition: {

lib/www/client/source/src/lib/durations.js

@@ -0,0 +1,47 @@
+
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+
+export {
+  duration_to_ms,
+  ms_to_duration,
+  normalise_duration,
+  add_durations
+}

lib/www/client/source/src/router/index.js

@@ -20,6 +20,9 @@ import ProjectSettings from '../views/ProjectSettings.vue'
 import Users from '../views/Users.vue'
 import DougalAppBarExtensionProject from '../components/app-bar-extension-project'
 import DougalAppBarExtensionProjectList from '../components/app-bar-extension-project-list'
+import GroupList from '../views/GroupList.vue'
+import Group from '../views/Group.vue'
+
 
 Vue.use(VueRouter)
 
@@ -196,7 +199,57 @@ Vue.use(VueRouter)
         component: ProjectSettings
       }
     ]
-  }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups",
+    redirect: "/groups/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/",
+    component: GroupList,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups", disabled: true }
+      ],
+      appBarExtension: {
+        // component: DougalAppBarExtensionProjectList
+      }
+    }
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group",
+    redirect: "/groups/:group/"
+  },
+  {
+    pathToRegexpOptions: { strict: true },
+    path: "/groups/:group/",
+    name: "Group",
+    component: Group,
+    meta: {
+      breadcrumbs: [
+        { text: "Comparisons", href: "/groups" },
+        { text: (ctx) => ctx.$route.params.group }
+        /*
+        {
+          text: (ctx) => ctx.$store.state.project.projectName || "…",
+          href: (ctx) => `/projects/${ctx.$store.state.project.projectId || ctx.$route.params.project || ""}/`,
+          title: (ctx) => Object.entries(ctx.$store.getters.projectConfiguration?.organisations ?? {}).map( ([org, ops]) => `* ${org}: ${Object.entries(ops).filter( ([k, v]) => v ).map( ([k, v]) => k ).join(", ")}`).join("\n"),
+          organisations: (ctx) => ctx.$store.getters.projectConfiguration?.organisations ?? {}
+        }
+        */
+      ],
+      /*
+      appBarExtension: {
+        component: DougalAppBarExtensionGroup
+      }
+      */
+    },
+    children: [
+    ]
+  },
 ]
 
 const router = new VueRouter({

lib/www/client/source/src/store/modules/project/actions.js

@@ -1,5 +1,10 @@
 
 async function getProject ({commit, dispatch}, projectId) {
+  if (projectId == null) {
+    console.log(`Skipping call to getProject${projectId})`);
+    return;
+  }
+
   const init = {
     headers: {
       cache: "reload",

lib/www/client/source/src/store/modules/projects/actions.js

@@ -1,13 +1,27 @@
+import * as d3a from 'd3-array';
+import { duration_to_ms, ms_to_duration, normalise_duration, add_durations } from '@/lib/durations';
 
 /** Fetch projects from server
  */
 async function refreshProjects ({commit, dispatch, state, rootState}) {
 
+  async function getSummary (project) {
+      const url = `/project/${project.pid}/summary`;
+      const init = {};
+      const summary = await dispatch('api', [url, init, null, {silent:true}]);
+      if (summary) {
+        return {...project, ...summary};
+      } else {
+        return project;
+      }
+  }
+
   if (state.loading) {
     commit('abortProjectsLoading');
   }
 
   commit('setProjectsLoading');
+  const tstamp = new Date();
   const pid = rootState.project.projectId;
   const url = `/project`;
   const init = {
@@ -17,10 +31,25 @@ async function refreshProjects ({commit, dispatch, state, rootState}) {
   const res = await dispatch('api', [url, init, null, {silent:true}]);
 
   if (res) {
-    commit('setProjects', res);
-    commit('setProjectsTimestamp');
+
+    let projects;
+
+    if (res.some( project => project.pid == null )) {
+      console.warn("At least one project found with no PID!");
+      projects = res.filter( project => project.pid != null );
+    } else {
+      projects = res;
+    }
+
+    commit('setProjects', projects); // First without summaries
+    commit('setProjectsTimestamp', tstamp);
+
+    projects = await Promise.all(projects.map( getSummary ));
+
+    commit('setProjects', projects); // Then with summaries
   }
   commit('clearProjectsLoading');
+  dispatch('prepareGroups');
 }
 
 /** Return a subset of projects from state.projects
@@ -118,4 +147,83 @@ async function getProjects ({commit, dispatch, state}, [{pid, name, schema, grou
   return {projects: filteredProjects, count};
 }
 
-export default { refreshProjects, getProjects };
+
+
+async function prepareGroups ({commit, dispatch, state, rootState}) {
+  const groups = {};
+
+  for (const project of state.projects) {
+
+    if (!project.prod_distance) {
+      // This project has no production data (either not started yet
+      // or production data has not been imported) so we skip it.
+      continue;
+    }
+
+    if (!project.prod_duration.days) {
+      project.prod_duration = normalise_duration(project.prod_duration);
+    }
+
+    for (const name of project.groups) {
+      if (!(name in groups)) {
+        groups[name] = {
+          group: name,
+          num_projects: 0,
+          lines: 0,
+          points: 0,
+          sequences: 0,
+          // Shots:
+          prime: 0,
+          other: 0,
+          ntba: 0,
+          prod_duration: {
+            days: 0,
+            hours: 0,
+            minutes: 0,
+            seconds: 0,
+            milliseconds: 0
+          },
+          prod_distance: 0,
+          shooting_rate: [],
+          projects: []
+        };
+      }
+      const group = groups[name];
+
+      group.num_projects++;
+      group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+      group.points = Math.max(group.points, project.total); // Idem
+      group.sequences += project.seq_final;
+      group.prime += project.prime;
+      group.other += project.other;
+      //group.ntba += project.ntba;
+      group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+      group.prod_distance += project.prod_distance;
+      group.shooting_rate.push(project.shooting_rate);
+      group.projects.push(project);
+    }
+  }
+
+  const grouplist = [];
+  for (const group of Object.values(groups)) {
+    group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+    group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+    delete group.shooting_rate;
+
+    grouplist.push(group);
+  }
+
+  commit('setGroups', grouplist);
+
+}
+
+async function getGroups({commit, dispatch, state, rootState}) {
+  if (!state.groups.length) {
+    await dispatch('refreshProjects');
+  }
+
+  return state.groups;
+}
+
+
+export default { refreshProjects, getProjects, prepareGroups, getGroups };

lib/www/client/source/src/store/modules/projects/getters.js

@@ -3,7 +3,7 @@ function projects (state) {
   return state.projects;
 }
 
-function projectGroups (state) {
+function projectGroupNames (state) {
   return [...new Set(state.projects.map(i => i.groups).flat())].sort();
 }
 
@@ -15,4 +15,8 @@ function projectsLoading (state) {
   return !!state.loading;
 }
 
-export default { projects, projectGroups, projectCount, projectsLoading };
+function groups (state) {
+  return state.groups;
+}
+
+export default { projects, projectGroupNames, projectCount, projectsLoading, groups };

lib/www/client/source/src/store/modules/projects/mutations.js

@@ -39,10 +39,15 @@ function abortProjectsLoading (state) {
   state.loading = null;
 }
 
+function setGroups (state, groups) {
+  state.groups = Object.freeze(groups);
+}
+
 export default {
   setProjects,
   setProjectsLoading,
   clearProjectsLoading,
   setProjectsTimestamp,
-  setProjectsETag
+  setProjectsETag,
+  setGroups
 };

lib/www/client/source/src/store/modules/projects/state.js

@@ -1,5 +1,6 @@
 const state = () => ({
   projects: Object.freeze([]),
+  groups: Object.freeze([]),
   loading: null,
   timestamp: null,
   etag: null,

lib/www/client/source/src/views/Group.vue

@@ -0,0 +1,339 @@
+<template>
+    <dougal-group-map v-if="mapView"
+      :baseline="baseline"
+      :monitor="monitor"
+      :monitors="monitors"
+      @input="mapView=$event"
+    ></dougal-group-map>
+  <v-container fluid fill-height class="ma-0 pa-0" v-else>
+
+    <v-overlay :value="loading && !comparisons.length" absolute>
+      <v-progress-circular
+        indeterminate
+        size="64"
+      ></v-progress-circular>
+    </v-overlay>
+
+    <v-overlay :value="!loading && !groupFound" absolute opacity="0.8">
+      <v-row justify="center">
+        <v-alert
+          type="error"
+        >
+          Group not found
+        </v-alert>
+      </v-row>
+      <v-row justify="center">
+        <v-btn color="primary" @click="refreshProjects">Retry</v-btn>
+      </v-row>
+    </v-overlay>
+
+    <v-row no-gutters align="stretch" class="fill-height">
+      <v-col cols="12" v-if="groupFound">
+
+        <v-data-table class="ma-1"
+          :headers="projectHeaders"
+          :items="projects"
+          dense
+        >
+
+          <template v-slot:item.baseline="{item, value, index}">
+            <v-simple-checkbox v-if="index+1 < projects.length"
+              color="primary"
+              :value="baseline === item"
+              @input="setBaseline(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.monitor="{item, value, index}">
+            <v-simple-checkbox v-if="index > 0 && !(index <= baselineIndex)"
+              color="primary"
+              :value="monitor === item"
+              @input="setMonitor(item)"
+            ></v-simple-checkbox>
+          </template>
+
+          <template v-slot:item.pid="{item, value}">
+            <v-chip
+              label
+              small
+              outlined
+              :href="`/projects/${item.pid}`"
+              :color="!item.archived ? 'primary' : ''"
+            >{{ value }}</v-chip>
+          </template>
+
+          <template v-slot:item.fsp="{item, value}">
+            <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.lsp="{item, value}">
+            <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+          </template>
+
+          <template v-slot:item.prod_duration="{item, value}">
+            <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+              {{ value.days }} d
+            </span>
+            <span v-else>
+              {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+            </span>
+          </template>
+
+          <template v-slot:item.prod_distance="{item, value}">
+            {{ (value/1000).toFixed(1) }} km
+          </template>
+
+          <template v-slot:footer.prepend>
+
+            <v-btn v-if="baseline && !mapView"
+              text
+              color="primary"
+              title="Switch to map view"
+              @click="mapView = true"
+            >View map</v-btn>
+
+            <v-btn v-if="comparison"
+              text
+              color="primary"
+              title="Back to summary"
+              @click="clearComparison"
+            >Back</v-btn>
+          </template>
+
+        </v-data-table>
+
+<!-- BEGIN TEST -->
+
+    <dougal-group-comparison-summary v-if="comparison"
+      :baseline="baseline"
+      :monitor="monitor"
+      :comparison="comparison"
+    ></dougal-group-comparison-summary>
+
+    <dougal-group-repeatability-summary v-else-if="comparisons.length"
+      :comparisons="comparisons"
+      :projects="projects"
+      @input="setComparison"
+    ></dougal-group-repeatability-summary>
+
+<!-- END   TEST -->
+
+
+      </v-col>
+      <v-col cols="12" v-else>
+        <v-card>
+          <v-card-text>
+            Group does not exist.
+          </v-card-text>
+        </v-card>
+      </v-col>
+    </v-row>
+  </v-container>
+</template>
+
+<script>
+import { mapActions, mapGetters } from 'vuex'
+import AccessMixin from '@/mixins/access';
+import DougalGroupRepeatabilitySummary from '@/components/groups/group-repeatability-summary.vue';
+import DougalGroupComparisonSummary from '@/components/groups/group-comparison-summary';
+import DougalGroupMap from '@/components/groups/group-map';
+
+export default {
+  name: 'Group',
+
+  mixins: [
+    AccessMixin
+  ],
+
+  components: {
+    DougalGroupRepeatabilitySummary,
+    DougalGroupComparisonSummary,
+    DougalGroupMap
+  },
+
+  data () {
+    return {
+      projectHeaders: [
+        {
+          value: "baseline",
+          text: "Baseline"
+        },
+        {
+          value: "monitor",
+          text: "Monitor"
+        },
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      mapView: false,
+
+      baseline: null,
+      monitor: null,
+      comparisons: []
+    }
+  },
+
+  computed: {
+
+    groupName () {
+      return this.$route.params.group;
+    },
+
+    group () {
+      return this.groups.find( i => i.group === this.groupName );
+    },
+
+    groupFound () {
+      return !!(this.loading || this.group);
+    },
+
+    projects () {
+      return this.group?.projects.toSorted((a, b) => a.pid.localeCompare(b.pid));
+    },
+
+    baselineIndex () {
+      return this.projects.indexOf(this.baseline);
+    },
+
+    monitors () {
+      if (this.baseline && this.comparisons) {
+        return this.comparisons
+          .filter( i => i.baseline_pid == this.baseline.pid )
+          .map( i => this.projects.find( p => p.pid == i.monitor_pid ));
+      } else {
+        return null;
+      }
+    },
+
+    comparison () {
+      return this.comparisons.find( row =>
+        row.baseline_pid == this.baseline?.pid && row.monitor_pid == this.monitor?.pid
+      )?.meta;
+    },
+
+    ...mapGetters(["loading", "groups"])
+  },
+
+  methods: {
+
+    setBaseline (project) {
+      if (project === this.baseline) {
+        this.baseline = null;
+      } else {
+        this.baseline = project;
+        if (this.monitor) {
+          if (this.projects.indexOf(this.monitor) <= this.projects.indexOf(this.baseline)) {
+            this.monitor = null;
+          }
+        }
+      }
+    },
+
+    setMonitor (project) {
+      if (project === this.monitor) {
+        this.monitor = null;
+      } else {
+        this.monitor = project;
+      }
+    },
+
+    clearComparison () {
+      this.baseline = null;
+      this.monitor = null;
+    },
+
+    setComparison (baseline, monitor) {
+      this.clearComparison();
+      this.setBaseline(baseline);
+      this.setMonitor(monitor);
+    },
+
+    async getComparisons () {
+      const url = `/comparison/group/${this.$route.params.group}`;
+      this.comparisons = await this.api([url]);
+    },
+
+    // TODO Should this go in a Vuex action rather?
+    async refreshComparisons () {
+      await this.getGroups();
+      if (this.groupFound) {
+        await this.getComparisons();
+      }
+    },
+
+    /*
+    async getComparison () {
+      if (this.baseline && this.monitor) {
+        const url = `/comparison/group/${this.$route.params.group}/baseline/${this.baseline.pid}/monitor/${this.monitor.pid}`;
+        const comparison = await this.api([url]);
+        if (comparison) {
+          this.comparison = comparison;
+        }
+      }
+    },
+    */
+
+    handleComparisons (context, {payload}) {
+      this.refreshComparisons();
+    },
+
+    registerNotificationHandlers (action = "registerHandler") {
+
+      this.$store.dispatch(action, {
+        table: 'comparisons',
+        handler: this.handleComparisons
+      });
+
+    },
+
+    unregisterNotificationHandlers () {
+      return this.registerNotificationHandlers("unregisterHandler");
+    },
+
+
+    ...mapActions(["api", "getGroups", "refreshProjects"])
+  },
+
+  async mounted () {
+    this.registerNotificationHandlers();
+    this.refreshComparisons()
+  },
+
+  beforeDestroy () {
+    this.unregisterNotificationHandlers();
+  }
+
+
+}
+</script>

lib/www/client/source/src/views/GroupList.vue

@@ -0,0 +1,396 @@
+<template>
+  <v-container fluid>
+
+    <v-data-table
+    :headers="headers"
+    :items="displayItems"
+    item-key="group"
+    :options.sync="options"
+    :expanded.sync="expanded"
+    show-expand
+    :loading="loading"
+    >
+
+      <template v-slot:item.group="{item, value}">
+        <v-chip
+          label
+          small
+          :href="`./${value}`"
+        >{{ value }}</v-chip>
+      </template>
+
+      <template v-slot:item.shots_total="{item, value}">
+        <div>{{ item.prime + item.other }}</div>
+        <v-progress-linear
+          background-color="secondary"
+          color="primary"
+          :value="item.prime/(item.prime+item.other)*100"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:item.prime="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.other="{item, value}">
+        {{ value }}
+        ({{ (value / (item.prime + item.other) * 100).toFixed(1) }}%)
+      </template>
+
+      <template v-slot:item.prod_duration="{item, value}">
+        <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+          {{ value.days }} d
+        </span>
+        <span v-else>
+          {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+        </span>
+      </template>
+
+      <template v-slot:item.prod_distance="{item, value}">
+        {{ (value/1000).toFixed(1) }} km
+      </template>
+
+      <template v-slot:item.shooting_rate_mean="{item, value}">
+        {{ (value).toFixed(2) }} s ±{{ (item.shooting_rate_sd).toFixed(3) }} s
+      </template>
+
+      <template v-slot:item.shots_per_point="{item, value}">
+        <div>
+          {{ ((item.prime + item.other)/item.points).toFixed(1) }}
+          ({{ ((((item.prime + item.other)/item.points) / item.num_projects)*100).toFixed(2) }}%)
+        </div>
+        <v-progress-linear
+          :value="((((item.prime + item.other)/item.points) / item.num_projects)*100)"
+        ></v-progress-linear>
+      </template>
+
+      <template v-slot:expanded-item="{ headers, item }">
+        <td :colspan="headers.length">
+          <v-data-table class="ma-1"
+            :headers="projectHeaders"
+            :items="item.projects"
+            dense
+            hide-default-footer
+          >
+
+            <template v-slot:item.pid="{item, value}">
+              <a :href="`/projects/${value}`" title="Go to project">{{ value }}</a>
+            </template>
+
+            <template v-slot:item.fsp="{item, value}">
+              <span title="First production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.lsp="{item, value}">
+              <span title="Last production shot">{{value.tstamp.substr(0, 10)}}</span>
+            </template>
+
+            <template v-slot:item.prod_duration="{item, value}">
+              <span v-if="value.days > 2" :title="`${value.days} d ${value.hours} h ${value.minutes} m ${(value.seconds + value.milliseconds/1000).toFixed(3)} s`">
+                {{ value.days }} d
+              </span>
+              <span v-else>
+                {{ value.days }} d {{ value.hours }} h {{ value.minutes }} m {{ (value.seconds + value.milliseconds/1000).toFixed(1) }} s
+              </span>
+            </template>
+
+            <template v-slot:item.prod_distance="{item, value}">
+              {{ (value/1000).toFixed(1) }} km
+            </template>
+
+          </v-data-table>
+        </td>
+      </template>
+
+    </v-data-table>
+
+  </v-container>
+</template>
+
+<style>
+td p:last-of-type {
+  margin-bottom: 0;
+}
+</style>
+
+<script>
+import { mapActions, mapGetters } from 'vuex';
+import AccessMixin from '@/mixins/access';
+
+
+// FIXME send to lib/utils or so
+/*
+function duration_to_ms(v) {
+  if (v instanceof Object) {
+    return (
+      (v.days || 0) * 86400000 +
+      (v.hours || 0) * 3600000 +
+      (v.minutes || 0) * 60000 +
+      (v.seconds || 0) * 1000 +
+      (v.milliseconds || 0)
+    );
+  } else {
+    return {
+      days: 0,
+      hours: 0,
+      minutes: 0,
+      seconds: 0,
+      milliseconds: 0
+    }
+  }
+}
+
+function ms_to_duration(v) {
+  const days = Math.floor(v / 86400000);
+  v %= 86400000;
+  const hours = Math.floor(v / 3600000);
+  v %= 3600000;
+  const minutes = Math.floor(v / 60000);
+  v %= 60000;
+  const seconds = Math.floor(v / 1000);
+  const milliseconds = v % 1000;
+  return { days, hours, minutes, seconds, milliseconds };
+}
+
+function normalise_duration (v) {
+  return ms_to_duration(duration_to_ms(v));
+}
+
+function add_durations(a, b) {
+  return ms_to_duration(duration_to_ms(a) + duration_to_ms(b));
+}
+*/
+
+export default {
+  name: "GroupList",
+
+  components: {
+  },
+
+  mixins: [
+    AccessMixin
+  ],
+
+  data () {
+    return {
+      headers: [
+        {
+          value: "group",
+          text: "Group name"
+        },
+        {
+          value: "num_projects",
+          text: "Number of campaigns"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "points",
+          text: "Preplot points"
+        },
+        {
+          value: "sequences",
+          text: "Total sequences"
+        },
+        {
+          value: "shots_total",
+          text: "Total shots"
+        },
+        {
+          value: "prime",
+          text: "Total prime"
+        },
+        {
+          value: "other",
+          text: "Total reshoot + infill"
+        },
+        /*
+        {
+          value: "ntba",
+          text: "Total NTBA"
+        },
+        */
+        {
+          value: "prod_duration",
+          text: "Total duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Total distance"
+        },
+        {
+          value: "shooting_rate_mean",
+          text: "Shooting rate (mean)"
+        },
+        {
+          value: "shots_per_point",
+          text: "Shots per point"
+        },
+      ],
+      items: [],
+      expanded: [],
+      options: { sortBy: ["group"], sortDesc: [false] },
+
+      projectHeaders: [
+        {
+          value: "pid",
+          text: "ID"
+        },
+        {
+          value: "name",
+          text: "Name"
+        },
+        {
+          value: "fsp",
+          text: "Start"
+        },
+        {
+          value: "lsp",
+          text: "Finish"
+        },
+        {
+          value: "lines",
+          text: "Preplot lines"
+        },
+        {
+          value: "seq_final",
+          text: "Num. of sequences"
+        },
+        {
+          value: "prod_duration",
+          text: "Duration"
+        },
+        {
+          value: "prod_distance",
+          text: "Distance"
+        },
+      ],
+
+      // Context menu stuff
+      contextMenuShow: false,
+      contextMenuX: 0,
+      contextMenuY: 0,
+      contextMenuItem: null,
+
+      /*
+      // FIXME Eventually need to move this into Vuex
+      groups: []
+      */
+    }
+  },
+
+  computed: {
+    displayItems () {
+      return this.items.filter(i => i.prod_distance);
+    },
+
+    ...mapGetters(['loading', 'groups'])
+  },
+
+  methods: {
+
+    /*
+    async prepareGroups () {
+      //const groups = await this.api(["/prospects"]);
+      //console.log("groups", groups);
+      const groups = {};
+
+      for (const project of this.projects) {
+
+        if (!project.prod_distance) {
+          // This project has no production data (either not started yet
+          // or production data has not been imported) so we skip it.
+          continue;
+        }
+
+        if (!project.prod_duration.days) {
+          project.prod_duration = normalise_duration(project.prod_duration);
+        }
+
+        for (const name of project.groups) {
+          if (!(name in groups)) {
+            groups[name] = {
+              group: name,
+              num_projects: 0,
+              lines: 0,
+              points: 0,
+              sequences: 0,
+              // Shots:
+              prime: 0,
+              other: 0,
+              ntba: 0,
+              prod_duration: {
+                days: 0,
+                hours: 0,
+                minutes: 0,
+                seconds: 0,
+                milliseconds: 0
+              },
+              prod_distance: 0,
+              shooting_rate: [],
+              projects: []
+            };
+          }
+          const group = groups[name];
+
+          group.num_projects++;
+          group.lines = Math.max(group.lines, project.lines); // In case preplots changed
+          group.points = Math.max(group.points, project.total); // Idem
+          group.sequences += project.seq_final;
+          group.prime += project.prime;
+          group.other += project.other;
+          //group.ntba += project.ntba;
+          group.prod_duration = add_durations(group.prod_duration, project.prod_duration);
+          group.prod_distance += project.prod_distance;
+          group.shooting_rate.push(project.shooting_rate);
+          group.projects.push(project);
+        }
+      }
+
+      this.groups = [];
+      for (const group of Object.values(groups)) {
+        group.shooting_rate_mean = d3a.mean(group.shooting_rate);
+        group.shooting_rate_sd = d3a.deviation(group.shooting_rate);
+        delete group.shooting_rate;
+
+        this.groups.push(group);
+      }
+
+    },
+    */
+
+    async list () {
+      this.items = [...this.groups];
+    },
+
+    async load () {
+      await this.refreshProjects();
+      //await this.prepareGroups();
+      await this.list();
+    },
+
+    registerNotificationHandlers () {
+      this.$store.dispatch('registerHandler', {
+        table: 'project`',
+
+        handler: (context, message) => {
+          if (message.payload?.table == "public") {
+            this.load();
+          }
+        }
+      });
+    },
+
+    ...mapActions(["api", "showSnack", "refreshProjects"])
+  },
+
+  mounted () {
+    this.registerNotificationHandlers();
+    this.load();
+  }
+}
+
+</script>

lib/www/client/source/src/views/ProjectList.vue

@@ -27,6 +27,8 @@
         <v-chip v-for="group in value"
           label
           small
+          :title="`View repeatability data for ${group}`"
+          :href="`/groups/${group}`"
         >{{ group }}</v-chip>
       </template>
 
@@ -187,19 +189,20 @@ export default {
     ...mapGetters(['loading', 'projects'])
   },
 
+  watch: {
+
+    async projects () {
+      await this.load();
+    }
+
+  },
+
   methods: {
 
     async list () {
       this.items = [...this.projects];
     },
 
-    async summary (item) {
-      const details = await this.api([`/project/${item.pid}/summary`]);
-      if (details) {
-        return Object.assign({}, details, item);
-      }
-    },
-
     title (item) {
       if (item.organisations) {
         return "Access:\n" + Object.entries(item.organisations).map( org =>
@@ -210,30 +213,22 @@ export default {
     },
 
     async load () {
-      await this.refreshProjects();
+      if (!this.projects.length) {
+        this.refreshProjects();
+      }
       await this.list();
-      const promises = [];
-      for (const key in this.items) {
-        const item = this.items[key];
-        const promise = this.summary(item)
-        .then( expanded => {
-          if (expanded) {
-            this.$set(this.items, key, expanded);
-          }
-        });
-        promises.push(promise);
+    },
+
+    handlerLoad (context, {payload}) {
+      if (payload?.table == "public") {
+        this.load();
       }
     },
 
     registerNotificationHandlers () {
       this.$store.dispatch('registerHandler', {
         table: 'project`',
-
-        handler: (context, message) => {
-          if (message.payload?.table == "public") {
-            this.load();
-          }
-        }
+        handler: this.handlerLoad
       });
     },
 

lib/www/server/api/index.js

@@ -357,6 +357,26 @@ app.map({
 			delete: [ mw.auth.operations, mw.auth.access.write, mw.info.delete ]
 		}
 	},
+
+	/*
+	 * 4D comparisons
+	 */
+
+	// FIXME no authentication yet!
+
+	'/comparison/group': {
+		get: [ mw.etag.noSave, mw.comparisons.groups.list ],
+		'/:group': {
+			get: [ mw.etag.noSave, mw.comparisons.groups.get ],
+		},
+	},
+
+
+
+	/*
+	 * Other endpoints
+	 */
+
 	'/queue/outgoing/': {
 		'asaqc': {
 			get: [ mw.etag.noSave, mw.queue.asaqc.get ],

lib/www/server/api/middleware/comparisons/groups/get.js

@@ -0,0 +1,20 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		// const data = await comparisons.groups()
+		// if (data?.[req.params.group]) {
+		const data = await comparisons.getGroup(req.params.group);
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(404).send({message: "Group does not exist"});
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/groups/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+	list: require('./list'),
+	get: require('./get'),
+}

lib/www/server/api/middleware/comparisons/groups/list.js

@@ -0,0 +1,18 @@
+const comparisons = require('../../../../lib/comparisons');
+
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const data = await comparisons.groups()
+		if (data) {
+			res.status(200).send(data);
+		} else {
+			res.status(204).end();
+		}
+		return next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/comparisons/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	groups: require('./groups')
+}

lib/www/server/api/middleware/index.js

@@ -23,4 +23,5 @@ module.exports = {
 	version: require('./version'),
 	admin: require('./admin'),
 	compress: require('./compress'),
+	comparisons: require('./comparisons'),
 };

lib/www/server/lib/binary/bundle.js

@@ -38,7 +38,7 @@ function bundle (json, opts = {}) {
 
 		return encode.sequential(json, el => el.sailline, el => el.point, deltas, values, type)
 
-	} if (type == 1) {
+	} else if (type == 1) {
 		/* Preplot information – source line points
 		 *
 		 * elem 0: Float32Array Longitude
@@ -273,6 +273,113 @@ function bundle (json, opts = {}) {
 		});
 
 		return encode.sequential(json, el => el.sequence, el => el.point, deltas, values, type)
+	} else if (type == 4) {
+		/* Bare final positions
+		 *
+		 * Δelem    0: Sequence no. (Uint16Array, Uint8Array)
+		 * elem   0‒1: Float32Array, Float32Array – Final positions (x, y)
+		 *
+		 */
+
+		deltas.push({
+			key: el => el[2],
+			baseType: Uint16Array,
+			incrType: Int8Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[4],
+			type: Float32Array
+		});
+
+		return encode.sequential(json, el => el[0], el => el[1], deltas, values, type)
+	} else if (type == 0xa) {
+		/* 4D comparison data:
+		 *
+		 * elem0: i differences
+		 * elem1: j differences
+		 *
+		 * Note that line/point may not be unique.
+		 *
+		 */
+
+		/*
+		deltas.push({
+			key: el => el.baseTStamp,
+			baseType: BigUint64Array,
+			incrType: Int32Array
+		});
+
+		deltas.push({
+			key: el => el.monTStamp,
+			baseType: BigUint64Array,
+			incrType: Int32Array
+		})
+		*/
+
+		values.push({
+			key: el => el[2],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		/*
+		values.push({
+			key: el => el.baseSeq,
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el.monSeq,
+			type: Uint16Array
+		});
+		*/
+
+		return encode.sequential(json, el => el[0], el => el[1], deltas, values, type)
+	} else if (type == 0xc) {
+		/* 4D comparison data (reduced sample)
+		 *
+		 * Input is comparison records, i.e.:
+		 * [ [ line, point, δi, δj ], … ]
+		 *
+		 * elem0: line
+		 * elem1: point
+		 * elem2: δi
+		 * elem3: δj
+		 *
+		 * Note that the chunk's `i` and `j` values are not used
+		 */
+
+		values.push({
+			key: el => el[0],
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el[1],
+			type: Uint16Array
+		});
+
+		values.push({
+			key: el => el[2],
+			type: Float32Array
+		});
+
+		values.push({
+			key: el => el[3],
+			type: Float32Array
+		});
+
+		return encode.sequential(json, el => 0, el => 0, deltas, values, type)
 	}
 }
 

lib/www/server/lib/comparisons/geometric-differences.js

@@ -0,0 +1,578 @@
+const d3a = require('d3-array');
+const { DougalBinaryBundle } = require('@dougal/binary');
+const { pool, setSurvey } = require('../db/connection');
+const db = require('../db');
+const { bundle } = require('../binary/bundle');
+const setops = require('../utils/setops');
+const { ijRMS, combinations, computeSample } = require('./utils');
+const { computePCA } = require('./pca');
+const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+
+async function fetchErrors (pid) {
+
+	const client = await setSurvey(pid);
+
+	try {
+		const text = `
+			SELECT
+				fs.line, fs.point,
+				ij_error(fs.line::double precision, fs.point::double precision, fs.geometry)::json AS errorfinal
+			FROM
+				final_shots fs
+			ORDER BY fs.line, fs.point;
+		`;
+
+		const res = await client.query(text);
+
+		return res.rows.map( row =>
+			[row.line, row.point, row.errorfinal.coordinates[0], row.errorfinal.coordinates[1]]
+		);
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function groups () {
+	const projects = await db.project.get();
+	const groupNames = [
+		...projects
+			.reduce( (acc, cur) => acc.add(...cur.groups), new Set() )
+	].filter( i => !!i );
+
+	return Object.fromEntries(groupNames.map( g => [g, projects.filter( p => p.groups.includes(g) )] ));
+}
+
+/*
+async function compare (baselineProjectID, monitorProjectID) {
+	console.log("Getting baseline", baselineProjectID);
+	const baselineData = await db.sequence.get(baselineProjectID);
+	console.log("Getting monitor", monitorProjectID);
+	const monitorData = await db.sequence.get(monitorProjectID);
+	console.log("Comparing");
+
+	const comparison = comparisonGeometricDifferences(baselineData, monitorData);
+	return comparison;
+}
+*/
+
+function geometric_differences (baseline, monitor) {
+
+	if (!baseline || !baseline.length) {
+		throw new Error("No baseline data");
+	}
+
+	if (!monitor || !monitor.length) {
+		throw new Error("No monitor data");
+	}
+
+	const comparison = []; // An array of { line, point, εi, εj }; line + point may be repeated
+
+	for (const bp of baseline) {
+		const monitor_points = monitor.filter( mp => mp[0] === bp[0] && mp[1] === bp[1] );
+
+		if (!monitor_points.length) {
+			// console.log(`No match for L${bp[0]} P${bp[1]}`);
+			continue;
+		}
+
+		for (const mp of monitor_points) {
+			const εi = mp[2] - bp[2], εj = mp[3] - bp[3];
+			comparison.push([bp[0], bp[1], εi, εj]);
+		}
+
+	}
+
+	return comparison;
+}
+
+async function compare (baselineProjectID, monitorProjectID, infoObj) {
+	console.log("Getting baseline", baselineProjectID);
+	const baselineData = await fetchErrors(baselineProjectID);
+	console.log("Getting monitor", monitorProjectID);
+	const monitorData = await fetchErrors(monitorProjectID);
+	console.log("Comparing");
+
+	const comparison = geometric_differences(baselineData, monitorData);
+
+	if (infoObj instanceof Object) {
+		const baselineIJ = baselineData.map(i => i.slice(0,2));
+		const monitorIJ = monitorData.map(i => i.slice(0,2));
+
+		infoObj.compared = comparison.length;
+		infoObj.baselineLength = baselineData.length;
+		infoObj.monitorLength = monitorData.length;
+		infoObj.baselineUniqueLength = setops.unique(baselineIJ).length;
+		infoObj.monitorUniqueLength = setops.unique(monitorIJ).length;
+		infoObj.common = setops.intersection(baselineIJ, monitorIJ).length;
+	}
+
+	return comparison;
+}
+
+
+async function save (baselineProjectID, monitorProjectID, bundle, meta) {
+	const info = {};
+	if (!bundle) {
+		const comparison = await compare(baselineProjectID, monitorProjectID, info);
+		if (comparison.length) {
+			bundle = asBundle(comparison);
+		} else {
+			console.warn(`No matching points between ${baselineProjectID} and ${monitorProjectID}`);
+			return;
+		}
+	} else if (!(bundle instanceof DougalBinaryBundle)) {
+		throw new Error("Illegal data: `bundle` must of null or of type DougalBinaryBundle");
+	}
+
+	if (!bundle.byteLength) {
+		console.warn(`Empty comparison results between ${baselineProjectID} and ${monitorProjectID}. Refusing to store`);
+		return;
+	}
+
+	meta = {tstamp: (new Date()), ...info, ...stats(bundle), ...meta};
+
+	console.log("Storing in database");
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			INSERT INTO comparisons.comparisons
+				(type, baseline_pid, monitor_pid, data, meta)
+			VALUES ('geometric_difference', $1, $2, $3, $4)
+			ON CONFLICT (type, baseline_pid, monitor_pid)
+			DO UPDATE SET
+				data = EXCLUDED.data,
+				meta = EXCLUDED.meta;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, Buffer.from(bundle), meta ];
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function saveSample (baselineProjectID, monitorProjectID, opts = {}) {
+	DEBUG("Not bothering to save samples. This feature will be removed.");
+}
+
+/*
+async function saveSample (baselineProjectID, monitorProjectID, opts = {}) {
+	let sample = opts.sample;
+	let populationStats = opts.populationStats;
+	let sampleStats = opts.sampleStats;
+
+	if (!sample?.length) {
+		const sampleSize = opts.sampleSize ?? 2000;
+		const record = await get(baselineProjectID, monitorProjectID);
+		let data;
+
+		if (record) {
+			data = record.data;
+		} else {
+			console.log("Full data not found in database");
+			data = asBundle(await compare(baselineProjectID, monitorProjectID));
+		}
+
+		sample = computeSample(data, opts);
+
+		if (!populationStats) {
+			populationStats = stats(data);
+		}
+	}
+
+	const bundle = asBundle(sample, {type: 0x0c});
+
+	if (!sampleStats) {
+		sampleStats = stats(bundle);
+	}
+
+	meta = {tstamp: (new Date()), populationStats, sampleStats, ...(opts.meta??{})};
+
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			INSERT INTO comparisons.comparisons
+				(type, baseline_pid, monitor_pid, data, meta)
+			VALUES ('geometric_difference_sample', $1, $2, $3, $4)
+			ON CONFLICT (type, baseline_pid, monitor_pid)
+			DO UPDATE SET
+				data = EXCLUDED.data,
+				meta = EXCLUDED.meta;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, Buffer.from(bundle), meta ];
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+*/
+
+async function get (baselineProjectID, monitorProjectID, type = 'geometric_difference') {
+
+	const client = await pool.connect();
+
+	try {
+
+		const text = `
+			SELECT data, meta
+			FROM comparisons.comparisons
+			WHERE type = $3 AND baseline_pid = $1 AND monitor_pid = $2;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID, type ];
+		const res = await client.query(text, values);
+		if (!res.rows.length) {
+			console.log("Comparison not found in database");
+			return;
+		}
+
+		const { data, meta } = res.rows[0];
+		return {
+			data: DougalBinaryBundle.clone(data),
+			meta
+		};
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+
+async function getSample (baselineProjectID, monitorProjectID) {
+	return await get(baselineProjectID, monitorProjectID, 'geometric_difference_sample');
+}
+
+
+async function remove (baselineProjectID, monitorProjectID) {
+	const client = await pool.connect();
+
+	try {
+		const text = `
+			DELETE
+			FROM comparisons.comparisons
+			WHERE
+				(type = 'geometric_difference' OR type = 'geometric_difference_sample')
+				AND baseline_pid = $1
+				AND monitor_pid = $2;
+		`;
+
+		const values = [ baselineProjectID, monitorProjectID ];
+
+		const res = await client.query(text, values);
+		return res.rowCount;
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+function stats (comparison) {
+	let i, j, δi, δj;
+
+	if (comparison instanceof DougalBinaryBundle) {
+		console.log("Computing stats");
+		const udv = comparison.chunks()[0]?.udv;
+
+		if (!udv) {
+			console.error("Could not determine udv from first chunk");
+			console.log(comparison.chunks());
+			return;
+		}
+
+		let records;
+
+		if (udv == 0xa) {
+			records = comparison.records;
+
+			// Transpose the records
+			[ i, j, δi, δj ] = Array.from({ length: 4 }, (_, i) => records.map(row => row[i]));
+		} else if (udv == 0xc) {
+			records = comparison.records;
+			let _;
+			[ _, _, i, j, δi, δj ] = Array.from({ length: 6 }, (_, i) => records.map(row => row[i]));
+		} else {
+			throw new Error(`Unrecognised DougalBinaryBundle User Defined Value: ${udv}`);
+		}
+
+		return {
+			length: records.length,
+			μ: [ d3a.mean(δi), d3a.mean(δj) ],
+			σ: [ d3a.deviation(δi), d3a.deviation(δj) ],
+			//rms: ijRMS(δi, δj),
+			...computePCA(records)
+		}
+	} else if (Array.isArray(comparison)) {
+		if (Array.isArray(comparison[0])) {
+			return stats(asBundle(comparison, {type: 0xc}));
+		} else {
+			// Assume object
+			return stats(asBundle(comparison));
+		}
+	}
+}
+
+
+/** Compare two projects' errorfinal quantities.
+ *
+ * Assumes that the preplots are the same.
+ * It is not a terribly efficient way of doing it, but considering
+ * that this is, by and large only going to be done once every few
+ * hours for an active prospect, and never for inactive ones, I
+ * think and hope we can live with that.
+ *
+ * `baseline` and `monitor` are the result of calling
+ * db.sequence.get(projectId) on each of the respective
+ * projects.
+ */
+/*
+function comparisonGeometricDifferences (baseline, monitor) {
+
+	if (!baseline || !baseline.length) {
+		throw new Error("No baseline data");
+	}
+
+	if (!monitor || !monitor.length) {
+		throw new Error("No monitor data");
+	}
+
+	const comparison = []; // An array of { line, point, εi, εj, δts }; line + point may be repeated
+
+	for (const bp of baseline) {
+		if (!bp.errorfinal) {
+			console.log(`No final data for baseline point L${bp.line} S${bp.sequence} P${bp.point}`);
+			continue;
+		}
+
+		const monitor_points = monitor.filter( mp => mp.line === bp.line && mp.point === bp.point );
+		for (const mp of monitor_points) {
+			if (!mp.errorfinal) {
+				console.log(`No final data for monitor point L${mp.line} S${mp.sequence} P${mp.point}`);
+				continue;
+			}
+
+			const line = bp.line;
+			const point = bp.point;
+			const baseSeq = bp.sequence;
+			const monSeq = mp.sequence;
+			const baseTStamp = bp.tstamp;
+			const monTStamp = mp.tstamp;
+			const δi = bp.errorfinal.coordinates[0] - mp.errorfinal.coordinates[0];
+			const δj = bp.errorfinal.coordinates[1] - mp.errorfinal.coordinates[1];
+
+			const obj = {line, point, baseSeq, monSeq, baseTStamp, monTStamp, δi, δj};
+			comparison.push(obj);
+			// console.log(obj);
+		}
+	}
+
+	return comparison.sort(sortFn);
+}
+
+function sortComparison (comparison) {
+	return comparison.sort( (a, b) => {
+		if (a.line == b.line) {
+			if (a.point == b.point) {
+				return a.baseTStamp - b.baseTStamp;
+			} else {
+				return a.point - b.point;
+			}
+		} else {
+			return a.line - b.line;
+		}
+	})
+}
+*/
+
+function sortFn (a, b) {
+	if (a.line == b.line) {
+		if (a.point == b.point) {
+			return a.baseTStamp - b.baseTStamp;
+		} else {
+			return a.point - b.point;
+		}
+	} else {
+		return a.line - b.line;
+	}
+}
+
+function asBundle (comparison, opts = {type: 0x0a}) {
+	return DougalBinaryBundle.clone(bundle(comparison, opts));
+}
+
+function fromBundle (bundle) {
+	if (!(bundle instanceof DougalBinaryBundle)) {
+		bundle = DougalBinaryBundle.clone(bundle);
+	}
+
+	const json = [];
+	for (const record of bundle) {
+		record.shift();
+		json.push(record);
+	}
+	return json;
+}
+
+async function saveGroup (group, opts = {}) {
+	if (group == null) {
+		// Save everything
+		const g = await groups();
+		for (const group of Object.values(g)) {
+			await saveGroup(group)
+		}
+	} if (typeof group === "string") {
+		// This is a group name
+		const g = await groups();
+		group = groups[g];
+	}
+
+	if (Array.isArray(group)) {
+		const pids = group.map( i => i.pid ).sort();
+
+		for (const [ baselineProjectID, monitorProjectID ] of combinations(pids, 2)) {
+			try {
+				if (!opts.overwrite) {
+					const exists = await get(baselineProjectID, monitorProjectID);
+					if (exists) {
+						DEBUG("Not overwriting existing comparison between %s and %s. Skipping", baselineProjectID, monitorProjectID);
+						continue;
+					}
+				}
+
+				const isSaved = await save(baselineProjectID, monitorProjectID);
+				if (isSaved) {
+					await saveSample(baselineProjectID, monitorProjectID, opts.sampleOpts);
+				} else {
+					await remove(baselineProjectID, monitorProjectID);
+				}
+				DEBUG("Saved comparison between %s and %s", baselineProjectID, monitorProjectID);
+			} catch (err) {
+				console.error(err);
+				ERROR("Error saving comparison between %s and %s", baselineProjectID, monitorProjectID);
+			}
+		}
+	}
+}
+
+/*
+async function getGroup (groupName, opts = {}) {
+
+	const group = (await groups())?.[groupName]?.map( i => i.pid)?.sort();
+
+	if (!group?.length) return;
+
+	const client = await pool.connect();
+
+	try {
+
+		const text = `
+			-- SQL query goes here
+		`;
+
+		const values = combinations(group, 2);
+		const res = await client.query(text, values);
+		if (!res.rows.length) {
+			console.log("Comparison not found in database");
+			return;
+		}
+
+		if (opts.returnData) {
+			return res.rows.map( row => ({
+				data: DougalBinaryBundle.clone(row.data),
+				meta: row.meta
+			});
+		} else {
+			return res.rows.map( row => row.meta );
+		}
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+*/
+
+
+async function getGroup (groupName, opts = {}) {
+
+	const group = (await groups())?.[groupName]?.map( i => i.pid)?.sort();
+
+	if (!group?.length) return;
+
+	const client = await pool.connect();
+
+	try {
+
+		const pairs = combinations(group, 2);
+		const flatValues = pairs.flat();
+		const placeholders = [];
+		for (let i = 0; i < pairs.length; i++) {
+			placeholders.push(`($${i * 2 + 1}, $${i * 2 + 2})`);
+		}
+		const inClause = placeholders.join(',');
+		const selectFields = opts.returnData ? 'data, meta' : 'meta';
+
+		const text = `
+			SELECT baseline_pid, monitor_pid, ${selectFields}
+			FROM comparisons.comparisons
+			WHERE type = 'geometric_difference'
+			AND (baseline_pid, monitor_pid) IN (VALUES ${inClause})
+			ORDER BY baseline_pid, monitor_pid
+		`;
+
+		console.log(text);
+		console.log(flatValues);
+
+		const res = await client.query(text, flatValues);
+		if (!res.rows.length) {
+			console.log("Comparison not found in database");
+			return;
+		}
+
+		if (opts.returnData) {
+			return res.rows.map( row => ({
+				...row,
+				data: DougalBinaryBundle.clone(row.data),
+			}));
+		} else {
+			return res.rows;
+		}
+	} catch (err) {
+		console.error(err);
+	} finally {
+		client.release();
+	}
+}
+
+module.exports = {
+	groups,
+	fetchErrors,
+	compare,
+	computeSample,
+	get,
+	save,
+	getSample,
+	saveSample,
+	saveGroup,
+	getGroup,
+	remove,
+	stats,
+	// comparisonGeometricDifferences,
+	asBundle,
+	fromBundle
+};

lib/www/server/lib/comparisons/index.js

@@ -0,0 +1,4 @@
+
+module.exports = {
+	...require('./geometric-differences')
+}

lib/www/server/lib/comparisons/pca.js

@@ -0,0 +1,83 @@
+const math = require('mathjs');
+
+/**
+ * Compute PCA (eigenvectors and eigenvalues) for deviation data to assess geometric repeatability.
+ * @param {Array<Array<number>>} deviations - Array of [point, line, i_deviation, j_deviation]
+ * @returns {Object} - { eigenvalues, eigenvectors, rms, anisotropy, primaryDirection }
+ */
+function computePCA(deviations) {
+	// Extract i_deviation and j_deviation
+	const deviationMatrix = deviations.map(row => [row[2], row[3]]);
+
+	// Convert to mathjs matrix
+	const D = math.matrix(deviationMatrix);
+
+	// Compute mean for centering (1 x 2 matrix)
+	const mean = math.mean(D, 0);
+
+	// Manually repeat-mean to match D's shape (n x 2)
+	const n = deviationMatrix.length;
+	const meanArr = mean.toArray();
+	const meanRepeated = math.matrix(
+		Array(n).fill().map(() => [meanArr[0], meanArr[1]])
+	);
+
+	// Center the data
+	const centered = math.subtract(D, meanRepeated);
+
+	// Compute covariance matrix: (1/(n-1)) * (D_centered^T * D_centered)
+	const covMatrix = math.multiply(
+		math.multiply(1 / (n - 1), math.transpose(centered)),
+		centered
+	);
+
+	// Perform eigen decomposition
+	const result = math.eigs(covMatrix);
+	let eigenvalues = result.values;
+	const evObjs = result.eigenvectors;
+
+	// Convert eigenvalues to array if it's a matrix
+	eigenvalues = Array.isArray(eigenvalues) ? eigenvalues : eigenvalues.toArray();
+
+	// Create pairs and convert vector to array if necessary
+	const pairs = eigenvalues.map((val, i) => {
+		let vec = evObjs[i].vector;
+		if (vec.toArray) vec = vec.toArray();
+		return { val, vec };
+	});
+
+	// Sort by descending eigenvalues
+	pairs.sort((a, b) => b.val - a.val);
+
+	// Sorted eigenvalues
+	const sortedEigenvalues = pairs.map(p => p.val);
+
+	// Build eigenvector matrix: rows as components, columns as eigenvectors
+	const dimension = pairs[0].vec.length; // e.g., 2
+	const evecRows = [];
+	for (let comp = 0; comp < dimension; comp++) {
+		evecRows.push(pairs.map(p => p.vec[comp]));
+	}
+	const sortedEigenvectors = math.matrix(evecRows);
+
+	// Compute RMS errors along principal axes
+	const rms = sortedEigenvalues.map(val => Math.sqrt(Math.max(val, 0)));
+
+	// Compute anisotropy (ratio of major to minor axis variance)
+	const anisotropy = sortedEigenvalues[0] / (sortedEigenvalues[1] || 1); // Avoid division by zero
+
+	// Primary direction (angle in degrees of major eigenvector)
+	const primaryVector = sortedEigenvectors.subset(math.index([0, 1], 0)).toArray();
+	const primaryDirection = Math.atan2(primaryVector[1], primaryVector[0]) * 180 / Math.PI;
+
+	return {
+		eigenvalues: sortedEigenvalues,
+		eigenvectors: sortedEigenvectors.toArray(),
+		rms: rms, // RMS errors along major/minor axes
+		anisotropy: anisotropy, // Ratio of variances
+		primaryDirection: primaryDirection // Angle of major axis (degrees)
+	};
+}
+
+
+module.exports = { computePCA };

lib/www/server/lib/comparisons/utils.js

@@ -0,0 +1,310 @@
+const d3 = require('d3-array');
+
+// Function to calculate the root mean square (RMS) of position deviations
+// This computes the RMS of the Euclidean distances: sqrt( (1/n) * sum(δi² + δj²) )
+// Assumes deviations are already centered (mean deviation ~0); if normalization by std dev or range is needed, adjust accordingly
+function ijRMS(δi, δj) {
+	if (!δi.length || !δj.length) return 0;
+
+	if (δi.length != δj.length) {
+		console.warn(`δi and δj have different lengths!`);
+	}
+
+	let sumSquares = 0;
+	const n = Math.min(δi.length, δj.length);
+
+	for (let i=0; i < n; i++) {
+		sumSquares += (δi[i] * δi[i]) + (δj[i] * δj[i]);
+	}
+
+	const meanSquare = sumSquares / n;
+	const rms = Math.sqrt(meanSquare);
+
+	return rms;
+}
+
+
+/**
+ * Performs stratified sampling on an array of [line, point, δi, δj] data points.
+ * Groups by line and samples proportionally to preserve shape and spread.
+ *
+ * @param {Array<Array<number>>} data - Input data: [[line, point, δi, δj], ...]
+ * @param {number} sampleSize - Target number of samples (e.g., 2000)
+ * @returns {Array<Array<number>>} Sampled data in same format
+ */
+function old_stratifiedSample(data, sampleSize) {
+	if (!Array.isArray(data) || data.length === 0) return [];
+	if (!Number.isInteger(sampleSize) || sampleSize <= 0) {
+		throw new Error('sampleSize must be a positive integer');
+	}
+
+	// Group data by line (first element)
+	const grouped = d3.group(data, d => d[0]);
+	const totalSize = data.length;
+	const sampled = [];
+
+	// Ensure sampleSize doesn't exceed data size
+	const effectiveSampleSize = Math.min(sampleSize, totalSize);
+
+	// Iterate over each line group
+	for (const [line, group] of grouped) {
+		// Calculate proportional sample size for this group
+		const groupSize = group.length;
+		const groupSampleSize = Math.max(1, Math.round((groupSize / totalSize) * effectiveSampleSize));
+
+		// Shuffle group and take first N elements
+		const shuffled = d3.shuffle([...group]);
+		sampled.push(...shuffled.slice(0, groupSampleSize));
+	}
+
+	// If sampled size is slightly off due to rounding, adjust
+	if (sampled.length > effectiveSampleSize) {
+		return d3.shuffle(sampled).slice(0, effectiveSampleSize);
+	} else if (sampled.length < effectiveSampleSize) {
+		// Pad with random samples from entire dataset if needed
+		const remaining = effectiveSampleSize - sampled.length;
+		const additional = d3.shuffle(data.filter(d => !sampled.includes(d))).slice(0, remaining);
+		sampled.push(...additional);
+	}
+
+	return sampled;
+}
+
+/**
+ * Performs stratified sampling on an array of [line, point, δi, δj] data points.
+ * Stratifies by line and δi quantiles to preserve shape and spread, with outlier control.
+ *
+ * @param {Array<Array<number>>} data - Input data: [[line, point, δi, δj], ...]
+ * @param {number} sampleSize - Target number of samples (e.g., 2000)
+ * @param {number} [binsPerLine=10] - Number of δi quantile bins per line
+ * @returns {Array<Array<number>>} Sampled data in same format
+ */
+function stratifiedSample(data, sampleSize, binsPerLine = 10) {
+	if (!Array.isArray(data) || data.length === 0) return [];
+	if (!Number.isInteger(sampleSize) || sampleSize <= 0) {
+		throw new Error('sampleSize must be a positive integer');
+	}
+	if (!Number.isInteger(binsPerLine) || binsPerLine <= 0) {
+		throw new Error('binsPerLine must be a positive integer');
+	}
+
+	const totalSize = data.length;
+	const effectiveSampleSize = Math.min(sampleSize, totalSize);
+	const sampled = [];
+
+	// Group by line
+	const groupedByLine = d3.group(data, d => d[0]);
+
+	// Compute population stats for validation
+	const populationStats = computeStats(data);
+
+	// Iterate over each line
+	for (const [line, group] of groupedByLine) {
+		const groupSize = group.length;
+		const lineSampleSize = Math.max(1, Math.round((groupSize / totalSize) * effectiveSampleSize));
+
+		// Create quantile-based bins for δi
+		const δiValues = group.map(d => d[2]).sort(d3.ascending);
+		const quantiles = d3.range(0, binsPerLine + 1).map(i => d3.quantile(δiValues, i / binsPerLine));
+		const binnedData = group.map(d => {
+			const δi = d[2];
+			let binIndex = 0;
+			for (let i = 0; i < binsPerLine; i++) {
+				if (δi >= quantiles[i] && δi < quantiles[i + 1]) {
+					binIndex = i;
+					break;
+				}
+			}
+			return { data: d, bin: binIndex };
+		});
+		const groupedByBin = d3.group(binnedData, d => d.bin);
+
+		// Allocate samples across bins, inversely weighted by density to control outliers
+		const binSampleSizes = new Map();
+		let remainingLineSamples = lineSampleSize;
+		const binCounts = Array(binsPerLine).fill(0);
+		for (const [bin, binGroup] of groupedByBin) {
+			binCounts[bin] = binGroup.length;
+		}
+		const maxBinCount = d3.max(binCounts);
+		for (const [bin, binGroup] of groupedByBin) {
+			const binSize = binGroup.length;
+			// Inverse weighting: smaller bins (outliers) get fewer samples
+			const weight = binSize > 0 ? Math.max(0.1, 1 - (binSize / maxBinCount) * 0.5) : 1;
+			const binSampleSize = Math.max(1, Math.round(lineSampleSize * (binSize / groupSize) * weight));
+			binSampleSizes.set(bin, Math.min(binSampleSize, binSize));
+			remainingLineSamples -= binSampleSizes.get(bin);
+		}
+
+		// Distribute remaining samples
+		if (remainingLineSamples > 0) {
+			const nonEmptyBins = Array.from(groupedByBin.keys());
+			for (let i = 0; i < remainingLineSamples && nonEmptyBins.length > 0; i++) {
+				const bin = nonEmptyBins[i % nonEmptyBins.length];
+				binSampleSizes.set(bin, binSampleSizes.get(bin) + 1);
+			}
+		}
+
+		// Sample from each bin
+		for (const [bin, binGroup] of groupedByBin) {
+			const samples = d3.shuffle([...binGroup]).slice(0, binSampleSizes.get(bin)).map(s => s.data);
+			sampled.push(...samples);
+		}
+	}
+
+	// Adjust sample size
+	let finalSample = sampled;
+	if (sampled.length > effectiveSampleSize) {
+		finalSample = d3.shuffle(sampled).slice(0, effectiveSampleSize);
+	} else if (sampled.length < effectiveSampleSize) {
+		const remaining = effectiveSampleSize - sampled.length;
+		const additional = d3.shuffle(data.filter(d => !sampled.includes(d))).slice(0, remaining);
+		finalSample = [...sampled, ...additional];
+	}
+
+	// Validate and adjust if stats are off
+	const sampleStats = computeStats(finalSample);
+	const statTolerance = { μ: 0.1, σ: 0.2 }; // Allowable relative deviation
+	const needsAdjustment =
+		Math.abs(sampleStats.μ[0] - populationStats.μ[0]) / populationStats.μ[0] > statTolerance.μ ||
+		Math.abs(sampleStats.μ[1] - populationStats.μ[1]) / populationStats.μ[1] > statTolerance.μ ||
+		Math.abs(sampleStats.σ[0] - populationStats.σ[0]) / populationStats.σ[0] > statTolerance.σ ||
+		Math.abs(sampleStats.σ[1] - populationStats.σ[1]) / populationStats.σ[1] > statTolerance.σ;
+
+	if (needsAdjustment) {
+		// Add points from underrepresented regions
+		const δiSample = finalSample.map(d => d[2]);
+		const δiPopulation = data.map(d => d[2]);
+		const quantiles = d3.range(0, binsPerLine + 1).map(i => d3.quantile(δiPopulation, i / binsPerLine));
+		const sampleBins = d3.histogram().domain(d3.extent(δiPopulation)).thresholds(quantiles)(δiSample);
+		const populationBins = d3.histogram().domain(d3.extent(δiPopulation)).thresholds(quantiles)(δiPopulation);
+		const underSampledBins = sampleBins
+			.map((b, i) => ({ bin: i, diff: populationBins[i].length / totalSize - b.length / finalSample.length }))
+			.filter(b => b.diff > 0.1); // Significant under-sampling
+
+		if (underSampledBins.length > 0) {
+			const additionalSamples = [];
+			for (const { bin } of underSampledBins) {
+				const binData = data.filter(d => d[2] >= quantiles[bin] && d[2] < quantiles[bin + 1] && !finalSample.includes(d));
+				const needed = Math.round((underSampledBins[0].diff * effectiveSampleSize) / 2);
+				additionalSamples.push(...d3.shuffle(binData).slice(0, needed));
+			}
+			finalSample = d3.shuffle([...finalSample, ...additionalSamples]).slice(0, effectiveSampleSize);
+		}
+	}
+
+	return finalSample;
+}
+
+function decimate (data, decimationCount = 20) {
+	return data.filter( (row, index) => (index % decimationCount) == 0 );
+}
+
+function computeSample (data, opts = {}) {
+	const DEFAULT_SAMPLE_SIZE = 2000;
+	let sample;
+
+	if (opts.decimate === true) {
+		if (opts.sampleSize > 0) {
+			sample = decimate(data.records, Math.floor(data.records.length / opts.sampleSize));
+		} else {
+			sample = decimate(data.records, Math.floor(data.records.length / DEFAULT_SAMPLE_SIZE));
+		}
+	} else if (opts.decimate > 0) {
+		sample = decimate(data.records, opts.decimate);
+	} else if (opts.sampleSize) {
+		sample = stratifiedSample(data.records, opt.sampleSize);
+	} else {
+		sample = stratifiedSample(data.records, DEFAULT_SAMPLE_SIZE);
+	}
+
+	return sample;
+}
+
+// Optional: Utility to compute stats for validation
+function computeStats(data) {
+	const δi = data.map(d => d[2]);
+	const δj = data.map(d => d[3]);
+	const rms = Math.sqrt(d3.mean(data, d => d[2] ** 2 + d[3] ** 2));
+	return {
+		l: data.length,
+		μ: [d3.mean(δi), d3.mean(δj)],
+		σ: [d3.deviation(δi), d3.deviation(δj)],
+		rms
+	};
+}
+
+function centre (data) {
+	const stats = computeStats(data);
+
+	return data.map( row => [row[0], row[1], row[2]-stats.μ[0], row[3]-stats.μ[1]] )
+}
+
+function outliers (data, sd=1.96) {
+	const stats = computeStats(data);
+
+	function fn ([l, p, i, j]) {
+		return (i - stats.μ[0]) > stats.σ[0]*sd ||
+			(j - stats.μ[1]) > stats.σ[1]*sd;
+	}
+
+	return data.filter(fn)
+}
+
+function inliers (data, sd=1.96) {
+	const stats = computeStats(data);
+
+	function fn ([l, p, i, j]) {
+		return (i - stats.μ[0]) <= stats.σ[0]*sd &&
+			(j - stats.μ[1]) <= stats.σ[1]*sd;
+	}
+
+	return data.filter(fn)
+}
+
+function difference (a, b) {
+	const obj = Array.isArray(a) ? [] : {};
+	for (const k in a) {
+		const v0 = a[k];
+		const v1 = b[k]
+		if (v0 instanceof Object && v1 instanceof Object) {
+			obj[k] = difference (v0, v1);
+		} else if (!isNaN(Number(v0)) && !isNaN(Number(v1))) {
+			obj[k] = v1 - v0;
+		}
+	}
+	return obj;
+}
+
+function combinations (a, n) {
+	const results = [];
+
+	function combine(current, start) {
+		if (current.length === n) {
+			results.push([...current]);
+			return;
+		}
+		for (let i = start; i < a.length; i++) {
+			current.push(a[i]);
+			combine(current, i + 1);
+			current.pop();
+		}
+	}
+
+	combine([], 0);
+	return results;
+}
+
+module.exports = {
+	combinations,
+	centre,
+	ijRMS,
+	computeStats,
+	computeSample,
+	stratifiedSample,
+	old_stratifiedSample,
+	decimate,
+	difference,
+	outliers,
+	inliers
+}

lib/www/server/lib/db/channels.js

@@ -10,5 +10,6 @@ module.exports = [
 	"planned_lines",
 	"raw_lines", "raw_shots",
 	"final_lines", "final_shots", "info",
-	"queue_items"
+	"queue_items",
+	"comparisons",
 ];

lib/www/server/lib/db/sequence/get.js

@@ -15,11 +15,50 @@ async function getSummary (projectId, sequence, opts = {}) {
 	return res.rows[0];
 }
 
+async function getPoints (projectId, sequence, opts = {}) {
+
+	const offset = Math.abs(opts.offset) || Math.abs((opts.page-1)*opts.itemsPerPage) || 0;
+	const limit = Math.abs(opts.limit) || Math.abs(Number(opts.itemsPerPage)) || null;
+
+	const client = await setSurvey(projectId);
+
+	const restriction = sequence
+		? "sequence = $3"
+		: "TRUE OR $3";
+
+	const text = `
+		SELECT line, point, sequence, st_x(ST_Transform(geometry, 4326)) longitude, st_y(ST_Transform(geometry, 4326)) latitude
+		FROM final_shots
+		WHERE ${restriction}
+		ORDER BY sequence, point
+		OFFSET $1
+		LIMIT $2;
+	`;
+
+	try {
+		const res = await client.query({text, values: [offset, limit, sequence], rowMode: 'array'});
+		return res.rows;
+	} catch (err) {
+		console.error(err);
+		// throw { status: 500, message: "Internal error" };
+	} finally {
+		client.release();
+	}
+
+}
 
 async function get (projectId, sequence, opts = {}) {
 	if (opts.summary) {
 		return await getSummary(projectId, sequence, opts);
 	}
+	if (opts.type == 4) {
+		// The user is request that we send just the bare details:
+		// sequence, sailline, line, longitude, latitude.
+		//
+		// This will probably be a binary data request (though doesn't
+		// need to).
+		return await getPoints(projectId, sequence, opts);
+	}
 
 	const client = await setSurvey(projectId);
 

lib/www/server/lib/utils/setops.js

@@ -0,0 +1,52 @@
+
+function unique(arr) {
+	const set = new Set(arr.map(JSON.stringify));
+	return Array.from(set).map(JSON.parse);
+}
+
+function duplicates(arr) {
+	const seen = new Set();
+	const dups = new Set();
+	for (const item of arr.map(JSON.stringify)) {
+		if (seen.has(item)) {
+			dups.add(item);
+		} else {
+			seen.add(item);
+		}
+	}
+	return Array.from(dups).map(JSON.parse);
+}
+
+function union(arr1, arr2) {
+	const set = new Set([...arr1, ...arr2].map(JSON.stringify));
+	return Array.from(set).map(JSON.parse);
+}
+
+
+function intersection(arr1, arr2) {
+	const set2 = new Set(arr2.map(JSON.stringify));
+	return arr1.filter(item => set2.has(JSON.stringify(item)));
+}
+
+function difference(arr1, arr2) {
+	const set2 = new Set(arr2.map(JSON.stringify));
+	return arr1.filter(item => !set2.has(JSON.stringify(item)));
+}
+
+function symmetricDifference(arr1, arr2) {
+	const set1 = new Set(arr1.map(JSON.stringify));
+	const set2 = new Set(arr2.map(JSON.stringify));
+	return [
+		...arr1.filter(item => !set2.has(JSON.stringify(item))),
+		...arr2.filter(item => !set1.has(JSON.stringify(item)))
+	];
+}
+
+module.exports = {
+	unique,
+	duplicates,
+	union,
+	intersection,
+	difference,
+	symmetricDifference
+}

lib/www/server/package.json

@@ -16,7 +16,7 @@
       "api": "0.4.0"
     },
     "wanted": {
-      "db_schema": "^0.6.0"
+      "db_schema": "^0.6.5"
     }
   },
   "engines": {
@@ -44,6 +44,7 @@
     "jsonwebtoken": "^9.0.2",
     "leaflet-headless": "git+https://git@gitlab.com/aaltronav/contrib/leaflet-headless.git#devel",
     "marked": "^4.0.12",
+    "mathjs": "^14.6.0",
     "node-fetch": "^2.6.1",
     "nunjucks": "^3.2.3",
     "path-to-regexp": "^6.2.1",

package-lock.json

@@ -1678,17 +1678,6 @@
       "dev": true,
       "license": "MIT"
     },
-    "lib/www/client/source/node_modules/@babel/runtime": {
-      "version": "7.23.2",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "regenerator-runtime": "^0.14.0"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      }
-    },
     "lib/www/client/source/node_modules/@babel/template": {
       "version": "7.27.2",
       "dev": true,
@@ -7524,11 +7513,6 @@
         "node": ">=4"
       }
     },
-    "lib/www/client/source/node_modules/regenerator-runtime": {
-      "version": "0.14.0",
-      "dev": true,
-      "license": "MIT"
-    },
     "lib/www/client/source/node_modules/regenerator-transform": {
       "version": "0.15.2",
       "dev": true,
@@ -9374,6 +9358,7 @@
         "jsonwebtoken": "^9.0.2",
         "leaflet-headless": "git+https://git@gitlab.com/aaltronav/contrib/leaflet-headless.git#devel",
         "marked": "^4.0.12",
+        "mathjs": "^14.6.0",
         "node-fetch": "^2.6.1",
         "nunjucks": "^3.2.3",
         "path-to-regexp": "^6.2.1",
@@ -10527,17 +10512,6 @@
         "node": ">=6.0.0"
       }
     },
-    "lib/www/server/node_modules/redoc-cli/node_modules/@babel/runtime": {
-      "version": "7.16.7",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "regenerator-runtime": "^0.13.4"
-      },
-      "engines": {
-        "node": ">=6.9.0"
-      }
-    },
     "lib/www/server/node_modules/redoc-cli/node_modules/@babel/template": {
       "version": "7.12.13",
       "dev": true,
@@ -12532,11 +12506,6 @@
         "url": "https://github.com/Mermade/oas-kit?sponsor=1"
       }
     },
-    "lib/www/server/node_modules/redoc-cli/node_modules/regenerator-runtime": {
-      "version": "0.13.9",
-      "dev": true,
-      "license": "MIT"
-    },
     "lib/www/server/node_modules/redoc-cli/node_modules/require-directory": {
       "version": "2.1.1",
       "dev": true,
@@ -13297,6 +13266,15 @@
         "node": ">=0.4"
       }
     },
+    "node_modules/@babel/runtime": {
+      "version": "7.28.3",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.3.tgz",
+      "integrity": "sha512-9uIQ10o0WGdpP6GDhXcdOJPJuDgFtIDtN/9+ArJQ2NAfAmiuhTQdzkaTGR33v43GYS2UrSA0eX2pPPHoFVvpxA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
     "node_modules/@deck.gl/aggregation-layers": {
       "version": "9.1.13",
       "resolved": "https://registry.npmjs.org/@deck.gl/aggregation-layers/-/aggregation-layers-9.1.13.tgz",
@@ -14389,6 +14367,19 @@
         "node": ">= 10"
       }
     },
+    "node_modules/complex.js": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.2.tgz",
+      "integrity": "sha512-qtx7HRhPGSCBtGiST4/WGHuW+zeaND/6Ld+db6PbrulIB1i2Ev/2UPiqcmpQNPSyfBKraC0EOvOKCB5dGZKt3g==",
+      "license": "MIT",
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/rawify"
+      }
+    },
     "node_modules/compressible": {
       "version": "2.0.18",
       "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
@@ -15089,6 +15080,12 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/escape-latex": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz",
+      "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==",
+      "license": "MIT"
+    },
     "node_modules/escodegen": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz",
@@ -15205,6 +15202,19 @@
         "node": ">= 6"
       }
     },
+    "node_modules/fraction.js": {
+      "version": "5.3.1",
+      "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.1.tgz",
+      "integrity": "sha512-PhqCuhSKIGbbkJ+cojHv47eEWClU71FIOhiUsYdZYTwhIzCeIN8rXeEjserTvPat5JLJChumn8chHz64WkZgTw==",
+      "license": "MIT",
+      "engines": {
+        "node": "*"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/rawify"
+      }
+    },
     "node_modules/fs-minipass": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
@@ -15635,6 +15645,12 @@
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
       "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g=="
     },
+    "node_modules/javascript-natural-sort": {
+      "version": "0.7.1",
+      "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz",
+      "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==",
+      "license": "MIT"
+    },
     "node_modules/jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
@@ -15848,6 +15864,29 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/mathjs": {
+      "version": "14.6.0",
+      "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.6.0.tgz",
+      "integrity": "sha512-5vI2BLB5GKQmiSK9BH6hVkZ+GgqpdnOgEfmHl7mqVmdQObLynr63KueyYYLCQMzj66q69mV2XZZGQqqxeftQbA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@babel/runtime": "^7.26.10",
+        "complex.js": "^2.2.5",
+        "decimal.js": "^10.4.3",
+        "escape-latex": "^1.2.0",
+        "fraction.js": "^5.2.1",
+        "javascript-natural-sort": "^0.7.1",
+        "seedrandom": "^3.0.5",
+        "tiny-emitter": "^2.1.0",
+        "typed-function": "^4.2.1"
+      },
+      "bin": {
+        "mathjs": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/md5": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
@@ -16343,6 +16382,12 @@
         "node": ">=10"
       }
     },
+    "node_modules/seedrandom": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+      "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==",
+      "license": "MIT"
+    },
     "node_modules/semver": {
       "version": "7.7.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
@@ -16606,6 +16651,12 @@
         "texture-compressor": "bin/texture-compressor.js"
       }
     },
+    "node_modules/tiny-emitter": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz",
+      "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==",
+      "license": "MIT"
+    },
     "node_modules/toidentifier": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -16667,6 +16718,15 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/typed-function": {
+      "version": "4.2.1",
+      "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.2.1.tgz",
+      "integrity": "sha512-EGjWssW7Tsk4DGfE+5yluuljS1OGYWiI1J6e8puZz9nTMM51Oug8CD5Zo4gWMsOhq5BI+1bF+rWTm4Vbj3ivRA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 18"
+      }
+    },
     "node_modules/undici-types": {
       "version": "7.8.0",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.8.0.tgz",