Add custom binary format packing / unpacking.

This series of custom binary messages are an alternative to JSON /
GeoJSON when huge amounts of data needs to be transferred to and
processed by the client, such as a GPU-based map view showing all
the points for a prospect, or QC graphs, etc.

lib/www/server/lib/binary/constants.js

@@ -0,0 +1,51 @@
+
+
+// Message types
+
+const MSGTYPE = {
+  BUNDLE:               0,
+  PREPLOT:              1,
+  RAW:                  2,
+  FINAL:                3,
+  PREPLOT_RAWERROR_OPT: 4,
+  RAW_OPT:              5,
+  FINAL_OPT:            6
+};
+
+// Packet format types
+
+const PKTTYPE = {
+  A: Symbol('PKTTYPE_A'),
+  B: Symbol('PKTTYPE_B'),
+  C: Symbol('PKTTYPE_C'),
+  D: Symbol('PKTTYPE_D'),
+  E: Symbol('PKTTYPE_E')
+};
+
+// Header offsets
+
+const HEADER_OFFSET = {
+  MSGTYPE:   0,
+  COUNT:     1,
+  SEQUENCE:  5,
+  POINT:     7,
+  DPOINT:    9,
+  DTSTAMP:  11
+};
+
+// Packet offsets (for some packet types)
+
+const PACKET_OFFSET = {
+  LONGITUDE:  0,
+  LATITUDE:   4,
+  DI:         8,
+  DJ:        12,
+  DTS:       16
+};
+
+module.exports = {
+	MSGTYPE,
+	PKTTYPE,
+	HEADER_OFFSET,
+	PACKET_OFFSET
+};

lib/www/server/lib/binary/decode.js

@@ -0,0 +1,109 @@
+const { MSGTYPE, PKTTYPE, HEADER_OFFSET, PACKET_OFFSET } = require('./constants');
+const { headerSize, packetSize, packetFormatType, isLittleEndian } = require('./info');
+
+const platformEndianness = isLittleEndian();
+
+function unbundle (data, index=0, endianness = platformEndianness) {
+
+	const view = new DataView(data);
+	const msgType = view.getUint8(0);
+	const HEADER_SIZE = headerSize(msgType);
+	const PACKET_TYPE = packetFormatType(msgType);
+
+// 	console.log("THE DATA", data);
+	if (PACKET_TYPE == PKTTYPE.A) {
+// 		console.log("Seen packet A");
+
+		const LENGTH_SIZE = 4; // 1 * Uint32
+
+		let offset = HEADER_SIZE;
+
+		let count = 0;
+		while (offset < data.byteLength) {
+			const length = view.getUint32(offset, endianness);
+			offset += LENGTH_SIZE;
+// 			console.log("unbundle", msgType, PACKET_TYPE, data.byteLength, "offset", offset, "length", length, "index", index, "count", count);
+
+			if (index == count) {
+				const buffer = data.slice(offset, offset+length);
+// 				console.log("buffer", buffer.byteLength, "from", offset, "to", offset+length);
+// 				console.log(buffer);
+				return buffer;
+			}
+
+			offset += length;
+			count++;
+// 			console.log("offset now", offset, data.byteLength, offset < data.byteLength);
+
+		}
+	} else if (index==0) {
+		if (PACKET_TYPE == PKTTYPE.D) {
+	// 		console.log("Seen packet D");
+			const offset = 0;
+
+			const type = msgType;
+			const count = view.getUint32(offset+HEADER_OFFSET.COUNT, endianness);
+			const seq = view.getUint16(offset+HEADER_OFFSET.SEQUENCE, endianness);
+			const sp0 = view.getUint16(offset+HEADER_OFFSET.POINT, endianness);
+			const Δsp = view.getInt16(offset+HEADER_OFFSET.DPOINT, endianness);
+
+	// 		console.log("posArray", data.byteLength, count, offset, offset+HEADER_SIZE, count*8, offset+HEADER_SIZE+count*8);
+			const positions = new Float32Array(data, offset+HEADER_SIZE, count*2);
+			const radii = new Float32Array(data, offset+HEADER_SIZE + count*8, count);
+
+			return {
+				type, count, seq, sp0, Δsp, positions, radii
+			};
+		} else if (PACKET_TYPE == PKTTYPE.E) {
+	// 		console.log("Seen packet D");
+			const offset = 0;
+
+			const type = msgType;
+			const count = view.getUint32(offset+HEADER_OFFSET.COUNT, endianness);
+			const seq = view.getUint16(offset+HEADER_OFFSET.SEQUENCE, endianness);
+			const sp0 = view.getUint16(offset+HEADER_OFFSET.POINT, endianness);
+			const Δsp = view.getInt16(offset+HEADER_OFFSET.DPOINT, endianness);
+			const ts0 = view.getBigInt64(offset+HEADER_OFFSET.DTSTAMP, endianness);
+
+	// 		console.log("posArray", data.byteLength, count, offset, offset+HEADER_SIZE, count*8, offset+HEADER_SIZE+count*8);
+			const positions = new Float32Array(data, offset+HEADER_SIZE, count*2);
+			const Δij = new Float32Array(data, offset+HEADER_SIZE + count*8, count*2);
+			const Δts = new Int32Array(data, offset+HEADER_SIZE + count*16, count);
+
+			return {
+				type, count, seq, sp0, Δsp, ts0, positions, Δij, Δts
+			};
+		}
+	}
+// 	console.log("Should return undefined");
+}
+
+function unpack (data, index=0, endianness = platformEndianness) {
+	const bundles = [];
+	while (true) {
+// 		console.log("iteration starts", index, data);
+		const bundle = unbundle(data, index++, endianness);
+// 		console.log("bundle", index, bundle);
+		if (bundle) {
+			if (bundle instanceof ArrayBuffer) {
+// 				console.log("bundle is ArrayBuffer");
+				const decoded = unpack(bundle, 0, endianness);
+// 				console.log("decoded", decoded);
+				bundles.push(...decoded.flat());
+			} else {
+// 				console.log("bundle is something else");
+				bundles.push(bundle);
+				if (bundles.length > 100) { break; }
+			}
+		} else {
+			break;
+		}
+// 		console.log("BUNDLES", bundles);
+	}
+	return bundles;
+}
+
+module.exports = {
+	unbundle,
+	unpack
+};

lib/www/server/lib/binary/encode.js

@@ -0,0 +1,266 @@
+const { MSGTYPE, PKTTYPE, HEADER_OFFSET, PACKET_OFFSET } = require('./constants');
+const { headerSize, packetSize, packetFormatType, isLittleEndian } = require('./info');
+
+function getGeometries (json, geometry, error) {
+	return json.map( feature => {
+
+		const obj = {
+			sequence: feature.sequence,
+			point: feature.point,
+			tstamp: (new Date(feature.tstamp)).valueOf()
+		};
+
+		if (feature[geometry]) {
+			obj.longitude = feature[geometry].coordinates[0];
+			obj.latitude = feature[geometry].coordinates[1];
+		} else {
+			return;
+		}
+
+		if (error) {
+			if (Array.isArray(error)) {
+				for (key of error) {
+					if (feature[key]) {
+						obj.Δj = feature[key].coordinates[0];
+						obj.Δi = feature[key].coordinates[1];
+						break;
+					}
+				}
+			} else {
+				if (feature[error]) {
+					obj.Δj = feature[error].coordinates[0];
+					obj.Δi = feature[error].coordinates[1];
+				}
+			}
+		}
+
+		return obj;
+	}).filter(i => !!i);
+}
+
+function getRun (json) {
+	if (json.length > 1) {
+		const first = json[0];
+		const second = json[1];
+		const tstamp = first.tstamp.valueOf();
+		const sequence = first.sequence;
+		const point = first.point;
+		const Δpoint = second.point - first.point;
+		let idx = 2;
+		while (idx < json.length) {
+			const a = json[idx-1].point;
+			const b = json[idx].point;
+			if ((b-a) == Δpoint) {
+				idx++;
+			} else {
+				break;
+			}
+		}
+		return {
+			sequence, point, Δpoint, tstamp, count: idx
+		};
+	} else if (json.length == 1) {
+		const first = json[0];
+		const tstamp = first.tstamp.valueOf();
+		const sequence = first.sequence;
+		const point = first.point;
+
+		return {
+			sequence, point, Δpoint: 0, tstamp, count: 1
+		}
+	}
+	// return undefined;
+};
+
+function getRuns (json, acc=[]) {
+	const run = getRun(json);
+	if (run) {
+		acc.push(run);
+		if (run.count < json.length) {
+			getRuns(json.slice(run.count), acc);
+		}
+	}
+	return acc;
+}
+
+
+function compactRun (json, run, msgType=MSGTYPE.PREPLOT, start=0, endianness) {
+
+	const HEADER_SIZE = headerSize(msgType);
+	const PACKET_SIZE = packetSize(msgType);
+	const PACKET_TYPE = packetFormatType(msgType);
+
+	// Header offsets
+	const MSGTYPE_OFFSET=0;
+	const COUNT_OFFSET=1;
+	const SEQUENCE_OFFSET=5;
+	const POINT_OFFSET=7;
+	const DPOINT_OFFSET=9;
+	const DTSTAMP_OFFSET=11;
+
+	// Packet offsets (for some packet types)
+	const LONGITUDE_OFFSET=0;
+	const LATITUDE_OFFSET=4;
+	const DI_OFFSET=8;
+	const DJ_OFFSET=12;
+	const DTS_OFFSET=16;
+
+// 	console.log("Allocating", HEADER_SIZE + PACKET_SIZE*run.count, "bytes");
+	const buffer = new ArrayBuffer(HEADER_SIZE + PACKET_SIZE*run.count);
+	const view = new DataView(buffer);
+
+	// Set the common header
+	view.setUint8(0, msgType);
+	view.setUint32(1, run.count, endianness);
+	view.setUint16(5, run.sequence, endianness);
+	view.setUint16(7, run.point, endianness);
+	view.setInt16(9, run.Δpoint, endianness);
+
+	if (PACKET_TYPE == PKTTYPE.B) {
+
+		for (let idx=start; idx<(start+run.count); idx++) {
+			const offset = HEADER_SIZE + PACKET_SIZE * (idx-start);
+
+			const a = json[idx];
+			const Δts = a.tstamp - run.tstamp;
+
+			view.setFloat32(offset+PACKET_OFFSET.LONGITUDE, a.longitude, endianness);
+			view.setFloat32(offset+PACKET_OFFSET.LATITUDE, a.latitude, endianness);
+
+		}
+
+	} else if (PACKET_TYPE == PKTTYPE.C) {
+
+		view.setBigInt64(HEADER_OFFSET.DTSTAMP, BigInt(run.tstamp), endianness)
+
+		for (let idx=start; idx<(start+run.count); idx++) {
+			const offset = HEADER_SIZE + PACKET_SIZE * (idx-start);
+
+			const a = json[idx];
+			const Δts = a.tstamp - run.tstamp;
+
+			view.setFloat32(offset+PACKET_OFFSET.LONGITUDE, a.longitude, endianness);
+			view.setFloat32(offset+PACKET_OFFSET.LATITUDE, a.latitude, endianness);
+			view.setFloat32(offset+PACKET_OFFSET.DI, a.Δi, endianness);
+			view.setFloat32(offset+PACKET_OFFSET.DJ, a.Δj, endianness);
+			view.setInt32(offset+PACKET_OFFSET.DTS, Δts, endianness);
+
+		}
+	} else if (PACKET_TYPE == PKTTYPE.D) {
+
+		for (let idx=start; idx<(start+run.count); idx++) {
+			const count = idx-start;
+			// <lon>
+			const offset0 = HEADER_SIZE + count * 8; // 2 * Float32
+			// <lat>
+			const offset1 = offset0 + 4;
+			// <radius>
+			const offset2 = HEADER_SIZE + run.count * 8 + count * 4; // 1 * Float32
+
+			const a = json[idx];
+			const longitude = a
+			const radius = Math.sqrt(Math.pow(a.Δi, 2) + Math.pow(a.Δj, 2));
+
+			view.setFloat32(offset0, a.longitude, endianness);
+			view.setFloat32(offset1, a.latitude, endianness);
+			view.setFloat32(offset2, radius, endianness);
+
+// 			console.log("offsets", count, offset0, offset1, offset2);
+// 			console.log("values", a.longitude, a.latitude, radius);
+// 			console.log("a", idx, a);
+		}
+
+	} else if (PACKET_TYPE == PKTTYPE.E) {
+
+		view.setBigInt64(HEADER_OFFSET.DTSTAMP, BigInt(run.tstamp), endianness)
+
+		for (let idx=start; idx<(start+run.count); idx++) {
+			const count = idx-start;
+			// <lon>
+			const offset0 = HEADER_SIZE + count * 8; // 2 * Float32
+			// <lat>
+			const offset1 = offset0 + 4;
+			// <Δi>
+			const offset2 = HEADER_SIZE + run.count * 8 + count * 8;
+			// <Δj>
+			const offset3 = offset2 + 4;
+			// <Δts>
+			const offset4 = HEADER_SIZE + run.count * 16 + count * 4;
+
+			const a = json[idx];
+			const Δts = a.tstamp - run.tstamp;
+
+			view.setFloat32(offset0, a.longitude, endianness);
+			view.setFloat32(offset1, a.latitude, endianness);
+			view.setFloat32(offset2, a.Δi, endianness);
+			view.setFloat32(offset3, a.Δj, endianness);
+			view.setInt32(offset4, Δts, endianness);
+
+// 			console.log(run.count, count, offset0, offset1, offset2, offset3, offset4, view.getFloat32(offset2, endianness), view.getFloat32(offset3, endianness));
+
+		}
+
+	}
+
+	return buffer;
+
+}
+
+function compactRuns (json, runs, msgType = MSGTYPE.PREPLOT, endianness) {
+	let offset = 0;
+	return runs.map(run => {
+		const start = offset;
+		offset += run.count;
+		return compactRun(json, run, msgType, start, endianness);
+	});
+}
+
+function bundle (json, runs, msgType = MSGTYPE.PREPLOT, endianness) {
+	const HEADER_SIZE = headerSize(MSGTYPE.BUNDLE);
+	const LENGTH_SIZE = 4; // 1 * Uint32
+
+	const buffers = compactRuns(json, runs, msgType, endianness);
+	const length =
+		HEADER_SIZE +
+		buffers.length * LENGTH_SIZE +
+		buffers.reduce( (acc, cur) => acc + cur.byteLength, 0 );
+// 	console.log("bundle length", HEADER_SIZE, buffers.length * LENGTH_SIZE, buffers.reduce( (acc, cur) => acc + cur.byteLength, 0 ));
+// 	console.log("bundle length = ", HEADER_SIZE + buffers.length * LENGTH_SIZE + buffers.reduce( (acc, cur) => acc + cur.byteLength, 0 ));
+
+	const buffer = new ArrayBuffer(length);
+	const view = new DataView(buffer);
+
+	view.setUint8(0, MSGTYPE.BUNDLE);
+
+	let offset = HEADER_SIZE;
+	for (const chunk of buffers) {
+		view.setUint32(offset, chunk.byteLength, endianness);
+		offset += LENGTH_SIZE;
+
+		console.log("target", buffer.byteLength, offset, chunk.byteLength, offset+chunk.byteLength);
+		const target = new Uint8Array(buffer, offset, chunk.byteLength);
+// 		console.log("source", chunk);
+		target.set(new Uint8Array(chunk));
+// 		console.log("target = ", target);
+// 		console.log("buffer = ", buffer);
+		offset += chunk.byteLength;
+	}
+
+	return buffer;
+}
+
+function encode (json, geometry, error, msgType, endianness) {
+	const geom = getGeometries(json, geometry, error);
+	const data = bundle(geom, getRuns(geom), msgType, endianness);
+	return data;
+};
+
+module.exports = {
+	getGeometries,
+	getRun,
+	getRuns,
+	compactRun,
+	compactRuns,
+	bundle,
+	encode
+};

lib/www/server/lib/binary/index.js

@@ -0,0 +1,46 @@
+
+/* Format:
+ *
+ * Bundle (MSGTYPE.BUNDLE):
+ *
+ * <msg-type: Uint8> <padding: 3*Uint8>
+ * <length: Uint32> <data: Uint8Array>
+ *
+ * Preplots (MSGTYPE.PREPLOT):
+ *
+ * <msg-type: Uint8> <count: Uint32> <seq: Uint16> <sp0: Uint16> <Δsp: Int16>
+ * <lon> <lat>
+ *
+ * Raw (MSGTYPE.RAW) / final (MSGTYPE.FINAL):
+ *
+ * <msg-type: Uint8> <count: Uint32> <seq: Uint16> <sp0: Uint16> <Δsp: Int16> <ts0: BigInt64>
+ * <lon> <lat> <Δi> <Δj> <Δts>
+ *
+ * Preplot + Raw Error, optimised (MSGTYPE.PREPLOT_RAWERROR_OPT):
+ *
+ * <msg-type: Uint8> <count: Uint32> <seq: Uint16> <sp0: Uint16> <Δsp: Int16> <padding: Uint8>
+ *   <lon: Float32> <lat: Float32> …
+ *   <radius: Float32> …
+ *
+ * Raw points + error, optimised (MSGTYPE.RAW_OPT):
+ *
+ * <msg-type> <count> <seq> <sp0> <Δsp> <ts0>
+ *   <lon> <lat> …
+ *   <Δi> <Δj> …
+ *   <Δts> …
+ *
+ * Final points + error, optimised (MSGTYPE.FINAL_OPT):
+ *
+ * <msg-type> <count> <seq> <sp0> <Δsp> <ts0>
+ *   <lon> <lat> …
+ *   <Δi> <Δj> …
+ *   <Δts> …
+ *
+ */
+
+module.exports = {
+	...require('./constants'),
+	...require('./info'),
+	...require('./encode'),
+	...require('./decode')
+};

lib/www/server/lib/binary/info.js

@@ -0,0 +1,82 @@
+const { MSGTYPE, PKTTYPE, HEADER_OFFSET, PACKET_OFFSET } = require('./constants');
+
+function headerSize (msgType) {
+	switch (msgType) {
+		case MSGTYPE.BUNDLE:
+			return 4;
+		case MSGTYPE.PREPLOT:
+			// 1 * Uint8 + 1 * Uint32 + 1 * Uint16 + 1 * Uint16 + 1 * Int16
+			// 1 * 1     + 1 * 4      + 1 * 2      + 1 * 2      + 1 * 2      = 11
+			return 11;
+		case MSGTYPE.PREPLOT_RAWERROR_OPT:
+			return 12;
+		case MSGTYPE.RAW:
+		case MSGTYPE.FINAL:
+		case MSGTYPE.RAW_OPT:
+		case MSGTYPE.FINAL_OPT:
+			// 1 * Uint8 + 1 * Uint32 + 1 * Uint16 + 1 * Uint16 + 1 * Int16 + 1 * BigInt64
+			// 1 * 1     + 1 * 4      + 1 * 2      + 1 * 2      + 1 * 2     + 1 * 8         = 19
+			return 20;
+		default:
+			return; // undefined
+	}
+}
+
+function packetSize (msgType) {
+	switch (msgType) {
+		case MSGTYPE.BUNDLE:
+			return; // Doesn't apply to this msgType
+		case MSGTYPE.PREPLOT:
+			// 2 * Float32
+			return 8;
+		case MSGTYPE.RAW:
+		case MSGTYPE.FINAL:
+			// 2 * Float32 + 2 * Float32 + 1 * Int32
+			return 20;
+		case MSGTYPE.PREPLOT_RAWERROR_OPT:
+			// 2 * Float32 + 1 * Float32
+			return 12;
+		case MSGTYPE.RAW_OPT:
+		case MSGTYPE.FINAL_OPT:
+			// 2 * Float32 + 2 * Float32 + 1 * Int32
+			return 20;
+		default:
+			return; // undefined
+	}
+}
+
+function packetFormatType (msgType) {
+	switch (msgType) {
+		case MSGTYPE.BUNDLE:
+			return PKTTYPE.A;
+		case MSGTYPE.PREPLOT:
+			return PKTTYPE.B;
+		case MSGTYPE.RAW:
+		case MSGTYPE.FINAL:
+			return PKTTYPE.C;
+		case MSGTYPE.PREPLOT_RAWERROR_OPT:
+			return PKTTYPE.D;
+		case MSGTYPE.RAW_OPT:
+		case MSGTYPE.FINAL_OPT:
+			return PKTTYPE.E;
+		default:
+			return; // undefined
+	}
+}
+
+function isLittleEndian () {
+	const buffer   = new ArrayBuffer(2);
+	const asUint8  = new Uint8Array(buffer);
+	const asUint16 = new Uint16Array(buffer);
+
+	asUint8[1] = 0xFF;
+
+	return asUint16[0] == 0xFF00;
+}
+
+module.exports = {
+	headerSize,
+	packetSize,
+	packetFormatType,
+	isLittleEndian
+};