Add @dougal/binary module.

It encodes / decodes sequence / preplot data using an efficient
binary format for sending large amounts of data across the wire
and for (relatively) memory efficient client-side use.

lib/modules/@dougal/binary/classes.js

@@ -0,0 +1,719 @@
+const codeToType = {
+	0: Int8Array,
+	1: Uint8Array,
+	2: Int16Array,
+	3: Uint16Array,
+	4: Int32Array,
+	5: Uint32Array,
+	7: Float32Array,
+	8: Float64Array,
+	9: BigInt64Array,
+	10: BigUint64Array
+};
+
+const typeToBytes = {
+	Int8Array: 1,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 2,
+	Int32Array: 4,
+	Uint32Array: 4,
+	Float32Array: 4,
+	Float64Array: 8,
+	BigInt64Array: 8,
+	BigUint64Array: 8
+};
+
+function readTypedValue(view, offset, type) {
+	switch (type) {
+		case Int8Array: return view.getInt8(offset);
+		case Uint8Array: return view.getUint8(offset);
+		case Int16Array: return view.getInt16(offset, true);
+		case Uint16Array: return view.getUint16(offset, true);
+		case Int32Array: return view.getInt32(offset, true);
+		case Uint32Array: return view.getUint32(offset, true);
+		case Float32Array: return view.getFloat32(offset, true);
+		case Float64Array: return view.getFloat64(offset, true);
+		case BigInt64Array: return view.getBigInt64(offset, true);
+		case BigUint64Array: return view.getBigUint64(offset, true);
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+class DougalBinaryBundle extends ArrayBuffer {
+
+	static HEADER_LENGTH = 4; // Length of a bundle header
+
+	/** Clone an existing ByteArray into a DougalBinaryBundle
+	 */
+	static clone (buffer) {
+		const clone = new DougalBinaryBundle(buffer.byteLength);
+		const uint8Array = new Uint8Array(buffer);
+		const uint8ArrayClone = new Uint8Array(clone);
+		uint8ArrayClone.set(uint8Array);
+		return clone;
+	}
+
+	constructor (length, options) {
+		super (length, options);
+	}
+
+	/** Get the count of bundles in this ByteArray.
+	 *
+	 * Stops at the first non-bundle looking offset
+	 */
+	get bundleCount () {
+		let count = 0;
+		let currentBundleOffset = 0;
+		const view = new DataView(this);
+
+		while (currentBundleOffset < this.byteLength) {
+
+			const currentBundleHeader = view.getUint32(currentBundleOffset, true);
+			if ((currentBundleHeader & 0xff) !== 0x1c) {
+				// This is not a bundle
+				return count;
+			}
+			let currentBundleLength =	currentBundleHeader >>> 8;
+
+			currentBundleOffset += currentBundleLength + DougalBinaryBundle.HEADER_LENGTH;
+			count++;
+		}
+
+		return count;
+	}
+
+
+	/** Get the number of chunks in the bundles of this ByteArray
+	 */
+	get chunkCount () {
+		let count = 0;
+		let bundleOffset = 0;
+		const view = new DataView(this);
+
+		while (bundleOffset < this.byteLength) {
+			const header = view.getUint32(bundleOffset, true);
+			if ((header & 0xFF) !== 0x1C) break;
+			const length = header >>> 8;
+			if (bundleOffset + 4 + length > this.byteLength) break;
+
+			let chunkOffset = bundleOffset + 4; // relative to buffer start
+
+			while (chunkOffset < bundleOffset + 4 + length) {
+				const chunkType = view.getUint8(chunkOffset);
+				if (chunkType !== 0x11 && chunkType !== 0x12) break;
+
+				const cCount = view.getUint16(chunkOffset + 2, true);
+				const ΔelemC = view.getUint8(chunkOffset + 10);
+				const elemC = view.getUint8(chunkOffset + 11);
+
+				let localOffset = 12; // header size
+
+				localOffset += ΔelemC + elemC; // preface
+
+				// initial values
+				for (let k = 0; k < ΔelemC; k++) {
+					const typeByte = view.getUint8(chunkOffset + 12 + k);
+					const baseCode = typeByte & 0xF;
+					const baseType = codeToType[baseCode];
+					if (!baseType) throw new Error('Invalid base type code');
+					localOffset += typeToBytes[baseType.name];
+				}
+
+				// pad after initial
+				while (localOffset % 4 !== 0) localOffset++;
+
+				if (chunkType === 0x11) { // Sequential
+					// record data: Δelems incrs
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						localOffset += cCount * typeToBytes[incrType.name];
+					}
+
+					// elems
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						localOffset += cCount * typeToBytes[type.name];
+					}
+				} else { // Interleaved
+					// Compute exact stride for interleaved record data
+					let ΔelemStride = 0;
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						ΔelemStride += typeToBytes[incrType.name];
+					}
+					let elemStride = 0;
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						elemStride += typeToBytes[type.name];
+					}
+					const recordStride = ΔelemStride + elemStride;
+					localOffset += cCount * recordStride;
+				}
+
+				// pad after record
+				while (localOffset % 4 !== 0) localOffset++;
+
+				chunkOffset += localOffset;
+				count++;
+			}
+
+			bundleOffset += 4 + length;
+		}
+
+		return count;
+	}
+
+	/** Return an array of DougalBinaryChunkSequential or DougalBinaryChunkInterleaved instances
+	 */
+	chunks () {
+		const chunks = [];
+		let bundleOffset = 0;
+		const view = new DataView(this);
+
+		while (bundleOffset < this.byteLength) {
+			const header = view.getUint32(bundleOffset, true);
+			if ((header & 0xFF) !== 0x1C) break;
+			const length = header >>> 8;
+			if (bundleOffset + 4 + length > this.byteLength) break;
+
+			let chunkOffset = bundleOffset + 4;
+
+			while (chunkOffset < bundleOffset + 4 + length) {
+				const chunkType = view.getUint8(chunkOffset);
+				if (chunkType !== 0x11 && chunkType !== 0x12) break;
+
+				const cCount = view.getUint16(chunkOffset + 2, true);
+				const ΔelemC = view.getUint8(chunkOffset + 10);
+				const elemC = view.getUint8(chunkOffset + 11);
+
+				let localOffset = 12;
+
+				localOffset += ΔelemC + elemC;
+
+				// initial values
+				for (let k = 0; k < ΔelemC; k++) {
+					const typeByte = view.getUint8(chunkOffset + 12 + k);
+					const baseCode = typeByte & 0xF;
+					const baseType = codeToType[baseCode];
+					if (!baseType) throw new Error('Invalid base type code');
+					localOffset += typeToBytes[baseType.name];
+				}
+
+				// pad after initial
+				while (localOffset % 4 !== 0) localOffset++;
+
+				if (chunkType === 0x11) { // Sequential
+					// record data: Δelems incrs
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						localOffset += cCount * typeToBytes[incrType.name];
+					}
+
+					// elems
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						localOffset += cCount * typeToBytes[type.name];
+					}
+				} else { // Interleaved
+					// Compute exact stride for interleaved record data
+					let ΔelemStride = 0;
+					for (let k = 0; k < ΔelemC; k++) {
+						const typeByte = view.getUint8(chunkOffset + 12 + k);
+						const incrCode = typeByte >> 4;
+						const incrType = codeToType[incrCode];
+						if (!incrType) throw new Error('Invalid incr type code');
+						ΔelemStride += typeToBytes[incrType.name];
+					}
+					let elemStride = 0;
+					for (let k = 0; k < elemC; k++) {
+						const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
+						const type = codeToType[typeCode];
+						if (!type) throw new Error('Invalid elem type code');
+						elemStride += typeToBytes[type.name];
+					}
+					const recordStride = ΔelemStride + elemStride;
+					localOffset += cCount * recordStride;
+				}
+
+				// pad after record
+				while (localOffset % 4 !== 0) localOffset++;
+
+				switch (chunkType) {
+					case 0x11:
+						chunks.push(new DougalBinaryChunkSequential(this, chunkOffset, localOffset));
+						break;
+					case 0x12:
+						chunks.push(new DougalBinaryChunkInterleaved(this, chunkOffset, localOffset));
+						break;
+					default:
+						throw new Error('Invalid chunk type');
+				}
+
+				chunkOffset += localOffset;
+			}
+
+			bundleOffset += 4 + length;
+		}
+
+		return chunks;
+	}
+
+	/** Return a ByteArray containing all data from all
+	 * chunks including reconstructed i, j and incremental
+	 * values as follows:
+	 *
+	 * <i_0> <i_1> … <i_x> // i values (constant)
+	 * <j_0> <j_1> … <j_x> // j values (j0 + Δj*i)
+	 * <Δelem_0_0> <Δelem_0_1> … <Δelem_0_x> // reconstructed Δelem0 (uses baseType)
+	 * <Δelem_1_0> <Δelem_1_1> … <Δelem_1_x> // reconstructed Δelem1
+	 * …
+	 * <Δelem_y_0> <Δelem_y_1> … <Δelem_y_x> // reconstructed Δelem1
+	 * <elem_0_0> <elem_0_1> … <elem_0_x> // First elem
+	 * <elem_1_0> <elem_1_1> … <elem_1_x> // Second elem
+	 * …
+	 * <elem_z_0> <elem_z_1> … <elem_z_x> // Last elem
+	 *
+	 * It does not matter whether the underlying chunks are
+	 * sequential or interleaved. This function will transform
+	 * as necessary.
+	 *
+	 */
+	getDataSequentially () {
+		// TODO
+	}
+
+	/** Return a ByteArray containing all data from all
+	 * chunks including reconstructed i, j and incremental
+	 * values, interleaved as follows:
+	 *
+	 * <i_0> <j_0> <Δelem_0_0> <Δelem_1_0> … <Δelem_y_0> <elem_0_0> <elem_1_0> … <elem_z_0>
+	 * <i_1> <j_1> <Δelem_0_1> <Δelem_1_1> … <Δelem_y_1> <elem_0_1> <elem_1_1> … <elem_z_1>
+	 * <i_x> <j_x> <Δelem_0_x> <Δelem_1_x> … <Δelem_y_x> <elem_0_x> <elem_1_x> … <elem_z_x>
+	 *
+	 * It does not matter whether the underlying chunks are
+	 * sequential or interleaved. This function will transform
+	 * as necessary.
+	 *
+	 */
+	getDataInterleaved () {
+		// TODO
+	}
+
+}
+
+
+class DougalBinaryChunkSequential extends ArrayBuffer {
+
+	constructor (buffer, offset, length) {
+		super(length);
+		new Uint8Array(this).set(new Uint8Array(buffer, offset, length));
+		this._ΔelemCaches = new Array(this.ΔelemCount);
+		this._elemCaches = new Array(this.elemCount);
+		this._ΔelemBlockOffsets = null;
+		this._elemBlockOffsets = null;
+		this._recordOffset = null;
+	}
+
+	_getRecordOffset() {
+		if (this._recordOffset !== null) return this._recordOffset;
+		const view = new DataView(this);
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		let recordOffset = 12 + ΔelemC + elemC;
+		for (let k = 0; k < ΔelemC; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			recordOffset += typeToBytes[bt.name];
+		}
+		while (recordOffset % 4 !== 0) recordOffset++;
+		this._recordOffset = recordOffset;
+		return recordOffset;
+	}
+
+	_initBlockOffsets() {
+		if (this._ΔelemBlockOffsets !== null) return;
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		const recordOffset = this._getRecordOffset();
+
+		this._ΔelemBlockOffsets = [];
+		let o = recordOffset;
+		for (let k = 0; k < ΔelemC; k++) {
+			this._ΔelemBlockOffsets[k] = o;
+			const tb = view.getUint8(12 + k);
+			const ic = tb >> 4;
+			const it = codeToType[ic];
+			o += count * typeToBytes[it.name];
+		}
+
+		this._elemBlockOffsets = [];
+		for (let k = 0; k < elemC; k++) {
+			this._elemBlockOffsets[k] = o;
+			const tc = view.getUint8(12 + ΔelemC + k);
+			const t = codeToType[tc];
+			o += count * typeToBytes[t.name];
+		}
+	}
+
+	/** Return the user-defined value
+	 */
+	get udv () {
+		return new DataView(this).getUint8(1);
+	}
+
+	/** Return the number of j elements in this chunk
+	 */
+	get jCount () {
+		return new DataView(this).getUint16(2, true);
+	}
+
+	/** Return the i value in this chunk
+	 */
+	get i () {
+		return new DataView(this).getUint16(4, true);
+	}
+
+	/** Return the j0 value in this chunk
+	 */
+	get j0 () {
+		return new DataView(this).getUint16(6, true);
+	}
+
+	/** Return the Δj value in this chunk
+	 */
+	get Δj () {
+		return new DataView(this).getInt16(8, true);
+	}
+
+	/** Return the Δelem_count value in this chunk
+	 */
+	get ΔelemCount () {
+		return new DataView(this).getUint8(10);
+	}
+
+	/** Return the elem_count value in this chunk
+	 */
+	get elemCount () {
+		return new DataView(this).getUint8(11);
+	}
+
+	/** Return a TypedArray (e.g., Uint16Array, …) for the n-th Δelem in the chunk
+	 */
+	Δelem (n) {
+		if (this._ΔelemCaches[n]) return this._ΔelemCaches[n];
+
+		if (n < 0 || n >= this.ΔelemCount) throw new Error(`Invalid Δelem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+
+		const typeByte = view.getUint8(12 + n);
+		const baseCode = typeByte & 0xF;
+		const incrCode = typeByte >> 4;
+		const baseType = codeToType[baseCode];
+		const incrType = codeToType[incrCode];
+		if (!baseType || !incrType) throw new Error('Invalid type codes for Δelem');
+
+		// Find offset for initial value of this Δelem
+		let initialOffset = 12 + ΔelemC + this.elemCount;
+		for (let k = 0; k < n; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			initialOffset += typeToBytes[bt.name];
+		}
+
+		let current = readTypedValue(view, initialOffset, baseType);
+
+		// Advance to start of record data (after all initials and pad)
+		const recordOffset = this._getRecordOffset();
+
+		// Find offset for deltas of this Δelem (skip previous Δelems' delta blocks)
+		this._initBlockOffsets();
+		const deltaOffset = this._ΔelemBlockOffsets[n];
+
+		// Reconstruct the array
+		const arr = new baseType(count);
+		const isBigInt = baseType === BigInt64Array || baseType === BigUint64Array;
+		arr[0] = current;
+		for (let idx = 1; idx < count; idx++) {
+			let delta = readTypedValue(view, deltaOffset + idx * typeToBytes[incrType.name], incrType);
+			if (isBigInt) {
+				delta = BigInt(delta);
+				current += delta;
+			} else {
+				current += delta;
+			}
+			arr[idx] = current;
+		}
+
+		this._ΔelemCaches[n] = arr;
+		return arr;
+	}
+
+	/** Return a TypedArray (e.g., Uint16Array, …) for the n-th elem in the chunk
+	 */
+	elem (n) {
+		if (this._elemCaches[n]) return this._elemCaches[n];
+
+		if (n < 0 || n >= this.elemCount) throw new Error(`Invalid elem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		const typeCode = view.getUint8(12 + ΔelemC + n);
+		const type = codeToType[typeCode];
+		if (!type) throw new Error('Invalid type code for elem');
+
+		// Find offset for this elem's data block
+		this._initBlockOffsets();
+		const elemOffset = this._elemBlockOffsets[n];
+
+		// Create and populate the array
+		const arr = new type(count);
+		const bytes = typeToBytes[type.name];
+		for (let idx = 0; idx < count; idx++) {
+			arr[idx] = readTypedValue(view, elemOffset + idx * bytes, type);
+		}
+
+		this._elemCaches[n] = arr;
+		return arr;
+	}
+
+	getRecord (index) {
+		if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
+
+		const arr = [thid.udv, this.i, this.j0 + index * this.Δj];
+
+		for (let m = 0; m < this.ΔelemCount; m++) {
+			const values = this.Δelem(m);
+			arr.push(values[index]);
+		}
+
+		for (let m = 0; m < this.elemCount; m++) {
+			const values = this.elem(m);
+			arr.push(values[index]);
+		}
+
+		return arr;
+	}
+}
+
+
+class DougalBinaryChunkInterleaved extends ArrayBuffer {
+	constructor(buffer, offset, length) {
+		super(length);
+		new Uint8Array(this).set(new Uint8Array(buffer, offset, length));
+		this._incrStrides = [];
+		this._elemStrides = [];
+		this._incrOffsets = [];
+		this._elemOffsets = [];
+		this._recordStride = 0;
+		this._recordOffset = null;
+		this._initStrides();
+		this._ΔelemCaches = new Array(this.ΔelemCount);
+		this._elemCaches = new Array(this.elemCount);
+	}
+
+	_getRecordOffset() {
+		if (this._recordOffset !== null) return this._recordOffset;
+		const view = new DataView(this);
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		let recordOffset = 12 + ΔelemC + elemC;
+		for (let k = 0; k < ΔelemC; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			recordOffset += typeToBytes[bt.name];
+		}
+		while (recordOffset % 4 !== 0) recordOffset++;
+		this._recordOffset = recordOffset;
+		return recordOffset;
+	}
+
+	_initStrides() {
+		const view = new DataView(this);
+		const ΔelemC = this.ΔelemCount;
+		const elemC = this.elemCount;
+
+		// Compute incr strides and offsets
+		let incrOffset = 0;
+		for (let k = 0; k < ΔelemC; k++) {
+			const typeByte = view.getUint8(12 + k);
+			const incrCode = typeByte >> 4;
+			const incrType = codeToType[incrCode];
+			if (!incrType) throw new Error('Invalid incr type code');
+			this._incrOffsets.push(incrOffset);
+			const bytes = typeToBytes[incrType.name];
+			this._incrStrides.push(bytes);
+			incrOffset += bytes;
+			this._recordStride += bytes;
+		}
+
+		// Compute elem strides and offsets
+		let elemOffset = incrOffset;
+		for (let k = 0; k < elemC; k++) {
+			const typeCode = view.getUint8(12 + ΔelemC + k);
+			const type = codeToType[typeCode];
+			if (!type) throw new Error('Invalid elem type code');
+			this._elemOffsets.push(elemOffset);
+			const bytes = typeToBytes[type.name];
+			this._elemStrides.push(bytes);
+			elemOffset += bytes;
+			this._recordStride += bytes;
+		}
+	}
+
+	get udv() {
+		return new DataView(this).getUint8(1);
+	}
+
+	get jCount() {
+		return new DataView(this).getUint16(2, true);
+	}
+
+	get i() {
+		return new DataView(this).getUint16(4, true);
+	}
+
+	get j0() {
+		return new DataView(this).getUint16(6, true);
+	}
+
+	get Δj() {
+		return new DataView(this).getInt16(8, true);
+	}
+
+	get ΔelemCount() {
+		return new DataView(this).getUint8(10);
+	}
+
+	get elemCount() {
+		return new DataView(this).getUint8(11);
+	}
+
+	Δelem(n) {
+		if (this._ΔelemCaches[n]) return this._ΔelemCaches[n];
+
+		if (n < 0 || n >= this.ΔelemCount) throw new Error(`Invalid Δelem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+
+		const typeByte = view.getUint8(12 + n);
+		const baseCode = typeByte & 0xF;
+		const incrCode = typeByte >> 4;
+		const baseType = codeToType[baseCode];
+		const incrType = codeToType[incrCode];
+		if (!baseType || !incrType) throw new Error('Invalid type codes for Δelem');
+
+		// Find offset for initial value of this Δelem
+		let initialOffset = 12 + ΔelemC + this.elemCount;
+		for (let k = 0; k < n; k++) {
+			const tb = view.getUint8(12 + k);
+			const bc = tb & 0xF;
+			const bt = codeToType[bc];
+			initialOffset += typeToBytes[bt.name];
+		}
+
+		let current = readTypedValue(view, initialOffset, baseType);
+
+		// Find offset to start of record data
+		const recordOffset = this._getRecordOffset();
+
+		// Use precomputed offset for this Δelem
+		const deltaOffset = recordOffset + this._incrOffsets[n];
+
+		// Reconstruct the array
+		const arr = new baseType(count);
+		const isBigInt = baseType === BigInt64Array || baseType === BigUint64Array;
+		arr[0] = current;
+		for (let idx = 1; idx < count; idx++) {
+			let delta = readTypedValue(view, deltaOffset + idx * this._recordStride, incrType);
+			if (isBigInt) {
+				delta = BigInt(delta);
+				current += delta;
+			} else {
+				current += delta;
+			}
+			arr[idx] = current;
+		}
+
+		this._ΔelemCaches[n] = arr;
+		return arr;
+	}
+
+	elem(n) {
+		if (this._elemCaches[n]) return this._elemCaches[n];
+
+		if (n < 0 || n >= this.elemCount) throw new Error(`Invalid elem index: ${n}`);
+		const view = new DataView(this);
+		const count = this.jCount;
+		const ΔelemC = this.ΔelemCount;
+
+		const typeCode = view.getUint8(12 + ΔelemC + n);
+		const type = codeToType[typeCode];
+		if (!type) throw new Error('Invalid type code for elem');
+
+		// Find offset to start of record data
+		const recordOffset = this._getRecordOffset();
+
+		// Use precomputed offset for this elem (relative to start of record data)
+		const elemOffset = recordOffset + this._elemOffsets[n];
+
+		// Create and populate the array
+		const arr = new type(count);
+		const bytes = typeToBytes[type.name];
+		for (let idx = 0; idx < count; idx++) {
+			arr[idx] = readTypedValue(view, elemOffset + idx * this._recordStride, type);
+		}
+
+		this._elemCaches[n] = arr;
+		return arr;
+	}
+
+	getRecord (index) {
+		if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
+
+		const arr = [this.udv, this.i, this.j0 + index * this.Δj];
+
+		for (let m = 0; m < this.ΔelemCount; m++) {
+			const values = this.Δelem(m);
+			arr.push(values[index]);
+		}
+
+		for (let m = 0; m < this.elemCount; m++) {
+			const values = this.elem(m);
+			arr.push(values[index]);
+		}
+
+		return arr;
+	}
+}
+
+
+module.exports = { DougalBinaryBundle, DougalBinaryChunkSequential, DougalBinaryChunkInterleaved }

lib/modules/@dougal/binary/decode.js

@@ -0,0 +1,327 @@
+const codeToType = {
+	0: Int8Array,
+	1: Uint8Array,
+	2: Int16Array,
+	3: Uint16Array,
+	4: Int32Array,
+	5: Uint32Array,
+	7: Float32Array,
+	8: Float64Array,
+	9: BigInt64Array,
+	10: BigUint64Array
+};
+
+const typeToBytes = {
+	Int8Array: 1,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 2,
+	Int32Array: 4,
+	Uint32Array: 4,
+	Float32Array: 4,
+	Float64Array: 8,
+	BigInt64Array: 8,
+	BigUint64Array: 8
+};
+
+function sequential(binary) {
+	if (!(binary instanceof Uint8Array) || binary.length < 4) {
+		throw new Error('Invalid binary input');
+	}
+
+	const view = new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
+	let offset = 0;
+
+	// Initialize result (assuming single i value for simplicity; extend for multiple i values if needed)
+	const result = { i: null, j: [], Δelems: [], elems: [] };
+
+	// Process bundles
+	while (offset < binary.length) {
+		// Read bundle header
+		if (offset + 4 > binary.length) throw new Error('Incomplete bundle header');
+
+		const bundleHeader = view.getUint32(offset, true);
+		if ((bundleHeader & 0xFF) !== 0x1C) throw new Error('Invalid bundle marker');
+		const bundleLength = bundleHeader >> 8;
+		offset += 4;
+		const bundleEnd = offset + bundleLength;
+
+		if (bundleEnd > binary.length) throw new Error('Bundle length exceeds input size');
+
+		// Process chunks in bundle
+		while (offset < bundleEnd) {
+			// Read chunk header
+			if (offset + 12 > bundleEnd) throw new Error('Incomplete chunk header');
+			const chunkType = view.getUint8(offset);
+			if (chunkType !== 0x11) throw new Error(`Unsupported chunk type: ${chunkType}`);
+			offset += 1; // Skip udv
+			offset += 1;
+			const count = view.getUint16(offset, true); offset += 2;
+			if (count > 65535) throw new Error('Chunk count exceeds 65535');
+			const iValue = view.getUint16(offset, true); offset += 2;
+			const j0 = view.getUint16(offset, true); offset += 2;
+			const Δj = view.getInt16(offset, true); offset += 2;
+			const ΔelemCount = view.getUint8(offset++); // Δelem_count
+			const elemCount = view.getUint8(offset++); // elem_count
+
+			// Set i value (assuming all chunks share the same i)
+			if (result.i === null) result.i = iValue;
+			else if (result.i !== iValue) throw new Error('Multiple i values not supported');
+
+			// Read preface (element types)
+			const ΔelemTypes = [];
+			for (let i = 0; i < ΔelemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete Δelem types');
+				const typeByte = view.getUint8(offset++);
+				const baseCode = typeByte & 0x0F;
+				const incrCode = typeByte >> 4;
+				if (!codeToType[baseCode] || !codeToType[incrCode]) {
+					throw new Error(`Invalid type code in Δelem: ${typeByte}`);
+				}
+				ΔelemTypes.push({ baseType: codeToType[baseCode], incrType: codeToType[incrCode] });
+			}
+			const elemTypes = [];
+			for (let i = 0; i < elemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete elem types');
+				const typeCode = view.getUint8(offset++);
+				if (!codeToType[typeCode]) throw new Error(`Invalid type code in elem: ${typeCode}`);
+				elemTypes.push(codeToType[typeCode]);
+			}
+
+			// Initialize Δelems and elems arrays if first chunk
+			if (!result.Δelems.length && ΔelemCount > 0) {
+				result.Δelems = Array(ΔelemCount).fill().map(() => []);
+			}
+			if (!result.elems.length && elemCount > 0) {
+				result.elems = Array(elemCount).fill().map(() => []);
+			}
+
+			// Read initial values for Δelems
+			const initialValues = [];
+			for (const { baseType } of ΔelemTypes) {
+				if (offset + typeToBytes[baseType.name] > bundleEnd) {
+					throw new Error('Incomplete initial values');
+				}
+				initialValues.push(readTypedValue(view, offset, baseType));
+				offset += typeToBytes[baseType.name];
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after initial values');
+				offset++;
+			}
+
+			// Reconstruct j values
+			for (let idx = 0; idx < count; idx++) {
+				result.j.push(j0 + idx * Δj);
+			}
+
+			// Read record data (non-interleaved)
+			for (let i = 0; i < ΔelemCount; i++) {
+				let current = initialValues[i];
+				const values = result.Δelems[i];
+				const incrType = ΔelemTypes[i].incrType;
+				const isBigInt = typeof current === 'bigint';
+				for (let idx = 0; idx < count; idx++) {
+					if (offset + typeToBytes[incrType.name] > bundleEnd) {
+						throw new Error('Incomplete Δelem data');
+					}
+					let delta = readTypedValue(view, offset, incrType);
+					if (idx === 0) {
+						values.push(isBigInt ? Number(current) : current);
+					} else {
+						if (isBigInt) {
+							delta = BigInt(delta);
+							current += delta;
+							values.push(Number(current));
+						} else {
+							current += delta;
+							values.push(current);
+						}
+					}
+					offset += typeToBytes[incrType.name];
+				}
+			}
+			for (let i = 0; i < elemCount; i++) {
+				const values = result.elems[i];
+				const type = elemTypes[i];
+				const isBigInt = type === BigInt64Array || type === BigUint64Array;
+				for (let idx = 0; idx < count; idx++) {
+					if (offset + typeToBytes[type.name] > bundleEnd) {
+						throw new Error('Incomplete elem data');
+					}
+					let value = readTypedValue(view, offset, type);
+					values.push(isBigInt ? Number(value) : value);
+					offset += typeToBytes[type.name];
+				}
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after record data');
+				offset++;
+			}
+		}
+	}
+
+	return result;
+}
+
+
+function interleaved(binary) {
+	if (!(binary instanceof Uint8Array) || binary.length < 4) {
+		throw new Error('Invalid binary input');
+	}
+
+	const view = new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
+	let offset = 0;
+
+	// Initialize result (assuming single i value for simplicity; extend for multiple i values if needed)
+	const result = { i: null, j: [], Δelems: [], elems: [] };
+
+	// Process bundles
+	while (offset < binary.length) {
+		// Read bundle header
+		if (offset + 4 > binary.length) throw new Error('Incomplete bundle header');
+
+		const bundleHeader = view.getUint32(offset, true);
+		if ((bundleHeader & 0xFF) !== 0x1C) throw new Error('Invalid bundle marker');
+		const bundleLength = bundleHeader >> 8;
+		offset += 4;
+		const bundleEnd = offset + bundleLength;
+
+		if (bundleEnd > binary.length) throw new Error('Bundle length exceeds input size');
+
+		// Process chunks in bundle
+		while (offset < bundleEnd) {
+			// Read chunk header
+			if (offset + 12 > bundleEnd) throw new Error('Incomplete chunk header');
+			const chunkType = view.getUint8(offset);
+			if (chunkType !== 0x12) throw new Error(`Unsupported chunk type: ${chunkType}`);
+			offset += 1; // Skip udv
+			offset += 1;
+			const count = view.getUint16(offset, true); offset += 2;
+			if (count > 65535) throw new Error('Chunk count exceeds 65535');
+			const iValue = view.getUint16(offset, true); offset += 2;
+			const j0 = view.getUint16(offset, true); offset += 2;
+			const Δj = view.getInt16(offset, true); offset += 2;
+			const ΔelemCount = view.getUint8(offset++); // Δelem_count
+			const elemCount = view.getUint8(offset++); // elem_count
+
+			// Set i value (assuming all chunks share the same i)
+			if (result.i === null) result.i = iValue;
+			else if (result.i !== iValue) throw new Error('Multiple i values not supported');
+
+			// Read preface (element types)
+			const ΔelemTypes = [];
+			for (let i = 0; i < ΔelemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete Δelem types');
+				const typeByte = view.getUint8(offset++);
+				const baseCode = typeByte & 0x0F;
+				const incrCode = typeByte >> 4;
+				if (!codeToType[baseCode] || !codeToType[incrCode]) {
+					throw new Error(`Invalid type code in Δelem: ${typeByte}`);
+				}
+				ΔelemTypes.push({ baseType: codeToType[baseCode], incrType: codeToType[incrCode] });
+			}
+			const elemTypes = [];
+			for (let i = 0; i < elemCount; i++) {
+				if (offset >= bundleEnd) throw new Error('Incomplete elem types');
+				const typeCode = view.getUint8(offset++);
+				if (!codeToType[typeCode]) throw new Error(`Invalid type code in elem: ${typeCode}`);
+				elemTypes.push(codeToType[typeCode]);
+			}
+
+			// Initialize Δelems and elems arrays if first chunk
+			if (!result.Δelems.length && ΔelemCount > 0) {
+				result.Δelems = Array(ΔelemCount).fill().map(() => []);
+			}
+			if (!result.elems.length && elemCount > 0) {
+				result.elems = Array(elemCount).fill().map(() => []);
+			}
+
+			// Read initial values for Δelems
+			const initialValues = [];
+			for (const { baseType } of ΔelemTypes) {
+				if (offset + typeToBytes[baseType.name] > bundleEnd) {
+					throw new Error('Incomplete initial values');
+				}
+				initialValues.push(readTypedValue(view, offset, baseType));
+				offset += typeToBytes[baseType.name];
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after initial values');
+				offset++;
+			}
+
+			// Reconstruct j values
+			for (let idx = 0; idx < count; idx++) {
+				result.j.push(j0 + idx * Δj);
+			}
+
+			// Read interleaved record data
+			for (let idx = 0; idx < count; idx++) {
+				// Read Δelems
+				for (let i = 0; i < ΔelemCount; i++) {
+					const values = result.Δelems[i];
+					const incrType = ΔelemTypes[i].incrType;
+					const isBigInt = typeof initialValues[i] === 'bigint';
+					if (offset + typeToBytes[incrType.name] > bundleEnd) {
+						throw new Error('Incomplete Δelem data');
+					}
+					let delta = readTypedValue(view, offset, incrType);
+					offset += typeToBytes[incrType.name];
+					if (idx === 0) {
+						values.push(isBigInt ? Number(initialValues[i]) : initialValues[i]);
+					} else {
+						if (isBigInt) {
+							delta = BigInt(delta);
+							initialValues[i] += delta;
+							values.push(Number(initialValues[i]));
+						} else {
+							initialValues[i] += delta;
+							values.push(initialValues[i]);
+						}
+					}
+				}
+				// Read elems
+				for (let i = 0; i < elemCount; i++) {
+					const values = result.elems[i];
+					const type = elemTypes[i];
+					const isBigInt = type === BigInt64Array || type === BigUint64Array;
+					if (offset + typeToBytes[type.name] > bundleEnd) {
+						throw new Error('Incomplete elem data');
+					}
+					let value = readTypedValue(view, offset, type);
+					values.push(isBigInt ? Number(value) : value);
+					offset += typeToBytes[type.name];
+				}
+			}
+			// Skip padding
+			while (offset % 4 !== 0) {
+				if (offset >= bundleEnd) throw new Error('Incomplete padding after record data');
+				offset++;
+			}
+		}
+	}
+
+	return result;
+}
+
+function readTypedValue(view, offset, type) {
+	switch (type) {
+		case Int8Array: return view.getInt8(offset);
+		case Uint8Array: return view.getUint8(offset);
+		case Int16Array: return view.getInt16(offset, true);
+		case Uint16Array: return view.getUint16(offset, true);
+		case Int32Array: return view.getInt32(offset, true);
+		case Uint32Array: return view.getUint32(offset, true);
+		case Float32Array: return view.getFloat32(offset, true);
+		case Float64Array: return view.getFloat64(offset, true);
+		case BigInt64Array: return view.getBigInt64(offset, true);
+		case BigUint64Array: return view.getBigUint64(offset, true);
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+module.exports = { sequential, interleaved };

lib/modules/@dougal/binary/encode.js

@@ -0,0 +1,380 @@
+const typeToCode = {
+	Int8Array: 0,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 3,
+	Int32Array: 4,
+	Uint32Array: 5,
+	Float32Array: 7, // Float16 not natively supported in JS, use Float32
+	Float64Array: 8,
+	BigInt64Array: 9,
+	BigUint64Array: 10
+};
+
+const typeToBytes = {
+	Int8Array: 1,
+	Uint8Array: 1,
+	Int16Array: 2,
+	Uint16Array: 2,
+	Int32Array: 4,
+	Uint32Array: 4,
+	Float32Array: 4,
+	Float64Array: 8,
+	BigInt64Array: 8,
+	BigUint64Array: 8
+};
+
+function sequential(json, iGetter, jGetter, Δelems = [], elems = [], udv = 0) {
+	if (!Array.isArray(json) || !json.length) return new Uint8Array(0);
+	if (typeof iGetter !== 'function' || typeof jGetter !== 'function') throw new Error('i and j must be getter functions');
+	Δelems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`Δelems[${idx}].key must be a getter function`);
+	});
+	elems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`elems[${idx}].key must be a getter function`);
+	});
+
+	// Group records by i value
+	const groups = new Map();
+	for (const record of json) {
+		const iValue = iGetter(record);
+		if (iValue == null) throw new Error('Missing i value from getter');
+		if (!groups.has(iValue)) groups.set(iValue, []);
+		groups.get(iValue).push(record);
+	}
+
+	const maxBundleSize = 0xFFFFFF; // Max bundle length (24 bits)
+	const buffers = [];
+
+	// Process each group (i value)
+	for (const [iValue, records] of groups) {
+		// Sort records by j to ensure consistent order
+		records.sort((a, b) => jGetter(a) - jGetter(b));
+		const jValues = records.map(jGetter);
+		if (jValues.some(v => v == null)) throw new Error('Missing j value from getter');
+
+		// Split records into chunks based on Δj continuity
+		const chunks = [];
+		let currentChunk = [records[0]];
+		let currentJ0 = jValues[0];
+		let currentΔj = records.length > 1 ? jValues[1] - jValues[0] : 0;
+
+		for (let idx = 1; idx < records.length; idx++) {
+			const chunkIndex = chunks.reduce((sum, c) => sum + c.records.length, 0);
+			const expectedJ = currentJ0 + (idx - chunkIndex) * currentΔj;
+			if (jValues[idx] !== expectedJ || idx - chunkIndex >= 65536) {
+				chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+				currentChunk = [records[idx]];
+				currentJ0 = jValues[idx];
+				currentΔj = idx + 1 < records.length ? jValues[idx + 1] - jValues[idx] : 0;
+			} else {
+				currentChunk.push(records[idx]);
+			}
+		}
+		if (currentChunk.length > 0) {
+			chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+		}
+
+		// Calculate total size for all chunks in this group by simulating offsets
+		const chunkSizes = chunks.map(({ records: chunkRecords }) => {
+			if (chunkRecords.length > 65535) throw new Error(`Chunk size exceeds 65535 for i=${iValue}`);
+			let simulatedOffset = 0; // Relative to chunk start
+			simulatedOffset += 12; // Header
+			simulatedOffset += Δelems.length + elems.length; // Preface
+			simulatedOffset += Δelems.reduce((sum, e) => sum + typeToBytes[e.baseType.name], 0); // Initial values
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after initial
+			simulatedOffset += chunkRecords.length * (
+				Δelems.reduce((sum, e) => sum + typeToBytes[e.incrType.name], 0) +
+				elems.reduce((sum, e) => sum + typeToBytes[e.type.name], 0)
+			); // Record data
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after record
+			return simulatedOffset;
+		});
+		const totalChunkSize = chunkSizes.reduce((sum, size) => sum + size, 0);
+
+		// Start a new bundle if needed
+		const lastBundle = buffers[buffers.length - 1];
+		if (!lastBundle || lastBundle.offset + totalChunkSize > maxBundleSize) {
+			buffers.push({ offset: 4, buffer: null, view: null });
+		}
+
+		// Initialize DataView for current bundle
+		const currentBundle = buffers[buffers.length - 1];
+		if (!currentBundle.buffer) {
+			const requiredSize = totalChunkSize + 4;
+			currentBundle.buffer = new ArrayBuffer(requiredSize);
+			currentBundle.view = new DataView(currentBundle.buffer);
+		}
+
+		// Process each chunk
+		for (const { records: chunkRecords, j0, Δj } of chunks) {
+			const chunkSize = chunkSizes.shift();
+
+			// Ensure buffer is large enough
+			if (currentBundle.offset + chunkSize > currentBundle.buffer.byteLength) {
+				const newSize = currentBundle.offset + chunkSize;
+				const newBuffer = new ArrayBuffer(newSize);
+				new Uint8Array(newBuffer).set(new Uint8Array(currentBundle.buffer));
+				currentBundle.buffer = newBuffer;
+				currentBundle.view = new DataView(newBuffer);
+			}
+
+			// Write chunk header
+			let offset = currentBundle.offset;
+			currentBundle.view.setUint8(offset++, 0x11); // Chunk type
+			currentBundle.view.setUint8(offset++, udv); // udv
+			currentBundle.view.setUint16(offset, chunkRecords.length, true); offset += 2; // count
+			currentBundle.view.setUint16(offset, iValue, true); offset += 2; // i
+			currentBundle.view.setUint16(offset, j0, true); offset += 2; // j0
+			currentBundle.view.setInt16(offset, Δj, true); offset += 2; // Δj
+			currentBundle.view.setUint8(offset++, Δelems.length); // Δelem_count
+			currentBundle.view.setUint8(offset++, elems.length); // elem_count
+
+			// Write chunk preface (element types)
+			for (const elem of Δelems) {
+				const baseCode = typeToCode[elem.baseType.name];
+				const incrCode = typeToCode[elem.incrType.name];
+				currentBundle.view.setUint8(offset++, (incrCode << 4) | baseCode);
+			}
+			for (const elem of elems) {
+				currentBundle.view.setUint8(offset++, typeToCode[elem.type.name]);
+			}
+
+			// Write initial values for Δelems
+			for (const elem of Δelems) {
+				const value = elem.key(chunkRecords[0]);
+				if (value == null) throw new Error('Missing Δelem value from getter');
+				writeTypedValue(currentBundle.view, offset, value, elem.baseType);
+				offset += typeToBytes[elem.baseType.name];
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Write record data (non-interleaved)
+			for (const elem of Δelems) {
+				let prev = elem.key(chunkRecords[0]);
+				for (let idx = 0; idx < chunkRecords.length; idx++) {
+					const value = idx === 0 ? 0 : elem.key(chunkRecords[idx]) - prev;
+					writeTypedValue(currentBundle.view, offset, value, elem.incrType);
+					offset += typeToBytes[elem.incrType.name];
+					prev = elem.key(chunkRecords[idx]);
+				}
+			}
+			for (const elem of elems) {
+				for (const record of chunkRecords) {
+					const value = elem.key(record);
+					if (value == null) throw new Error('Missing elem value from getter');
+					writeTypedValue(currentBundle.view, offset, value, elem.type);
+					offset += typeToBytes[elem.type.name];
+				}
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Update bundle offset
+			currentBundle.offset = offset;
+		}
+
+		// Update bundle header
+		currentBundle.view.setUint32(0, 0x1C | ((currentBundle.offset - 4) << 8), true);
+	}
+
+	// Combine buffers into final Uint8Array
+	const finalLength = buffers.reduce((sum, b) => sum + b.offset, 0);
+	const result = new Uint8Array(finalLength);
+	let offset = 0;
+	for (const { buffer, offset: bundleOffset } of buffers) {
+		result.set(new Uint8Array(buffer, 0, bundleOffset), offset);
+		offset += bundleOffset;
+	}
+
+	return result;
+}
+
+
+function interleaved(json, iGetter, jGetter, Δelems = [], elems = [], udv = 0) {
+	if (!Array.isArray(json) || !json.length) return new Uint8Array(0);
+	if (typeof iGetter !== 'function' || typeof jGetter !== 'function') throw new Error('i and j must be getter functions');
+	Δelems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`Δelems[${idx}].key must be a getter function`);
+	});
+	elems.forEach((elem, idx) => {
+		if (typeof elem.key !== 'function') throw new Error(`elems[${idx}].key must be a getter function`);
+	});
+
+	// Group records by i value
+	const groups = new Map();
+	for (const record of json) {
+		const iValue = iGetter(record);
+		if (iValue == null) throw new Error('Missing i value from getter');
+		if (!groups.has(iValue)) groups.set(iValue, []);
+		groups.get(iValue).push(record);
+	}
+
+	const maxBundleSize = 0xFFFFFF; // Max bundle length (24 bits)
+	const buffers = [];
+
+	// Process each group (i value)
+	for (const [iValue, records] of groups) {
+		// Sort records by j to ensure consistent order
+		records.sort((a, b) => jGetter(a) - jGetter(b));
+		const jValues = records.map(jGetter);
+		if (jValues.some(v => v == null)) throw new Error('Missing j value from getter');
+
+		// Split records into chunks based on Δj continuity
+		const chunks = [];
+		let currentChunk = [records[0]];
+		let currentJ0 = jValues[0];
+		let currentΔj = records.length > 1 ? jValues[1] - jValues[0] : 0;
+
+		for (let idx = 1; idx < records.length; idx++) {
+			const chunkIndex = chunks.reduce((sum, c) => sum + c.records.length, 0);
+			const expectedJ = currentJ0 + (idx - chunkIndex) * currentΔj;
+			if (jValues[idx] !== expectedJ || idx - chunkIndex >= 65536) {
+				chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+				currentChunk = [records[idx]];
+				currentJ0 = jValues[idx];
+				currentΔj = idx + 1 < records.length ? jValues[idx + 1] - jValues[idx] : 0;
+			} else {
+				currentChunk.push(records[idx]);
+			}
+		}
+		if (currentChunk.length > 0) {
+			chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
+		}
+
+		// Calculate total size for all chunks in this group by simulating offsets
+		const chunkSizes = chunks.map(({ records: chunkRecords }) => {
+			if (chunkRecords.length > 65535) throw new Error(`Chunk size exceeds 65535 for i=${iValue}`);
+			let simulatedOffset = 0; // Relative to chunk start
+			simulatedOffset += 12; // Header
+			simulatedOffset += Δelems.length + elems.length; // Preface
+			simulatedOffset += Δelems.reduce((sum, e) => sum + typeToBytes[e.baseType.name], 0); // Initial values
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after initial
+			simulatedOffset += chunkRecords.length * (
+				Δelems.reduce((sum, e) => sum + typeToBytes[e.incrType.name], 0) +
+				elems.reduce((sum, e) => sum + typeToBytes[e.type.name], 0)
+			); // Interleaved record data
+			while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after record
+			return simulatedOffset;
+		});
+		const totalChunkSize = chunkSizes.reduce((sum, size) => sum + size, 0);
+
+		// Start a new bundle if needed
+		const lastBundle = buffers[buffers.length - 1];
+		if (!lastBundle || lastBundle.offset + totalChunkSize > maxBundleSize) {
+			buffers.push({ offset: 4, buffer: null, view: null });
+		}
+
+		// Initialize DataView for current bundle
+		const currentBundle = buffers[buffers.length - 1];
+		if (!currentBundle.buffer) {
+			const requiredSize = totalChunkSize + 4;
+			currentBundle.buffer = new ArrayBuffer(requiredSize);
+			currentBundle.view = new DataView(currentBundle.buffer);
+		}
+
+		// Process each chunk
+		for (const { records: chunkRecords, j0, Δj } of chunks) {
+			const chunkSize = chunkSizes.shift();
+
+			// Ensure buffer is large enough
+			if (currentBundle.offset + chunkSize > currentBundle.buffer.byteLength) {
+				const newSize = currentBundle.offset + chunkSize;
+				const newBuffer = new ArrayBuffer(newSize);
+				new Uint8Array(newBuffer).set(new Uint8Array(currentBundle.buffer));
+				currentBundle.buffer = newBuffer;
+				currentBundle.view = new DataView(newBuffer);
+			}
+
+			// Write chunk header
+			let offset = currentBundle.offset;
+			currentBundle.view.setUint8(offset++, 0x12); // Chunk type
+			currentBundle.view.setUint8(offset++, udv); // udv
+			currentBundle.view.setUint16(offset, chunkRecords.length, true); offset += 2; // count
+			currentBundle.view.setUint16(offset, iValue, true); offset += 2; // i
+			currentBundle.view.setUint16(offset, j0, true); offset += 2; // j0
+			currentBundle.view.setInt16(offset, Δj, true); offset += 2; // Δj
+			currentBundle.view.setUint8(offset++, Δelems.length); // Δelem_count
+			currentBundle.view.setUint8(offset++, elems.length); // elem_count
+
+			// Write chunk preface (element types)
+			for (const elem of Δelems) {
+				const baseCode = typeToCode[elem.baseType.name];
+				const incrCode = typeToCode[elem.incrType.name];
+				currentBundle.view.setUint8(offset++, (incrCode << 4) | baseCode);
+			}
+			for (const elem of elems) {
+				currentBundle.view.setUint8(offset++, typeToCode[elem.type.name]);
+			}
+
+			// Write initial values for Δelems
+			for (const elem of Δelems) {
+				const value = elem.key(chunkRecords[0]);
+				if (value == null) throw new Error('Missing Δelem value from getter');
+				writeTypedValue(currentBundle.view, offset, value, elem.baseType);
+				offset += typeToBytes[elem.baseType.name];
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Write interleaved record data
+			const prevValues = Δelems.map(elem => elem.key(chunkRecords[0]));
+			for (let idx = 0; idx < chunkRecords.length; idx++) {
+				// Write Δelems increments
+				for (let i = 0; i < Δelems.length; i++) {
+					const elem = Δelems[i];
+					const value = idx === 0 ? 0 : elem.key(chunkRecords[idx]) - prevValues[i];
+					writeTypedValue(currentBundle.view, offset, value, elem.incrType);
+					offset += typeToBytes[elem.incrType.name];
+					prevValues[i] = elem.key(chunkRecords[idx]);
+				}
+				// Write elems
+				for (const elem of elems) {
+					const value = elem.key(chunkRecords[idx]);
+					if (value == null) throw new Error('Missing elem value from getter');
+					writeTypedValue(currentBundle.view, offset, value, elem.type);
+					offset += typeToBytes[elem.type.name];
+				}
+			}
+			// Pad to 4-byte boundary
+			while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
+
+			// Update bundle offset
+			currentBundle.offset = offset;
+		}
+
+		// Update bundle header
+		currentBundle.view.setUint32(0, 0x1C | ((currentBundle.offset - 4) << 8), true);
+	}
+
+	// Combine buffers into final Uint8Array
+	const finalLength = buffers.reduce((sum, b) => sum + b.offset, 0);
+	const result = new Uint8Array(finalLength);
+	let offset = 0;
+	for (const { buffer, offset: bundleOffset } of buffers) {
+		result.set(new Uint8Array(buffer, 0, bundleOffset), offset);
+		offset += bundleOffset;
+	}
+
+	return result;
+}
+
+
+function writeTypedValue(view, offset, value, type) {
+	switch (type) {
+		case Int8Array: view.setInt8(offset, value); break;
+		case Uint8Array: view.setUint8(offset, value); break;
+		case Int16Array: view.setInt16(offset, value, true); break;
+		case Uint16Array: view.setUint16(offset, value, true); break;
+		case Int32Array: view.setInt32(offset, value, true); break;
+		case Uint32Array: view.setUint32(offset, value, true); break;
+		case Float32Array: view.setFloat32(offset, value, true); break;
+		case Float64Array: view.setFloat64(offset, value, true); break;
+		case BigInt64Array: view.setBigInt64(offset, BigInt(value), true); break;
+		case BigUint64Array: view.setBigUint64(offset, BigInt(value), true); break;
+		default: throw new Error(`Unsupported type: ${type.name}`);
+	}
+}
+
+module.exports = { sequential, interleaved };

lib/modules/@dougal/binary/index.js

@@ -0,0 +1,139 @@
+
+/** Binary encoder
+ *
+ * This module encodes scalar data from a grid-like source
+ * into a packed binary format for bandwidth efficiency and
+ * speed of access.
+ *
+ * Data are indexed by i & j values, with "i" being constant
+ * (e.g., a sequence or line number) and "j" expected to change
+ * by a constant, linear amount (e.g., point numbers). All data
+ * from consecutive "j" values will be encoded as a single array
+ * (or series of arrays if multiple values are encoded).
+ * If there is a jump in the "j" progression, a new "chunk" will
+ * be started with a new array (or series of arrays).
+ *
+ * Multiple values may be encoded per (i, j) pair, using any of
+ * the types supported by JavaScript's TypedArray except for
+ * Float16 and Uint8Clamped. Each variable can be encoded with
+ * a different size.
+ *
+ * Values may be encoded directly or as deltas from an initial
+ * value. The latter is particularly efficient when dealing with
+ * monotonically incrementing data, such as timestamps.
+ *
+ * The conceptual packet format for sequentially encoded data
+ * looks like this:
+ *
+ * <msg-type> <count: x> <i> <j0> <Δj>
+ *
+ * <Δelement_count: y>
+ * <element_count: z>
+ *
+ * <Δelement_1_type_base> … <Δelement_y_type_base>
+ * <Δelement_1_type_incr> … <Δelement_y_type_incr>
+ * <elem_1_type> … <elem_z_type>
+ *
+ * <Δelement_1_first> … <Δelement_z_first>
+ *
+ * <Δelem_1_0> … <Δelem_1_x>
+ * …
+ * <Δelem_y_0> … <Δelem_y_x>
+ * <elem_1_0> … <elem_1_x>
+ * …
+ * <elem_z_0> … <elem_z_x>
+ *
+ *
+ * The conceptual packet format for interleaved encoded data
+ * looks like this:
+ *
+ *
+ * <msg-type> <count: x> <i> <j0> <Δj>
+ *
+ * <Δelement_count: y>
+ * <element_count: z>
+ *
+ * <Δelement_1_type_base> … <Δelement_y_type_base>
+ * <Δelement_1_type_incr> … <Δelement_y_type_incr>
+ * <elem_1_type> … <elem_z_type>
+ *
+ * <Δelement_1_first> … <Δelement_y_first>
+ *
+ * <Δelem_1_0> <Δelem_2_0> … <Δelem_y_0> <elem_1_0> <elem_2_0> … <elem_z_0>
+ * <Δelem_1_1> <Δelem_2_1> … <Δelem_y_1> <elem_1_1> <elem_2_1> … <elem_z_1>
+ * …
+ * <Δelem_1_x> <Δelem_2_x> … <Δelem_y_x> <elem_1_x> <elem_2_x> … <elem_z_x>
+ *
+ *
+ * Usage example:
+ *
+ * json = [
+ *   {
+ *     sequence: 7,
+ *     sailline: 5354,
+ *     line: 5356,
+ *     point: 1068,
+ *     tstamp: 1695448704372,
+ *     objrefraw: 3,
+ *     objreffinal: 4
+ *   },
+ *   {
+ *     sequence: 7,
+ *     sailline: 5354,
+ *     line: 5352,
+ *     point: 1070,
+ *     tstamp: 1695448693612,
+ *     objrefraw: 2,
+ *     objreffinal: 3
+ *   },
+ *   {
+ *     sequence: 7,
+ *     sailline: 5354,
+ *     line: 5356,
+ *     point: 1072,
+ *     tstamp: 1695448684624,
+ *     objrefraw: 3,
+ *     objreffinal: 4
+ *   }
+ * ];
+ *
+ * deltas = [
+ *   { key: el => el.tstamp, baseType: BigUint64Array, incrType: Int16Array }
+ * ];
+ *
+ * elems = [
+ *   { key: el => el.objrefraw, type: Uint8Array },
+ *   { key: el => el.objreffinal, type: Uint8Array }
+ * ];
+ *
+ * i = el => el.sequence;
+ *
+ * j = el => el.point;
+ *
+ * bundle = encode(json, i, j, deltas, elems);
+ *
+ * // bundle:
+ *
+ * Uint8Array(40) [
+ *    36,   0,   0,  28, 17, 0,  3, 0, 7,   0,
+ *    44,   4,   2,   0,  1, 2, 42, 1, 1, 116,
+ *    37, 158, 192, 138,  1, 0,  0, 0, 0,   0,
+ *   248, 213, 228, 220,  3, 2,  3, 4, 3,   4
+ * ]
+ *
+ * decode(bundle);
+ *
+ * {
+ *   i: 7,
+ *   j: [ 1068, 1070, 1072 ],
+ *   'Δelems': [ [ 1695448704372, 1695448693612, 1695448684624 ] ],
+ *   elems: [ [ 3, 2, 3 ], [ 4, 3, 4 ] ]
+ * }
+ *
+ */
+
+module.exports = {
+	encode: {...require('./encode')},
+	decode: {...require('./decode')},
+	...require('./classes')
+};

lib/modules/@dougal/binary/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@dougal/binary",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "description": ""
+}