Aaltronav/dougal-software
1
0
Fork 0
mirror of https://gitlab.com/wgp/dougal/software.git synced 2025-12-06 11:37:08 +00:00
Code Issues Packages Projects Releases Activity

Add @dougal/binary module.

Browse Source 
It encodes / decodes sequence / preplot data using an efficient
binary format for sending large amounts of data across the wire
and for (relatively) memory efficient client-side use.
This commit is contained in:
D. Berge
2025-07-30 17:37:00 +02:00
parent 2a0025cdbf
commit 1cb9d4b1e2
5 changed files with 1577 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

719
lib/modules/@dougal/binary/classes.js Normal file
View File

@@ -0,0 +1,719 @@
const codeToType = {
0: Int8Array,
1: Uint8Array,
2: Int16Array,
3: Uint16Array,
4: Int32Array,
5: Uint32Array,
7: Float32Array,
8: Float64Array,
9: BigInt64Array,
10: BigUint64Array
};
const typeToBytes = {
Int8Array: 1,
Uint8Array: 1,
Int16Array: 2,
Uint16Array: 2,
Int32Array: 4,
Uint32Array: 4,
Float32Array: 4,
Float64Array: 8,
BigInt64Array: 8,
BigUint64Array: 8
};
function readTypedValue(view, offset, type) {
switch (type) {
case Int8Array: return view.getInt8(offset);
case Uint8Array: return view.getUint8(offset);
case Int16Array: return view.getInt16(offset, true);
case Uint16Array: return view.getUint16(offset, true);
case Int32Array: return view.getInt32(offset, true);
case Uint32Array: return view.getUint32(offset, true);
case Float32Array: return view.getFloat32(offset, true);
case Float64Array: return view.getFloat64(offset, true);
case BigInt64Array: return view.getBigInt64(offset, true);
case BigUint64Array: return view.getBigUint64(offset, true);
default: throw new Error(`Unsupported type: ${type.name}`);
}
}
class DougalBinaryBundle extends ArrayBuffer {
static HEADER_LENGTH = 4; // Length of a bundle header
/** Clone an existing ByteArray into a DougalBinaryBundle
*/
static clone (buffer) {
const clone = new DougalBinaryBundle(buffer.byteLength);
const uint8Array = new Uint8Array(buffer);
const uint8ArrayClone = new Uint8Array(clone);
uint8ArrayClone.set(uint8Array);
return clone;
}
constructor (length, options) {
super (length, options);
}
/** Get the count of bundles in this ByteArray.
*
* Stops at the first non-bundle looking offset
*/
get bundleCount () {
let count = 0;
let currentBundleOffset = 0;
const view = new DataView(this);
while (currentBundleOffset < this.byteLength) {
const currentBundleHeader = view.getUint32(currentBundleOffset, true);
if ((currentBundleHeader & 0xff) !== 0x1c) {
// This is not a bundle
return count;
}
let currentBundleLength = currentBundleHeader >>> 8;
currentBundleOffset += currentBundleLength + DougalBinaryBundle.HEADER_LENGTH;
count++;
}
return count;
}
/** Get the number of chunks in the bundles of this ByteArray
*/
get chunkCount () {
let count = 0;
let bundleOffset = 0;
const view = new DataView(this);
while (bundleOffset < this.byteLength) {
const header = view.getUint32(bundleOffset, true);
if ((header & 0xFF) !== 0x1C) break;
const length = header >>> 8;
if (bundleOffset + 4 + length > this.byteLength) break;
let chunkOffset = bundleOffset + 4; // relative to buffer start
while (chunkOffset < bundleOffset + 4 + length) {
const chunkType = view.getUint8(chunkOffset);
if (chunkType !== 0x11 && chunkType !== 0x12) break;
const cCount = view.getUint16(chunkOffset + 2, true);
const ΔelemC = view.getUint8(chunkOffset + 10);
const elemC = view.getUint8(chunkOffset + 11);
let localOffset = 12; // header size
localOffset += ΔelemC + elemC; // preface
// initial values
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(chunkOffset + 12 + k);
const baseCode = typeByte & 0xF;
const baseType = codeToType[baseCode];
if (!baseType) throw new Error('Invalid base type code');
localOffset += typeToBytes[baseType.name];
}
// pad after initial
while (localOffset % 4 !== 0) localOffset++;
if (chunkType === 0x11) { // Sequential
// record data: Δelems incrs
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(chunkOffset + 12 + k);
const incrCode = typeByte >> 4;
const incrType = codeToType[incrCode];
if (!incrType) throw new Error('Invalid incr type code');
localOffset += cCount * typeToBytes[incrType.name];
}
// elems
for (let k = 0; k < elemC; k++) {
const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid elem type code');
localOffset += cCount * typeToBytes[type.name];
}
} else { // Interleaved
// Compute exact stride for interleaved record data
let ΔelemStride = 0;
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(chunkOffset + 12 + k);
const incrCode = typeByte >> 4;
const incrType = codeToType[incrCode];
if (!incrType) throw new Error('Invalid incr type code');
ΔelemStride += typeToBytes[incrType.name];
}
let elemStride = 0;
for (let k = 0; k < elemC; k++) {
const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid elem type code');
elemStride += typeToBytes[type.name];
}
const recordStride = ΔelemStride + elemStride;
localOffset += cCount * recordStride;
}
// pad after record
while (localOffset % 4 !== 0) localOffset++;
chunkOffset += localOffset;
count++;
}
bundleOffset += 4 + length;
}
return count;
}
/** Return an array of DougalBinaryChunkSequential or DougalBinaryChunkInterleaved instances
*/
chunks () {
const chunks = [];
let bundleOffset = 0;
const view = new DataView(this);
while (bundleOffset < this.byteLength) {
const header = view.getUint32(bundleOffset, true);
if ((header & 0xFF) !== 0x1C) break;
const length = header >>> 8;
if (bundleOffset + 4 + length > this.byteLength) break;
let chunkOffset = bundleOffset + 4;
while (chunkOffset < bundleOffset + 4 + length) {
const chunkType = view.getUint8(chunkOffset);
if (chunkType !== 0x11 && chunkType !== 0x12) break;
const cCount = view.getUint16(chunkOffset + 2, true);
const ΔelemC = view.getUint8(chunkOffset + 10);
const elemC = view.getUint8(chunkOffset + 11);
let localOffset = 12;
localOffset += ΔelemC + elemC;
// initial values
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(chunkOffset + 12 + k);
const baseCode = typeByte & 0xF;
const baseType = codeToType[baseCode];
if (!baseType) throw new Error('Invalid base type code');
localOffset += typeToBytes[baseType.name];
}
// pad after initial
while (localOffset % 4 !== 0) localOffset++;
if (chunkType === 0x11) { // Sequential
// record data: Δelems incrs
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(chunkOffset + 12 + k);
const incrCode = typeByte >> 4;
const incrType = codeToType[incrCode];
if (!incrType) throw new Error('Invalid incr type code');
localOffset += cCount * typeToBytes[incrType.name];
}
// elems
for (let k = 0; k < elemC; k++) {
const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid elem type code');
localOffset += cCount * typeToBytes[type.name];
}
} else { // Interleaved
// Compute exact stride for interleaved record data
let ΔelemStride = 0;
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(chunkOffset + 12 + k);
const incrCode = typeByte >> 4;
const incrType = codeToType[incrCode];
if (!incrType) throw new Error('Invalid incr type code');
ΔelemStride += typeToBytes[incrType.name];
}
let elemStride = 0;
for (let k = 0; k < elemC; k++) {
const typeCode = view.getUint8(chunkOffset + 12 + ΔelemC + k);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid elem type code');
elemStride += typeToBytes[type.name];
}
const recordStride = ΔelemStride + elemStride;
localOffset += cCount * recordStride;
}
// pad after record
while (localOffset % 4 !== 0) localOffset++;
switch (chunkType) {
case 0x11:
chunks.push(new DougalBinaryChunkSequential(this, chunkOffset, localOffset));
break;
case 0x12:
chunks.push(new DougalBinaryChunkInterleaved(this, chunkOffset, localOffset));
break;
default:
throw new Error('Invalid chunk type');
}
chunkOffset += localOffset;
}
bundleOffset += 4 + length;
}
return chunks;
}
/** Return a ByteArray containing all data from all
* chunks including reconstructed i, j and incremental
* values as follows:
*
* <i_0> <i_1> … <i_x> // i values (constant)
* <j_0> <j_1> … <j_x> // j values (j0 + Δj*i)
* <Δelem_0_0> <Δelem_0_1> … <Δelem_0_x> // reconstructed Δelem0 (uses baseType)
* <Δelem_1_0> <Δelem_1_1> … <Δelem_1_x> // reconstructed Δelem1
* …
* <Δelem_y_0> <Δelem_y_1> … <Δelem_y_x> // reconstructed Δelem1
* <elem_0_0> <elem_0_1> … <elem_0_x> // First elem
* <elem_1_0> <elem_1_1> … <elem_1_x> // Second elem
* …
* <elem_z_0> <elem_z_1> … <elem_z_x> // Last elem
*
* It does not matter whether the underlying chunks are
* sequential or interleaved. This function will transform
* as necessary.
*
*/
getDataSequentially () {
// TODO
}
/** Return a ByteArray containing all data from all
* chunks including reconstructed i, j and incremental
* values, interleaved as follows:
*
* <i_0> <j_0> <Δelem_0_0> <Δelem_1_0> … <Δelem_y_0> <elem_0_0> <elem_1_0> … <elem_z_0>
* <i_1> <j_1> <Δelem_0_1> <Δelem_1_1> … <Δelem_y_1> <elem_0_1> <elem_1_1> … <elem_z_1>
* <i_x> <j_x> <Δelem_0_x> <Δelem_1_x> … <Δelem_y_x> <elem_0_x> <elem_1_x> … <elem_z_x>
*
* It does not matter whether the underlying chunks are
* sequential or interleaved. This function will transform
* as necessary.
*
*/
getDataInterleaved () {
// TODO
}
}
class DougalBinaryChunkSequential extends ArrayBuffer {
constructor (buffer, offset, length) {
super(length);
new Uint8Array(this).set(new Uint8Array(buffer, offset, length));
this._ΔelemCaches = new Array(this.ΔelemCount);
this._elemCaches = new Array(this.elemCount);
this._ΔelemBlockOffsets = null;
this._elemBlockOffsets = null;
this._recordOffset = null;
}
_getRecordOffset() {
if (this._recordOffset !== null) return this._recordOffset;
const view = new DataView(this);
const ΔelemC = this.ΔelemCount;
const elemC = this.elemCount;
let recordOffset = 12 + ΔelemC + elemC;
for (let k = 0; k < ΔelemC; k++) {
const tb = view.getUint8(12 + k);
const bc = tb & 0xF;
const bt = codeToType[bc];
recordOffset += typeToBytes[bt.name];
}
while (recordOffset % 4 !== 0) recordOffset++;
this._recordOffset = recordOffset;
return recordOffset;
}
_initBlockOffsets() {
if (this._ΔelemBlockOffsets !== null) return;
const view = new DataView(this);
const count = this.jCount;
const ΔelemC = this.ΔelemCount;
const elemC = this.elemCount;
const recordOffset = this._getRecordOffset();
this._ΔelemBlockOffsets = [];
let o = recordOffset;
for (let k = 0; k < ΔelemC; k++) {
this._ΔelemBlockOffsets[k] = o;
const tb = view.getUint8(12 + k);
const ic = tb >> 4;
const it = codeToType[ic];
o += count * typeToBytes[it.name];
}
this._elemBlockOffsets = [];
for (let k = 0; k < elemC; k++) {
this._elemBlockOffsets[k] = o;
const tc = view.getUint8(12 + ΔelemC + k);
const t = codeToType[tc];
o += count * typeToBytes[t.name];
}
}
/** Return the user-defined value
*/
get udv () {
return new DataView(this).getUint8(1);
}
/** Return the number of j elements in this chunk
*/
get jCount () {
return new DataView(this).getUint16(2, true);
}
/** Return the i value in this chunk
*/
get i () {
return new DataView(this).getUint16(4, true);
}
/** Return the j0 value in this chunk
*/
get j0 () {
return new DataView(this).getUint16(6, true);
}
/** Return the Δj value in this chunk
*/
get Δj () {
return new DataView(this).getInt16(8, true);
}
/** Return the Δelem_count value in this chunk
*/
get ΔelemCount () {
return new DataView(this).getUint8(10);
}
/** Return the elem_count value in this chunk
*/
get elemCount () {
return new DataView(this).getUint8(11);
}
/** Return a TypedArray (e.g., Uint16Array, …) for the n-th Δelem in the chunk
*/
Δelem (n) {
if (this._ΔelemCaches[n]) return this._ΔelemCaches[n];
if (n < 0 || n >= this.ΔelemCount) throw new Error(`Invalid Δelem index: ${n}`);
const view = new DataView(this);
const count = this.jCount;
const ΔelemC = this.ΔelemCount;
const typeByte = view.getUint8(12 + n);
const baseCode = typeByte & 0xF;
const incrCode = typeByte >> 4;
const baseType = codeToType[baseCode];
const incrType = codeToType[incrCode];
if (!baseType || !incrType) throw new Error('Invalid type codes for Δelem');
// Find offset for initial value of this Δelem
let initialOffset = 12 + ΔelemC + this.elemCount;
for (let k = 0; k < n; k++) {
const tb = view.getUint8(12 + k);
const bc = tb & 0xF;
const bt = codeToType[bc];
initialOffset += typeToBytes[bt.name];
}
let current = readTypedValue(view, initialOffset, baseType);
// Advance to start of record data (after all initials and pad)
const recordOffset = this._getRecordOffset();
// Find offset for deltas of this Δelem (skip previous Δelems' delta blocks)
this._initBlockOffsets();
const deltaOffset = this._ΔelemBlockOffsets[n];
// Reconstruct the array
const arr = new baseType(count);
const isBigInt = baseType === BigInt64Array || baseType === BigUint64Array;
arr[0] = current;
for (let idx = 1; idx < count; idx++) {
let delta = readTypedValue(view, deltaOffset + idx * typeToBytes[incrType.name], incrType);
if (isBigInt) {
delta = BigInt(delta);
current += delta;
} else {
current += delta;
}
arr[idx] = current;
}
this._ΔelemCaches[n] = arr;
return arr;
}
/** Return a TypedArray (e.g., Uint16Array, …) for the n-th elem in the chunk
*/
elem (n) {
if (this._elemCaches[n]) return this._elemCaches[n];
if (n < 0 || n >= this.elemCount) throw new Error(`Invalid elem index: ${n}`);
const view = new DataView(this);
const count = this.jCount;
const ΔelemC = this.ΔelemCount;
const elemC = this.elemCount;
const typeCode = view.getUint8(12 + ΔelemC + n);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid type code for elem');
// Find offset for this elem's data block
this._initBlockOffsets();
const elemOffset = this._elemBlockOffsets[n];
// Create and populate the array
const arr = new type(count);
const bytes = typeToBytes[type.name];
for (let idx = 0; idx < count; idx++) {
arr[idx] = readTypedValue(view, elemOffset + idx * bytes, type);
}
this._elemCaches[n] = arr;
return arr;
}
getRecord (index) {
if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
const arr = [thid.udv, this.i, this.j0 + index * this.Δj];
for (let m = 0; m < this.ΔelemCount; m++) {
const values = this.Δelem(m);
arr.push(values[index]);
}
for (let m = 0; m < this.elemCount; m++) {
const values = this.elem(m);
arr.push(values[index]);
}
return arr;
}
}
class DougalBinaryChunkInterleaved extends ArrayBuffer {
constructor(buffer, offset, length) {
super(length);
new Uint8Array(this).set(new Uint8Array(buffer, offset, length));
this._incrStrides = [];
this._elemStrides = [];
this._incrOffsets = [];
this._elemOffsets = [];
this._recordStride = 0;
this._recordOffset = null;
this._initStrides();
this._ΔelemCaches = new Array(this.ΔelemCount);
this._elemCaches = new Array(this.elemCount);
}
_getRecordOffset() {
if (this._recordOffset !== null) return this._recordOffset;
const view = new DataView(this);
const ΔelemC = this.ΔelemCount;
const elemC = this.elemCount;
let recordOffset = 12 + ΔelemC + elemC;
for (let k = 0; k < ΔelemC; k++) {
const tb = view.getUint8(12 + k);
const bc = tb & 0xF;
const bt = codeToType[bc];
recordOffset += typeToBytes[bt.name];
}
while (recordOffset % 4 !== 0) recordOffset++;
this._recordOffset = recordOffset;
return recordOffset;
}
_initStrides() {
const view = new DataView(this);
const ΔelemC = this.ΔelemCount;
const elemC = this.elemCount;
// Compute incr strides and offsets
let incrOffset = 0;
for (let k = 0; k < ΔelemC; k++) {
const typeByte = view.getUint8(12 + k);
const incrCode = typeByte >> 4;
const incrType = codeToType[incrCode];
if (!incrType) throw new Error('Invalid incr type code');
this._incrOffsets.push(incrOffset);
const bytes = typeToBytes[incrType.name];
this._incrStrides.push(bytes);
incrOffset += bytes;
this._recordStride += bytes;
}
// Compute elem strides and offsets
let elemOffset = incrOffset;
for (let k = 0; k < elemC; k++) {
const typeCode = view.getUint8(12 + ΔelemC + k);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid elem type code');
this._elemOffsets.push(elemOffset);
const bytes = typeToBytes[type.name];
this._elemStrides.push(bytes);
elemOffset += bytes;
this._recordStride += bytes;
}
}
get udv() {
return new DataView(this).getUint8(1);
}
get jCount() {
return new DataView(this).getUint16(2, true);
}
get i() {
return new DataView(this).getUint16(4, true);
}
get j0() {
return new DataView(this).getUint16(6, true);
}
get Δj() {
return new DataView(this).getInt16(8, true);
}
get ΔelemCount() {
return new DataView(this).getUint8(10);
}
get elemCount() {
return new DataView(this).getUint8(11);
}
Δelem(n) {
if (this._ΔelemCaches[n]) return this._ΔelemCaches[n];
if (n < 0 || n >= this.ΔelemCount) throw new Error(`Invalid Δelem index: ${n}`);
const view = new DataView(this);
const count = this.jCount;
const ΔelemC = this.ΔelemCount;
const typeByte = view.getUint8(12 + n);
const baseCode = typeByte & 0xF;
const incrCode = typeByte >> 4;
const baseType = codeToType[baseCode];
const incrType = codeToType[incrCode];
if (!baseType || !incrType) throw new Error('Invalid type codes for Δelem');
// Find offset for initial value of this Δelem
let initialOffset = 12 + ΔelemC + this.elemCount;
for (let k = 0; k < n; k++) {
const tb = view.getUint8(12 + k);
const bc = tb & 0xF;
const bt = codeToType[bc];
initialOffset += typeToBytes[bt.name];
}
let current = readTypedValue(view, initialOffset, baseType);
// Find offset to start of record data
const recordOffset = this._getRecordOffset();
// Use precomputed offset for this Δelem
const deltaOffset = recordOffset + this._incrOffsets[n];
// Reconstruct the array
const arr = new baseType(count);
const isBigInt = baseType === BigInt64Array || baseType === BigUint64Array;
arr[0] = current;
for (let idx = 1; idx < count; idx++) {
let delta = readTypedValue(view, deltaOffset + idx * this._recordStride, incrType);
if (isBigInt) {
delta = BigInt(delta);
current += delta;
} else {
current += delta;
}
arr[idx] = current;
}
this._ΔelemCaches[n] = arr;
return arr;
}
elem(n) {
if (this._elemCaches[n]) return this._elemCaches[n];
if (n < 0 || n >= this.elemCount) throw new Error(`Invalid elem index: ${n}`);
const view = new DataView(this);
const count = this.jCount;
const ΔelemC = this.ΔelemCount;
const typeCode = view.getUint8(12 + ΔelemC + n);
const type = codeToType[typeCode];
if (!type) throw new Error('Invalid type code for elem');
// Find offset to start of record data
const recordOffset = this._getRecordOffset();
// Use precomputed offset for this elem (relative to start of record data)
const elemOffset = recordOffset + this._elemOffsets[n];
// Create and populate the array
const arr = new type(count);
const bytes = typeToBytes[type.name];
for (let idx = 0; idx < count; idx++) {
arr[idx] = readTypedValue(view, elemOffset + idx * this._recordStride, type);
}
this._elemCaches[n] = arr;
return arr;
}
getRecord (index) {
if (index < 0 || index >= this.jCount) throw new Error(`Invalid record index: ${index}`);
const arr = [this.udv, this.i, this.j0 + index * this.Δj];
for (let m = 0; m < this.ΔelemCount; m++) {
const values = this.Δelem(m);
arr.push(values[index]);
}
for (let m = 0; m < this.elemCount; m++) {
const values = this.elem(m);
arr.push(values[index]);
}
return arr;
}
}
module.exports = { DougalBinaryBundle, DougalBinaryChunkSequential, DougalBinaryChunkInterleaved }

327
lib/modules/@dougal/binary/decode.js Normal file
View File

@@ -0,0 +1,327 @@
const codeToType = {
0: Int8Array,
1: Uint8Array,
2: Int16Array,
3: Uint16Array,
4: Int32Array,
5: Uint32Array,
7: Float32Array,
8: Float64Array,
9: BigInt64Array,
10: BigUint64Array
};
const typeToBytes = {
Int8Array: 1,
Uint8Array: 1,
Int16Array: 2,
Uint16Array: 2,
Int32Array: 4,
Uint32Array: 4,
Float32Array: 4,
Float64Array: 8,
BigInt64Array: 8,
BigUint64Array: 8
};
function sequential(binary) {
if (!(binary instanceof Uint8Array) || binary.length < 4) {
throw new Error('Invalid binary input');
}
const view = new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
let offset = 0;
// Initialize result (assuming single i value for simplicity; extend for multiple i values if needed)
const result = { i: null, j: [], Δelems: [], elems: [] };
// Process bundles
while (offset < binary.length) {
// Read bundle header
if (offset + 4 > binary.length) throw new Error('Incomplete bundle header');
const bundleHeader = view.getUint32(offset, true);
if ((bundleHeader & 0xFF) !== 0x1C) throw new Error('Invalid bundle marker');
const bundleLength = bundleHeader >> 8;
offset += 4;
const bundleEnd = offset + bundleLength;
if (bundleEnd > binary.length) throw new Error('Bundle length exceeds input size');
// Process chunks in bundle
while (offset < bundleEnd) {
// Read chunk header
if (offset + 12 > bundleEnd) throw new Error('Incomplete chunk header');
const chunkType = view.getUint8(offset);
if (chunkType !== 0x11) throw new Error(`Unsupported chunk type: ${chunkType}`);
offset += 1; // Skip udv
offset += 1;
const count = view.getUint16(offset, true); offset += 2;
if (count > 65535) throw new Error('Chunk count exceeds 65535');
const iValue = view.getUint16(offset, true); offset += 2;
const j0 = view.getUint16(offset, true); offset += 2;
const Δj = view.getInt16(offset, true); offset += 2;
const ΔelemCount = view.getUint8(offset++); // Δelem_count
const elemCount = view.getUint8(offset++); // elem_count
// Set i value (assuming all chunks share the same i)
if (result.i === null) result.i = iValue;
else if (result.i !== iValue) throw new Error('Multiple i values not supported');
// Read preface (element types)
const ΔelemTypes = [];
for (let i = 0; i < ΔelemCount; i++) {
if (offset >= bundleEnd) throw new Error('Incomplete Δelem types');
const typeByte = view.getUint8(offset++);
const baseCode = typeByte & 0x0F;
const incrCode = typeByte >> 4;
if (!codeToType[baseCode] || !codeToType[incrCode]) {
throw new Error(`Invalid type code in Δelem: ${typeByte}`);
}
ΔelemTypes.push({ baseType: codeToType[baseCode], incrType: codeToType[incrCode] });
}
const elemTypes = [];
for (let i = 0; i < elemCount; i++) {
if (offset >= bundleEnd) throw new Error('Incomplete elem types');
const typeCode = view.getUint8(offset++);
if (!codeToType[typeCode]) throw new Error(`Invalid type code in elem: ${typeCode}`);
elemTypes.push(codeToType[typeCode]);
}
// Initialize Δelems and elems arrays if first chunk
if (!result.Δelems.length && ΔelemCount > 0) {
result.Δelems = Array(ΔelemCount).fill().map(() => []);
}
if (!result.elems.length && elemCount > 0) {
result.elems = Array(elemCount).fill().map(() => []);
}
// Read initial values for Δelems
const initialValues = [];
for (const { baseType } of ΔelemTypes) {
if (offset + typeToBytes[baseType.name] > bundleEnd) {
throw new Error('Incomplete initial values');
}
initialValues.push(readTypedValue(view, offset, baseType));
offset += typeToBytes[baseType.name];
}
// Skip padding
while (offset % 4 !== 0) {
if (offset >= bundleEnd) throw new Error('Incomplete padding after initial values');
offset++;
}
// Reconstruct j values
for (let idx = 0; idx < count; idx++) {
result.j.push(j0 + idx * Δj);
}
// Read record data (non-interleaved)
for (let i = 0; i < ΔelemCount; i++) {
let current = initialValues[i];
const values = result.Δelems[i];
const incrType = ΔelemTypes[i].incrType;
const isBigInt = typeof current === 'bigint';
for (let idx = 0; idx < count; idx++) {
if (offset + typeToBytes[incrType.name] > bundleEnd) {
throw new Error('Incomplete Δelem data');
}
let delta = readTypedValue(view, offset, incrType);
if (idx === 0) {
values.push(isBigInt ? Number(current) : current);
} else {
if (isBigInt) {
delta = BigInt(delta);
current += delta;
values.push(Number(current));
} else {
current += delta;
values.push(current);
}
}
offset += typeToBytes[incrType.name];
}
}
for (let i = 0; i < elemCount; i++) {
const values = result.elems[i];
const type = elemTypes[i];
const isBigInt = type === BigInt64Array || type === BigUint64Array;
for (let idx = 0; idx < count; idx++) {
if (offset + typeToBytes[type.name] > bundleEnd) {
throw new Error('Incomplete elem data');
}
let value = readTypedValue(view, offset, type);
values.push(isBigInt ? Number(value) : value);
offset += typeToBytes[type.name];
}
}
// Skip padding
while (offset % 4 !== 0) {
if (offset >= bundleEnd) throw new Error('Incomplete padding after record data');
offset++;
}
}
}
return result;
}
function interleaved(binary) {
if (!(binary instanceof Uint8Array) || binary.length < 4) {
throw new Error('Invalid binary input');
}
const view = new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
let offset = 0;
// Initialize result (assuming single i value for simplicity; extend for multiple i values if needed)
const result = { i: null, j: [], Δelems: [], elems: [] };
// Process bundles
while (offset < binary.length) {
// Read bundle header
if (offset + 4 > binary.length) throw new Error('Incomplete bundle header');
const bundleHeader = view.getUint32(offset, true);
if ((bundleHeader & 0xFF) !== 0x1C) throw new Error('Invalid bundle marker');
const bundleLength = bundleHeader >> 8;
offset += 4;
const bundleEnd = offset + bundleLength;
if (bundleEnd > binary.length) throw new Error('Bundle length exceeds input size');
// Process chunks in bundle
while (offset < bundleEnd) {
// Read chunk header
if (offset + 12 > bundleEnd) throw new Error('Incomplete chunk header');
const chunkType = view.getUint8(offset);
if (chunkType !== 0x12) throw new Error(`Unsupported chunk type: ${chunkType}`);
offset += 1; // Skip udv
offset += 1;
const count = view.getUint16(offset, true); offset += 2;
if (count > 65535) throw new Error('Chunk count exceeds 65535');
const iValue = view.getUint16(offset, true); offset += 2;
const j0 = view.getUint16(offset, true); offset += 2;
const Δj = view.getInt16(offset, true); offset += 2;
const ΔelemCount = view.getUint8(offset++); // Δelem_count
const elemCount = view.getUint8(offset++); // elem_count
// Set i value (assuming all chunks share the same i)
if (result.i === null) result.i = iValue;
else if (result.i !== iValue) throw new Error('Multiple i values not supported');
// Read preface (element types)
const ΔelemTypes = [];
for (let i = 0; i < ΔelemCount; i++) {
if (offset >= bundleEnd) throw new Error('Incomplete Δelem types');
const typeByte = view.getUint8(offset++);
const baseCode = typeByte & 0x0F;
const incrCode = typeByte >> 4;
if (!codeToType[baseCode] || !codeToType[incrCode]) {
throw new Error(`Invalid type code in Δelem: ${typeByte}`);
}
ΔelemTypes.push({ baseType: codeToType[baseCode], incrType: codeToType[incrCode] });
}
const elemTypes = [];
for (let i = 0; i < elemCount; i++) {
if (offset >= bundleEnd) throw new Error('Incomplete elem types');
const typeCode = view.getUint8(offset++);
if (!codeToType[typeCode]) throw new Error(`Invalid type code in elem: ${typeCode}`);
elemTypes.push(codeToType[typeCode]);
}
// Initialize Δelems and elems arrays if first chunk
if (!result.Δelems.length && ΔelemCount > 0) {
result.Δelems = Array(ΔelemCount).fill().map(() => []);
}
if (!result.elems.length && elemCount > 0) {
result.elems = Array(elemCount).fill().map(() => []);
}
// Read initial values for Δelems
const initialValues = [];
for (const { baseType } of ΔelemTypes) {
if (offset + typeToBytes[baseType.name] > bundleEnd) {
throw new Error('Incomplete initial values');
}
initialValues.push(readTypedValue(view, offset, baseType));
offset += typeToBytes[baseType.name];
}
// Skip padding
while (offset % 4 !== 0) {
if (offset >= bundleEnd) throw new Error('Incomplete padding after initial values');
offset++;
}
// Reconstruct j values
for (let idx = 0; idx < count; idx++) {
result.j.push(j0 + idx * Δj);
}
// Read interleaved record data
for (let idx = 0; idx < count; idx++) {
// Read Δelems
for (let i = 0; i < ΔelemCount; i++) {
const values = result.Δelems[i];
const incrType = ΔelemTypes[i].incrType;
const isBigInt = typeof initialValues[i] === 'bigint';
if (offset + typeToBytes[incrType.name] > bundleEnd) {
throw new Error('Incomplete Δelem data');
}
let delta = readTypedValue(view, offset, incrType);
offset += typeToBytes[incrType.name];
if (idx === 0) {
values.push(isBigInt ? Number(initialValues[i]) : initialValues[i]);
} else {
if (isBigInt) {
delta = BigInt(delta);
initialValues[i] += delta;
values.push(Number(initialValues[i]));
} else {
initialValues[i] += delta;
values.push(initialValues[i]);
}
}
}
// Read elems
for (let i = 0; i < elemCount; i++) {
const values = result.elems[i];
const type = elemTypes[i];
const isBigInt = type === BigInt64Array || type === BigUint64Array;
if (offset + typeToBytes[type.name] > bundleEnd) {
throw new Error('Incomplete elem data');
}
let value = readTypedValue(view, offset, type);
values.push(isBigInt ? Number(value) : value);
offset += typeToBytes[type.name];
}
}
// Skip padding
while (offset % 4 !== 0) {
if (offset >= bundleEnd) throw new Error('Incomplete padding after record data');
offset++;
}
}
}
return result;
}
function readTypedValue(view, offset, type) {
switch (type) {
case Int8Array: return view.getInt8(offset);
case Uint8Array: return view.getUint8(offset);
case Int16Array: return view.getInt16(offset, true);
case Uint16Array: return view.getUint16(offset, true);
case Int32Array: return view.getInt32(offset, true);
case Uint32Array: return view.getUint32(offset, true);
case Float32Array: return view.getFloat32(offset, true);
case Float64Array: return view.getFloat64(offset, true);
case BigInt64Array: return view.getBigInt64(offset, true);
case BigUint64Array: return view.getBigUint64(offset, true);
default: throw new Error(`Unsupported type: ${type.name}`);
}
}
module.exports = { sequential, interleaved };

380
lib/modules/@dougal/binary/encode.js Normal file
View File

@@ -0,0 +1,380 @@
const typeToCode = {
Int8Array: 0,
Uint8Array: 1,
Int16Array: 2,
Uint16Array: 3,
Int32Array: 4,
Uint32Array: 5,
Float32Array: 7, // Float16 not natively supported in JS, use Float32
Float64Array: 8,
BigInt64Array: 9,
BigUint64Array: 10
};
const typeToBytes = {
Int8Array: 1,
Uint8Array: 1,
Int16Array: 2,
Uint16Array: 2,
Int32Array: 4,
Uint32Array: 4,
Float32Array: 4,
Float64Array: 8,
BigInt64Array: 8,
BigUint64Array: 8
};
function sequential(json, iGetter, jGetter, Δelems = [], elems = [], udv = 0) {
if (!Array.isArray(json) || !json.length) return new Uint8Array(0);
if (typeof iGetter !== 'function' || typeof jGetter !== 'function') throw new Error('i and j must be getter functions');
Δelems.forEach((elem, idx) => {
if (typeof elem.key !== 'function') throw new Error(`Δelems[${idx}].key must be a getter function`);
});
elems.forEach((elem, idx) => {
if (typeof elem.key !== 'function') throw new Error(`elems[${idx}].key must be a getter function`);
});
// Group records by i value
const groups = new Map();
for (const record of json) {
const iValue = iGetter(record);
if (iValue == null) throw new Error('Missing i value from getter');
if (!groups.has(iValue)) groups.set(iValue, []);
groups.get(iValue).push(record);
}
const maxBundleSize = 0xFFFFFF; // Max bundle length (24 bits)
const buffers = [];
// Process each group (i value)
for (const [iValue, records] of groups) {
// Sort records by j to ensure consistent order
records.sort((a, b) => jGetter(a) - jGetter(b));
const jValues = records.map(jGetter);
if (jValues.some(v => v == null)) throw new Error('Missing j value from getter');
// Split records into chunks based on Δj continuity
const chunks = [];
let currentChunk = [records[0]];
let currentJ0 = jValues[0];
let currentΔj = records.length > 1 ? jValues[1] - jValues[0] : 0;
for (let idx = 1; idx < records.length; idx++) {
const chunkIndex = chunks.reduce((sum, c) => sum + c.records.length, 0);
const expectedJ = currentJ0 + (idx - chunkIndex) * currentΔj;
if (jValues[idx] !== expectedJ || idx - chunkIndex >= 65536) {
chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
currentChunk = [records[idx]];
currentJ0 = jValues[idx];
currentΔj = idx + 1 < records.length ? jValues[idx + 1] - jValues[idx] : 0;
} else {
currentChunk.push(records[idx]);
}
}
if (currentChunk.length > 0) {
chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
}
// Calculate total size for all chunks in this group by simulating offsets
const chunkSizes = chunks.map(({ records: chunkRecords }) => {
if (chunkRecords.length > 65535) throw new Error(`Chunk size exceeds 65535 for i=${iValue}`);
let simulatedOffset = 0; // Relative to chunk start
simulatedOffset += 12; // Header
simulatedOffset += Δelems.length + elems.length; // Preface
simulatedOffset += Δelems.reduce((sum, e) => sum + typeToBytes[e.baseType.name], 0); // Initial values
while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after initial
simulatedOffset += chunkRecords.length * (
Δelems.reduce((sum, e) => sum + typeToBytes[e.incrType.name], 0) +
elems.reduce((sum, e) => sum + typeToBytes[e.type.name], 0)
); // Record data
while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after record
return simulatedOffset;
});
const totalChunkSize = chunkSizes.reduce((sum, size) => sum + size, 0);
// Start a new bundle if needed
const lastBundle = buffers[buffers.length - 1];
if (!lastBundle || lastBundle.offset + totalChunkSize > maxBundleSize) {
buffers.push({ offset: 4, buffer: null, view: null });
}
// Initialize DataView for current bundle
const currentBundle = buffers[buffers.length - 1];
if (!currentBundle.buffer) {
const requiredSize = totalChunkSize + 4;
currentBundle.buffer = new ArrayBuffer(requiredSize);
currentBundle.view = new DataView(currentBundle.buffer);
}
// Process each chunk
for (const { records: chunkRecords, j0, Δj } of chunks) {
const chunkSize = chunkSizes.shift();
// Ensure buffer is large enough
if (currentBundle.offset + chunkSize > currentBundle.buffer.byteLength) {
const newSize = currentBundle.offset + chunkSize;
const newBuffer = new ArrayBuffer(newSize);
new Uint8Array(newBuffer).set(new Uint8Array(currentBundle.buffer));
currentBundle.buffer = newBuffer;
currentBundle.view = new DataView(newBuffer);
}
// Write chunk header
let offset = currentBundle.offset;
currentBundle.view.setUint8(offset++, 0x11); // Chunk type
currentBundle.view.setUint8(offset++, udv); // udv
currentBundle.view.setUint16(offset, chunkRecords.length, true); offset += 2; // count
currentBundle.view.setUint16(offset, iValue, true); offset += 2; // i
currentBundle.view.setUint16(offset, j0, true); offset += 2; // j0
currentBundle.view.setInt16(offset, Δj, true); offset += 2; // Δj
currentBundle.view.setUint8(offset++, Δelems.length); // Δelem_count
currentBundle.view.setUint8(offset++, elems.length); // elem_count
// Write chunk preface (element types)
for (const elem of Δelems) {
const baseCode = typeToCode[elem.baseType.name];
const incrCode = typeToCode[elem.incrType.name];
currentBundle.view.setUint8(offset++, (incrCode << 4) | baseCode);
}
for (const elem of elems) {
currentBundle.view.setUint8(offset++, typeToCode[elem.type.name]);
}
// Write initial values for Δelems
for (const elem of Δelems) {
const value = elem.key(chunkRecords[0]);
if (value == null) throw new Error('Missing Δelem value from getter');
writeTypedValue(currentBundle.view, offset, value, elem.baseType);
offset += typeToBytes[elem.baseType.name];
}
// Pad to 4-byte boundary
while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
// Write record data (non-interleaved)
for (const elem of Δelems) {
let prev = elem.key(chunkRecords[0]);
for (let idx = 0; idx < chunkRecords.length; idx++) {
const value = idx === 0 ? 0 : elem.key(chunkRecords[idx]) - prev;
writeTypedValue(currentBundle.view, offset, value, elem.incrType);
offset += typeToBytes[elem.incrType.name];
prev = elem.key(chunkRecords[idx]);
}
}
for (const elem of elems) {
for (const record of chunkRecords) {
const value = elem.key(record);
if (value == null) throw new Error('Missing elem value from getter');
writeTypedValue(currentBundle.view, offset, value, elem.type);
offset += typeToBytes[elem.type.name];
}
}
// Pad to 4-byte boundary
while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
// Update bundle offset
currentBundle.offset = offset;
}
// Update bundle header
currentBundle.view.setUint32(0, 0x1C | ((currentBundle.offset - 4) << 8), true);
}
// Combine buffers into final Uint8Array
const finalLength = buffers.reduce((sum, b) => sum + b.offset, 0);
const result = new Uint8Array(finalLength);
let offset = 0;
for (const { buffer, offset: bundleOffset } of buffers) {
result.set(new Uint8Array(buffer, 0, bundleOffset), offset);
offset += bundleOffset;
}
return result;
}
function interleaved(json, iGetter, jGetter, Δelems = [], elems = [], udv = 0) {
if (!Array.isArray(json) || !json.length) return new Uint8Array(0);
if (typeof iGetter !== 'function' || typeof jGetter !== 'function') throw new Error('i and j must be getter functions');
Δelems.forEach((elem, idx) => {
if (typeof elem.key !== 'function') throw new Error(`Δelems[${idx}].key must be a getter function`);
});
elems.forEach((elem, idx) => {
if (typeof elem.key !== 'function') throw new Error(`elems[${idx}].key must be a getter function`);
});
// Group records by i value
const groups = new Map();
for (const record of json) {
const iValue = iGetter(record);
if (iValue == null) throw new Error('Missing i value from getter');
if (!groups.has(iValue)) groups.set(iValue, []);
groups.get(iValue).push(record);
}
const maxBundleSize = 0xFFFFFF; // Max bundle length (24 bits)
const buffers = [];
// Process each group (i value)
for (const [iValue, records] of groups) {
// Sort records by j to ensure consistent order
records.sort((a, b) => jGetter(a) - jGetter(b));
const jValues = records.map(jGetter);
if (jValues.some(v => v == null)) throw new Error('Missing j value from getter');
// Split records into chunks based on Δj continuity
const chunks = [];
let currentChunk = [records[0]];
let currentJ0 = jValues[0];
let currentΔj = records.length > 1 ? jValues[1] - jValues[0] : 0;
for (let idx = 1; idx < records.length; idx++) {
const chunkIndex = chunks.reduce((sum, c) => sum + c.records.length, 0);
const expectedJ = currentJ0 + (idx - chunkIndex) * currentΔj;
if (jValues[idx] !== expectedJ || idx - chunkIndex >= 65536) {
chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
currentChunk = [records[idx]];
currentJ0 = jValues[idx];
currentΔj = idx + 1 < records.length ? jValues[idx + 1] - jValues[idx] : 0;
} else {
currentChunk.push(records[idx]);
}
}
if (currentChunk.length > 0) {
chunks.push({ records: currentChunk, j0: currentJ0, Δj: currentΔj });
}
// Calculate total size for all chunks in this group by simulating offsets
const chunkSizes = chunks.map(({ records: chunkRecords }) => {
if (chunkRecords.length > 65535) throw new Error(`Chunk size exceeds 65535 for i=${iValue}`);
let simulatedOffset = 0; // Relative to chunk start
simulatedOffset += 12; // Header
simulatedOffset += Δelems.length + elems.length; // Preface
simulatedOffset += Δelems.reduce((sum, e) => sum + typeToBytes[e.baseType.name], 0); // Initial values
while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after initial
simulatedOffset += chunkRecords.length * (
Δelems.reduce((sum, e) => sum + typeToBytes[e.incrType.name], 0) +
elems.reduce((sum, e) => sum + typeToBytes[e.type.name], 0)
); // Interleaved record data
while (simulatedOffset % 4 !== 0) simulatedOffset++; // Pad after record
return simulatedOffset;
});
const totalChunkSize = chunkSizes.reduce((sum, size) => sum + size, 0);
// Start a new bundle if needed
const lastBundle = buffers[buffers.length - 1];
if (!lastBundle || lastBundle.offset + totalChunkSize > maxBundleSize) {
buffers.push({ offset: 4, buffer: null, view: null });
}
// Initialize DataView for current bundle
const currentBundle = buffers[buffers.length - 1];
if (!currentBundle.buffer) {
const requiredSize = totalChunkSize + 4;
currentBundle.buffer = new ArrayBuffer(requiredSize);
currentBundle.view = new DataView(currentBundle.buffer);
}
// Process each chunk
for (const { records: chunkRecords, j0, Δj } of chunks) {
const chunkSize = chunkSizes.shift();
// Ensure buffer is large enough
if (currentBundle.offset + chunkSize > currentBundle.buffer.byteLength) {
const newSize = currentBundle.offset + chunkSize;
const newBuffer = new ArrayBuffer(newSize);
new Uint8Array(newBuffer).set(new Uint8Array(currentBundle.buffer));
currentBundle.buffer = newBuffer;
currentBundle.view = new DataView(newBuffer);
}
// Write chunk header
let offset = currentBundle.offset;
currentBundle.view.setUint8(offset++, 0x12); // Chunk type
currentBundle.view.setUint8(offset++, udv); // udv
currentBundle.view.setUint16(offset, chunkRecords.length, true); offset += 2; // count
currentBundle.view.setUint16(offset, iValue, true); offset += 2; // i
currentBundle.view.setUint16(offset, j0, true); offset += 2; // j0
currentBundle.view.setInt16(offset, Δj, true); offset += 2; // Δj
currentBundle.view.setUint8(offset++, Δelems.length); // Δelem_count
currentBundle.view.setUint8(offset++, elems.length); // elem_count
// Write chunk preface (element types)
for (const elem of Δelems) {
const baseCode = typeToCode[elem.baseType.name];
const incrCode = typeToCode[elem.incrType.name];
currentBundle.view.setUint8(offset++, (incrCode << 4) | baseCode);
}
for (const elem of elems) {
currentBundle.view.setUint8(offset++, typeToCode[elem.type.name]);
}
// Write initial values for Δelems
for (const elem of Δelems) {
const value = elem.key(chunkRecords[0]);
if (value == null) throw new Error('Missing Δelem value from getter');
writeTypedValue(currentBundle.view, offset, value, elem.baseType);
offset += typeToBytes[elem.baseType.name];
}
// Pad to 4-byte boundary
while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
// Write interleaved record data
const prevValues = Δelems.map(elem => elem.key(chunkRecords[0]));
for (let idx = 0; idx < chunkRecords.length; idx++) {
// Write Δelems increments
for (let i = 0; i < Δelems.length; i++) {
const elem = Δelems[i];
const value = idx === 0 ? 0 : elem.key(chunkRecords[idx]) - prevValues[i];
writeTypedValue(currentBundle.view, offset, value, elem.incrType);
offset += typeToBytes[elem.incrType.name];
prevValues[i] = elem.key(chunkRecords[idx]);
}
// Write elems
for (const elem of elems) {
const value = elem.key(chunkRecords[idx]);
if (value == null) throw new Error('Missing elem value from getter');
writeTypedValue(currentBundle.view, offset, value, elem.type);
offset += typeToBytes[elem.type.name];
}
}
// Pad to 4-byte boundary
while (offset % 4 !== 0) currentBundle.view.setUint8(offset++, 0);
// Update bundle offset
currentBundle.offset = offset;
}
// Update bundle header
currentBundle.view.setUint32(0, 0x1C | ((currentBundle.offset - 4) << 8), true);
}
// Combine buffers into final Uint8Array
const finalLength = buffers.reduce((sum, b) => sum + b.offset, 0);
const result = new Uint8Array(finalLength);
let offset = 0;
for (const { buffer, offset: bundleOffset } of buffers) {
result.set(new Uint8Array(buffer, 0, bundleOffset), offset);
offset += bundleOffset;
}
return result;
}
function writeTypedValue(view, offset, value, type) {
switch (type) {
case Int8Array: view.setInt8(offset, value); break;
case Uint8Array: view.setUint8(offset, value); break;
case Int16Array: view.setInt16(offset, value, true); break;
case Uint16Array: view.setUint16(offset, value, true); break;
case Int32Array: view.setInt32(offset, value, true); break;
case Uint32Array: view.setUint32(offset, value, true); break;
case Float32Array: view.setFloat32(offset, value, true); break;
case Float64Array: view.setFloat64(offset, value, true); break;
case BigInt64Array: view.setBigInt64(offset, BigInt(value), true); break;
case BigUint64Array: view.setBigUint64(offset, BigInt(value), true); break;
default: throw new Error(`Unsupported type: ${type.name}`);
}
}
module.exports = { sequential, interleaved };

139
lib/modules/@dougal/binary/index.js Normal file
View File

@@ -0,0 +1,139 @@
/** Binary encoder
*
* This module encodes scalar data from a grid-like source
* into a packed binary format for bandwidth efficiency and
* speed of access.
*
* Data are indexed by i & j values, with "i" being constant
* (e.g., a sequence or line number) and "j" expected to change
* by a constant, linear amount (e.g., point numbers). All data
* from consecutive "j" values will be encoded as a single array
* (or series of arrays if multiple values are encoded).
* If there is a jump in the "j" progression, a new "chunk" will
* be started with a new array (or series of arrays).
*
* Multiple values may be encoded per (i, j) pair, using any of
* the types supported by JavaScript's TypedArray except for
* Float16 and Uint8Clamped. Each variable can be encoded with
* a different size.
*
* Values may be encoded directly or as deltas from an initial
* value. The latter is particularly efficient when dealing with
* monotonically incrementing data, such as timestamps.
*
* The conceptual packet format for sequentially encoded data
* looks like this:
*
* <msg-type> <count: x> <i> <j0> <Δj>
*
* <Δelement_count: y>
* <element_count: z>
*
* <Δelement_1_type_base> … <Δelement_y_type_base>
* <Δelement_1_type_incr> … <Δelement_y_type_incr>
* <elem_1_type> … <elem_z_type>
*
* <Δelement_1_first> … <Δelement_z_first>
*
* <Δelem_1_0> … <Δelem_1_x>
* …
* <Δelem_y_0> … <Δelem_y_x>
* <elem_1_0> … <elem_1_x>
* …
* <elem_z_0> … <elem_z_x>
*
*
* The conceptual packet format for interleaved encoded data
* looks like this:
*
*
* <msg-type> <count: x> <i> <j0> <Δj>
*
* <Δelement_count: y>
* <element_count: z>
*
* <Δelement_1_type_base> … <Δelement_y_type_base>
* <Δelement_1_type_incr> … <Δelement_y_type_incr>
* <elem_1_type> … <elem_z_type>
*
* <Δelement_1_first> … <Δelement_y_first>
*
* <Δelem_1_0> <Δelem_2_0> … <Δelem_y_0> <elem_1_0> <elem_2_0> … <elem_z_0>
* <Δelem_1_1> <Δelem_2_1> … <Δelem_y_1> <elem_1_1> <elem_2_1> … <elem_z_1>
* …
* <Δelem_1_x> <Δelem_2_x> … <Δelem_y_x> <elem_1_x> <elem_2_x> … <elem_z_x>
*
*
* Usage example:
*
* json = [
* {
* sequence: 7,
* sailline: 5354,
* line: 5356,
* point: 1068,
* tstamp: 1695448704372,
* objrefraw: 3,
* objreffinal: 4
* },
* {
* sequence: 7,
* sailline: 5354,
* line: 5352,
* point: 1070,
* tstamp: 1695448693612,
* objrefraw: 2,
* objreffinal: 3
* },
* {
* sequence: 7,
* sailline: 5354,
* line: 5356,
* point: 1072,
* tstamp: 1695448684624,
* objrefraw: 3,
* objreffinal: 4
* }
* ];
*
* deltas = [
* { key: el => el.tstamp, baseType: BigUint64Array, incrType: Int16Array }
* ];
*
* elems = [
* { key: el => el.objrefraw, type: Uint8Array },
* { key: el => el.objreffinal, type: Uint8Array }
* ];
*
* i = el => el.sequence;
*
* j = el => el.point;
*
* bundle = encode(json, i, j, deltas, elems);
*
* // bundle:
*
* Uint8Array(40) [
* 36, 0, 0, 28, 17, 0, 3, 0, 7, 0,
* 44, 4, 2, 0, 1, 2, 42, 1, 1, 116,
* 37, 158, 192, 138, 1, 0, 0, 0, 0, 0,
* 248, 213, 228, 220, 3, 2, 3, 4, 3, 4
* ]
*
* decode(bundle);
*
* {
* i: 7,
* j: [ 1068, 1070, 1072 ],
* 'Δelems': [ [ 1695448704372, 1695448693612, 1695448684624 ] ],
* elems: [ [ 3, 2, 3 ], [ 4, 3, 4 ] ]
* }
*
*/
module.exports = {
encode: {...require('./encode')},
decode: {...require('./decode')},
...require('./classes')
};

12
lib/modules/@dougal/binary/package.json Normal file
View File

@@ -0,0 +1,12 @@
{
"name": "@dougal/binary",
"version": "1.0.0",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": ""
}