Save online and offline real-time data to database

lib/www/server/lib/db/index.js

@@ -6,5 +6,6 @@ module.exports = {
 	event: require('./event'),
 	gis: require('./gis'),
 	label: require('./label'),
-	configuration: require('./configuration')
+	configuration: require('./configuration'),
+	navdata: require('./navdata')
 };

lib/www/server/lib/db/navdata/index.js

@@ -0,0 +1,5 @@
+
+module.exports = {
+	save: require('./save')
+};
+

lib/www/server/lib/db/navdata/save.js

@@ -0,0 +1,226 @@
+// FIXME This code is in painful need of refactoring
+ 
+const { setSurvey, transaction, pool } = require('../connection');
+
+async function getAllProjectConfigs () {
+	const client = await pool.connect();
+
+	const res0 = await client.query("SELECT schema FROM projects;");
+	const text = res0.rows.map(r => {
+		return `SELECT '${r.schema}' AS schema, data FROM ${r.schema}.file_data WHERE (data->>'archived')::boolean IS NOT true AND data->>'id' IS NOT NULL`;
+	}).join("\nUNION ALL ");
+
+	console.log(text);
+	const res1 = await client.query(text);
+	client.release();
+	return res1.rows.map(r => Object.assign(r.data, {schema: r.schema}));
+}
+
+async function getNearestPreplot (candidates) {
+
+	const pointsText = candidates.map(c => {
+		return `
+			SELECT '${c.schema}' AS schema, *
+			FROM ${c.schema}.preplot_points
+			WHERE line = ${c.line} AND point = ${c.point}
+		`;
+	}).join("\nUNION ");
+
+	let text, values;
+
+	if ("latitude" in candidates[0] && "longitude" in candidates[0]) {
+		text = `
+			WITH points AS (
+				${pointsText}
+			)
+			SELECT
+				*,
+				ST_Distance(ST_Transform(ST_SetSRID(ST_MakePoint($1, $2), 4326), ST_SRID(geometry)), geometry) AS distance
+			FROM points
+			ORDER BY distance ASC, schema DESC
+			LIMIT 1;
+		`;
+		values = [ candidates[0].longitude, candidates[0].latitude ];
+	} else if ("easting" in candidates[0] && "northing" in candidates[0]) {
+		text = `
+			WITH points AS (
+				${pointsText}
+			)
+			SELECT
+				*,
+				ST_Distance(ST_SetSRID(ST_MakePoint($1, $2), ST_SRID(geometry)), geometry) AS distance
+			FROM points
+			ORDER BY distance ASC, schema DESC
+			LIMIT 1;
+		`;
+		values = [ candidates[0].easting, candidates[0].northing ];
+	} else {
+		// Missing a position, shouldn't happen at this point
+		return;
+	}
+
+	const client = await pool.connect();
+	const res = await client.query(text, values);
+	client.release();
+	return res.rows[0].schema;
+}
+
+async function saveOnline (dataset) {
+
+	const client = await pool.connect();
+
+	try {
+		await transaction.begin(client);
+		for (const item of dataset) {
+
+			// Set schema
+			await client.query(`SET search_path TO ${item.schema},public;`);
+
+			// Add *online* pseudo-file
+			await client.query(`INSERT INTO files (path, hash) VALUES ('', '*online*') ON CONFLICT DO NOTHING;`);
+
+			// Add sequence if need be
+			await client.query(`
+				INSERT INTO raw_lines (sequence, line, incr)
+					VALUES ($1, $2, false)
+					ON CONFLICT DO NOTHING;
+			`, [item.sequence, item.line]);
+
+			// Add *online* to raw_lines_files
+			await client.query(`INSERT INTO raw_lines_files (sequence, hash) VALUES ($1, '*online*') ON CONFLICT DO NOTHING;`, [item.sequence]);
+
+			// Finally, add the actual shotpoint
+			// FIXME Use grid coordinates whenever possible
+			if (item.easting && item.northing) {
+				await client.query(`
+					INSERT INTO raw_shots
+						(sequence, line, point, objref, tstamp, geometry, hash)
+						VALUES ($1, $2, $3, $4, $5, ST_SetSRID(ST_MakePoint($6, $7), (SELECT (data->>'epsg')::integer AS epsg FROM file_data)), '*online*')
+						ON CONFLICT DO NOTHING;
+				`, [item.sequence, item.line, item.point, 0, item.tstamp, item.easting, item.northing]);
+			} else if (item.latitude && item.longitude) {
+				await client.query(`
+					INSERT INTO raw_shots
+						(sequence, line, point, objref, tstamp, geometry, hash)
+						VALUES ($1, $2, $3, $4, $5, ST_Transform(ST_SetSRID(ST_MakePoint($6, $7), 4326), (SELECT (data->>'epsg')::integer AS epsg FROM file_data)), '*online*')
+						ON CONFLICT DO NOTHING;
+				`, [item.sequence, item.line, item.point, 0, item.tstamp, item.longitude, item.latitude]);
+			} else {
+				throw new Error("Real time data has neither geographical nor grid positions");
+			}
+
+		}
+		await transaction.commit(client);
+	} catch (error) {
+		console.error("ONLINE DATA INSERT ERROR");
+		console.error(error);
+		await transaction.rollback(client);
+	} finally {
+		client.release();
+	}
+}
+
+async function saveOffline (navData) {
+	const client = await pool.connect();
+
+	if ("latitude" in navData && "longitude" in navData) {
+		const text = `
+			INSERT INTO real_time_inputs (tstamp, geometry, meta)
+			VALUES ($1, ST_SetSRID(ST_MakePoint($2, $3), 4326), $4);
+		`;
+
+		const values = [navData.tstamp, navData.longitude, navData.latitude, navData.payload];
+
+		await client.query(text, values)
+	} else {
+		const text = `
+			INSERT INTO real_time_inputs (tstamp, meta)
+			VALUES ($1, $2);
+		`;
+
+		const values = [navData.tstamp, navData.payload];
+
+		await client.query(text, values)
+	}
+
+	client.release();
+}
+
+async function save (navData, opts = {}) {
+
+	const hasLatLon = ("latitude" in navData && "longitude" in navData);
+	const hasEastNorth = ("easting" in navData && "northing" in navData);
+	const hasLinePoint = ("lineName" in navData && "point" in navData);
+	if (!(hasLinePoint || hasLatLon || hasEastNorth)) {
+		// This is of no interest to us
+		console.warning("Ignoring data without useful values", navData);
+		return;
+	}
+
+	if (navData.online === true) {
+
+		// So we have a lineName, see which projects match the line pattern.
+		// For this we need to get all the project configs
+		const configs = await getAllProjectConfigs();
+
+		// We just get the bits of interest: pattern and schema
+		const candidates = configs.map(c => {
+			if (!(c && c.online && c.online.line)) {
+				return null;
+			}
+
+			const p = c.online.line.pattern; // For short
+
+			const rx = new RegExp(p.regex, p.flags);
+			const matches = navData.lineName.match(rx);
+
+			if (!matches || ((matches.length+1) < p.captures.length)) {
+				return null;
+			}
+
+			matches.shift(); // Get rid of the full matched text
+			const obj = Object.assign({}, navData, {schema: c.schema});
+			p.captures.forEach( (k, i) => {
+				obj[k] = matches[i];
+			});
+			return obj;
+		}).filter(c => !!c);
+		console.log("CANDIDATES", candidates);
+
+		if (candidates.length == 0) {
+			// This is probably a test line, so we treat it as offline
+			console.log("No match");
+		} else if (candidates.length == 1) {
+			// Only one candidate, associate with it
+			console.log("Save into schema", candidates[0].match.schema);
+		} else {
+			// More than one candidate, go for the closest. If more than one active
+			// project with the same preplots, highest numbered schema.
+			console.log("Choose nearest");
+//
+			const destinationSchema = await getNearestPreplot(candidates);
+			if (destinationSchema) {
+				await saveOnline(candidates.filter(c => c.schema == destinationSchema));
+			} else {
+				console.log("Nowhere to save to");
+			}
+		}
+
+	} else {
+		console.log("Save offline");
+		await saveOffline(navData);
+	}
+
+// 	const client = await setSurvey(projectId);
+//
+// 	const text = `
+// 		SELECT *
+// 		FROM project_summary;
+// 	`;
+//
+// 	const res = await client.query(text);
+// 	client.release();
+// 	return res.rows[0];
+}
+
+module.exports = save;