From c0c395d9a049b0fd49348eea4150337736b78b29 Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Mon, 31 Aug 2020 13:44:43 +0200 Subject: [PATCH] Save online and offline real-time data to database --- lib/www/server/lib/db/index.js | 3 +- lib/www/server/lib/db/navdata/index.js | 5 + lib/www/server/lib/db/navdata/save.js | 226 +++++++++++++++++++++++++ 3 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 lib/www/server/lib/db/navdata/index.js create mode 100644 lib/www/server/lib/db/navdata/save.js diff --git a/lib/www/server/lib/db/index.js b/lib/www/server/lib/db/index.js index 247e8f3..5a65e15 100644 --- a/lib/www/server/lib/db/index.js +++ b/lib/www/server/lib/db/index.js @@ -6,5 +6,6 @@ module.exports = { event: require('./event'), gis: require('./gis'), label: require('./label'), - configuration: require('./configuration') + configuration: require('./configuration'), + navdata: require('./navdata') }; diff --git a/lib/www/server/lib/db/navdata/index.js b/lib/www/server/lib/db/navdata/index.js new file mode 100644 index 0000000..f166a13 --- /dev/null +++ b/lib/www/server/lib/db/navdata/index.js @@ -0,0 +1,5 @@ + +module.exports = { + save: require('./save') +}; + diff --git a/lib/www/server/lib/db/navdata/save.js b/lib/www/server/lib/db/navdata/save.js new file mode 100644 index 0000000..e36b3e8 --- /dev/null +++ b/lib/www/server/lib/db/navdata/save.js @@ -0,0 +1,226 @@ +// FIXME This code is in painful need of refactoring + +const { setSurvey, transaction, pool } = require('../connection'); + +async function getAllProjectConfigs () { + const client = await pool.connect(); + + const res0 = await client.query("SELECT schema FROM projects;"); + const text = res0.rows.map(r => { + return `SELECT '${r.schema}' AS schema, data FROM ${r.schema}.file_data WHERE (data->>'archived')::boolean IS NOT true AND data->>'id' IS NOT NULL`; + }).join("\nUNION ALL "); + + console.log(text); + const res1 = await client.query(text); + client.release(); + return res1.rows.map(r => Object.assign(r.data, {schema: r.schema})); +} + +async function getNearestPreplot (candidates) { + + const pointsText = candidates.map(c => { + return ` + SELECT '${c.schema}' AS schema, * + FROM ${c.schema}.preplot_points + WHERE line = ${c.line} AND point = ${c.point} + `; + }).join("\nUNION "); + + let text, values; + + if ("latitude" in candidates[0] && "longitude" in candidates[0]) { + text = ` + WITH points AS ( + ${pointsText} + ) + SELECT + *, + ST_Distance(ST_Transform(ST_SetSRID(ST_MakePoint($1, $2), 4326), ST_SRID(geometry)), geometry) AS distance + FROM points + ORDER BY distance ASC, schema DESC + LIMIT 1; + `; + values = [ candidates[0].longitude, candidates[0].latitude ]; + } else if ("easting" in candidates[0] && "northing" in candidates[0]) { + text = ` + WITH points AS ( + ${pointsText} + ) + SELECT + *, + ST_Distance(ST_SetSRID(ST_MakePoint($1, $2), ST_SRID(geometry)), geometry) AS distance + FROM points + ORDER BY distance ASC, schema DESC + LIMIT 1; + `; + values = [ candidates[0].easting, candidates[0].northing ]; + } else { + // Missing a position, shouldn't happen at this point + return; + } + + const client = await pool.connect(); + const res = await client.query(text, values); + client.release(); + return res.rows[0].schema; +} + +async function saveOnline (dataset) { + + const client = await pool.connect(); + + try { + await transaction.begin(client); + for (const item of dataset) { + + // Set schema + await client.query(`SET search_path TO ${item.schema},public;`); + + // Add *online* pseudo-file + await client.query(`INSERT INTO files (path, hash) VALUES ('', '*online*') ON CONFLICT DO NOTHING;`); + + // Add sequence if need be + await client.query(` + INSERT INTO raw_lines (sequence, line, incr) + VALUES ($1, $2, false) + ON CONFLICT DO NOTHING; + `, [item.sequence, item.line]); + + // Add *online* to raw_lines_files + await client.query(`INSERT INTO raw_lines_files (sequence, hash) VALUES ($1, '*online*') ON CONFLICT DO NOTHING;`, [item.sequence]); + + // Finally, add the actual shotpoint + // FIXME Use grid coordinates whenever possible + if (item.easting && item.northing) { + await client.query(` + INSERT INTO raw_shots + (sequence, line, point, objref, tstamp, geometry, hash) + VALUES ($1, $2, $3, $4, $5, ST_SetSRID(ST_MakePoint($6, $7), (SELECT (data->>'epsg')::integer AS epsg FROM file_data)), '*online*') + ON CONFLICT DO NOTHING; + `, [item.sequence, item.line, item.point, 0, item.tstamp, item.easting, item.northing]); + } else if (item.latitude && item.longitude) { + await client.query(` + INSERT INTO raw_shots + (sequence, line, point, objref, tstamp, geometry, hash) + VALUES ($1, $2, $3, $4, $5, ST_Transform(ST_SetSRID(ST_MakePoint($6, $7), 4326), (SELECT (data->>'epsg')::integer AS epsg FROM file_data)), '*online*') + ON CONFLICT DO NOTHING; + `, [item.sequence, item.line, item.point, 0, item.tstamp, item.longitude, item.latitude]); + } else { + throw new Error("Real time data has neither geographical nor grid positions"); + } + + } + await transaction.commit(client); + } catch (error) { + console.error("ONLINE DATA INSERT ERROR"); + console.error(error); + await transaction.rollback(client); + } finally { + client.release(); + } +} + +async function saveOffline (navData) { + const client = await pool.connect(); + + if ("latitude" in navData && "longitude" in navData) { + const text = ` + INSERT INTO real_time_inputs (tstamp, geometry, meta) + VALUES ($1, ST_SetSRID(ST_MakePoint($2, $3), 4326), $4); + `; + + const values = [navData.tstamp, navData.longitude, navData.latitude, navData.payload]; + + await client.query(text, values) + } else { + const text = ` + INSERT INTO real_time_inputs (tstamp, meta) + VALUES ($1, $2); + `; + + const values = [navData.tstamp, navData.payload]; + + await client.query(text, values) + } + + client.release(); +} + +async function save (navData, opts = {}) { + + const hasLatLon = ("latitude" in navData && "longitude" in navData); + const hasEastNorth = ("easting" in navData && "northing" in navData); + const hasLinePoint = ("lineName" in navData && "point" in navData); + if (!(hasLinePoint || hasLatLon || hasEastNorth)) { + // This is of no interest to us + console.warning("Ignoring data without useful values", navData); + return; + } + + if (navData.online === true) { + + // So we have a lineName, see which projects match the line pattern. + // For this we need to get all the project configs + const configs = await getAllProjectConfigs(); + + // We just get the bits of interest: pattern and schema + const candidates = configs.map(c => { + if (!(c && c.online && c.online.line)) { + return null; + } + + const p = c.online.line.pattern; // For short + + const rx = new RegExp(p.regex, p.flags); + const matches = navData.lineName.match(rx); + + if (!matches || ((matches.length+1) < p.captures.length)) { + return null; + } + + matches.shift(); // Get rid of the full matched text + const obj = Object.assign({}, navData, {schema: c.schema}); + p.captures.forEach( (k, i) => { + obj[k] = matches[i]; + }); + return obj; + }).filter(c => !!c); + console.log("CANDIDATES", candidates); + + if (candidates.length == 0) { + // This is probably a test line, so we treat it as offline + console.log("No match"); + } else if (candidates.length == 1) { + // Only one candidate, associate with it + console.log("Save into schema", candidates[0].match.schema); + } else { + // More than one candidate, go for the closest. If more than one active + // project with the same preplots, highest numbered schema. + console.log("Choose nearest"); +// + const destinationSchema = await getNearestPreplot(candidates); + if (destinationSchema) { + await saveOnline(candidates.filter(c => c.schema == destinationSchema)); + } else { + console.log("Nowhere to save to"); + } + } + + } else { + console.log("Save offline"); + await saveOffline(navData); + } + +// const client = await setSurvey(projectId); +// +// const text = ` +// SELECT * +// FROM project_summary; +// `; +// +// const res = await client.query(text); +// client.release(); +// return res.rows[0]; +} + +module.exports = save;