From 99ac082f00f7c0617eac231df79e418a5646f21a Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Fri, 29 Sep 2023 16:11:44 +0200 Subject: [PATCH 1/3] Use common naming convention both online and offline --- lib/www/server/lib/db/navdata/save.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/www/server/lib/db/navdata/save.js b/lib/www/server/lib/db/navdata/save.js index 55df5c6..215b8b3 100644 --- a/lib/www/server/lib/db/navdata/save.js +++ b/lib/www/server/lib/db/navdata/save.js @@ -70,9 +70,9 @@ async function getNearestOfflinePreplot (candidates) { if ("latitude" in candidates[0] && "longitude" in candidates[0]) { text = ` SELECT - '${c._schema}' AS _schema, + '${c.schema}' AS schema, ST_Distance(ST_Transform(ST_SetSRID(ST_MakePoint($1, $2), 4326), ST_SRID(geometry)), geometry) AS distance - FROM ${c._schema}.preplot_points + FROM ${c.schema}.preplot_points ORDER BY distance ASC LIMIT 1; `; @@ -80,9 +80,9 @@ async function getNearestOfflinePreplot (candidates) { } else if ("easting" in candidates[0] && "northing" in candidates[0]) { text = ` SELECT - '${c._schema}' AS _schema, + '${c.schema}' AS schema, ST_Distance(ST_SetSRID(ST_MakePoint($1, $2), ST_SRID(geometry)), geometry) AS distance - FROM ${c._schema}.preplot_points + FROM ${c.schema}.preplot_points ORDER BY distance ASC LIMIT 1; `; @@ -98,13 +98,13 @@ async function getNearestOfflinePreplot (candidates) { const results = []; for (const qry of queries) { const res = await client.query(qry.text, qry.values); - if (res.rows[0] && res.rows[0]._schema) { + if (res.rows[0] && res.rows[0].schema) { results.push(res.rows[0]); } } client.release(); - const _schema = results.sort( (a, b) => a.distance - b.distance).shift()?._schema; - return candidates.find(c => c._schema == _schema); + const schema = results.sort( (a, b) => a.distance - b.distance).shift()?.schema; + return candidates.find(c => c.schema == schema); } async function saveOnline (dataset, opts = {}) { @@ -292,7 +292,7 @@ async function save (navData, opts = {}) { const candidates = configs.map(c => Object.assign({}, navData, {_schema: c.schema})); const bestCandidate = await getNearestOfflinePreplot(candidates); if (bestCandidate) { - navData.payload._schema = bestCandidate._schema; + navData.payload._schema = bestCandidate.schema; last_tstamp = now; } } From 0e58b8fa5b3a120942d26b4d70e5d9095b4d9507 Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Fri, 29 Sep 2023 16:13:35 +0200 Subject: [PATCH 2/3] Refactor code to identify candidate schemas. As part of the refactoring, we took into account a slight payload format change (project configuration details are under the `data` attribute). --- lib/www/server/lib/db/navdata/save.js | 60 +++++++++++++++------------ 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/lib/www/server/lib/db/navdata/save.js b/lib/www/server/lib/db/navdata/save.js index 215b8b3..c6d16f6 100644 --- a/lib/www/server/lib/db/navdata/save.js +++ b/lib/www/server/lib/db/navdata/save.js @@ -211,6 +211,37 @@ async function saveOffline (navData, opts = {}) { client.release(); } +async function getCandidates (navData) { + + const configs = await getAllProjectConfigs(); + + // We just get the bits of interest: pattern and schema + const candidates = configs.map(c => { + if (!c?.data?.online?.line || c?.archived === true) { + return null; + } + + const p = c.data.online.line.pattern; // For short + + const rx = new RegExp(p.regex, p.flags); + const matches = navData.lineName.match(rx); + + if (!matches || ((matches.length+1) < p.captures.length)) { + return null; + } + + matches.shift(); // Get rid of the full matched text + const obj = Object.assign({}, navData, {schema: c.schema}); + p.captures.forEach( (k, i) => { + obj[k] = matches[i]; + }); + return obj; + }).filter(c => !!c); + DEBUG("Candidates: %j", candidates.map(c => c.schema)); + + return candidates; +} + async function save (navData, opts = {}) { const hasLatLon = ("latitude" in navData && "longitude" in navData); @@ -226,32 +257,7 @@ async function save (navData, opts = {}) { // So we have a lineName, see which projects match the line pattern. // For this we need to get all the project configs - const configs = await getAllProjectConfigs(); - - // We just get the bits of interest: pattern and schema - const candidates = configs.map(c => { - if (!(c && c.online && c.online.line)) { - return null; - } - - const p = c.online.line.pattern; // For short - - const rx = new RegExp(p.regex, p.flags); - const matches = navData.lineName.match(rx); - - if (!matches || ((matches.length+1) < p.captures.length)) { - return null; - } - - matches.shift(); // Get rid of the full matched text - const obj = Object.assign({}, navData, {schema: c.schema}); - p.captures.forEach( (k, i) => { - obj[k] = matches[i]; - }); - return obj; - }).filter(c => !!c); - DEBUG("Candidates: %j", candidates); -// console.log("CANDIDATES", candidates); + const candidates = await getCandidates(navData); if (candidates.length == 0) { // This is probably a test line, so we treat it as offline @@ -289,7 +295,7 @@ async function save (navData, opts = {}) { if (do_save) { const configs = await getAllProjectConfigs(); - const candidates = configs.map(c => Object.assign({}, navData, {_schema: c.schema})); + const candidates = await getCandidates(navData); const bestCandidate = await getNearestOfflinePreplot(candidates); if (bestCandidate) { navData.payload._schema = bestCandidate.schema; From ab58e578c939cbe139feef7a3cdab88f728150ba Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Fri, 29 Sep 2023 16:16:33 +0200 Subject: [PATCH 3/3] Use DEBUG library throughout --- .../server/events/handlers/detect-soleol.js | 24 +++++++++++++++---- lib/www/server/lib/db/navdata/save.js | 14 ++++++----- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/lib/www/server/events/handlers/detect-soleol.js b/lib/www/server/events/handlers/detect-soleol.js index 1622838..543e526 100644 --- a/lib/www/server/events/handlers/detect-soleol.js +++ b/lib/www/server/events/handlers/detect-soleol.js @@ -1,23 +1,24 @@ const { schema2pid } = require('../../lib/db/connection'); const { event } = require('../../lib/db'); +const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename); class DetectSOLEOL { /* Data may come much faster than we can process it, so we put it * in a queue and process it at our own pace. - * + * * The run() method fills the queue with the necessary data and then * calls processQueue(). - * + * * The processQueue() method looks takes the first two elements in * the queue and processes them if they are not already being taken * care of by a previous processQueue() call – this will happen when * data is coming in faster than it can be processed. - * + * * If the processQueue() call is the first to see the two bottommost * two elements, it will process them and, when finished, it will set * the `isPending` flag of the bottommost element to `false`, thus * letting the next call know that it has work to do. - * + * * If the queue was empty, run() will set the `isPending` flag of its * first element to a falsy value, thus bootstrapping the process. */ @@ -26,8 +27,10 @@ class DetectSOLEOL { queue = []; async processQueue () { + DEBUG("Queue length", this.queue.length) while (this.queue.length > 1) { if (this.queue[0].isPending) { + DEBUG("Queue busy"); setImmediate(() => this.processQueue()); return; } @@ -38,9 +41,15 @@ class DetectSOLEOL { const sequence = Number(cur._sequence); try { + DEBUG("Sequence", sequence); + // DEBUG("Previous", prev); + // DEBUG("Current", cur); if (prev.lineName == cur.lineName && prev._sequence == cur._sequence && prev.lineStatus != "online" && cur.lineStatus == "online" && sequence) { + INFO("Transition to ONLINE detected"); + // DEBUG(cur); + // DEBUG(prev); // console.log("TRANSITION TO ONLINE", prev, cur); // Check if there are already FSP, FGSP events for this sequence @@ -63,12 +72,17 @@ class DetectSOLEOL { } // console.log(projectId, payload); + INFO("Posting event", projectId, payload); await event.post(projectId, payload); } else { // A first shot point has been already entered in the log, // so we have nothing to do here. + INFO("FSP already in the log. Doing nothing"); } } else if (prev.lineStatus == "online" && cur.lineStatus != "online") { + INFO("Transition to OFFLINE detected"); + // DEBUG(cur); + // DEBUG(prev); // console.log("TRANSITION TO OFFLINE", prev, cur); // Check if there are already LSP, LGSP events for this sequence @@ -91,10 +105,12 @@ class DetectSOLEOL { } // console.log(projectId, payload); + INFO("Posting event", projectId, payload); await event.post(projectId, payload); } else { // A first shot point has been already entered in the log, // so we have nothing to do here. + INFO("LSP already in the log. Doing nothing"); } } // Processing of this shot has already been completed. diff --git a/lib/www/server/lib/db/navdata/save.js b/lib/www/server/lib/db/navdata/save.js index c6d16f6..f1ece0b 100644 --- a/lib/www/server/lib/db/navdata/save.js +++ b/lib/www/server/lib/db/navdata/save.js @@ -1,6 +1,6 @@ // FIXME This code is in painful need of refactoring -const { DEBUG } = require("DOUGAL_ROOT/debug")(__filename); +const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename); const { setSurvey, transaction, pool } = require('../connection'); let last_tstamp = 0; @@ -154,8 +154,8 @@ async function saveOnline (dataset, opts = {}) { } await transaction.commit(client); } catch (error) { - console.error("ONLINE DATA INSERT ERROR"); - console.error(error); + ERROR("ONLINE DATA INSERT ERROR"); + ERROR(error); await transaction.rollback(client); } finally { client.release(); @@ -249,10 +249,12 @@ async function save (navData, opts = {}) { const hasLinePoint = ("lineName" in navData && "point" in navData); if (!(hasLinePoint || hasLatLon || hasEastNorth)) { // This is of no interest to us - console.warning("Ignoring data without useful values", navData); + NOTICE("Ignoring data without useful values", navData); return; } + // DEBUG("navData", navData); + if (navData.online === true) { // So we have a lineName, see which projects match the line pattern. @@ -261,7 +263,7 @@ async function save (navData, opts = {}) { if (candidates.length == 0) { // This is probably a test line, so we treat it as offline - console.log("No match"); + WARNING("No match"); } else { if (candidates.length == 1) { // Only one candidate, associate with it @@ -277,7 +279,7 @@ async function save (navData, opts = {}) { await saveOnline(candidates.filter(c => c.schema == destinationSchema), opts); navData.payload._schema = destinationSchema; } else { - console.log("Nowhere to save to"); + WARNING("Nowhere to save to"); } }