Aaltronav/dougal-software
1
0
Fork 0
mirror of https://gitlab.com/wgp/dougal/software.git synced 2025-12-06 10:27:09 +00:00
Code Issues Packages Projects Releases Activity

Refactor QC execution and results saving.

Browse Source 
The results are now saved as follows:

For shot QCs, failing tests result in an event being created in
the event_log table. The text of the event is the QC result message,
while the labels are as set in the QC definition. It is conventionally
expected that these include a `QC` label. The event `meta` contains a
`qc_id` attribute with the ID of the failing QC.

For sequences, failing tests result in a `meta` entry under `qc`, with
the QC ID as the key and the result message as the value.

Finally, the project's `info` table still has a `qc` key, but unlike
with the old code, which stored all the QC results in a huge object
under this key, now only the timestamp of the last time a QC was run on
this project is stored, as `{ "updatedOn": timestamp }`.

The QCs are launched by calling the main() function in /lib/qc/index.js.
This function will first check the timestamp of the files imported into
the project and only run QCs if any of the file timestamps are later
than `info.qc.updatedOn`. Likewise, for each sequence, the timestamp of
the files conforming that sequence is checked against
`info.qc.updatedOn` and only those which are newer are actually
processed. This cuts down the running time very considerably.

The logic now is much easier on memory too, as it doesn't load the
whole project at once into memory. Instead, shotpoint QCs are processed
first, and for this a cursor is used, fetching one shotpoint at a
time. Then the sequence QCs are run, also one sequence at a time
(fetched via an individual query touching the `sequences_summary` view,
rather than via a cursor; we reuse some of the lib/db functions here),
for each sequence all its shotpoints and a list of missing shots are
also fetched (via lib/db function reuse) and passed to the QC functions
as predefined variables.

The logic of the QC functions is also changed. Now they can return:

* If a QC passes, the function MUST return boolean `true`.

* If a QC fails, the function MAY return a string describing the nature
  of the failure, or in the case of an `iterate: sequence` type test,
  it may return an object with these attributes:

  - `remarks`: a string describing the nature of the failure;
  - `labels`: a set of labels to associate with this failure;
  - `shots`: a object in which each attribute denotes a shotpoint number
    and the value consists of either a string or an object with
`remarks` (string), `labels` (array of strings) attributes. This allows
us to add detail about which shotpoints exactly contribute to cause a
sequence-wide test failure (this may not be applicable to every
sequence-wide QC) and it's also a handy way to detect and insert events
for missing shots.

* For QCs which may give false positives, such as missing gun data, a
  new QC definition attribute is introduced: if `ignoreAllFailed` is
boolean `true` and all shots fail the test for a sequence, or all
sequences fail the test for a prospect, the results of the QC will be
ignored, as if the test had passed. This is mostly to deal with gun or
any other data that may be temporarily missing.
This commit is contained in:
D. Berge
2022-03-07 21:41:10 +01:00
parent 67f8b9c6dd
commit 14a2f57c8d
10 changed files with 528 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

473
lib/www/server/lib/qc.js
View File

@@ -1,473 +0,0 @@
const fs = require('fs');
const YAML = require('yaml');
const vm = require('vm');
const { pool, setSurvey, transaction } = require('./db/connection')
const { project, configuration } = require('./db')
/**
* Retrieves information for all shots useful for QCing.
*
* It retrieves the “best” version of each shot, i.e.,
* from the final line if available and if not from raw
* (which may be real-time data even). It returns *all*
* shots for the prospect.
*/
async function byShot (client) {
console.error("byShot");
const text = `
SELECT
CASE
WHEN fs.sequence IS NOT NULL THEN 'final'
WHEN fs.sequence IS NULL AND rsp.hash = '*online*' THEN 'real-time'
WHEN fs.sequence IS NULL AND rsp.hash <> '*online*' THEN 'raw'
END AS type,
ARRAY[sequence, point] _id,
rsp.sequence,
rsp.line,
rsp.point,
rsp.objref,
rsp.tstamp,
COALESCE(
fs.hash,
rsp.hash
) hash,
COALESCE(
ST_AsGeoJSON(ST_Transform(fs.geometry, 4326)),
ST_AsGeoJSON(ST_Transform(rsp.geometry, 4326))
) geometry,
COALESCE(
fse.error_i,
rse.error_i
) error_i,
COALESCE(
fse.error_j,
rse.error_j
) error_j,
ST_AsGeoJSON(rsp.pp_geometry) preplot_geometry,
ST_AsGeoJSON(rsp.geometry) raw_geometry,
ST_AsGeoJSON(fs.geometry) final_geometry,
rsp.pp_meta,
rsp.meta raw_meta,
fs.meta final_meta
FROM raw_shots_preplots rsp
INNER JOIN raw_shots_ij_error rse USING (sequence, point)
LEFT JOIN final_shots fs USING (sequence, point)
LEFT JOIN final_shots_ij_error fse USING (sequence, point)
WHERE
rsp.hash != '*online*'
--WHERE
-- NOT COALESCE(fs.meta, rsp.meta) ? 'qc'
-- OR COALESCE(fs.meta, rsp.meta) @? '$.qc.sbs'
ORDER BY sequence, point;
`;
const res = await client.query(text);
return res.rows;
}
/**
* Retrieves information for all sequences.
*
*/
async function bySequence (client) {
console.error("bySequence");
const text = `
SELECT sequence _id, 'raw' AS type, *,
EXISTS(SELECT 1
FROM raw_shots
WHERE sequence = ss.sequence
HAVING count(*) FILTER (WHERE meta ? 'smsrc') > 0
) has_smsrc_data
FROM sequences_summary ss
WHERE NOT EXISTS (
SELECT sequence
FROM raw_lines_files rlf
WHERE hash = '*online*' AND rlf.sequence = ss.sequence)
ORDER BY sequence;
`;
const res = await client.query(text);
return res.rows;
}
/**
* Retrieves all project preplots.
*
* It also includes a count of the number of times a given preplot
* has been acquired. May be useful for things such as checking
* overlaps.
*
*/
async function byPreplot (client) {
console.error("byPreplot");
const text = `
SELECT ARRAY[NULL, line, point] _id,
'preplot' AS type,
line,
point,
class,
ntba,
ST_AsGeoJSON(geometry) geometry,
meta,
count
FROM preplot_points pp
INNER JOIN preplot_points_count ppc USING (line, point)
ORDER BY line, point;
`;
const res = await client.query(text);
return res.rows;
}
/**
* Run QC checks.
*
* If called with a projectId, it runs the QC checks in that project.
* If called without a projectId, it runs QC checks for all projects.
*
* The optional parameters @a qc and @a parameters can be used to
* supply the test definitions and test parameters to be run. If not
* supplied, the function will look for those in YAML files, the path
* to which should be supplied in the survey configuration file under
* qc.definitions and qc.parameters respectively.
*
* Projects that have no QC definitions will not be QC'ed unless @a qc
* and @a parameters are explicitly provided.
*
*/
async function qcCheck(elementIn, elementOut, variables, parameters, opts = {}) {
if (elementIn.disabled) return;
console.error(elementIn.name);
elementOut.name = elementIn.name;
elementOut.iterate = elementIn.iterate || "shots";
elementOut.labels = elementIn.labels;
if ("check" in elementIn) {
elementOut.check = [];
elementOut.id = elementIn.id;
const rows = elementIn.iterate in variables
? variables[elementIn.iterate]
: variables.shots;
console.error(Object.keys(variables), elementIn.iterate, rows.length);
for (const currentItem of rows) {
currentItem._ = (k) => k.split(".").reduce((a, b) => typeof a != "undefined" ? a[b] : a, currentItem);
const ctx = {...variables, currentItem, parameters};
const res = vm.runInNewContext(elementIn.check, ctx);
if (opts.skipPassed && res === true) continue;
elementOut.check.push({
_id: currentItem._id,
type: currentItem.type,
results: res
});
}
}
if ("children" in elementIn) {
elementOut.children = [];
for (const child of elementIn.children) {
if (child.disabled)
continue;
const childOut = {};
elementOut.children.push(childOut);
await qcCheck(child, childOut, variables, parameters, opts);
}
}
}
async function runQCs (projectId, qc=undefined, parameters=undefined) {
if (!projectId) {
console.log("Getting projects list");
const projects = await project.list();
const results = [];
for (const project of projects) {
const cfg = await configuration.get(project.pid);
// FIXME We should have a proper call to get the project config
if (cfg.id && cfg.id.toLowerCase() == project.pid.toLowerCase()) {
if (cfg.archived == true) {
continue;
}
}
results.push({projectId: project.pid, results: await runQCs(project.pid)});
}
return results;
}
console.log("Project", projectId);
const client = await setSurvey(projectId);
const results = [];
try {
if (!qc || !parameters) {
console.log("Fetching QC definitions and parameters")
qcConfig = await configuration.get(projectId, "qc");
if (!qc && qcConfig && qcConfig.definitions) {
qc = YAML.parse(fs.readFileSync(qcConfig.definitions).toString())
}
if (!parameters && qcConfig && qcConfig.parameters) {
parameters = YAML.parse(fs.readFileSync(qcConfig.parameters).toString())
}
}
if (qc && qc.length) {
console.log("Got", qc.length, "top-level QC groups");
const shots = await byShot(client);
const sequences = await bySequence(client);
const preplots = await byPreplot(client);
for (const sequence of sequences) {
sequence.shots = shots.filter(shot => shot.sequence == sequence.sequence);
}
const variables = { sequences, shots, preplots };
for (elementIn of qc) {
if (elementIn.disabled)
continue;
const elementOut = {};
results.push(elementOut);
await qcCheck(elementIn, elementOut, variables, parameters, {skipPassed: true});
}
const text = `
INSERT INTO info (key, value)
VALUES ('qc', $1)
ON CONFLICT (key) DO UPDATE
SET value = EXCLUDED.value;
`;
await client.query(text, [{results, updatedOn: new Date()}]);
} else {
console.log("No QC definitions found or provided");
}
} catch (error) {
console.error(error);
throw error;
} finally {
client.release();
}
return results;
}
async function saveQCs(projectId, qc) {
console.log("saveQCs", projectId, qc.results.length, "top level branches");
const client = await setSurvey(projectId);
const updatedOn = qc.updatedOn || (new Date())
async function saveResult (result, id) {
try {
if (typeof result._id == 'number') {
// This is a sequence-wide result
const sequence = result._id;
// const table = result.type == 'final'
// ? 'final_lines'
// : 'raw_lines';
const text = `
UPDATE raw_lines
SET meta = jsonb_set(jsonb_set(meta, '{qc}', COALESCE(meta->'qc', '{}'), true), ('{qc,'||$1||'}')::text[], $2::jsonb, true)
WHERE sequence = $3
AND meta->'qc'->$1->'results' IS DISTINCT FROM ($2::jsonb)->'results';
`;
const values = [ id, {results: result.results, updatedOn}, sequence ];
await client.query(text, values);
} else if (result._id.length == 2) {
// console.log(id, "Save to seq.", result._id[0], "point", result._id[1], result.results);
const sequence = result._id[0];
const point = result._id[1];
// const table = result.type == 'final'
// ? 'final_shots'
// : 'raw_shots';
const text = `
UPDATE raw_shots
SET meta = jsonb_set(jsonb_set(meta, '{qc}', COALESCE(meta->'qc', '{}'), true), ('{qc,'||$1||'}')::text[], $2::jsonb, true)
WHERE sequence = $3 AND point = $4
AND meta->'qc'->$1->'results' IS DISTINCT FROM ($2::jsonb)->'results';
`;
const values = [ id, {results: result.results, updatedOn}, sequence, point ];
await client.query(text, values);
// Sequence + point
} else if (result._id.length == 3) {
// Null + point + line
console.log(id, "Save to line", result._id[2], "point", result._id[1], result.results);
} else {
console.log(id, "NOT SAVED", result._id);
}
} catch (err) {
console.error("saveResult ERROR", err);
throw err;
}
}
async function iterate (item) {
try {
if (item.check) {
if (item.id) {
for (const result of item.check) {
// Save this to the corresponding shotpoint / sequence
await saveResult(result, item.id);
}
} else {
console.log("Warning!", item.name, "Tests with no item.id will not be saved");
}
}
if (item.children) {
for (const child of item.children) {
await iterate(child);
}
}
} catch (err) {
throw err;
}
}
try {
transaction.begin(client);
for (const item of qc.results) {
await iterate(item);
}
transaction.commit(client);
console.log("Results committed");
} catch (err) {
transaction.rollback(client);
console.error("Error saving QC results");
throw err;
} finally {
console.log("Client released");
client.release();
}
}
async function purgeQCs(projectId, qc) {
console.log("purgeQCs", projectId, qc.results.length, "top level branches");
const client = await setSurvey(projectId);
const updatedOn = qc.updatedOn || (new Date())
const resultObjects = {
sequence: {},
point: {},
line: {}
};
function addTo(branch, key, value) {
// console.log("adding", branch, key, value)
if (!branch[key]) {
branch[key] = [];
}
branch[key].push(value);
}
function filterResults (item, id) {
// console.log("item", item);
if (item.check && item.id) {
for (const child of item.check) {
filterResults(child, item.id);
}
} else if (item._id && id) {
const sequence = item._id.length
? item._id[0]
: item._id;
const point = item._id.length >= 2
? item._id[1]
: undefined;
const line = item._id.length == 3
? item._id[2]
: undefined;
if (sequence && !point) {
addTo(resultObjects.sequence, id, sequence);
} else if (sequence && point) {
addTo(resultObjects.point, id, [sequence, point]);
} else if (point && line) {
addTo(resultObjects.line, id, [point, line]);
}
}
}
for (const item of qc.results) {
filterResults(item);
}
// console.log("resultObjects", resultObjects);
const queries = {
sequence: `
-- SELECT $1::text id, sequence
-- FROM raw_lines
UPDATE raw_lines
SET meta = meta #- ('{qc, ' || $1 || '}')::text[]
WHERE meta @? ('$.qc.' || $1)::jsonpath
AND NOT (sequence = ANY($2));
`,
point: `
-- SELECT $1::text id, sequence, point
-- FROM raw_shots
UPDATE raw_shots
SET meta = meta #- ('{qc, ' || $1 || '}')::text[]
WHERE meta @? ('$.qc.' || $1)::jsonpath
AND NOT ([sequence, point] = ANY($2));
`,
line: `
-- SELECT $1::text id, point, line
-- FROM preplot_points
UPDATE preplot_points
SET meta = meta #- ('{qc, ' || $1 || '}')::text[]
WHERE meta @? ('$.qc.' || $1)::jsonpath
AND NOT ([point, line] = ANY($2));
`
}
try {
console.log("resultObjects", resultObjects);
transaction.begin(client);
for (const type in resultObjects) {
const text = queries[type];
for (const id in resultObjects[type]) {
const values = [id, resultObjects[type][id]];
const res = await client.query(text, values);
console.log("id, values", id, values);
console.log("DATA", type, id, res.rows);
}
}
transaction.commit(client);
console.log("Results committed");
} catch (err) {
transaction.rollback(client);
console.error("Error purging QC results");
throw err;
} finally {
console.log("Client released");
client.release();
}
}
if (require.main === module) {
runQCs() //.then(res => console.log("res", res));
.then(res => Promise.all(
res.map(r => [
saveQCs(r.projectId, {results: r.results}),
purgeQCs(r.projectId, {results: r.results})
]).flat()
))
.then(res => console.log("Done"));
} else {
module.exports = {
byShot,
bySequence,
byPreplot,
setSurvey,
runQCs,
saveQCs,
purgeQCs
};
}

27
lib/www/server/lib/qc/flatten.js Normal file
View File

@@ -0,0 +1,27 @@
/** Convert QC definitions tree into a flat list.
*/
function flattenQCDefinitions(items, keywords=[], labels=[], disabled=false) {
const result = [];
if (items) {
for (const item of items) {
if (!item.children) {
result.push({
...item,
iterate: (item.iterate ?? "shots"),
labels: labels.concat(item.labels??[]),
disabled: (item.disabled ?? disabled)
});
} else {
const k = [...keywords, item.name];
const l = [...labels, ...(item.labels??[])];
const d = item.disabled ?? disabled
result.push(...flattenQCDefinitions(item.children, k, l, d))
}
}
}
return result;
}
module.exports = flattenQCDefinitions;

98
lib/www/server/lib/qc/index.js Normal file
View File

@@ -0,0 +1,98 @@
const fs = require('fs');
const YAML = require('yaml');
const vm = require('vm');
const Cursor = require('pg-cursor');
const { pool, setSurvey, transaction, fetchRow } = require('../db/connection')
const { project, sequence, configuration, info } = require('../db')
const flattenQCDefinitions = require('./flatten');
const { projectLastModified, sequenceLastModified } = require('./last-modified');
const { runShotsQC, saveShotsQC } = require('./shots');
const { runSequenceQCs, saveSequenceQCs } = require('./sequences');
async function getProjectQCConfig (projectId) {
console.log("getProjectQCConfig");
const qcConfig = await configuration.get(projectId, "qc");
console.log("qcConfig", qcConfig);
if (qcConfig?.definitions && qcConfig?.parameters) {
const definitions =
flattenQCDefinitions(YAML.parse(fs.readFileSync(qcConfig.definitions).toString()));
const parameters = YAML.parse(fs.readFileSync(qcConfig.parameters).toString());
return { definitions, parameters };
}
}
async function main () {
// Fetch list of projects
console.log("GET PROJECTS");
const projects = await project.list();
console.log("PROJECTS", projects);
for (const proj of projects) {
const projectId = proj.pid;
console.log("PROJECT ID", projectId);
if (!project.archived) {
const QCTstamp = new Date();
const projectTstamp = await projectLastModified(projectId);
const updatedOn = await info.get(projectId, "qc/updatedOn");
const lastQCTstamp = isNaN(new Date(updatedOn)) ? -Infinity : new Date(updatedOn);
console.log("QCTstamp", QCTstamp);
console.log("projectTstamp", projectTstamp);
console.log("lastQCTstamp", lastQCTstamp);
if (projectTstamp >= lastQCTstamp) {
console.log("projectTstamp >= lastQCTstamp", projectId, projectTstamp, lastQCTstamp, projectTstamp >= lastQCTstamp);
// Fetch definitions and parameters
const { definitions, parameters } = await getProjectQCConfig(projectId);
if (definitions && parameters) {
console.log("PROJECT ID", projectId);
console.log("definitions, parameters", !!definitions, !!parameters);
const sequences = await sequence.list(projectId);
console.log("SEQUENCES", sequences.length);
const sequenceQCs = definitions.filter(i => i.iterate == "sequences" && !i.disabled);
const shotQCs = definitions.filter(i => i.iterate == "shots" && !i.disabled);
// Run shot QCs
for (const seq of sequences) {
const sequenceNumber = seq.sequence;
const sequenceTstamp = await sequenceLastModified(projectId, sequenceNumber);
if (sequenceTstamp >= lastQCTstamp) {
const results = await runShotsQC(projectId, sequenceNumber, shotQCs, parameters);
await saveShotsQC(projectId, {[sequenceNumber]: results});
// console.log("Saved", sequenceNumber);
} else {
console.log("NOT MODIFIED: SEQ", sequenceNumber);
}
}
// Run sequence-wide QCs
const results = await runSequenceQCs(projectId, sequenceQCs, parameters);
await saveSequenceQCs(projectId, results);
// Run survey-wide QCs TODO maybe
await info.put(projectId, "qc", {updatedOn: QCTstamp}, {}, null);
}
}
}
}
console.log("main done");
}
if (require.main === module) {
main()
.then(() => console.log("Done"))
.catch( err => {
console.error(err);
process.exit(1);
});
}

42
lib/www/server/lib/qc/last-modified.js Normal file
View File

@@ -0,0 +1,42 @@
const { pool, setSurvey, transaction, fetchRow } = require('../db/connection')
async function projectLastModified (projectId) {
const client = await setSurvey(projectId);
const text = `
SELECT to_timestamp(max(greatest(hash[2]::numeric, hash[3]::numeric))) AS tstamp
FROM (
SELECT string_to_array(hash, ':') hash
FROM files
) AS h;
`;
const res = ((await client.query(text))?.rows ?? [])[0]?.tstamp;
await client.release();
return res;
}
async function sequenceLastModified (projectId, sequence) {
const client = await setSurvey(projectId);
const text = `
SELECT to_timestamp(max(greatest(hash[2]::numeric, hash[3]::numeric))) AS tstamp
FROM (
SELECT sequence, string_to_array(hash, ':') AS hash
FROM raw_shots
) AS h
GROUP BY sequence
HAVING sequence = $1;
`;
const values = [ sequence ];
const res = ((await client.query(text, values))?.rows ?? [])[0]?.tstamp;
await client.release();
return res;
}
module.exports = {
projectLastModified,
sequenceLastModified
};

6
lib/www/server/lib/qc/sequences/index.js Normal file
View File

@@ -0,0 +1,6 @@
module.exports = {
runSequenceQCs: require('./run-qc'),
saveSequenceQCs: require('./save-qc')
};

68
lib/www/server/lib/qc/sequences/run-qc.js Normal file
View File

@@ -0,0 +1,68 @@
const vm = require('vm');
const { project, sequence, configuration, info } = require('../../db')
/** Run sequence QCs for project
*
* @a projectId The project in which to run the QCs
* @a definitions The QC definitions, as a flat array
* @a parameters The QC parameters (variables)
*
* @a return results array
*/
async function runSequenceQCs (projectId, definitions, parameters) {
console.log("runSequenceQCs", projectId);
const results = {}; // The structure will be { sequence: { qc_id: test_result } }
const sequences = await sequence.list(projectId, {missing:true});
for (const currentItem of sequences) {
const sequenceNumber = currentItem.sequence;
currentItem._ = (k) => k.split(".").reduce((a, b) => typeof a != "undefined" ? a[b] : a, currentItem);
// These two objects are passed to runInNewContext so that they are
// available to the QC functions
const shotpoints = await sequence.get(projectId, sequenceNumber);
const missingShotpoints = currentItem.fsp_final && currentItem.lsp_final
? currentItem.missing_final
: currentItem.missing_raw;
// Let us run QCs
for (qc of definitions.filter(i => i.iterate == "sequences")) {
if (!qc.disabled) {
console.log("CURRENT ITEM", sequenceNumber, qc.id);
const variables = { shotpoints, missingShotpoints };
const ctx = {...variables, currentItem, parameters};
const res = vm.runInNewContext(qc.check, ctx);
console.log("COMING OUT", res);
if (!(sequenceNumber in results)) {
results[sequenceNumber] = {};
}
results[sequenceNumber][qc.id] = res === true ? res : {
remarks: (typeof res === "string" ? res : res.remarks),
shots: res.shots,
labels: qc.labels
};
}
}
for (qc of definitions.filter(i => i.iterate == "sequences")) {
if (qc.ignoreAllFailed) {
const allFailed = Object.values(results).map(i => i[qc.id]).every(i => i !== true);
if (allFailed) {
console.log("IGNORING ALL FAILED", qc.id);
for (const sequenceNumber in results) {
delete results[sequenceNumber][qc.id];
}
}
}
}
}
return results;
}
module.exports = runSequenceQCs;

121
lib/www/server/lib/qc/sequences/save-qc.js Normal file
View File

@@ -0,0 +1,121 @@
const { setSurvey, transaction } = require('../../db/connection')
async function saveSequenceQCs (projectId, results) {
const client = await setSurvey(projectId);
await transaction.begin(client);
async function deleteQCSequence (sequence, qc_id) {
console.log("deleteQCSequence", sequence, qc_id);
const text = `
UPDATE raw_lines
SET
meta = meta #- ARRAY[ 'qc', $2 ]
WHERE
sequence = $1;
`;
const values = [ sequence, qc_id ];
return await client.query(text, values);
}
async function deleteQCEvent (sequence, qc_id) {
console.log("deleteQCEvent", sequence, qc_id);
// NOTE that we delete from event_log_full
// because the event is readonly and we can't otherwise
// modify it via event_log (though it'd be nice to)
const text = `
DELETE
FROM event_log_full
WHERE
sequence = $1
AND meta->>'qc_id' = $2;
`;
return await client.query(text, [ sequence, qc_id ]);
}
async function updateQCSequence (sequence, qc_id, remarks, labels) {
console.log("updateQCSequence", sequence, qc_id, remarks);
const text = `
UPDATE raw_lines
SET
meta = jsonb_set(meta, '{qc}',
coalesce(meta->'qc', '{}'::jsonb) || jsonb_build_object($2::text, $3::text)
)
WHERE
sequence = $1
`;
// NOTE We ignore labels for now.
// The idea is that one day we will associate labels with
// sequences just like we associate labels with shotpoints
// in the event log.
const values = [ sequence, qc_id, remarks, /*labels*/ ];
return await client.query(text, values);
}
async function updateQCEvent (sequence, shot, qc_id, remarks, labels) {
console.log("updateQCEvent", sequence, shot, qc_id, remarks);
const text = `
INSERT INTO event_log (sequence, point, remarks, labels, meta)
SELECT $1, $2, $4, $5, $6
WHERE NOT EXISTS (
SELECT 1 FROM event_log WHERE sequence = $1 AND point = $2 AND meta->>'qc_id' = $3
);
`;
const values = [ sequence, shot, qc_id, remarks, labels, {qc_id, readonly: true} ];
return await client.query(text, values);
};
for (const sequence in results) {
for (const qc_id in results[sequence]) {
const result = results[sequence][qc_id];
await deleteQCSequence(sequence, qc_id);
// For shot QCs we get a pass/fail for every shot, but
// for sequence QCs we may only get individual shot results
// for failed shots. So what we do is remove all data related
// to this QC id from the event log, then we (re-)insert data
// for failed shots. Not ideal.
await deleteQCEvent(sequence, qc_id);
if (result !== true) {
const remarks = typeof result === "string"
? result
: result?.remarks;
const labels = result?.labels ?? [];
await updateQCSequence(sequence, qc_id, remarks, labels);
if (result?.shots) {
for (const shot in result.shots) {
const shotResult = result.shots[shot];
// `true` means QC passed. Otherwise expect string or object.
if (shotResult !== true) {
// Add or replace an existing event for this QC
const remarks = typeof shotResult === "string"
? shotResult
: shotResult?.remarks;
const labels = shotResult?.labels ?? [];
await updateQCEvent(sequence, shot, qc_id, remarks, labels);
}
}
}
}
}
console.log("Done sequence", sequence);
}
await transaction.commit(client);
await client.release();
}
module.exports = saveSequenceQCs;

6
lib/www/server/lib/qc/shots/index.js Normal file
View File

@@ -0,0 +1,6 @@
module.exports = {
runShotsQC: require('./run-qc'),
saveShotsQC: require('./save-qc')
};

101
lib/www/server/lib/qc/shots/run-qc.js Normal file
View File

@@ -0,0 +1,101 @@
const vm = require('vm');
const Cursor = require('pg-cursor');
const { pool, setSurvey, transaction, fetchRow } = require('../../db/connection')
/** Run shot QCs for project
*
* @a projectId The project in which to run the QCs
* @a sequence The sequence number in which to run the QCs
* @a definitions The QC definitions, as a flat array
* @a parameters The QC parameters (variables)
*
* @a return results array
*/
async function runShotQCs (projectId, sequence, definitions, parameters) {
const client = await setSurvey(projectId);
const results = {}; // The structure will be { point: { qc_id: test_result } }
const text = `
SELECT
CASE
WHEN fs.sequence IS NOT NULL THEN 'final'
WHEN fs.sequence IS NULL AND rsp.hash = '*online*' THEN 'real-time'
WHEN fs.sequence IS NULL AND rsp.hash <> '*online*' THEN 'raw'
END AS type,
rsp.sequence,
rsp.line,
rsp.point,
rsp.objref,
rsp.tstamp,
COALESCE(
fs.hash,
rsp.hash
) hash,
COALESCE(
ST_AsGeoJSON(ST_Transform(fs.geometry, 4326)),
ST_AsGeoJSON(ST_Transform(rsp.geometry, 4326))
) geometry,
COALESCE(
fse.error_i,
rse.error_i
) error_i,
COALESCE(
fse.error_j,
rse.error_j
) error_j,
ST_AsGeoJSON(rsp.pp_geometry) preplot_geometry,
ST_AsGeoJSON(rsp.geometry) raw_geometry,
ST_AsGeoJSON(fs.geometry) final_geometry,
rsp.pp_meta,
rsp.meta raw_meta,
fs.meta final_meta
FROM raw_shots_preplots rsp
INNER JOIN raw_shots_ij_error rse USING (sequence, point)
LEFT JOIN final_shots fs USING (sequence, point)
LEFT JOIN final_shots_ij_error fse USING (sequence, point)
WHERE
sequence = $1
ORDER BY sequence, point;
`;
const values = [ sequence ];
// Let us run shotpoint QCs
for (qc of definitions.filter(i => i.iterate == "shots")) {
if (!qc.disabled) {
const cursor = client.query(new Cursor(text, values));
let currentItem;
while (currentItem = await fetchRow(cursor)) {
currentItem._ = (k) => k.split(".").reduce((a, b) => typeof a != "undefined" ? a[b] : a, currentItem);
const point = currentItem.point;
const ctx = {currentItem, parameters}; // No variables for now
const res = vm.runInNewContext(qc.check, ctx);
if (!(point in results)) {
results[point] = {};
}
results[point][qc.id] = res === true ? res : {remarks: res, labels: qc.labels};
}
if (qc.ignoreAllFailed) {
const allFailed = Object.values(results).map(i => i[qc.id]).every(i => i !== true);
if (allFailed) {
console.info("IGNORING ALL FAILED", qc.id);
for (const point in results) {
delete results[point][qc.id];
}
}
}
cursor.close();
}
}
client.release();
return results;
}
module.exports = runShotQCs;

59
lib/www/server/lib/qc/shots/save-qc.js Normal file
View File

@@ -0,0 +1,59 @@
const { setSurvey, transaction } = require('../../db/connection')
async function saveShotsQC (projectId, results) {
const client = await setSurvey(projectId);
await transaction.begin(client);
async function deleteQCEvent (sequence, shot, qc_id) {
// NOTE that we delete from event_log_full
// because the event is readonly and we can't otherwise
// modify it via event_log (though it'd be nice to)
const text = `
DELETE
FROM event_log_full
WHERE
sequence = $1 AND point = $2
AND meta->>'qc_id' = $3;
`;
// console.log("DELETE QUERY", projectId, sequence, shot, qc_id);
return await client.query(text, [ sequence, shot, qc_id ]);
}
async function updateQCEvent (sequence, shot, qc_id, result) {
const text = `
INSERT INTO event_log (sequence, point, remarks, labels, meta)
SELECT $1, $2, $4, $5, $6
WHERE NOT EXISTS (
SELECT 1 FROM event_log WHERE sequence = $1 AND point = $2 AND meta->>'qc_id' = $3
);
`;
// console.log("INSERT QUERY", projectId, sequence, shot, qc_id, result);
const values = [ sequence, shot, qc_id, result.remarks, result.labels, {qc_id, readonly: true} ];
await client.query(text, values);
};
for (const sequence in results) {
for (const shot in results[sequence]) {
for (const qc_id in results[sequence][shot]) {
const result = results[sequence][shot][qc_id];
// Remove any existing event for this QC
await deleteQCEvent(sequence, shot, qc_id);
if (result !== true) { // `true` means QC passed. Otherwise expect string or array.
// Add or replace an existing event for this QC
await updateQCEvent(sequence, shot, qc_id, result);
}
}
}
console.log("Done sequence", sequence);
}
await transaction.commit(client);
await client.release();
}
module.exports = saveShotsQC;