From 49b7747dedcd61b728bdc4109ba3a67792a463cb Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Thu, 17 Mar 2022 20:07:11 +0100 Subject: [PATCH 1/2] Remove *all* QC events when saving sequence results. When saving shot-by-shot results for a sequence, *all* existing QC events for that sequence will be removed first. We do this because otherwise we may end up with QC data for shots that no longer exist. Also, in the case that we have QCed based on raw data, QC results for shots which are not in the final data would stay around even though those shots are no longer valid. --- lib/www/server/lib/qc/shots/save-qc.js | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/lib/www/server/lib/qc/shots/save-qc.js b/lib/www/server/lib/qc/shots/save-qc.js index ba69d54..0c9748a 100644 --- a/lib/www/server/lib/qc/shots/save-qc.js +++ b/lib/www/server/lib/qc/shots/save-qc.js @@ -5,20 +5,16 @@ async function saveShotsQC (projectId, results) { const client = await setSurvey(projectId); await transaction.begin(client); - async function deleteQCEvent (sequence, shot, qc_id) { - // NOTE that we delete from event_log_full - // because the event is readonly and we can't otherwise - // modify it via event_log (though it'd be nice to) + async function deleteQCEvents (sequence) { const text = ` DELETE FROM event_log_full - WHERE - sequence = $1 AND point = $2 - AND meta->>'qc_id' = $3; + WHERE sequence = $1 AND meta ? 'qc_id'; `; -// console.log("DELETE QUERY", projectId, sequence, shot, qc_id); - return await client.query(text, [ sequence, shot, qc_id ]); + const values = [ sequence ]; + + return await client.query(text, values); } async function updateQCEvent (sequence, shot, qc_id, result) { @@ -37,11 +33,11 @@ async function saveShotsQC (projectId, results) { }; for (const sequence in results) { + // Remove *all* QC events for this sequence + await deleteQCEvents(sequence); for (const shot in results[sequence]) { for (const qc_id in results[sequence][shot]) { const result = results[sequence][shot][qc_id]; - // Remove any existing event for this QC - await deleteQCEvent(sequence, shot, qc_id); if (result !== true) { // `true` means QC passed. Otherwise expect string or array. // Add or replace an existing event for this QC await updateQCEvent(sequence, shot, qc_id, result); From 913606e7f16707352c669bc9e3f4f109e8a1248e Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Thu, 17 Mar 2022 20:10:26 +0100 Subject: [PATCH 2/2] Allow forcing QCs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QCs may be re-run for specific sequences or for a whole project by defining an environment variable, as follows: For an entire project: * DOUGAL_FORCE_QC="project-id" For specific sequences: * DOUGAL_FORCE_QC="project-id sequence1 sequence2 … sequenceN" --- lib/www/server/lib/qc/index.js | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/lib/www/server/lib/qc/index.js b/lib/www/server/lib/qc/index.js index 7b444bf..038b773 100644 --- a/lib/www/server/lib/qc/index.js +++ b/lib/www/server/lib/qc/index.js @@ -10,6 +10,34 @@ const { projectHash, sequenceHash } = require('./last-modified'); const { runShotsQC, saveShotsQC } = require('./shots'); const { runSequenceQCs, saveSequenceQCs } = require('./sequences'); +/** Return true if the user has requested force running + * a QC that would normally not be scheduled to be run. + * + * This is done via the DOUGAL_FORCE_QC environment + * variable. The format is as follows: + * + * DOUGAL_FORCE_QC="project-id [sequence1 sequence2 … sequenceN]" + * + * A value of "project-id" re-runs QCs for all sequences of that + * project, whereas specifying sequences to be run affects only + * those sequences. + */ +function forceQC (projectId, sequenceNumber) { + if (process.env.DOUGAL_FORCE_QC) { + const [force_projectID, ...force_sequences ] = process.env.DOUGAL_FORCE_QC.split(/\s+/); + if (projectId == force_projectID) { + if (!sequenceNumber) { + return true; + } else if (!force_sequences.length) { + return true; + } else { + return force_sequences.map(i => Number(i)).some(i => i == sequenceNumber); + } + } + } + return false; +} + async function getProjectQCConfig (projectId) { console.log("getProjectQCConfig"); const qcConfig = await configuration.get(projectId, "qc"); @@ -41,7 +69,7 @@ async function main () { console.log("projectHash", projectHash); console.log("lastQCHash", lastQCHash); - if (currentQCHash != lastQCHash) { + if (currentQCHash != lastQCHash || forceQC(projectId)) { console.log("currentQCHash != lastQCHash", projectId, currentQCHash, lastQCHash); // Fetch definitions and parameters @@ -64,7 +92,7 @@ async function main () { console.log("sequenceCurrentHash", sequenceCurrentHash); console.log("sequenceLastQCHash", sequenceLastQCHash); - if (sequenceCurrentHash != sequenceLastQCHash) { + if (sequenceCurrentHash != sequenceLastQCHash || forceQC(projectId, sequenceNumber)) { const results = await runShotsQC(projectId, sequenceNumber, shotQCs, parameters);