dougal-software/lib/www/server/lib/qc/index.js

const fs = require('fs');
const YAML = require('yaml');
const vm = require('vm');
const Cursor = require('pg-cursor');
const { pool, setSurvey, transaction, fetchRow } = require('../db/connection')
const { project, sequence, configuration, info } = require('../db')
const flattenQCDefinitions = require('./flatten');
const { projectHash, sequenceHash } = require('./last-modified');

const { runShotsQC, saveShotsQC } = require('./shots');
const { runSequenceQCs, saveSequenceQCs } = require('./sequences');

/** Return true if the user has requested force running
 * a QC that would normally not be scheduled to be run.
 *
 * This is done via the DOUGAL_FORCE_QC environment
 * variable. The format is as follows:
 *
 * DOUGAL_FORCE_QC="project-id [sequence1 sequence2 … sequenceN]"
 *
 * A value of "project-id" re-runs QCs for all sequences of that
 * project, whereas specifying sequences to be run affects only
 * those sequences.
 */
function forceQC (projectId, sequenceNumber) {
	if (process.env.DOUGAL_FORCE_QC) {
		const [force_projectID, ...force_sequences ] = process.env.DOUGAL_FORCE_QC.split(/\s+/);
		if (projectId == force_projectID) {
			if (!sequenceNumber) {
				return true;
			} else if (!force_sequences.length) {
				return true;
			} else {
				return force_sequences.map(i => Number(i)).some(i => i == sequenceNumber);
			}
		}
	}
	return false;
}

async function getProjectQCConfig (projectId) {
	console.log("getProjectQCConfig");
	const qcConfig = await configuration.get(projectId, "qc");
	console.log("qcConfig", qcConfig);
	if (qcConfig?.definitions && qcConfig?.parameters) {
		const definitions =
			flattenQCDefinitions(YAML.parse(fs.readFileSync(qcConfig.definitions).toString()));
		const parameters = YAML.parse(fs.readFileSync(qcConfig.parameters).toString());

		return { definitions, parameters };
	}
}

async function main () {
	// Fetch list of projects
	console.log("GET PROJECTS");
	const projects = await project.list();
	console.log("PROJECTS", projects);

	for (const proj of projects) {
		const projectId = proj.pid;
		console.log("PROJECT ID", projectId);

		if (!project.archived) {
			const QCTstamp = new Date();

			const currentQCHash = await projectHash(projectId);
			const lastQCHash = await info.get(projectId, "qc/hash");
			console.log("projectHash", projectHash);
			console.log("lastQCHash", lastQCHash);

			if (currentQCHash != lastQCHash || forceQC(projectId)) {
				console.log("currentQCHash != lastQCHash", projectId, currentQCHash, lastQCHash);

				// Fetch definitions and parameters
				const { definitions, parameters } = await getProjectQCConfig(projectId);

				if (definitions && parameters) {
					console.log("PROJECT ID", projectId);
					console.log("definitions, parameters", !!definitions, !!parameters);
					const sequences = await sequence.list(projectId);
					console.log("SEQUENCES", sequences.length);

					const sequenceQCs = definitions.filter(i => i.iterate == "sequences" && !i.disabled);
					const shotQCs = definitions.filter(i => i.iterate == "shots" && !i.disabled);

					// Run shot QCs
					for (const seq of sequences) {
						const sequenceNumber = seq.sequence;
						const sequenceCurrentHash = await sequenceHash(projectId, sequenceNumber);
						const sequenceLastQCHash = seq.meta?.lastQCHash;
						console.log("sequenceCurrentHash", sequenceCurrentHash);
						console.log("sequenceLastQCHash", sequenceLastQCHash);

						if (sequenceCurrentHash != sequenceLastQCHash || forceQC(projectId, sequenceNumber)) {

							const results = await runShotsQC(projectId, sequenceNumber, shotQCs, parameters);

							await saveShotsQC(projectId, {[sequenceNumber]: results});
							await sequenceHash(projectId, sequenceNumber, sequenceCurrentHash);

						} else {
							console.log("NOT MODIFIED: SEQ", sequenceNumber);
						}
					}

					// Run sequence-wide QCs
					const results = await runSequenceQCs(projectId, sequenceQCs, parameters);
					await saveSequenceQCs(projectId, results);

					// Run survey-wide QCs TODO maybe

					await info.put(projectId, "qc", {updatedOn: QCTstamp, hash: currentQCHash}, {}, null);
				}
			}
		}
	}
	console.log("main done");
}

if (require.main === module) {
	main()
	.then(() => console.log("Done"))
	.catch( err => {
		console.error(err);
		process.exit(1);
	});
}