dougal-software/sbin/upgrade-project-configurations-20231113.js

#!/usr/bin/node

const path = require('path');
const fs = require('fs');
const YAML = dougal_require("yaml");
const db = dougal_require("db");
const { deepSet } = dougal_require("utils");

function dougal_require(id) {
	try {
		return require(path.join(__dirname, "../lib/www/server/lib", id));
	} catch (err) {
		if (err.code == "MODULE_NOT_FOUND") {
			console.log("Trying alternative path");
			return require(path.join(__dirname, "../lib/www/server/node_modules", id));
		} else {
			console.error(err);
			throw err;
		}
	}
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/291
//

function check_asaqc (cfg) {
	if (!cfg.cloud?.asaqc?.id) {
		return apply_cloud_asaqc;
	}
}

function apply_cloud_asaqc (cfg) {
	const asaqc = cfg.asaqc;
	if (asaqc) {
		console.log("Applying ASAQC changes");
		deepSet(cfg, [ "cloud", "asaqc" ], asaqc);
	} else {
		console.log("ASAQC configuration not found. Will create empty ASAQC object");
		deepSet(cfg, [ "cloud", "asaqc" ], {
			id: null,
			imo: null,
			mmsi: null
		});
	}
	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/296
//

function check_asaqc_subscription_key (cfg) {
	if (!cfg.cloud?.asaqc?.subscriptionKey) {
		return apply_asaqc_subscription_key;
	}
}

function apply_asaqc_subscription_key (cfg) {
	console.log("Adding subscriptionKey to ASAQC configuration");
	const subscriptionKey = process.env.DOUGAL_ASAQC_SUBSCRIPTION_KEY;
	if (subscriptionKey) {
		deepSet(cfg, [ "cloud", "asaqc", "subscriptionKey" ] , subscriptionKey);
	} else {
		throw new Error("The ASAQC subscription key must be supplied via the DOUGAL_ASAQC_SUBSCRIPTION_KEY environment variable");
	}
	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/297
//

function check_online_line_name_info (cfg) {
	if (!cfg.online?.line?.lineNameInfo?.fields) {
		return apply_online_line_name_info;
	}
}

function apply_online_line_name_info (cfg) {
	console.log("Applying online line name info changes");
	let lineNameInfo = {
		example: null,
		fields: {
			line: {
				offset: null,
				length: 4,
				type: "int",
			},
			sequence: {
				offset: null,
				length: 3,
				type: "int"
			},
			incr: {
				offset: null,
				length: 2,
				type: "bool",
				enum: {}
			},
			attempt: {
				offset: null,
				length: 1,
				type: "int"
			}
		}
	};
	switch (process.env.HOST) {
		case "dougal04":
			lineNameInfo = {
				"example": "EQ22200-2213130-007",
				"fields": {
					"line": {
						"length": 4,
						"type": "int",
						"offset": 10
					},
					"sequence": {
						"length": 3,
						"type": "int",
						"offset": 16
					},
					"incr": {
						"enum": {
							"1": true,
							"2": false
						},
						"length": 1,
						"type": "bool",
						"offset": 8
					},
					"attempt": {
						"length": 1,
						"type": "int",
						"offset": 14
					},
					"file_no": {
						"length": 3,
						"type": "int",
						"offset": 20
					},
					"year": {
						"offset": 2,
						"length": 2,
						"type": "int"
					},
					"survey_type": {
						"enum": {
							"0": "Marine",
							"2": "OBS/PRM"
						},
						"offset": 4,
						"length": 1,
						"default": "Unknown",
						"type": "str"
					},
					"project_number": {
						"offset": 5,
						"length": 2,
						"type": "int"
					},
					"num_sources": {
						"enum": {
							"0": "2",
							"1": "1",
							"2": "3"
						},
						"offset": 9,
						"length": 1,
						"type": "int"
					}
				}
			}
			break;
		case "dougal03":
			// Don't know what they use
			break;
		case "dougal02":
		case "dougal01":
		default: // Includes dev servers
			lineNameInfo = {
				example: "1054282180S00000",
				fields: {
					line: {
						offset: 2,
						length: 4,
						type: "int",
					},
					sequence: {
						offset: 7,
						length: 3,
						type: "int"
					},
					incr: {
						offset: 0,
						length: 2,
						type: "bool",
						enum: {
							"10": true,
							"20": false
						}
					},
					attempt: {
						offset: 6,
						length: 1,
						type: "int"
					}
				}
			};
	}

	deepSet(cfg, [ "online", "line", "lineNameInfo" ], lineNameInfo);
	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/292
//

function check_raw_p111_line_name_info (cfg) {
	if (!cfg.raw?.p111?.lineNameInfo?.fields) {
		return apply_raw_p111_line_name_info;
	}
}

function apply_raw_p111_line_name_info (cfg) {
	console.log("Applying raw P1/11 name info changes");
	let lineNameInfo = {
		example: null,
		fields: {
			line: {
				offset: null,
				length: 4,
				type: "int",
			},
			sequence: {
				offset: null,
				length: 3,
				type: "int"
			},
			incr: {
				offset: null,
				length: 2,
				type: "bool",
				enum: {}
			},
			attempt: {
				offset: null,
				length: 1,
				type: "int"
			}
		}
	};
	switch (process.env.HOST) {
		case "dougal04":
			lineNameInfo = {
				"example": "EQ22200-2213130-007.000.P111",
				"fields": {
					"line": {
						"length": 4,
						"type": "int",
						"offset": 10
					},
					"sequence": {
						"length": 3,
						"type": "int",
						"offset": 16
					},
					"incr": {
						"enum": {
							"1": true,
							"2": false
						},
						"length": 1,
						"type": "bool",
						"offset": 8
					},
					"attempt": {
						"length": 1,
						"type": "int",
						"offset": 14
					},
					"file_no": {
						"length": 3,
						"type": "int",
						"offset": 20
					},
					"year": {
						"offset": 2,
						"length": 2,
						"type": "int"
					},
					"survey_type": {
						"enum": {
							"0": "Marine",
							"2": "OBS/PRM"
						},
						"offset": 4,
						"length": 1,
						"default": "Unknown",
						"type": "str"
					},
					"project_number": {
						"offset": 5,
						"length": 2,
						"type": "int"
					},
					"num_sources": {
						"enum": {
							"0": "2",
							"1": "1",
							"2": "3"
						},
						"offset": 9,
						"length": 1,
						"type": "int"
					}
				}
			}
			break;
		case "dougal03":
			// Don't know what they use
			break;
		case "dougal02":
		case "dougal01":
		default: // Includes dev servers
			lineNameInfo = {
				example: "1054282180S00000.000.p111",
				fields: {
					line: {
						offset: 2,
						length: 4,
						type: "int",
					},
					sequence: {
						offset: 7,
						length: 3,
						type: "int"
					},
					incr: {
						offset: 0,
						length: 2,
						type: "bool",
						enum: {
							"10": true,
							"20": false
						}
					},
					attempt: {
						offset: 6,
						length: 1,
						type: "int"
					}
				}
			};
	}

	deepSet(cfg, [ "raw", "p111", "lineNameInfo" ], lineNameInfo);
	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/293
//

function check_final_p111_line_name_info (cfg) {
	if (!cfg.final?.p111?.lineNameInfo?.fields) {
		return apply_final_p111_line_name_info;
	}
}

function apply_final_p111_line_name_info (cfg) {
	console.log("Applying final P1/11 name info changes");
	let lineNameInfo = {
		example: null,
		fields: {
			line: {
				offset: null,
				length: 4,
				type: "int",
			},
			sequence: {
				offset: null,
				length: 3,
				type: "int"
			},
			incr: {
				offset: null,
				length: 2,
				type: "bool",
				enum: {}
			},
			attempt: {
				offset: null,
				length: 1,
				type: "int"
			}
		}
	};
	switch (process.env.HOST) {
		case "dougal04":
			lineNameInfo = {
				"example": "EQ22200-2213130-007.000.P111",
				"fields": {
					"line": {
						"length": 4,
						"type": "int",
						"offset": 10
					},
					"sequence": {
						"length": 3,
						"type": "int",
						"offset": 16
					},
					"incr": {
						"enum": {
							"1": true,
							"2": false
						},
						"length": 1,
						"type": "bool",
						"offset": 8
					},
					"attempt": {
						"length": 1,
						"type": "int",
						"offset": 14
					},
					"file_no": {
						"length": 3,
						"type": "int",
						"offset": 20
					},
					"year": {
						"offset": 2,
						"length": 2,
						"type": "int"
					},
					"survey_type": {
						"enum": {
							"0": "Marine",
							"2": "OBS/PRM"
						},
						"offset": 4,
						"length": 1,
						"default": "Unknown",
						"type": "str"
					},
					"project_number": {
						"offset": 5,
						"length": 2,
						"type": "int"
					},
					"num_sources": {
						"enum": {
							"0": "2",
							"1": "1",
							"2": "3"
						},
						"offset": 9,
						"length": 1,
						"type": "int"
					}
				}
			}
			break;
		case "dougal03":
			// Don't know what they use
			break;
		case "dougal02":
		case "dougal01":
		default: // Includes dev servers
			lineNameInfo = {
				example: "1054282180S00000.000.p111",
				fields: {
					line: {
						offset: 2,
						length: 4,
						type: "int",
					},
					sequence: {
						offset: 7,
						length: 3,
						type: "int"
					},
					incr: {
						offset: 0,
						length: 2,
						type: "bool",
						enum: {
							"10": true,
							"20": false
						}
					},
					attempt: {
						offset: 6,
						length: 1,
						type: "int"
					}
				}
			};
	}

	deepSet(cfg, [ "final", "p111", "lineNameInfo" ], lineNameInfo);
	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/294
//

function check_smsrc_headers_glob_path (cfg) {
	if (!cfg.raw?.source?.smsrc?.header?.glob?.length) {
		return apply_smsrc_headers_glob_path;
	}
}

function apply_smsrc_headers_glob_path (cfg) {
	console.log("Copying Smartsource header glob and path values to new location");

	const globs = cfg?.raw?.smsrc?.globs;
	const paths = cfg?.raw?.smsrc?.paths;

	if (globs) {
		deepSet(cfg, [ "raw", "source", "smsrc", "header", "globs" ], globs);
	}

	if (paths) {
		deepSet(cfg, [ "raw", "source", "smsrc", "header", "paths" ], paths);
	}

	return cfg;
}



//
// https://gitlab.com/wgp/dougal/software/-/work_items/294
//

function check_smsrc_headers_line_name_info (cfg) {
	if (!cfg.raw?.source?.smsrc?.header?.lineNameInfo?.fields) {
		return apply_smsrc_headers_line_name_info;
	}
}

function apply_smsrc_headers_line_name_info (cfg) {
	console.log("Applying raw P1/11 name info changes");
	let lineNameInfo = {
		example: null,
		fields: {
			line: {
				offset: null,
				length: 4,
				type: "int",
			},
			sequence: {
				offset: null,
				length: 3,
				type: "int"
			},
			incr: {
				offset: null,
				length: 2,
				type: "bool",
				enum: {}
			},
			attempt: {
				offset: null,
				length: 1,
				type: "int"
			}
		}
	};
	switch (process.env.HOST) {
		case "dougal04":
			lineNameInfo = {
				"example": "EQ22200-2213130-007.000.P111",
				"fields": {
					"line": {
						"length": 4,
						"type": "int",
						"offset": 10
					},
					"sequence": {
						"length": 3,
						"type": "int",
						"offset": 16
					},
					"incr": {
						"enum": {
							"1": true,
							"2": false
						},
						"length": 1,
						"type": "bool",
						"offset": 8
					},
					"attempt": {
						"length": 1,
						"type": "int",
						"offset": 14
					},
					"file_no": {
						"length": 3,
						"type": "int",
						"offset": 20
					},
					"year": {
						"offset": 2,
						"length": 2,
						"type": "int"
					},
					"survey_type": {
						"enum": {
							"0": "Marine",
							"2": "OBS/PRM"
						},
						"offset": 4,
						"length": 1,
						"default": "Unknown",
						"type": "str"
					},
					"project_number": {
						"offset": 5,
						"length": 2,
						"type": "int"
					},
					"num_sources": {
						"enum": {
							"0": "2",
							"1": "1",
							"2": "3"
						},
						"offset": 9,
						"length": 1,
						"type": "int"
					}
				}
			}
			break;
		case "dougal03":
			// Don't know what they use
			break;
		case "dougal02":
		case "dougal01":
		default: // Includes dev servers
			lineNameInfo = {
				example: "1054282180S00000.HDR",
				fields: {
					line: {
						offset: 2,
						length: 4,
						type: "int",
					},
					sequence: {
						offset: 7,
						length: 3,
						type: "int"
					},
					incr: {
						offset: 0,
						length: 2,
						type: "bool",
						enum: {
							"10": true,
							"20": false
						}
					},
					attempt: {
						offset: 6,
						length: 1,
						type: "int"
					}
				}
			};
	}

	deepSet(cfg, [ "raw", "source", "smsrc", "header", "lineNameInfo" ], lineNameInfo);
	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/295
//

function check_smsrc_segy (cfg) {
	// We only do this on installations where we know there is, or there
	// might be, SEG-Y data available.
	const supported_hosts = [
		"dougal02",
		"dougal01"
	];
	if (supported_hosts.includes(process.env.HOST)) {
		if (!cfg.raw?.source?.smsrc?.segy?.lineNameInfo?.fields) {
			return apply_smsrc_segy;
		}
	}
}

function apply_smsrc_segy (cfg) {
	// We don't need to run a switch() for hosts here, since
	// we've already done that in check_smsrc_segy().

	// Use the paths for *.HDR files as a reference
	const paths = cfg.raw?.source?.smsrc?.header?.paths?.map( p =>
		path.join(path.dirname(p), "10 SEG-Y"));
	const globs = [ "**/*-hyd.sgy" ];
	const lineNameInfo = {
		"example": "1051460070S00000-hyd.sgy",
		"fields": {
			"sequence": {
				"length": 3,
				"type": "int",
				"offset": 7
			},
			"line": {
				"length": 4,
				"offset": 2
			}
		}
	};
	const segy = { paths, globs, lineNameInfo };

	deepSet(cfg, [ "raw", "source", "smsrc", "segy" ], segy);

	return cfg;
}


//
// https://gitlab.com/wgp/dougal/software/-/work_items/298
//

function check_preplots_fields (cfg) {
	if (cfg.preplots?.length) {
		const indices = [];
		for (const idx in cfg.preplots) {
			const preplot = cfg.preplots[idx];
			if (!preplot?.fields?.line_name) {
				indices.push(idx);
			}
		}
		if (indices.length) {
			return apply_preplots_fieldsλ(indices);
		}
	}
}

function apply_preplots_fieldsλ (indices) {
	function fix_preplot (preplot) {
		const names = preplot.format.names;
		const types = preplot.format.types;
		const widths = preplot.format.widths;
		const offsets_widths = widths.reduce ((acc, cur) => {
			if (cur < 0) {
				acc.p -= cur; // Advances the position by -cur
			} else {
				acc.f.push({offset: acc.p, width: cur});
				acc.p += cur;
			}
			return acc;
		}, {f: [], p: 0})

		const fields = {};
		names.forEach( (name, ι) => {
			const field = {
				type: types[ι],
				 ...offsets_widths[ι]
			};
			fields[name] = field;
		});
		preplot.fields = fields;
		return preplot;
	}

	return function apply_preplots_fields (cfg) {
		for (const idx of indices) {
			console.log("Fixing preplot", idx);
			const preplot = fix_preplot(cfg.preplots[idx]);
			cfg.preplots.splice(idx, 1, preplot);
		}
	}
}


/* Template for more upgrade actions

//
// https://gitlab.com/wgp/dougal/software/-/work_items/
//

function check_ (cfg) {
}

function apply_ (cfg) {
}

*/


const checkers = [
	check_asaqc,
	check_asaqc_subscription_key,
	check_online_line_name_info,
	check_raw_p111_line_name_info,
	check_final_p111_line_name_info,
	check_smsrc_headers_glob_path,
	check_smsrc_headers_line_name_info,
	check_smsrc_segy,
	check_preplots_fields
]

const now = new Date();
const tstamp = now.toISOString().substr(0, 19)+"Z";

function fnames(pid) {
	return {
		backup: `${pid}-configuration-${tstamp}.yaml`,
		upgrade: `NEW-${pid}-configuration-${tstamp}.yaml`
	};
}

function save_scripts (pid) {
	const cwd = process.cwd();
	const fn_backup = path.resolve(path.join(cwd, fnames(pid).backup));
	const fn_upgrade = path.resolve(path.join(cwd, fnames(pid).upgrade));

	console.log("Creating script to restore old / new configurations");
	const backup = `# Restore pre-upgrade configuration for ${pid}
curl -vs "http://localhost:3000/api/project/${pid}/configuration" -X PUT -H "Content-Type: application/yaml" --data-binary @${fn_backup}\n`;
	const upgrade = `# Restore post-upgrade configuration for ${pid}
curl -vs "http://localhost:3000/api/project/${pid}/configuration" -X PUT -H "Content-Type: application/yaml" --data-binary @${fn_upgrade}\n`;

	fs.writeFileSync(`restore-20231113-pre-${pid}.sh`, backup);
	fs.writeFileSync(`restore-20231113-post-${pid}.sh`, upgrade);
}

async function backup (pid, cfg) {
	const fname = fnames(pid).backup;
	console.log(`Backing up configuration for ${pid} as ${fname} into current directory`);
	const text = YAML.stringify(cfg);
	fs.writeFileSync(fname, text);
}

async function save_configuration (pid, cfg) {
	console.log("Saving configuration for", pid);

	console.log("Saving copy of NEW configuration to file");
	const fname = fnames(pid).upgrade;
	const text = YAML.stringify(cfg);
	fs.writeFileSync(fname, text);
	save_scripts(pid);

	//console.log("Uploading configuration to server");
	try {
		//await db.project.configuration.put(pid);
	} catch (err) {
		console.log("Configuration upload failed");
		console.error(err);
		throw err;
	}
}

async function upgrade_configuration (pid) {
	const configuration = await db.project.configuration.get(pid);

	console.log(`Checking configuration for ${configuration.id} (${configuration.schema})`);

	const appliers = checkers.map( checker => checker(configuration) ).filter( i => !!i );

	if (appliers.length) {
		console.log("Configuration needs changes.");
		await backup(pid, configuration);

		console.log("Applying changes");
		console.log(appliers);
		for (const applier of appliers) {
			applier(configuration);
		}

		await save_configuration(pid, configuration);
	}

}

async function main () {

	const cla = process.argv.slice(2).map(i => i.toLowerCase());

	function project_filter (project) {
		if (cla.length == 0)
			return true;
		return cla.includes(project.pid.toLowerCase());
	}

	const projects = (await db.project.get()).filter(project_filter);

	projects.sort( (a, b) =>
		a.pid > b.pid
			? 1
			: a.pid < b.pid
				? -1
				: 0);

	console.log(projects);

	for (const project of projects) {
		await upgrade_configuration(project.pid);
	}

	console.log("All done");
	process.exit(0);
}


main();