Merge branch '76-add-configuration-gui' into 'devel'

Resolve "Add configuration GUI"

Closes #294, #295, #296, #298, #76, #297, #129, #313, #312, #305, #264, #307, #303, #300, #301, #302, #290, #291, #292, and #293

See merge request wgp/dougal/software!17
This commit is contained in:
D. Berge
2025-07-09 18:11:50 +00:00
103 changed files with 11493 additions and 228 deletions

View File

@@ -0,0 +1,924 @@
#!/usr/bin/node
const path = require('path');
const fs = require('fs');
const YAML = dougal_require("yaml");
const db = dougal_require("db");
const { deepSet } = dougal_require("utils");
function dougal_require(id) {
try {
return require(path.join(__dirname, "../lib/www/server/lib", id));
} catch (err) {
if (err.code == "MODULE_NOT_FOUND") {
console.log("Trying alternative path");
return require(path.join(__dirname, "../lib/www/server/node_modules", id));
} else {
console.error(err);
throw err;
}
}
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/291
//
function check_asaqc (cfg) {
if (!cfg.cloud?.asaqc?.id) {
return apply_cloud_asaqc;
}
}
function apply_cloud_asaqc (cfg) {
const asaqc = cfg.asaqc;
if (asaqc) {
console.log("Applying ASAQC changes");
deepSet(cfg, [ "cloud", "asaqc" ], asaqc);
} else {
console.log("ASAQC configuration not found. Will create empty ASAQC object");
deepSet(cfg, [ "cloud", "asaqc" ], {
id: null,
imo: null,
mmsi: null
});
}
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/296
//
function check_asaqc_subscription_key (cfg) {
if (!cfg.cloud?.asaqc?.subscriptionKey) {
return apply_asaqc_subscription_key;
}
}
function apply_asaqc_subscription_key (cfg) {
console.log("Adding subscriptionKey to ASAQC configuration");
const subscriptionKey = process.env.DOUGAL_ASAQC_SUBSCRIPTION_KEY;
if (subscriptionKey) {
deepSet(cfg, [ "cloud", "asaqc", "subscriptionKey" ] , subscriptionKey);
} else {
throw new Error("The ASAQC subscription key must be supplied via the DOUGAL_ASAQC_SUBSCRIPTION_KEY environment variable");
}
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/297
//
function check_online_line_name_info (cfg) {
if (!cfg.online?.line?.lineNameInfo?.fields) {
return apply_online_line_name_info;
}
}
function apply_online_line_name_info (cfg) {
console.log("Applying online line name info changes");
let lineNameInfo = {
example: null,
fields: {
line: {
offset: null,
length: 4,
type: "int",
},
sequence: {
offset: null,
length: 3,
type: "int"
},
incr: {
offset: null,
length: 2,
type: "bool",
enum: {}
},
attempt: {
offset: null,
length: 1,
type: "int"
}
}
};
switch (process.env.HOST) {
case "dougal04":
lineNameInfo = {
"example": "EQ22200-2213130-007",
"fields": {
"line": {
"length": 4,
"type": "int",
"offset": 10
},
"sequence": {
"length": 3,
"type": "int",
"offset": 16
},
"incr": {
"enum": {
"1": true,
"2": false
},
"length": 1,
"type": "bool",
"offset": 8
},
"attempt": {
"length": 1,
"type": "int",
"offset": 14
},
"file_no": {
"length": 3,
"type": "int",
"offset": 20
},
"year": {
"offset": 2,
"length": 2,
"type": "int"
},
"survey_type": {
"enum": {
"0": "Marine",
"2": "OBS/PRM"
},
"offset": 4,
"length": 1,
"default": "Unknown",
"type": "str"
},
"project_number": {
"offset": 5,
"length": 2,
"type": "int"
},
"num_sources": {
"enum": {
"0": "2",
"1": "1",
"2": "3"
},
"offset": 9,
"length": 1,
"type": "int"
}
}
}
break;
case "dougal03":
// Don't know what they use
break;
case "dougal02":
case "dougal01":
default: // Includes dev servers
lineNameInfo = {
example: "1054282180S00000",
fields: {
line: {
offset: 2,
length: 4,
type: "int",
},
sequence: {
offset: 7,
length: 3,
type: "int"
},
incr: {
offset: 0,
length: 2,
type: "bool",
enum: {
"10": true,
"20": false
}
},
attempt: {
offset: 6,
length: 1,
type: "int"
}
}
};
}
deepSet(cfg, [ "online", "line", "lineNameInfo" ], lineNameInfo);
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/292
//
function check_raw_p111_line_name_info (cfg) {
if (!cfg.raw?.p111?.lineNameInfo?.fields) {
return apply_raw_p111_line_name_info;
}
}
function apply_raw_p111_line_name_info (cfg) {
console.log("Applying raw P1/11 name info changes");
let lineNameInfo = {
example: null,
fields: {
line: {
offset: null,
length: 4,
type: "int",
},
sequence: {
offset: null,
length: 3,
type: "int"
},
incr: {
offset: null,
length: 2,
type: "bool",
enum: {}
},
attempt: {
offset: null,
length: 1,
type: "int"
}
}
};
switch (process.env.HOST) {
case "dougal04":
lineNameInfo = {
"example": "EQ22200-2213130-007.000.P111",
"fields": {
"line": {
"length": 4,
"type": "int",
"offset": 10
},
"sequence": {
"length": 3,
"type": "int",
"offset": 16
},
"incr": {
"enum": {
"1": true,
"2": false
},
"length": 1,
"type": "bool",
"offset": 8
},
"attempt": {
"length": 1,
"type": "int",
"offset": 14
},
"file_no": {
"length": 3,
"type": "int",
"offset": 20
},
"year": {
"offset": 2,
"length": 2,
"type": "int"
},
"survey_type": {
"enum": {
"0": "Marine",
"2": "OBS/PRM"
},
"offset": 4,
"length": 1,
"default": "Unknown",
"type": "str"
},
"project_number": {
"offset": 5,
"length": 2,
"type": "int"
},
"num_sources": {
"enum": {
"0": "2",
"1": "1",
"2": "3"
},
"offset": 9,
"length": 1,
"type": "int"
}
}
}
break;
case "dougal03":
// Don't know what they use
break;
case "dougal02":
case "dougal01":
default: // Includes dev servers
lineNameInfo = {
example: "1054282180S00000.000.p111",
fields: {
line: {
offset: 2,
length: 4,
type: "int",
},
sequence: {
offset: 7,
length: 3,
type: "int"
},
incr: {
offset: 0,
length: 2,
type: "bool",
enum: {
"10": true,
"20": false
}
},
attempt: {
offset: 6,
length: 1,
type: "int"
}
}
};
}
deepSet(cfg, [ "raw", "p111", "lineNameInfo" ], lineNameInfo);
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/293
//
function check_final_p111_line_name_info (cfg) {
if (!cfg.final?.p111?.lineNameInfo?.fields) {
return apply_final_p111_line_name_info;
}
}
function apply_final_p111_line_name_info (cfg) {
console.log("Applying final P1/11 name info changes");
let lineNameInfo = {
example: null,
fields: {
line: {
offset: null,
length: 4,
type: "int",
},
sequence: {
offset: null,
length: 3,
type: "int"
},
incr: {
offset: null,
length: 2,
type: "bool",
enum: {}
},
attempt: {
offset: null,
length: 1,
type: "int"
}
}
};
switch (process.env.HOST) {
case "dougal04":
lineNameInfo = {
"example": "EQ22200-2213130-007.000.P111",
"fields": {
"line": {
"length": 4,
"type": "int",
"offset": 10
},
"sequence": {
"length": 3,
"type": "int",
"offset": 16
},
"incr": {
"enum": {
"1": true,
"2": false
},
"length": 1,
"type": "bool",
"offset": 8
},
"attempt": {
"length": 1,
"type": "int",
"offset": 14
},
"file_no": {
"length": 3,
"type": "int",
"offset": 20
},
"year": {
"offset": 2,
"length": 2,
"type": "int"
},
"survey_type": {
"enum": {
"0": "Marine",
"2": "OBS/PRM"
},
"offset": 4,
"length": 1,
"default": "Unknown",
"type": "str"
},
"project_number": {
"offset": 5,
"length": 2,
"type": "int"
},
"num_sources": {
"enum": {
"0": "2",
"1": "1",
"2": "3"
},
"offset": 9,
"length": 1,
"type": "int"
}
}
}
break;
case "dougal03":
// Don't know what they use
break;
case "dougal02":
case "dougal01":
default: // Includes dev servers
lineNameInfo = {
example: "1054282180S00000.000.p111",
fields: {
line: {
offset: 2,
length: 4,
type: "int",
},
sequence: {
offset: 7,
length: 3,
type: "int"
},
incr: {
offset: 0,
length: 2,
type: "bool",
enum: {
"10": true,
"20": false
}
},
attempt: {
offset: 6,
length: 1,
type: "int"
}
}
};
}
deepSet(cfg, [ "final", "p111", "lineNameInfo" ], lineNameInfo);
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/294
//
function check_smsrc_headers_glob_path (cfg) {
if (!cfg.raw?.source?.smsrc?.header?.glob?.length) {
return apply_smsrc_headers_glob_path;
}
}
function apply_smsrc_headers_glob_path (cfg) {
console.log("Copying Smartsource header glob and path values to new location");
const globs = cfg?.raw?.smsrc?.globs;
const paths = cfg?.raw?.smsrc?.paths;
if (globs) {
deepSet(cfg, [ "raw", "source", "smsrc", "header", "globs" ], globs);
}
if (paths) {
deepSet(cfg, [ "raw", "source", "smsrc", "header", "paths" ], paths);
}
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/294
//
function check_smsrc_headers_line_name_info (cfg) {
if (!cfg.raw?.source?.smsrc?.header?.lineNameInfo?.fields) {
return apply_smsrc_headers_line_name_info;
}
}
function apply_smsrc_headers_line_name_info (cfg) {
console.log("Applying raw P1/11 name info changes");
let lineNameInfo = {
example: null,
fields: {
line: {
offset: null,
length: 4,
type: "int",
},
sequence: {
offset: null,
length: 3,
type: "int"
},
incr: {
offset: null,
length: 2,
type: "bool",
enum: {}
},
attempt: {
offset: null,
length: 1,
type: "int"
}
}
};
switch (process.env.HOST) {
case "dougal04":
lineNameInfo = {
"example": "EQ22200-2213130-007.000.P111",
"fields": {
"line": {
"length": 4,
"type": "int",
"offset": 10
},
"sequence": {
"length": 3,
"type": "int",
"offset": 16
},
"incr": {
"enum": {
"1": true,
"2": false
},
"length": 1,
"type": "bool",
"offset": 8
},
"attempt": {
"length": 1,
"type": "int",
"offset": 14
},
"file_no": {
"length": 3,
"type": "int",
"offset": 20
},
"year": {
"offset": 2,
"length": 2,
"type": "int"
},
"survey_type": {
"enum": {
"0": "Marine",
"2": "OBS/PRM"
},
"offset": 4,
"length": 1,
"default": "Unknown",
"type": "str"
},
"project_number": {
"offset": 5,
"length": 2,
"type": "int"
},
"num_sources": {
"enum": {
"0": "2",
"1": "1",
"2": "3"
},
"offset": 9,
"length": 1,
"type": "int"
}
}
}
break;
case "dougal03":
// Don't know what they use
break;
case "dougal02":
case "dougal01":
default: // Includes dev servers
lineNameInfo = {
example: "1054282180S00000.HDR",
fields: {
line: {
offset: 2,
length: 4,
type: "int",
},
sequence: {
offset: 7,
length: 3,
type: "int"
},
incr: {
offset: 0,
length: 2,
type: "bool",
enum: {
"10": true,
"20": false
}
},
attempt: {
offset: 6,
length: 1,
type: "int"
}
}
};
}
deepSet(cfg, [ "raw", "source", "smsrc", "header", "lineNameInfo" ], lineNameInfo);
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/295
//
function check_smsrc_segy (cfg) {
// We only do this on installations where we know there is, or there
// might be, SEG-Y data available.
const supported_hosts = [
"dougal02",
"dougal01"
];
if (supported_hosts.includes(process.env.HOST)) {
if (!cfg.raw?.source?.smsrc?.segy?.lineNameInfo?.fields) {
return apply_smsrc_segy;
}
}
}
function apply_smsrc_segy (cfg) {
// We don't need to run a switch() for hosts here, since
// we've already done that in check_smsrc_segy().
// Use the paths for *.HDR files as a reference
const paths = cfg.raw?.source?.smsrc?.header?.paths?.map( p =>
path.join(path.dirname(p), "10 SEG-Y"));
const globs = [ "**/*-hyd.sgy" ];
const lineNameInfo = {
"example": "1051460070S00000-hyd.sgy",
"fields": {
"sequence": {
"length": 3,
"type": "int",
"offset": 7
},
"line": {
"length": 4,
"offset": 2
}
}
};
const segy = { paths, globs, lineNameInfo };
deepSet(cfg, [ "raw", "source", "smsrc", "segy" ], segy);
return cfg;
}
//
// https://gitlab.com/wgp/dougal/software/-/work_items/298
//
function check_preplots_fields (cfg) {
if (cfg.preplots?.length) {
const indices = [];
for (const idx in cfg.preplots) {
const preplot = cfg.preplots[idx];
if (!preplot?.fields?.line_name) {
indices.push(idx);
}
}
if (indices.length) {
return apply_preplots_fieldsλ(indices);
}
}
}
function apply_preplots_fieldsλ (indices) {
function fix_preplot (preplot) {
const names = preplot.format.names;
const types = preplot.format.types;
const widths = preplot.format.widths;
const offsets_widths = widths.reduce ((acc, cur) => {
if (cur < 0) {
acc.p -= cur; // Advances the position by -cur
} else {
acc.f.push({offset: acc.p, width: cur});
acc.p += cur;
}
return acc;
}, {f: [], p: 0})
const fields = {};
names.forEach( (name, ι) => {
const field = {
type: types[ι],
...offsets_widths[ι]
};
fields[name] = field;
});
preplot.fields = fields;
return preplot;
}
return function apply_preplots_fields (cfg) {
for (const idx of indices) {
console.log("Fixing preplot", idx);
const preplot = fix_preplot(cfg.preplots[idx]);
cfg.preplots.splice(idx, 1, preplot);
}
}
}
/* Template for more upgrade actions
//
// https://gitlab.com/wgp/dougal/software/-/work_items/
//
function check_ (cfg) {
}
function apply_ (cfg) {
}
*/
const checkers = [
check_asaqc,
check_asaqc_subscription_key,
check_online_line_name_info,
check_raw_p111_line_name_info,
check_final_p111_line_name_info,
check_smsrc_headers_glob_path,
check_smsrc_headers_line_name_info,
check_smsrc_segy,
check_preplots_fields
]
const now = new Date();
const tstamp = now.toISOString().substr(0, 19)+"Z";
function fnames(pid) {
return {
backup: `${pid}-configuration-${tstamp}.yaml`,
upgrade: `NEW-${pid}-configuration-${tstamp}.yaml`
};
}
function save_scripts (pid) {
const cwd = process.cwd();
const fn_backup = path.resolve(path.join(cwd, fnames(pid).backup));
const fn_upgrade = path.resolve(path.join(cwd, fnames(pid).upgrade));
console.log("Creating script to restore old / new configurations");
const backup = `# Restore pre-upgrade configuration for ${pid}
curl -vs "http://localhost:3000/api/project/${pid}/configuration" -X PUT -H "Content-Type: application/yaml" --data-binary @${fn_backup}\n`;
const upgrade = `# Restore post-upgrade configuration for ${pid}
curl -vs "http://localhost:3000/api/project/${pid}/configuration" -X PUT -H "Content-Type: application/yaml" --data-binary @${fn_upgrade}\n`;
fs.writeFileSync(`restore-20231113-pre-${pid}.sh`, backup);
fs.writeFileSync(`restore-20231113-post-${pid}.sh`, upgrade);
}
async function backup (pid, cfg) {
const fname = fnames(pid).backup;
console.log(`Backing up configuration for ${pid} as ${fname} into current directory`);
const text = YAML.stringify(cfg);
fs.writeFileSync(fname, text);
}
async function save_configuration (pid, cfg) {
console.log("Saving configuration for", pid);
console.log("Saving copy of NEW configuration to file");
const fname = fnames(pid).upgrade;
const text = YAML.stringify(cfg);
fs.writeFileSync(fname, text);
save_scripts(pid);
//console.log("Uploading configuration to server");
try {
//await db.project.configuration.put(pid);
} catch (err) {
console.log("Configuration upload failed");
console.error(err);
throw err;
}
}
async function upgrade_configuration (pid) {
const configuration = await db.project.configuration.get(pid);
console.log(`Checking configuration for ${configuration.id} (${configuration.schema})`);
const appliers = checkers.map( checker => checker(configuration) ).filter( i => !!i );
if (appliers.length) {
console.log("Configuration needs changes.");
await backup(pid, configuration);
console.log("Applying changes");
console.log(appliers);
for (const applier of appliers) {
applier(configuration);
}
await save_configuration(pid, configuration);
}
}
async function main () {
const cla = process.argv.slice(2).map(i => i.toLowerCase());
function project_filter (project) {
if (cla.length == 0)
return true;
return cla.includes(project.pid.toLowerCase());
}
const projects = (await db.project.get()).filter(project_filter);
projects.sort( (a, b) =>
a.pid > b.pid
? 1
: a.pid < b.pid
? -1
: 0);
console.log(projects);
for (const project of projects) {
await upgrade_configuration(project.pid);
}
console.log("All done");
process.exit(0);
}
main();