Merge branch '175-add-database-versioning-and-migration-mechanism' into 'devel'

Resolve "Add database versioning and migration mechanism"

Closes #175

See merge request wgp/dougal/software!18

etc/db/upgrades/README.md

@@ -0,0 +1,34 @@
+# Database schema upgrades
+
+When the database schema needs to be upgraded in order to provide new functionality, fix errors, etc., an upgrade script should be added to this directory.
+
+The script can be SQL (preferred) or anything else (Bash, Python, …) in the event of complex upgrades.
+
+The script itself should:
+
+* document what the intended changes are;
+* contain instructions on how to run it;
+* make the user aware of any non-obvious side effects; and
+* say if it is safe to run the script multiple times on the
+* same schema / database.
+
+## Naming
+
+Script files should be named `upgrade-<index>-<commit-id-old>-<commit-id-new>-v<schema-version>.sql`, where:
+
+* `<index>` is a correlative two-digit index. When reaching 99, existing files will be renamed to a three digit index (001-099) and new files will use three digits.
+* `<commit-id-old>` is the ID of the Git commit that last introduced a schema change.
+* `<commit-id-new>` is the ID of the first Git commit expecting the updated schema.
+* `<schema-version>` is the version of the schema.
+
+Note: the `<schema-version>` value should be updated with every change and it should be the same as reported by:
+
+```sql
+select value->>'db_schema' as db_schema from public.info where key = 'version';
+```
+
+If necessary, the wanted schema version must also be updated in `package.json`.
+
+## Running
+
+Schema upgrades are always run manually.

etc/db/upgrades/upgrade09-74b3de5c→83be83e4-v0.1.0.sql

@@ -0,0 +1,24 @@
+-- Upgrade the database from commit 74b3de5c to commit 83be83e4.
+--
+-- NOTE: This upgrade only affects the `public` schema.
+--
+-- This inserts a database schema version into the database.
+-- Note that we are not otherwise changing the schema, so older
+-- server code will continue to run against this version.
+--
+-- ATTENTION!
+--
+-- This value should be incremented every time that the database
+-- schema changes (either `public` or any of the survey schemas)
+-- and is used by the server at start-up to detect if it is
+-- running against a compatible schema version.
+--
+-- To apply, run as the dougal user:
+--
+-- psql < $THIS_FILE
+--
+-- NOTE: It can be applied multiple times without ill effect.
+
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.1.0"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.1.0"}' WHERE public.info.key = 'version';

lib/www/server/api/middleware/info/delete.js

@@ -4,7 +4,7 @@ const { info } = require('../../../lib/db');
 module.exports = async function (req, res, next) {
 
 	try {
-		await info.delete(req.params.project, req.params.path);
+		await info.delete(req.params.project, req.params.path, undefined, req.user.role);
 		res.status(204).send();
 		next();
 	} catch (err) {

lib/www/server/api/middleware/info/get.js

@@ -4,7 +4,7 @@ const { info } = require('../../../lib/db');
 module.exports = async function (req, res, next) {
 
 	try {
-		res.status(200).json(await info.get(req.params.project, req.params.path, req.query));
+		res.status(200).json(await info.get(req.params.project, req.params.path, req.query, req.user.role));
 	} catch (err) {
 		if (err instanceof TypeError) {
 			res.status(404).json(null);

lib/www/server/api/middleware/info/post.js

@@ -6,7 +6,7 @@ module.exports = async function (req, res, next) {
 	try {
 		const payload = req.body;
 
-		await info.post(req.params.project, req.params.path, payload);
+		await info.post(req.params.project, req.params.path, payload, undefined, req.user.role);
 		res.status(201).send();
 		next();
 	} catch (err) {

lib/www/server/api/middleware/info/put.js

@@ -6,7 +6,7 @@ module.exports = async function (req, res, next) {
 	try {
 		const payload = req.body;
 
-		await info.put(req.params.project, req.params.path, payload);
+		await info.put(req.params.project, req.params.path, payload, undefined, req.user.role);
 		res.status(201).send();
 		next();
 	} catch (err) {

lib/www/server/index.js

@@ -1,12 +1,16 @@
 #!/usr/bin/node
 
+async function main () {
+	// Check that we're running against the correct database version
+	const version = require('./lib/version');
+	console.log("Running version", await version.describe());
+	version.compatible()
+	.then( () => {
 		const api = require('./api');
 		const ws = require('./ws');
 
 		const { fork } = require('child_process');
 
-// const em = require('./events');
-
 		const server = api.start(process.env.HTTP_PORT || 3000, process.env.HTTP_PATH);
 		ws.start(server);
 
@@ -14,5 +18,11 @@ const eventManagerPath = [__dirname, "events"].join("/");
 		const eventManager = fork(eventManagerPath, /*{ stdio: 'ignore' }*/);
 
 		process.on('exit', () => eventManager.kill());
+	})
+	.catch( ({current, wanted}) => {
+		console.error(`Fatal error: incompatible database schema version ${current} (wanted: ${wanted})`);
+		process.exit(1);
+	});
+}
 
-// em.start();
+main();

lib/www/server/lib/db/info/check-permission.js

@@ -0,0 +1,83 @@
+/** Check permission to read or write certain keys.
+ *
+ * The global and survey `info` tables can be used to
+ * store and retrieve arbitrary data, but it is also
+ * used by the software, with some keys being reserved
+ * for specific purposes.
+ *
+ * This module lists those keys which are in some way
+ * reserved and reports on who should be allowed what
+ * type of access to them.
+ */
+
+
+/** Reserved keys.
+ *
+ * The structure of this dictionary is
+ * object.verb.subject = Boolean.
+ *
+ * The special value `_` is a wildcard
+ * denoting the default condition for
+ * a verb or a subject.
+ */
+const dictionary = {
+	version: {
+		// Database or schema version string.
+		// Everyone can read, nobody can alter.
+		get: { _: true  },
+		_  : { _: false }
+	},
+	config: {
+		// Configuration (site-wide or survey)
+		// Nobody except admin can access
+		_: { _: false, admin: true }
+	},
+	qc: {
+		// QC results (survey)
+		// Everyone can read, nobody can write
+		get: { _: true  },
+		_  : { _: false }
+	},
+	equipment: {
+		// Equipment info (site)
+		// Everyone can read, user + admin can alter
+		get: { _: true },
+		_  : { _: false, user: true, admin: true }
+	},
+	contact: {
+		// Contact details (basically an example entry)
+		// Everyone can read, admin can alter
+		get: { _: true },
+		_  : { _: false, admin: true },
+	}
+}
+
+/** Check if access is allowed to an info entry.
+ *
+ * @a key {String} is the object of the action.
+ * @a verb {String} is the action.
+ * @a role {String} is the subject of the action.
+ *
+ * @returns {Boolean} `true` is the action is allowed,
+ * `false` if it is not.
+ *
+ * By default, all actions are allowed on a key that's
+ * not listed in the dictionary. For a key that is listed,
+ * the result for a default action or subject is denoted
+ * by `_`, others are entered explicitly.
+ *
+ */
+function checkPermission (key, verb, role) {
+	const entry = dictionary[key]
+	if (entry) {
+		const action = entry[verb] ?? entry._
+		if (action) {
+			return action[role] ?? action._ ?? false;
+		}
+		return false;
+	}
+	return true;
+}
+
+
+module.exports = checkPermission;

lib/www/server/lib/db/info/delete.js

@@ -1,9 +1,15 @@
 const { setSurvey, transaction } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function del (projectId, path, opts = {}) {
+async function del (projectId, path, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...jsonpath] = (path||"").split("/").filter(i => i.length);
 
+	if (!checkPermission(key, "delete", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	try {
 		const text = jsonpath.length
 			? `

lib/www/server/lib/db/info/get.js

@@ -1,9 +1,15 @@
 const { setSurvey } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function get (projectId, path, opts = {}) {
+async function get (projectId, path, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...subkey] = path.split("/").filter(i => i.trim().length);
 
+	if (!checkPermission(key, "get", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	const text = `
 		SELECT value
 		FROM info

lib/www/server/lib/db/info/post.js

@@ -1,9 +1,15 @@
 const { setSurvey, transaction } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function post (projectId, path, payload, opts = {}) {
+async function post (projectId, path, payload, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...jsonpath] = (path||"").split("/").filter(i => i.length);
 
+	if (!checkPermission(key, "post", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	try {
 		const text = jsonpath.length
 			? `

lib/www/server/lib/db/info/put.js

@@ -1,9 +1,15 @@
 const { setSurvey, transaction } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function put (projectId, path, payload, opts = {}) {
+async function put (projectId, path, payload, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...jsonpath] = (path||"").split("/").filter(i => i.length);
 
+	if (!checkPermission(key, "put", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	try {
 		const text = jsonpath.length
 			? `

lib/www/server/lib/version.js

@@ -0,0 +1,68 @@
+const semver = require("semver");
+const { exec } = require("child_process");
+const pkg = require("../package.json");
+
+/** Report whether the database schema version is
+ * compatible with the version required by this server.
+ *
+ * The current schema version is retrieved from the
+ * public.info table.
+ *
+ * The wanted version is retrieved from package.json
+ * (config.db_schema).
+ *
+ * @returns true if the versions are compatible,
+ * false otherwise.
+ */
+function compatible () {
+	const { info } = require('./db');
+	return new Promise ( async (resolve, reject) => {
+		const current = await info.get(null, "version/db_schema");
+		const wanted = pkg.config.db_schema;
+		if (semver.satisfies(current, wanted)) {
+			resolve({current, wanted});
+		} else {
+			reject({current, wanted});
+		}
+	});
+}
+
+
+/** Return software name.
+ *
+ */
+function name () {
+	const pkg = require("../package.json");
+	return pkg.name ?? pkg.description ?? "Unknown";
+}
+
+/** Return software version, from Git if possible.
+ *
+ */
+async function describe () {
+	return new Promise( (resolve, reject) => {
+		const cmd = `git describe || echo git+$(git describe --always);`;
+		exec(cmd, {cwd: __dirname}, (error, stdout, stderr) => {
+			if (error) {
+				reject(error);
+			}
+
+			if (stdout) {
+				resolve(stdout.trim());
+			} else {
+				// Most likely not installed from Git, use the
+				// version number in package.json.
+				resolve(pkg.version ?? "Unknown")
+			}
+		})
+	});
+}
+
+function version () {
+	return pkg.version;
+}
+version.compatible = compatible;
+version.name = name;
+version.describe = describe;
+
+module.exports = version;

lib/www/server/package.json

@@ -9,6 +9,16 @@
   },
   "author": "Aaltronav s.r.o.",
   "license": "UNLICENSED",
+  "private": true,
+  "config": {
+    "db_schema": "^0.1.0"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "os": [
+    "linux"
+  ],
   "dependencies": {
     "cookie-parser": "^1.4.5",
     "express": "^4.17.1",