Add database upgrade file 10.

NOTE: this is the first time we modify the actual data
in the database, as opposed to adding to the schema.

bin/datastore.py

@@ -4,6 +4,7 @@ import psycopg2
 import configuration
 import preplots
 import p111
+from hashlib import md5 # Because it's good enough
 
 """
 Interface to the PostgreSQL database.
@@ -11,13 +12,16 @@ Interface to the PostgreSQL database.
 
 def file_hash(file):
 	"""
-	Calculate a file hash based on its size, inode, modification and creation times.
+	Calculate a file hash based on its name, size, modification and creation times.
 
 	The hash is used to uniquely identify files in the database and detect if they
 	have changed.
 	"""
+	h = md5()
+	h.update(file.encode())
+	name_digest = h.hexdigest()[:16]
 	st = os.stat(file)
-	return ":".join([str(v) for v in [st.st_size, st.st_mtime, st.st_ctime, st.st_ino]])
+	return ":".join([str(v) for v in [st.st_size, st.st_mtime, st.st_ctime, name_digest]])
 
 class Datastore:
 	"""
@@ -390,9 +394,9 @@ class Datastore:
 
 		with self.conn.cursor() as cursor:
 			cursor.execute("BEGIN;")
-			
+
 			hash = self.add_file(filepath, cursor)
-			
+
 			if not records or len(records) == 0:
 				print("File has no records (or none have been detected)")
 				# We add the file to the database anyway to signal that we have
@@ -412,13 +416,13 @@ class Datastore:
 			"""
 
 			cursor.execute(qry, (fileinfo["sequence"], fileinfo["line"], ntbp, incr, json.dumps(fileinfo["meta"])))
-			
+
 			qry = """
 				UPDATE raw_lines
 				SET meta = meta || %s
 				WHERE sequence = %s;
 			"""
-			
+
 			cursor.execute(qry, (json.dumps(fileinfo["meta"]), fileinfo["sequence"]))
 
 			qry = """
@@ -452,7 +456,7 @@ class Datastore:
 
 		with self.conn.cursor() as cursor:
 			cursor.execute("BEGIN;")
-			
+
 			hash = self.add_file(filepath, cursor)
 
 			qry = """
@@ -462,13 +466,13 @@ class Datastore:
 			"""
 
 			cursor.execute(qry, (fileinfo["sequence"], fileinfo["line"], json.dumps(fileinfo["meta"])))
-			
+
 			qry = """
 				UPDATE raw_lines
 				SET meta = meta || %s
 				WHERE sequence = %s;
 			"""
-			
+
 			cursor.execute(qry, (json.dumps(fileinfo["meta"]), fileinfo["sequence"]))
 
 			qry = """
@@ -495,7 +499,7 @@ class Datastore:
 
 			if filedata is not None:
 				self.save_file_data(filepath, json.dumps(filedata), cursor)
-				
+
 			cursor.execute("CALL final_line_post_import(%s);", (fileinfo["sequence"],))
 
 		self.maybe_commit()
@@ -662,7 +666,7 @@ class Datastore:
 		"""
 		Remove final data for a sequence.
 		"""
-		
+
 		if cursor is None:
 			cur = self.conn.cursor()
 		else:
@@ -674,4 +678,4 @@ class Datastore:
 			self.maybe_commit()
 			# We do not commit if we've been passed a cursor, instead
 			# we assume that we are in the middle of a transaction
-		
+

bin/runner.sh

@@ -114,6 +114,9 @@ run $BINDIR/human_exports_qc.py
 print_log "Export sequence data"
 run $BINDIR/human_exports_seis.py
 
+print_log "Process ASAQC queue"
+run $DOUGAL_ROOT/lib/www/server/queues/asaqc/index.js
+
 
 rm "$LOCKFILE"
 print_info "End run"

etc/config.yaml

@@ -35,7 +35,7 @@ imports:
 queues:
     asaqc:
         request:
-            url: "https://localhost:3077/vt/v1/api/upload-file-encoded"
+            url: "https://api.gateway.equinor.com/vt/v1/api/upload-file-encoded"
             args:
                 method: POST
                 headers:

etc/db/README.md

@@ -19,3 +19,79 @@ Created with:
 ```bash
 SCHEMA_NAME=survey_X EPSG_CODE=XXXXX $DOUGAL_ROOT/sbin/dump_schema.sh
 ```
+
+## To create a new Dougal database
+
+Ensure that the following packages are installed:
+
+* `postgresql*-postgis-utils`
+* `postgresql*-postgis`
+* `postgresql*-contrib` # For B-trees
+
+```bash
+psql -U postgres <./database-template.sql
+```
+
+---
+
+# Upgrading PostgreSQL
+
+The following is based on https://en.opensuse.org/SDB:PostgreSQL#Upgrading_major_PostgreSQL_version
+
+```bash
+# The following bash code should be checked and executed
+# line for line whenever you do an upgrade. The example
+# shows the upgrade process from an original installation
+# of version 12 up to version 14.
+
+# install the new server as well as the required postgresql-contrib packages:
+zypper in postgresql14-server postgresql14-contrib postgresql12-contrib
+
+# If not yet done, copy the configuration create a new PostgreSQL configuration directory...
+mkdir /etc/postgresql
+# and copy the original file to this global directory
+cd /srv/pgsql/data
+for i in  pg_hba.conf pg_ident.conf postgresql.conf postgresql.auto.conf ; do cp -a $i /etc/postgresql/$i ; done
+
+# Now create a new data-directory and initialize it for usage with the new server
+install -d -m 0700 -o postgres -g postgres /srv/pgsql/data14
+cd /srv/pgsql/data14
+sudo -u postgres /usr/lib/postgresql14/bin/initdb .
+
+# replace the newly generated files by a symlink to the global files.
+# After doing so, you may check the difference of the created backup files and
+# the files from the former installation
+for i in  pg_hba.conf pg_ident.conf postgresql.conf postgresql.auto.conf ; do old $i ; ln -s /etc/postgresql/$i .; done
+
+# Copy over special thesaurus files if some exists.
+#cp -a /usr/share/postgresql12/tsearch_data/my_thesaurus_german.ths /usr/share/postgresql14/tsearch_data/
+
+# Now it's time to disable the service...
+systemctl stop postgresql.service
+
+# And to start the migration. Please ensure, the directories fit to your upgrade path
+sudo -u postgres /usr/lib/postgresql14/bin/pg_upgrade --link \
+  --old-bindir="/usr/lib/postgresql12/bin" \
+  --new-bindir="/usr/lib/postgresql14/bin" \
+  --old-datadir="/srv/pgsql/data/" \
+  --new-datadir="/srv/pgsql/data14/"
+
+# After successfully migrating the data...
+cd ..
+# if not already symlinked move the old data to a versioned directory matching
+# your old installation...
+mv data data12
+# and set a symlink to the new data directory
+ln -sf data14/ data
+
+# Now start the new service
+systemctl start postgresql.service
+
+# If everything has been sucessful, you should uninstall old packages...
+#zypper rm -u postgresql12 postgresql13
+# and remove old data directories
+#rm -rf /srv/pgsql/data_OLD_POSTGRES_VERSION_NUMBER
+
+# For good measure:
+sudo -u postgres /usr/lib/postgresql14/bin/vacuumdb --all --analyze-in-stages
+```

etc/db/upgrades/README.md

@@ -0,0 +1,34 @@
+# Database schema upgrades
+
+When the database schema needs to be upgraded in order to provide new functionality, fix errors, etc., an upgrade script should be added to this directory.
+
+The script can be SQL (preferred) or anything else (Bash, Python, …) in the event of complex upgrades.
+
+The script itself should:
+
+* document what the intended changes are;
+* contain instructions on how to run it;
+* make the user aware of any non-obvious side effects; and
+* say if it is safe to run the script multiple times on the
+* same schema / database.
+
+## Naming
+
+Script files should be named `upgrade-<index>-<commit-id-old>-<commit-id-new>-v<schema-version>.sql`, where:
+
+* `<index>` is a correlative two-digit index. When reaching 99, existing files will be renamed to a three digit index (001-099) and new files will use three digits.
+* `<commit-id-old>` is the ID of the Git commit that last introduced a schema change.
+* `<commit-id-new>` is the ID of the first Git commit expecting the updated schema.
+* `<schema-version>` is the version of the schema.
+
+Note: the `<schema-version>` value should be updated with every change and it should be the same as reported by:
+
+```sql
+select value->>'db_schema' as db_schema from public.info where key = 'version';
+```
+
+If necessary, the wanted schema version must also be updated in `package.json`.
+
+## Running
+
+Schema upgrades are always run manually.

etc/db/upgrades/upgrade09-74b3de5c→83be83e4-v0.1.0.sql

@@ -0,0 +1,24 @@
+-- Upgrade the database from commit 74b3de5c to commit 83be83e4.
+--
+-- NOTE: This upgrade only affects the `public` schema.
+--
+-- This inserts a database schema version into the database.
+-- Note that we are not otherwise changing the schema, so older
+-- server code will continue to run against this version.
+--
+-- ATTENTION!
+--
+-- This value should be incremented every time that the database
+-- schema changes (either `public` or any of the survey schemas)
+-- and is used by the server at start-up to detect if it is
+-- running against a compatible schema version.
+--
+-- To apply, run as the dougal user:
+--
+-- psql < $THIS_FILE
+--
+-- NOTE: It can be applied multiple times without ill effect.
+
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.1.0"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.1.0"}' WHERE public.info.key = 'version';

etc/db/upgrades/upgrade10-83be83e4→53ed096e-v0.2.0.sql

@@ -0,0 +1,84 @@
+-- Upgrade the database from commit 83be83e4 to 53ed096e.
+--
+-- New schema version: 0.2.0
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This migrates the file hashes to address issue #173.
+-- The new hashes use size, modification time, creation time and the
+-- first half of the MD5 hex digest of the file's absolute path.
+--
+-- It's a minor (rather than patch) version number increment because
+-- changes to `bin/datastore.py` mean that the data is no longer
+-- compatible with the hashing function.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can take a while if run on a large database.
+-- NOTE: It can be applied multiple times without ill effect.
+
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE migrate_hashes (schema_name text) AS $$
+BEGIN
+	RAISE NOTICE 'Migrating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+	EXECUTE format('UPDATE %I.files SET hash = array_to_string(array_append(trim_array(string_to_array(hash, '':''), 1), left(md5(path), 16)), '':'')', schema_name);
+	EXECUTE 'SET search_path TO public'; -- Back to the default search path for good measure
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE upgrade_10 () AS $$
+DECLARE
+	row RECORD;
+BEGIN
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL migrate_hashes(row.schema_name);
+	END LOOP;
+END;
+$$ LANGUAGE plpgsql;
+
+CALL upgrade_10();
+
+CALL show_notice('Cleaning up');
+DROP PROCEDURE migrate_hashes (schema_name text);
+DROP PROCEDURE upgrade_10 ();
+
+CALL show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.2.0"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.2.0"}' WHERE public.info.key = 'version';
+
+
+CALL show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/www/server/api/middleware/info/delete.js

@@ -4,7 +4,7 @@ const { info } = require('../../../lib/db');
 module.exports = async function (req, res, next) {
 
 	try {
-		await info.delete(req.params.project, req.params.path);
+		await info.delete(req.params.project, req.params.path, undefined, req.user.role);
 		res.status(204).send();
 		next();
 	} catch (err) {

lib/www/server/api/middleware/info/get.js

@@ -4,7 +4,7 @@ const { info } = require('../../../lib/db');
 module.exports = async function (req, res, next) {
 
 	try {
-		res.status(200).json(await info.get(req.params.project, req.params.path, req.query));
+		res.status(200).json(await info.get(req.params.project, req.params.path, req.query, req.user.role));
 	} catch (err) {
 		if (err instanceof TypeError) {
 			res.status(404).json(null);

lib/www/server/api/middleware/info/post.js

@@ -6,7 +6,7 @@ module.exports = async function (req, res, next) {
 	try {
 		const payload = req.body;
 
-		await info.post(req.params.project, req.params.path, payload);
+		await info.post(req.params.project, req.params.path, payload, undefined, req.user.role);
 		res.status(201).send();
 		next();
 	} catch (err) {

lib/www/server/api/middleware/info/put.js

@@ -6,7 +6,7 @@ module.exports = async function (req, res, next) {
 	try {
 		const payload = req.body;
 
-		await info.put(req.params.project, req.params.path, payload);
+		await info.put(req.params.project, req.params.path, payload, undefined, req.user.role);
 		res.status(201).send();
 		next();
 	} catch (err) {

lib/www/server/index.js

@@ -1,18 +1,28 @@
 #!/usr/bin/node
 
-const api = require('./api');
+async function main () {
-const ws = require('./ws');
+	// Check that we're running against the correct database version
+	const version = require('./lib/version');
+	console.log("Running version", await version.describe());
+	version.compatible()
+	.then( () => {
+		const api = require('./api');
+		const ws = require('./ws');
 
-const { fork } = require('child_process');
+		const { fork } = require('child_process');
 
-// const em = require('./events');
+		const server = api.start(process.env.HTTP_PORT || 3000, process.env.HTTP_PATH);
+		ws.start(server);
 
-const server = api.start(process.env.HTTP_PORT || 3000, process.env.HTTP_PATH);
+		const eventManagerPath = [__dirname, "events"].join("/");
-ws.start(server);
+		const eventManager = fork(eventManagerPath, /*{ stdio: 'ignore' }*/);
 
-const eventManagerPath = [__dirname, "events"].join("/");
+		process.on('exit', () => eventManager.kill());
-const eventManager = fork(eventManagerPath, /*{ stdio: 'ignore' }*/);
+	})
+	.catch( ({current, wanted}) => {
+		console.error(`Fatal error: incompatible database schema version ${current} (wanted: ${wanted})`);
+		process.exit(1);
+	});
+}
 
-process.on('exit', () => eventManager.kill());
+main();
-
-// em.start();

lib/www/server/lib/db/info/check-permission.js

@@ -0,0 +1,83 @@
+/** Check permission to read or write certain keys.
+ *
+ * The global and survey `info` tables can be used to
+ * store and retrieve arbitrary data, but it is also
+ * used by the software, with some keys being reserved
+ * for specific purposes.
+ *
+ * This module lists those keys which are in some way
+ * reserved and reports on who should be allowed what
+ * type of access to them.
+ */
+
+
+/** Reserved keys.
+ *
+ * The structure of this dictionary is
+ * object.verb.subject = Boolean.
+ *
+ * The special value `_` is a wildcard
+ * denoting the default condition for
+ * a verb or a subject.
+ */
+const dictionary = {
+	version: {
+		// Database or schema version string.
+		// Everyone can read, nobody can alter.
+		get: { _: true  },
+		_  : { _: false }
+	},
+	config: {
+		// Configuration (site-wide or survey)
+		// Nobody except admin can access
+		_: { _: false, admin: true }
+	},
+	qc: {
+		// QC results (survey)
+		// Everyone can read, nobody can write
+		get: { _: true  },
+		_  : { _: false }
+	},
+	equipment: {
+		// Equipment info (site)
+		// Everyone can read, user + admin can alter
+		get: { _: true },
+		_  : { _: false, user: true, admin: true }
+	},
+	contact: {
+		// Contact details (basically an example entry)
+		// Everyone can read, admin can alter
+		get: { _: true },
+		_  : { _: false, admin: true },
+	}
+}
+
+/** Check if access is allowed to an info entry.
+ *
+ * @a key {String} is the object of the action.
+ * @a verb {String} is the action.
+ * @a role {String} is the subject of the action.
+ *
+ * @returns {Boolean} `true` is the action is allowed,
+ * `false` if it is not.
+ *
+ * By default, all actions are allowed on a key that's
+ * not listed in the dictionary. For a key that is listed,
+ * the result for a default action or subject is denoted
+ * by `_`, others are entered explicitly.
+ *
+ */
+function checkPermission (key, verb, role) {
+	const entry = dictionary[key]
+	if (entry) {
+		const action = entry[verb] ?? entry._
+		if (action) {
+			return action[role] ?? action._ ?? false;
+		}
+		return false;
+	}
+	return true;
+}
+
+
+module.exports = checkPermission;

lib/www/server/lib/db/info/delete.js

@@ -1,9 +1,15 @@
 const { setSurvey, transaction } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function del (projectId, path, opts = {}) {
+async function del (projectId, path, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...jsonpath] = (path||"").split("/").filter(i => i.length);
 
+	if (!checkPermission(key, "delete", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	try {
 		const text = jsonpath.length
 			? `

lib/www/server/lib/db/info/get.js

@@ -1,9 +1,15 @@
 const { setSurvey } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function get (projectId, path, opts = {}) {
+async function get (projectId, path, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...subkey] = path.split("/").filter(i => i.trim().length);
 
+	if (!checkPermission(key, "get", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	const text = `
 		SELECT value
 		FROM info

lib/www/server/lib/db/info/post.js

@@ -1,9 +1,15 @@
 const { setSurvey, transaction } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function post (projectId, path, payload, opts = {}) {
+async function post (projectId, path, payload, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...jsonpath] = (path||"").split("/").filter(i => i.length);
 
+	if (!checkPermission(key, "post", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	try {
 		const text = jsonpath.length
 			? `

lib/www/server/lib/db/info/put.js

@@ -1,9 +1,15 @@
 const { setSurvey, transaction } = require('../connection');
+const checkPermission = require('./check-permission');
 
-async function put (projectId, path, payload, opts = {}) {
+async function put (projectId, path, payload, opts = {}, role) {
 	const client = await setSurvey(projectId);
 	const [key, ...jsonpath] = (path||"").split("/").filter(i => i.length);
 
+	if (!checkPermission(key, "put", role)) {
+		throw {status: 403, message: "Forbidden"};
+		return;
+	}
+
 	try {
 		const text = jsonpath.length
 			? `

lib/www/server/lib/db/plan/lib/index.js

@@ -1,3 +1,4 @@
+const alert = require("../../../alerts");
 const configuration = require('../../configuration');
 
 async function getDistance (client, payload) {
@@ -88,6 +89,13 @@ async function getPlanned (client) {
 
 async function getLineName (client, projectId, payload) {
 	// FIXME TODO Get line name script from configuration
+	// Ref.: https://gitlab.com/wgp/dougal/software/-/issues/129
+
+	// This is to monitor #165
+	// https://gitlab.com/wgp/dougal/software/-/issues/incident/165
+	if (!payload?.line) {
+		alert({function: "getLineName", client, projectId, payload});
+	}
 
 	const planned = await getPlanned(client);
 	const previous = await getSequencesForLine(client, payload.line);

lib/www/server/lib/version.js

@@ -0,0 +1,68 @@
+const semver = require("semver");
+const { exec } = require("child_process");
+const pkg = require("../package.json");
+
+/** Report whether the database schema version is
+ * compatible with the version required by this server.
+ *
+ * The current schema version is retrieved from the
+ * public.info table.
+ *
+ * The wanted version is retrieved from package.json
+ * (config.db_schema).
+ *
+ * @returns true if the versions are compatible,
+ * false otherwise.
+ */
+function compatible () {
+	const { info } = require('./db');
+	return new Promise ( async (resolve, reject) => {
+		const current = await info.get(null, "version/db_schema");
+		const wanted = pkg.config.db_schema;
+		if (semver.satisfies(current, wanted)) {
+			resolve({current, wanted});
+		} else {
+			reject({current, wanted});
+		}
+	});
+}
+
+
+/** Return software name.
+ *
+ */
+function name () {
+	const pkg = require("../package.json");
+	return pkg.name ?? pkg.description ?? "Unknown";
+}
+
+/** Return software version, from Git if possible.
+ *
+ */
+async function describe () {
+	return new Promise( (resolve, reject) => {
+		const cmd = `git describe || echo git+$(git describe --always);`;
+		exec(cmd, {cwd: __dirname}, (error, stdout, stderr) => {
+			if (error) {
+				reject(error);
+			}
+
+			if (stdout) {
+				resolve(stdout.trim());
+			} else {
+				// Most likely not installed from Git, use the
+				// version number in package.json.
+				resolve(pkg.version ?? "Unknown")
+			}
+		})
+	});
+}
+
+function version () {
+	return pkg.version;
+}
+version.compatible = compatible;
+version.name = name;
+version.describe = describe;
+
+module.exports = version;

lib/www/server/package.json

@@ -9,6 +9,16 @@
   },
   "author": "Aaltronav s.r.o.",
   "license": "UNLICENSED",
+  "private": true,
+  "config": {
+    "db_schema": "^0.2.0"
+  },
+  "engines": {
+    "node": ">=14.0.0"
+  },
+  "os": [
+    "linux"
+  ],
   "dependencies": {
     "cookie-parser": "^1.4.5",
     "express": "^4.17.1",