Adapt Project component to Vuex use for fetching data.

The Project component is now responsible for fetching and
updating the data used by most project tabs, with the
exception of ProjectSummary, QC, Graphs and Map. It is
also the only one listening for server events and reacting
to them.

Individual tabs are still responsible for sending data to
the server, at least for the time being.

.gitignore

@@ -11,3 +11,5 @@ lib/www/client/dist/
 etc/surveys/*.yaml
 !etc/surveys/_*.yaml
 etc/ssl/*
+etc/config.yaml
+var/*

bin/check_mounts_present.py

@@ -0,0 +1,27 @@
+#!/usr/bin/python3
+
+"""
+Check if any of the directories provided in the imports.mounts configuration
+section are empty.
+
+Returns 0 if all arguments are non-empty, 1 otherwise. It stops at the first
+empty directory.
+"""
+
+import os
+import configuration
+
+cfg = configuration.read()
+
+if cfg and "imports" in cfg and "mounts" in cfg["imports"]:
+
+	mounts = cfg["imports"]["mounts"]
+	for item in mounts:
+		with os.scandir(item) as contents:
+			if not any(contents):
+				exit(1)
+
+else:
+	print("No mounts in configuration")
+
+exit(0)

bin/configuration.py

@@ -1,4 +1,5 @@
 import os
+import pathlib
 from glob import glob
 from yaml import full_load as _load
 
@@ -11,6 +12,18 @@ surveys should be under $HOME/etc/surveys/*.yaml. In both cases,
 $HOME is the home directory of the user running this script.
 """
 
+def is_relative_to(it, other):
+	"""
+	is_relative_to() is not present version Python 3.9, so we
+	need this kludge to get Dougal to run on OpenSUSE 15.4
+	"""
+
+	if "is_relative_to" in dir(it):
+		return it.is_relative_to(other)
+
+	return str(it.absolute()).startswith(str(other.absolute()))
+
+
 prefix = os.environ.get("DOUGAL_ROOT", os.environ.get("HOME", ".")+"/software")
 
 DOUGAL_ROOT = os.environ.get("DOUGAL_ROOT", os.environ.get("HOME", ".")+"/software")
@@ -54,6 +67,10 @@ def files (globspec = None, include_archived = False):
 	quickly and temporarily “disabling” a survey configuration by renaming
 	the relevant file.
 	"""
+
+	print("This method is obsolete")
+	return
+
 	tuples = []
 
 	if globspec is None:
@@ -87,3 +104,73 @@ def rxflags (flagstr):
 	for flag in flagstr:
 		flags |= cases.get(flag, 0)
 	return flags
+
+def translate_path (file):
+	"""
+	Translate a path from a Dougal import directory to an actual
+	physical path on disk.
+
+	Any user files accessible by Dougal must be under a path prefixed
+	by `(config.yaml).imports.paths`. The value of `imports.paths` may
+	be either a string, in which case this represents the prefix under
+	which all Dougal data resides, or a dictionary where the keys are
+	logical paths and their values the corresponding physical path.
+	"""
+	cfg = read()
+	root = pathlib.Path(DOUGAL_ROOT)
+	filepath = pathlib.Path(file).resolve()
+	import_paths = cfg["imports"]["paths"]
+
+	if filepath.is_absolute():
+		if type(import_paths) == str:
+			# Substitute the root for the real physical path
+			# NOTE: `root` deals with import_paths not being absolute
+			prefix = root.joinpath(pathlib.Path(import_paths)).resolve()
+			return str(pathlib.Path(prefix).joinpath(*filepath.parts[2:]))
+		else:
+			# Look for a match on the second path element
+			if filepath.parts[1] in import_paths:
+				# NOTE: `root` deals with import_paths[…] not being absolute
+				prefix = root.joinpath(import_paths[filepath.parts[1]])
+				return str(pathlib.Path(prefix).joinpath(*filepath.parts[2:]))
+			else:
+				# This path is invalid
+				raise TypeError("invalid path or file: {0!r}".format(filepath))
+	else:
+		# A relative filepath is always resolved relative to the logical root
+		root = pathlib.Path("/")
+		return translate_path(root.joinpath(filepath))
+
+def untranslate_path (file):
+	"""
+	Attempt to convert a physical path into a logical one.
+	See `translate_path()` above for details.
+	"""
+	cfg = read()
+	dougal_root = pathlib.Path(DOUGAL_ROOT)
+	filepath = pathlib.Path(file).resolve()
+	import_paths = cfg["imports"]["paths"]
+	physical_root = pathlib.Path("/")
+
+	if filepath.is_absolute():
+		if type(import_paths) == str:
+			if is_relative_to(filepath, import_paths):
+				physical_root = pathlib.Path("/")
+				physical_prefix = pathlib.Path(import_paths)
+				return str(root.joinpath(filepath.relative_to(physical_prefix)))
+			else:
+				raise TypeError("invalid path or file: {0!r}".format(filepath))
+		else:
+			for key, value in import_paths.items():
+				value = dougal_root.joinpath(value)
+				physical_prefix = pathlib.Path(value)
+				if is_relative_to(filepath, physical_prefix):
+					logical_prefix = physical_root.joinpath(pathlib.Path(key)).resolve()
+					return str(logical_prefix.joinpath(filepath.relative_to(physical_prefix)))
+
+			# If we got here with no matches, this is not a valid
+			# Dougal data path
+			raise TypeError("invalid path or file: {0!r}".format(filepath))
+	else:
+		# A relative filepath is always resolved relative to DOUGAL_ROOT
+		return untranslate_path(root.joinpath(filepath))

bin/daily_tasks.py

@@ -11,11 +11,9 @@ from datastore import Datastore
 
 if __name__ == '__main__':
 
-	print("Reading configuration")
-	surveys = configuration.surveys()
-
 	print("Connecting to database")
 	db = Datastore()
+	surveys = db.surveys()
 
 	print("Reading surveys")
 	for survey in surveys:

bin/datastore.py

@@ -52,7 +52,7 @@ class Datastore:
 
 		self.conn = psycopg2.connect(configuration.read()["db"]["connection_string"], **opts)
 
-	def set_autocommit(value = True):
+	def set_autocommit(self, value = True):
 		"""
 		Enable or disable autocommit.
 
@@ -95,7 +95,7 @@ class Datastore:
 				cursor.execute(qry, (filepath,))
 				results = cursor.fetchall()
 				if len(results):
-					return (filepath, file_hash(filepath)) in results
+					return (filepath, file_hash(configuration.translate_path(filepath))) in results
 
 
 	def add_file(self, path, cursor = None):
@@ -107,7 +107,8 @@ class Datastore:
 		else:
 			cur = cursor
 
-		hash = file_hash(path)
+		realpath = configuration.translate_path(path)
+		hash = file_hash(realpath)
 		qry = "CALL add_file(%s, %s);"
 		cur.execute(qry, (path, hash))
 		if cursor is None:
@@ -176,7 +177,7 @@ class Datastore:
 		else:
 			cur = cursor
 
-		hash = file_hash(path)
+		hash = file_hash(configuration.translate_path(path))
 		qry = """
 			UPDATE raw_lines rl
 			SET ntbp = %s
@@ -588,7 +589,63 @@ class Datastore:
 			# We do not commit if we've been passed a cursor, instead
 			# we assume that we are in the middle of a transaction
 
+	def get_file_data(self, path, cursor = None):
+		"""
+		Retrieve arbitrary data associated with a file.
+		"""
 
+		if cursor is None:
+			cur = self.conn.cursor()
+		else:
+			cur = cursor
+
+		realpath = configuration.translate_path(path)
+		hash = file_hash(realpath)
+
+		qry = """
+			SELECT data
+			FROM file_data
+			WHERE hash = %s;
+		"""
+
+		cur.execute(qry, (hash,))
+		res = cur.fetchone()
+
+		if cursor is None:
+			self.maybe_commit()
+			# We do not commit if we've been passed a cursor, instead
+			# we assume that we are in the middle of a transaction
+		return res[0]
+
+	def surveys (self, include_archived = False):
+		"""
+		Return list of survey definitions.
+		"""
+
+		if self.conn is None:
+			self.connect()
+
+		if include_archived:
+			qry = """
+				SELECT meta
+				FROM public.projects;
+			"""
+		else:
+			qry = """
+				SELECT meta
+				FROM public.projects
+				WHERE NOT (meta->'archived')::boolean IS true
+			"""
+
+		with self.conn:
+			with self.conn.cursor() as cursor:
+
+				cursor.execute(qry)
+				results = cursor.fetchall()
+				return [r[0] for r in results if r[0]]
+
+
+	# TODO Does this need tweaking on account of #246?
 	def apply_survey_configuration(self, cursor = None):
 		if cursor is None:
 			cur = self.conn.cursor()

bin/housekeep_database.py

@@ -9,11 +9,9 @@ from datastore import Datastore
 
 if __name__ == '__main__':
 
-	print("Reading configuration")
-	surveys = configuration.surveys()
-
 	print("Connecting to database")
 	db = Datastore()
+	surveys = db.surveys()
 
 	print("Reading surveys")
 	for survey in surveys:

bin/import_final_p111.py

@@ -51,12 +51,11 @@ def del_pending_remark(db, sequence):
 if __name__ == '__main__':
 
 	print("Reading configuration")
-	surveys = configuration.surveys()
 	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
-	db.connect()
+	surveys = db.surveys()
 
 	print("Reading surveys")
 	for survey in surveys:
@@ -77,29 +76,31 @@ if __name__ == '__main__':
 			pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])
 
 		for fileprefix in final_p111["paths"]:
-			print(f"Path prefix: {fileprefix}")
+			realprefix = configuration.translate_path(fileprefix)
+			print(f"Path prefix: {fileprefix} → {realprefix}")
 
 			for globspec in final_p111["globs"]:
-				for filepath in pathlib.Path(fileprefix).glob(globspec):
-					filepath = str(filepath)
-					print(f"Found {filepath}")
+				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
+					physical_filepath = str(physical_filepath)
+					logical_filepath = configuration.untranslate_path(physical_filepath)
+					print(f"Found {logical_filepath}")
 
 					pending = False
 					if pendingRx:
-						pending = pendingRx.search(filepath) is not None
+						pending = pendingRx.search(physical_filepath) is not None
 
-					if not db.file_in_db(filepath):
+					if not db.file_in_db(logical_filepath):
 
-						age = time.time() - os.path.getmtime(filepath)
+						age = time.time() - os.path.getmtime(physical_filepath)
 						if age < file_min_age:
-							print("Skipping file because too new", filepath)
+							print("Skipping file because too new", logical_filepath)
 							continue
 
 						print("Importing")
 
-						match = rx.match(os.path.basename(filepath))
+						match = rx.match(os.path.basename(logical_filepath))
 						if not match:
-							error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})"
+							error_message = f"File path not match the expected format! ({logical_filepath} ~ {pattern['regex']})"
 							print(error_message, file=sys.stderr)
 							print("This file will be ignored!")
 							continue
@@ -108,21 +109,21 @@ if __name__ == '__main__':
 						file_info["meta"] = {}
 
 						if pending:
-							print("Skipping / removing final file because marked as PENDING", filepath)
+							print("Skipping / removing final file because marked as PENDING", logical_filepath)
 							db.del_sequence_final(file_info["sequence"])
 							add_pending_remark(db, file_info["sequence"])
 							continue
 						else:
 							del_pending_remark(db, file_info["sequence"])
 
-						p111_data = p111.from_file(filepath)
+						p111_data = p111.from_file(physical_filepath)
 
 						print("Saving")
 
 						p111_records = p111.p111_type("S", p111_data)
 						file_info["meta"]["lineName"] = p111.line_name(p111_data)
 
-						db.save_final_p111(p111_records, file_info, filepath, survey["epsg"])
+						db.save_final_p111(p111_records, file_info, logical_filepath, survey["epsg"])
 					else:
 						print("Already in DB")
 						if pending:

bin/import_map_layers.py

@@ -0,0 +1,127 @@
+#!/usr/bin/python3
+
+"""
+Import SmartSource data.
+
+For each survey in configuration.surveys(), check for new
+or modified final gun header files and (re-)import them into the
+database.
+"""
+
+import os
+import sys
+import pathlib
+import re
+import time
+import json
+import configuration
+from datastore import Datastore
+
+if __name__ == '__main__':
+	"""
+	Imports map layers from the directories defined in the configuration object
+	`import.map.layers`. The content of that key is an object with the following
+	structure:
+
+	{
+		layer1Name: [
+			format: "geojson",
+			path: "…", // Logical path to a directory
+			globs: [
+				"**/*.geojson", // List of globs matching map data files
+				…
+			]
+		],
+
+		layer2Name: …
+		…
+	}
+	"""
+
+
+	def process (layer_name, layer, physical_filepath):
+		physical_filepath = str(physical_filepath)
+		logical_filepath = configuration.untranslate_path(physical_filepath)
+		print(f"Found {logical_filepath}")
+
+		if not db.file_in_db(logical_filepath):
+
+			age = time.time() - os.path.getmtime(physical_filepath)
+			if age < file_min_age:
+				print("Skipping file because too new", logical_filepath)
+				return
+
+			print("Importing")
+
+			file_info = {
+				"type": "map_layer",
+				"format": layer["format"],
+				"name": layer_name,
+				"tooltip": layer.get("tooltip"),
+				"popup": layer.get("popup")
+			}
+
+			db.save_file_data(logical_filepath, json.dumps(file_info))
+
+		else:
+			file_info = db.get_file_data(logical_filepath)
+			dirty = False
+			if file_info:
+				if file_info["name"] != layer_name:
+					print("Renaming to", layer_name)
+					file_info["name"] = layer_name
+					dirty = True
+				if file_info.get("tooltip") != layer.get("tooltip"):
+					print("Changing tooltip to", layer.get("tooltip") or "null")
+					file_info["tooltip"] = layer.get("tooltip")
+					dirty = True
+				if file_info.get("popup") != layer.get("popup"):
+					print("Changing popup to", layer.get("popup") or "null")
+					file_info["popup"] = layer.get("popup")
+					dirty = True
+
+			if dirty:
+				db.save_file_data(logical_filepath, json.dumps(file_info))
+			else:
+				print("Already in DB")
+
+
+	print("Reading configuration")
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
+
+	print("Connecting to database")
+	db = Datastore()
+	surveys = db.surveys()
+
+	print("Reading surveys")
+	for survey in surveys:
+		print(f'Survey: {survey["id"]} ({survey["schema"]})')
+
+		db.set_survey(survey["schema"])
+
+		try:
+			map_layers = survey["imports"]["map"]["layers"]
+		except KeyError:
+			print("No map layers defined")
+			continue
+
+		for layer_name, layer_items in map_layers.items():
+
+			for layer in layer_items:
+				fileprefix = layer["path"]
+				realprefix = configuration.translate_path(fileprefix)
+
+				if os.path.isfile(realprefix):
+
+					process(layer_name, layer, realprefix)
+
+				elif os.path.isdir(realprefix):
+
+					if not "globs" in layer:
+						layer["globs"] = [ "**/*.geojson" ]
+
+					for globspec in layer["globs"]:
+						for physical_filepath in pathlib.Path(realprefix).glob(globspec):
+							process(layer_name, layer, physical_filepath)
+
+	print("Done")

bin/import_preplots.py

@@ -17,29 +17,31 @@ from datastore import Datastore
 
 if __name__ == '__main__':
 
-	print("Reading configuration")
-	surveys = configuration.surveys()
-	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
-
 	print("Connecting to database")
 	db = Datastore()
+	surveys = db.surveys()
+
+	print("Reading configuration")
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Reading surveys")
 	for survey in surveys:
 		print(f'Survey: {survey["id"]} ({survey["schema"]})')
 		db.set_survey(survey["schema"])
 		for file in survey["preplots"]:
+			realpath = configuration.translate_path(file["path"])
+
 			print(f"Preplot: {file['path']}")
 			if not db.file_in_db(file["path"]):
 
-				age = time.time() - os.path.getmtime(file["path"])
+				age = time.time() - os.path.getmtime(realpath)
 				if age < file_min_age:
 					print("Skipping file because too new", file["path"])
 					continue
 
 				print("Importing")
 				try:
-					preplot = preplots.from_file(file)
+					preplot = preplots.from_file(file, realpath)
 				except FileNotFoundError:
 					print(f"File does not exist: {file['path']}", file=sys.stderr)
 					continue

bin/import_raw_p111.py

@@ -20,12 +20,11 @@ from datastore import Datastore
 if __name__ == '__main__':
 
 	print("Reading configuration")
-	surveys = configuration.surveys()
 	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
-	db.connect()
+	surveys = db.surveys()
 
 	print("Reading surveys")
 	for survey in surveys:
@@ -46,30 +45,32 @@ if __name__ == '__main__':
 			ntbpRx = re.compile(survey["raw"]["ntbp"]["pattern"]["regex"])
 
 		for fileprefix in raw_p111["paths"]:
-			print(f"Path prefix: {fileprefix}")
+			realprefix = configuration.translate_path(fileprefix)
+			print(f"Path prefix: {fileprefix} → {realprefix}")
 
 			for globspec in raw_p111["globs"]:
-				for filepath in pathlib.Path(fileprefix).glob(globspec):
-					filepath = str(filepath)
-					print(f"Found {filepath}")
+				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
+					physical_filepath = str(physical_filepath)
+					logical_filepath = configuration.untranslate_path(physical_filepath)
+					print(f"Found {logical_filepath}")
 
 					if ntbpRx:
-						ntbp = ntbpRx.search(filepath) is not None
+						ntbp = ntbpRx.search(physical_filepath) is not None
 					else:
 						ntbp = False
 
-					if not db.file_in_db(filepath):
+					if not db.file_in_db(logical_filepath):
 
-						age = time.time() - os.path.getmtime(filepath)
+						age = time.time() - os.path.getmtime(physical_filepath)
 						if age < file_min_age:
-							print("Skipping file because too new", filepath)
+							print("Skipping file because too new", logical_filepath)
 							continue
 
 						print("Importing")
 
-						match = rx.match(os.path.basename(filepath))
+						match = rx.match(os.path.basename(logical_filepath))
 						if not match:
-							error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})"
+							error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
 							print(error_message, file=sys.stderr)
 							print("This file will be ignored!")
 							continue
@@ -77,7 +78,7 @@ if __name__ == '__main__':
 						file_info = dict(zip(pattern["captures"], match.groups()))
 						file_info["meta"] = {}
 
-						p111_data = p111.from_file(filepath)
+						p111_data = p111.from_file(physical_filepath)
 
 						print("Saving")
 
@@ -85,7 +86,7 @@ if __name__ == '__main__':
 						if len(p111_records):
 							file_info["meta"]["lineName"] = p111.line_name(p111_data)
 
-							db.save_raw_p111(p111_records, file_info, filepath, survey["epsg"], ntbp=ntbp)
+							db.save_raw_p111(p111_records, file_info, logical_filepath, survey["epsg"], ntbp=ntbp)
 						else:
 							print("No source records found in file")
 					else:
@@ -93,7 +94,7 @@ if __name__ == '__main__':
 
 					# Update the NTBP status to whatever the latest is,
 					# as it might have changed.
-					db.set_ntbp(filepath, ntbp)
+					db.set_ntbp(logical_filepath, ntbp)
 					if ntbp:
 						print("Sequence is NTBP")
 

bin/import_smsrc.py

@@ -20,12 +20,11 @@ from datastore import Datastore
 if __name__ == '__main__':
 
 	print("Reading configuration")
-	surveys = configuration.surveys()
 	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
-	db.connect()
+	surveys = db.surveys()
 
 	print("Reading surveys")
 	for survey in surveys:
@@ -47,36 +46,38 @@ if __name__ == '__main__':
 		rx = re.compile(pattern["regex"], flags)
 
 		for fileprefix in raw_smsrc["paths"]:
-			print(f"Path prefix: {fileprefix}")
+			realprefix = configuration.translate_path(fileprefix)
+			print(f"Path prefix: {fileprefix} → {realprefix}")
 
 			for globspec in raw_smsrc["globs"]:
-				for filepath in pathlib.Path(fileprefix).glob(globspec):
-					filepath = str(filepath)
-					print(f"Found {filepath}")
+				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
+					physical_filepath = str(physical_filepath)
+					logical_filepath = configuration.untranslate_path(physical_filepath)
+					print(f"Found {logical_filepath}")
 
-					if not db.file_in_db(filepath):
+					if not db.file_in_db(logical_filepath):
 
-						age = time.time() - os.path.getmtime(filepath)
+						age = time.time() - os.path.getmtime(physical_filepath)
 						if age < file_min_age:
-							print("Skipping file because too new", filepath)
+							print("Skipping file because too new", logical_filepath)
 							continue
 
 						print("Importing")
 
-						match = rx.match(os.path.basename(filepath))
+						match = rx.match(os.path.basename(logical_filepath))
 						if not match:
-							error_message = f"File path not matching the expected format! ({filepath} ~ {pattern['regex']})"
+							error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
 							print(error_message, file=sys.stderr)
 							print("This file will be ignored!")
 							continue
 
 						file_info = dict(zip(pattern["captures"], match.groups()))
 
-						smsrc_records = smsrc.from_file(filepath)
+						smsrc_records = smsrc.from_file(physical_filepath)
 
 						print("Saving")
 
-						db.save_raw_smsrc(smsrc_records, file_info, filepath)
+						db.save_raw_smsrc(smsrc_records, file_info, logical_filepath)
 					else:
 						print("Already in DB")
 

bin/import_survey_config.py

@@ -15,25 +15,4 @@ from datastore import Datastore
 
 if __name__ == '__main__':
 
-	print("Reading configuration")
-	configs = configuration.files(include_archived = True)
-
-	print("Connecting to database")
-	db = Datastore()
-	#db.connect()
-
-	print("Reading surveys")
-	for config in configs:
-		filepath = config[0]
-		survey = config[1]
-		print(f'Survey: {survey["id"]} ({filepath})')
-		db.set_survey(survey["schema"])
-		if not db.file_in_db(filepath):
-			print("Saving to DB")
-			db.save_file_data(filepath, json.dumps(survey))
-			print("Applying survey configuration")
-			db.apply_survey_configuration()
-		else:
-			print("Already in DB")
-
-	print("Done")
+	print("This function is obsolete. Returning with no action")

bin/preplots.py

@@ -4,9 +4,10 @@ import sps
 Preplot importing functions.
 """
 
-def from_file (file):
+def from_file (file, realpath = None):
+	filepath = realpath or file["path"]
 	if not "type" in file or file["type"] == "sps":
-		records = sps.from_file(file["path"], file["format"] if "format" in file else None )
+		records = sps.from_file(filepath, file["format"] if "format" in file else None )
 	else:
 		return "Not an SPS file"
 

bin/purge_deleted_files.py

@@ -13,21 +13,27 @@ from datastore import Datastore
 
 if __name__ == '__main__':
 
-	print("Reading configuration")
-	surveys = configuration.surveys()
-
 	print("Connecting to database")
 	db = Datastore()
 
+	print("Reading configuration")
+	surveys = db.surveys()
+
 	print("Reading surveys")
 	for survey in surveys:
 		print(f'Survey: {survey["id"]} ({survey["schema"]})')
 		db.set_survey(survey["schema"])
 
 		for file in db.list_files():
-			path = file[0]
-			if not os.path.exists(path):
-				print(path, "NOT FOUND")
-				db.del_file(path)
+			try:
+				path = configuration.translate_path(file[0])
+				if not os.path.exists(path):
+					print(path, "NOT FOUND")
+					db.del_file(file[0])
+			except TypeError:
+				# In case the logical path no longer matches
+				# the Dougal configuration.
+				print(file[0], "COULD NOT BE TRANSLATED TO A PHYSICAL PATH. DELETING")
+				db.del_file(file[0])
 
 	print("Done")

bin/runner.sh

@@ -90,6 +90,12 @@ function run () {
 	rm $STDOUTLOG $STDERRLOG
 }
 
+function cleanup () {
+	if [[ -f $LOCKFILE ]]; then
+		rm "$LOCKFILE"
+	fi
+}
+
 if [[ -f $LOCKFILE ]]; then
 	PID=$(cat "$LOCKFILE")
 	if pgrep -F "$LOCKFILE"; then
@@ -107,6 +113,13 @@ echo "$$" > "$LOCKFILE" || {
 }
 print_info "Start run"
 
+print_log "Check if data is accessible"
+$BINDIR/check_mounts_present.py || {
+	print_warning "Import mounts not accessible. Inhibiting all tasks!"
+	cleanup
+	exit 253
+}
+
 print_log "Purge deleted files"
 run $BINDIR/purge_deleted_files.py
 
@@ -119,18 +132,21 @@ run $BINDIR/import_preplots.py
 print_log "Import raw P1/11"
 run $BINDIR/import_raw_p111.py
 
-print_log "Import raw P1/90"
-run $BINDIR/import_raw_p190.py
+#print_log "Import raw P1/90"
+#run $BINDIR/import_raw_p190.py
 
 print_log "Import final P1/11"
 run $BINDIR/import_final_p111.py
 
-print_log "Import final P1/90"
-run $BINDIR/import_final_p190.py
+#print_log "Import final P1/90"
+#run $BINDIR/import_final_p190.py
 
 print_log "Import SmartSource data"
 run $BINDIR/import_smsrc.py
 
+print_log "Import map user layers"
+run $BINDIR/import_map_layers.py
+
 # if [[ -z "$RUNNER_NOEXPORT" ]]; then
 # 	print_log "Export system data"
 # 	run $BINDIR/system_exports.py

etc/config.example.yaml

@@ -32,6 +32,25 @@ imports:
     # least this many seconds ago.
     file_min_age: 60
 
+    # These paths refer to remote mounts which must be present in order
+    # for imports to work. If any of these paths are empty, import actions
+    # (including data deletion) will be inhibited. This is to cope with
+    # things like transient network failures.
+    mounts:
+        - /srv/mnt/Data
+
+    # These paths can be exposed to end users via the API. They should
+    # contain the locations were project data, or any other user data
+    # that needs to be accessible by Dougal, is located.
+    #
+    # This key can be either a string or an object:
+    # - If a string, it points to the root path for Dougal-accessible data.
+    # - If an object, there is an implicit root and the first-level
+    #   paths are denoted by the keys, with the values being their
+    #   respective physical paths.
+    # Non-absolute paths are relative to $DOUGAL_ROOT.
+    paths: /srv/mnt/Data
+
 queues:
     asaqc:
         request:

etc/db/database-version.sql

@@ -1,3 +1,5 @@
-INSERT INTO public.info VALUES ('version', '{"db_schema": "0.3.12"}')
+\connect dougal
+
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.2"}')
 ON CONFLICT (key) DO UPDATE
-	SET value = public.info.value || '{"db_schema": "0.3.12"}' WHERE public.info.key = 'version';
+	SET value = public.info.value || '{"db_schema": "0.4.2"}' WHERE public.info.key = 'version';

etc/db/schema-template.sql

@@ -2,8 +2,8 @@
 -- PostgreSQL database dump
 --
 
--- Dumped from database version 14.2
--- Dumped by pg_dump version 14.2
+-- Dumped from database version 14.8
+-- Dumped by pg_dump version 14.9
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -70,173 +70,171 @@ If the path matches that of an existing entry, delete that entry (which cascades
 CREATE PROCEDURE _SURVEY__TEMPLATE_.adjust_planner()
     LANGUAGE plpgsql
     AS $$
-	DECLARE
-		_planner_config jsonb;
-		_planned_line planned_lines%ROWTYPE;
-		_lag interval;
-		_last_sequence sequences_summary%ROWTYPE;
-		_deltatime interval;
-		_shotinterval interval;
-		_tstamp timestamptz;
-		_incr integer;
-	BEGIN
+		DECLARE
+			_planner_config jsonb;
+			_planned_line planned_lines%ROWTYPE;
+			_lag interval;
+			_last_sequence sequences_summary%ROWTYPE;
+			_deltatime interval;
+			_shotinterval interval;
+			_tstamp timestamptz;
+			_incr integer;
+		BEGIN
 
-		SET CONSTRAINTS planned_lines_pkey DEFERRED;
+			SET CONSTRAINTS planned_lines_pkey DEFERRED;
 
-		SELECT data->'planner'
-		INTO _planner_config
-		FROM file_data
-		WHERE data ? 'planner';
+			SELECT project_configuration()->'planner'
+			INTO _planner_config;
 
-		SELECT *
-		INTO _last_sequence
-		FROM sequences_summary
-		ORDER BY sequence DESC
-		LIMIT 1;
+			SELECT *
+			INTO _last_sequence
+			FROM sequences_summary
+			ORDER BY sequence DESC
+			LIMIT 1;
 
-		SELECT *
-		INTO _planned_line
-		FROM planned_lines
-		WHERE sequence = _last_sequence.sequence AND line = _last_sequence.line;
+			SELECT *
+			INTO _planned_line
+			FROM planned_lines
+			WHERE sequence = _last_sequence.sequence AND line = _last_sequence.line;
 
-		SELECT
-			COALESCE(
-				((lead(ts0) OVER (ORDER BY sequence)) - ts1),
-				make_interval(mins => (_planner_config->>'defaultLineChangeDuration')::integer)
-			)
-		INTO _lag
-		FROM planned_lines
-		WHERE sequence = _last_sequence.sequence AND line = _last_sequence.line;
-
-		_incr = sign(_last_sequence.lsp - _last_sequence.fsp);
-
-		RAISE NOTICE '_planner_config: %', _planner_config;
-		RAISE NOTICE '_last_sequence: %', _last_sequence;
-		RAISE NOTICE '_planned_line: %', _planned_line;
-		RAISE NOTICE '_incr: %', _incr;
-
-		-- Does the latest sequence match a planned sequence?
-		IF _planned_line IS NULL THEN -- No it doesn't
-			RAISE NOTICE 'Latest sequence shot does not match a planned sequence';
-			SELECT * INTO _planned_line FROM planned_lines ORDER BY sequence ASC LIMIT 1;
-			RAISE NOTICE '_planned_line: %', _planned_line;
-
-			IF _planned_line.sequence <= _last_sequence.sequence THEN
-				RAISE NOTICE 'Renumbering the planned sequences starting from %', _planned_line.sequence + 1;
-				-- Renumber the planned sequences starting from last shot sequence number + 1
-				UPDATE planned_lines
-				SET sequence = sequence + _last_sequence.sequence - _planned_line.sequence + 1;
-			END IF;
-
-			-- The correction to make to the first planned line's ts0 will be based on either the last
-			-- sequence's EOL + default line change time or the current time, whichever is later.
-			_deltatime := GREATEST(COALESCE(_last_sequence.ts1_final, _last_sequence.ts1) + make_interval(mins => (_planner_config->>'defaultLineChangeDuration')::integer), current_timestamp) - _planned_line.ts0;
-
-			-- Is the first of the planned lines start time in the past? (±5 mins)
-			IF _planned_line.ts0 < (current_timestamp - make_interval(mins => 5)) THEN
-				RAISE NOTICE 'First planned line is in the past. Adjusting times by %', _deltatime;
-				-- Adjust the start / end time of the planned lines by assuming that we are at
-				-- `defaultLineChangeDuration` minutes away from SOL of the first planned line.
-				UPDATE planned_lines
-				SET
-					ts0 = ts0 + _deltatime,
-					ts1 = ts1 + _deltatime;
-			END IF;
-
-		ELSE -- Yes it does
-			RAISE NOTICE 'Latest sequence does match a planned sequence: %, %', _planned_line.sequence, _planned_line.line;
-
-			-- Is it online?
-			IF EXISTS(SELECT 1 FROM raw_lines_files WHERE sequence = _last_sequence.sequence AND hash = '*online*') THEN
-				-- Yes it is
-				RAISE NOTICE 'Sequence % is online', _last_sequence.sequence;
-
-				-- Let us get the SOL from the events log if we can
-				RAISE NOTICE 'Trying to set fsp, ts0 from events log FSP, FGSP';
-				WITH e AS (
-					SELECT * FROM event_log
-					WHERE
-						sequence = _last_sequence.sequence
-						AND ('FSP' = ANY(labels) OR 'FGSP' = ANY(labels))
-						ORDER BY tstamp LIMIT 1
+			SELECT
+				COALESCE(
+					((lead(ts0) OVER (ORDER BY sequence)) - ts1),
+					make_interval(mins => (_planner_config->>'defaultLineChangeDuration')::integer)
 				)
-				UPDATE planned_lines
-				SET
-					fsp = COALESCE(e.point, fsp),
-					ts0 = COALESCE(e.tstamp, ts0)
-				FROM e
-				WHERE planned_lines.sequence = _last_sequence.sequence;
+			INTO _lag
+			FROM planned_lines
+			WHERE sequence = _last_sequence.sequence AND line = _last_sequence.line;
 
-				-- Shot interval
-				_shotinterval := (_last_sequence.ts1 - _last_sequence.ts0) / abs(_last_sequence.lsp - _last_sequence.fsp);
+			_incr = sign(_last_sequence.lsp - _last_sequence.fsp);
 
-				RAISE NOTICE 'Estimating EOL from current shot interval: %', _shotinterval;
+			RAISE NOTICE '_planner_config: %', _planner_config;
+			RAISE NOTICE '_last_sequence: %', _last_sequence;
+			RAISE NOTICE '_planned_line: %', _planned_line;
+			RAISE NOTICE '_incr: %', _incr;
 
-				SELECT (abs(lsp-fsp) * _shotinterval + ts0) - ts1
-				INTO _deltatime
-				FROM planned_lines
-				WHERE sequence = _last_sequence.sequence;
+			-- Does the latest sequence match a planned sequence?
+			IF _planned_line IS NULL THEN -- No it doesn't
+				RAISE NOTICE 'Latest sequence shot does not match a planned sequence';
+				SELECT * INTO _planned_line FROM planned_lines ORDER BY sequence ASC LIMIT 1;
+				RAISE NOTICE '_planned_line: %', _planned_line;
 
-				---- Set ts1 for the current sequence
-				--UPDATE planned_lines
-				--SET
-					--ts1 = (abs(lsp-fsp) * _shotinterval) + ts0
-				--WHERE sequence = _last_sequence.sequence;
-
-				RAISE NOTICE 'Adjustment is %', _deltatime;
-
-				IF abs(EXTRACT(EPOCH FROM _deltatime)) < 8 THEN
-					RAISE NOTICE 'Adjustment too small (< 8 s), so not applying it';
-					RETURN;
+				IF _planned_line.sequence <= _last_sequence.sequence THEN
+					RAISE NOTICE 'Renumbering the planned sequences starting from %', _planned_line.sequence + 1;
+					-- Renumber the planned sequences starting from last shot sequence number + 1
+					UPDATE planned_lines
+					SET sequence = sequence + _last_sequence.sequence - _planned_line.sequence + 1;
 				END IF;
 
-				-- Adjust ts1 for the current sequence
-				UPDATE planned_lines
-				SET ts1 = ts1 + _deltatime
-				WHERE sequence = _last_sequence.sequence;
+				-- The correction to make to the first planned line's ts0 will be based on either the last
+				-- sequence's EOL + default line change time or the current time, whichever is later.
+				_deltatime := GREATEST(COALESCE(_last_sequence.ts1_final, _last_sequence.ts1) + make_interval(mins => (_planner_config->>'defaultLineChangeDuration')::integer), current_timestamp) - _planned_line.ts0;
 
-				-- Now shift all sequences after
-				UPDATE planned_lines
-				SET ts0 = ts0 + _deltatime, ts1 = ts1 + _deltatime
-				WHERE sequence > _last_sequence.sequence;
+				-- Is the first of the planned lines start time in the past? (±5 mins)
+				IF _planned_line.ts0 < (current_timestamp - make_interval(mins => 5)) THEN
+					RAISE NOTICE 'First planned line is in the past. Adjusting times by %', _deltatime;
+					-- Adjust the start / end time of the planned lines by assuming that we are at
+					-- `defaultLineChangeDuration` minutes away from SOL of the first planned line.
+					UPDATE planned_lines
+					SET
+						ts0 = ts0 + _deltatime,
+						ts1 = ts1 + _deltatime;
+				END IF;
 
-				RAISE NOTICE 'Deleting planned sequences before %', _planned_line.sequence;
-				-- Remove all previous planner entries.
-				DELETE
-				FROM planned_lines
-				WHERE sequence < _last_sequence.sequence;
+			ELSE -- Yes it does
+				RAISE NOTICE 'Latest sequence does match a planned sequence: %, %', _planned_line.sequence, _planned_line.line;
 
-			ELSE
-				-- No it isn't
-				RAISE NOTICE 'Sequence % is offline', _last_sequence.sequence;
+				-- Is it online?
+				IF EXISTS(SELECT 1 FROM raw_lines_files WHERE sequence = _last_sequence.sequence AND hash = '*online*') THEN
+					-- Yes it is
+					RAISE NOTICE 'Sequence % is online', _last_sequence.sequence;
 
-				-- We were supposed to finish at _planned_line.ts1 but we finished at:
-				_tstamp := GREATEST(COALESCE(_last_sequence.ts1_final, _last_sequence.ts1), current_timestamp);
-				-- WARNING Next line is for testing only
-				--_tstamp := COALESCE(_last_sequence.ts1_final, _last_sequence.ts1);
-				-- So we need to adjust timestamps by:
-				_deltatime := _tstamp - _planned_line.ts1;
+					-- Let us get the SOL from the events log if we can
+					RAISE NOTICE 'Trying to set fsp, ts0 from events log FSP, FGSP';
+					WITH e AS (
+						SELECT * FROM event_log
+						WHERE
+							sequence = _last_sequence.sequence
+							AND ('FSP' = ANY(labels) OR 'FGSP' = ANY(labels))
+							ORDER BY tstamp LIMIT 1
+					)
+					UPDATE planned_lines
+					SET
+						fsp = COALESCE(e.point, fsp),
+						ts0 = COALESCE(e.tstamp, ts0)
+					FROM e
+					WHERE planned_lines.sequence = _last_sequence.sequence;
 
-				RAISE NOTICE 'Planned end: %, actual end: % (%, %)', _planned_line.ts1, _tstamp, _planned_line.sequence, _last_sequence.sequence;
-				RAISE NOTICE 'Shifting times by % for sequences > %', _deltatime, _planned_line.sequence;
-				-- NOTE: This won't work if sequences are not, err… sequential.
-				-- NOTE: This has been known to happen in 2020.
-				UPDATE planned_lines
-				SET
-					ts0 = ts0 + _deltatime,
-					ts1 = ts1 + _deltatime
-				WHERE sequence > _planned_line.sequence;
+					-- Shot interval
+					_shotinterval := (_last_sequence.ts1 - _last_sequence.ts0) / abs(_last_sequence.lsp - _last_sequence.fsp);
 
-				RAISE NOTICE 'Deleting planned sequences up to %', _planned_line.sequence;
-				-- Remove all previous planner entries.
-				DELETE
-				FROM planned_lines
-				WHERE sequence <= _last_sequence.sequence;
+					RAISE NOTICE 'Estimating EOL from current shot interval: %', _shotinterval;
+
+					SELECT (abs(lsp-fsp) * _shotinterval + ts0) - ts1
+					INTO _deltatime
+					FROM planned_lines
+					WHERE sequence = _last_sequence.sequence;
+
+					---- Set ts1 for the current sequence
+					--UPDATE planned_lines
+					--SET
+						--ts1 = (abs(lsp-fsp) * _shotinterval) + ts0
+					--WHERE sequence = _last_sequence.sequence;
+
+					RAISE NOTICE 'Adjustment is %', _deltatime;
+
+					IF abs(EXTRACT(EPOCH FROM _deltatime)) < 8 THEN
+						RAISE NOTICE 'Adjustment too small (< 8 s), so not applying it';
+						RETURN;
+					END IF;
+
+					-- Adjust ts1 for the current sequence
+					UPDATE planned_lines
+					SET ts1 = ts1 + _deltatime
+					WHERE sequence = _last_sequence.sequence;
+
+					-- Now shift all sequences after
+					UPDATE planned_lines
+					SET ts0 = ts0 + _deltatime, ts1 = ts1 + _deltatime
+					WHERE sequence > _last_sequence.sequence;
+
+					RAISE NOTICE 'Deleting planned sequences before %', _planned_line.sequence;
+					-- Remove all previous planner entries.
+					DELETE
+					FROM planned_lines
+					WHERE sequence < _last_sequence.sequence;
+
+				ELSE
+					-- No it isn't
+					RAISE NOTICE 'Sequence % is offline', _last_sequence.sequence;
+
+					-- We were supposed to finish at _planned_line.ts1 but we finished at:
+					_tstamp := GREATEST(COALESCE(_last_sequence.ts1_final, _last_sequence.ts1), current_timestamp);
+					-- WARNING Next line is for testing only
+					--_tstamp := COALESCE(_last_sequence.ts1_final, _last_sequence.ts1);
+					-- So we need to adjust timestamps by:
+					_deltatime := _tstamp - _planned_line.ts1;
+
+					RAISE NOTICE 'Planned end: %, actual end: % (%, %)', _planned_line.ts1, _tstamp, _planned_line.sequence, _last_sequence.sequence;
+					RAISE NOTICE 'Shifting times by % for sequences > %', _deltatime, _planned_line.sequence;
+					-- NOTE: This won't work if sequences are not, err… sequential.
+					-- NOTE: This has been known to happen in 2020.
+					UPDATE planned_lines
+					SET
+						ts0 = ts0 + _deltatime,
+						ts1 = ts1 + _deltatime
+					WHERE sequence > _planned_line.sequence;
+
+					RAISE NOTICE 'Deleting planned sequences up to %', _planned_line.sequence;
+					-- Remove all previous planner entries.
+					DELETE
+					FROM planned_lines
+					WHERE sequence <= _last_sequence.sequence;
+
+				END IF;
 
 			END IF;
-
-		END IF;
-	END;
+		END;
 	$$;
 
 
@@ -367,8 +365,8 @@ COMMENT ON PROCEDURE _SURVEY__TEMPLATE_.augment_event_data(IN maxspan numeric) I
 CREATE FUNCTION _SURVEY__TEMPLATE_.binning_parameters() RETURNS jsonb
     LANGUAGE sql STABLE LEAKPROOF PARALLEL SAFE
     AS $$
-SELECT data->'binning' binning FROM file_data WHERE data->>'binning' IS NOT NULL LIMIT 1;
-$$;
+	SELECT project_configuration()->'binning' binning;
+	$$;
 
 
 ALTER FUNCTION _SURVEY__TEMPLATE_.binning_parameters() OWNER TO postgres;
@@ -1042,6 +1040,39 @@ ALTER PROCEDURE _SURVEY__TEMPLATE_.log_midnight_shots(IN dt0 date, IN dt1 date)
 COMMENT ON PROCEDURE _SURVEY__TEMPLATE_.log_midnight_shots(IN dt0 date, IN dt1 date) IS 'Add midnight shots between two dates dt0 and dt1 to the event_log, unless the events already exist.';
 
 
+--
+-- Name: project_configuration(); Type: FUNCTION; Schema: _SURVEY__TEMPLATE_; Owner: postgres
+--
+
+CREATE FUNCTION _SURVEY__TEMPLATE_.project_configuration() RETURNS jsonb
+    LANGUAGE plpgsql
+    AS $$
+	DECLARE
+		schema_name text;
+		configuration jsonb;
+	BEGIN
+
+		SELECT nspname
+		INTO schema_name
+		FROM pg_namespace
+		WHERE oid = (
+			SELECT pronamespace
+			FROM pg_proc
+			WHERE oid = 'project_configuration'::regproc::oid
+		);
+
+		SELECT meta
+		INTO configuration
+		FROM public.projects
+		WHERE schema = schema_name;
+
+		RETURN configuration;
+	END
+	$$;
+
+
+ALTER FUNCTION _SURVEY__TEMPLATE_.project_configuration() OWNER TO postgres;
+
 --
 -- Name: replace_placeholders(text, timestamp with time zone, integer, integer); Type: FUNCTION; Schema: _SURVEY__TEMPLATE_; Owner: postgres
 --
@@ -1488,9 +1519,9 @@ CREATE VIEW _SURVEY__TEMPLATE_.final_lines_summary AS
     s.ts1,
     (s.ts1 - s.ts0) AS duration,
     s.num_points,
-    (( SELECT count(*) AS count
-           FROM _SURVEY__TEMPLATE_.preplot_points
-          WHERE ((preplot_points.line = fl.line) AND (((preplot_points.point >= s.fsp) AND (preplot_points.point <= s.lsp)) OR ((preplot_points.point >= s.lsp) AND (preplot_points.point <= s.fsp))))) - s.num_points) AS missing_shots,
+    ( SELECT count(*) AS count
+      FROM _SURVEY__TEMPLATE_.missing_sequence_final_points
+      WHERE missing_sequence_final_points.sequence = s.sequence) AS missing_shots,
     s.length,
     s.azimuth,
     fl.remarks,
@@ -2137,9 +2168,9 @@ CREATE VIEW _SURVEY__TEMPLATE_.raw_lines_summary AS
     (s.ts1 - s.ts0) AS duration,
     s.num_points,
     s.num_preplots,
-    (( SELECT count(*) AS count
-           FROM _SURVEY__TEMPLATE_.preplot_points
-          WHERE ((preplot_points.line = rl.line) AND (((preplot_points.point >= s.fsp) AND (preplot_points.point <= s.lsp)) OR ((preplot_points.point >= s.lsp) AND (preplot_points.point <= s.fsp))))) - s.num_preplots) AS missing_shots,
+    (SELECT count(*) AS count
+      FROM _SURVEY__TEMPLATE_.missing_sequence_raw_points
+      WHERE missing_sequence_raw_points.sequence = s.sequence) AS missing_shots,
     s.length,
     s.azimuth,
     rl.remarks,

etc/db/upgrades/upgrade26-v0.3.13-fix-missing-shots-summary.sql

@@ -0,0 +1,162 @@
+-- Fix wrong number of missing shots in summary views
+--
+-- New schema version: 0.3.13
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- Fixes a bug in the `final_lines_summary` and `raw_lines_summary` views
+-- which results in the number of missing shots being miscounted on jobs
+-- using three sources.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_survey_schema (schema_name text) AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+
+
+	CREATE OR REPLACE VIEW raw_lines_summary AS
+	WITH summary AS (
+			SELECT DISTINCT rs.sequence,
+				first_value(rs.point) OVER w AS fsp,
+				last_value(rs.point) OVER w AS lsp,
+				first_value(rs.tstamp) OVER w AS ts0,
+				last_value(rs.tstamp) OVER w AS ts1,
+				count(rs.point) OVER w AS num_points,
+				count(pp.point) OVER w AS num_preplots,
+				public.st_distance(first_value(rs.geometry) OVER w, last_value(rs.geometry) OVER w) AS length,
+				((public.st_azimuth(first_value(rs.geometry) OVER w, last_value(rs.geometry) OVER w) * (180)::double precision) / pi()) AS azimuth
+			FROM (raw_shots rs
+				LEFT JOIN preplot_points pp USING (line, point))
+			WINDOW w AS (PARTITION BY rs.sequence ORDER BY rs.tstamp ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+			)
+	SELECT rl.sequence,
+		rl.line,
+		s.fsp,
+		s.lsp,
+		s.ts0,
+		s.ts1,
+		(s.ts1 - s.ts0) AS duration,
+		s.num_points,
+		s.num_preplots,
+		(SELECT count(*) AS count
+			FROM missing_sequence_raw_points
+			WHERE missing_sequence_raw_points.sequence = s.sequence) AS missing_shots,
+		s.length,
+		s.azimuth,
+		rl.remarks,
+		rl.ntbp,
+		rl.meta
+	FROM (summary s
+		JOIN raw_lines rl USING (sequence));
+
+
+	CREATE OR REPLACE VIEW final_lines_summary AS
+	WITH summary AS (
+			SELECT DISTINCT fs.sequence,
+				first_value(fs.point) OVER w AS fsp,
+				last_value(fs.point) OVER w AS lsp,
+				first_value(fs.tstamp) OVER w AS ts0,
+				last_value(fs.tstamp) OVER w AS ts1,
+				count(fs.point) OVER w AS num_points,
+				public.st_distance(first_value(fs.geometry) OVER w, last_value(fs.geometry) OVER w) AS length,
+				((public.st_azimuth(first_value(fs.geometry) OVER w, last_value(fs.geometry) OVER w) * (180)::double precision) / pi()) AS azimuth
+			FROM final_shots fs
+			WINDOW w AS (PARTITION BY fs.sequence ORDER BY fs.tstamp ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+			)
+	SELECT fl.sequence,
+		fl.line,
+		s.fsp,
+		s.lsp,
+		s.ts0,
+		s.ts1,
+		(s.ts1 - s.ts0) AS duration,
+		s.num_points,
+		( SELECT count(*) AS count
+			FROM missing_sequence_final_points
+			WHERE missing_sequence_final_points.sequence = s.sequence) AS missing_shots,
+		s.length,
+		s.azimuth,
+		fl.remarks,
+		fl.meta
+	FROM (summary s
+		JOIN final_lines fl USING (sequence));
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.3.13' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.3.12' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL pg_temp.upgrade_survey_schema(row.schema_name);
+	END LOOP;
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_survey_schema (schema_name text);
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.3.13"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.3.13"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade27-v0.4.0-add-project_configuration-function.sql

@@ -0,0 +1,122 @@
+-- Fix wrong number of missing shots in summary views
+--
+-- New schema version: 0.4.0
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This adapts the schema to the change in how project configurations are
+-- handled (https://gitlab.com/wgp/dougal/software/-/merge_requests/29)
+-- by creating a project_configuration() function which returns the
+-- current project's configuration data.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_survey_schema (schema_name text) AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+
+	CREATE OR REPLACE FUNCTION project_configuration()
+		RETURNS jsonb
+		LANGUAGE plpgsql
+		AS $$
+	DECLARE
+		schema_name text;
+		configuration jsonb;
+	BEGIN
+
+		SELECT nspname
+		INTO schema_name
+		FROM pg_namespace
+		WHERE oid = (
+			SELECT pronamespace
+			FROM pg_proc
+			WHERE oid = 'project_configuration'::regproc::oid
+		);
+
+		SELECT meta
+		INTO configuration
+		FROM public.projects
+		WHERE schema = schema_name;
+
+		RETURN configuration;
+	END
+	$$;
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.4.0' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.3.12' AND current_db_version != '0.3.13' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL pg_temp.upgrade_survey_schema(row.schema_name);
+	END LOOP;
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_survey_schema (schema_name text);
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.0"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.4.0"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade28-v0.4.1-use-project-configuration-in-adjust_planner.sql

@@ -0,0 +1,264 @@
+-- Fix wrong number of missing shots in summary views
+--
+-- New schema version: 0.4.1
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This modifies adjust_planner() to use project_configuration()
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_survey_schema (schema_name text) AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+
+
+	CREATE OR REPLACE PROCEDURE adjust_planner()
+		LANGUAGE plpgsql
+		AS $$
+		DECLARE
+			_planner_config jsonb;
+			_planned_line planned_lines%ROWTYPE;
+			_lag interval;
+			_last_sequence sequences_summary%ROWTYPE;
+			_deltatime interval;
+			_shotinterval interval;
+			_tstamp timestamptz;
+			_incr integer;
+		BEGIN
+
+			SET CONSTRAINTS planned_lines_pkey DEFERRED;
+
+			SELECT project_configuration()->'planner'
+			INTO _planner_config;
+
+			SELECT *
+			INTO _last_sequence
+			FROM sequences_summary
+			ORDER BY sequence DESC
+			LIMIT 1;
+
+			SELECT *
+			INTO _planned_line
+			FROM planned_lines
+			WHERE sequence = _last_sequence.sequence AND line = _last_sequence.line;
+
+			SELECT
+				COALESCE(
+					((lead(ts0) OVER (ORDER BY sequence)) - ts1),
+					make_interval(mins => (_planner_config->>'defaultLineChangeDuration')::integer)
+				)
+			INTO _lag
+			FROM planned_lines
+			WHERE sequence = _last_sequence.sequence AND line = _last_sequence.line;
+
+			_incr = sign(_last_sequence.lsp - _last_sequence.fsp);
+
+			RAISE NOTICE '_planner_config: %', _planner_config;
+			RAISE NOTICE '_last_sequence: %', _last_sequence;
+			RAISE NOTICE '_planned_line: %', _planned_line;
+			RAISE NOTICE '_incr: %', _incr;
+
+			-- Does the latest sequence match a planned sequence?
+			IF _planned_line IS NULL THEN -- No it doesn't
+				RAISE NOTICE 'Latest sequence shot does not match a planned sequence';
+				SELECT * INTO _planned_line FROM planned_lines ORDER BY sequence ASC LIMIT 1;
+				RAISE NOTICE '_planned_line: %', _planned_line;
+
+				IF _planned_line.sequence <= _last_sequence.sequence THEN
+					RAISE NOTICE 'Renumbering the planned sequences starting from %', _planned_line.sequence + 1;
+					-- Renumber the planned sequences starting from last shot sequence number + 1
+					UPDATE planned_lines
+					SET sequence = sequence + _last_sequence.sequence - _planned_line.sequence + 1;
+				END IF;
+
+				-- The correction to make to the first planned line's ts0 will be based on either the last
+				-- sequence's EOL + default line change time or the current time, whichever is later.
+				_deltatime := GREATEST(COALESCE(_last_sequence.ts1_final, _last_sequence.ts1) + make_interval(mins => (_planner_config->>'defaultLineChangeDuration')::integer), current_timestamp) - _planned_line.ts0;
+
+				-- Is the first of the planned lines start time in the past? (±5 mins)
+				IF _planned_line.ts0 < (current_timestamp - make_interval(mins => 5)) THEN
+					RAISE NOTICE 'First planned line is in the past. Adjusting times by %', _deltatime;
+					-- Adjust the start / end time of the planned lines by assuming that we are at
+					-- `defaultLineChangeDuration` minutes away from SOL of the first planned line.
+					UPDATE planned_lines
+					SET
+						ts0 = ts0 + _deltatime,
+						ts1 = ts1 + _deltatime;
+				END IF;
+
+			ELSE -- Yes it does
+				RAISE NOTICE 'Latest sequence does match a planned sequence: %, %', _planned_line.sequence, _planned_line.line;
+
+				-- Is it online?
+				IF EXISTS(SELECT 1 FROM raw_lines_files WHERE sequence = _last_sequence.sequence AND hash = '*online*') THEN
+					-- Yes it is
+					RAISE NOTICE 'Sequence % is online', _last_sequence.sequence;
+
+					-- Let us get the SOL from the events log if we can
+					RAISE NOTICE 'Trying to set fsp, ts0 from events log FSP, FGSP';
+					WITH e AS (
+						SELECT * FROM event_log
+						WHERE
+							sequence = _last_sequence.sequence
+							AND ('FSP' = ANY(labels) OR 'FGSP' = ANY(labels))
+							ORDER BY tstamp LIMIT 1
+					)
+					UPDATE planned_lines
+					SET
+						fsp = COALESCE(e.point, fsp),
+						ts0 = COALESCE(e.tstamp, ts0)
+					FROM e
+					WHERE planned_lines.sequence = _last_sequence.sequence;
+
+					-- Shot interval
+					_shotinterval := (_last_sequence.ts1 - _last_sequence.ts0) / abs(_last_sequence.lsp - _last_sequence.fsp);
+
+					RAISE NOTICE 'Estimating EOL from current shot interval: %', _shotinterval;
+
+					SELECT (abs(lsp-fsp) * _shotinterval + ts0) - ts1
+					INTO _deltatime
+					FROM planned_lines
+					WHERE sequence = _last_sequence.sequence;
+
+					---- Set ts1 for the current sequence
+					--UPDATE planned_lines
+					--SET
+						--ts1 = (abs(lsp-fsp) * _shotinterval) + ts0
+					--WHERE sequence = _last_sequence.sequence;
+
+					RAISE NOTICE 'Adjustment is %', _deltatime;
+
+					IF abs(EXTRACT(EPOCH FROM _deltatime)) < 8 THEN
+						RAISE NOTICE 'Adjustment too small (< 8 s), so not applying it';
+						RETURN;
+					END IF;
+
+					-- Adjust ts1 for the current sequence
+					UPDATE planned_lines
+					SET ts1 = ts1 + _deltatime
+					WHERE sequence = _last_sequence.sequence;
+
+					-- Now shift all sequences after
+					UPDATE planned_lines
+					SET ts0 = ts0 + _deltatime, ts1 = ts1 + _deltatime
+					WHERE sequence > _last_sequence.sequence;
+
+					RAISE NOTICE 'Deleting planned sequences before %', _planned_line.sequence;
+					-- Remove all previous planner entries.
+					DELETE
+					FROM planned_lines
+					WHERE sequence < _last_sequence.sequence;
+
+				ELSE
+					-- No it isn't
+					RAISE NOTICE 'Sequence % is offline', _last_sequence.sequence;
+
+					-- We were supposed to finish at _planned_line.ts1 but we finished at:
+					_tstamp := GREATEST(COALESCE(_last_sequence.ts1_final, _last_sequence.ts1), current_timestamp);
+					-- WARNING Next line is for testing only
+					--_tstamp := COALESCE(_last_sequence.ts1_final, _last_sequence.ts1);
+					-- So we need to adjust timestamps by:
+					_deltatime := _tstamp - _planned_line.ts1;
+
+					RAISE NOTICE 'Planned end: %, actual end: % (%, %)', _planned_line.ts1, _tstamp, _planned_line.sequence, _last_sequence.sequence;
+					RAISE NOTICE 'Shifting times by % for sequences > %', _deltatime, _planned_line.sequence;
+					-- NOTE: This won't work if sequences are not, err… sequential.
+					-- NOTE: This has been known to happen in 2020.
+					UPDATE planned_lines
+					SET
+						ts0 = ts0 + _deltatime,
+						ts1 = ts1 + _deltatime
+					WHERE sequence > _planned_line.sequence;
+
+					RAISE NOTICE 'Deleting planned sequences up to %', _planned_line.sequence;
+					-- Remove all previous planner entries.
+					DELETE
+					FROM planned_lines
+					WHERE sequence <= _last_sequence.sequence;
+
+				END IF;
+
+			END IF;
+		END;
+	$$;
+
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.4.1' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.4.0' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL pg_temp.upgrade_survey_schema(row.schema_name);
+	END LOOP;
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_survey_schema (schema_name text);
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.1"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.4.1"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade29-v0.4.2-use-project-configuration-in-binning_parameters.sql

@@ -0,0 +1,98 @@
+-- Fix wrong number of missing shots in summary views
+--
+-- New schema version: 0.4.2
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This modifies binning_parameters() to use project_configuration()
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_survey_schema (schema_name text) AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+
+	CREATE OR REPLACE FUNCTION binning_parameters() RETURNS jsonb
+		LANGUAGE sql STABLE LEAKPROOF PARALLEL SAFE
+		AS $$
+	SELECT project_configuration()->'binning' binning;
+	$$;
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.4.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.4.1' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL pg_temp.upgrade_survey_schema(row.schema_name);
+	END LOOP;
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_survey_schema (schema_name text);
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.2"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.4.2"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade30-v0.4.3-large-notification-payloads.sql

@@ -0,0 +1,164 @@
+-- Support notification payloads larger than Postgres' NOTIFY limit.
+--
+-- New schema version: 0.4.3
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects the public schema only.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This creates a new table where large notification payloads are stored
+-- temporarily and from which they might be recalled by the notification
+-- listeners. It also creates a purge_notifications() procedure used to
+-- clean up old notifications from the notifications log and finally,
+-- modifies notify() to support these changes. When a large payload is
+-- encountered, the payload is stored in the notify_payloads table and
+-- a trimmed down version containing a notification_id is sent to listeners
+-- instead. Listeners can then query notify_payloads to retrieve the full
+-- payloads. It is the application layer's responsibility to delete old
+-- notifications.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_schema () AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating public schema';
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO public');
+
+	CREATE TABLE IF NOT EXISTS public.notify_payloads (
+		id SERIAL,
+		tstamp timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
+		payload text NOT NULL DEFAULT '',
+		PRIMARY KEY (id)
+	);
+
+	CREATE INDEX IF NOT EXISTS notify_payload_tstamp ON notify_payloads (tstamp);
+
+	CREATE OR REPLACE FUNCTION public.notify() RETURNS trigger
+		LANGUAGE plpgsql
+		AS $$
+	DECLARE
+	channel text := TG_ARGV[0];
+	pid text;
+	payload text;
+	notification text;
+	payload_id integer;
+	BEGIN
+
+	SELECT projects.pid INTO pid FROM projects WHERE schema = TG_TABLE_SCHEMA;
+
+	payload := json_build_object(
+	'tstamp', CURRENT_TIMESTAMP,
+	'operation', TG_OP,
+	'schema', TG_TABLE_SCHEMA,
+	'table', TG_TABLE_NAME,
+	'old', row_to_json(OLD),
+	'new', row_to_json(NEW),
+	'pid', pid
+	)::text;
+
+	IF octet_length(payload) < 1000 THEN
+	PERFORM pg_notify(channel, payload);
+	ELSE
+	-- We need to find another solution
+	-- FIXME Consider storing the payload in a temporary memory table,
+	-- referenced by some form of autogenerated ID. Then send the ID
+	-- as the payload and then it's up to the user to fetch the original
+	-- payload if interested. This needs a mechanism to expire older payloads
+	-- in the interest of conserving memory.
+
+	INSERT INTO notify_payloads (payload) VALUES (payload) RETURNING id INTO payload_id;
+
+	notification := json_build_object(
+	'tstamp', CURRENT_TIMESTAMP,
+	'operation', TG_OP,
+	'schema', TG_TABLE_SCHEMA,
+	'table', TG_TABLE_NAME,
+	'pid', pid,
+	'payload_id', payload_id
+	)::text;
+
+	PERFORM pg_notify(channel, notification);
+	RAISE INFO 'Payload over limit';
+	END IF;
+	RETURN NULL;
+	END;
+	$$;
+
+	CREATE PROCEDURE public.purge_notifications (age_seconds numeric DEFAULT 120) AS $$
+		DELETE FROM notify_payloads WHERE EXTRACT(epoch FROM CURRENT_TIMESTAMP - tstamp) > age_seconds;
+	$$ LANGUAGE sql;
+
+
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.4.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.4.2' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	-- This upgrade modified the `public` schema only, not individual
+	-- project schemas.
+	CALL pg_temp.upgrade_schema();
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_schema ();
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.3"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.4.3"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

etc/db/upgrades/upgrade31-v0.4.4-add-event_log_changes-function.sql

@@ -0,0 +1,104 @@
+-- Add event_log_changes function
+--
+-- New schema version: 0.4.4
+--
+-- ATTENTION:
+--
+-- ENSURE YOU HAVE BACKED UP THE DATABASE BEFORE RUNNING THIS SCRIPT.
+--
+--
+-- NOTE: This upgrade affects all schemas in the database.
+-- NOTE: Each application starts a transaction, which must be committed
+-- or rolled back.
+--
+-- This adds a function event_log_changes which returns the subset of
+-- events from event_log_full which have been modified on or after a
+-- given timestamp.
+--
+-- To apply, run as the dougal user:
+--
+-- psql <<EOF
+-- \i $THIS_FILE
+-- COMMIT;
+-- EOF
+--
+-- NOTE: It can be applied multiple times without ill effect.
+--
+
+BEGIN;
+
+CREATE OR REPLACE PROCEDURE pg_temp.show_notice (notice text) AS $$
+BEGIN
+	RAISE NOTICE '%', notice;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade_survey_schema (schema_name text) AS $outer$
+BEGIN
+
+	RAISE NOTICE 'Updating schema %', schema_name;
+	-- We need to set the search path because some of the trigger
+	-- functions reference other tables in survey schemas assuming
+	-- they are in the search path.
+	EXECUTE format('SET search_path TO %I,public', schema_name);
+
+	CREATE OR REPLACE FUNCTION event_log_changes(ts0 timestamptz)
+		RETURNS SETOF event_log_full
+		LANGUAGE sql
+		AS $$
+		SELECT *
+		FROM event_log_full
+		WHERE lower(validity) > ts0 OR upper(validity) IS NOT NULL AND upper(validity) > ts0
+		ORDER BY lower(validity);
+	$$;
+
+END;
+$outer$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE PROCEDURE pg_temp.upgrade () AS $outer$
+DECLARE
+	row RECORD;
+	current_db_version TEXT;
+BEGIN
+
+	SELECT value->>'db_schema' INTO current_db_version FROM public.info WHERE key = 'version';
+
+	IF current_db_version >= '0.4.4' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Patch already applied';
+	END IF;
+
+	IF current_db_version != '0.4.3' THEN
+		RAISE EXCEPTION
+			USING MESSAGE='Invalid database version: ' || current_db_version,
+				HINT='Ensure all previous patches have been applied.';
+	END IF;
+
+	FOR row IN
+		SELECT schema_name FROM information_schema.schemata
+		WHERE schema_name LIKE 'survey_%'
+		ORDER BY schema_name
+	LOOP
+		CALL pg_temp.upgrade_survey_schema(row.schema_name);
+	END LOOP;
+END;
+$outer$ LANGUAGE plpgsql;
+
+CALL pg_temp.upgrade();
+
+CALL pg_temp.show_notice('Cleaning up');
+DROP PROCEDURE pg_temp.upgrade_survey_schema (schema_name text);
+DROP PROCEDURE pg_temp.upgrade ();
+
+CALL pg_temp.show_notice('Updating db_schema version');
+INSERT INTO public.info VALUES ('version', '{"db_schema": "0.4.4"}')
+ON CONFLICT (key) DO UPDATE
+	SET value = public.info.value || '{"db_schema": "0.4.4"}' WHERE public.info.key = 'version';
+
+
+CALL pg_temp.show_notice('All done. You may now run "COMMIT;" to persist the changes');
+DROP PROCEDURE pg_temp.show_notice (notice text);
+
+--
+--NOTE Run `COMMIT;` now if all went well
+--

lib/www/client/source/package-lock.json

@@ -9,7 +9,7 @@
       "version": "0.0.0",
       "license": "UNLICENSED",
       "dependencies": {
-        "@mdi/font": "^5.6.55",
+        "@mdi/font": "^7.2.96",
         "core-js": "^3.6.5",
         "d3": "^7.0.1",
         "jwt-decode": "^3.0.0",
@@ -1763,9 +1763,9 @@
       }
     },
     "node_modules/@mdi/font": {
-      "version": "5.9.55",
-      "resolved": "https://registry.npmjs.org/@mdi/font/-/font-5.9.55.tgz",
-      "integrity": "sha512-jswRF6q3eq8NWpWiqct6q+6Fg/I7nUhrxYJfiEM8JJpap0wVJLQdbKtyS65GdlK7S7Ytnx3TTi/bmw+tBhkGmg=="
+      "version": "7.2.96",
+      "resolved": "https://registry.npmjs.org/@mdi/font/-/font-7.2.96.tgz",
+      "integrity": "sha512-e//lmkmpFUMZKhmCY9zdjRe4zNXfbOIJnn6xveHbaV2kSw5aJ5dLXUxcRt1Gxfi7ZYpFLUWlkG2MGSFAiqAu7w=="
     },
     "node_modules/@mrmlnc/readdir-enhanced": {
       "version": "2.2.1",
@@ -3844,14 +3844,24 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001317",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001317.tgz",
-      "integrity": "sha512-xIZLh8gBm4dqNX0gkzrBeyI86J2eCjWzYAs40q88smG844YIrN4tVQl/RhquHvKEKImWWFIVh1Lxe5n1G/N+GQ==",
+      "version": "1.0.30001476",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001476.tgz",
+      "integrity": "sha512-JmpktFppVSvyUN4gsLS0bShY2L9ZUslHLE72vgemBkS43JD2fOvKTKs+GtRwuxrtRGnwJFW0ye7kWRRlLJS9vQ==",
       "dev": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/browserslist"
-      }
+      "funding": [
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/browserslist"
+        },
+        {
+          "type": "tidelift",
+          "url": "https://tidelift.com/funding/github/npm/caniuse-lite"
+        },
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/ai"
+        }
+      ]
     },
     "node_modules/case-sensitive-paths-webpack-plugin": {
       "version": "2.4.0",
@@ -16432,9 +16442,9 @@
       }
     },
     "@mdi/font": {
-      "version": "5.9.55",
-      "resolved": "https://registry.npmjs.org/@mdi/font/-/font-5.9.55.tgz",
-      "integrity": "sha512-jswRF6q3eq8NWpWiqct6q+6Fg/I7nUhrxYJfiEM8JJpap0wVJLQdbKtyS65GdlK7S7Ytnx3TTi/bmw+tBhkGmg=="
+      "version": "7.2.96",
+      "resolved": "https://registry.npmjs.org/@mdi/font/-/font-7.2.96.tgz",
+      "integrity": "sha512-e//lmkmpFUMZKhmCY9zdjRe4zNXfbOIJnn6xveHbaV2kSw5aJ5dLXUxcRt1Gxfi7ZYpFLUWlkG2MGSFAiqAu7w=="
     },
     "@mrmlnc/readdir-enhanced": {
       "version": "2.2.1",
@@ -18175,9 +18185,9 @@
       }
     },
     "caniuse-lite": {
-      "version": "1.0.30001317",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001317.tgz",
-      "integrity": "sha512-xIZLh8gBm4dqNX0gkzrBeyI86J2eCjWzYAs40q88smG844YIrN4tVQl/RhquHvKEKImWWFIVh1Lxe5n1G/N+GQ==",
+      "version": "1.0.30001476",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001476.tgz",
+      "integrity": "sha512-JmpktFppVSvyUN4gsLS0bShY2L9ZUslHLE72vgemBkS43JD2fOvKTKs+GtRwuxrtRGnwJFW0ye7kWRRlLJS9vQ==",
       "dev": true
     },
     "case-sensitive-paths-webpack-plugin": {

lib/www/client/source/package.json

@@ -3,11 +3,11 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "serve": "vue-cli-service serve",
+    "serve": "vue-cli-service serve --host=0.0.0.0",
     "build": "vue-cli-service build"
   },
   "dependencies": {
-    "@mdi/font": "^5.6.55",
+    "@mdi/font": "^7.2.96",
     "core-js": "^3.6.5",
     "d3": "^7.0.1",
     "jwt-decode": "^3.0.0",

lib/www/client/source/src/components/event-edit.vue

@@ -44,7 +44,7 @@
               <template v-slot:activator="{ on, attrs }">
                 <v-text-field
                   v-model="tsDate"
-                  :disabled="!!(sequence || point || entrySequence || entryPoint)"
+                  :disabled="!!(entrySequence || entryPoint)"
                   label="Date"
                   suffix="UTC"
                   prepend-icon="mdi-calendar"
@@ -64,7 +64,7 @@
           <v-col>
             <v-text-field
               v-model="tsTime"
-              :disabled="!!(sequence || point || entrySequence || entryPoint)"
+              :disabled="!!(entrySequence || entryPoint)"
               label="Time"
               suffix="UTC"
               prepend-icon="mdi-clock-outline"
@@ -256,6 +256,15 @@
         >
           Cancel
         </v-btn>
+        <v-btn v-if="!id && (entrySequence || entryPoint)"
+          color="info"
+          text
+          title="Enter an event by time"
+          @click="timed"
+        >
+        <v-icon left small>mdi-clock-outline</v-icon>
+        Timed
+        </v-btn>
         <v-spacer></v-spacer>
         <v-btn
           :disabled="!canSave"
@@ -632,6 +641,14 @@ export default {
       }
     },
 
+    timed () {
+      const tstamp = (new Date()).toISOString();
+      this.entrySequence = null;
+      this.entryPoint = null;
+      this.tsDate = tstamp.substr(0, 10);
+      this.tsTime = tstamp.substr(11, 8);
+    },
+
     close () {
       this.entryLabels = this.selectedLabels.map(this.labelToItem)
       this.$emit("input", false);

lib/www/client/source/src/components/line-status.vue

@@ -2,8 +2,8 @@
 <div class="line-status" v-if="sequences.length == 0">
   <slot name="empty"></slot>
 </div>
-<div class="line-status" v-else-if="sequenceHref">
-  <router-link v-for="sequence in sequences" :key="sequence.sequence"
+<div class="line-status" v-else-if="sequenceHref || plannedSequenceHref || pendingReshootHref">
+  <router-link v-for="sequence in sequences" :key="sequence.sequence" v-if="sequenceHref"
     class="sequence"
     :class="sequence.status"
     :style="style(sequence)"
@@ -11,15 +11,41 @@
     :to="sequenceHref(sequence)"
   >
   </router-link>
+  <router-link v-for="sequence in plannedSequences" :key="sequence.sequence" v-if="plannedSequenceHref"
+    class="sequence planned"
+    :style="style(sequence)"
+    :title="title(sequence, 'planned')"
+    :to="plannedSequenceHref(sequence)"
+  >
+  </router-link>
+  <router-link v-for="(line, key) in pendingReshoots" :key="key" v-if="pendingReshootHref"
+    class="sequence reshoot"
+    :style="style(line)"
+    :title="title(line, 'reshoot')"
+    :to="pendingReshootHref(line)"
+  >
+  </router-link>
 </div>
 <div class="line-status" v-else>
-  <div v-for="sequence in sequences"
+  <div v-for="sequence in sequences" :key="sequence.sequence"
     class="sequence"
     :class="sequence.status"
     :style="style(sequence)"
     :title="title(sequence)"
   >
   </div>
+  <div v-for="sequence in plannedSequences" :key="sequence.sequence"
+    class="sequence planned"
+    :style="style(sequence)"
+    :title="title(sequence, 'planned')"
+  >
+  </div>
+  <div v-for="(line, key) in pendingReshoots" :key="key"
+    class="sequence reshoot"
+    :style="style(line)"
+    :title="title(line, 'reshoot')"
+  >
+  </div>
 </div>
 </template>
 
@@ -48,6 +74,8 @@
       background-color blue
     &.planned
       background-color magenta
+    &.reshoot
+      background repeating-linear-gradient(-45deg, rgba(255,0,255,0.302), brown 5px, rgba(247, 247, 247, 0.1) 5px, rgba(242, 241, 241, 0.08) 10px), repeating-linear-gradient(45deg, rgba(255,0,255,0.302), brown 5px, rgba(247, 247, 247, 0.1) 5px, rgba(242, 241, 241, 0.08) 10px)
 </style>
 
 <script>
@@ -58,7 +86,11 @@ export default {
   props: {
     preplot: Object,
     sequences: Array,
-    "sequence-href": Function
+    "sequence-href": Function,
+    "planned-sequences": Array,
+    "planned-sequence-href": Function,
+    "pending-reshoots": Array,
+    "pending-reshoot-href": Function
   },
 
   methods: {
@@ -68,13 +100,13 @@ export default {
         ? s.fsp_final
         : s.status == "ntbp"
           ? (s.fsp_final || s.fsp)
-          : s.fsp; /* status == "raw" */
+          : s.fsp; /* status == "raw" or planned sequence or pending reshoot */
 
       const lsp = s.status == "final"
         ? s.lsp_final
         : s.status == "ntbp"
           ? (s.lsp_final || s.lsp)
-          : s.lsp; /* status == "raw" */
+          : s.lsp; /* status == "raw" or planned sequence or pending reshoot */
 
       const pp0 = Math.min(this.preplot.fsp, this.preplot.lsp);
       const pp1 = Math.max(this.preplot.fsp, this.preplot.lsp);
@@ -91,20 +123,24 @@ export default {
       return values;
     },
 
-    title (s) {
-      const status = s.status == "final"
-        ? "Final"
-        : s.status == "raw"
-          ? "Acquired"
-          : s.status == "ntbp"
-            ? "NTBP"
-            : s.status == "planned"
-              ? "Planned"
-              : s.status;
+    title (s, type) {
+      if (s.status || type == "planned") {
+        const status = s.status == "final"
+          ? "Final"
+          : s.status == "raw"
+            ? "Acquired"
+            : s.status == "ntbp"
+              ? "NTBP"
+              : type == "planned"
+                ? "Planned"
+                : s.status;
 
-      const remarks = "\n"+[s.remarks, s.remarks_final].join("\n").trim()
+        const remarks = "\n"+[s.remarks, s.remarks_final].join("\n").trim()
 
-      return `Sequence ${s.sequence} – ${status} (${s.fsp_final || s.fsp}−${s.lsp_final || s.lsp})${remarks}`;
+        return `Sequence ${s.sequence} – ${status} (${s.fsp_final || s.fsp}−${s.lsp_final || s.lsp})${remarks}`;
+      } else if (type == "reshoot") {
+        return `Pending reshoot (${s.fsp}‒${s.lsp})${s.remarks? "\n"+s.remarks : ""}`;
+      }
     }
   }
 

lib/www/client/source/src/store/index.js

@@ -5,6 +5,11 @@ import api from './modules/api'
 import user from './modules/user'
 import snack from './modules/snack'
 import project from './modules/project'
+import event from './modules/event'
+import label from './modules/label'
+import sequence from './modules/sequence'
+import plan from './modules/plan'
+import line from './modules/line'
 import notify from './modules/notify'
 
 Vue.use(Vuex)
@@ -15,6 +20,11 @@ export default new Vuex.Store({
     user,
     snack,
     project,
+    event,
+    label,
+    sequence,
+    plan,
+    line,
     notify
   }
 })

lib/www/client/source/src/store/modules/api/actions.js

@@ -16,7 +16,7 @@ async function api ({state, commit, dispatch}, [resource, init = {}, cb]) {
     const url = /^https?:\/\//i.test(resource) ? resource : (state.apiUrl + resource);
     const res = await fetch(url, init);
     if (typeof cb === 'function') {
-      cb(null, res);
+      await cb(null, res);
     }
     if (res.ok) {
 
@@ -35,7 +35,14 @@ async function api ({state, commit, dispatch}, [resource, init = {}, cb]) {
         throw err;
       }
     } else {
-      await dispatch('showSnack', [res.statusText, "warning"]);
+      let message = res.statusText;
+      if (res.headers.get("Content-Type").match(/^application\/json/i)) {
+        const body = await res.json();
+        if (body.message) {
+          message = body.message;
+        }
+      }
+      await dispatch('showSnack', [message, "warning"]);
     }
   } catch (err) {
     if (err && err.name == "AbortError") return;

lib/www/client/source/src/store/modules/event/actions.js

@@ -0,0 +1,129 @@
+
+/** Fetch events from server
+ */
+async function refreshEvents ({commit, dispatch, state, rootState}, [modifiedAfter] = []) {
+
+  if (!modifiedAfter) {
+    modifiedAfter = state.timestamp;
+  }
+
+  if (state.loading) {
+    commit('abortEventsLoading');
+  }
+
+  commit('setEventsLoading');
+  const pid = rootState.project.projectId;
+  const url = modifiedAfter
+    ? `/project/${pid}/event/changes/${(new Date(modifiedAfter)).toISOString()}?unique=t`
+    : `/project/${pid}/event`;
+  const init = {
+    signal: state.loading.signal
+  };
+  const res = await dispatch('api', [url, init]);
+
+  if (res) {
+    if (modifiedAfter) {
+      commit('setModifiedEvents', res);
+    } else {
+      commit('setEvents', res);
+    }
+    commit('setEventsTimestamp');
+  }
+  commit('clearEventsLoading');
+
+}
+
+/** Return a subset of events from state.events
+ */
+async function getEvents ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text, label}]) {
+  let filteredEvents = [...state.events];
+
+  if (sortBy) {
+
+    sortBy.forEach( (key, idx) => {
+      filteredEvents.sort( (el0, el1) => {
+        const a = el0?.[key];
+        const b = el1?.[key];
+        if (a < b) {
+          return -1;
+        } else if (a > b) {
+          return 1;
+        } else if (a == b) {
+          return 0;
+        } else if (a && !b) {
+          return 1;
+        } else if (!a && b) {
+          return -1;
+        } else {
+          return 0;
+        }
+      });
+      if (sortDesc && sortDesc[idx] === true) {
+        filteredEvents.reverse();
+      }
+    });
+
+  }
+
+  if (sequence) {
+    filteredEvents = filteredEvents.filter( event => event.sequence == sequence );
+  }
+
+  if (date0 && date1) {
+    filteredEvents = filteredEvents.filter( event =>
+      event.tstamp.substr(0, 10) >= date0 && event.tstamp.substr(0, 10) <= date1
+    );
+  } else if (date0) {
+    filteredEvents = filteredEvents.filter( event => event.tstamp.substr(0, 10) == date0 );
+  }
+
+  if (text) {
+    const tstampFilter = (value, search, item) => {
+      return textFilter(value, search, item);
+    };
+
+    const numberFilter = (value, search, item) => {
+      return value == search;
+    };
+
+    const textFilter = (value, search, item) => {
+      return String(value).toLowerCase().includes(search.toLowerCase());
+    };
+
+    const searchFunctions = {
+      tstamp: tstampFilter,
+      sequence: numberFilter,
+      point: numberFilter,
+      remarks: textFilter,
+      labels: (value, search, item) => value.some(label => textFilter(label, search, item))
+    };
+
+    filteredEvents = filteredEvents.filter ( event => {
+      for (let key in searchFunctions) {
+        const fn = searchFunctions[key];
+        if (fn(event[key], text, event)) {
+          return true;
+        }
+      }
+      return false;
+    });
+  }
+
+  if (label) {
+    filteredEvents = filteredEvents.filter( event => event.labels?.includes(label) );
+  }
+
+  const count = filteredEvents.length;
+
+  if (itemsPerPage && itemsPerPage > 0) {
+    const offset = (page > 0)
+      ? (page-1) * itemsPerPage
+      : 0;
+
+    filteredEvents = filteredEvents.slice(offset, offset+itemsPerPage);
+  }
+
+  return {events: filteredEvents, count};
+}
+
+export default { refreshEvents, getEvents };

lib/www/client/source/src/store/modules/event/getters.js

@@ -0,0 +1,14 @@
+
+function events (state) {
+  return state.events;
+}
+
+function eventCount (state) {
+  return state.events?.length ?? 0;
+}
+
+function eventsLoading (state) {
+  return !!state.loading;
+}
+
+export default { events, eventCount, eventsLoading };

lib/www/client/source/src/store/modules/event/index.js

@@ -0,0 +1,6 @@
+import state from './state'
+import getters from './getters'
+import actions from './actions'
+import mutations from './mutations'
+
+export default { state, getters, actions, mutations };

lib/www/client/source/src/store/modules/event/mutations.js

@@ -0,0 +1,73 @@
+
+function setEvents (state, events) {
+  // We don't need or want the events array to be reactive, since
+  // it can be tens of thousands of items long.
+  state.events = Object.freeze(events);
+}
+
+/** Selectively replace / insert / delete events
+ * from state.events.
+ *
+ * modifiedEvents is the result of
+ * /api/project/:project/event/changes?unique=t
+ */
+function setModifiedEvents (state, modifiedEvents) {
+  const events = [...state.events];
+  for (let evt of modifiedEvents) {
+    const idx = events.findIndex(i => i.id == evt.id);
+    if (idx != -1) {
+      if (evt.is_deleted) {
+        events.splice(idx, 1);
+      } else {
+        delete evt.is_deleted;
+        events.splice(idx, 1, evt);
+      }
+    } else {
+      if (!evt.is_deleted) {
+        delete evt.is_deleted;
+        events.unshift(evt);
+      }
+    }
+  }
+  setEvents(state, events);
+}
+
+function setEventsLoading (state, abortController = new AbortController()) {
+  state.loading = abortController;
+}
+
+function clearEventsLoading (state) {
+  state.loading = null;
+}
+
+function setEventsTimestamp (state, timestamp = new Date()) {
+  if (timestamp === true) {
+    const tstamp = state.events
+      .map( event => event.modified_on )
+      .reduce( (acc, cur) => acc > cur ? acc : cur );
+    state.timestamp = tstamp ? new Date(tstamp) : new Date();
+  } else {
+    state.timestamp = timestamp;
+  }
+}
+
+function setEventsETag (state, etag) {
+  state.etag = etag;
+}
+
+function abortEventsLoading (state) {
+  if (state.loading) {
+    state.loading.abort();
+  }
+  state.loading = null;
+}
+
+export default {
+  setEvents,
+  setModifiedEvents,
+  setEventsLoading,
+  clearEventsLoading,
+  abortEventsLoading,
+  setEventsTimestamp,
+  setEventsETag
+};

lib/www/client/source/src/store/modules/event/state.js

@@ -0,0 +1,8 @@
+const state = () => ({
+  events: Object.freeze([]),
+  loading: null,
+  timestamp: null,
+  etag: null,
+});
+
+export default state;

lib/www/client/source/src/store/modules/label/actions.js

@@ -0,0 +1,106 @@
+
+/** Fetch labels from server
+ */
+async function refreshLabels ({commit, dispatch, state, rootState}) {
+
+  if (state.loading) {
+    commit('abortLabelsLoading');
+  }
+
+  commit('setLabelsLoading');
+  const pid = rootState.project.projectId;
+  const url = `/project/${pid}/label`;
+  const init = {
+    signal: state.loading.signal
+  };
+  const res = await dispatch('api', [url, init]);
+
+  if (res) {
+    commit('setLabels', res);
+    commit('setLabelsTimestamp');
+  }
+  commit('clearLabelsLoading');
+}
+
+/** Return a subset of labels from state.labels.
+ *
+ * Note that, unlike other actions in the get* family,
+ * the return value is not isomorphic to the state.
+ *
+ * While state.labels is an object, getLabels() returns
+ * an array with each item have the shape:
+ *
+ * { label: "labelName", view: {…}, model: {…} }
+ *
+ * This is intended to be useful, for instance, for a table
+ * of labels.
+ */
+async function getLabels ({commit, dispatch, state}, [projectId, {sortBy, sortDesc, itemsPerPage, page, text, label}]) {
+
+  let filteredLabels = Object.entries(state.labels).map(i => {
+    return {
+      label: i[0],
+      ...i[1]
+    }
+  });
+
+  if (sortBy) {
+
+    sortBy.forEach( (key, idx) => {
+      filteredLabels.sort( (el0, el1) => {
+        const a = key == "label" ? el0[0] : el0[1].view[key];
+        const b = key == "label" ? el1[0] : el1[1].view[key];
+        if (a < b) {
+          return -1;
+        } else if (a > b) {
+          return 1;
+        } else if (a == b) {
+          return 0;
+        } else if (a && !b) {
+          return 1;
+        } else if (!a && b) {
+          return -1;
+        } else {
+          return 0;
+        }
+      });
+      if (sortDesc && sortDesc[idx] === true) {
+        filteredLabels.reverse();
+      }
+    });
+
+  }
+
+  if (label) {
+    filteredLabels = filteredLabels.filter( label => label.label == label );
+  }
+
+  if (text) {
+    const textFilter = (value, search, item) => {
+      return String(value).toLowerCase().includes(search.toLowerCase());
+    };
+
+    const searchFunctions = {
+      label: numberFilter,
+      description: textFilter,
+    };
+
+    filteredLabels = filteredLabels.filter ( item => {
+      return textFilter(item.label, text, item) ?? textFilter(item.view.description, text, item);
+    });
+  }
+
+  const count = filteredLabels.length;
+
+  if (itemsPerPage && itemsPerPage > 0) {
+    const offset = (page > 0)
+      ? (page-1) * itemsPerPage
+      : 0;
+
+    filteredLabels = filteredLabels.slice(offset, offset+itemsPerPage);
+  }
+
+  return {labels: filteredLabels, count};
+}
+
+export default { refreshLabels, getLabels };

lib/www/client/source/src/store/modules/label/getters.js

@@ -0,0 +1,22 @@
+
+function labels (state) {
+  return state.labels;
+}
+
+/** Return labels that can be added by users.
+ *
+ * As opposed to system labels.
+ */
+function userLabels (state) {
+  return Object.fromEntries(Object.entries(state.labels).filter(i => i[1].model.user));
+}
+
+function labelCount (state) {
+  return state.labels?.length ?? 0;
+}
+
+function labelsLoading (state) {
+  return !!state.loading;
+}
+
+export default { labels, userLabels, labelCount, labelsLoading };

lib/www/client/source/src/store/modules/label/index.js

@@ -0,0 +1,6 @@
+import state from './state'
+import getters from './getters'
+import actions from './actions'
+import mutations from './mutations'
+
+export default { state, getters, actions, mutations };

lib/www/client/source/src/store/modules/label/mutations.js

@@ -0,0 +1,49 @@
+
+function setLabels (state, labels) {
+  // We don't need or want the events array to be reactive, since
+  // it can be tens of thousands of items long.
+  state.labels = Object.freeze(labels);
+}
+
+function setLabelsLoading (state, abortController = new AbortController()) {
+  state.loading = abortController;
+}
+
+// This assumes that we know any transactions have finished or we
+// don't care about aborting.
+function clearLabelsLoading (state) {
+  state.loading = null;
+}
+
+function setLabelsTimestamp (state, timestamp = new Date()) {
+  // NOTE: There is no `modified_on` property in the labels
+  // result or in the database schema, but we could add
+  // one.
+  if (timestamp === true) {
+    const tstamp = state.labels
+      .map( i => i.modified_on )
+      .reduce( (acc, cur) => acc > cur ? acc : cur );
+    state.timestamp = tstamp ? new Date(tstamp) : new Date();
+  } else {
+    state.timestamp = timestamp;
+  }
+}
+
+function setLabelsETag (state, etag) {
+  state.etag = etag;
+}
+
+function abortLabelsLoading (state) {
+  if (state.loading) {
+    state.loading.abort();
+  }
+  state.loading = null;
+}
+
+export default {
+  setLabels,
+  setLabelsLoading,
+  clearLabelsLoading,
+  setLabelsTimestamp,
+  setLabelsETag
+};

lib/www/client/source/src/store/modules/label/state.js

@@ -0,0 +1,8 @@
+const state = () => ({
+  labels: Object.freeze([]),
+  loading: null,
+  timestamp: null,
+  etag: null,
+});
+
+export default state;

lib/www/client/source/src/store/modules/line/actions.js

@@ -0,0 +1,117 @@
+
+/** Fetch lines from server
+ */
+async function refreshLines ({commit, dispatch, state, rootState}) {
+
+  if (state.loading) {
+    commit('abortLinesLoading');
+  }
+
+  commit('setLinesLoading');
+  const pid = rootState.project.projectId;
+  const url = `/project/${pid}/line`;
+  const init = {
+    signal: state.loading.signal
+  };
+  const res = await dispatch('api', [url, init]);
+
+  if (res) {
+    commit('setLines', res);
+    commit('setLinesTimestamp');
+  }
+  commit('clearLinesLoading');
+}
+
+/** Return a subset of lines from state.lines
+ */
+async function getLines ({commit, dispatch, state}, [projectId, {line, fsp, lsp, incr, sortBy, sortDesc, itemsPerPage, page, text}]) {
+  let filteredLines = [...state.lines];
+
+  if (sortBy) {
+
+    sortBy.forEach( (key, idx) => {
+      filteredLines.sort( (el0, el1) => {
+        const a = el0?.[key];
+        const b = el1?.[key];
+        if (a < b) {
+          return -1;
+        } else if (a > b) {
+          return 1;
+        } else if (a == b) {
+          return 0;
+        } else if (a && !b) {
+          return 1;
+        } else if (!a && b) {
+          return -1;
+        } else {
+          return 0;
+        }
+      });
+      if (sortDesc && sortDesc[idx] === true) {
+        filteredLines.reverse();
+      }
+    });
+
+  }
+
+  if (line) {
+    filteredLines = filteredLines.filter( line => line.line == line );
+  }
+
+  if (fsp) {
+    filteredLines = filteredLines.filter( line => line.fsp == fsp );
+  }
+
+  if (lsp) {
+    filteredLines = filteredLines.filter( line => line.lsp == lsp );
+  }
+
+  if (text) {
+    const numberFilter = (value, search, item) => {
+      return value == search;
+    };
+
+    const textFilter = (value, search, item) => {
+      return String(value).toLowerCase().includes(search.toLowerCase());
+    };
+
+    const incrFilter = (value, search, item) => {
+      const inc = /^(incr(ement)?|↑|\+)/i;
+      const dec = /^(decr(ement)?|↓|-)/i;
+      return (inc.test(search) && value) || (dec.test(search) && !value)
+    }
+
+    const searchFunctions = {
+      line: numberFilter,
+      fsp: numberFilter,
+      lsp: numberFilter,
+      remarks: textFilter,
+      incr: incrFilter,
+      ntba: (value, search, item) => text.toLowerCase() == "ntba" && value
+    };
+
+    filteredLines = filteredLines.filter ( line => {
+      for (let key in searchFunctions) {
+        const fn = searchFunctions[key];
+        if (fn(line[key], text, line)) {
+          return true;
+        }
+      }
+      return false;
+    });
+  }
+
+  const count = filteredLines.length;
+
+  if (itemsPerPage && itemsPerPage > 0) {
+    const offset = (page > 0)
+      ? (page-1) * itemsPerPage
+      : 0;
+
+    filteredLines = filteredLines.slice(offset, offset+itemsPerPage);
+  }
+
+  return {lines: filteredLines, count};
+}
+
+export default { refreshLines, getLines };

lib/www/client/source/src/store/modules/line/getters.js

@@ -0,0 +1,14 @@
+
+function lines (state) {
+  return state.lines;
+}
+
+function lineCount (state) {
+  return state.lines?.length ?? 0;
+}
+
+function linesLoading (state) {
+  return !!state.loading;
+}
+
+export default { lines, lineCount, linesLoading };

lib/www/client/source/src/store/modules/line/index.js

@@ -0,0 +1,6 @@
+import state from './state'
+import getters from './getters'
+import actions from './actions'
+import mutations from './mutations'
+
+export default { state, getters, actions, mutations };

lib/www/client/source/src/store/modules/line/mutations.js

@@ -0,0 +1,49 @@
+
+function setLines (state, lines) {
+  // We don't need or want the events array to be reactive, since
+  // it can be tens of thousands of items long.
+  state.lines = Object.freeze(lines);
+}
+
+function setLinesLoading (state, abortController = new AbortController()) {
+  state.loading = abortController;
+}
+
+// This assumes that we know any transactions have finished or we
+// don't care about aborting.
+function clearLinesLoading (state) {
+  state.loading = null;
+}
+
+function setLinesTimestamp (state, timestamp = new Date()) {
+  // NOTE: There is no `modified_on` property in the lines
+  // result or in the database schema, but we could perhaps add
+  // one.
+  if (timestamp === true) {
+    const tstamp = state.lines
+      .map( event => event.modified_on )
+      .reduce( (acc, cur) => acc > cur ? acc : cur );
+    state.timestamp = tstamp ? new Date(tstamp) : new Date();
+  } else {
+    state.timestamp = timestamp;
+  }
+}
+
+function setLinesETag (state, etag) {
+  state.etag = etag;
+}
+
+function abortLinesLoading (state) {
+  if (state.loading) {
+    state.loading.abort();
+  }
+  state.loading = null;
+}
+
+export default {
+  setLines,
+  setLinesLoading,
+  clearLinesLoading,
+  setLinesTimestamp,
+  setLinesETag
+};

lib/www/client/source/src/store/modules/line/state.js

@@ -0,0 +1,8 @@
+const state = () => ({
+  lines: Object.freeze([]),
+  loading: null,
+  timestamp: null,
+  etag: null,
+});
+
+export default state;

lib/www/client/source/src/store/modules/plan/actions.js

@@ -0,0 +1,114 @@
+
+/** Fetch sequences from server
+ */
+async function refreshPlan ({commit, dispatch, state, rootState}) {
+
+  if (state.loading) {
+    commit('abortPlanLoading');
+  }
+
+  commit('setPlanLoading');
+  const pid = rootState.project.projectId;
+  const url = `/project/${pid}/plan`;
+  const init = {
+    signal: state.loading.signal
+  };
+  const res = await dispatch('api', [url, init]);
+
+  if (res) {
+    commit('setPlan', res);
+    commit('setPlanTimestamp');
+  }
+  commit('clearPlanLoading');
+}
+
+/** Return a subset of sequences from state.sequences
+ */
+async function getPlannedSequences ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text}]) {
+  let filteredPlannedSequences = [...state.sequences];
+
+  if (sortBy) {
+
+    sortBy.forEach( (key, idx) => {
+      filteredPlannedSequences.sort( (el0, el1) => {
+        const a = el0?.[key];
+        const b = el1?.[key];
+        if (a < b) {
+          return -1;
+        } else if (a > b) {
+          return 1;
+        } else if (a == b) {
+          return 0;
+        } else if (a && !b) {
+          return 1;
+        } else if (!a && b) {
+          return -1;
+        } else {
+          return 0;
+        }
+      });
+      if (sortDesc && sortDesc[idx] === true) {
+        filteredPlannedSequences.reverse();
+      }
+    });
+
+  }
+
+  if (sequence) {
+    filteredPlannedSequences = filteredPlannedSequences.filter( sequence => sequence.sequence == sequence );
+  }
+
+  if (date0 && date1) {
+    filteredPlannedSequences = filteredPlannedSequences.filter( sequence =>
+      sequence.ts0.substr(0, 10) >= date0 && sequence.ts1.substr(0, 10) <= date1
+    );
+  } else if (date0) {
+    filteredPlannedSequences = filteredPlannedSequences.filter( sequence => sequence.ts0.substr(0, 10) == date0 || sequence.ts1.substr(0, 10) );
+  }
+
+  if (text) {
+    const tstampFilter = (value, search, item) => {
+      return textFilter(value.toISOString(), search, item);
+    };
+
+    const numberFilter = (value, search, item) => {
+      return value == search;
+    };
+
+    const textFilter = (value, search, item) => {
+      return String(value).toLowerCase().includes(search.toLowerCase());
+    };
+
+    const searchFunctions = {
+      sequence: numberFilter,
+      line: numberFilter,
+      remarks: textFilter,
+      ts0: tstampFilter,
+      ts1: tstampFilter
+    };
+
+    filteredPlannedSequences = filteredPlannedSequences.filter ( sequence => {
+      for (let key in searchFunctions) {
+        const fn = searchFunctions[key];
+        if (fn(sequence[key], text, sequence)) {
+          return true;
+        }
+      }
+      return false;
+    });
+  }
+
+  const count = filteredPlannedSequences.length;
+
+  if (itemsPerPage && itemsPerPage > 0) {
+    const offset = (page > 0)
+      ? (page-1) * itemsPerPage
+      : 0;
+
+    filteredPlannedSequences = filteredPlannedSequences.slice(offset, offset+itemsPerPage);
+  }
+
+  return {sequences: filteredPlannedSequences, count};
+}
+
+export default { refreshPlan, getPlannedSequences };

lib/www/client/source/src/store/modules/plan/getters.js

@@ -0,0 +1,18 @@
+
+function planRemarks (state) {
+  return state.remarks;
+}
+
+function plannedSequences (state) {
+  return state.sequences;
+}
+
+function plannedSequenceCount (state) {
+  return state.sequences?.length ?? 0;
+}
+
+function plannedSequencesLoading (state) {
+  return !!state.loading;
+}
+
+export default { planRemarks, plannedSequences, plannedSequenceCount, plannedSequencesLoading };

lib/www/client/source/src/store/modules/plan/index.js

@@ -0,0 +1,6 @@
+import state from './state'
+import getters from './getters'
+import actions from './actions'
+import mutations from './mutations'
+
+export default { state, getters, actions, mutations };

lib/www/client/source/src/store/modules/plan/mutations.js

@@ -0,0 +1,59 @@
+
+
+function transform (item) {
+  item.ts0 = new Date(item.ts0);
+  item.ts1 = new Date(item.ts1);
+  return item;
+}
+
+// ATTENTION: This relies on the new planner endpoint
+// as per issue #281.
+
+function setPlan (state, plan) {
+  // We don't need or want the planned sequences array to be reactive
+  state.sequences = Object.freeze(plan.sequences.map(transform));
+  state.remarks = plan.remarks;
+}
+
+function setPlanLoading (state, abortController = new AbortController()) {
+  state.loading = abortController;
+}
+
+// This assumes that we know any transactions have finished or we
+// don't care about aborting.
+function clearPlanLoading (state) {
+  state.loading = null;
+}
+
+function setPlanTimestamp (state, timestamp = new Date()) {
+  // NOTE: There is no `modified_on` property in the plan
+  // result or in the database schema, but we should probably add
+  // one.
+  if (timestamp === true) {
+    const tstamp = state.plan
+      .map( item => item.modified_on )
+      .reduce( (acc, cur) => acc > cur ? acc : cur );
+    state.timestamp = tstamp ? new Date(tstamp) : new Date();
+  } else {
+    state.timestamp = timestamp;
+  }
+}
+
+function setPlanETag (state, etag) {
+  state.etag = etag;
+}
+
+function abortPlanLoading (state) {
+  if (state.loading) {
+    state.loading.abort();
+  }
+  state.loading = null;
+}
+
+export default {
+  setPlan,
+  setPlanLoading,
+  clearPlanLoading,
+  setPlanTimestamp,
+  setPlanETag
+};

lib/www/client/source/src/store/modules/plan/state.js

@@ -0,0 +1,9 @@
+const state = () => ({
+  sequences: Object.freeze([]),
+  remarks: null,
+  loading: null,
+  timestamp: null,
+  etag: null,
+});
+
+export default state;

lib/www/client/source/src/store/modules/project/actions.js

@@ -1,13 +1,19 @@
 
 async function getProject ({commit, dispatch}, projectId) {
-  const res = await dispatch('api', [`/project/${String(projectId).toLowerCase()}`]);
+  const res = await dispatch('api', [`/project/${String(projectId).toLowerCase()}/configuration`]);
   if (res) {
     commit('setProjectName', res.name);
-    commit('setProjectId', res.pid);
+    commit('setProjectId', res.id?.toLowerCase());
     commit('setProjectSchema', res.schema);
+    commit('setProjectConfiguration', res);
     const recentProjects = JSON.parse(localStorage.getItem("recentProjects") || "[]")
     recentProjects.unshift(res);
     localStorage.setItem("recentProjects", JSON.stringify(recentProjects.slice(0, 3)));
+  } else {
+    commit('setProjectName', null);
+    commit('setProjectId', null);
+    commit('setProjectSchema', null);
+    commit('setProjectConfiguration', {});
   }
 }
 

lib/www/client/source/src/store/modules/project/getters.js

@@ -0,0 +1,18 @@
+
+function projectId (state) {
+  return state.projectId;
+}
+
+function projectName (state) {
+  return state.projectName;
+}
+
+function projectSchema (state) {
+  return state.projectSchema;
+}
+
+function projectConfiguration (state) {
+  return state.projectConfiguration;
+}
+
+export default { projectId, projectName, projectSchema, projectConfiguration };

lib/www/client/source/src/store/modules/project/mutations.js

@@ -11,4 +11,8 @@ function setProjectSchema (state, schema) {
   state.projectSchema = schema;
 }
 
-export default { setProjectId, setProjectName, setProjectSchema };
+function setProjectConfiguration (state, configuration) {
+  state.projectConfiguration = Object.freeze(configuration);
+}
+
+export default { setProjectId, setProjectName, setProjectSchema, setProjectConfiguration };

lib/www/client/source/src/store/modules/project/state.js

@@ -1,7 +1,8 @@
 const state = () => ({
   projectId: null,
   projectName: null,
-  projectSchema: null
+  projectSchema: null,
+  projectConfiguration: {}
 });
 
 export default state;

lib/www/client/source/src/store/modules/sequence/actions.js

@@ -0,0 +1,122 @@
+
+/** Fetch sequences from server
+ */
+async function refreshSequences ({commit, dispatch, state, rootState}) {
+
+  if (state.loading) {
+    commit('abortSequencesLoading');
+  }
+
+  commit('setSequencesLoading');
+  const pid = rootState.project.projectId;
+  const url = `/project/${pid}/sequence?files=true`;
+  const init = {
+    signal: state.loading.signal
+  };
+  const res = await dispatch('api', [url, init]);
+
+  if (res) {
+    commit('setSequences', res);
+    commit('setSequencesTimestamp');
+  }
+  commit('clearSequencesLoading');
+}
+
+/** Return a subset of sequences from state.sequences
+ */
+async function getSequences ({commit, dispatch, state}, [projectId, {sequence, date0, date1, sortBy, sortDesc, itemsPerPage, page, text}]) {
+  let filteredSequences = [...state.sequences];
+
+  if (sortBy) {
+
+    sortBy.forEach( (key, idx) => {
+      filteredSequences.sort( (el0, el1) => {
+        const a = el0?.[key];
+        const b = el1?.[key];
+        if (a < b) {
+          return -1;
+        } else if (a > b) {
+          return 1;
+        } else if (a == b) {
+          return 0;
+        } else if (a && !b) {
+          return 1;
+        } else if (!a && b) {
+          return -1;
+        } else {
+          return 0;
+        }
+      });
+      if (sortDesc && sortDesc[idx] === true) {
+        filteredSequences.reverse();
+      }
+    });
+
+  }
+
+  if (sequence) {
+    filteredSequences = filteredSequences.filter( sequence => sequence.sequence == sequence );
+  }
+
+  if (date0 && date1) {
+    filteredSequences = filteredSequences.filter( sequence =>
+      (sequence.ts0_final ?? sequence.ts0)?.substr(0, 10) >= date0 &&
+      (sequence.ts1_final ?? sequence.ts1)?.substr(0, 10) <= date1
+    );
+  } else if (date0) {
+    filteredSequences = filteredSequences.filter( sequence => (sequence.ts0_final ?? sequence.ts0)?.substr(0, 10) == date0 || (sequence.ts1_final ?? sequence.ts1)?.substr(0, 10) );
+  }
+
+  if (text) {
+    const tstampFilter = (value, search, item) => {
+      return search?.length >= 5 && textFilter(value, search, item);
+    };
+
+    const numberFilter = (value, search, item) => {
+      return value == search;
+    };
+
+    const textFilter = (value, search, item) => {
+      return String(value).toLowerCase().includes(search.toLowerCase());
+    };
+
+    const searchFunctions = {
+      ts0: tstampFilter,
+      ts1: tstampFilter,
+      ts0_final: tstampFilter,
+      ts1_final: tstampFilter,
+      sequence: numberFilter,
+      line: numberFilter,
+      fsp: numberFilter,
+      lsp: numberFilter,
+      fsp_final: numberFilter,
+      fsp_final: numberFilter,
+      remarks: textFilter,
+      remarks_final: textFilter
+    };
+
+    filteredSequences = filteredSequences.filter ( sequence => {
+      for (let key in searchFunctions) {
+        const fn = searchFunctions[key];
+        if (fn(sequence[key], text, sequence)) {
+          return true;
+        }
+      }
+      return false;
+    });
+  }
+
+  const count = filteredSequences.length;
+
+  if (itemsPerPage && itemsPerPage > 0) {
+    const offset = (page > 0)
+      ? (page-1) * itemsPerPage
+      : 0;
+
+    filteredSequences = filteredSequences.slice(offset, offset+itemsPerPage);
+  }
+
+  return {sequences: filteredSequences, count};
+}
+
+export default { refreshSequences, getSequences };

lib/www/client/source/src/store/modules/sequence/getters.js

@@ -0,0 +1,14 @@
+
+function sequences (state) {
+  return state.sequences;
+}
+
+function sequenceCount (state) {
+  return state.sequences?.length ?? 0;
+}
+
+function sequencesLoading (state) {
+  return !!state.loading;
+}
+
+export default { sequences, sequenceCount, sequencesLoading };

lib/www/client/source/src/store/modules/sequence/index.js

@@ -0,0 +1,6 @@
+import state from './state'
+import getters from './getters'
+import actions from './actions'
+import mutations from './mutations'
+
+export default { state, getters, actions, mutations };

lib/www/client/source/src/store/modules/sequence/mutations.js

@@ -0,0 +1,49 @@
+
+function setSequences (state, sequences) {
+  // We don't need or want the events array to be reactive, since
+  // it can be tens of thousands of items long.
+  state.sequences = Object.freeze(sequences);
+}
+
+function setSequencesLoading (state, abortController = new AbortController()) {
+  state.loading = abortController;
+}
+
+// This assumes that we know any transactions have finished or we
+// don't care about aborting.
+function clearSequencesLoading (state) {
+  state.loading = null;
+}
+
+function setSequencesTimestamp (state, timestamp = new Date()) {
+  // NOTE: There is no `modified_on` property in the sequences
+  // result or in the database schema, but we should probably add
+  // one.
+  if (timestamp === true) {
+    const tstamp = state.sequences
+      .map( event => event.modified_on )
+      .reduce( (acc, cur) => acc > cur ? acc : cur );
+    state.timestamp = tstamp ? new Date(tstamp) : new Date();
+  } else {
+    state.timestamp = timestamp;
+  }
+}
+
+function setSequencesETag (state, etag) {
+  state.etag = etag;
+}
+
+function abortSequencesLoading (state) {
+  if (state.loading) {
+    state.loading.abort();
+  }
+  state.loading = null;
+}
+
+export default {
+  setSequences,
+  setSequencesLoading,
+  clearSequencesLoading,
+  setSequencesTimestamp,
+  setSequencesETag
+};

lib/www/client/source/src/store/modules/sequence/state.js

@@ -0,0 +1,8 @@
+const state = () => ({
+  sequences: Object.freeze([]),
+  loading: null,
+  timestamp: null,
+  etag: null,
+});
+
+export default state;

lib/www/client/source/src/views/Calendar.vue

@@ -39,6 +39,12 @@
           {{ $refs.calendar.title }}
         </v-toolbar-title>
         <v-spacer></v-spacer>
+        <v-btn v-if="categoriesAvailable"
+          small
+          class="mx-4"
+          v-model="useCategories"
+          @click="useCategories = !useCategories"
+        >Labels {{useCategories ? "On" : "Off"}}</v-btn>
         <v-menu bottom right>
           <template v-slot:activator="{ on, attrs }">
             <v-btn
@@ -72,16 +78,23 @@
       <v-calendar
         ref="calendar"
         v-model="focus"
-        :events="events"
+        :events="items"
         :event-color="getEventColour"
         color="primary"
-        :type="type"
+        :type="view"
         :locale-first-day-of-year="4"
         :weekdays="weekdays"
         :show-week="true"
+        :category-days="categoryDays"
+        :categories="categories"
         @click:date="showLogForDate"
         @click:event="showLogForEvent"
-      ></v-calendar>
+        @change="setSpan"
+      >
+        <template v-slot:event="{ event }">
+            <div style="height:100%;overflow:scroll;" v-html="event.name"></div>
+        </template>
+      </v-calendar>
     </v-sheet>
   </div>
 </template>
@@ -97,8 +110,9 @@ export default {
       weekdays: [1, 2, 3, 4, 5, 6, 0],
       type: "week",
       focus: "",
-      events: [
-      ],
+      items: [],
+      useCategories: false,
+      span: {},
       options: {
         sortBy: "sequence"
       }
@@ -117,28 +131,126 @@ export default {
       return labels[this.type];
     },
 
-    ...mapGetters(['loading'])
+    view () {
+      return this.useCategories ? "category" : this.type;
+    },
+
+    categoriesAvailable () {
+      return this.type == "day" || this.type == "4day";
+    },
+
+    categoryDays () {
+      if (this.useCategories) {
+        const days = {
+          month: 30,
+          week: 7,
+          "4day": 4,
+          day: 1
+        };
+
+        return days[this.type];
+      }
+    },
+
+    visibleItems () {
+      return this.items.filter(i => {
+        const end = i.end ?? i.start;
+        if (i.start > this.span.end) {
+          return false;
+        }
+        if (end < this.span.start) {
+          return false;
+        }
+        return true;
+      });
+    },
+
+    categories () {
+      return [...new Set(this.visibleItems.map(i => i.category ?? "General"))];
+    },
+
+    ...mapGetters(['sequencesLoading', 'sequences', 'events'])
+  },
+
+  watch: {
+
+    sequences () {
+      const isFirstLoad = !this.items.length;
+
+      this.getEvents();
+
+      if (isFirstLoad) {
+        this.setLast();
+      }
+
+    },
+
+    events () {
+      const isFirstLoad = !this.items.length;
+
+      this.getEvents();
+
+      if (isFirstLoad) {
+        this.setLast();
+      }
+
+    },
+
+    type () {
+      this.getEvents();
+    },
+
+    categoriesAvailable (value) {
+      if (!value) {
+        this.useCategories = false;
+      }
+    }
+
   },
 
   methods: {
 
     async getEvents () {
-      const query = new URLSearchParams(this.options);
-      const url = `/project/${this.$route.params.project}/sequence?${query.toString()}`;
-
-      const finalSequences = await this.api([url]) || [];
-      this.events = finalSequences.map(s => {
+      const sequences = this.sequences.map(s => {
         const e = {};
-        //e.start = s.ts0.substring(0,10)+" "+s.ts0.substring(11,19)
-        //e.end = s.ts1.substring(0,10)+" "+s.ts1.substring(11,19)
         e.routerLink = { name: "logBySequence", params: { sequence: s.sequence } };
         e.start = new Date(s.ts0);
         e.end = new Date(s.ts1);
         e.timed = true;
         e.colour = "orange";
-        e.name = `Sequence ${s.sequence}`;
+        e.name = `<b>Sequence ${s.sequence}</b><br/>Line ${s.line}<br/><abbr title="Shotpoints">SP</abbr> ${s.fgsp ?? s.fsp}‒${s.lgsp ?? s.lsp}`;
+        e.category = "Sequence"
         return e;
       });
+
+      const lineChanges = this.events.filter(i => i.meta?.["*ReportLineChangeTime*"]?.value && i.meta?.["*ReportLineChangeTime*"]?.type != "excess").map(i => {
+        const e = {};
+        const duration = i.meta?.["*ReportLineChangeTime*"]?.value;
+        e.end = new Date(i.tstamp);
+        e.start = new Date(e.end - duration);
+        e.timed = true;
+        e.colour = "pink";
+        e.name = "Line change";
+        e.category = "Production"
+        return e;
+      });
+
+      const excludedLabels = [ "FSP", "FGSP", "LSP", "LGSP", "QC" ];
+      const otherEvents = this.events.filter(i => !excludedLabels.some(l => i.labels.includes(l))).map(i => {
+        const e = {};
+        e.start = new Date(i.tstamp);
+        e.colour = "brown";
+        e.timed = true;
+        e.name = this.$options.filters.markdownInline(i.remarks);
+        e.category = i.labels[0];
+        return e;
+      });
+
+      this.items = [...sequences];
+
+      if (this.type == "day" || this.type == "4day") {
+        this.items.push(...lineChanges, ...otherEvents);
+      }
     },
 
     getEventColour (event) {
@@ -150,11 +262,15 @@ export default {
     },
 
     setFirst () {
-      this.focus = this.events[this.events.length-1].start;
+      if (this.items.length) {
+        this.focus = this.items[this.items.length-1].start;
+      }
     },
 
     setLast () {
-      this.focus = this.events[0].start;
+      if (this.items.length) {
+        this.focus = this.items[0].start;
+      }
     },
 
     prev () {
@@ -175,6 +291,13 @@ export default {
       }
     },
 
+    setSpan (span) {
+      this.span = {
+        start: new Date(span.start.date),
+        end:   new Date((new Date(span.end.date)).valueOf() + 86400000)
+      };
+    },
+
 
     ...mapActions(["api"])
 
@@ -182,9 +305,7 @@ export default {
 
   async mounted () {
     await this.getEvents();
-    if (this.events.length) {
-      this.setLast();
-    }
+    this.setLast();
   }
 }
 

lib/www/client/source/src/views/LineList.vue

@@ -11,6 +11,7 @@
             label="Filter"
             single-line
             clearable
+            hint="Filter by line number, first or last shotpoint or remarks. Use ‘incr’ or ‘+’ / ‘decr’ or ‘-’ to show only incrementing / decrementing lines"
             ></v-text-field>
         </v-toolbar>
       </v-card-title>
@@ -106,12 +107,14 @@
         <v-data-table
         :headers="headers"
         :items="items"
-        item-key="line"
         :items-per-page.sync="itemsPerPage"
+        :server-items-length="lineCount"
+        item-key="line"
         :search="filter"
-        :loading="loading"
-        :fixed-header="true"
-        :footer-props='{itemsPerPageOptions: [ 10, 25, 50, 100, 500, -1 ]}'
+        :loading="linesLoading"
+        :options.sync="options"
+        fixed-header
+        :footer-props='{itemsPerPageOptions: [ 10, 25, 50, 100, 500, -1 ], showFirstLastPage: true}'
         :item-class="itemClass"
         :show-select="selectOn"
         v-model="selectedRows"
@@ -124,6 +127,10 @@
               :preplot="item"
               :sequences="sequences.filter(s => s.line == item.line)"
               :sequence-href="(s) => `/projects/${$route.params.project}/log/sequence/${s.sequence}`"
+              :planned-sequences="plannedSequences.filter(s => s.line == item.line)"
+              :planned-sequence-href="() => `/projects/${$route.params.project}/plan`"
+              :pending-reshoots="null"
+              :pending-reshoot-href="null"
             >
               <template v-slot:empty>
                 <div v-if="!item.ntba" class="sequence" title="Virgin"></div>
@@ -161,7 +168,7 @@
                 icon
                 small
                 title="Edit"
-                :disabled="loading"
+                :disabled="linesLoading"
                 @click="editItem(item, 'remarks')"
               >
                 <v-icon small>mdi-square-edit-outline</v-icon>
@@ -251,9 +258,10 @@ export default {
       items: [],
       selectOn: false,
       selectedRows: [],
-      filter: null,
-      num_lines: null,
-      sequences: [],
+      filter: "",
+      options: {},
+      lineCount: null,
+      //sequences: [],
       activeItem: null,
       edit: null, // {line, key, value}
       queuedReload: false,
@@ -273,11 +281,22 @@ export default {
   },
 
   computed: {
-    ...mapGetters(['user', 'writeaccess', 'loading', 'serverEvent'])
+    ...mapGetters(['user', 'writeaccess', 'linesLoading', 'lines', 'sequences', 'plannedSequences'])
   },
 
   watch: {
 
+    options: {
+      handler () {
+        this.fetchLines();
+      },
+      deep: true
+    },
+
+    async lines () {
+      await this.fetchLines();
+    },
+
     async edit (newVal, oldVal) {
       if (newVal === null && oldVal !== null) {
         const item = this.items.find(i => i.line == oldVal.line);
@@ -296,39 +315,9 @@ export default {
       }
     },
 
-    async serverEvent (event) {
-      if (event.payload.pid == this.$route.params.project) {
-        if (event.channel == "preplot_lines" || event.channel == "preplot_points") {
-          if (!this.loading && !this.queuedReload) {
-            // Do not force a non-cached response if refreshing as a result
-            // of an event notification. We will assume that the server has
-            // already had time to update the cache by the time our request
-            // gets back to it.
-            this.getLines();
-          } else {
-            this.queuedReload = true;
-          }
-        } else if ([ "planned_lines", "raw_lines", "final_lines" ].includes(event.channel)) {
-          if (!this.loading && !this.queuedReload) {
-            this.getSequences();
-          } else {
-            this.queuedReload = true;
-          }
-        }
-      }
-    },
-
-    queuedReload (newVal, oldVal) {
-      if (newVal && !oldVal && !this.loading) {
-        this.getLines();
-        this.getSequences();
-      }
-    },
-
-    loading (newVal, oldVal) {
-      if (!newVal && oldVal && this.queuedReload) {
-        this.getLines();
-        this.getSequences();
+    filter (newVal, oldVal) {
+      if (newVal?.toLowerCase() != oldVal?.toLowerCase()) {
+        this.fetchLines();
       }
     },
 
@@ -468,43 +457,28 @@ export default {
       }
     },
 
-    async getNumLines () {
-      const projectInfo = await this.api([`/project/${this.$route.params.project}`]);
-      this.num_lines = projectInfo.lines;
-    },
-
-    async getLines () {
-
-      const url = `/project/${this.$route.params.project}/line`;
-
-      this.queuedReload = false;
-      this.items = await this.api([url]) || [];
-
-    },
-
-    async getSequences () {
-      const urlS = `/project/${this.$route.params.project}/sequence`;
-      this.sequences = await this.api([urlS]) || [];
-
-      const urlP = `/project/${this.$route.params.project}/plan`;
-      const planned = await this.api([urlP]) || [];
-      planned.forEach(i => i.status = "planned");
-      this.sequences.push(...planned);
-    },
-
     setActiveItem (item) {
       this.activeItem = this.activeItem == item
         ? null
         : item;
     },
 
-    ...mapActions(["api"])
+
+    async fetchLines (opts = {}) {
+      const options = {
+        text: this.filter,
+        ...this.options
+      };
+      const res = await this.getLines([this.$route.params.project, options]);
+      this.items = res.lines;
+      this.lineCount = res.count;
+    },
+
+    ...mapActions(["api", "getLines"])
   },
 
   mounted () {
-    this.getLines();
-    this.getNumLines();
-    this.getSequences();
+    this.fetchLines();
 
     // Initialise stylesheet
     const el = document.createElement("style");

lib/www/client/source/src/views/Log.vue

@@ -72,6 +72,10 @@
                 :href="`/api/project/${$route.params.project}/event/-/${$route.params.sequence}?mime=application%2Fjson`"
                 title="Download as a generic JSON file"
               >JSON</v-list-item>
+              <v-list-item
+                :href="`/api/project/${$route.params.project}/event/-/${$route.params.sequence}?mime=text%2Fcsv`"
+                title="Download as Comma Separated Values file"
+              >CSV</v-list-item>
               <v-list-item
                 :href="`/api/project/${$route.params.project}/event/-/${$route.params.sequence}?mime=text%2Fhtml`"
                 title="Download as an HTML formatted file"
@@ -89,7 +93,21 @@
             append-icon="mdi-magnify"
             label="Filter"
             single-line
-            hide-details></v-text-field>
+            clearable
+            hide-details>
+            <template v-slot:prepend-inner>
+              <v-chip v-if="labelSearch"
+                class="mr-1"
+                small
+                close
+                @click:close="labelSearch=null"
+                :color="labels[labelSearch] && labels[labelSearch].view.colour"
+                :title="labels[labelSearch] && labels[labelSearch].view.description"
+                :dark="labels[labelSearch] && labels[labelSearch].view.dark"
+                :light="labels[labelSearch] && labels[labelSearch].view.light"
+              >{{labelSearch}}</v-chip>
+            </template>
+          </v-text-field>
         </v-toolbar>
       </v-card-title>
       <v-card-text>
@@ -211,13 +229,14 @@
     :headers="headers"
     :items="rows"
     :items-per-page.sync="itemsPerPage"
+    :server-items-length="eventCount"
     item-key="key"
     :item-class="itemClass"
     sort-by="tstamp"
     :sort-desc="true"
     :search="filter"
-    :custom-filter="searchTable"
-    :loading="loading"
+    :loading="eventsLoading"
+    :options.sync="options"
     fixed-header
     :footer-props='{itemsPerPageOptions: [ 10, 25, 50, 100, 500, -1 ], showFirstLastPage: true}'
     @click:row="setActiveItem"
@@ -245,12 +264,12 @@
                   :dark="labels[label] && labels[label].view.dark"
                   :light="labels[label] && labels[label].view.light"
                   :key="label"
-                  :href="$route.path+'?label='+encodeURIComponent(label)"
+                  @click="labelSearch=label"
                 >{{label}}</v-chip>
               </span>
               <dougal-event-edit-history v-if="entry.has_edits && writeaccess"
                 :id="entry.id"
-                :disabled="loading"
+                :disabled="eventsLoading"
                 :labels="labels"
               ></dougal-event-edit-history>
               <span v-if="entry.meta.readonly"
@@ -381,7 +400,6 @@ export default {
         }
       ],
       items: [],
-      labels: {},
       options: {},
       filter: "",
       filterableLabels: [ "QC", "QCAccepted" ],
@@ -390,7 +408,6 @@ export default {
       eventDialog: false,
       eventLabelsDialog: false,
       defaultEventTimestamp: null,
-      presetRemarks: null,
       remarksMenu: null,
       remarksMenuItem: null,
       editedEvent: {},
@@ -440,17 +457,6 @@ export default {
       return Object.values(rows);
     },
 
-    userLabels () {
-      const filtered = {};
-      for (const key in this.labels) {
-        if (this.labels[key].model.user) {
-          filtered[key] = this.labels[key];
-        }
-      }
-      return filtered;
-
-    },
-
     popularLabels () {
       const tuples = this.items.flatMap( i => i.labels )
       .filter( l => (this.labels[l]??{})?.model?.user )
@@ -462,6 +468,10 @@ export default {
       .sort( (a, b) => b[1]-a[1] );
     },
 
+    presetRemarks () {
+      return this.projectConfiguration?.events?.presetRemarks ?? [];
+    },
+
     defaultSequence () {
       if (this.$route.params.sequence) {
         return Number(this.$route.params.sequence.split(";").pop());
@@ -470,19 +480,24 @@ export default {
       }
     },
 
-    ...mapGetters(['user', 'writeaccess', 'loading', 'online', 'sequence', 'line', 'point', 'position', 'timestamp', 'lineName', 'serverEvent']),
+    ...mapGetters(['user', 'writeaccess', 'eventsLoading', 'online', 'sequence', 'line', 'point', 'position', 'timestamp', 'lineName', 'serverEvent', 'events', 'labels', 'userLabels']),
     ...mapState({projectSchema: state => state.project.projectSchema})
 
   },
 
   watch: {
     options: {
-      handler () {
-        //this.getEvents();
+      async handler () {
+        await this.fetchEvents();
       },
       deep: true
     },
 
+    async events () {
+      console.log("Events changed");
+      await this.fetchEvents();
+    },
+
     eventDialog (val) {
       if (val) {
         // If not online
@@ -490,30 +505,14 @@ export default {
       }
     },
 
-    async serverEvent (event) {
-      if (event.channel == "event" && event.payload.schema == this.projectSchema) {
-        if (!this.loading && !this.queuedReload) {
-          // Do not force a non-cached response if refreshing as a result
-          // of an event notification. We will assume that the server has
-          // already had time to update the cache by the time our request
-          // gets back to it.
-          this.getEvents();
-        } else {
-          this.queuedReload = true;
-        }
+    filter (newVal, oldVal) {
+      if (newVal?.toLowerCase() != oldVal?.toLowerCase()) {
+        this.fetchEvents();
       }
     },
 
-    queuedReload (newVal, oldVal) {
-      if (newVal && !oldVal && !this.loading) {
-        this.getEvents();
-      }
-    },
-
-    loading (newVal, oldVal) {
-      if (!newVal && oldVal && this.queuedReload) {
-        this.getEvents();
-      }
+    labelSearch () {
+      this.fetchEvents();
     },
 
     itemsPerPage (newVal, oldVal) {
@@ -570,50 +569,15 @@ export default {
       }
     },
 
-    async getEventCount () {
-      //this.eventCount = await this.api([`/project/${this.$route.params.project}/event/?count`]);
-      this.eventCount = null;
-    },
-
-    async getEvents (opts = {}) {
-
-      const query = new URLSearchParams(this.options);
-      if (this.options.itemsPerPage < 0) {
-        query.delete("itemsPerPage");
-      }
-
-      if (this.$route.params.sequence) {
-        query.set("sequence", this.$route.params.sequence);
-      }
-
-      if (this.$route.params.date0) {
-        query.set("date0", this.$route.params.date0);
-      }
-
-      if (this.$route.params.date1) {
-        query.set("date1", this.$route.params.date1);
-      }
-
-      const url = `/project/${this.$route.params.project}/event?${query.toString()}`;
-
-      this.queuedReload = false;
-      this.items = await this.api([url, opts]) || [];
-
-    },
-
-    async getLabelDefinitions () {
-      const url = `/project/${this.$route.params.project}/label`;
-
-      const labelSet = {};
-      const labels = await this.api([url]) || [];
-      labels.forEach( l => labelSet[l.name] = l.data );
-      this.labels = labelSet;
-    },
-
-    async getPresetRemarks () {
-      const url = `/project/${this.$route.params.project}/configuration/events/presetRemarks`;
-
-      this.presetRemarks = await this.api([url]);
+    async fetchEvents (opts = {}) {
+      const options = {
+        text: this.filter,
+        label: this.labelSearch,
+        ...this.options
+      };
+      const res = await this.getEvents([this.$route.params.project, options]);
+      this.items = res.events;
+      this.eventCount = res.count;
     },
 
     newItem (from = {}) {
@@ -687,7 +651,7 @@ export default {
         if (!err && res.ok) {
           this.showSnack(["Event saved", "success"]);
           this.queuedReload = true;
-          this.getEvents({cache: "reload"});
+          this.fetchEvents({cache: "reload"});
         }
       }
 
@@ -705,7 +669,7 @@ export default {
         if (!err && res.ok) {
           this.showSnack(["Event saved", "success"]);
           this.queuedReload = true;
-          this.getEvents({cache: "reload"});
+          this.fetchEvents({cache: "reload"});
         }
       }
 
@@ -752,7 +716,7 @@ export default {
           if (!err && res.ok) {
             this.showSnack([`${ids.length} events deleted`, "red"]);
             this.queuedReload = true;
-            this.getEvents({cache: "reload"});
+            this.fetchEvents({cache: "reload"});
           }
         }
 
@@ -768,7 +732,7 @@ export default {
           if (!err && res.ok) {
             this.showSnack(["Event deleted", "red"]);
             this.queuedReload = true;
-            this.getEvents({cache: "reload"});
+            this.fetchEvents({cache: "reload"});
           }
         }
 
@@ -802,19 +766,6 @@ export default {
 
     },
 
-    searchTable (value, search, item) {
-      if (!value && !search) return true;
-      const s = search.toLowerCase();
-      if (typeof value === 'string') {
-        return value.toLowerCase().includes(s);
-      } else if (typeof value === 'number') {
-        return value == search;
-      } else {
-        return item.items.some( i => i.remarks.toLowerCase().includes(s) ) ||
-          item.items.some( i => i.labels.some( l => l.toLowerCase().includes(s) ));
-      }
-    },
-
     viewOnMap(item) {
       if (item?.meta && item.meta?.geometry?.type == "Point") {
         const [ lon, lat ] = item.meta.geometry.coordinates;
@@ -853,14 +804,11 @@ export default {
       */
     },
 
-    ...mapActions(["api", "showSnack"])
+    ...mapActions(["api", "showSnack", "refreshEvents", "getEvents"])
   },
 
   async mounted () {
-    await this.getLabelDefinitions();
-    this.getEventCount();
-    this.getEvents();
-    this.getPresetRemarks();
+    this.fetchEvents();
 
     window.addEventListener('keyup', this.handleKeyboardEvent);
   },

lib/www/client/source/src/views/Map.vue

@@ -375,7 +375,8 @@ export default {
         }
       ],
       labels: {},
-      hashMarker: null
+      hashMarker: null,
+      references: {}
     };
   },
 
@@ -474,7 +475,7 @@ export default {
         bounds._northEast.lng,
         bounds._northEast.lat
       ].map(i => i.toFixed(bboxScale)).join(",");
-      const limit = 10000;
+      const limit = 10000; // Empirical value
 
       const query = new URLSearchParams({bbox, limit});
 
@@ -511,7 +512,9 @@ export default {
             }
 
             l.layer.clearLayers();
-            if (layer instanceof L.Layer || (layer.features && layer.features.length < limit) || ("length" in layer && layer.length < limit)) {
+            //if (layer instanceof L.Layer || (layer.features && layer.features.length < limit) || ("length" in layer && layer.length < limit)) {
+            if (layer instanceof L.Layer || ((layer.features?.length ?? layer?.length) < limit)) {
+
               if (l.layer.addData) {
                 l.layer.addData(layer);
               } else if (l.layer.addLayer) {
@@ -519,8 +522,12 @@ export default {
               }
 
               l.layer.lastRequestURL = url;
+            } else if (!layer.features) {
+              console.log(`Layer ${url} is empty`);
             } else {
-              console.warn("Too much data from", url);
+              console.warn(`Too much data from ${url} (${layer.features?.length ?? layer.length} ≥ ${limit} features)`);
+
+              this.showSnack([`Layer ${l.layer.options.userLayerName ? "‘"+l.layer.options.userLayerName+"’ " : ""}is too large: ${layer.features?.length ?? layer.length} features; maximum is ${limit}`, "error"]);
             }
           })
           .finally( () => {
@@ -674,13 +681,140 @@ export default {
     async getLabelDefinitions () {
       const url = `/project/${this.$route.params.project}/label`;
 
-      const labelSet = {};
-      const labels = await this.api([url]) || [];
-      labels.forEach( l => labelSet[l.name] = l.data );
-      this.labels = labelSet;
+      this.labels = await this.api([url]) || [];
     },
 
-    ...mapActions(["api"])
+
+    removeUserLayers () {
+      map.eachLayer( layer => {
+        if (layer.options.userLayer === true) {
+          console.log("Removing", layer);
+          layer.eachLayer( sublayer => {
+            const idx = this.layerRefreshConfig.findIndex(i => i.layer == layer);
+            if (idx != -1) {
+              this.layerRefreshConfig.splice(idx, 1);
+            }
+          });
+          map.removeLayer(layer);
+          this.references.layerControl.removeLayer(layer);
+        }
+      });
+    },
+
+    async addUserLayers (userLayers) {
+
+      const options = {
+        userLayer: true,
+        style (feature) {
+          const style = {
+            stroke: undefined,
+            color: "grey",
+            weight: 2,
+            opacity: 0.5,
+            lineCap: undefined,
+            lineJoin: undefined,
+            dashArray: undefined,
+            dashOffset: undefined,
+            fill: undefined,
+            fillColor: "lightgrey",
+            fillOpacity: 0.5,
+            fillRule: undefined
+          };
+
+          for (let key in style) {
+            switch (key) {
+              case "color":
+                style[key] = feature.properties?.colour ?? feature.properties?.color ?? style[key];
+                break;
+              case "fillColor":
+                style[key] = feature.properties?.fillColour ?? feature.properties?.fillColor ?? style[key];
+                break;
+              default:
+                style[key] = feature.properties?.[key] ?? style[key];
+            }
+
+            if (typeof style[key] === "undefined") {
+              delete style[key];
+            }
+          }
+
+          return style;
+        }
+      };
+
+      const userLayerGroups = {};
+      userLayers.forEach(layer => {
+        if (!(layer.name in userLayerGroups)) {
+          userLayerGroups[layer.name] = [];
+        }
+        userLayerGroups[layer.name].push(layer);
+      });
+
+      for (let userLayerName in userLayerGroups) {
+        const userLayerGroup = userLayerGroups[userLayerName];
+
+        const layer = L.featureGroup(null, {userLayer: true, userLayerGroup: true, userLayerName});
+        userLayerGroup.forEach(l => {
+          const sublayer = L.geoJSON(null, {...options, userLayerName});
+          layer.addLayer(sublayer);
+          sublayer.on('add', ({target}) => {
+            this.refreshLayers([target])
+          });
+
+          if (l.tooltip) {
+            sublayer.bindTooltip((layer) => {
+              return layer?.feature?.properties?.[l.tooltip] ?? userLayerName;
+            });
+          }
+
+          if (l.popup) {
+            if (l.popup === true) {
+              sublayer.bindPopup((layer) => {
+                const p = layer?.feature?.properties;
+                let t = "";
+                if (p) {
+                  t += "<table>";
+                  for (let [k, v] of Object.entries(p)) {
+                    t += `<tr><td><b>${k}: </b></td><td>${v}</td></tr>`;
+                  }
+                  t += "</table>";
+                  return t;
+                }
+                return userLayerName;
+              });
+            } else {
+              sublayer.binPopup((layer) => {
+                return layer?.feature?.properties?.[l.popup] ?? userLayerName;
+              });
+            }
+          }
+
+          const refreshConfig = {
+            layer: sublayer,
+            url: (query = "") => {
+              return `/files/${l.path}`;
+            }
+          };
+
+          this.layerRefreshConfig.push(refreshConfig);
+        });
+
+        layer.on('add', ({target}) => {
+          this.refreshLayers(target.getLayers())
+        });
+        this.references.layerControl.addOverlay(layer, `<span title="User layer" style="text-decoration: dotted underline;">${userLayerName}</span>`);
+      }
+    },
+
+    async fetchUserLayers () {
+      const url = `/project/${this.$route.params.project}/gis/layer`;
+      const userLayers = await this.api([url]) || [];
+
+      this.removeUserLayers();
+      this.addUserLayers(userLayers);
+    },
+
+    ...mapActions(["api", "showSnack"])
 
   },
 
@@ -750,7 +884,7 @@ export default {
     }
 
     if (init.activeLayers) {
-      init.activeLayers.forEach(l => layers[l].addTo(map));
+      init.activeLayers.forEach(l => layers[l]?.addTo(map));
     } else {
       layers.OpenSeaMap.addTo(map);
       layers.Preplots.addTo(map);
@@ -759,6 +893,9 @@ export default {
     const layerControl = L.control.layers(tileMaps, layers).addTo(map);
     const scaleControl = L.control.scale().addTo(map);
 
+    this.references.layerControl = layerControl;
+    this.references.scaleControl = scaleControl;
+
     if (init.position) {
       map.setView(init.position.slice(1), init.position[0]);
     } else {
@@ -786,10 +923,13 @@ export default {
     map.on('layeradd', this.updateURL);
     map.on('layerremove', this.updateURL);
 
+
     this.layerRefreshConfig.forEach( l => {
       l.layer.on('add', ({target}) => this.refreshLayers([target]));
     });
 
+    this.fetchUserLayers();
+
     if (init.position) {
       this.refreshLayers();
     } else {

lib/www/client/source/src/views/Plan.vue

@@ -44,6 +44,7 @@
             label="Filter"
             single-line
             clearable
+            hint="Filter by sequence, line, first or last shotpoints, remarks or start/end time"
             ></v-text-field>
         </v-toolbar>
       </v-card-title>
@@ -109,11 +110,14 @@
         :headers="headers"
         :items="items"
         :items-per-page.sync="itemsPerPage"
+        :server-items-length="sequenceCount"
+        item-key="sequence"
         :search="filter"
-        :loading="loading"
-        :fixed-header="true"
+        :loading="plannedSequencesLoading"
+        fixed-header
         no-data-text="No planned lines. Add lines via the context menu from either the Lines or Sequences view."
         :item-class="(item) => (activeItem == item && !edit) ? 'blue accent-1 elevation-3' : ''"
+        :footer-props="{showFirstLastPage: true}"
         @click:row="setActiveItem"
         @contextmenu:row="contextMenu"
         >
@@ -275,7 +279,7 @@
                 icon
                 small
                 title="Edit"
-                :disabled="loading"
+                :disabled="plannedSequencesLoading"
                 @click="editItem(item, 'remarks')"
               >
                 <v-icon small>mdi-square-edit-outline</v-icon>
@@ -417,7 +421,8 @@ export default {
       remarks: null,
       editRemarks: false,
       filter: null,
-      num_lines: null,
+      options: {},
+      sequenceCount: null,
       activeItem: null,
       edit: null, // {sequence, key, value}
       queuedReload: false,
@@ -552,11 +557,22 @@ export default {
   },
 
   computed: {
-    ...mapGetters(['user', 'writeaccess', 'loading', 'serverEvent'])
+    ...mapGetters(['user', 'writeaccess', 'plannedSequencesLoading', 'plannedSequences', 'planRemarks'])
   },
 
   watch: {
 
+    options: {
+      handler () {
+        this.fetchPlannedSequences();
+      },
+      deep: true
+    },
+
+    async plannedSequences () {
+      await this.fetchPlannedSequences();
+    },
+
     async edit (newVal, oldVal) {
       if (newVal === null && oldVal !== null) {
         const item = this.items.find(i => i.sequence == oldVal.sequence);
@@ -587,41 +603,9 @@ export default {
       }
     },
 
-    async serverEvent (event) {
-      if (event.channel == "planned_lines" && event.payload.pid == this.$route.params.project) {
-
-        // Ignore non-ops
-        /*
-        if (event.payload.old === null && event.payload.new === null) {
-          return;
-        }
-        */
-
-        if (!this.loading && !this.queuedReload) {
-          // Do not force a non-cached response if refreshing as a result
-          // of an event notification. We will assume that the server has
-          // already had time to update the cache by the time our request
-          // gets back to it.
-          this.getPlannedLines();
-        } else {
-          this.queuedReload = true;
-        }
-      } else if (event.channel == "info" && event.payload.pid == this.$route.params.project) {
-        if (event.payload?.new?.key == "plan" && ("remarks" in (event.payload?.new?.value || {}))) {
-          this.remarks = event.payload?.new.value.remarks;
-        }
-      }
-    },
-
-    queuedReload (newVal, oldVal) {
-      if (newVal && !oldVal && !this.loading) {
-        this.getPlannedLines();
-      }
-    },
-
-    loading (newVal, oldVal) {
-      if (!newVal && oldVal && this.queuedReload) {
-        this.getPlannedLines();
+    filter (newVal, oldVal) {
+      if (newVal?.toLowerCase() != oldVal?.toLowerCase()) {
+        this.fetchPlannedSequences();
       }
     },
 
@@ -890,7 +874,6 @@ export default {
       const url = `/project/${this.$route.params.project}/plan/${this.contextMenuItem.sequence}`;
       const init = {method: "DELETE"};
       await this.api([url, init]);
-      await this.getPlannedLines();
     },
 
     editItem (item, key, value) {
@@ -942,21 +925,6 @@ export default {
       }
     },
 
-    async getPlannedLines () {
-
-      const url = `/project/${this.$route.params.project}/plan`;
-
-      this.queuedReload = false;
-      this.items = await this.api([url]) || [];
-      for (const item of this.items) {
-        item.ts0 = new Date(item.ts0);
-        item.ts1 = new Date(item.ts1);
-        this.wxQuery(item).then( (wx) => {
-          item.meta = {...item.meta, wx};
-        });
-      }
-    },
-
     async getPlannerConfig () {
       const url = `/project/${this.$route.params.project}/configuration/planner`;
       this.plannerConfig = await this.api([url]) || {
@@ -967,14 +935,15 @@ export default {
       }
     },
 
-    async getPlannerRemarks () {
-      const url = `/project/${this.$route.params.project}/info/plan/remarks`;
-      this.remarks = await this.api([url]) || "";
-    },
-
-    async getSequences () {
-      const url = `/project/${this.$route.params.project}/sequence`;
-      this.sequences = await this.api([url]) || [];
+    async fetchPlannedSequences (opts = {}) {
+      const options = {
+        text: this.filter,
+        ...this.options
+      };
+      const res = await this.getPlannedSequences([this.$route.params.project, options]);
+      this.items = res.sequences;
+      this.sequenceCount = res.count;
+      this.remarks = this.planRemarks;
     },
 
     setActiveItem (item) {
@@ -983,13 +952,12 @@ export default {
         : item;
     },
 
-    ...mapActions(["api", "showSnack"])
+    ...mapActions(["api", "showSnack", "getPlannedSequences"])
   },
 
   async mounted () {
     await this.getPlannerConfig();
-    this.getPlannedLines();
-    this.getPlannerRemarks();
+    await this.fetchPlannedSequences();
   }
 
 }

lib/www/client/source/src/views/Project.vue

@@ -1,18 +1,25 @@
 <template>
   <v-container fluid fill-height class="ma-0 pa-0">
     <v-row no-gutters align="stretch" class="fill-height">
-      <v-col cols="12">
+      <v-col cols="12" v-if="projectFound">
         <!-- Show component here according to selected route -->
         <keep-alive>
           <router-view :key="$route.path"></router-view>
         </keep-alive>
       </v-col>
+      <v-col cols="12" v-else>
+        <v-card>
+          <v-card-text>
+            Project does not exist.
+          </v-card-text>
+        </v-card>
+      </v-col>
     </v-row>
   </v-container>
 </template>
 
 <script>
-import { mapActions } from 'vuex'
+import { mapActions, mapGetters } from 'vuex'
 
 export default {
   name: 'Project',
@@ -24,12 +31,53 @@ export default {
     }
   },
 
+  computed: {
+
+    projectFound () {
+      return this.loading || this.projectId;
+    },
+
+    ...mapGetters(["loading", "projectId", "projectSchema", "serverEvent"])
+  },
+
+  watch: {
+    async serverEvent (event) {
+      if (event.channel == "project" && event.payload?.operation == "DELETE" && event.payload?.schema == "public") {
+        // Project potentially deleted
+        await this.getProject(this.$route.params.project);
+      } else if (event.payload?.schema == this.projectSchema) {
+        if (event.channel == "event") {
+          this.refreshEvents();
+        } else if (event.channel == "planned_lines") {
+          this.refreshPlan();
+        } else if (["raw_lines", "final_lines", "final_shots"].includes(event.channel)) {
+          this.refreshSequences();
+        } else if (["preplot_lines", "preplot_points"].includes(event.channel)) {
+          this.refreshLines();
+        } else if (event.channel == "info") {
+          if ((event.payload?.new ?? event.payload?.old)?.key == "plan") {
+            this.refreshPlan();
+          }
+        } else if (event.channel == "project") {
+          this.getProject(this.$route.params.project);
+        }
+      }
+    }
+  },
+
   methods: {
-    ...mapActions(["getProject"])
+    ...mapActions(["getProject", "refreshLines", "refreshSequences", "refreshEvents", "refreshLabels", "refreshPlan"])
   },
 
   async mounted () {
     await this.getProject(this.$route.params.project);
+    if (this.projectFound) {
+      this.refreshLines();
+      this.refreshSequences();
+      this.refreshEvents();
+      this.refreshLabels();
+      this.refreshPlan();
+    }
   }
 
 }

lib/www/client/source/src/views/ProjectList.vue

@@ -83,12 +83,22 @@ export default {
   },
 
   computed: {
-    ...mapGetters(['loading'])
+    ...mapGetters(['loading', 'serverEvent'])
+  },
+
+  watch: {
+    async serverEvent (event) {
+      if (event.channel == "project" && event.payload?.schema == "public") {
+        if (event.payload?.operation == "DELETE" || event.payload?.operation == "INSERT") {
+          await this.load();
+        }
+      }
+    }
   },
 
   methods: {
     async list () {
-      this.items = await this.api(["/project/"]) || [];
+      this.items = await this.api(["/project"]) || [];
     },
 
     async summary (item) {

lib/www/client/source/src/views/QC.vue

@@ -433,10 +433,7 @@ export default {
     async getLabelDefinitions () {
       const url = `/project/${this.$route.params.project}/label`;
 
-      const labelSet = {};
-      const labels = await this.api([url]) || [];
-      labels.forEach( l => labelSet[l.name] = l.data );
-      this.labels = labelSet;
+      this.labels = await this.api([url]) || {};
     },
 
     async getQCData () {

lib/www/client/source/src/views/SequenceList.vue

@@ -148,15 +148,16 @@
           :headers="headers"
           :items="items"
           :items-per-page.sync="itemsPerPage"
+          :server-items-length="sequenceCount"
           item-key="sequence"
-          :server-items-length="num_rows"
-          :search="filter"
-          :custom-filter="customFilter"
-          :loading="loading"
-          :fixed-header="true"
-          :footer-props='{itemsPerPageOptions: [ 10, 25, 50, 100, 500, -1 ]}'
-          show-expand
           :item-class="(item) => activeItem == item ? 'blue accent-1 elevation-3' : ''"
+          :search="filter"
+          x-custom-filter="customFilter"
+          :loading="sequencesLoading"
+          :options.sync="options"
+          fixed-header
+          :footer-props='{itemsPerPageOptions: [ 10, 25, 50, 100, 500, -1 ], showFirstLastPage: true}'
+          show-expand
           @click:row="setActiveItem"
           @contextmenu:row="contextMenu"
         >
@@ -176,7 +177,7 @@
                                   icon
                                   small
                                   title="Cancel edit"
-                                  :disabled="loading"
+                                  :disabled="sequencesLoading"
                                   @click="edit.value = item.remarks; edit = null"
                                 >
                                   <v-icon small>mdi-close</v-icon>
@@ -185,7 +186,7 @@
                                   icon
                                   small
                                   title="Save edits"
-                                  :disabled="loading"
+                                  :disabled="sequencesLoading"
                                   @click="edit = null"
                                 >
                                   <v-icon small>mdi-content-save-edit-outline</v-icon>
@@ -196,7 +197,7 @@
                                 icon
                                 small
                                 title="Edit"
-                                :disabled="loading"
+                                :disabled="sequencesLoading"
                                 @click="editItem(item, 'remarks')"
                               >
                                 <v-icon small>mdi-square-edit-outline</v-icon>
@@ -210,7 +211,7 @@
                               class="markdown"
                               autofocus
                               placeholder="Enter your text here"
-                              :disabled="loading"
+                              :disabled="sequencesLoading"
                               v-model="edit.value"
                             >
                             </v-textarea>
@@ -228,7 +229,7 @@
                                   icon
                                   small
                                   title="Cancel edit"
-                                  :disabled="loading"
+                                  :disabled="sequencesLoading"
                                   @click="edit.value = item.remarks_final; edit = null"
                                 >
                                   <v-icon small>mdi-close</v-icon>
@@ -237,7 +238,7 @@
                                   icon
                                   small
                                   title="Save edits"
-                                  :disabled="loading"
+                                  :disabled="sequencesLoading"
                                   @click="edit = null"
                                 >
                                   <v-icon small>mdi-content-save-edit-outline</v-icon>
@@ -248,7 +249,7 @@
                                 icon
                                 small
                                 title="Edit"
-                                :disabled="loading"
+                                :disabled="sequencesLoading"
                                 @click="editItem(item, 'remarks_final')"
                               >
                                 <v-icon small>mdi-square-edit-outline</v-icon>
@@ -262,7 +263,7 @@
                               class="markdown"
                               autofocus
                               placeholder="Enter your text here"
-                              :disabled="loading"
+                              :disabled="sequencesLoading"
                               v-model="edit.value"
                             >
                             </v-textarea>
@@ -292,9 +293,13 @@
                             <v-list-item v-for="(path, index) in item.raw_files"
                               key="index"
                               link
-                              title="View the shot log"
+                              title="Download file"
+                              :href="`/api/files${path}`"
                             >
                               {{ basename(path) }}
+                              <v-list-item-action>
+                                <v-icon right small>mdi-cloud-download</v-icon>
+                              </v-list-item-action>
                             </v-list-item>
                           </v-list-group>
                           <v-list-group value="true" v-if="item.final_files">
@@ -308,10 +313,13 @@
                             </template>
                             <v-list-item v-for="(path, index) in item.final_files"
                               key="index"
-                              link
-                              title="View the shot log"
+                              title="Download file"
+                              :href="`/api/files${path}`"
                             >
                               {{ basename(path) }}
+                              <v-list-item-action>
+                                <v-icon right small>mdi-cloud-download</v-icon>
+                              </v-list-item-action>
                             </v-list-item>
                           </v-list-group>
                         </v-list>
@@ -559,7 +567,7 @@ export default {
       items: [],
       filter: "",
       options: {},
-      num_rows: null,
+      sequenceCount: null,
       activeItem: null,
       edit: null, // {sequence, key, value}
       queuedReload: false,
@@ -586,17 +594,22 @@ export default {
       return this.queuedItems.find(i => i.payload.sequence == this.contextMenuItem.sequence);
     },
 
-    ...mapGetters(['user', 'writeaccess', 'loading', 'serverEvent'])
+    ...mapGetters(['user', 'writeaccess', 'sequencesLoading', 'sequences'])
   },
 
   watch: {
+
     options: {
       handler () {
-        this.getSequences();
+        this.fetchSequences();
       },
       deep: true
     },
 
+    async sequences () {
+      await this.fetchSequences();
+    },
+
     async edit (newVal, oldVal) {
       if (newVal === null && oldVal !== null) {
         const item = this.items.find(i => i.sequence == oldVal.sequence);
@@ -610,39 +623,9 @@ export default {
       }
     },
 
-    async serverEvent (event) {
-      const subscriptions = ["raw_lines", "final_lines", "final_shots"];
-      if (subscriptions.includes(event.channel) && event.payload.pid == this.$route.params.project) {
-        if (!this.loading && !this.queuedReload) {
-          // Do not force a non-cached response if refreshing as a result
-          // of an event notification. We will assume that the server has
-          // already had time to update the cache by the time our request
-          // gets back to it.
-          this.getSequences();
-        } else {
-          this.queuedReload = true;
-        }
-      } else if (event.channel == "queue_items") {
-        const project =
-          event.payload?.project ??
-          event.payload?.new?.payload?.project ??
-          event.payload?.old?.payload?.project;
-
-        if (project == this.$route.params.project) {
-          this.getQueuedItems();
-        }
-      }
-    },
-
-    queuedReload (newVal, oldVal) {
-      if (newVal && !oldVal && !this.loading) {
-        this.getSequences();
-      }
-    },
-
-    loading (newVal, oldVal) {
-      if (!newVal && oldVal && this.queuedReload) {
-        this.getSequences();
+    filter (newVal, oldVal) {
+      if (newVal?.toLowerCase() != oldVal?.toLowerCase()) {
+        this.fetchSequences();
       }
     },
 
@@ -811,19 +794,14 @@ export default {
       this.num_rows = projectInfo.sequences;
     },
 
-    async getSequences () {
-
-      const query = new URLSearchParams(this.options);
-      query.set("filter", this.filter);
-      query.set("files", true);
-      if (this.options.itemsPerPage < 0) {
-        query.delete("itemsPerPage");
-      }
-      const url = `/project/${this.$route.params.project}/sequence?${query.toString()}`;
-
-      this.queuedReload = false;
-      this.items = await this.api([url]) || [];
-
+    async fetchSequences (opts = {}) {
+      const options = {
+        text: this.filter,
+        ...this.options
+      };
+      const res = await this.getSequences([this.$route.params.project, options]);
+      this.items = res.sequences;
+      this.sequenceCount = res.count;
     },
 
     async getQueuedItems () {
@@ -871,11 +849,11 @@ export default {
       return false;
     },
 
-    ...mapActions(["api", "showSnack"])
+    ...mapActions(["api", "showSnack", "getSequences"])
   },
 
   mounted () {
-    this.getSequences();
+    this.fetchSequences();
     this.getNumLines();
     this.getQueuedItems();
   }

lib/www/client/source/vue.config.js

@@ -4,6 +4,7 @@ module.exports = {
     "leaflet-arrowheads"
   ],
   devServer: {
+    host: "0.0.0.0",
     proxy: {
       "^/api(/|$)": {
         target: "http://localhost:3000",

lib/www/server/api/index.js

@@ -1,6 +1,7 @@
 
 const http = require('http');
 const express = require('express');
+express.yaml ??= require('body-parser').yaml; // NOTE: Use own customised body-parser
 const cookieParser = require('cookie-parser')
 
 const maybeSendAlert = require("../lib/alerts");
@@ -9,7 +10,7 @@ const mw = require('./middleware');
 const { ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
 const verbose = process.env.NODE_ENV != 'test';
 const app = express();
-app.locals.version = "0.3.1"; // API version
+app.locals.version = "0.4.0"; // API version
 
 app.map = function(a, route){
 	route = route || '';
@@ -31,6 +32,7 @@ app.map = function(a, route){
 };
 
 app.use(express.json({type: "application/json", strict: false, limit: '10mb'}));
+app.use(express.yaml({type: "application/yaml", limit: '10mb'}));
 app.use(express.urlencoded({ type: "application/x-www-form-urlencoded", extended: true }));
 app.use(express.text({type: "text/*", limit: '10mb'}));
 app.use((req, res, next) => {
@@ -87,13 +89,19 @@ app.use(mw.etag.ifNoneMatch);
 // We must be authenticated before we can access these
 app.map({
 	'/project': {
-		get: [ mw.project.list ],  // Get list of projects
+		get: [ mw.project.get ],  // Get list of projects
+		post: [ mw.auth.access.admin, mw.project.post ], // Create a new project
 	},
 	'/project/:project': {
-		get: [ mw.project.get ],      // Get project data
+		get: [ mw.project.summary.get ],      // Get project data
+		delete: [ mw.auth.access.admin, mw.project.delete ], // Delete a project (only if empty)
 	},
 	'/project/:project/summary': {
-		get: [ mw.project.get ],
+		get: [ mw.project.summary.get ],
+	},
+	'/project/:project/configuration': {
+		get: [ mw.project.configuration.get ],	// Get project configuration
+		patch: [ mw.auth.access.admin, mw.project.configuration.patch ],	// Modify project configuration
 	},
 
 	/*
@@ -101,19 +109,25 @@ app.map({
 	 */
 
 	'/project/:project/gis': {
-		get: [ mw.gis.project.bbox ]
+		get: [ mw.etag.noSave, mw.gis.project.bbox ]
 	},
 	'/project/:project/gis/preplot': {
-		get: [ mw.gis.project.preplot ]
+		get: [ mw.etag.noSave, mw.gis.project.preplot ]
 	},
 	'/project/:project/gis/preplot/:featuretype(line|point)': {
-		get: [ mw.gis.project.preplot ]
+		get: [ mw.etag.noSave, mw.gis.project.preplot ]
 	},
 	'/project/:project/gis/raw/:featuretype(line|point)': {
-		get: [ mw.gis.project.raw ]
+		get: [ mw.etag.noSave, mw.gis.project.raw ]
 	},
 	'/project/:project/gis/final/:featuretype(line|point)': {
-		get: [ mw.gis.project.final ]
+		get: [ mw.etag.noSave, mw.gis.project.final ]
+	},
+	'/project/:project/gis/layer': {
+		get: [ mw.etag.noSave, mw.gis.project.layer.get ]
+	},
+	'/project/:project/gis/layer/:name': {
+		get: [ mw.etag.noSave, mw.gis.project.layer.get ]
 	},
 
 	/*
@@ -167,6 +181,9 @@ app.map({
 		post: [ mw.auth.access.write, mw.event.post ],
 		put: [ mw.auth.access.write, mw.event.put ],
 		delete: [ mw.auth.access.write, mw.event.delete ],
+		'changes/:since': {
+			get: [ mw.event.changes ]
+		},
 		// TODO Rename -/:sequence → sequence/:sequence
 		'-/:sequence/': { // NOTE: We need to avoid conflict with the next endpoint ☹
 			get: [ mw.event.sequence.get ],
@@ -186,25 +203,25 @@ app.map({
 	'/project/:project/qc': {
 		'/results': {
 			// Get all QC results for :project
-			get: [ mw.qc.results.get ],
+			get: [ mw.etag.noSave, mw.qc.results.get ],
 
 			// Delete all QC results for :project
-			delete: [ mw.auth.access.write, mw.qc.results.delete ],
+			delete: [ mw.etag.noSave, mw.auth.access.write, mw.qc.results.delete ],
 
 			'/accept': {
-				post: [ mw.auth.access.write, mw.qc.results.accept ]
+				post: [ mw.etag.noSave, mw.auth.access.write, mw.qc.results.accept ]
 			},
 
 			'/unaccept': {
-				post: [ mw.auth.access.write, mw.qc.results.unaccept ]
+				post: [ mw.etag.noSave, mw.auth.access.write, mw.qc.results.unaccept ]
 			},
 
 			'/sequence/:sequence': {
 				// Get QC results for :project, :sequence
-				get: [ mw.qc.results.get ],
+				get: [ mw.etag.noSave, mw.qc.results.get ],
 
 				// Delete QC results for :project, :sequence
-				delete: [ mw.auth.access.write, mw.qc.results.delete ]
+				delete: [ mw.etag.noSave, mw.auth.access.write, mw.qc.results.delete ]
 			}
 		}
 	},
@@ -247,10 +264,16 @@ app.map({
 // // 		post: [ mw.permissions.post ],
 // // 		delete: [ mw.permissions.delete ]
 // 	},
+	'/project/:project/files/:path(*)': {
+		get: [ mw.auth.access.write, mw.files.get ]
+	},
+	'/files/?:path(*)': {
+		get: [ mw.auth.access.write, mw.etag.noSave, mw.files.get ]
+	},
 	'/navdata/': {
-		get: [ mw.navdata.get ],
+		get: [ mw.etag.noSave, mw.navdata.get ],
 		'gis/:featuretype(line|point)': {
-			get: [ mw.gis.navdata.get ]
+			get: [ mw.etag.noSave, mw.gis.navdata.get ]
 		}
 	},
 	'/info/': {
@@ -329,15 +352,14 @@ app.disable('x-powered-by');
 app.enable('trust proxy');
 INFO('trust proxy is ' + (app.get('trust proxy')? 'on' : 'off'));
 
-const addr = "127.0.0.1";
-
 if (!module.parent) {
-	var port = process.env.HTTP_PORT || 3000;
-	var server = http.createServer(app).listen(port, addr);
+	const port = process.env.HTTP_PORT || 3000;
+	const host = process.env.HTTP_HOST || "127.0.0.1";
+	var server = http.createServer(app).listen(port, host);
 
 	INFO('API started on port ' + port);
 } else {
-	app.start = function (port = 3000, path) {
+	app.start = function (port = 3000, host = "127.0.0.1", path) {
 
 		var root = app;
 		if (path) {
@@ -346,9 +368,9 @@ if (!module.parent) {
 			root.use(path, app);
 		}
 
-		const server = http.createServer(root).listen(port, addr);
+		const server = http.createServer(root).listen(port, host);
 		if (server) {
-// 			console.log(`API started on port ${port}, prefix: ${path || "/"}`);
+			console.log(`API started on port ${port}, prefix: ${path || "/"}`);
 			INFO(`API started on port ${port}, prefix: ${path || "/"}`);
 		}
 		return server;

lib/www/server/api/middleware/auth/jwt.js

@@ -1,4 +1,4 @@
-const expressJWT = require('express-jwt');
+const {expressjwt: expressJWT} = require('express-jwt');
 
 const cfg = require("../../../lib/config").jwt;
 
@@ -15,11 +15,12 @@ const options = {
 	secret: cfg.secret,
 	credentialsRequired: false,
 	algorithms: ['HS256'],
+	requestProperty: "user",
 	getToken
 };
 
 const allow = {
-	path: [/\/login$/, /\/logout$/],
+	path: [/\/login$/, /\/logout$/, /\/$/, /\/version$/],
 	useOriginalUrl: false
 };
 

lib/www/server/api/middleware/etag/cache.js

@@ -33,7 +33,9 @@ function saveResponse (res) {
 			const cache = getCache(res);
 			const req = res.req;
 			console.log(`Saving ETag: ${req.method} ${req.url} → ${etag}`);
-			cache[req.url] = {etag, headers: res.getHeaders()};
+			const headers = structuredClone(res.getHeaders());
+			delete headers["set-cookie"];
+			cache[req.url] = {etag, headers};
 		}
 	}
 };

lib/www/server/api/middleware/etag/watch.js

@@ -43,15 +43,26 @@ const rels = [
 		matches: [ ],
 		callback (url, data) {
 			if (data.payload?.table == "info") {
+				const pid = data.payload?.pid;
+				const key = (data.payload?.new ?? data.payload?.old)?.key;
+
 				const rx = /^\/project\/([^\/]+)\/info\/([^\/?]+)[\/?]?/;
 				const match = url.match(rx);
 				if (match) {
-					if (match[1] == data.payload.pid) {
+					if (match[1] == pid) {
 						if (match[2] == data.payload?.old?.key || match[2] == data.payload?.new?.key) {
 							return true;
 						}
 					}
 				}
+
+				if (key == "plan") {
+					const rx = /^\/project\/([^\/]+)\/plan[\/?]?/;
+					const match = url.match(rx);
+					if (match) {
+						return match[1] == pid;
+					}
+				}
 			}
 			return false;
 		}

lib/www/server/api/middleware/event/changes.js

@@ -0,0 +1,14 @@
+
+const { event } = require('../../../lib/db');
+
+const json = async function (req, res, next) {
+	try {
+		const response = await event.changes(req.params.project, req.params.since, req.query);
+		res.status(200).send(response);
+		next();
+	} catch (err) {
+		next(err);
+	}
+};
+
+module.exports = json;

lib/www/server/api/middleware/event/index.js

@@ -6,5 +6,6 @@ module.exports = {
 	post: require('./post'),
 	put: require('./put'),
 	patch: require('./patch'),
-	delete: require('./delete')
+	delete: require('./delete'),
+	changes: require('./changes')
 }

lib/www/server/api/middleware/event/sequence/get/csv.js

@@ -0,0 +1,85 @@
+const { stringify } = require('csv');
+const { transform, prepare } = require('../../../../../lib/sse');
+
+const json = async function (req, res, next) {
+	try {
+		const query = req.query;
+		query.sequence = req.params.sequence;
+		const {events, sequences} = await prepare(req.params.project, query);
+		if ("download" in query || "d" in query) {
+			const extension = "csv";
+			// Get the sequence number(s) (more than one sequence can be selected)
+			const seqNums = query.sequence.split(";");
+			// If we've only been asked for a single sequence, get its line name
+			const lineName = (sequences.find(i => i.sequence == seqNums[0]) || {})?.meta?.lineName;
+			const filename = (seqNums.length  == 1 && lineName)
+				? `${lineName}-NavLog.${extension}`
+				: `${req.params.project}-${query.sequence}.${extension}`;
+			res.set("Content-Disposition", `attachment; filename="${filename}"`);
+		}
+
+		const columns = {
+			id: "id",
+			unix_epoch: (row) => Math.floor(row.tstamp/1000),
+			timestamp: (row) => (new Date(row.tstamp)).toISOString(),
+			sequence: "sequence",
+			point: "point",
+			text: "remarks",
+			labels: (row) => row.labels.join(";"),
+			latitude: (row) => {
+				if (row.meta.geometry?.type == "Point" && row.meta.geometry?.coordinates) {
+					return row.meta.geometry.coordinates[1];
+				}
+			},
+			longitude: (row) => {
+				if (row.meta.geometry?.type == "Point" && row.meta.geometry?.coordinates) {
+					return row.meta.geometry.coordinates[0];
+				}
+			}
+		};
+
+		let fields = [ "timestamp", "sequence", "point", "text", "labels", "latitude", "longitude", "id" ];
+
+		if (req.query.fields) {
+			fields = req.query.fields.split(/[,;:.\s+*|]+/);
+		}
+
+		let delimiter = req.query.delimiter || ",";
+
+		const stringifier = stringify({delimiter});
+		stringifier.on('error', (err) => {
+			console.error(err.message);
+		});
+
+		stringifier.on('readable', () => {
+			while((row = stringifier.read()) !== null) {
+				res.write(row);
+			}
+		});
+
+		res.status(200);
+
+		if (!req.query.header || req.query.header.toLowerCase() == "true" || req.query.header == "1") {
+			// Send header
+			stringifier.write(fields);
+		}
+
+		events.forEach( event => {
+			stringifier.write(fields.map( field => {
+				if (typeof columns[field] === "function") {
+					return columns[field](event);
+				} else {
+					return event[columns[field]];
+				}
+			}));
+		});
+
+		stringifier.end();
+		res.end();
+		next();
+	} catch (err) {
+		next(err);
+	}
+};
+
+module.exports = json;

lib/www/server/api/middleware/event/sequence/get/index.js

@@ -2,6 +2,7 @@ const json = require('./json');
 const geojson = require('./geojson');
 const seis = require('./seis');
 const html = require('./html');
+const csv = require('./csv');
 const pdf = require('./pdf');
 
 module.exports = async function (req, res, next) {
@@ -11,6 +12,7 @@ module.exports = async function (req, res, next) {
 			"application/geo+json": geojson,
 			"application/vnd.seis+json": seis,
 			"text/html": html,
+			"text/csv": csv,
 			"application/pdf": pdf
 		};
 

lib/www/server/api/middleware/files/get.js

@@ -0,0 +1,29 @@
+const files = require('../../../lib/files');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const entity = await files.get(req.params.path, req.params.project, req.query);
+		if (entity) {
+			if (entity.download) {
+				res.download(...entity.download, (err) => next(err));
+			} else {
+				// Directory listing
+				res.status(203).json(entity);
+				next();
+			}
+		} else {
+			throw {
+				status: 404,
+				code: "ENOENT"
+			};
+		}
+	} catch (err) {
+		if (err.code == 'ENOENT') {
+			res.status(404).json({message: err.code});
+		} else {
+			next(err);
+		}
+	}
+
+};

lib/www/server/api/middleware/files/index.js

@@ -0,0 +1,7 @@
+
+module.exports = {
+	get: require('./get'),
+	post: require('./post'),
+	put: require('./put'),
+	delete: require('./delete')
+}

lib/www/server/api/middleware/gis/project/index.js

@@ -2,5 +2,6 @@ module.exports = {
 	bbox: require('./bbox'),
 	preplot: require('./preplot'),
 	raw: require('./raw'),
-	final: require('./final')
+	final: require('./final'),
+	layer: require('./layer')
 };

lib/www/server/api/middleware/gis/project/layer/get.js

@@ -0,0 +1,18 @@
+
+const { gis } = require('../../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const layers = await gis.project.layer.get(req.params.project, req.params.name);
+		if (req.params.name && (!layers || !layers.length)) {
+			res.status(404).json({message: "Not found"});
+		} else {
+			res.status(200).send(layers ?? []);
+		}
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/gis/project/layer/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	get: require('./get')
+};

lib/www/server/api/middleware/index.js

@@ -1,5 +1,6 @@
 module.exports = {
 	event: require('./event'),
+	files: require('./files'),
 	plan: require('./plan'),
 	line: require('./line'),
 	project: require('./project'),

lib/www/server/api/middleware/label/list.js

@@ -1,10 +1,11 @@
 
-const { label } = require('../../../lib/db');
+const { project } = require('../../../lib/db');
 
 module.exports = async function (req, res, next) {
 
 	try {
-		res.status(200).send(await label.list(req.params.project, req.query));
+		const labels = (await project.configuration.get(req.params.project))?.labels ?? {};
+		res.status(200).send(labels);
 		next();
 	} catch (err) {
 		next(err);

lib/www/server/api/middleware/plan/list/json.js

@@ -1,9 +1,14 @@
 
-const { plan } = require('../../../../lib/db');
+const { plan, info } = require('../../../../lib/db');
 
 const json = async function (req, res, next) {
 	try {
-		const response = await plan.list(req.params.project, req.query);
+		const sequences = await plan.list(req.params.project, req.query) ?? [];
+		const remarks = await info.get(req.params.project, "plan/remarks", req.query, req.user.role) ?? null;
+		const response = {
+			remarks,
+			sequences
+		};
 		res.status(200).send(response);
 		next();
 	} catch (err) {

lib/www/server/api/middleware/project/configuration/get.js

@@ -0,0 +1,13 @@
+
+const { project } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		res.status(200).send(await project.configuration.get(req.params.project));
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/project/configuration/index.js

@@ -0,0 +1,8 @@
+
+module.exports = {
+	get: require('./get'),
+	// post: require('./post'),
+	// put: require('./put'),
+	patch: require('./patch'),
+	// delete: require('./delete'),
+};

lib/www/server/api/middleware/project/configuration/patch.js

@@ -0,0 +1,16 @@
+
+const { project } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		// TODO
+		// Implement If-Match header requirements
+		res.send(await project.configuration.patch(req.params.project, req.body));
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+
+};

lib/www/server/api/middleware/project/delete.js

@@ -0,0 +1,15 @@
+
+const { project } = require('../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		await project.delete(req.params.project)
+		res.status(204).send();
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+
+};

lib/www/server/api/middleware/project/get.js

@@ -4,10 +4,11 @@ const { project} = require('../../../lib/db');
 module.exports = async function (req, res, next) {
 
 	try {
-		res.status(200).send(await project.get(req.params.project));
+		res.status(200).send(await project.get());
 		next();
 	} catch (err) {
 		next(err);
 	}
 
+
 };

lib/www/server/api/middleware/project/index.js

@@ -1,4 +1,7 @@
 module.exports = {
-	list: require('./list'),
-	get: require('./get')
+	get: require('./get'),
+	post: require('./post'),
+	delete: require('./delete'),
+	summary: require('./summary'),
+	configuration: require('./configuration'),
 };

lib/www/server/api/middleware/project/list.js

@@ -1,14 +0,0 @@
-
-const { project} = require('../../../lib/db');
-
-module.exports = async function (req, res, next) {
-
-	try {
-		res.status(200).send(await project.list());
-		next();
-	} catch (err) {
-		next(err);
-	}
-
-
-};

lib/www/server/api/middleware/project/post.js

@@ -0,0 +1,16 @@
+
+const { project } = require('../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		const payload = req.body;
+
+		const projectDefinition = await project.post(payload);
+		res.status(201).send(projectDefinition);
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/project/summary/get.js

@@ -0,0 +1,13 @@
+
+const { project } = require('../../../../lib/db');
+
+module.exports = async function (req, res, next) {
+
+	try {
+		res.status(200).send(await project.summary.get(req.params.project));
+		next();
+	} catch (err) {
+		next(err);
+	}
+
+};

lib/www/server/api/middleware/project/summary/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+	get: require('./get'),
+};

lib/www/server/events/handlers/detect-fdsp.js

@@ -9,105 +9,16 @@ const { ALERT, ERROR, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
  * the last shot and first shot of the previous and current dates, respectively.
  */
 class DetectFDSP {
-	/* Data may come much faster than we can process it, so we put it
-	 * in a queue and process it at our own pace.
-	 *
-	 * The run() method fills the queue with the necessary data and then
-	 * calls processQueue().
-	 *
-	 * The processQueue() method looks at the first two elements in
-	 * the queue and processes them if they are not already being taken
-	 * care of by a previous processQueue() call – this will happen when
-	 * data is coming in faster than it can be processed.
-	 *
-	 * If the processQueue() call is the first to see the two bottommost
-	 * two elements, it will process them and, when finished, it will set
-	 * the `isPending` flag of the bottommost element to `false`, thus
-	 * letting the next call know that it has work to do.
-	 *
-	 * If the queue was empty, run() will set the `isPending` flag of its
-	 * first element to a falsy value, thus bootstrapping the process.
-	 */
-	static MAX_QUEUE_SIZE = 125000;
 
-	queue = [];
+	author = `*${this.constructor.name}*`;
+	prev = null;
 
-	async processQueue () {
-		DEBUG("Queue length", this.queue.length)
-		while (this.queue.length > 1) {
-
-			if (this.queue[0].isPending) {
-				setImmediate(() => this.processQueue());
-				return;
-			}
-
-			const prev = this.queue.shift();
-			const cur  = this.queue[0];
-
-			const sequence = Number(cur._sequence);
-
-			try {
-
-				if (prev.lineName == cur.lineName && prev._sequence == cur._sequence &&
-					prev.lineStatus == "online" && cur.lineStatus == "online" && sequence) {
-
-// 					DEBUG("Previous", prev);
-// 					DEBUG("Current", cur);
-
-					if (prev.time.substr(0, 10) != cur.time.substr(0, 10)) {
-						// Possible a date change, but could also be a missing timestamp
-						// or something else.
-
-						const ts0 = new Date(prev.time)
-						const ts1 = new Date(cur.time);
-
-						if (!isNaN(ts0) && !isNaN(ts1) &&  ts0.getUTCDay() != ts1.getUTCDay()) {
-							INFO("Sequence shot across midnight UTC detected", cur._sequence, cur.lineName);
-
-							const ldsp = {
-								sequence: prev._sequence,
-								point: prev._point,
-								remarks: "Last shotpoint of the day",
-								labels: ["LDSP", "Prod"],
-								meta: {auto: true, insertedBy: this.constructor.name}
-							};
-
-							const fdsp = {
-								sequence: cur._sequence,
-								point: cur._point,
-								remarks: "First shotpoint of the day",
-								labels: ["FDSP", "Prod"],
-								meta: {auto: true, insertedBy: this.constructor.name}
-							};
-
-							INFO("LDSP", ldsp);
-							INFO("FDSP", fdsp);
-
-							const projectId = await schema2pid(prev._schema);
-
-							if (projectId) {
-								await event.post(projectId, ldsp);
-								await event.post(projectId, fdsp);
-							} else {
-								ERROR("projectId not found for", prev._schema);
-							}
-						} else {
-							WARNING("False positive on these timestamps", prev.time, cur.time);
-							WARNING("No events were created");
-						}
-					}
-				}
-				// Processing of this shot has already been completed.
-				// The queue can now move forward.
-			} catch (err) {
-				ERROR(err);
-			} finally {
-				cur.isPending = false;
-			}
-		}
+	constructor () {
+		DEBUG(`${this.author} instantiated`);
 	}
 
-	async run (data) {
+	async run (data, ctx) {
+
 		if (!data || data.channel !== "realtime") {
 			return;
 		}
@@ -116,27 +27,70 @@ class DetectFDSP {
 			return;
 		}
 
-		const meta = data.payload.new.meta;
-
-		if (this.queue.length < DetectFDSP.MAX_QUEUE_SIZE) {
-
-			const event = {
-				isPending: this.queue.length,
-				_schema: meta._schema,
-				time: meta.time,
-				lineStatus: meta.lineStatus,
-				_sequence: meta._sequence,
-				_point: meta._point,
-				lineName: meta.lineName
-			};
-			this.queue.push(event);
-// 			DEBUG("EVENT", event);
-
-		} else {
-			ALERT("Queue full at", this.queue.length);
+		if (!this.prev) {
+			DEBUG("Initialising `prev`");
+			this.prev = data;
+			return;
 		}
 
-		this.processQueue();
+		try {
+			DEBUG("Running");
+			const cur = data;
+			const sequence = Number(cur._sequence);
+
+			if (this.prev.lineName == cur.lineName && this.prev._sequence == cur._sequence &&
+				this.prev.lineStatus == "online" && cur.lineStatus == "online" && sequence) {
+
+				if (this.prev.time.substr(0, 10) != cur.time.substr(0, 10)) {
+					// Possibly a date change, but could also be a missing timestamp
+					// or something else.
+
+					const ts0 = new Date(this.prev.time)
+					const ts1 = new Date(cur.time);
+
+					if (!isNaN(ts0) && !isNaN(ts1) &&  ts0.getUTCDay() != ts1.getUTCDay()) {
+						INFO("Sequence shot across midnight UTC detected", cur._sequence, cur.lineName);
+
+						const ldsp = {
+							sequence: this.prev._sequence,
+							point: this.prev._point,
+							remarks: "Last shotpoint of the day",
+							labels: ["LDSP", "Prod"],
+							meta: {auto: true, author: `*${this.constructor.name}*`}
+						};
+
+						const fdsp = {
+							sequence: cur._sequence,
+							point: cur._point,
+							remarks: "First shotpoint of the day",
+							labels: ["FDSP", "Prod"],
+							meta: {auto: true, author: `*${this.constructor.name}*`}
+						};
+
+						INFO("LDSP", ldsp);
+						INFO("FDSP", fdsp);
+
+						const projectId = await schema2pid(this.prev._schema);
+
+						if (projectId) {
+							await event.post(projectId, ldsp);
+							await event.post(projectId, fdsp);
+						} else {
+							ERROR("projectId not found for", this.prev._schema);
+						}
+					} else {
+						WARNING("False positive on these timestamps", this.prev.time, cur.time);
+						WARNING("No events were created");
+					}
+				}
+
+			}
+		} catch (err) {
+			DEBUG(`${this.author} error`, err);
+			throw err;
+		} finally {
+			this.prev = data;
+		}
 	}
 }
 

lib/www/server/events/handlers/detect-project-configuration-change.js

@@ -0,0 +1,60 @@
+const project = require('../../lib/db/project');
+const { ALERT, ERROR, WARNING, NOTICE, INFO, DEBUG } = require('DOUGAL_ROOT/debug')(__filename);
+
+class DetectProjectConfigurationChange {
+
+	author = `*${this.constructor.name}*`;
+
+	constructor (ctx) {
+		DEBUG(`${this.author} instantiated`);
+
+		// Grab project configurations.
+		// NOTE that this will run asynchronously
+		this.run({channel: "project"}, ctx);
+	}
+
+	async run (data, ctx) {
+
+		if (!data || data.channel !== "project") {
+			return;
+		}
+
+		// Project notifications, as of this writing, most likely
+		// do not carry payloads as those exceed the notification
+		// size limit.
+		// For our purposes, we do not care as we just re-read all
+		// the configurations for all non-archived projects.
+
+		try {
+			DEBUG("Project configuration change detected")
+
+			const projects = await project.get();
+
+			const _ctx_data = {};
+			for (let pid of projects.map(i => i.pid)) {
+				DEBUG("Retrieving configuration for", pid);
+				const cfg = await project.configuration.get(pid);
+				if (cfg?.archived === true) {
+					DEBUG(pid, "is archived. Ignoring");
+					continue;
+				}
+
+				DEBUG("Saving configuration for", pid);
+				_ctx_data[pid] = cfg;
+			}
+
+			if (! ("projects" in ctx)) {
+				ctx.projects = {};
+			}
+
+			ctx.projects.configuration = _ctx_data;
+			DEBUG("Committed project configuration to ctx.projects.configuration");
+
+		} catch (err) {
+			DEBUG(`${this.author} error`, err);
+			throw err;
+		}
+	}
+}
+
+module.exports = DetectProjectConfigurationChange;