Recognise PENDING status in sequence imports.

If a final sequence file or directory name matches a pattern
which is recognised to indicate a ‘pending acceptance’ status,
the final data (if any exists) for that sequence will be deleted
and a comment added to the effect that the sequence has been
marked as ‘pending’.

To accept the sequence, rename its final file or directory name
accordingly.

Note: it is the *final* data that is searched for a matching
pattern, not the raw.

Closes #91.

bin/datastore.py

@@ -639,3 +639,21 @@ class Datastore:
 			self.maybe_commit()
 			# We do not commit if we've been passed a cursor, instead
 			# we assume that we are in the middle of a transaction
+
+	def del_sequence_final(self, sequence, cursor = None):
+		"""
+		Remove final data for a sequence.
+		"""
+		
+		if cursor is None:
+			cur = self.conn.cursor()
+		else:
+			cur = cursor
+
+		qry = "DELETE FROM files WHERE hash = (SELECT hash FROM final_lines_files WHERE sequence = %s);"
+		cur.execute(qry, (sequence,))
+		if cursor is None:
+			self.maybe_commit()
+			# We do not commit if we've been passed a cursor, instead
+			# we assume that we are in the middle of a transaction
+		

bin/import_final_p111.py

@@ -17,6 +17,35 @@ import configuration
 import p111
 from datastore import Datastore
 
+def add_pending_remark(db, sequence):
+	text = '<!-- @@DGL:PENDING@@ --><h4 style="color:red;cursor:help;" title="Edit the sequence file or directory name to import final data">Marked as <code>PENDING</code>.</h4><!-- @@/DGL:PENDING@@ -->\n'
+	
+	with db.conn.cursor() as cursor:
+		qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
+		cursor.execute(qry, (sequence,))
+		remarks = cursor.fetchone()[0]
+		rx = re.compile("^(<!-- @@DGL:PENDING@@ -->.*<!-- @@/DGL:PENDING@@ -->\n)")
+		m = rx.match(remarks)
+		if m is None:
+			remarks = text + remarks
+			qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
+			cursor.execute(qry, (remarks, sequence))
+			db.maybe_commit()
+
+def del_pending_remark(db, sequence):
+	
+	with db.conn.cursor() as cursor:
+		qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
+		cursor.execute(qry, (sequence,))
+		remarks = cursor.fetchone()[0]
+		rx = re.compile("^(<!-- @@DGL:PENDING@@ -->.*<!-- @@/DGL:PENDING@@ -->\n)")
+		m = rx.match(remarks)
+		if m is not None:
+			remarks = rx.sub("",remarks)
+			qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
+			cursor.execute(qry, (remarks, sequence))
+			db.maybe_commit()
+
 if __name__ == '__main__':
 
 	print("Reading configuration")
@@ -42,6 +71,9 @@ if __name__ == '__main__':
 		pattern = final_p111["pattern"]
 		rx = re.compile(pattern["regex"])
 
+		if "pending" in survey["final"]:
+			pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])
+
 		for fileprefix in final_p111["paths"]:
 			print(f"Path prefix: {fileprefix}")
 
@@ -50,6 +82,10 @@ if __name__ == '__main__':
 					filepath = str(filepath)
 					print(f"Found {filepath}")
 
+					pending = False
+					if pendingRx:
+						pending = pendingRx.search(filepath) is not None
+
 					if not db.file_in_db(filepath):
 						
 						age = time.time() - os.path.getmtime(filepath)
@@ -68,6 +104,14 @@ if __name__ == '__main__':
 
 						file_info = dict(zip(pattern["captures"], match.groups()))
 						
+						if pending:
+							print("Skipping / removing final file because marked as PENDING", filepath)
+							db.del_sequence_final(file_info["sequence"])
+							add_pending_remark(db, file_info["sequence"])
+							continue
+						else:
+							del_pending_remark(db, file_info["sequence"])
+
 						p111_data = p111.from_file(filepath)
 
 						print("Saving")
@@ -77,6 +121,10 @@ if __name__ == '__main__':
 						db.save_final_p111(p111_records, file_info, filepath, survey["epsg"])
 					else:
 						print("Already in DB")
+						if pending:
+							print("Removing from database because marked as PENDING")
+							db.del_sequence_final(file_info["sequence"])
+							add_pending_remark(db, file_info["sequence"])
 
 	print("Done")
 

bin/system_imports.py

@@ -40,6 +40,10 @@ if __name__ == '__main__':
 				continue
 
 			try:
+				for table in exportables:
+					path = os.path.join(pathPrefix, table)
+					if os.path.exists(path):
+						cursor.execute(f"DELETE FROM {table};")
 				for table in exportables:
 					path = os.path.join(pathPrefix, table)
 					print(" ← ", path, " → ", table)