dougal-software/bin/import_final_p111.py

#!/usr/bin/python3

"""
Import final p111.

For each survey in configuration.surveys(), check for new
or modified final P1/11 files and (re-)import them into the
database.
"""

import os
import sys
import pathlib
import re
import time
import configuration
import p111
import fwr
from datastore import Datastore

def add_pending_remark(db, sequence):
	text = '<!-- @@DGL:PENDING@@ --><h4 style="color:red;cursor:help;" title="Edit the sequence file or directory name to import final data">Marked as <code>PENDING</code>.</h4><!-- @@/DGL:PENDING@@ -->\n'

	with db.conn.cursor() as cursor:
		qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
		cursor.execute(qry, (sequence,))
		remarks = cursor.fetchone()[0]
		rx = re.compile("^(<!-- @@DGL:PENDING@@ -->.*<!-- @@/DGL:PENDING@@ -->\n)")
		m = rx.match(remarks)
		if m is None:
			remarks = text + remarks
			qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
			cursor.execute(qry, (remarks, sequence))
			db.maybe_commit()

def del_pending_remark(db, sequence):

	with db.conn.cursor() as cursor:
		qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
		cursor.execute(qry, (sequence,))
		row = cursor.fetchone()
		if row is not None:
			remarks = row[0]
			rx = re.compile("^(<!-- @@DGL:PENDING@@ -->.*<!-- @@/DGL:PENDING@@ -->\n)")
			m = rx.match(remarks)
			if m is not None:
				remarks = rx.sub("",remarks)
				qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
				cursor.execute(qry, (remarks, sequence))
				db.maybe_commit()

if __name__ == '__main__':

	print("Reading configuration")
	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)

	print("Connecting to database")
	db = Datastore()
	surveys = db.surveys()

	print("Reading surveys")
	for survey in surveys:
		print(f'Survey: {survey["id"]} ({survey["schema"]})')

		db.set_survey(survey["schema"])

		try:
			final_p111 = survey["final"]["p111"]
		except KeyError:
			print("No final P1/11 configuration")
			exit(0)


		lineNameInfo = final_p111.get("lineNameInfo")
		pattern = final_p111.get("pattern")
		if not lineNameInfo:
			if not pattern:
				print("ERROR! Missing final.p111.lineNameInfo in project configuration. Cannot import final P111")
				raise Exception("Missing final.p111.lineNameInfo")
			else:
				print("WARNING! No `lineNameInfo` in project configuration (final.p111). You should add it to the settings.")
		rx = None
		if pattern and pattern.get("regex"):
			rx = re.compile(pattern["regex"])

		if "pending" in survey["final"]:
			pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])

		for fileprefix in final_p111["paths"]:
			realprefix = configuration.translate_path(fileprefix)
			print(f"Path prefix: {fileprefix} → {realprefix}")

			for globspec in final_p111["globs"]:
				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
					physical_filepath = str(physical_filepath)
					logical_filepath = configuration.untranslate_path(physical_filepath)
					print(f"Found {logical_filepath}")

					pending = False
					if pendingRx:
						pending = pendingRx.search(physical_filepath) is not None

					if not db.file_in_db(logical_filepath):

						age = time.time() - os.path.getmtime(physical_filepath)
						if age < file_min_age:
							print("Skipping file because too new", logical_filepath)
							continue

						print("Importing")

						if rx:
							match = rx.match(os.path.basename(logical_filepath))
							if not match:
								error_message = f"File path not match the expected format! ({logical_filepath} ~ {pattern['regex']})"
								print(error_message, file=sys.stderr)
								print("This file will be ignored!")
								continue

							file_info = dict(zip(pattern["captures"], match.groups()))
							file_info["meta"] = {}

							if lineNameInfo:
								basename = os.path.basename(physical_filepath)
								fields = lineNameInfo.get("fields", {})
								fixed = lineNameInfo.get("fixed")
								try:
									parsed_line = fwr.parse_line(basename, fields, fixed)
								except ValueError as err:
									parsed_line = "Line format error: " + str(err)
								if type(parsed_line) == str:
									print(parsed_line, file=sys.stderr)
									print("This file will be ignored!")
									continue

								file_info = {}
								file_info["sequence"] = parsed_line["sequence"]
								file_info["line"] = parsed_line["line"]
								del(parsed_line["sequence"])
								del(parsed_line["line"])
								file_info["meta"] = {
									"fileInfo": parsed_line
								}

						if pending:
							print("Skipping / removing final file because marked as PENDING", logical_filepath)
							db.del_sequence_final(file_info["sequence"])
							add_pending_remark(db, file_info["sequence"])
							continue
						else:
							del_pending_remark(db, file_info["sequence"])

						p111_data = p111.from_file(physical_filepath)

						print("Saving")

						p111_records = p111.p111_type("S", p111_data)
						file_info["meta"]["lineName"] = p111.line_name(p111_data)

						db.save_final_p111(p111_records, file_info, logical_filepath, survey["epsg"])
					else:
						print("Already in DB")
						if pending:
							print("Removing from database because marked as PENDING")
							db.del_sequence_final(file_info["sequence"])
							add_pending_remark(db, file_info["sequence"])

	print("Done")