dougal-software/bin/import_preplots.py

#!/usr/bin/python3

"""
Import preplot data.

For each preplot file defined in each survey, check for new
or modified preplots and (re-)import them into the database.
"""

from glob import glob
import os
import sys
import time
import configuration
import preplots
from datastore import Datastore

def preplots_sorter (preplot):
	rank = {
		"x-sl+csv": 10
	}
	return rank.get(preplot.get("type"), 0)

if __name__ == '__main__':

	print("Connecting to database")
	db = Datastore()
	surveys = db.surveys()

	print("Reading configuration")
	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)

	print("Reading surveys")
	for survey in surveys:
		print(f'Survey: {survey["id"]} ({survey["schema"]})')
		db.set_survey(survey["schema"])

		# We sort the preplots so that ancillary line info always comes last,
		# after the actual line + point data has been imported
		for file in sorted(survey["preplots"], key=preplots_sorter):
			realpath = configuration.translate_path(file["path"])

			print(f"Preplot: {file['path']}")
			if not db.file_in_db(file["path"]):

				age = time.time() - os.path.getmtime(realpath)
				if age < file_min_age:
					print("Skipping file because too new", file["path"])
					continue

				print("Importing")
				try:
					preplot = preplots.from_file(file, realpath)
				except FileNotFoundError:
					print(f"File does not exist: {file['path']}", file=sys.stderr)
					continue

				if type(preplot) is list:
					print("Saving to DB")
					if file.get("type") == "x-sl+csv":
						db.save_preplot_line_info(preplot, file["path"], file)
					else:
						db.save_preplots(preplot, file["path"], file["class"], survey["epsg"], file)
				elif type(preplot) is str:
					print(preplot)
			else:
				print("Already in DB")

	print("Done")