dougal-software/bin/import_smsrc.py

#!/usr/bin/python3

"""
Import SmartSource data.

For each survey in configuration.surveys(), check for new
or modified final gun header files and (re-)import them into the
database.
"""

import os
import sys
import pathlib
import re
import time
import configuration
import smsrc
import fwr
from datastore import Datastore

if __name__ == '__main__':

	print("Reading configuration")
	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)

	print("Connecting to database")
	db = Datastore()
	surveys = db.surveys()

	print("Reading surveys")
	for survey in surveys:
		print(f'Survey: {survey["id"]} ({survey["schema"]})')

		db.set_survey(survey["schema"])

		try:
			raw_smsrc = survey["raw"]["source"]["smsrc"]["header"]
		except KeyError:
			print("No SmartSource data configuration")
			continue

		# NOTE I've no idea what this is 🤔
		# flags = 0
		# if "flags" in raw_smsrc:
			# configuration.rxflags(raw_smsrc["flags"])

		lineNameInfo = raw_smsrc.get("lineNameInfo")
		pattern = raw_smsrc.get("pattern")
		rx = None
		if pattern and pattern.get("regex"):
			rx = re.compile(pattern["regex"])

		for fileprefix in raw_smsrc["paths"]:
			realprefix = configuration.translate_path(fileprefix)
			print(f"Path prefix: {fileprefix} → {realprefix}")

			for globspec in raw_smsrc["globs"]:
				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
					physical_filepath = str(physical_filepath)
					logical_filepath = configuration.untranslate_path(physical_filepath)
					print(f"Found {logical_filepath}")

					if not db.file_in_db(logical_filepath):

						age = time.time() - os.path.getmtime(physical_filepath)
						if age < file_min_age:
							print("Skipping file because too new", logical_filepath)
							continue

						print("Importing")

						if rx:
							match = rx.match(os.path.basename(logical_filepath))
							if not match:
								error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
								print(error_message, file=sys.stderr)
								print("This file will be ignored!")
								continue

							file_info = dict(zip(pattern["captures"], match.groups()))
							file_info["meta"] = {}


						if lineNameInfo:
							basename = os.path.basename(physical_filepath)
							fields = lineNameInfo.get("fields", {})
							fixed = lineNameInfo.get("fixed")
							try:
								parsed_line = fwr.parse_line(basename, fields, fixed)
							except ValueError as err:
								parsed_line = "Line format error: " + str(err)
							if type(parsed_line) == str:
								print(parsed_line, file=sys.stderr)
								print("This file will be ignored!")
								continue

							file_info = {}
							file_info["sequence"] = parsed_line["sequence"]
							file_info["line"] = parsed_line["line"]
							del(parsed_line["sequence"])
							del(parsed_line["line"])
							file_info["meta"] = {
								"fileInfo": parsed_line
							}

						smsrc_records = smsrc.from_file(physical_filepath)

						print("Saving")

						db.save_raw_smsrc(smsrc_records, file_info, logical_filepath)
					else:
						print("Already in DB")

	print("Done")