Add map layer importer

bin/import_map_layers.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+
+"""
+Import SmartSource data.
+
+For each survey in configuration.surveys(), check for new
+or modified final gun header files and (re-)import them into the
+database.
+"""
+
+import os
+import sys
+import pathlib
+import re
+import time
+import json
+import configuration
+from datastore import Datastore
+
+if __name__ == '__main__':
+
+	print("Reading configuration")
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
+
+	print("Connecting to database")
+	db = Datastore()
+	surveys = db.surveys()
+
+	print("Reading surveys")
+	for survey in surveys:
+		print(f'Survey: {survey["id"]} ({survey["schema"]})')
+
+		db.set_survey(survey["schema"])
+
+		try:
+			map_layers = survey["imports"]["map"]["layers"]
+		except KeyError:
+			print("No map layers defined")
+			continue
+
+		for layer_name, layer_items in map_layers.items():
+
+			for layer in layer_items:
+				fileprefix = layer["path"]
+				realprefix = configuration.translate_path(fileprefix)
+
+				for globspec in layer["globs"]:
+					for physical_filepath in pathlib.Path(realprefix).glob(globspec):
+						physical_filepath = str(physical_filepath)
+						logical_filepath = configuration.untranslate_path(physical_filepath)
+						print(f"Found {logical_filepath}")
+
+						if not db.file_in_db(logical_filepath):
+
+							age = time.time() - os.path.getmtime(physical_filepath)
+							if age < file_min_age:
+								print("Skipping file because too new", logical_filepath)
+								continue
+
+							print("Importing")
+
+							file_info = {
+								"type": "map_layer",
+								"format": layer["format"],
+								"name": layer_name,
+								"data": pathlib.Path(physical_filepath).read_text()
+							}
+
+							db.save_file_data(logical_filepath, json.dumps(file_info))
+
+						else:
+							print("Already in DB")
+
+	print("Done")