dougal-software/bin/import_map_layers.py

#!/usr/bin/python3

"""
Import SmartSource data.

For each survey in configuration.surveys(), check for new
or modified final gun header files and (re-)import them into the
database.
"""

import os
import sys
import pathlib
import re
import time
import json
import configuration
from datastore import Datastore

if __name__ == '__main__':
	"""
	Imports map layers from the directories defined in the configuration object
	`import.map.layers`. The content of that key is an object with the following
	structure:

	{
		layer1Name: [
			format: "geojson",
			path: "…", // Logical path to a directory
			globs: [
				"**/*.geojson", // List of globs matching map data files
				…
			]
		],

		layer2Name: …
		…
	}
	"""


	def process (layer_name, layer, physical_filepath):
		physical_filepath = str(physical_filepath)
		logical_filepath = configuration.untranslate_path(physical_filepath)
		print(f"Found {logical_filepath}")

		if not db.file_in_db(logical_filepath):

			age = time.time() - os.path.getmtime(physical_filepath)
			if age < file_min_age:
				print("Skipping file because too new", logical_filepath)
				return

			print("Importing")

			file_info = {
				"type": "map_layer",
				"format": layer["format"],
				"name": layer_name,
				"tooltip": layer.get("tooltip"),
				"popup": layer.get("popup")
			}

			db.save_file_data(logical_filepath, json.dumps(file_info))

		else:
			file_info = db.get_file_data(logical_filepath)
			dirty = False
			if file_info:
				if file_info["name"] != layer_name:
					print("Renaming to", layer_name)
					file_info["name"] = layer_name
					dirty = True
				if file_info.get("tooltip") != layer.get("tooltip"):
					print("Changing tooltip to", layer.get("tooltip") or "null")
					file_info["tooltip"] = layer.get("tooltip")
					dirty = True
				if file_info.get("popup") != layer.get("popup"):
					print("Changing popup to", layer.get("popup") or "null")
					file_info["popup"] = layer.get("popup")
					dirty = True

			if dirty:
				db.save_file_data(logical_filepath, json.dumps(file_info))
			else:
				print("Already in DB")


	print("Reading configuration")
	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)

	print("Connecting to database")
	db = Datastore()
	surveys = db.surveys()

	print("Reading surveys")
	for survey in surveys:
		print(f'Survey: {survey["id"]} ({survey["schema"]})')

		db.set_survey(survey["schema"])

		try:
			map_layers = survey["imports"]["map"]["layers"]
		except KeyError:
			print("No map layers defined")
			continue

		for layer_name, layer_items in map_layers.items():

			for layer in layer_items:
				fileprefix = layer["path"]
				realprefix = configuration.translate_path(fileprefix)

				if os.path.isfile(realprefix):

					process(layer_name, layer, realprefix)

				elif os.path.isdir(realprefix):

					if not "globs" in layer:
						layer["globs"] = [ "**/*.geojson" ]

					for globspec in layer["globs"]:
						for physical_filepath in pathlib.Path(realprefix).glob(globspec):
							process(layer_name, layer, physical_filepath)

	print("Done")