mirror of
https://gitlab.com/wgp/dougal/software.git
synced 2025-12-06 09:07:09 +00:00
128 lines
3.0 KiB
Python
Executable File
128 lines
3.0 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
"""
|
|
Import SmartSource data.
|
|
|
|
For each survey in configuration.surveys(), check for new
|
|
or modified final gun header files and (re-)import them into the
|
|
database.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import pathlib
|
|
import re
|
|
import time
|
|
import json
|
|
import configuration
|
|
from datastore import Datastore
|
|
|
|
if __name__ == '__main__':
|
|
"""
|
|
Imports map layers from the directories defined in the configuration object
|
|
`import.map.layers`. The content of that key is an object with the following
|
|
structure:
|
|
|
|
{
|
|
layer1Name: [
|
|
format: "geojson",
|
|
path: "…", // Logical path to a directory
|
|
globs: [
|
|
"**/*.geojson", // List of globs matching map data files
|
|
…
|
|
]
|
|
],
|
|
|
|
layer2Name: …
|
|
…
|
|
}
|
|
"""
|
|
|
|
|
|
def process (layer_name, layer, physical_filepath):
|
|
physical_filepath = str(physical_filepath)
|
|
logical_filepath = configuration.untranslate_path(physical_filepath)
|
|
print(f"Found {logical_filepath}")
|
|
|
|
if not db.file_in_db(logical_filepath):
|
|
|
|
age = time.time() - os.path.getmtime(physical_filepath)
|
|
if age < file_min_age:
|
|
print("Skipping file because too new", logical_filepath)
|
|
return
|
|
|
|
print("Importing")
|
|
|
|
file_info = {
|
|
"type": "map_layer",
|
|
"format": layer["format"],
|
|
"name": layer_name,
|
|
"tooltip": layer.get("tooltip"),
|
|
"popup": layer.get("popup")
|
|
}
|
|
|
|
db.save_file_data(logical_filepath, json.dumps(file_info))
|
|
|
|
else:
|
|
file_info = db.get_file_data(logical_filepath)
|
|
dirty = False
|
|
if file_info:
|
|
if file_info["name"] != layer_name:
|
|
print("Renaming to", layer_name)
|
|
file_info["name"] = layer_name
|
|
dirty = True
|
|
if file_info.get("tooltip") != layer.get("tooltip"):
|
|
print("Changing tooltip to", layer.get("tooltip") or "null")
|
|
file_info["tooltip"] = layer.get("tooltip")
|
|
dirty = True
|
|
if file_info.get("popup") != layer.get("popup"):
|
|
print("Changing popup to", layer.get("popup") or "null")
|
|
file_info["popup"] = layer.get("popup")
|
|
dirty = True
|
|
|
|
if dirty:
|
|
db.save_file_data(logical_filepath, json.dumps(file_info))
|
|
else:
|
|
print("Already in DB")
|
|
|
|
|
|
print("Reading configuration")
|
|
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
|
|
|
|
print("Connecting to database")
|
|
db = Datastore()
|
|
surveys = db.surveys()
|
|
|
|
print("Reading surveys")
|
|
for survey in surveys:
|
|
print(f'Survey: {survey["id"]} ({survey["schema"]})')
|
|
|
|
db.set_survey(survey["schema"])
|
|
|
|
try:
|
|
map_layers = survey["imports"]["map"]["layers"]
|
|
except KeyError:
|
|
print("No map layers defined")
|
|
continue
|
|
|
|
for layer_name, layer_items in map_layers.items():
|
|
|
|
for layer in layer_items:
|
|
fileprefix = layer["path"]
|
|
realprefix = configuration.translate_path(fileprefix)
|
|
|
|
if os.path.isfile(realprefix):
|
|
|
|
process(layer_name, layer, realprefix)
|
|
|
|
elif os.path.isdir(realprefix):
|
|
|
|
if not "globs" in layer:
|
|
layer["globs"] = [ "**/*.geojson" ]
|
|
|
|
for globspec in layer["globs"]:
|
|
for physical_filepath in pathlib.Path(realprefix).glob(globspec):
|
|
process(layer_name, layer, physical_filepath)
|
|
|
|
print("Done")
|