mirror of
https://gitlab.com/wgp/dougal/software.git
synced 2025-12-06 10:27:09 +00:00
129 lines
3.5 KiB
Python
Executable File
129 lines
3.5 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
"""
|
|
Import raw p111.
|
|
|
|
For each survey in configuration.surveys(), check for new
|
|
or modified final P1/11 files and (re-)import them into the
|
|
database.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import pathlib
|
|
import re
|
|
import time
|
|
import configuration
|
|
import p111
|
|
import fwr
|
|
from datastore import Datastore
|
|
|
|
if __name__ == '__main__':
|
|
|
|
print("Reading configuration")
|
|
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
|
|
|
|
print("Connecting to database")
|
|
db = Datastore()
|
|
surveys = db.surveys()
|
|
|
|
print("Reading surveys")
|
|
for survey in surveys:
|
|
print(f'Survey: {survey["id"]} ({survey["schema"]})')
|
|
|
|
db.set_survey(survey["schema"])
|
|
|
|
try:
|
|
raw_p111 = survey["raw"]["p111"]
|
|
except KeyError:
|
|
print("No raw P1/11 configuration")
|
|
exit(0)
|
|
|
|
lineNameInfo = raw_p111.get("lineNameInfo")
|
|
pattern = raw_p111.get("pattern")
|
|
rx = None
|
|
if pattern and pattern.get("regex"):
|
|
rx = re.compile(pattern["regex"])
|
|
|
|
if "ntbp" in survey["raw"]:
|
|
ntbpRx = re.compile(survey["raw"]["ntbp"]["pattern"]["regex"])
|
|
|
|
for fileprefix in raw_p111["paths"]:
|
|
realprefix = configuration.translate_path(fileprefix)
|
|
print(f"Path prefix: {fileprefix} → {realprefix}")
|
|
|
|
for globspec in raw_p111["globs"]:
|
|
for physical_filepath in pathlib.Path(realprefix).glob(globspec):
|
|
physical_filepath = str(physical_filepath)
|
|
logical_filepath = configuration.untranslate_path(physical_filepath)
|
|
print(f"Found {logical_filepath}")
|
|
|
|
if ntbpRx:
|
|
ntbp = ntbpRx.search(physical_filepath) is not None
|
|
else:
|
|
ntbp = False
|
|
|
|
if not db.file_in_db(logical_filepath):
|
|
|
|
age = time.time() - os.path.getmtime(physical_filepath)
|
|
if age < file_min_age:
|
|
print("Skipping file because too new", logical_filepath)
|
|
continue
|
|
|
|
print("Importing")
|
|
|
|
if rx:
|
|
match = rx.match(os.path.basename(logical_filepath))
|
|
if not match:
|
|
error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
|
|
print(error_message, file=sys.stderr)
|
|
print("This file will be ignored!")
|
|
continue
|
|
|
|
file_info = dict(zip(pattern["captures"], match.groups()))
|
|
file_info["meta"] = {}
|
|
|
|
if lineNameInfo:
|
|
basename = os.path.basename(physical_filepath)
|
|
fields = lineNameInfo.get("fields", {})
|
|
fixed = lineNameInfo.get("fixed")
|
|
try:
|
|
parsed_line = fwr.parse_line(basename, fields, fixed)
|
|
except ValueError as err:
|
|
parsed_line = "Line format error: " + str(err)
|
|
if type(parsed_line) == str:
|
|
print(parsed_line, file=sys.stderr)
|
|
print("This file will be ignored!")
|
|
continue
|
|
|
|
file_info = {}
|
|
file_info["sequence"] = parsed_line["sequence"]
|
|
file_info["line"] = parsed_line["line"]
|
|
del(parsed_line["sequence"])
|
|
del(parsed_line["line"])
|
|
file_info["meta"] = {
|
|
"fileInfo": parsed_line
|
|
}
|
|
p111_data = p111.from_file(physical_filepath)
|
|
|
|
print("Saving")
|
|
|
|
p111_records = p111.p111_type("S", p111_data)
|
|
if len(p111_records):
|
|
file_info["meta"]["lineName"] = p111.line_name(p111_data)
|
|
|
|
db.save_raw_p111(p111_records, file_info, logical_filepath, survey["epsg"], ntbp=ntbp)
|
|
else:
|
|
print("No source records found in file")
|
|
else:
|
|
print("Already in DB")
|
|
|
|
# Update the NTBP status to whatever the latest is,
|
|
# as it might have changed.
|
|
db.set_ntbp(logical_filepath, ntbp)
|
|
if ntbp:
|
|
print("Sequence is NTBP")
|
|
|
|
print("Done")
|
|
|