mirror of
https://gitlab.com/wgp/dougal/software.git
synced 2025-12-06 08:57:08 +00:00
136 lines
4.1 KiB
Python
Executable File
136 lines
4.1 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
"""
|
|
Import final p111.
|
|
|
|
For each survey in configuration.surveys(), check for new
|
|
or modified final P1/11 files and (re-)import them into the
|
|
database.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import pathlib
|
|
import re
|
|
import time
|
|
import configuration
|
|
import p111
|
|
from datastore import Datastore
|
|
|
|
def add_pending_remark(db, sequence):
|
|
text = '<!-- @@DGL:PENDING@@ --><h4 style="color:red;cursor:help;" title="Edit the sequence file or directory name to import final data">Marked as <code>PENDING</code>.</h4><!-- @@/DGL:PENDING@@ -->\n'
|
|
|
|
with db.conn.cursor() as cursor:
|
|
qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
|
|
cursor.execute(qry, (sequence,))
|
|
remarks = cursor.fetchone()[0]
|
|
rx = re.compile("^(<!-- @@DGL:PENDING@@ -->.*<!-- @@/DGL:PENDING@@ -->\n)")
|
|
m = rx.match(remarks)
|
|
if m is None:
|
|
remarks = text + remarks
|
|
qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
|
|
cursor.execute(qry, (remarks, sequence))
|
|
db.maybe_commit()
|
|
|
|
def del_pending_remark(db, sequence):
|
|
|
|
with db.conn.cursor() as cursor:
|
|
qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
|
|
cursor.execute(qry, (sequence,))
|
|
row = cursor.fetchone()
|
|
if row is not None:
|
|
remarks = row[0]
|
|
rx = re.compile("^(<!-- @@DGL:PENDING@@ -->.*<!-- @@/DGL:PENDING@@ -->\n)")
|
|
m = rx.match(remarks)
|
|
if m is not None:
|
|
remarks = rx.sub("",remarks)
|
|
qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
|
|
cursor.execute(qry, (remarks, sequence))
|
|
db.maybe_commit()
|
|
|
|
if __name__ == '__main__':
|
|
|
|
print("Reading configuration")
|
|
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
|
|
|
|
print("Connecting to database")
|
|
db = Datastore()
|
|
surveys = db.surveys()
|
|
|
|
print("Reading surveys")
|
|
for survey in surveys:
|
|
print(f'Survey: {survey["id"]} ({survey["schema"]})')
|
|
|
|
db.set_survey(survey["schema"])
|
|
|
|
try:
|
|
final_p111 = survey["final"]["p111"]
|
|
except KeyError:
|
|
print("No final P1/11 configuration")
|
|
exit(0)
|
|
|
|
pattern = final_p111["pattern"]
|
|
rx = re.compile(pattern["regex"])
|
|
|
|
if "pending" in survey["final"]:
|
|
pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])
|
|
|
|
for fileprefix in final_p111["paths"]:
|
|
realprefix = configuration.translate_path(fileprefix)
|
|
print(f"Path prefix: {fileprefix} → {realprefix}")
|
|
|
|
for globspec in final_p111["globs"]:
|
|
for physical_filepath in pathlib.Path(realprefix).glob(globspec):
|
|
physical_filepath = str(physical_filepath)
|
|
logical_filepath = configuration.untranslate_path(physical_filepath)
|
|
print(f"Found {logical_filepath}")
|
|
|
|
pending = False
|
|
if pendingRx:
|
|
pending = pendingRx.search(physical_filepath) is not None
|
|
|
|
if not db.file_in_db(logical_filepath):
|
|
|
|
age = time.time() - os.path.getmtime(physical_filepath)
|
|
if age < file_min_age:
|
|
print("Skipping file because too new", logical_filepath)
|
|
continue
|
|
|
|
print("Importing")
|
|
|
|
match = rx.match(os.path.basename(logical_filepath))
|
|
if not match:
|
|
error_message = f"File path not match the expected format! ({logical_filepath} ~ {pattern['regex']})"
|
|
print(error_message, file=sys.stderr)
|
|
print("This file will be ignored!")
|
|
continue
|
|
|
|
file_info = dict(zip(pattern["captures"], match.groups()))
|
|
file_info["meta"] = {}
|
|
|
|
if pending:
|
|
print("Skipping / removing final file because marked as PENDING", logical_filepath)
|
|
db.del_sequence_final(file_info["sequence"])
|
|
add_pending_remark(db, file_info["sequence"])
|
|
continue
|
|
else:
|
|
del_pending_remark(db, file_info["sequence"])
|
|
|
|
p111_data = p111.from_file(physical_filepath)
|
|
|
|
print("Saving")
|
|
|
|
p111_records = p111.p111_type("S", p111_data)
|
|
file_info["meta"]["lineName"] = p111.line_name(p111_data)
|
|
|
|
db.save_final_p111(p111_records, file_info, logical_filepath, survey["epsg"])
|
|
else:
|
|
print("Already in DB")
|
|
if pending:
|
|
print("Removing from database because marked as PENDING")
|
|
db.del_sequence_final(file_info["sequence"])
|
|
add_pending_remark(db, file_info["sequence"])
|
|
|
|
print("Done")
|
|
|