diff --git a/bin/datastore.py b/bin/datastore.py
index 4a9967e..3d0efb8 100644
--- a/bin/datastore.py
+++ b/bin/datastore.py
@@ -639,3 +639,21 @@ class Datastore:
self.maybe_commit()
# We do not commit if we've been passed a cursor, instead
# we assume that we are in the middle of a transaction
+
+ def del_sequence_final(self, sequence, cursor = None):
+ """
+ Remove final data for a sequence.
+ """
+
+ if cursor is None:
+ cur = self.conn.cursor()
+ else:
+ cur = cursor
+
+ qry = "DELETE FROM files WHERE hash = (SELECT hash FROM final_lines_files WHERE sequence = %s);"
+ cur.execute(qry, (sequence,))
+ if cursor is None:
+ self.maybe_commit()
+ # We do not commit if we've been passed a cursor, instead
+ # we assume that we are in the middle of a transaction
+
diff --git a/bin/import_final_p111.py b/bin/import_final_p111.py
index a06c365..9a45524 100755
--- a/bin/import_final_p111.py
+++ b/bin/import_final_p111.py
@@ -17,6 +17,35 @@ import configuration
import p111
from datastore import Datastore
+def add_pending_remark(db, sequence):
+ text = '
Marked as PENDING.
\n'
+
+ with db.conn.cursor() as cursor:
+ qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
+ cursor.execute(qry, (sequence,))
+ remarks = cursor.fetchone()[0]
+ rx = re.compile("^(.*\n)")
+ m = rx.match(remarks)
+ if m is None:
+ remarks = text + remarks
+ qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
+ cursor.execute(qry, (remarks, sequence))
+ db.maybe_commit()
+
+def del_pending_remark(db, sequence):
+
+ with db.conn.cursor() as cursor:
+ qry = "SELECT remarks FROM raw_lines WHERE sequence = %s;"
+ cursor.execute(qry, (sequence,))
+ remarks = cursor.fetchone()[0]
+ rx = re.compile("^(.*\n)")
+ m = rx.match(remarks)
+ if m is not None:
+ remarks = rx.sub("",remarks)
+ qry = "UPDATE raw_lines SET remarks = %s WHERE sequence = %s;"
+ cursor.execute(qry, (remarks, sequence))
+ db.maybe_commit()
+
if __name__ == '__main__':
print("Reading configuration")
@@ -42,6 +71,9 @@ if __name__ == '__main__':
pattern = final_p111["pattern"]
rx = re.compile(pattern["regex"])
+ if "pending" in survey["final"]:
+ pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])
+
for fileprefix in final_p111["paths"]:
print(f"Path prefix: {fileprefix}")
@@ -50,6 +82,10 @@ if __name__ == '__main__':
filepath = str(filepath)
print(f"Found {filepath}")
+ pending = False
+ if pendingRx:
+ pending = pendingRx.search(filepath) is not None
+
if not db.file_in_db(filepath):
age = time.time() - os.path.getmtime(filepath)
@@ -67,6 +103,14 @@ if __name__ == '__main__':
continue
file_info = dict(zip(pattern["captures"], match.groups()))
+
+ if pending:
+ print("Skipping / removing final file because marked as PENDING", filepath)
+ db.del_sequence_final(file_info["sequence"])
+ add_pending_remark(db, file_info["sequence"])
+ continue
+ else:
+ del_pending_remark(db, file_info["sequence"])
p111_data = p111.from_file(filepath)
@@ -77,6 +121,10 @@ if __name__ == '__main__':
db.save_final_p111(p111_records, file_info, filepath, survey["epsg"])
else:
print("Already in DB")
+ if pending:
+ print("Removing from database because marked as PENDING")
+ db.del_sequence_final(file_info["sequence"])
+ add_pending_remark(db, file_info["sequence"])
print("Done")
diff --git a/bin/system_imports.py b/bin/system_imports.py
index 2201169..03e1008 100755
--- a/bin/system_imports.py
+++ b/bin/system_imports.py
@@ -40,6 +40,10 @@ if __name__ == '__main__':
continue
try:
+ for table in exportables:
+ path = os.path.join(pathPrefix, table)
+ if os.path.exists(path):
+ cursor.execute(f"DELETE FROM {table};")
for table in exportables:
path = os.path.join(pathPrefix, table)
print(" ← ", path, " → ", table)