From 0fdb42c5936e929f72f2cc351283c4624399a4d3 Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Fri, 7 May 2021 13:50:32 +0200 Subject: [PATCH] Do not import files that have just been modified. We now check that a file is at least a few seconds old before attempting to import it. The actual minimum age can be configured in etc/config.yaml or else is defaults to 10 seconds. The idea is that this should give the OS enough time to fully write the file before we import it. The timestamp being looked at is the modification time. Fixes #92. --- bin/import_final_p111.py | 8 ++++++++ bin/import_final_p190.py | 8 ++++++++ bin/import_preplots.py | 9 +++++++++ bin/import_raw_p111.py | 8 ++++++++ bin/import_raw_p190.py | 8 ++++++++ bin/import_smsrc.py | 8 ++++++++ etc/config.yaml | 6 ++++++ 7 files changed, 55 insertions(+) diff --git a/bin/import_final_p111.py b/bin/import_final_p111.py index ca6b3df..a06c365 100755 --- a/bin/import_final_p111.py +++ b/bin/import_final_p111.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p111 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -49,6 +51,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_final_p190.py b/bin/import_final_p190.py index e8f948d..278ca31 100755 --- a/bin/import_final_p190.py +++ b/bin/import_final_p190.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p190 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -49,6 +51,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_preplots.py b/bin/import_preplots.py index 5fb4887..fda9883 100755 --- a/bin/import_preplots.py +++ b/bin/import_preplots.py @@ -8,7 +8,9 @@ or modified preplots and (re-)import them into the database. """ from glob import glob +import os import sys +import time import configuration import preplots from datastore import Datastore @@ -17,6 +19,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -28,6 +31,12 @@ if __name__ == '__main__': for file in survey["preplots"]: print(f"Preplot: {file['path']}") if not db.file_in_db(file["path"]): + + age = time.time() - os.path.getmtime(file["path"]) + if age < file_min_age: + print("Skipping file because too new", file["path"]) + continue + print("Importing") try: preplot = preplots.from_file(file) diff --git a/bin/import_raw_p111.py b/bin/import_raw_p111.py index e8e781c..8f15c1a 100755 --- a/bin/import_raw_p111.py +++ b/bin/import_raw_p111.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p111 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -57,6 +59,12 @@ if __name__ == '__main__': ntbp = False if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_raw_p190.py b/bin/import_raw_p190.py index 4d48c4e..141e5ee 100755 --- a/bin/import_raw_p190.py +++ b/bin/import_raw_p190.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p190 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -52,6 +54,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_smsrc.py b/bin/import_smsrc.py index f634f6e..a6b7ba8 100755 --- a/bin/import_smsrc.py +++ b/bin/import_smsrc.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import smsrc from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -53,6 +55,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/etc/config.yaml b/etc/config.yaml index e618201..f943b31 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -22,3 +22,9 @@ navigation: # saving routine. epsg: 23031 # Assume this CRS for unqualified E/N data + +imports: + # For a file to be imported, it must have been last modified at + # least this many seconds ago. + file_min_age: 60 +