diff --git a/bin/import_final_p111.py b/bin/import_final_p111.py index ca6b3df..a06c365 100755 --- a/bin/import_final_p111.py +++ b/bin/import_final_p111.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p111 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -49,6 +51,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_final_p190.py b/bin/import_final_p190.py index e8f948d..278ca31 100755 --- a/bin/import_final_p190.py +++ b/bin/import_final_p190.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p190 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -49,6 +51,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_preplots.py b/bin/import_preplots.py index 5fb4887..fda9883 100755 --- a/bin/import_preplots.py +++ b/bin/import_preplots.py @@ -8,7 +8,9 @@ or modified preplots and (re-)import them into the database. """ from glob import glob +import os import sys +import time import configuration import preplots from datastore import Datastore @@ -17,6 +19,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -28,6 +31,12 @@ if __name__ == '__main__': for file in survey["preplots"]: print(f"Preplot: {file['path']}") if not db.file_in_db(file["path"]): + + age = time.time() - os.path.getmtime(file["path"]) + if age < file_min_age: + print("Skipping file because too new", file["path"]) + continue + print("Importing") try: preplot = preplots.from_file(file) diff --git a/bin/import_raw_p111.py b/bin/import_raw_p111.py index e8e781c..8f15c1a 100755 --- a/bin/import_raw_p111.py +++ b/bin/import_raw_p111.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p111 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -57,6 +59,12 @@ if __name__ == '__main__': ntbp = False if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_raw_p190.py b/bin/import_raw_p190.py index 4d48c4e..141e5ee 100755 --- a/bin/import_raw_p190.py +++ b/bin/import_raw_p190.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import p190 from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -52,6 +54,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/bin/import_smsrc.py b/bin/import_smsrc.py index f634f6e..a6b7ba8 100755 --- a/bin/import_smsrc.py +++ b/bin/import_smsrc.py @@ -12,6 +12,7 @@ import os import sys import pathlib import re +import time import configuration import smsrc from datastore import Datastore @@ -20,6 +21,7 @@ if __name__ == '__main__': print("Reading configuration") surveys = configuration.surveys() + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() @@ -53,6 +55,12 @@ if __name__ == '__main__': print(f"Found {filepath}") if not db.file_in_db(filepath): + + age = time.time() - os.path.getmtime(filepath) + if age < file_min_age: + print("Skipping file because too new", filepath) + continue + print("Importing") match = rx.match(os.path.basename(filepath)) diff --git a/etc/config.yaml b/etc/config.yaml index e618201..f943b31 100644 --- a/etc/config.yaml +++ b/etc/config.yaml @@ -22,3 +22,9 @@ navigation: # saving routine. epsg: 23031 # Assume this CRS for unqualified E/N data + +imports: + # For a file to be imported, it must have been last modified at + # least this many seconds ago. + file_min_age: 60 +