Do not import files that have just been modified.

We now check that a file is at least a few seconds old
before attempting to import it.

The actual minimum age can be configured in etc/config.yaml or
else is defaults to 10 seconds.

The idea is that this should give the OS enough time to fully
write the file before we import it.

The timestamp being looked at is the modification time.

Fixes #92.

bin/import_final_p111.py

@@ -12,6 +12,7 @@ import os
 import sys
 import pathlib
 import re
+import time
 import configuration
 import p111
 from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
 
 	print("Reading configuration")
 	surveys = configuration.surveys()
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
@@ -49,6 +51,12 @@ if __name__ == '__main__':
 					print(f"Found {filepath}")
 
 					if not db.file_in_db(filepath):
+						
+						age = time.time() - os.path.getmtime(filepath)
+						if age < file_min_age:
+							print("Skipping file because too new", filepath)
+							continue
+						
 						print("Importing")
 
 						match = rx.match(os.path.basename(filepath))

bin/import_final_p190.py

@@ -12,6 +12,7 @@ import os
 import sys
 import pathlib
 import re
+import time
 import configuration
 import p190
 from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
 
 	print("Reading configuration")
 	surveys = configuration.surveys()
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
@@ -49,6 +51,12 @@ if __name__ == '__main__':
 					print(f"Found {filepath}")
 
 					if not db.file_in_db(filepath):
+						
+						age = time.time() - os.path.getmtime(filepath)
+						if age < file_min_age:
+							print("Skipping file because too new", filepath)
+							continue
+						
 						print("Importing")
 
 						match = rx.match(os.path.basename(filepath))

bin/import_preplots.py

@@ -8,7 +8,9 @@ or modified preplots and (re-)import them into the database.
 """
 
 from glob import glob
+import os
 import sys
+import time
 import configuration
 import preplots
 from datastore import Datastore
@@ -17,6 +19,7 @@ if __name__ == '__main__':
 
 	print("Reading configuration")
 	surveys = configuration.surveys()
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
@@ -28,6 +31,12 @@ if __name__ == '__main__':
 		for file in survey["preplots"]:
 			print(f"Preplot: {file['path']}")
 			if not db.file_in_db(file["path"]):
+				
+				age = time.time() - os.path.getmtime(file["path"])
+				if age < file_min_age:
+					print("Skipping file because too new", file["path"])
+					continue
+				
 				print("Importing")
 				try:
 					preplot = preplots.from_file(file)

bin/import_raw_p111.py

@@ -12,6 +12,7 @@ import os
 import sys
 import pathlib
 import re
+import time
 import configuration
 import p111
 from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
 
 	print("Reading configuration")
 	surveys = configuration.surveys()
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
@@ -57,6 +59,12 @@ if __name__ == '__main__':
 						ntbp = False
 
 					if not db.file_in_db(filepath):
+						
+						age = time.time() - os.path.getmtime(filepath)
+						if age < file_min_age:
+							print("Skipping file because too new", filepath)
+							continue
+						
 						print("Importing")
 
 						match = rx.match(os.path.basename(filepath))

bin/import_raw_p190.py

@@ -12,6 +12,7 @@ import os
 import sys
 import pathlib
 import re
+import time
 import configuration
 import p190
 from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
 
 	print("Reading configuration")
 	surveys = configuration.surveys()
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
@@ -52,6 +54,12 @@ if __name__ == '__main__':
 					print(f"Found {filepath}")
 
 					if not db.file_in_db(filepath):
+						
+						age = time.time() - os.path.getmtime(filepath)
+						if age < file_min_age:
+							print("Skipping file because too new", filepath)
+							continue
+						
 						print("Importing")
 
 						match = rx.match(os.path.basename(filepath))

bin/import_smsrc.py

@@ -12,6 +12,7 @@ import os
 import sys
 import pathlib
 import re
+import time
 import configuration
 import smsrc
 from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
 
 	print("Reading configuration")
 	surveys = configuration.surveys()
+	file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
 
 	print("Connecting to database")
 	db = Datastore()
@@ -53,6 +55,12 @@ if __name__ == '__main__':
 					print(f"Found {filepath}")
 
 					if not db.file_in_db(filepath):
+						
+						age = time.time() - os.path.getmtime(filepath)
+						if age < file_min_age:
+							print("Skipping file because too new", filepath)
+							continue
+						
 						print("Importing")
 
 						match = rx.match(os.path.basename(filepath))

etc/config.yaml

@@ -22,3 +22,9 @@ navigation:
                 # saving routine.
                 epsg: 23031 # Assume this CRS for unqualified E/N data
 
+
+imports:
+    # For a file to be imported, it must have been last modified at
+    # least this many seconds ago.
+    file_min_age: 60
+