Adapt import functions to use logical paths

bin/import_final_p111.py

@@ -76,29 +76,31 @@ if __name__ == '__main__':
 			pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])
 
 		for fileprefix in final_p111["paths"]:
-			print(f"Path prefix: {fileprefix}")
+			realprefix = configuration.translate_path(fileprefix)
+			print(f"Path prefix: {fileprefix} → {realprefix}")
 
 			for globspec in final_p111["globs"]:
-				for filepath in pathlib.Path(fileprefix).glob(globspec):
+				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
-					filepath = str(filepath)
+					physical_filepath = str(physical_filepath)
-					print(f"Found {filepath}")
+					logical_filepath = configuration.untranslate_path(physical_filepath)
+					print(f"Found {logical_filepath}")
 
 					pending = False
 					if pendingRx:
-						pending = pendingRx.search(filepath) is not None
+						pending = pendingRx.search(physical_filepath) is not None
 
-					if not db.file_in_db(filepath):
+					if not db.file_in_db(logical_filepath):
 
-						age = time.time() - os.path.getmtime(filepath)
+						age = time.time() - os.path.getmtime(physical_filepath)
 						if age < file_min_age:
-							print("Skipping file because too new", filepath)
+							print("Skipping file because too new", logical_filepath)
 							continue
 
 						print("Importing")
 
-						match = rx.match(os.path.basename(filepath))
+						match = rx.match(os.path.basename(logical_filepath))
 						if not match:
-							error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})"
+							error_message = f"File path not match the expected format! ({logical_filepath} ~ {pattern['regex']})"
 							print(error_message, file=sys.stderr)
 							print("This file will be ignored!")
 							continue
@@ -107,21 +109,21 @@ if __name__ == '__main__':
 						file_info["meta"] = {}
 
 						if pending:
-							print("Skipping / removing final file because marked as PENDING", filepath)
+							print("Skipping / removing final file because marked as PENDING", logical_filepath)
 							db.del_sequence_final(file_info["sequence"])
 							add_pending_remark(db, file_info["sequence"])
 							continue
 						else:
 							del_pending_remark(db, file_info["sequence"])
 
-						p111_data = p111.from_file(filepath)
+						p111_data = p111.from_file(physical_filepath)
 
 						print("Saving")
 
 						p111_records = p111.p111_type("S", p111_data)
 						file_info["meta"]["lineName"] = p111.line_name(p111_data)
 
-						db.save_final_p111(p111_records, file_info, filepath, survey["epsg"])
+						db.save_final_p111(p111_records, file_info, logical_filepath, survey["epsg"])
 					else:
 						print("Already in DB")
 						if pending:

bin/import_preplots.py

@@ -29,17 +29,19 @@ if __name__ == '__main__':
 		print(f'Survey: {survey["id"]} ({survey["schema"]})')
 		db.set_survey(survey["schema"])
 		for file in survey["preplots"]:
+			realpath = configuration.translate_path(file["path"])
+
 			print(f"Preplot: {file['path']}")
 			if not db.file_in_db(file["path"]):
 
-				age = time.time() - os.path.getmtime(file["path"])
+				age = time.time() - os.path.getmtime(realpath)
 				if age < file_min_age:
 					print("Skipping file because too new", file["path"])
 					continue
 
 				print("Importing")
 				try:
-					preplot = preplots.from_file(file)
+					preplot = preplots.from_file(file, realpath)
 				except FileNotFoundError:
 					print(f"File does not exist: {file['path']}", file=sys.stderr)
 					continue

bin/import_raw_p111.py

@@ -45,30 +45,32 @@ if __name__ == '__main__':
 			ntbpRx = re.compile(survey["raw"]["ntbp"]["pattern"]["regex"])
 
 		for fileprefix in raw_p111["paths"]:
-			print(f"Path prefix: {fileprefix}")
+			realprefix = configuration.translate_path(fileprefix)
+			print(f"Path prefix: {fileprefix} → {realprefix}")
 
 			for globspec in raw_p111["globs"]:
-				for filepath in pathlib.Path(fileprefix).glob(globspec):
+				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
-					filepath = str(filepath)
+					physical_filepath = str(physical_filepath)
-					print(f"Found {filepath}")
+					logical_filepath = configuration.untranslate_path(physical_filepath)
+					print(f"Found {logical_filepath}")
 
 					if ntbpRx:
-						ntbp = ntbpRx.search(filepath) is not None
+						ntbp = ntbpRx.search(physical_filepath) is not None
 					else:
 						ntbp = False
 
-					if not db.file_in_db(filepath):
+					if not db.file_in_db(logical_filepath):
 
-						age = time.time() - os.path.getmtime(filepath)
+						age = time.time() - os.path.getmtime(physical_filepath)
 						if age < file_min_age:
-							print("Skipping file because too new", filepath)
+							print("Skipping file because too new", logical_filepath)
 							continue
 
 						print("Importing")
 
-						match = rx.match(os.path.basename(filepath))
+						match = rx.match(os.path.basename(logical_filepath))
 						if not match:
-							error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})"
+							error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
 							print(error_message, file=sys.stderr)
 							print("This file will be ignored!")
 							continue
@@ -76,7 +78,7 @@ if __name__ == '__main__':
 						file_info = dict(zip(pattern["captures"], match.groups()))
 						file_info["meta"] = {}
 
-						p111_data = p111.from_file(filepath)
+						p111_data = p111.from_file(physical_filepath)
 
 						print("Saving")
 
@@ -84,7 +86,7 @@ if __name__ == '__main__':
 						if len(p111_records):
 							file_info["meta"]["lineName"] = p111.line_name(p111_data)
 
-							db.save_raw_p111(p111_records, file_info, filepath, survey["epsg"], ntbp=ntbp)
+							db.save_raw_p111(p111_records, file_info, logical_filepath, survey["epsg"], ntbp=ntbp)
 						else:
 							print("No source records found in file")
 					else:
@@ -92,7 +94,7 @@ if __name__ == '__main__':
 
 					# Update the NTBP status to whatever the latest is,
 					# as it might have changed.
-					db.set_ntbp(filepath, ntbp)
+					db.set_ntbp(logical_filepath, ntbp)
 					if ntbp:
 						print("Sequence is NTBP")
 

bin/import_smsrc.py

@@ -46,36 +46,38 @@ if __name__ == '__main__':
 		rx = re.compile(pattern["regex"], flags)
 
 		for fileprefix in raw_smsrc["paths"]:
-			print(f"Path prefix: {fileprefix}")
+			realprefix = configuration.translate_path(fileprefix)
+			print(f"Path prefix: {fileprefix} → {realprefix}")
 
 			for globspec in raw_smsrc["globs"]:
-				for filepath in pathlib.Path(fileprefix).glob(globspec):
+				for physical_filepath in pathlib.Path(realprefix).glob(globspec):
-					filepath = str(filepath)
+					physical_filepath = str(physical_filepath)
-					print(f"Found {filepath}")
+					logical_filepath = configuration.untranslate_path(physical_filepath)
+					print(f"Found {logical_filepath}")
 
-					if not db.file_in_db(filepath):
+					if not db.file_in_db(logical_filepath):
 
-						age = time.time() - os.path.getmtime(filepath)
+						age = time.time() - os.path.getmtime(physical_filepath)
 						if age < file_min_age:
-							print("Skipping file because too new", filepath)
+							print("Skipping file because too new", logical_filepath)
 							continue
 
 						print("Importing")
 
-						match = rx.match(os.path.basename(filepath))
+						match = rx.match(os.path.basename(logical_filepath))
 						if not match:
-							error_message = f"File path not matching the expected format! ({filepath} ~ {pattern['regex']})"
+							error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
 							print(error_message, file=sys.stderr)
 							print("This file will be ignored!")
 							continue
 
 						file_info = dict(zip(pattern["captures"], match.groups()))
 
-						smsrc_records = smsrc.from_file(filepath)
+						smsrc_records = smsrc.from_file(physical_filepath)
 
 						print("Saving")
 
-						db.save_raw_smsrc(smsrc_records, file_info, filepath)
+						db.save_raw_smsrc(smsrc_records, file_info, logical_filepath)
 					else:
 						print("Already in DB")
 

bin/preplots.py

@@ -4,9 +4,10 @@ import sps
 Preplot importing functions.
 """
 
-def from_file (file):
+def from_file (file, realpath = None):
+	filepath = realpath or file["path"]
 	if not "type" in file or file["type"] == "sps":
-		records = sps.from_file(file["path"], file["format"] if "format" in file else None )
+		records = sps.from_file(filepath, file["format"] if "format" in file else None )
 	else:
 		return "Not an SPS file"
 

bin/purge_deleted_files.py

@@ -25,9 +25,15 @@ if __name__ == '__main__':
 		db.set_survey(survey["schema"])
 
 		for file in db.list_files():
-			path = file[0]
+			try:
-			if not os.path.exists(path):
+				path = configuration.translate_path(file[0])
-				print(path, "NOT FOUND")
+				if not os.path.exists(path):
-				db.del_file(path)
+					print(path, "NOT FOUND")
+					db.del_file(file[0])
+			except TypeError:
+				# In case the logical path no longer matches
+				# the Dougal configuration.
+				print(file[0], "COULD NOT BE TRANSLATED TO A PHYSICAL PATH. DELETING")
+				db.del_file(file[0])
 
 	print("Done")