diff --git a/bin/import_final_p111.py b/bin/import_final_p111.py index cac0fe5..bc0c146 100755 --- a/bin/import_final_p111.py +++ b/bin/import_final_p111.py @@ -76,29 +76,31 @@ if __name__ == '__main__': pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"]) for fileprefix in final_p111["paths"]: - print(f"Path prefix: {fileprefix}") + realprefix = configuration.translate_path(fileprefix) + print(f"Path prefix: {fileprefix} → {realprefix}") for globspec in final_p111["globs"]: - for filepath in pathlib.Path(fileprefix).glob(globspec): - filepath = str(filepath) - print(f"Found {filepath}") + for physical_filepath in pathlib.Path(realprefix).glob(globspec): + physical_filepath = str(physical_filepath) + logical_filepath = configuration.untranslate_path(physical_filepath) + print(f"Found {logical_filepath}") pending = False if pendingRx: - pending = pendingRx.search(filepath) is not None + pending = pendingRx.search(physical_filepath) is not None - if not db.file_in_db(filepath): + if not db.file_in_db(logical_filepath): - age = time.time() - os.path.getmtime(filepath) + age = time.time() - os.path.getmtime(physical_filepath) if age < file_min_age: - print("Skipping file because too new", filepath) + print("Skipping file because too new", logical_filepath) continue print("Importing") - match = rx.match(os.path.basename(filepath)) + match = rx.match(os.path.basename(logical_filepath)) if not match: - error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})" + error_message = f"File path not match the expected format! ({logical_filepath} ~ {pattern['regex']})" print(error_message, file=sys.stderr) print("This file will be ignored!") continue @@ -107,21 +109,21 @@ if __name__ == '__main__': file_info["meta"] = {} if pending: - print("Skipping / removing final file because marked as PENDING", filepath) + print("Skipping / removing final file because marked as PENDING", logical_filepath) db.del_sequence_final(file_info["sequence"]) add_pending_remark(db, file_info["sequence"]) continue else: del_pending_remark(db, file_info["sequence"]) - p111_data = p111.from_file(filepath) + p111_data = p111.from_file(physical_filepath) print("Saving") p111_records = p111.p111_type("S", p111_data) file_info["meta"]["lineName"] = p111.line_name(p111_data) - db.save_final_p111(p111_records, file_info, filepath, survey["epsg"]) + db.save_final_p111(p111_records, file_info, logical_filepath, survey["epsg"]) else: print("Already in DB") if pending: diff --git a/bin/import_preplots.py b/bin/import_preplots.py index 5a1af2b..10c5c1b 100755 --- a/bin/import_preplots.py +++ b/bin/import_preplots.py @@ -29,17 +29,19 @@ if __name__ == '__main__': print(f'Survey: {survey["id"]} ({survey["schema"]})') db.set_survey(survey["schema"]) for file in survey["preplots"]: + realpath = configuration.translate_path(file["path"]) + print(f"Preplot: {file['path']}") if not db.file_in_db(file["path"]): - age = time.time() - os.path.getmtime(file["path"]) + age = time.time() - os.path.getmtime(realpath) if age < file_min_age: print("Skipping file because too new", file["path"]) continue print("Importing") try: - preplot = preplots.from_file(file) + preplot = preplots.from_file(file, realpath) except FileNotFoundError: print(f"File does not exist: {file['path']}", file=sys.stderr) continue diff --git a/bin/import_raw_p111.py b/bin/import_raw_p111.py index d528e6a..c6a38ab 100755 --- a/bin/import_raw_p111.py +++ b/bin/import_raw_p111.py @@ -45,30 +45,32 @@ if __name__ == '__main__': ntbpRx = re.compile(survey["raw"]["ntbp"]["pattern"]["regex"]) for fileprefix in raw_p111["paths"]: - print(f"Path prefix: {fileprefix}") + realprefix = configuration.translate_path(fileprefix) + print(f"Path prefix: {fileprefix} → {realprefix}") for globspec in raw_p111["globs"]: - for filepath in pathlib.Path(fileprefix).glob(globspec): - filepath = str(filepath) - print(f"Found {filepath}") + for physical_filepath in pathlib.Path(realprefix).glob(globspec): + physical_filepath = str(physical_filepath) + logical_filepath = configuration.untranslate_path(physical_filepath) + print(f"Found {logical_filepath}") if ntbpRx: - ntbp = ntbpRx.search(filepath) is not None + ntbp = ntbpRx.search(physical_filepath) is not None else: ntbp = False - if not db.file_in_db(filepath): + if not db.file_in_db(logical_filepath): - age = time.time() - os.path.getmtime(filepath) + age = time.time() - os.path.getmtime(physical_filepath) if age < file_min_age: - print("Skipping file because too new", filepath) + print("Skipping file because too new", logical_filepath) continue print("Importing") - match = rx.match(os.path.basename(filepath)) + match = rx.match(os.path.basename(logical_filepath)) if not match: - error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})" + error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})" print(error_message, file=sys.stderr) print("This file will be ignored!") continue @@ -76,7 +78,7 @@ if __name__ == '__main__': file_info = dict(zip(pattern["captures"], match.groups())) file_info["meta"] = {} - p111_data = p111.from_file(filepath) + p111_data = p111.from_file(physical_filepath) print("Saving") @@ -84,7 +86,7 @@ if __name__ == '__main__': if len(p111_records): file_info["meta"]["lineName"] = p111.line_name(p111_data) - db.save_raw_p111(p111_records, file_info, filepath, survey["epsg"], ntbp=ntbp) + db.save_raw_p111(p111_records, file_info, logical_filepath, survey["epsg"], ntbp=ntbp) else: print("No source records found in file") else: @@ -92,7 +94,7 @@ if __name__ == '__main__': # Update the NTBP status to whatever the latest is, # as it might have changed. - db.set_ntbp(filepath, ntbp) + db.set_ntbp(logical_filepath, ntbp) if ntbp: print("Sequence is NTBP") diff --git a/bin/import_smsrc.py b/bin/import_smsrc.py index 758e9c4..b3fd4e7 100755 --- a/bin/import_smsrc.py +++ b/bin/import_smsrc.py @@ -46,36 +46,38 @@ if __name__ == '__main__': rx = re.compile(pattern["regex"], flags) for fileprefix in raw_smsrc["paths"]: - print(f"Path prefix: {fileprefix}") + realprefix = configuration.translate_path(fileprefix) + print(f"Path prefix: {fileprefix} → {realprefix}") for globspec in raw_smsrc["globs"]: - for filepath in pathlib.Path(fileprefix).glob(globspec): - filepath = str(filepath) - print(f"Found {filepath}") + for physical_filepath in pathlib.Path(realprefix).glob(globspec): + physical_filepath = str(physical_filepath) + logical_filepath = configuration.untranslate_path(physical_filepath) + print(f"Found {logical_filepath}") - if not db.file_in_db(filepath): + if not db.file_in_db(logical_filepath): - age = time.time() - os.path.getmtime(filepath) + age = time.time() - os.path.getmtime(physical_filepath) if age < file_min_age: - print("Skipping file because too new", filepath) + print("Skipping file because too new", logical_filepath) continue print("Importing") - match = rx.match(os.path.basename(filepath)) + match = rx.match(os.path.basename(logical_filepath)) if not match: - error_message = f"File path not matching the expected format! ({filepath} ~ {pattern['regex']})" + error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})" print(error_message, file=sys.stderr) print("This file will be ignored!") continue file_info = dict(zip(pattern["captures"], match.groups())) - smsrc_records = smsrc.from_file(filepath) + smsrc_records = smsrc.from_file(physical_filepath) print("Saving") - db.save_raw_smsrc(smsrc_records, file_info, filepath) + db.save_raw_smsrc(smsrc_records, file_info, logical_filepath) else: print("Already in DB") diff --git a/bin/preplots.py b/bin/preplots.py index 51f92de..58be431 100644 --- a/bin/preplots.py +++ b/bin/preplots.py @@ -4,9 +4,10 @@ import sps Preplot importing functions. """ -def from_file (file): +def from_file (file, realpath = None): + filepath = realpath or file["path"] if not "type" in file or file["type"] == "sps": - records = sps.from_file(file["path"], file["format"] if "format" in file else None ) + records = sps.from_file(filepath, file["format"] if "format" in file else None ) else: return "Not an SPS file" diff --git a/bin/purge_deleted_files.py b/bin/purge_deleted_files.py index c7aa778..a8dd966 100755 --- a/bin/purge_deleted_files.py +++ b/bin/purge_deleted_files.py @@ -25,9 +25,15 @@ if __name__ == '__main__': db.set_survey(survey["schema"]) for file in db.list_files(): - path = file[0] - if not os.path.exists(path): - print(path, "NOT FOUND") - db.del_file(path) + try: + path = configuration.translate_path(file[0]) + if not os.path.exists(path): + print(path, "NOT FOUND") + db.del_file(file[0]) + except TypeError: + # In case the logical path no longer matches + # the Dougal configuration. + print(file[0], "COULD NOT BE TRANSLATED TO A PHYSICAL PATH. DELETING") + db.del_file(file[0]) print("Done")