Aaltronav/dougal-software
1
0
Fork 0
mirror of https://gitlab.com/wgp/dougal/software.git synced 2025-12-06 08:17:09 +00:00
Code Issues Packages Projects Releases Activity

Adapt import functions to use logical paths

Browse Source
This commit is contained in:
D. Berge
2023-08-30 14:56:09 +02:00
parent ccf4bbf547
commit 53f2108e37
6 changed files with 60 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
bin/import_final_p111.py
View File

@@ -76,29 +76,31 @@ if __name__ == '__main__':
pendingRx = re.compile(survey["final"]["pending"]["pattern"]["regex"])
for fileprefix in final_p111["paths"]:
print(f"Path prefix: {fileprefix}")
realprefix = configuration.translate_path(fileprefix)
print(f"Path prefix: {fileprefix} → {realprefix}")
for globspec in final_p111["globs"]:
for filepath in pathlib.Path(fileprefix).glob(globspec):
filepath = str(filepath)
print(f"Found {filepath}")
for physical_filepath in pathlib.Path(realprefix).glob(globspec):
physical_filepath = str(physical_filepath)
logical_filepath = configuration.untranslate_path(physical_filepath)
print(f"Found {logical_filepath}")
pending = False
if pendingRx:
pending = pendingRx.search(filepath) is not None
pending = pendingRx.search(physical_filepath) is not None
if not db.file_in_db(filepath):
if not db.file_in_db(logical_filepath):
age = time.time() - os.path.getmtime(filepath)
age = time.time() - os.path.getmtime(physical_filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
print("Skipping file because too new", logical_filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))
match = rx.match(os.path.basename(logical_filepath))
if not match:
error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})"
error_message = f"File path not match the expected format! ({logical_filepath} ~ {pattern['regex']})"
print(error_message, file=sys.stderr)
print("This file will be ignored!")
continue
@@ -107,21 +109,21 @@ if __name__ == '__main__':
file_info["meta"] = {}
if pending:
print("Skipping / removing final file because marked as PENDING", filepath)
print("Skipping / removing final file because marked as PENDING", logical_filepath)
db.del_sequence_final(file_info["sequence"])
add_pending_remark(db, file_info["sequence"])
continue
else:
del_pending_remark(db, file_info["sequence"])
p111_data = p111.from_file(filepath)
p111_data = p111.from_file(physical_filepath)
print("Saving")
p111_records = p111.p111_type("S", p111_data)
file_info["meta"]["lineName"] = p111.line_name(p111_data)
db.save_final_p111(p111_records, file_info, filepath, survey["epsg"])
db.save_final_p111(p111_records, file_info, logical_filepath, survey["epsg"])
else:
print("Already in DB")
if pending:

6
bin/import_preplots.py
View File

@@ -29,17 +29,19 @@ if __name__ == '__main__':
print(f'Survey: {survey["id"]} ({survey["schema"]})')
db.set_survey(survey["schema"])
for file in survey["preplots"]:
realpath = configuration.translate_path(file["path"])
print(f"Preplot: {file['path']}")
if not db.file_in_db(file["path"]):
age = time.time() - os.path.getmtime(file["path"])
age = time.time() - os.path.getmtime(realpath)
if age < file_min_age:
print("Skipping file because too new", file["path"])
continue
print("Importing")
try:
preplot = preplots.from_file(file)
preplot = preplots.from_file(file, realpath)
except FileNotFoundError:
print(f"File does not exist: {file['path']}", file=sys.stderr)
continue

28
bin/import_raw_p111.py
View File

@@ -45,30 +45,32 @@ if __name__ == '__main__':
ntbpRx = re.compile(survey["raw"]["ntbp"]["pattern"]["regex"])
for fileprefix in raw_p111["paths"]:
print(f"Path prefix: {fileprefix}")
realprefix = configuration.translate_path(fileprefix)
print(f"Path prefix: {fileprefix} → {realprefix}")
for globspec in raw_p111["globs"]:
for filepath in pathlib.Path(fileprefix).glob(globspec):
filepath = str(filepath)
print(f"Found {filepath}")
for physical_filepath in pathlib.Path(realprefix).glob(globspec):
physical_filepath = str(physical_filepath)
logical_filepath = configuration.untranslate_path(physical_filepath)
print(f"Found {logical_filepath}")
if ntbpRx:
ntbp = ntbpRx.search(filepath) is not None
ntbp = ntbpRx.search(physical_filepath) is not None
else:
ntbp = False
if not db.file_in_db(filepath):
if not db.file_in_db(logical_filepath):
age = time.time() - os.path.getmtime(filepath)
age = time.time() - os.path.getmtime(physical_filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
print("Skipping file because too new", logical_filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))
match = rx.match(os.path.basename(logical_filepath))
if not match:
error_message = f"File path not match the expected format! ({filepath} ~ {pattern['regex']})"
error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
print(error_message, file=sys.stderr)
print("This file will be ignored!")
continue
@@ -76,7 +78,7 @@ if __name__ == '__main__':
file_info = dict(zip(pattern["captures"], match.groups()))
file_info["meta"] = {}
p111_data = p111.from_file(filepath)
p111_data = p111.from_file(physical_filepath)
print("Saving")
@@ -84,7 +86,7 @@ if __name__ == '__main__':
if len(p111_records):
file_info["meta"]["lineName"] = p111.line_name(p111_data)
db.save_raw_p111(p111_records, file_info, filepath, survey["epsg"], ntbp=ntbp)
db.save_raw_p111(p111_records, file_info, logical_filepath, survey["epsg"], ntbp=ntbp)
else:
print("No source records found in file")
else:
@@ -92,7 +94,7 @@ if __name__ == '__main__':
# Update the NTBP status to whatever the latest is,
# as it might have changed.
db.set_ntbp(filepath, ntbp)
db.set_ntbp(logical_filepath, ntbp)
if ntbp:
print("Sequence is NTBP")

24
bin/import_smsrc.py
View File

@@ -46,36 +46,38 @@ if __name__ == '__main__':
rx = re.compile(pattern["regex"], flags)
for fileprefix in raw_smsrc["paths"]:
print(f"Path prefix: {fileprefix}")
realprefix = configuration.translate_path(fileprefix)
print(f"Path prefix: {fileprefix} → {realprefix}")
for globspec in raw_smsrc["globs"]:
for filepath in pathlib.Path(fileprefix).glob(globspec):
filepath = str(filepath)
print(f"Found {filepath}")
for physical_filepath in pathlib.Path(realprefix).glob(globspec):
physical_filepath = str(physical_filepath)
logical_filepath = configuration.untranslate_path(physical_filepath)
print(f"Found {logical_filepath}")
if not db.file_in_db(filepath):
if not db.file_in_db(logical_filepath):
age = time.time() - os.path.getmtime(filepath)
age = time.time() - os.path.getmtime(physical_filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
print("Skipping file because too new", logical_filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))
match = rx.match(os.path.basename(logical_filepath))
if not match:
error_message = f"File path not matching the expected format! ({filepath} ~ {pattern['regex']})"
error_message = f"File path not matching the expected format! ({logical_filepath} ~ {pattern['regex']})"
print(error_message, file=sys.stderr)
print("This file will be ignored!")
continue
file_info = dict(zip(pattern["captures"], match.groups()))
smsrc_records = smsrc.from_file(filepath)
smsrc_records = smsrc.from_file(physical_filepath)
print("Saving")
db.save_raw_smsrc(smsrc_records, file_info, filepath)
db.save_raw_smsrc(smsrc_records, file_info, logical_filepath)
else:
print("Already in DB")

5
bin/preplots.py
View File

@@ -4,9 +4,10 @@ import sps
Preplot importing functions.
"""
def from_file (file):
def from_file (file, realpath = None):
filepath = realpath or file["path"]
if not "type" in file or file["type"] == "sps":
records = sps.from_file(file["path"], file["format"] if "format" in file else None )
records = sps.from_file(filepath, file["format"] if "format" in file else None )
else:
return "Not an SPS file"

14
bin/purge_deleted_files.py
View File

@@ -25,9 +25,15 @@ if __name__ == '__main__':
db.set_survey(survey["schema"])
for file in db.list_files():
path = file[0]
if not os.path.exists(path):
print(path, "NOT FOUND")
db.del_file(path)
try:
path = configuration.translate_path(file[0])
if not os.path.exists(path):
print(path, "NOT FOUND")
db.del_file(file[0])
except TypeError:
# In case the logical path no longer matches
# the Dougal configuration.
print(file[0], "COULD NOT BE TRANSLATED TO A PHYSICAL PATH. DELETING")
db.del_file(file[0])
print("Done")