mirror of
https://gitlab.com/wgp/dougal/software.git
synced 2025-12-06 11:57:08 +00:00
19ce158329
Provided that the SmartSource headers are being saved to file, and that the path to those files is present in the survey configuration, we now import SmartSource information as metadata in raw_shots.meta->'smsrc'. Closes #19.
77 lines
1.7 KiB
Python
Executable File
77 lines
1.7 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
"""
|
|
Import SmartSource data.
|
|
|
|
For each survey in configuration.surveys(), check for new
|
|
or modified final gun header files and (re-)import them into the
|
|
database.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import pathlib
|
|
import re
|
|
import configuration
|
|
import smsrc
|
|
from datastore import Datastore
|
|
|
|
if __name__ == '__main__':
|
|
|
|
print("Reading configuration")
|
|
surveys = configuration.surveys()
|
|
|
|
print("Connecting to database")
|
|
db = Datastore()
|
|
db.connect()
|
|
|
|
print("Reading surveys")
|
|
for survey in surveys:
|
|
print(f'Survey: {survey["id"]} ({survey["schema"]})')
|
|
|
|
db.set_survey(survey["schema"])
|
|
|
|
try:
|
|
raw_smsrc = survey["raw"]["smsrc"]
|
|
except KeyError:
|
|
print("No SmartSource data configuration")
|
|
continue
|
|
|
|
flags = 0
|
|
if "flags" in raw_smsrc:
|
|
configuration.rxflags(raw_smsrc["flags"])
|
|
|
|
pattern = raw_smsrc["pattern"]
|
|
rx = re.compile(pattern["regex"], flags)
|
|
|
|
for fileprefix in raw_smsrc["paths"]:
|
|
print(f"Path prefix: {fileprefix}")
|
|
|
|
for globspec in raw_smsrc["globs"]:
|
|
for filepath in pathlib.Path(fileprefix).glob(globspec):
|
|
filepath = str(filepath)
|
|
print(f"Found {filepath}")
|
|
|
|
if not db.file_in_db(filepath):
|
|
print("Importing")
|
|
|
|
match = rx.match(os.path.basename(filepath))
|
|
if not match:
|
|
error_message = f"File path not matching the expected format! ({filepath} ~ {pattern['regex']})"
|
|
print(error_message, file=sys.stderr)
|
|
print("This file will be ignored!")
|
|
continue
|
|
|
|
file_info = dict(zip(pattern["captures"], match.groups()))
|
|
|
|
smsrc_records = smsrc.from_file(filepath)
|
|
|
|
print("Saving")
|
|
|
|
db.save_raw_smsrc(smsrc_records, file_info, filepath)
|
|
else:
|
|
print("Already in DB")
|
|
|
|
print("Done")
|
|
|