mirror of
https://gitlab.com/wgp/dougal/software.git
synced 2025-12-06 10:37:07 +00:00
0fdb42c593
We now check that a file is at least a few seconds old before attempting to import it. The actual minimum age can be configured in etc/config.yaml or else is defaults to 10 seconds. The idea is that this should give the OS enough time to fully write the file before we import it. The timestamp being looked at is the modification time. Fixes #92.
56 lines
1.3 KiB
Python
Executable File
56 lines
1.3 KiB
Python
Executable File
#!/usr/bin/python3
|
|
|
|
"""
|
|
Import preplot data.
|
|
|
|
For each preplot file defined in each survey, check for new
|
|
or modified preplots and (re-)import them into the database.
|
|
"""
|
|
|
|
from glob import glob
|
|
import os
|
|
import sys
|
|
import time
|
|
import configuration
|
|
import preplots
|
|
from datastore import Datastore
|
|
|
|
if __name__ == '__main__':
|
|
|
|
print("Reading configuration")
|
|
surveys = configuration.surveys()
|
|
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
|
|
|
|
print("Connecting to database")
|
|
db = Datastore()
|
|
|
|
print("Reading surveys")
|
|
for survey in surveys:
|
|
print(f'Survey: {survey["id"]} ({survey["schema"]})')
|
|
db.set_survey(survey["schema"])
|
|
for file in survey["preplots"]:
|
|
print(f"Preplot: {file['path']}")
|
|
if not db.file_in_db(file["path"]):
|
|
|
|
age = time.time() - os.path.getmtime(file["path"])
|
|
if age < file_min_age:
|
|
print("Skipping file because too new", file["path"])
|
|
continue
|
|
|
|
print("Importing")
|
|
try:
|
|
preplot = preplots.from_file(file)
|
|
except FileNotFoundError:
|
|
print(f"File does not exist: {file['path']}", file=sys.stderr)
|
|
continue
|
|
|
|
if type(preplot) is list:
|
|
print("Saving to DB")
|
|
db.save_preplots(preplot, file["path"], file["class"], survey["epsg"], file)
|
|
elif type(preplot) is str:
|
|
print(preplot)
|
|
else:
|
|
print("Already in DB")
|
|
|
|
print("Done")
|