Aaltronav/dougal-software
1
0
Fork 0
mirror of https://gitlab.com/wgp/dougal/software.git synced 2025-12-06 11:37:08 +00:00
Code Issues Packages Projects Releases Activity

Do not import files that have just been modified.

Browse Source 
We now check that a file is at least a few seconds old
before attempting to import it.

The actual minimum age can be configured in etc/config.yaml or
else is defaults to 10 seconds.

The idea is that this should give the OS enough time to fully
write the file before we import it.

The timestamp being looked at is the modification time.

Fixes #92.
This commit is contained in:
D. Berge
2021-05-07 13:50:32 +02:00
parent 6e5584a433
commit 0fdb42c593
7 changed files with 55 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bin/import_final_p111.py
View File

@@ -12,6 +12,7 @@ import os
import sys
import pathlib
import re
import time
import configuration
import p111
from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
print("Reading configuration")
surveys = configuration.surveys()
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
print("Connecting to database")
db = Datastore()
@@ -49,6 +51,12 @@ if __name__ == '__main__':
print(f"Found {filepath}")
if not db.file_in_db(filepath):
age = time.time() - os.path.getmtime(filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))

8
bin/import_final_p190.py
View File

@@ -12,6 +12,7 @@ import os
import sys
import pathlib
import re
import time
import configuration
import p190
from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
print("Reading configuration")
surveys = configuration.surveys()
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
print("Connecting to database")
db = Datastore()
@@ -49,6 +51,12 @@ if __name__ == '__main__':
print(f"Found {filepath}")
if not db.file_in_db(filepath):
age = time.time() - os.path.getmtime(filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))

9
bin/import_preplots.py
View File

@@ -8,7 +8,9 @@ or modified preplots and (re-)import them into the database.
"""
from glob import glob
import os
import sys
import time
import configuration
import preplots
from datastore import Datastore
@@ -17,6 +19,7 @@ if __name__ == '__main__':
print("Reading configuration")
surveys = configuration.surveys()
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
print("Connecting to database")
db = Datastore()
@@ -28,6 +31,12 @@ if __name__ == '__main__':
for file in survey["preplots"]:
print(f"Preplot: {file['path']}")
if not db.file_in_db(file["path"]):
age = time.time() - os.path.getmtime(file["path"])
if age < file_min_age:
print("Skipping file because too new", file["path"])
continue
print("Importing")
try:
preplot = preplots.from_file(file)

8
bin/import_raw_p111.py
View File

@@ -12,6 +12,7 @@ import os
import sys
import pathlib
import re
import time
import configuration
import p111
from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
print("Reading configuration")
surveys = configuration.surveys()
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
print("Connecting to database")
db = Datastore()
@@ -57,6 +59,12 @@ if __name__ == '__main__':
ntbp = False
if not db.file_in_db(filepath):
age = time.time() - os.path.getmtime(filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))

8
bin/import_raw_p190.py
View File

@@ -12,6 +12,7 @@ import os
import sys
import pathlib
import re
import time
import configuration
import p190
from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
print("Reading configuration")
surveys = configuration.surveys()
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
print("Connecting to database")
db = Datastore()
@@ -52,6 +54,12 @@ if __name__ == '__main__':
print(f"Found {filepath}")
if not db.file_in_db(filepath):
age = time.time() - os.path.getmtime(filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))

8
bin/import_smsrc.py
View File

@@ -12,6 +12,7 @@ import os
import sys
import pathlib
import re
import time
import configuration
import smsrc
from datastore import Datastore
@@ -20,6 +21,7 @@ if __name__ == '__main__':
print("Reading configuration")
surveys = configuration.surveys()
file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10)
print("Connecting to database")
db = Datastore()
@@ -53,6 +55,12 @@ if __name__ == '__main__':
print(f"Found {filepath}")
if not db.file_in_db(filepath):
age = time.time() - os.path.getmtime(filepath)
if age < file_min_age:
print("Skipping file because too new", filepath)
continue
print("Importing")
match = rx.match(os.path.basename(filepath))