Aaltronav/dougal-software
1
0
Fork 0
mirror of https://gitlab.com/wgp/dougal/software.git synced 2025-12-06 09:27:07 +00:00
Code Issues Packages Projects Releases Activity

Handle NTBP flags in raw data.

Browse Source 
This works by matching a regular expression against
the full path name (not just the file name). If it
matches, the NTBP flag is set to true.
This commit is contained in:
D. Berge
2020-08-31 13:04:46 +02:00
parent c32c408d12
commit 5363ce8bab
3 changed files with 20 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
bin/datastore.py
View File

@@ -207,7 +207,7 @@ class Datastore:
self.maybe_commit()
def save_raw_p190(self, records, fileinfo, filepath, epsg = 0, filedata = None):
def save_raw_p190(self, records, fileinfo, filepath, epsg = 0, filedata = None, ntbp = False):
"""
Save raw P1 data.
@@ -241,7 +241,7 @@ class Datastore:
ON CONFLICT DO NOTHING;
"""
cursor.execute(qry, (fileinfo["sequence"], fileinfo["line"], False, incr))
cursor.execute(qry, (fileinfo["sequence"], fileinfo["line"], ntbp, incr))
qry = """
INSERT INTO raw_lines_files (sequence, hash)
@@ -335,7 +335,7 @@ class Datastore:
self.maybe_commit()
def save_raw_p111 (self, records, fileinfo, filepath, epsg = 0, filedata = None):
def save_raw_p111 (self, records, fileinfo, filepath, epsg = 0, filedata = None, ntbp = False):
with self.conn.cursor() as cursor:
hash = self.add_file(filepath, cursor)
@@ -347,7 +347,7 @@ class Datastore:
ON CONFLICT DO NOTHING;
"""
cursor.execute(qry, (fileinfo["sequence"], fileinfo["line"], False, incr))
cursor.execute(qry, (fileinfo["sequence"], fileinfo["line"], ntbp, incr))
qry = """
INSERT INTO raw_lines_files (sequence, hash)

9
bin/import_raw_p111.py
View File

@@ -40,6 +40,9 @@ if __name__ == '__main__':
pattern = raw_p111["pattern"]
rx = re.compile(pattern["regex"])
if "ntbp" in survey["raw"]:
ntbpRx = re.compile(survey["raw"]["ntbp"])
for fileprefix in raw_p111["paths"]:
print(f"Path prefix: {fileprefix}")
@@ -59,6 +62,10 @@ if __name__ == '__main__':
continue
file_info = dict(zip(pattern["captures"], match.groups()))
if ntbpRx:
ntbp = ntbpRx.match(filepath) is not None
else:
ntbp = False
p111_data = p111.from_file(filepath)
@@ -66,7 +73,7 @@ if __name__ == '__main__':
p111_records = p111.p111_type("S", p111_data)
db.save_raw_p111(p111_records, file_info, filepath, survey["epsg"])
db.save_raw_p111(p111_records, file_info, filepath, survey["epsg"], ntbp=ntbp)
else:
print("Already in DB")

9
bin/import_raw_p190.py
View File

@@ -40,6 +40,9 @@ if __name__ == '__main__':
pattern = raw_p190["pattern"]
rx = re.compile(pattern["regex"])
if "ntbp" in survey["raw"]:
ntbpRx = re.compile(survey["raw"]["ntbp"])
for fileprefix in raw_p190["paths"]:
print(f"Path prefix: {fileprefix}")
@@ -59,6 +62,10 @@ if __name__ == '__main__':
continue
file_info = dict(zip(pattern["captures"], match.groups()))
if ntbpRx:
ntbp = ntbpRx.match(filepath) is not None
else:
ntbp = False
p190_data = p190.from_file(filepath, with_objrefs=True)
p190_data_timestamped = p190.apply_tstamps(
@@ -75,7 +82,7 @@ if __name__ == '__main__':
file_data = dict()
file_data["offsets_p190"] = [h["description"].strip().split(" to ")+h["data"].split() for h in p190_data_normalised if h["record_type"] == "H" and h["header_type"] == "09"]
db.save_raw_p190(p190_records, file_info, filepath, survey["epsg"], file_data)
db.save_raw_p190(p190_records, file_info, filepath, survey["epsg"], file_data, ntbp)
else:
print("Already in DB")