dougal-software/bin/human_exports_qc.py

#!/usr/bin/python3

"""
Export data that is entered directly into Dougal
as opposed to being read from external sources.

This data will be read back in when the database
is recreated for an existing survey.
"""

import os
from glob import glob
import pathlib
import string
import configuration
import preplots
from datastore import Datastore

def sane_name(filename):
	allowed_chars = string.ascii_letters + string.digits + " _-#+&^%$!();:.,"
	return ''.join([c for c in filename if c in allowed_chars])

def write_file (filename, items):
	filename.parent.mkdir(parents=True, exist_ok=True)
	with open(filename, "w") as fd:
		for item in items:
			sequence = point = line = ""
			if type(item["_id"]) == list:
				if len(item["_id"]) == 2:
					sequence, point = item["_id"]
				elif len(item["_id"]) == 3:
					sequence, point, line = item["_id"]
			else:
				sequence = item["_id"]

			line = f"{sequence}\t{point}\t{line}\t{item['results']}\n"
			fd.write(line)

def qc_item(item, prefixes = [], index = None):
	leader = "{:0>2d}".format(index) if index is not None else ""
	name = sane_name(leader+" "+item["name"])
	if "check" in item:
		filename = pathlib.Path(*prefixes, name+".txt")
		print("MKFILE", filename)
		print("Export", len(item["check"]), "results")
		write_file(filename, item["check"])

	if "children" in item:
		print("MKDIR", pathlib.Path(*prefixes, name))
		subindex = 0
		for child in item["children"]:
			subindex += 1
			qc_item(child, [*prefixes, name], subindex)

def qc_data (cursor, prefix):
	qc = db.get_info('qc', cursor)
	if qc is not None:
		qc = qc[0]
	else:
		print("No QC data found");
		return 

	#print("QC", qc)
	index = 0
	for item in qc["results"]:
		index += 1
		qc_item(item, [prefix, "QC"], index)

if __name__ == '__main__':

	print("Reading configuration")
	surveys = configuration.surveys()

	print("Connecting to database")
	db = Datastore()

	print("Reading surveys")
	for survey in surveys:
		print(f'Survey: {survey["id"]} ({survey["schema"]})')
		db.set_survey(survey["schema"])
		with db.conn.cursor() as cursor:

			try:
				pathPrefix = survey["exports"]["human"]["path"]
			except KeyError:
				print("Survey does not define an export path for human data")
				continue

			if not pathlib.Path(pathPrefix).exists():
				print(pathPrefix)
				raise ValueError("Export path does not exist")

			qc_data(cursor, pathPrefix)

	print("Done")