From c99a625b602792123d56281e55585900a6f586ca Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Wed, 30 Aug 2023 14:19:56 +0200 Subject: [PATCH] Add function to retrieve survey configurations from DB. As the survey definitions will no longer be stored in files under etc/surveys/ but directly on the database, this function replaces configuration.surveys() --- bin/configuration.py | 4 ++++ bin/datastore.py | 30 ++++++++++++++++++++++++++++++ bin/import_final_p111.py | 3 +-- bin/import_preplots.py | 8 ++++---- bin/import_raw_p111.py | 3 +-- bin/import_smsrc.py | 3 +-- bin/import_survey_config.py | 23 +---------------------- bin/purge_deleted_files.py | 6 +++--- 8 files changed, 45 insertions(+), 35 deletions(-) diff --git a/bin/configuration.py b/bin/configuration.py index d59d0ee..481f0b0 100644 --- a/bin/configuration.py +++ b/bin/configuration.py @@ -55,6 +55,10 @@ def files (globspec = None, include_archived = False): quickly and temporarily “disabling” a survey configuration by renaming the relevant file. """ + + print("This method is obsolete") + return + tuples = [] if globspec is None: diff --git a/bin/datastore.py b/bin/datastore.py index 31cc2d1..0c16928 100644 --- a/bin/datastore.py +++ b/bin/datastore.py @@ -589,6 +589,36 @@ class Datastore: # we assume that we are in the middle of a transaction + + def surveys (self, include_archived = False): + """ + Return list of survey definitions. + """ + + if self.conn is None: + self.connect() + + if include_archived: + qry = """ + SELECT meta + FROM public.projects; + """ + else: + qry = """ + SELECT meta + FROM public.projects + WHERE NOT (meta->'archived')::boolean IS true + """ + + with self.conn: + with self.conn.cursor() as cursor: + + cursor.execute(qry) + results = cursor.fetchall() + return [r[0] for r in results if r[0]] + + + # TODO Does this need tweaking on account of #246? def apply_survey_configuration(self, cursor = None): if cursor is None: cur = self.conn.cursor() diff --git a/bin/import_final_p111.py b/bin/import_final_p111.py index 4056924..cac0fe5 100755 --- a/bin/import_final_p111.py +++ b/bin/import_final_p111.py @@ -51,12 +51,11 @@ def del_pending_remark(db, sequence): if __name__ == '__main__': print("Reading configuration") - surveys = configuration.surveys() file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() - db.connect() + surveys = db.surveys() print("Reading surveys") for survey in surveys: diff --git a/bin/import_preplots.py b/bin/import_preplots.py index 8c89cee..5a1af2b 100755 --- a/bin/import_preplots.py +++ b/bin/import_preplots.py @@ -17,12 +17,12 @@ from datastore import Datastore if __name__ == '__main__': - print("Reading configuration") - surveys = configuration.surveys() - file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) - print("Connecting to database") db = Datastore() + surveys = db.surveys() + + print("Reading configuration") + file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Reading surveys") for survey in surveys: diff --git a/bin/import_raw_p111.py b/bin/import_raw_p111.py index bd59696..d528e6a 100755 --- a/bin/import_raw_p111.py +++ b/bin/import_raw_p111.py @@ -20,12 +20,11 @@ from datastore import Datastore if __name__ == '__main__': print("Reading configuration") - surveys = configuration.surveys() file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() - db.connect() + surveys = db.surveys() print("Reading surveys") for survey in surveys: diff --git a/bin/import_smsrc.py b/bin/import_smsrc.py index 8b5dc83..758e9c4 100755 --- a/bin/import_smsrc.py +++ b/bin/import_smsrc.py @@ -20,12 +20,11 @@ from datastore import Datastore if __name__ == '__main__': print("Reading configuration") - surveys = configuration.surveys() file_min_age = configuration.read().get('imports', {}).get('file_min_age', 10) print("Connecting to database") db = Datastore() - db.connect() + surveys = db.surveys() print("Reading surveys") for survey in surveys: diff --git a/bin/import_survey_config.py b/bin/import_survey_config.py index b0452f7..75fd149 100755 --- a/bin/import_survey_config.py +++ b/bin/import_survey_config.py @@ -15,25 +15,4 @@ from datastore import Datastore if __name__ == '__main__': - print("Reading configuration") - configs = configuration.files(include_archived = True) - - print("Connecting to database") - db = Datastore() - #db.connect() - - print("Reading surveys") - for config in configs: - filepath = config[0] - survey = config[1] - print(f'Survey: {survey["id"]} ({filepath})') - db.set_survey(survey["schema"]) - if not db.file_in_db(filepath): - print("Saving to DB") - db.save_file_data(filepath, json.dumps(survey)) - print("Applying survey configuration") - db.apply_survey_configuration() - else: - print("Already in DB") - - print("Done") + print("This function is obsolete. Returning with no action") diff --git a/bin/purge_deleted_files.py b/bin/purge_deleted_files.py index 512bf2a..c7aa778 100755 --- a/bin/purge_deleted_files.py +++ b/bin/purge_deleted_files.py @@ -13,12 +13,12 @@ from datastore import Datastore if __name__ == '__main__': - print("Reading configuration") - surveys = configuration.surveys() - print("Connecting to database") db = Datastore() + print("Reading configuration") + surveys = db.surveys() + print("Reading surveys") for survey in surveys: print(f'Survey: {survey["id"]} ({survey["schema"]})')