Mercurial > public > scc_access / changeset
changeset
Download of products for automatic upload to EARLINET db implemented
Thu, 05 Oct 2023 15:41:41 +0200
- author
- Claudio Dema <claudio.dema@imaa.cnr.it>
- date
- Thu, 05 Oct 2023 15:41:41 +0200
- changeset 71
- bf545d90784d
- parent 70
- 4ef6f2102a61
- child 72
- 3de925f654ad
Download of products for automatic upload to EARLINET db implemented
| requirements.txt | file | annotate | diff | comparison | revisions | |
| scc_access/scc_access.py | file | annotate | diff | comparison | revisions |
--- a/requirements.txt Tue Sep 26 16:56:35 2023 +0200 +++ b/requirements.txt Thu Oct 05 15:41:41 2023 +0200 @@ -1,1 +1,2 @@ -. \ No newline at end of file +mysql +mysql-connector-python \ No newline at end of file
--- a/scc_access/scc_access.py Tue Sep 26 16:56:35 2023 +0200 +++ b/scc_access/scc_access.py Thu Oct 05 15:41:41 2023 +0200 @@ -14,6 +14,7 @@ """ import sys +import mysql.connector import requests try: @@ -52,11 +53,15 @@ fails. """ - def __init__(self, auth, output_dir, base_url): + def __init__(self, auth, output_dir, base_url, sccdb_host=None, sccdb_credentials=None, sccdb=None, sccquery=None): self.auth = auth self.output_dir = output_dir self.base_url = base_url + self.sccdb_host = sccdb_host + self.sccdb_credentials = sccdb_credentials + self.sccdb = sccdb + self.sccquery = sccquery self.session = requests.Session() self.session.auth = auth self.session.verify = False @@ -300,44 +305,72 @@ with open(local_file, 'wb') as f: f.write(zip_file.read(ziped_name)) - def download_hirelpp(self, measurement_id): + def download_hirelpp(self, measurement_id, is_automatic_upload=False): """ Download hirelpp files for the measurement id. """ # Construct the download url download_url = self.download_hirelpp_pattern.format(measurement_id) try: self.download_files(measurement_id, 'hirelpp', download_url) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#hirelpp_transfer_status#", "true") + self.sccquery = self.sccquery.replace("#hirelpp_transfer_message#", "'HIRELPP products downloaded'") except Exception as e: logger.error("Could not download HiRElPP files. Error message: {}".format(e)) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#hirelpp_transfer_status#", "false") + self.sccquery = self.sccquery.replace("#hirelpp_transfer_message#", + "'Could not download HIRELPP files. Error message: {}".format(e))+"'" logger.debug('Download exception:', exc_info=True) - def download_cloudmask(self, measurement_id): + def download_cloudmask(self, measurement_id, is_automatic_upload=False): """ Download cloudmask files for the measurement id. """ # Construct the download url download_url = self.download_cloudmask_pattern.format(measurement_id) try: self.download_files(measurement_id, 'cloudscreen', download_url) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#cloudmask_transfer_status#", "true") + self.sccquery = self.sccquery.replace("#cloudmask_transfer_message#", "'CLODUMASK products downloaded'") except Exception as e: logger.error("Could not download cloudscreen files. Error message: {}".format(e)) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#cloudmask_transfer_status#", "false") + self.sccquery = self.sccquery.replace("#cloudmask_transfer_message#", + "'Could not download CLOUDMASK files. Error message: {}".format(e))+"'" logger.debug('Download exception:', exc_info=True) - def download_elpp(self, measurement_id): + def download_elpp(self, measurement_id, is_automatic_upload=False): """ Download preprocessed files for the measurement id. """ # Construct the download url download_url = self.download_elpp_pattern.format(measurement_id) try: self.download_files(measurement_id, 'elpp', download_url) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elpp_transfer_status#", "true") + self.sccquery = self.sccquery.replace("#elpp_transfer_message#", "'ELPP products downloaded'") except Exception as e: logger.error("Could not download ElPP files. Error message: {}".format(e)) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elpp_transfer_status#", "false") + self.sccquery = self.sccquery.replace("#elpp_transfer_message#", + "'Could not download ELPP files. Error message: {}".format(e))+"'" logger.debug('Download exception:', exc_info=True) - def download_elda(self, measurement_id): + def download_elda(self, measurement_id, is_automatic_upload=False): """ Download optical files for the measurement id. """ # Construct the download url download_url = self.download_elda_pattern.format(measurement_id) try: self.download_files(measurement_id, 'elda', download_url) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elda_transfer_status#", "true") + self.sccquery = self.sccquery.replace("#elda_transfer_message#", "'ELDA products downloaded'") except Exception as e: logger.error("Could not download ELDA files. Error message: {}".format(e)) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elda_transfer_status#", "false") + self.sccquery = self.sccquery.replace("#elda_transfer_message#", + "'Could not download ELDA files. Error message: {}".format(e))+"'" logger.debug('Download exception:', exc_info=True) def download_plots(self, measurement_id): @@ -350,24 +383,38 @@ logger.error("Could not download ELDA plots. Error message: {}".format(e)) logger.debug('Download exception:', exc_info=True) - def download_elic(self, measurement_id): + def download_elic(self, measurement_id, is_automatic_upload=False): """ Download ELIC files for the measurement id. """ # Construct the download url download_url = self.download_elic_pattern.format(measurement_id) try: self.download_files(measurement_id, 'elic', download_url) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elic_transfer_status#", "true") + self.sccquery = self.sccquery.replace("#elic_transfer_message#", "'ELIC products downloaded'") except Exception as e: logger.error("Could not download ELIC files. Error message: {}".format(e)) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elic_transfer_status#", "false") + self.sccquery = self.sccquery.replace("#elic_transfer_message#", + "'Could not download ELIC files. Error message: {}".format(e))+"'" logger.debug('Download exception:', exc_info=True) - def download_eldec(self, measurement_id): + def download_eldec(self, measurement_id, is_automatic_upload=False): """ Download ELDEC files for the measurement id. """ # Construct the download url download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now try: self.download_files(measurement_id, 'eldec', download_url) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "true") + self.sccquery = self.sccquery.replace("#eldec_transfer_message#", "'ELDEC products downloaded'") except Exception as e: logger.error("Could not download EDELC files. Error message: {}".format(e)) + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "false") + self.sccquery = self.sccquery.replace("#eldec_transfer_message#", + "'Could not download ELDEC files. Error message: {}".format(e))+"'" logger.debug('Download exception:', exc_info=True) def rerun_elpp(self, measurement_id, monitor=True): @@ -497,32 +544,63 @@ return measurement - def download_products(self, measurement, dir_name): - """ Download all the products of a measurement id (used only for E-SHAPE)""" + def download_products(self, measurement, dir_name, is_automatic_upload=False): + """ Download all the products of a measurement id (used for E-SHAPE and automatic_upload)""" + measurement_id = measurement.id + self.sccquery = self.sccquery.replace("#__measurements__ID#", "'"+measurement_id+"'") base_output_dir = self.output_dir self.output_dir = self.output_dir + dir_name + "/" if measurement.hirelpp == 127: logger.info("Downloading HiRElPP files.") - self.download_hirelpp(measurement_id) + self.download_hirelpp(measurement_id, is_automatic_upload) + else: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#hirelpp_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#hirelpp_transfer_message#", "'No HIRELPP products available'") if measurement.cloudmask == 127: logger.info("Downloading cloud screening files.") - self.download_cloudmask(measurement_id) + self.download_cloudmask(measurement_id, is_automatic_upload) + else: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#cloudmask_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#cloudmask_transfer_message#", "'No CLOUDMASK products available'") if measurement.elpp == 127: logger.info("Downloading ElPP files.") - self.download_elpp(measurement_id) + self.download_elpp(measurement_id, is_automatic_upload) + else: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elpp_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#elpp_transfer_message#", "'No ELPP products available'") if measurement.elda == 127: logger.info("Downloading ELDA files.") - self.download_elda(measurement_id) - logger.info("Downloading ELDA plots.") - self.download_plots(measurement_id) + self.download_elda(measurement_id, is_automatic_upload) + if not is_automatic_upload: + logger.info("Downloading ELDA plots.") + self.download_plots(measurement_id) + else: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elda_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#elda_transfer_message#", "'No ELDA products available'") if measurement.elic == 127: logger.info("Downloading ELIC files.") - self.download_elic(measurement_id) - if measurement.is_calibration and measurement.eldec == 0: + self.download_elic(measurement_id, is_automatic_upload) + else: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#elic_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#elic_transfer_message#", "'No ELIC products available'") + if measurement.is_calibration and measurement.eldec == 127: logger.info("Downloading ELDEC files.") - self.download_eldec(measurement_id) + self.download_eldec(measurement_id, is_automatic_upload) + elif not measurement.is_calibration: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#eldec_transfer_message#", "'The measurement is not a calibration'") + elif measurement.is_calibration and measurement.eldec != 127: + if is_automatic_upload: + self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "NULL") + self.sccquery = self.sccquery.replace("#eldec_transfer_message#", "'No ELDEC products available'") logger.info("--- Processing finished. ---") self.output_dir = base_output_dir @@ -1018,18 +1096,67 @@ def automatic_upload(settings): - date_time_start = datetime.datetime.now() - datetime.timedelta(days=1) + date_time_start = datetime.datetime.now() - datetime.timedelta(hours=8) + date_time_start = date_time_start.replace(minute=0, second=0) start_parameter = date_time_start.strftime("%Y-%m-%dT%H:%M:%S") - date_time_stop = datetime.datetime.now() + date_time_stop = date_time_start + datetime.timedelta(hours=6) stop_parameter = date_time_stop.strftime("%Y-%m-%dT%H:%M:%S") - with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: + # Altri esempi di utilizzo della libreria + #date_time_start = datetime.datetime.combine(date_time_start, datetime.time.min) + #date_time_stop = date_time_start.replace(hour=6, minute=0, second=0) + #date_time_stop = datetime.datetime.combine(date_time_stop, datetime.time.max) + + with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'], settings['sccdb_host'], settings['sccdb_credentials'], settings['sccdb']) as scc: scc.login(settings['website_credentials']) measurements = scc.available_measurements(start_gte=start_parameter, stop_lte=stop_parameter, is_queued=False, is_being_processed=False) + is_automatic_upload = True if measurements is not None: for meas in measurements: - scc.download_products(meas, "") + #Create SQL query to compose + mySql_insert_query = ("INSERT INTO scctoearlinettransfer_log (__measurements__ID, creation_date, " + "hirelpp_transfer_status, hirelpp_transfer_message, cloudmask_transfer_status, " + "cloudmask_transfer_message, elpp_transfer_status, elpp_transfer_message, " + "elda_transfer_status, elda_transfer_message, eldamwl_transfer_status, " + "eldamwl_transfer_message, eldec_transfer_status, eldec_transfer_message, " + "elic_transfer_status, elic_transfer_message, transfer_status, transfer_message) " + "VALUES " + "(#__measurements__ID#, #creation_date# , #hirelpp_transfer_status#, " + "#hirelpp_transfer_message# , #cloudmask_transfer_status#, #cloudmask_transfer_message#, " + "#elpp_transfer_status#, #elpp_transfer_message#, #elda_transfer_status#, " + "#elda_transfer_message#, #eldamwl_transfer_status#, #eldamwl_transfer_message#, " + "#eldec_transfer_status#, #eldec_transfer_message#, #elic_transfer_status#, " + "#elic_transfer_message#, #transfer_status#, #transfer_message#);") + scc.sccquery = mySql_insert_query + scc.download_products(meas, "", is_automatic_upload) + + scc.sccquery = scc.sccquery.replace("#creation_date#", "'"+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+"'") + scc.sccquery = scc.sccquery.replace("#eldamwl_transfer_status#", "NULL") + scc.sccquery = scc.sccquery.replace("#eldamwl_transfer_message#", "'No ELDAMWL products available'") + scc.sccquery = scc.sccquery.replace("#transfer_status#", "true") + scc.sccquery = scc.sccquery.replace("#transfer_message#", "NULL") + + #Execute SQL query + try: + connection = mysql.connector.connect(host=scc.sccdb_host, + database=scc.sccdb, + user=scc.sccdb_credentials[0], + password=scc.sccdb_credentials[1]) + + cursor = connection.cursor() + cursor.execute(scc.sccquery) + connection.commit() + logger.info("{} Record inserted successfully into scctoearlinettransfer_log table".format(cursor.rowcount)) + cursor.close() + + except mysql.connector.Error as error: + logger.error("Failed to insert record into scctoearlinettransfer_log table {}".format(error)) + + finally: + if connection.is_connected(): + connection.close() + logger.debug("MySQL connection is closed") scc.logout()
