Mercurial > public > scc_access / file comparison

comparison: scc_access/scc_access.py

scc_access/scc_access.py

changeset 71
bf545d90784d
parent 70
4ef6f2102a61
child 72
3de925f654ad
equal deleted inserted replaced
70:4ef6f2102a61 71:bf545d90784d
12 12
13 Several shortcut functions are defined to perform specific tasks using the SCC class (e.g. process_file, delete_measurements etc). 13 Several shortcut functions are defined to perform specific tasks using the SCC class (e.g. process_file, delete_measurements etc).
14 """ 14 """
15 import sys 15 import sys
16 16
17 import mysql.connector
17 import requests 18 import requests
18 19
19 try: 20 try:
20 import urllib.parse as urlparse # Python 3 21 import urllib.parse as urlparse # Python 3
21 except ImportError: 22 except ImportError:
50 Most interactions are by simulating a normal browser session. In the current 51 Most interactions are by simulating a normal browser session. In the current
51 version few checks are performed before upload a file, and no feedback is given in case the upload 52 version few checks are performed before upload a file, and no feedback is given in case the upload
52 fails. 53 fails.
53 """ 54 """
54 55
55 def __init__(self, auth, output_dir, base_url): 56 def __init__(self, auth, output_dir, base_url, sccdb_host=None, sccdb_credentials=None, sccdb=None, sccquery=None):
56 57
57 self.auth = auth 58 self.auth = auth
58 self.output_dir = output_dir 59 self.output_dir = output_dir
59 self.base_url = base_url 60 self.base_url = base_url
61 self.sccdb_host = sccdb_host
62 self.sccdb_credentials = sccdb_credentials
63 self.sccdb = sccdb
64 self.sccquery = sccquery
60 self.session = requests.Session() 65 self.session = requests.Session()
61 self.session.auth = auth 66 self.session.auth = auth
62 self.session.verify = False 67 self.session.verify = False
63 68
64 # Setup SCC server URLS for later use 69 # Setup SCC server URLS for later use
298 local_file = os.path.join(local_dir, basename) 303 local_file = os.path.join(local_dir, basename)
299 304
300 with open(local_file, 'wb') as f: 305 with open(local_file, 'wb') as f:
301 f.write(zip_file.read(ziped_name)) 306 f.write(zip_file.read(ziped_name))
302 307
303 def download_hirelpp(self, measurement_id): 308 def download_hirelpp(self, measurement_id, is_automatic_upload=False):
304 """ Download hirelpp files for the measurement id. """ 309 """ Download hirelpp files for the measurement id. """
305 # Construct the download url 310 # Construct the download url
306 download_url = self.download_hirelpp_pattern.format(measurement_id) 311 download_url = self.download_hirelpp_pattern.format(measurement_id)
307 try: 312 try:
308 self.download_files(measurement_id, 'hirelpp', download_url) 313 self.download_files(measurement_id, 'hirelpp', download_url)
314 if is_automatic_upload:
315 self.sccquery = self.sccquery.replace("#hirelpp_transfer_status#", "true")
316 self.sccquery = self.sccquery.replace("#hirelpp_transfer_message#", "'HIRELPP products downloaded'")
309 except Exception as e: 317 except Exception as e:
310 logger.error("Could not download HiRElPP files. Error message: {}".format(e)) 318 logger.error("Could not download HiRElPP files. Error message: {}".format(e))
319 if is_automatic_upload:
320 self.sccquery = self.sccquery.replace("#hirelpp_transfer_status#", "false")
321 self.sccquery = self.sccquery.replace("#hirelpp_transfer_message#",
322 "'Could not download HIRELPP files. Error message: {}".format(e))+"'"
311 logger.debug('Download exception:', exc_info=True) 323 logger.debug('Download exception:', exc_info=True)
312 324
313 def download_cloudmask(self, measurement_id): 325 def download_cloudmask(self, measurement_id, is_automatic_upload=False):
314 """ Download cloudmask files for the measurement id. """ 326 """ Download cloudmask files for the measurement id. """
315 # Construct the download url 327 # Construct the download url
316 download_url = self.download_cloudmask_pattern.format(measurement_id) 328 download_url = self.download_cloudmask_pattern.format(measurement_id)
317 try: 329 try:
318 self.download_files(measurement_id, 'cloudscreen', download_url) 330 self.download_files(measurement_id, 'cloudscreen', download_url)
331 if is_automatic_upload:
332 self.sccquery = self.sccquery.replace("#cloudmask_transfer_status#", "true")
333 self.sccquery = self.sccquery.replace("#cloudmask_transfer_message#", "'CLODUMASK products downloaded'")
319 except Exception as e: 334 except Exception as e:
320 logger.error("Could not download cloudscreen files. Error message: {}".format(e)) 335 logger.error("Could not download cloudscreen files. Error message: {}".format(e))
336 if is_automatic_upload:
337 self.sccquery = self.sccquery.replace("#cloudmask_transfer_status#", "false")
338 self.sccquery = self.sccquery.replace("#cloudmask_transfer_message#",
339 "'Could not download CLOUDMASK files. Error message: {}".format(e))+"'"
321 logger.debug('Download exception:', exc_info=True) 340 logger.debug('Download exception:', exc_info=True)
322 341
323 def download_elpp(self, measurement_id): 342 def download_elpp(self, measurement_id, is_automatic_upload=False):
324 """ Download preprocessed files for the measurement id. """ 343 """ Download preprocessed files for the measurement id. """
325 # Construct the download url 344 # Construct the download url
326 download_url = self.download_elpp_pattern.format(measurement_id) 345 download_url = self.download_elpp_pattern.format(measurement_id)
327 try: 346 try:
328 self.download_files(measurement_id, 'elpp', download_url) 347 self.download_files(measurement_id, 'elpp', download_url)
348 if is_automatic_upload:
349 self.sccquery = self.sccquery.replace("#elpp_transfer_status#", "true")
350 self.sccquery = self.sccquery.replace("#elpp_transfer_message#", "'ELPP products downloaded'")
329 except Exception as e: 351 except Exception as e:
330 logger.error("Could not download ElPP files. Error message: {}".format(e)) 352 logger.error("Could not download ElPP files. Error message: {}".format(e))
353 if is_automatic_upload:
354 self.sccquery = self.sccquery.replace("#elpp_transfer_status#", "false")
355 self.sccquery = self.sccquery.replace("#elpp_transfer_message#",
356 "'Could not download ELPP files. Error message: {}".format(e))+"'"
331 logger.debug('Download exception:', exc_info=True) 357 logger.debug('Download exception:', exc_info=True)
332 358
333 def download_elda(self, measurement_id): 359 def download_elda(self, measurement_id, is_automatic_upload=False):
334 """ Download optical files for the measurement id. """ 360 """ Download optical files for the measurement id. """
335 # Construct the download url 361 # Construct the download url
336 download_url = self.download_elda_pattern.format(measurement_id) 362 download_url = self.download_elda_pattern.format(measurement_id)
337 try: 363 try:
338 self.download_files(measurement_id, 'elda', download_url) 364 self.download_files(measurement_id, 'elda', download_url)
365 if is_automatic_upload:
366 self.sccquery = self.sccquery.replace("#elda_transfer_status#", "true")
367 self.sccquery = self.sccquery.replace("#elda_transfer_message#", "'ELDA products downloaded'")
339 except Exception as e: 368 except Exception as e:
340 logger.error("Could not download ELDA files. Error message: {}".format(e)) 369 logger.error("Could not download ELDA files. Error message: {}".format(e))
370 if is_automatic_upload:
371 self.sccquery = self.sccquery.replace("#elda_transfer_status#", "false")
372 self.sccquery = self.sccquery.replace("#elda_transfer_message#",
373 "'Could not download ELDA files. Error message: {}".format(e))+"'"
341 logger.debug('Download exception:', exc_info=True) 374 logger.debug('Download exception:', exc_info=True)
342 375
343 def download_plots(self, measurement_id): 376 def download_plots(self, measurement_id):
344 """ Download profile graphs for the measurement id. """ 377 """ Download profile graphs for the measurement id. """
345 # Construct the download url 378 # Construct the download url
348 self.download_files(measurement_id, 'elda_plots', download_url) 381 self.download_files(measurement_id, 'elda_plots', download_url)
349 except Exception as e: 382 except Exception as e:
350 logger.error("Could not download ELDA plots. Error message: {}".format(e)) 383 logger.error("Could not download ELDA plots. Error message: {}".format(e))
351 logger.debug('Download exception:', exc_info=True) 384 logger.debug('Download exception:', exc_info=True)
352 385
353 def download_elic(self, measurement_id): 386 def download_elic(self, measurement_id, is_automatic_upload=False):
354 """ Download ELIC files for the measurement id. """ 387 """ Download ELIC files for the measurement id. """
355 # Construct the download url 388 # Construct the download url
356 download_url = self.download_elic_pattern.format(measurement_id) 389 download_url = self.download_elic_pattern.format(measurement_id)
357 try: 390 try:
358 self.download_files(measurement_id, 'elic', download_url) 391 self.download_files(measurement_id, 'elic', download_url)
392 if is_automatic_upload:
393 self.sccquery = self.sccquery.replace("#elic_transfer_status#", "true")
394 self.sccquery = self.sccquery.replace("#elic_transfer_message#", "'ELIC products downloaded'")
359 except Exception as e: 395 except Exception as e:
360 logger.error("Could not download ELIC files. Error message: {}".format(e)) 396 logger.error("Could not download ELIC files. Error message: {}".format(e))
397 if is_automatic_upload:
398 self.sccquery = self.sccquery.replace("#elic_transfer_status#", "false")
399 self.sccquery = self.sccquery.replace("#elic_transfer_message#",
400 "'Could not download ELIC files. Error message: {}".format(e))+"'"
361 logger.debug('Download exception:', exc_info=True) 401 logger.debug('Download exception:', exc_info=True)
362 402
363 def download_eldec(self, measurement_id): 403 def download_eldec(self, measurement_id, is_automatic_upload=False):
364 """ Download ELDEC files for the measurement id. """ 404 """ Download ELDEC files for the measurement id. """
365 # Construct the download url 405 # Construct the download url
366 download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now 406 download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now
367 try: 407 try:
368 self.download_files(measurement_id, 'eldec', download_url) 408 self.download_files(measurement_id, 'eldec', download_url)
409 if is_automatic_upload:
410 self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "true")
411 self.sccquery = self.sccquery.replace("#eldec_transfer_message#", "'ELDEC products downloaded'")
369 except Exception as e: 412 except Exception as e:
370 logger.error("Could not download EDELC files. Error message: {}".format(e)) 413 logger.error("Could not download EDELC files. Error message: {}".format(e))
414 if is_automatic_upload:
415 self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "false")
416 self.sccquery = self.sccquery.replace("#eldec_transfer_message#",
417 "'Could not download ELDEC files. Error message: {}".format(e))+"'"
371 logger.debug('Download exception:', exc_info=True) 418 logger.debug('Download exception:', exc_info=True)
372 419
373 def rerun_elpp(self, measurement_id, monitor=True): 420 def rerun_elpp(self, measurement_id, monitor=True):
374 logger.debug("Started rerun_elpp procedure.") 421 logger.debug("Started rerun_elpp procedure.")
375 422
495 self.download_eldec(measurement_id) 542 self.download_eldec(measurement_id)
496 logger.info("--- Processing finished. ---") 543 logger.info("--- Processing finished. ---")
497 544
498 return measurement 545 return measurement
499 546
500 def download_products(self, measurement, dir_name): 547 def download_products(self, measurement, dir_name, is_automatic_upload=False):
501 """ Download all the products of a measurement id (used only for E-SHAPE)""" 548 """ Download all the products of a measurement id (used for E-SHAPE and automatic_upload)"""
549
502 measurement_id = measurement.id 550 measurement_id = measurement.id
551 self.sccquery = self.sccquery.replace("#__measurements__ID#", "'"+measurement_id+"'")
503 base_output_dir = self.output_dir 552 base_output_dir = self.output_dir
504 self.output_dir = self.output_dir + dir_name + "/" 553 self.output_dir = self.output_dir + dir_name + "/"
505 554
506 if measurement.hirelpp == 127: 555 if measurement.hirelpp == 127:
507 logger.info("Downloading HiRElPP files.") 556 logger.info("Downloading HiRElPP files.")
508 self.download_hirelpp(measurement_id) 557 self.download_hirelpp(measurement_id, is_automatic_upload)
558 else:
559 if is_automatic_upload:
560 self.sccquery = self.sccquery.replace("#hirelpp_transfer_status#", "NULL")
561 self.sccquery = self.sccquery.replace("#hirelpp_transfer_message#", "'No HIRELPP products available'")
509 if measurement.cloudmask == 127: 562 if measurement.cloudmask == 127:
510 logger.info("Downloading cloud screening files.") 563 logger.info("Downloading cloud screening files.")
511 self.download_cloudmask(measurement_id) 564 self.download_cloudmask(measurement_id, is_automatic_upload)
565 else:
566 if is_automatic_upload:
567 self.sccquery = self.sccquery.replace("#cloudmask_transfer_status#", "NULL")
568 self.sccquery = self.sccquery.replace("#cloudmask_transfer_message#", "'No CLOUDMASK products available'")
512 if measurement.elpp == 127: 569 if measurement.elpp == 127:
513 logger.info("Downloading ElPP files.") 570 logger.info("Downloading ElPP files.")
514 self.download_elpp(measurement_id) 571 self.download_elpp(measurement_id, is_automatic_upload)
572 else:
573 if is_automatic_upload:
574 self.sccquery = self.sccquery.replace("#elpp_transfer_status#", "NULL")
575 self.sccquery = self.sccquery.replace("#elpp_transfer_message#", "'No ELPP products available'")
515 if measurement.elda == 127: 576 if measurement.elda == 127:
516 logger.info("Downloading ELDA files.") 577 logger.info("Downloading ELDA files.")
517 self.download_elda(measurement_id) 578 self.download_elda(measurement_id, is_automatic_upload)
518 logger.info("Downloading ELDA plots.") 579 if not is_automatic_upload:
519 self.download_plots(measurement_id) 580 logger.info("Downloading ELDA plots.")
581 self.download_plots(measurement_id)
582 else:
583 if is_automatic_upload:
584 self.sccquery = self.sccquery.replace("#elda_transfer_status#", "NULL")
585 self.sccquery = self.sccquery.replace("#elda_transfer_message#", "'No ELDA products available'")
520 if measurement.elic == 127: 586 if measurement.elic == 127:
521 logger.info("Downloading ELIC files.") 587 logger.info("Downloading ELIC files.")
522 self.download_elic(measurement_id) 588 self.download_elic(measurement_id, is_automatic_upload)
523 if measurement.is_calibration and measurement.eldec == 0: 589 else:
590 if is_automatic_upload:
591 self.sccquery = self.sccquery.replace("#elic_transfer_status#", "NULL")
592 self.sccquery = self.sccquery.replace("#elic_transfer_message#", "'No ELIC products available'")
593 if measurement.is_calibration and measurement.eldec == 127:
524 logger.info("Downloading ELDEC files.") 594 logger.info("Downloading ELDEC files.")
525 self.download_eldec(measurement_id) 595 self.download_eldec(measurement_id, is_automatic_upload)
596 elif not measurement.is_calibration:
597 if is_automatic_upload:
598 self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "NULL")
599 self.sccquery = self.sccquery.replace("#eldec_transfer_message#", "'The measurement is not a calibration'")
600 elif measurement.is_calibration and measurement.eldec != 127:
601 if is_automatic_upload:
602 self.sccquery = self.sccquery.replace("#eldec_transfer_status#", "NULL")
603 self.sccquery = self.sccquery.replace("#eldec_transfer_message#", "'No ELDEC products available'")
526 logger.info("--- Processing finished. ---") 604 logger.info("--- Processing finished. ---")
527 605
528 self.output_dir = base_output_dir 606 self.output_dir = base_output_dir
529 return measurement 607 return measurement
530 608
1016 1094
1017 scc.logout() 1095 scc.logout()
1018 1096
1019 1097
1020 def automatic_upload(settings): 1098 def automatic_upload(settings):
1021 date_time_start = datetime.datetime.now() - datetime.timedelta(days=1) 1099 date_time_start = datetime.datetime.now() - datetime.timedelta(hours=8)
1100 date_time_start = date_time_start.replace(minute=0, second=0)
1022 start_parameter = date_time_start.strftime("%Y-%m-%dT%H:%M:%S") 1101 start_parameter = date_time_start.strftime("%Y-%m-%dT%H:%M:%S")
1023 date_time_stop = datetime.datetime.now() 1102 date_time_stop = date_time_start + datetime.timedelta(hours=6)
1024 stop_parameter = date_time_stop.strftime("%Y-%m-%dT%H:%M:%S") 1103 stop_parameter = date_time_stop.strftime("%Y-%m-%dT%H:%M:%S")
1025 1104
1026 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: 1105 # Altri esempi di utilizzo della libreria
1106 #date_time_start = datetime.datetime.combine(date_time_start, datetime.time.min)
1107 #date_time_stop = date_time_start.replace(hour=6, minute=0, second=0)
1108 #date_time_stop = datetime.datetime.combine(date_time_stop, datetime.time.max)
1109
1110 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'], settings['sccdb_host'], settings['sccdb_credentials'], settings['sccdb']) as scc:
1027 scc.login(settings['website_credentials']) 1111 scc.login(settings['website_credentials'])
1028 1112
1029 measurements = scc.available_measurements(start_gte=start_parameter, stop_lte=stop_parameter, is_queued=False, is_being_processed=False) 1113 measurements = scc.available_measurements(start_gte=start_parameter, stop_lte=stop_parameter, is_queued=False, is_being_processed=False)
1114 is_automatic_upload = True
1030 if measurements is not None: 1115 if measurements is not None:
1031 for meas in measurements: 1116 for meas in measurements:
1032 scc.download_products(meas, "") 1117 #Create SQL query to compose
1118 mySql_insert_query = ("INSERT INTO scctoearlinettransfer_log (__measurements__ID, creation_date, "
1119 "hirelpp_transfer_status, hirelpp_transfer_message, cloudmask_transfer_status, "
1120 "cloudmask_transfer_message, elpp_transfer_status, elpp_transfer_message, "
1121 "elda_transfer_status, elda_transfer_message, eldamwl_transfer_status, "
1122 "eldamwl_transfer_message, eldec_transfer_status, eldec_transfer_message, "
1123 "elic_transfer_status, elic_transfer_message, transfer_status, transfer_message) "
1124 "VALUES "
1125 "(#__measurements__ID#, #creation_date# , #hirelpp_transfer_status#, "
1126 "#hirelpp_transfer_message# , #cloudmask_transfer_status#, #cloudmask_transfer_message#, "
1127 "#elpp_transfer_status#, #elpp_transfer_message#, #elda_transfer_status#, "
1128 "#elda_transfer_message#, #eldamwl_transfer_status#, #eldamwl_transfer_message#, "
1129 "#eldec_transfer_status#, #eldec_transfer_message#, #elic_transfer_status#, "
1130 "#elic_transfer_message#, #transfer_status#, #transfer_message#);")
1131 scc.sccquery = mySql_insert_query
1132 scc.download_products(meas, "", is_automatic_upload)
1133
1134 scc.sccquery = scc.sccquery.replace("#creation_date#", "'"+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+"'")
1135 scc.sccquery = scc.sccquery.replace("#eldamwl_transfer_status#", "NULL")
1136 scc.sccquery = scc.sccquery.replace("#eldamwl_transfer_message#", "'No ELDAMWL products available'")
1137 scc.sccquery = scc.sccquery.replace("#transfer_status#", "true")
1138 scc.sccquery = scc.sccquery.replace("#transfer_message#", "NULL")
1139
1140 #Execute SQL query
1141 try:
1142 connection = mysql.connector.connect(host=scc.sccdb_host,
1143 database=scc.sccdb,
1144 user=scc.sccdb_credentials[0],
1145 password=scc.sccdb_credentials[1])
1146
1147 cursor = connection.cursor()
1148 cursor.execute(scc.sccquery)
1149 connection.commit()
1150 logger.info("{} Record inserted successfully into scctoearlinettransfer_log table".format(cursor.rowcount))
1151 cursor.close()
1152
1153 except mysql.connector.Error as error:
1154 logger.error("Failed to insert record into scctoearlinettransfer_log table {}".format(error))
1155
1156 finally:
1157 if connection.is_connected():
1158 connection.close()
1159 logger.debug("MySQL connection is closed")
1033 1160
1034 scc.logout() 1161 scc.logout()
1035 1162
1036 1163
1037 def settings_from_path(config_file_path): 1164 def settings_from_path(config_file_path):

EARLINET Repository: SCC access

mercurial