491 self.download_eldec(measurement_id) |
491 self.download_eldec(measurement_id) |
492 logger.info("--- Processing finished. ---") |
492 logger.info("--- Processing finished. ---") |
493 |
493 |
494 return measurement |
494 return measurement |
495 |
495 |
|
496 def download_products(self, measurement, dir_name): |
|
497 """ Download all the products of a measurement id (used only for E-SHAPE""" |
|
498 measurement_id = measurement.id |
|
499 base_output_dir = self.output_dir |
|
500 self.output_dir = self.output_dir + dir_name + "/" |
|
501 |
|
502 if measurement.hirelpp == 127: |
|
503 logger.info("Downloading HiRElPP files.") |
|
504 self.download_hirelpp(measurement_id) |
|
505 if measurement.cloudmask == 127: |
|
506 logger.info("Downloading cloud screening files.") |
|
507 self.download_cloudmask(measurement_id) |
|
508 if measurement.elpp == 127: |
|
509 logger.info("Downloading ElPP files.") |
|
510 self.download_elpp(measurement_id) |
|
511 if measurement.elda == 127: |
|
512 logger.info("Downloading ELDA files.") |
|
513 self.download_elda(measurement_id) |
|
514 logger.info("Downloading ELDA plots.") |
|
515 self.download_plots(measurement_id) |
|
516 if measurement.elic == 127: |
|
517 logger.info("Downloading ELIC files.") |
|
518 self.download_elic(measurement_id) |
|
519 if measurement.is_calibration and measurement.eldec == 0: |
|
520 logger.info("Downloading ELDEC files.") |
|
521 self.download_eldec(measurement_id) |
|
522 logger.info("--- Processing finished. ---") |
|
523 |
|
524 self.output_dir = base_output_dir |
|
525 return measurement |
|
526 |
496 def get_measurement(self, measurement_id): |
527 def get_measurement(self, measurement_id): |
497 """ Get a measurement information from the SCC API. |
528 """ Get a measurement information from the SCC API. |
498 |
529 |
499 Parameters |
530 Parameters |
500 ---------- |
531 ---------- |
579 return None |
610 return None |
580 |
611 |
581 logger.info("Deleted measurement {0}".format(measurement_id)) |
612 logger.info("Deleted measurement {0}".format(measurement_id)) |
582 return True |
613 return True |
583 |
614 |
584 def available_measurements(self): |
615 def available_measurements(self, start_gte=None, stop_lte=None): |
585 """ Get a list of available measurement on the SCC. |
616 """ Get a list of available measurement on the SCC. |
586 |
617 |
587 The methods is currently not used, could be merged with list_measurements. |
618 The methods is currently not used, could be merged with list_measurements. |
588 """ |
619 """ |
589 response = self.session.get(self.api_measurements_url) |
620 |
|
621 params = {} |
|
622 if start_gte is not None: |
|
623 params['start__gte'] = start_gte |
|
624 if stop_lte is not None: |
|
625 params['stop__lte'] = stop_lte |
|
626 |
|
627 response = self.session.get(self.api_measurements_url, params=params) |
590 response_dict = response.json() |
628 response_dict = response.json() |
591 |
629 |
592 if response_dict: |
630 if response_dict: |
593 measurement_list = response_dict['objects'] |
631 measurement_list = response_dict['objects'] |
594 measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] |
632 measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] |
597 logger.warning("No response received from the SCC when asked for available measurements.") |
635 logger.warning("No response received from the SCC when asked for available measurements.") |
598 measurements = None |
636 measurements = None |
599 |
637 |
600 return measurements |
638 return measurements |
601 |
639 |
602 def list_measurements(self, id_exact=None, id_startswith=None): |
640 def list_measurements(self, id_exact=None, id_startswith=None, |
|
641 start_exact=None, start_gte=None, start_lte=None, |
|
642 stop_exact=None, stop_gte=None, stop_lte=None, |
|
643 station_exact=None, station_in=None): |
603 """ Get the response text from the API. """ |
644 """ Get the response text from the API. """ |
604 |
645 |
605 # TODO: Add some error handling, e.g. as per available_measurements method |
646 # TODO: Add some error handling, e.g. as per available_measurements method |
606 |
647 |
607 # Need to set to empty string if not specified, we won't get any results |
648 # Need to set to empty string if not specified, we won't get any results |
608 params = {} |
649 params = {} |
609 |
650 |
610 if id_exact is not None: |
651 if id_exact is not None: |
611 params['id__exact'] = id_exact |
652 params['id__exact'] = id_exact |
612 else: |
653 elif id_startswith is not None: |
613 params['id__startswith'] = id_startswith |
654 params['id__startswith'] = id_startswith |
|
655 |
|
656 if start_exact is not None: |
|
657 params['start__exact'] = start_exact |
|
658 else: |
|
659 if start_gte is not None: |
|
660 params['start__gte'] = start_gte |
|
661 if start_lte is not None: |
|
662 params['start__lte'] = start_lte |
|
663 |
|
664 if stop_exact is not None: |
|
665 params['stop__exact'] = stop_exact |
|
666 else: |
|
667 if stop_gte is not None: |
|
668 params['stop__gte'] = stop_gte |
|
669 if stop_lte is not None: |
|
670 params['stop__lte'] = stop_lte |
|
671 |
|
672 if station_exact is not None: |
|
673 params['station__exact'] = station_exact |
|
674 elif station_in is not None: |
|
675 params['station__in'] = station_in |
614 |
676 |
615 response_json = self.session.get(self.api_measurements_url, params=params).text |
677 response_json = self.session.get(self.api_measurements_url, params=params).text |
616 |
678 |
617 return response_json |
679 return response_json |
618 |
680 |
746 self.eldec_exit_code = None |
808 self.eldec_exit_code = None |
747 self.elquick = None |
809 self.elquick = None |
748 self.elquick_exit_code = None |
810 self.elquick_exit_code = None |
749 |
811 |
750 self.id = None |
812 self.id = None |
|
813 self.num_id = None |
751 self.is_calibration = None |
814 self.is_calibration = None |
752 self.is_running = None |
815 self.is_running = None |
|
816 self.is_queued = None |
753 |
817 |
754 self.resource_uri = None |
818 self.resource_uri = None |
755 self.start = None |
819 self.start = None |
756 self.stop = None |
820 self.stop = None |
757 self.system = None |
821 self.system = None |
|
822 self.station = None |
758 self.upload = None |
823 self.upload = None |
759 |
824 |
760 super().__init__(base_url, dict_response) |
825 super().__init__(base_url, dict_response) |
761 |
826 |
762 @property |
827 @property |
768 else: |
833 else: |
769 return False |
834 return False |
770 |
835 |
771 def log_processing_status(self): |
836 def log_processing_status(self): |
772 """ Log module status. """ |
837 """ Log module status. """ |
773 logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}). Please wait.".format( |
838 logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}, {}). Please wait.".format( |
774 self.upload, |
839 self.upload, |
775 self.hirelpp, |
840 self.hirelpp, |
776 self.cloudmask, |
841 self.cloudmask, |
777 self.elpp, |
842 self.elpp, |
778 self.elda, |
843 self.elda, |
779 self.elic)) |
844 self.elic, |
|
845 self.elquick)) |
780 |
846 |
781 def log_detailed_status(self): |
847 def log_detailed_status(self): |
782 """ Log module exit and status codes.""" |
848 """ Log module exit and status codes.""" |
783 logger.info("Measurement exit status:".format(self.id)) |
849 logger.info("Measurement exit status:".format(self.id)) |
784 if self.is_calibration: |
850 if self.is_calibration: |
887 logger.info("Rerunning (optical) processing for %s" % m_id) |
953 logger.info("Rerunning (optical) processing for %s" % m_id) |
888 scc.rerun_elpp(m_id, monitor) |
954 scc.rerun_elpp(m_id, monitor) |
889 scc.logout() |
955 scc.logout() |
890 |
956 |
891 |
957 |
892 def list_measurements(settings, id_exact=None, id_startswith=None): |
958 def list_measurements(settings, id_exact=None, id_startswith=None, |
|
959 start_exact=None, start_gte=None, start_lte=None, |
|
960 stop_exact=None, stop_gte=None, stop_lte=None, |
|
961 station_exact=None, station_in=None): |
893 """List all available measurements""" |
962 """List all available measurements""" |
894 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
963 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
895 scc.login(settings['website_credentials']) |
964 scc.login(settings['website_credentials']) |
896 |
965 |
897 results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith) |
966 results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith, |
|
967 start_exact=start_exact, start_gte=start_gte, start_lte=start_lte, |
|
968 stop_exact=stop_exact, stop_gte=stop_gte, stop_lte=stop_lte, |
|
969 station_exact=station_exact, station_in=station_in) |
898 print(results_json) |
970 print(results_json) |
899 |
971 |
900 scc.logout() |
972 scc.logout() |
901 |
973 |
902 |
974 |
906 scc.login(settings['website_credentials']) |
978 scc.login(settings['website_credentials']) |
907 for m_id in measurement_ids: |
979 for m_id in measurement_ids: |
908 scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing) |
980 scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing) |
909 |
981 |
910 scc.logout() |
982 scc.logout() |
|
983 |
|
984 |
|
985 def eshape_downloader(settings): |
|
986 eshape_dir = settings['output_dir'] |
|
987 #directories = [[x[0] for x in os.walk(eshape_dir)]] |
|
988 all_directories = [f.name for f in os.scandir(eshape_dir) if f.is_dir()] |
|
989 folders_pattern = re.compile("^(\d{6}_{1}\d{12})+$") |
|
990 directories = [dir_name for dir_name in all_directories if folders_pattern.match(dir_name)] |
|
991 for dir_name in directories: |
|
992 start_string = dir_name.split("_")[1] |
|
993 date_time_start = datetime.datetime.strptime(start_string, '%Y%m%d%H%M') |
|
994 if (datetime.datetime.now() - datetime.timedelta(days=3)) <= date_time_start: |
|
995 start_parameter = date_time_start.strftime("%Y-%m-%dT%H:%M:%S") |
|
996 date_time_stop = datetime.datetime.now() |
|
997 stop_parameter = date_time_stop.strftime("%Y-%m-%dT%H:%M:%S") |
|
998 |
|
999 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
|
1000 scc.login(settings['website_credentials']) |
|
1001 |
|
1002 measurements = scc.available_measurements(start_gte=start_parameter, stop_lte=stop_parameter) |
|
1003 if measurements is not None: |
|
1004 for meas in measurements: |
|
1005 scc.download_products(meas, dir_name) |
|
1006 |
|
1007 scc.logout() |
911 |
1008 |
912 |
1009 |
913 def settings_from_path(config_file_path): |
1010 def settings_from_path(config_file_path): |
914 """ Read the configuration file. |
1011 """ Read the configuration file. |
915 |
1012 |
1001 parser.set_defaults(execute=upload_file_from_args) |
1098 parser.set_defaults(execute=upload_file_from_args) |
1002 |
1099 |
1003 |
1100 |
1004 def setup_list_measurements(parser): |
1101 def setup_list_measurements(parser): |
1005 def list_measurements_from_args(parsed): |
1102 def list_measurements_from_args(parsed): |
1006 list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith) |
1103 list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith, |
1007 |
1104 start_exact=parsed.start_exact, start_gte=parsed.start_gte, start_lte=parsed.start_lte, |
1008 group = parser.add_mutually_exclusive_group() |
1105 stop_exact=parsed.stop_exact, stop_gte=parsed.stop_gte, stop_lte=parsed.stop_lte, |
1009 group.add_argument("--id_exact", help="Exact measurement id.") |
1106 station_exact=parsed.station_exact, station_in=parsed.station_in) |
1010 group.add_argument("--id_startswith", help="Initial part of measurement id.") |
1107 |
|
1108 group = parser.add_argument_group() |
|
1109 |
|
1110 group_id = group.add_mutually_exclusive_group() |
|
1111 group_id.add_argument("--id_exact", help="Exact measurement id.") |
|
1112 group_id.add_argument("--id_startswith", help="Initial part of measurement id.") |
|
1113 |
|
1114 group_start = group.add_argument_group() |
|
1115 group_start.add_argument("--start_exact", help="Exact start date of the measurement.") |
|
1116 group_start.add_argument("--start_gte", help="Start date of the measurement after the given date.") |
|
1117 group_start.add_argument("--start_lte", help="Start date of the measurement before the given date.") |
|
1118 |
|
1119 group_stop = group.add_argument_group() |
|
1120 group_stop.add_argument("--stop_exact", help="Exact stop date of the measurement.") |
|
1121 group_stop.add_argument("--stop_gte", help="Stop date of the measurement after the given date.") |
|
1122 group_stop.add_argument("--stop_lte", help="Stop date of the measurement before the given date.") |
|
1123 |
|
1124 group_station = group.add_mutually_exclusive_group() |
|
1125 group_station.add_argument("--station_exact", help="Station the performed the measurement.") |
|
1126 group_station.add_argument("--station_in", help="List of stations (separated by comma) that performed the measurement.") |
1011 |
1127 |
1012 parser.set_defaults(execute=list_measurements_from_args) |
1128 parser.set_defaults(execute=list_measurements_from_args) |
1013 |
1129 |
1014 |
1130 |
1015 def setup_download_measurements(parser): |
1131 def setup_download_measurements(parser): |
1020 parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, |
1136 parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, |
1021 type=int) |
1137 type=int) |
1022 parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", |
1138 parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", |
1023 action="store_false") |
1139 action="store_false") |
1024 parser.set_defaults(execute=download_measurements_from_args) |
1140 parser.set_defaults(execute=download_measurements_from_args) |
|
1141 |
|
1142 |
|
1143 def setup_eshape_downloader(parser): |
|
1144 def run_eshape_downloader(parsed): |
|
1145 eshape_downloader(parsed.config) |
|
1146 |
|
1147 parser.set_defaults(execute=run_eshape_downloader) |
1025 |
1148 |
1026 |
1149 |
1027 def main(): |
1150 def main(): |
1028 # Define the command line arguments. |
1151 # Define the command line arguments. |
1029 parser = argparse.ArgumentParser() |
1152 parser = argparse.ArgumentParser() |
1036 help="Rerun low-resolution processing steps for the provided measurement ID.") |
1159 help="Rerun low-resolution processing steps for the provided measurement ID.") |
1037 upload_file_parser = subparsers.add_parser("upload-file", |
1160 upload_file_parser = subparsers.add_parser("upload-file", |
1038 help="Submit a file and, optionally, download the output products.") |
1161 help="Submit a file and, optionally, download the output products.") |
1039 list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") |
1162 list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") |
1040 download_parser = subparsers.add_parser("download", help="Download selected measurements.") |
1163 download_parser = subparsers.add_parser("download", help="Download selected measurements.") |
|
1164 eshape_parser = subparsers.add_parser("eshape-downloader", help="Search and download relevant products for E-SHAPE.") |
1041 |
1165 |
1042 setup_delete(delete_parser) |
1166 setup_delete(delete_parser) |
1043 setup_rerun_all(rerun_all_parser) |
1167 setup_rerun_all(rerun_all_parser) |
1044 setup_rerun_elpp(rerun_processing_parser) |
1168 setup_rerun_elpp(rerun_processing_parser) |
1045 |
1169 |
1046 setup_upload_file(upload_file_parser) |
1170 setup_upload_file(upload_file_parser) |
1047 setup_list_measurements(list_parser) |
1171 setup_list_measurements(list_parser) |
1048 setup_download_measurements(download_parser) |
1172 setup_download_measurements(download_parser) |
|
1173 setup_eshape_downloader(eshape_parser) |
1049 |
1174 |
1050 # Verbosity settings from http://stackoverflow.com/a/20663028 |
1175 # Verbosity settings from http://stackoverflow.com/a/20663028 |
1051 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", |
1176 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", |
1052 dest="loglevel", const=logging.DEBUG, default=logging.INFO, |
1177 dest="loglevel", const=logging.DEBUG, default=logging.INFO, |
1053 ) |
1178 ) |