--- a/scc_access/scc_access.py Mon May 24 11:43:31 2021 +0300
+++ b/scc_access/scc_access.py Wed May 25 23:00:57 2022 +0200
@@ -493,6 +493,37 @@
return measurement
+ def download_products(self, measurement, dir_name):
+ """ Download all the products of a measurement id (used only for E-SHAPE"""
+ measurement_id = measurement.id
+ base_output_dir = self.output_dir
+ self.output_dir = self.output_dir + dir_name + "/"
+
+ if measurement.hirelpp == 127:
+ logger.info("Downloading HiRElPP files.")
+ self.download_hirelpp(measurement_id)
+ if measurement.cloudmask == 127:
+ logger.info("Downloading cloud screening files.")
+ self.download_cloudmask(measurement_id)
+ if measurement.elpp == 127:
+ logger.info("Downloading ElPP files.")
+ self.download_elpp(measurement_id)
+ if measurement.elda == 127:
+ logger.info("Downloading ELDA files.")
+ self.download_elda(measurement_id)
+ logger.info("Downloading ELDA plots.")
+ self.download_plots(measurement_id)
+ if measurement.elic == 127:
+ logger.info("Downloading ELIC files.")
+ self.download_elic(measurement_id)
+ if measurement.is_calibration and measurement.eldec == 0:
+ logger.info("Downloading ELDEC files.")
+ self.download_eldec(measurement_id)
+ logger.info("--- Processing finished. ---")
+
+ self.output_dir = base_output_dir
+ return measurement
+
def get_measurement(self, measurement_id):
""" Get a measurement information from the SCC API.
@@ -581,12 +612,19 @@
logger.info("Deleted measurement {0}".format(measurement_id))
return True
- def available_measurements(self):
+ def available_measurements(self, start_gte=None, stop_lte=None):
""" Get a list of available measurement on the SCC.
The methods is currently not used, could be merged with list_measurements.
"""
- response = self.session.get(self.api_measurements_url)
+
+ params = {}
+ if start_gte is not None:
+ params['start__gte'] = start_gte
+ if stop_lte is not None:
+ params['stop__lte'] = stop_lte
+
+ response = self.session.get(self.api_measurements_url, params=params)
response_dict = response.json()
if response_dict:
@@ -599,7 +637,10 @@
return measurements
- def list_measurements(self, id_exact=None, id_startswith=None):
+ def list_measurements(self, id_exact=None, id_startswith=None,
+ start_exact=None, start_gte=None, start_lte=None,
+ stop_exact=None, stop_gte=None, stop_lte=None,
+ station_exact=None, station_in=None):
""" Get the response text from the API. """
# TODO: Add some error handling, e.g. as per available_measurements method
@@ -609,8 +650,29 @@
if id_exact is not None:
params['id__exact'] = id_exact
+ elif id_startswith is not None:
+ params['id__startswith'] = id_startswith
+
+ if start_exact is not None:
+ params['start__exact'] = start_exact
else:
- params['id__startswith'] = id_startswith
+ if start_gte is not None:
+ params['start__gte'] = start_gte
+ if start_lte is not None:
+ params['start__lte'] = start_lte
+
+ if stop_exact is not None:
+ params['stop__exact'] = stop_exact
+ else:
+ if stop_gte is not None:
+ params['stop__gte'] = stop_gte
+ if stop_lte is not None:
+ params['stop__lte'] = stop_lte
+
+ if station_exact is not None:
+ params['station__exact'] = station_exact
+ elif station_in is not None:
+ params['station__in'] = station_in
response_json = self.session.get(self.api_measurements_url, params=params).text
@@ -748,13 +810,16 @@
self.elquick_exit_code = None
self.id = None
+ self.num_id = None
self.is_calibration = None
self.is_running = None
+ self.is_queued = None
self.resource_uri = None
self.start = None
self.stop = None
self.system = None
+ self.station = None
self.upload = None
super().__init__(base_url, dict_response)
@@ -770,13 +835,14 @@
def log_processing_status(self):
""" Log module status. """
- logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}). Please wait.".format(
+ logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}, {}). Please wait.".format(
self.upload,
self.hirelpp,
self.cloudmask,
self.elpp,
self.elda,
- self.elic))
+ self.elic,
+ self.elquick))
def log_detailed_status(self):
""" Log module exit and status codes."""
@@ -889,12 +955,18 @@
scc.logout()
-def list_measurements(settings, id_exact=None, id_startswith=None):
+def list_measurements(settings, id_exact=None, id_startswith=None,
+ start_exact=None, start_gte=None, start_lte=None,
+ stop_exact=None, stop_gte=None, stop_lte=None,
+ station_exact=None, station_in=None):
"""List all available measurements"""
with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
scc.login(settings['website_credentials'])
- results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith)
+ results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith,
+ start_exact=start_exact, start_gte=start_gte, start_lte=start_lte,
+ stop_exact=stop_exact, stop_gte=stop_gte, stop_lte=stop_lte,
+ station_exact=station_exact, station_in=station_in)
print(results_json)
scc.logout()
@@ -910,6 +982,31 @@
scc.logout()
+def eshape_downloader(settings):
+ eshape_dir = settings['output_dir']
+ #directories = [[x[0] for x in os.walk(eshape_dir)]]
+ all_directories = [f.name for f in os.scandir(eshape_dir) if f.is_dir()]
+ folders_pattern = re.compile("^(\d{6}_{1}\d{12})+$")
+ directories = [dir_name for dir_name in all_directories if folders_pattern.match(dir_name)]
+ for dir_name in directories:
+ start_string = dir_name.split("_")[1]
+ date_time_start = datetime.datetime.strptime(start_string, '%Y%m%d%H%M')
+ if (datetime.datetime.now() - datetime.timedelta(days=3)) <= date_time_start:
+ start_parameter = date_time_start.strftime("%Y-%m-%dT%H:%M:%S")
+ date_time_stop = datetime.datetime.now()
+ stop_parameter = date_time_stop.strftime("%Y-%m-%dT%H:%M:%S")
+
+ with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
+ scc.login(settings['website_credentials'])
+
+ measurements = scc.available_measurements(start_gte=start_parameter, stop_lte=stop_parameter)
+ if measurements is not None:
+ for meas in measurements:
+ scc.download_products(meas, dir_name)
+
+ scc.logout()
+
+
def settings_from_path(config_file_path):
""" Read the configuration file.
@@ -1003,11 +1100,30 @@
def setup_list_measurements(parser):
def list_measurements_from_args(parsed):
- list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith)
+ list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith,
+ start_exact=parsed.start_exact, start_gte=parsed.start_gte, start_lte=parsed.start_lte,
+ stop_exact=parsed.stop_exact, stop_gte=parsed.stop_gte, stop_lte=parsed.stop_lte,
+ station_exact=parsed.station_exact, station_in=parsed.station_in)
+
+ group = parser.add_argument_group()
+
+ group_id = group.add_mutually_exclusive_group()
+ group_id.add_argument("--id_exact", help="Exact measurement id.")
+ group_id.add_argument("--id_startswith", help="Initial part of measurement id.")
- group = parser.add_mutually_exclusive_group()
- group.add_argument("--id_exact", help="Exact measurement id.")
- group.add_argument("--id_startswith", help="Initial part of measurement id.")
+ group_start = group.add_argument_group()
+ group_start.add_argument("--start_exact", help="Exact start date of the measurement.")
+ group_start.add_argument("--start_gte", help="Start date of the measurement after the given date.")
+ group_start.add_argument("--start_lte", help="Start date of the measurement before the given date.")
+
+ group_stop = group.add_argument_group()
+ group_stop.add_argument("--stop_exact", help="Exact stop date of the measurement.")
+ group_stop.add_argument("--stop_gte", help="Stop date of the measurement after the given date.")
+ group_stop.add_argument("--stop_lte", help="Stop date of the measurement before the given date.")
+
+ group_station = group.add_mutually_exclusive_group()
+ group_station.add_argument("--station_exact", help="Station the performed the measurement.")
+ group_station.add_argument("--station_in", help="List of stations (separated by comma) that performed the measurement.")
parser.set_defaults(execute=list_measurements_from_args)
@@ -1024,6 +1140,13 @@
parser.set_defaults(execute=download_measurements_from_args)
+def setup_eshape_downloader(parser):
+ def run_eshape_downloader(parsed):
+ eshape_downloader(parsed.config)
+
+ parser.set_defaults(execute=run_eshape_downloader)
+
+
def main():
# Define the command line arguments.
parser = argparse.ArgumentParser()
@@ -1038,6 +1161,7 @@
help="Submit a file and, optionally, download the output products.")
list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
download_parser = subparsers.add_parser("download", help="Download selected measurements.")
+ eshape_parser = subparsers.add_parser("eshape-downloader", help="Search and download relevant products for E-SHAPE.")
setup_delete(delete_parser)
setup_rerun_all(rerun_all_parser)
@@ -1046,6 +1170,7 @@
setup_upload_file(upload_file_parser)
setup_list_measurements(list_parser)
setup_download_measurements(download_parser)
+ setup_eshape_downloader(eshape_parser)
# Verbosity settings from http://stackoverflow.com/a/20663028
parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
