Mercurial > public > scc_access / changeset

--- a/.hgignore	Mon May 24 11:43:31 2021 +0300
+++ b/.hgignore	Wed May 25 23:00:57 2022 +0200
@@ -7,3 +7,5 @@
 scc_access.egg-info/
 re:^\.pytest_cache/
 dist/
+settings.yaml
+

--- a/scc_access/scc_access.py	Mon May 24 11:43:31 2021 +0300
+++ b/scc_access/scc_access.py	Wed May 25 23:00:57 2022 +0200
@@ -493,6 +493,37 @@
 
         return measurement
 
+    def download_products(self, measurement, dir_name):
+        """ Download all the products of a measurement id (used only for E-SHAPE"""
+        measurement_id = measurement.id
+        base_output_dir = self.output_dir
+        self.output_dir = self.output_dir + dir_name + "/"
+
+        if measurement.hirelpp == 127:
+            logger.info("Downloading HiRElPP files.")
+            self.download_hirelpp(measurement_id)
+        if measurement.cloudmask == 127:
+            logger.info("Downloading cloud screening files.")
+            self.download_cloudmask(measurement_id)
+        if measurement.elpp == 127:
+            logger.info("Downloading ElPP files.")
+            self.download_elpp(measurement_id)
+        if measurement.elda == 127:
+            logger.info("Downloading ELDA files.")
+            self.download_elda(measurement_id)
+            logger.info("Downloading ELDA plots.")
+            self.download_plots(measurement_id)
+        if measurement.elic == 127:
+            logger.info("Downloading ELIC files.")
+            self.download_elic(measurement_id)
+        if measurement.is_calibration and measurement.eldec == 0:
+            logger.info("Downloading ELDEC files.")
+            self.download_eldec(measurement_id)
+        logger.info("--- Processing finished. ---")
+
+        self.output_dir = base_output_dir
+        return measurement
+
     def get_measurement(self, measurement_id):
         """ Get a measurement information from the SCC API.
 
@@ -581,12 +612,19 @@
         logger.info("Deleted measurement {0}".format(measurement_id))
         return True
 
-    def available_measurements(self):
+    def available_measurements(self, start_gte=None, stop_lte=None):
         """ Get a list of available measurement on the SCC.
 
         The methods is currently not used, could be merged with list_measurements.
         """
-        response = self.session.get(self.api_measurements_url)
+
+        params = {}
+        if start_gte is not None:
+            params['start__gte'] = start_gte
+        if stop_lte is not None:
+            params['stop__lte'] = stop_lte
+
+        response = self.session.get(self.api_measurements_url, params=params)
         response_dict = response.json()
 
         if response_dict:
@@ -599,7 +637,10 @@
 
         return measurements
 
-    def list_measurements(self, id_exact=None, id_startswith=None):
+    def list_measurements(self, id_exact=None, id_startswith=None,
+                          start_exact=None, start_gte=None, start_lte=None,
+                          stop_exact=None, stop_gte=None, stop_lte=None,
+                          station_exact=None, station_in=None):
         """ Get the response text from the API. """
 
         # TODO: Add some error handling, e.g. as per available_measurements method
@@ -609,8 +650,29 @@
 
         if id_exact is not None:
             params['id__exact'] = id_exact
+        elif id_startswith is not None:
+            params['id__startswith'] = id_startswith
+
+        if start_exact is not None:
+            params['start__exact'] = start_exact
         else:
-            params['id__startswith'] = id_startswith
+            if start_gte is not None:
+                params['start__gte'] = start_gte
+            if start_lte is not None:
+                params['start__lte'] = start_lte
+
+        if stop_exact is not None:
+            params['stop__exact'] = stop_exact
+        else:
+            if stop_gte is not None:
+                params['stop__gte'] = stop_gte
+            if stop_lte is not None:
+                params['stop__lte'] = stop_lte
+
+        if station_exact is not None:
+            params['station__exact'] = station_exact
+        elif station_in is not None:
+            params['station__in'] = station_in
 
         response_json = self.session.get(self.api_measurements_url, params=params).text
 
@@ -748,13 +810,16 @@
         self.elquick_exit_code = None
 
         self.id = None
+        self.num_id = None
         self.is_calibration = None
         self.is_running = None
+        self.is_queued = None
 
         self.resource_uri = None
         self.start = None
         self.stop = None
         self.system = None
+        self.station = None
         self.upload = None
 
         super().__init__(base_url, dict_response)
@@ -770,13 +835,14 @@
 
     def log_processing_status(self):
         """ Log module status. """
-        logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}). Please wait.".format(
+        logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}, {}). Please wait.".format(
             self.upload,
             self.hirelpp,
             self.cloudmask,
             self.elpp,
             self.elda,
-            self.elic))
+            self.elic,
+            self.elquick))
 
     def log_detailed_status(self):
         """ Log module exit and status codes."""
@@ -889,12 +955,18 @@
         scc.logout()
 
 
-def list_measurements(settings, id_exact=None, id_startswith=None):
+def list_measurements(settings, id_exact=None, id_startswith=None,
+                      start_exact=None, start_gte=None, start_lte=None,
+                      stop_exact=None, stop_gte=None, stop_lte=None,
+                      station_exact=None, station_in=None):
     """List all available measurements"""
     with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
         scc.login(settings['website_credentials'])
 
-        results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith)
+        results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith,
+                                            start_exact=start_exact, start_gte=start_gte, start_lte=start_lte,
+                                            stop_exact=stop_exact, stop_gte=stop_gte, stop_lte=stop_lte,
+                                            station_exact=station_exact, station_in=station_in)
         print(results_json)
 
         scc.logout()
@@ -910,6 +982,31 @@
         scc.logout()
 
 
+def eshape_downloader(settings):
+    eshape_dir = settings['output_dir']
+    #directories = [[x[0] for x in os.walk(eshape_dir)]]
+    all_directories = [f.name for f in os.scandir(eshape_dir) if f.is_dir()]
+    folders_pattern = re.compile("^(\d{6}_{1}\d{12})+$")
+    directories = [dir_name for dir_name in all_directories if folders_pattern.match(dir_name)]
+    for dir_name in directories:
+        start_string = dir_name.split("_")[1]
+        date_time_start = datetime.datetime.strptime(start_string, '%Y%m%d%H%M')
+        if (datetime.datetime.now() - datetime.timedelta(days=3)) <= date_time_start:
+            start_parameter = date_time_start.strftime("%Y-%m-%dT%H:%M:%S")
+            date_time_stop = datetime.datetime.now()
+            stop_parameter = date_time_stop.strftime("%Y-%m-%dT%H:%M:%S")
+
+            with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
+                scc.login(settings['website_credentials'])
+
+                measurements = scc.available_measurements(start_gte=start_parameter, stop_lte=stop_parameter)
+                if measurements is not None:
+                    for meas in measurements:
+                        scc.download_products(meas, dir_name)
+
+                scc.logout()
+
+
 def settings_from_path(config_file_path):
     """ Read the configuration file.
 
@@ -1003,11 +1100,30 @@
 
 def setup_list_measurements(parser):
     def list_measurements_from_args(parsed):
-        list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith)
+        list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith,
+                          start_exact=parsed.start_exact, start_gte=parsed.start_gte, start_lte=parsed.start_lte,
+                          stop_exact=parsed.stop_exact, stop_gte=parsed.stop_gte, stop_lte=parsed.stop_lte,
+                          station_exact=parsed.station_exact, station_in=parsed.station_in)
+
+    group = parser.add_argument_group()
+
+    group_id = group.add_mutually_exclusive_group()
+    group_id.add_argument("--id_exact", help="Exact measurement id.")
+    group_id.add_argument("--id_startswith", help="Initial part of measurement id.")
 
-    group = parser.add_mutually_exclusive_group()
-    group.add_argument("--id_exact", help="Exact measurement id.")
-    group.add_argument("--id_startswith", help="Initial part of measurement id.")
+    group_start = group.add_argument_group()
+    group_start.add_argument("--start_exact", help="Exact start date of the measurement.")
+    group_start.add_argument("--start_gte", help="Start date of the measurement after the given date.")
+    group_start.add_argument("--start_lte", help="Start date of the measurement before the given date.")
+
+    group_stop = group.add_argument_group()
+    group_stop.add_argument("--stop_exact", help="Exact stop date of the measurement.")
+    group_stop.add_argument("--stop_gte", help="Stop date of the measurement after the given date.")
+    group_stop.add_argument("--stop_lte", help="Stop date of the measurement before the given date.")
+
+    group_station = group.add_mutually_exclusive_group()
+    group_station.add_argument("--station_exact", help="Station the performed the measurement.")
+    group_station.add_argument("--station_in", help="List of stations (separated by comma) that performed the measurement.")
 
     parser.set_defaults(execute=list_measurements_from_args)
 
@@ -1024,6 +1140,13 @@
     parser.set_defaults(execute=download_measurements_from_args)
 
 
+def setup_eshape_downloader(parser):
+    def run_eshape_downloader(parsed):
+        eshape_downloader(parsed.config)
+
+    parser.set_defaults(execute=run_eshape_downloader)
+
+
 def main():
     # Define the command line arguments.
     parser = argparse.ArgumentParser()
@@ -1038,6 +1161,7 @@
                                                help="Submit a file and, optionally, download the output products.")
     list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
     download_parser = subparsers.add_parser("download", help="Download selected measurements.")
+    eshape_parser = subparsers.add_parser("eshape-downloader", help="Search and download relevant products for E-SHAPE.")
 
     setup_delete(delete_parser)
     setup_rerun_all(rerun_all_parser)
@@ -1046,6 +1170,7 @@
     setup_upload_file(upload_file_parser)
     setup_list_measurements(list_parser)
     setup_download_measurements(download_parser)
+    setup_eshape_downloader(eshape_parser)
 
     # Verbosity settings from http://stackoverflow.com/a/20663028
     parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",