Mercurial > public > scc_access / changeset

--- a/scc_access/scc_access.py	Thu Apr 22 15:13:13 2021 +0300
+++ b/scc_access/scc_access.py	Mon Apr 26 12:15:02 2021 +0300
@@ -368,29 +368,33 @@
 
         return None
 
-    def monitor_processing(self, measurement_id):
+    def monitor_processing(self, measurement_id, retry_max=2, time_sleep=2, exit_if_missing=True):
         """ Monitor the processing progress of a measurement id"""
 
         # try to deal with error 404
-        error_count = 0
-        error_max = 3
-        time_sleep = 3
+        attempts_count = 0
+        max_attempts = retry_max + 1
 
         # try to wait for measurement to appear in API
         measurement = None
         logger.info("Looking for measurement %s on the SCC.", measurement_id)
-        while error_count < error_max:
-            time.sleep(time_sleep)
+        while attempts_count < max_attempts:
+            attempts_count += 1
             measurement, status = self.get_measurement(measurement_id)
-            if status != 200 and error_count < error_max:
-                logger.error("Measurement not found. waiting %ds", time_sleep)
-                error_count += 1
+            if status != 200:
+                logger.warning("Measurement not found.")
+                if attempts_count < max_attempts:
+                    logger.warning("Waiting %ds.", time_sleep)
+                    time.sleep(time_sleep)
             else:
                 break
 
-        if error_count == error_max:
-            logger.critical("Measurement %s doesn't seem to exist", measurement_id)
-            sys.exit(1)
+        if attempts_count == max_attempts:
+            logger.error("Measurement %s doesn't seem to exist.", measurement_id)
+            if exit_if_missing:
+                sys.exit(1)
+            else:
+                return measurement
 
         logger.info('Measurement %s found.', measurement_id)
 
@@ -836,23 +840,13 @@
         scc.logout()
 
 
-def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings):
+def download_measurements(measurement_ids, max_retries, exit_if_missing, settings):
     """Download all measurements for the specified IDs"""
     with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
         scc.login(settings['website_credentials'])
         for m_id in measurement_ids:
-            if download_preproc:
-                logger.info("Downloading preprocessed files for '%s'" % m_id)
-                scc.download_elpp(m_id)
-                logger.info("Complete")
-            if download_optical:
-                logger.info("Downloading optical files for '%s'" % m_id)
-                scc.download_elda(m_id)
-                logger.info("Complete")
-            if download_graph:
-                logger.info("Downloading profile graph files for '%s'" % m_id)
-                scc.download_plots(m_id)
-                logger.info("Complete")
+            scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing)
+
         scc.logout()
 
 
@@ -980,21 +974,11 @@
 
 def setup_download_measurements(parser):
     def download_measurements_from_args(parsed):
-        # TODO: Fix this
-        logger.warning("This method needs to be updated. Cross-check any results.")
-
-        preproc = parsed.download_preprocessed
-        optical = parsed.download_optical
-        graphs = parsed.download_profile_graphs
-        if not preproc and not graphs:
-            optical = True
-        download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config)
+        download_measurements(parsed.IDs, parsed.max_retries, parsed.ignore_errors, parsed.config)
 
     parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+")
-    parser.add_argument("--download-preprocessed", action="store_true", help="Download preprocessed files.")
-    parser.add_argument("--download-optical", action="store_true",
-                        help="Download optical files (default if no other download is used).")
-    parser.add_argument("--download-profile-graphs", action="store_true", help="Download profile graph files.")
+    parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, type=int)
+    parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", action="store_false")
     parser.set_defaults(execute=download_measurements_from_args)