--- a/scc_access/scc_access.py Thu Apr 22 15:13:13 2021 +0300
+++ b/scc_access/scc_access.py Mon Apr 26 12:15:02 2021 +0300
@@ -368,29 +368,33 @@
return None
- def monitor_processing(self, measurement_id):
+ def monitor_processing(self, measurement_id, retry_max=2, time_sleep=2, exit_if_missing=True):
""" Monitor the processing progress of a measurement id"""
# try to deal with error 404
- error_count = 0
- error_max = 3
- time_sleep = 3
+ attempts_count = 0
+ max_attempts = retry_max + 1
# try to wait for measurement to appear in API
measurement = None
logger.info("Looking for measurement %s on the SCC.", measurement_id)
- while error_count < error_max:
- time.sleep(time_sleep)
+ while attempts_count < max_attempts:
+ attempts_count += 1
measurement, status = self.get_measurement(measurement_id)
- if status != 200 and error_count < error_max:
- logger.error("Measurement not found. waiting %ds", time_sleep)
- error_count += 1
+ if status != 200:
+ logger.warning("Measurement not found.")
+ if attempts_count < max_attempts:
+ logger.warning("Waiting %ds.", time_sleep)
+ time.sleep(time_sleep)
else:
break
- if error_count == error_max:
- logger.critical("Measurement %s doesn't seem to exist", measurement_id)
- sys.exit(1)
+ if attempts_count == max_attempts:
+ logger.error("Measurement %s doesn't seem to exist.", measurement_id)
+ if exit_if_missing:
+ sys.exit(1)
+ else:
+ return measurement
logger.info('Measurement %s found.', measurement_id)
@@ -836,23 +840,13 @@
scc.logout()
-def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings):
+def download_measurements(measurement_ids, max_retries, exit_if_missing, settings):
"""Download all measurements for the specified IDs"""
with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
scc.login(settings['website_credentials'])
for m_id in measurement_ids:
- if download_preproc:
- logger.info("Downloading preprocessed files for '%s'" % m_id)
- scc.download_elpp(m_id)
- logger.info("Complete")
- if download_optical:
- logger.info("Downloading optical files for '%s'" % m_id)
- scc.download_elda(m_id)
- logger.info("Complete")
- if download_graph:
- logger.info("Downloading profile graph files for '%s'" % m_id)
- scc.download_plots(m_id)
- logger.info("Complete")
+ scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing)
+
scc.logout()
@@ -980,21 +974,11 @@
def setup_download_measurements(parser):
def download_measurements_from_args(parsed):
- # TODO: Fix this
- logger.warning("This method needs to be updated. Cross-check any results.")
-
- preproc = parsed.download_preprocessed
- optical = parsed.download_optical
- graphs = parsed.download_profile_graphs
- if not preproc and not graphs:
- optical = True
- download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config)
+ download_measurements(parsed.IDs, parsed.max_retries, parsed.ignore_errors, parsed.config)
parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+")
- parser.add_argument("--download-preprocessed", action="store_true", help="Download preprocessed files.")
- parser.add_argument("--download-optical", action="store_true",
- help="Download optical files (default if no other download is used).")
- parser.add_argument("--download-profile-graphs", action="store_true", help="Download profile graph files.")
+ parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, type=int)
+ parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", action="store_false")
parser.set_defaults(execute=download_measurements_from_args)
