Mercurial > public > scc_access / file comparison
comparison: scc_access/scc_access.py
scc_access/scc_access.py
- changeset 61
- c4e03940780a
- parent 60
- 2fe60db870df
- child 62
- 33a1ebad3f7e
equal
deleted
inserted
replaced
| 60:2fe60db870df | 61:c4e03940780a |
|---|---|
| 366 logger.info("Monitoring processing.") | 366 logger.info("Monitoring processing.") |
| 367 return self.monitor_processing(measurement_id) | 367 return self.monitor_processing(measurement_id) |
| 368 | 368 |
| 369 return None | 369 return None |
| 370 | 370 |
| 371 def monitor_processing(self, measurement_id): | 371 def monitor_processing(self, measurement_id, retry_max=2, time_sleep=2, exit_if_missing=True): |
| 372 """ Monitor the processing progress of a measurement id""" | 372 """ Monitor the processing progress of a measurement id""" |
| 373 | 373 |
| 374 # try to deal with error 404 | 374 # try to deal with error 404 |
| 375 error_count = 0 | 375 attempts_count = 0 |
| 376 error_max = 3 | 376 max_attempts = retry_max + 1 |
| 377 time_sleep = 3 | |
| 378 | 377 |
| 379 # try to wait for measurement to appear in API | 378 # try to wait for measurement to appear in API |
| 380 measurement = None | 379 measurement = None |
| 381 logger.info("Looking for measurement %s on the SCC.", measurement_id) | 380 logger.info("Looking for measurement %s on the SCC.", measurement_id) |
| 382 while error_count < error_max: | 381 while attempts_count < max_attempts: |
| 383 time.sleep(time_sleep) | 382 attempts_count += 1 |
| 384 measurement, status = self.get_measurement(measurement_id) | 383 measurement, status = self.get_measurement(measurement_id) |
| 385 if status != 200 and error_count < error_max: | 384 if status != 200: |
| 386 logger.error("Measurement not found. waiting %ds", time_sleep) | 385 logger.warning("Measurement not found.") |
| 387 error_count += 1 | 386 if attempts_count < max_attempts: |
| 387 logger.warning("Waiting %ds.", time_sleep) | |
| 388 time.sleep(time_sleep) | |
| 388 else: | 389 else: |
| 389 break | 390 break |
| 390 | 391 |
| 391 if error_count == error_max: | 392 if attempts_count == max_attempts: |
| 392 logger.critical("Measurement %s doesn't seem to exist", measurement_id) | 393 logger.error("Measurement %s doesn't seem to exist.", measurement_id) |
| 393 sys.exit(1) | 394 if exit_if_missing: |
| 395 sys.exit(1) | |
| 396 else: | |
| 397 return measurement | |
| 394 | 398 |
| 395 logger.info('Measurement %s found.', measurement_id) | 399 logger.info('Measurement %s found.', measurement_id) |
| 396 | 400 |
| 397 if measurement is not None: | 401 if measurement is not None: |
| 398 while not measurement.has_finished: | 402 while not measurement.has_finished: |
| 834 for entry in ret: | 838 for entry in ret: |
| 835 print("%s" % entry.id) | 839 print("%s" % entry.id) |
| 836 scc.logout() | 840 scc.logout() |
| 837 | 841 |
| 838 | 842 |
| 839 def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings): | 843 def download_measurements(measurement_ids, max_retries, exit_if_missing, settings): |
| 840 """Download all measurements for the specified IDs""" | 844 """Download all measurements for the specified IDs""" |
| 841 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: | 845 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
| 842 scc.login(settings['website_credentials']) | 846 scc.login(settings['website_credentials']) |
| 843 for m_id in measurement_ids: | 847 for m_id in measurement_ids: |
| 844 if download_preproc: | 848 scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing) |
| 845 logger.info("Downloading preprocessed files for '%s'" % m_id) | 849 |
| 846 scc.download_elpp(m_id) | |
| 847 logger.info("Complete") | |
| 848 if download_optical: | |
| 849 logger.info("Downloading optical files for '%s'" % m_id) | |
| 850 scc.download_elda(m_id) | |
| 851 logger.info("Complete") | |
| 852 if download_graph: | |
| 853 logger.info("Downloading profile graph files for '%s'" % m_id) | |
| 854 scc.download_plots(m_id) | |
| 855 logger.info("Complete") | |
| 856 scc.logout() | 850 scc.logout() |
| 857 | 851 |
| 858 | 852 |
| 859 def settings_from_path(config_file_path): | 853 def settings_from_path(config_file_path): |
| 860 """ Read the configuration file. | 854 """ Read the configuration file. |
| 978 parser.set_defaults(execute=list_measurements_from_args) | 972 parser.set_defaults(execute=list_measurements_from_args) |
| 979 | 973 |
| 980 | 974 |
| 981 def setup_download_measurements(parser): | 975 def setup_download_measurements(parser): |
| 982 def download_measurements_from_args(parsed): | 976 def download_measurements_from_args(parsed): |
| 983 # TODO: Fix this | 977 download_measurements(parsed.IDs, parsed.max_retries, parsed.ignore_errors, parsed.config) |
| 984 logger.warning("This method needs to be updated. Cross-check any results.") | |
| 985 | |
| 986 preproc = parsed.download_preprocessed | |
| 987 optical = parsed.download_optical | |
| 988 graphs = parsed.download_profile_graphs | |
| 989 if not preproc and not graphs: | |
| 990 optical = True | |
| 991 download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config) | |
| 992 | 978 |
| 993 parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+") | 979 parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+") |
| 994 parser.add_argument("--download-preprocessed", action="store_true", help="Download preprocessed files.") | 980 parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, type=int) |
| 995 parser.add_argument("--download-optical", action="store_true", | 981 parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", action="store_false") |
| 996 help="Download optical files (default if no other download is used).") | |
| 997 parser.add_argument("--download-profile-graphs", action="store_true", help="Download profile graph files.") | |
| 998 parser.set_defaults(execute=download_measurements_from_args) | 982 parser.set_defaults(execute=download_measurements_from_args) |
| 999 | 983 |
| 1000 | 984 |
| 1001 def main(): | 985 def main(): |
| 1002 # Define the command line arguments. | 986 # Define the command line arguments. |
