Mercurial > public > scc_access / file comparison
comparison: scc_access/scc_access.py
scc_access/scc_access.py
- branch
- development
- changeset 53
- 934a5e0a05c9
- parent 51
- a4ca2b6d67f5
- child 54
- 5a2e8efdaae9
equal
deleted
inserted
replaced
| 52:24dc95cbbf50 | 53:934a5e0a05c9 |
|---|---|
| 360 def monitor_processing(self, measurement_id): | 360 def monitor_processing(self, measurement_id): |
| 361 """ Monitor the processing progress of a measurement id""" | 361 """ Monitor the processing progress of a measurement id""" |
| 362 | 362 |
| 363 # try to deal with error 404 | 363 # try to deal with error 404 |
| 364 error_count = 0 | 364 error_count = 0 |
| 365 error_max = 6 | 365 error_max = 3 |
| 366 time_sleep = 10 | 366 time_sleep = 3 |
| 367 | 367 |
| 368 # try to wait for measurement to appear in API | 368 # try to wait for measurement to appear in API |
| 369 measurement = None | 369 measurement = None |
| 370 logger.info("Looking for measurement %s on the SCC.", measurement_id) | 370 logger.info("Looking for measurement %s on the SCC.", measurement_id) |
| 371 while error_count < error_max: | 371 while error_count < error_max: |
| 383 | 383 |
| 384 logger.info('Measurement %s found.', measurement_id) | 384 logger.info('Measurement %s found.', measurement_id) |
| 385 | 385 |
| 386 if measurement is not None: | 386 if measurement is not None: |
| 387 while measurement.is_running: | 387 while measurement.is_running: |
| 388 logger.info("Measurement is being processed. status: {}, {}, {}, {}, {}, {}). Please wait.".format( | 388 measurement.log_processing_status() |
| 389 measurement.upload, | |
| 390 measurement.hirelpp, | |
| 391 measurement.cloudmask, | |
| 392 measurement.elpp, | |
| 393 measurement.elda, | |
| 394 measurement.elic)) | |
| 395 time.sleep(10) | 389 time.sleep(10) |
| 396 measurement, status = self.get_measurement(measurement_id) | 390 measurement, status = self.get_measurement(measurement_id) |
| 397 | 391 |
| 398 logger.info("Measurement processing finished.") | 392 logger.info("Measurement processing finished.") |
| 393 measurement.log_detailed_status() | |
| 394 | |
| 399 if measurement.hirelpp == 127: | 395 if measurement.hirelpp == 127: |
| 400 logger.info("Downloading HiRElPP files.") | 396 logger.info("Downloading HiRElPP files.") |
| 401 self.download_hirelpp(measurement_id) | 397 self.download_hirelpp(measurement_id) |
| 402 if measurement.cloudmask == 127: | 398 if measurement.cloudmask == 127: |
| 403 logger.info("Downloading cloud screening files.") | 399 logger.info("Downloading cloud screening files.") |
| 411 logger.info("Downloading ELDA plots.") | 407 logger.info("Downloading ELDA plots.") |
| 412 self.download_plots(measurement_id) | 408 self.download_plots(measurement_id) |
| 413 if measurement.elic == 127: | 409 if measurement.elic == 127: |
| 414 logger.info("Downloading ELIC files.") | 410 logger.info("Downloading ELIC files.") |
| 415 self.download_elic(measurement_id) | 411 self.download_elic(measurement_id) |
| 416 | 412 if measurement.is_calibration and measurement.eldec==0: |
| 417 # TODO: Need to check ELDEC code (when it becomes available in the API) | |
| 418 if measurement.is_calibration: | |
| 419 logger.info("Downloading ELDEC files.") | 413 logger.info("Downloading ELDEC files.") |
| 420 self.download_eldec(measurement_id) | 414 self.download_eldec(measurement_id) |
| 421 logger.info("--- Processing finished. ---") | 415 logger.info("--- Processing finished. ---") |
| 422 | 416 |
| 423 return measurement | 417 return measurement |
| 658 """ | 652 """ |
| 659 | 653 |
| 660 def __init__(self, base_url, dict_response): | 654 def __init__(self, base_url, dict_response): |
| 661 | 655 |
| 662 # Define expected attributes to assist debugging | 656 # Define expected attributes to assist debugging |
| 657 | |
| 658 self.hirelpp = None | |
| 659 self.hirelpp_exit_code = None | |
| 663 self.cloudmask = None | 660 self.cloudmask = None |
| 661 self.cloudmask_exit_code = None | |
| 662 self.elpp = None | |
| 663 self.elpp_exit_code = None | |
| 664 self.elda = None | 664 self.elda = None |
| 665 self.elda_exit_code = None | |
| 665 self.elic = None | 666 self.elic = None |
| 666 self.elpp = None | 667 self.elic_exit_code = None |
| 667 self.hirelpp = None | 668 self.eldec = None |
| 669 self.eldec_exit_code = None | |
| 670 self.elquick = None | |
| 671 self.elquick_exit_code = None | |
| 672 | |
| 668 self.id = None | 673 self.id = None |
| 669 self.is_calibration = None | 674 self.is_calibration = None |
| 670 self.is_running = None | 675 self.is_running = None |
| 671 self.pre_processing_exit_code = None | 676 |
| 672 self.processing_exit_code = None | |
| 673 self.resource_uri = None | 677 self.resource_uri = None |
| 674 self.start = None | 678 self.start = None |
| 675 self.stop = None | 679 self.stop = None |
| 676 self.system = None | 680 self.system = None |
| 677 self.upload = None | 681 self.upload = None |
| 678 | 682 |
| 679 super().__init__(base_url, dict_response) | 683 super().__init__(base_url, dict_response) |
| 684 | |
| 685 def log_processing_status(self): | |
| 686 """ Log module status. """ | |
| 687 logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}). Please wait.".format( | |
| 688 self.upload, | |
| 689 self.hirelpp, | |
| 690 self.cloudmask, | |
| 691 self.elpp, | |
| 692 self.elda, | |
| 693 self.elic)) | |
| 694 | |
| 695 def log_detailed_status(self): | |
| 696 """ Log module exit and status codes.""" | |
| 697 logger.info("Measurement exit status:".format(self.id)) | |
| 698 if self.is_calibration: | |
| 699 self._log_module_status('ElPP', self.elpp, self.elpp_exit_code) | |
| 700 self._log_module_status('ElDEC', self.eldec, self.eldec_exit_code) | |
| 701 else: | |
| 702 self._log_module_status('HiRElPP', self.hirelpp, self.hirelpp_exit_code) | |
| 703 self._log_module_status('CloudScreen', self.cloudmask, self.cloudmask_exit_code) | |
| 704 self._log_module_status('ElPP', self.elpp, self.elpp_exit_code) | |
| 705 self._log_module_status('ELDA', self.elda, self.elda_exit_code) | |
| 706 self._log_module_status('ELIC', self.elic, self.elic_exit_code) | |
| 707 self._log_module_status('ELQuick', self.elquick, self.elquick_exit_code) | |
| 708 | |
| 709 def _log_module_status(self, name, status, exit_code): | |
| 710 if exit_code: | |
| 711 logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code)) | |
| 712 else: | |
| 713 logger.info("{0} exit code: {2}".format(name, status, exit_code)) | |
| 680 | 714 |
| 681 @property | 715 @property |
| 682 def rerun_elda_url(self): | 716 def rerun_elda_url(self): |
| 683 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') | 717 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') |
| 684 return url_pattern.format(self.id) | 718 return url_pattern.format(self.id) |
