51 self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, |
55 self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, |
52 'data_processing/measurements/{0}/download-hirelpp/') |
56 'data_processing/measurements/{0}/download-hirelpp/') |
53 self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, |
57 self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, |
54 'data_processing/measurements/{0}/download-cloudmask/') |
58 'data_processing/measurements/{0}/download-cloudmask/') |
55 |
59 |
56 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, |
60 self.download_elpp_pattern = urlparse.urljoin(self.base_url, |
57 'data_processing/measurements/{0}/download-preprocessed/') |
61 'data_processing/measurements/{0}/download-preprocessed/') |
58 self.download_optical_pattern = urlparse.urljoin(self.base_url, |
62 self.download_elda_pattern = urlparse.urljoin(self.base_url, |
59 'data_processing/measurements/{0}/download-optical/') |
63 'data_processing/measurements/{0}/download-optical/') |
60 self.download_graph_pattern = urlparse.urljoin(self.base_url, |
64 self.download_plots_pattern = urlparse.urljoin(self.base_url, |
61 'data_processing/measurements/{0}/download-plots/') |
65 'data_processing/measurements/{0}/download-plots/') |
62 self.download_elic_pattern = urlparse.urljoin(self.base_url, |
66 self.download_elic_pattern = urlparse.urljoin(self.base_url, |
63 'data_processing/measurements/{0}/download-elic/') |
67 'data_processing/measurements/{0}/download-elic/') |
64 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') |
68 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') |
65 |
69 |
190 |
229 |
191 with open(local_file, 'wb') as f: |
230 with open(local_file, 'wb') as f: |
192 f.write(zip_file.read(ziped_name)) |
231 f.write(zip_file.read(ziped_name)) |
193 |
232 |
194 def download_hirelpp(self, measurement_id): |
233 def download_hirelpp(self, measurement_id): |
195 """ Download HiRElPP files for the measurement id. """ |
234 """ Download hirelpp files for the measurement id. """ |
196 # Construct the download url |
235 # Construct the download url |
197 download_url = self.download_hirelpp_pattern.format(measurement_id) |
236 download_url = self.download_hirelpp_pattern.format(measurement_id) |
198 self.download_files(measurement_id, 'hirelpp', download_url) |
237 try: |
|
238 self.download_files(measurement_id, 'scc_hirelpp', download_url) |
|
239 except Exception as e: |
|
240 logger.error("Could not download HiRElPP files. Error message: {}".format(e)) |
|
241 logger.debug('Download exception:', exc_info=True) |
199 |
242 |
200 def download_cloudmask(self, measurement_id): |
243 def download_cloudmask(self, measurement_id): |
|
244 """ Download cloudmask files for the measurement id. """ |
|
245 # Construct the download url |
|
246 download_url = self.download_cloudmask_pattern.format(measurement_id) |
|
247 try: |
|
248 self.download_files(measurement_id, 'scc_cloudscreen', download_url) |
|
249 except Exception as e: |
|
250 logger.error("Could not download cloudscreen files. Error message: {}".format(e)) |
|
251 logger.debug('Download exception:', exc_info=True) |
|
252 |
|
253 def download_elpp(self, measurement_id): |
201 """ Download preprocessed files for the measurement id. """ |
254 """ Download preprocessed files for the measurement id. """ |
202 # Construct the download url |
255 # Construct the download url |
203 download_url = self.download_cloudmask_pattern.format(measurement_id) |
256 download_url = self.download_elpp_pattern.format(measurement_id) |
204 self.download_files(measurement_id, 'cloudmask', download_url) |
257 try: |
205 |
258 self.download_files(measurement_id, 'scc_preprocessed', download_url) |
206 def download_preprocessed(self, measurement_id): |
259 except Exception as e: |
207 """ Download preprocessed files for the measurement id. """ |
260 logger.error("Could not download ElPP files. Error message: {}".format(e)) |
208 # Construct the download url |
261 logger.debug('Download exception:', exc_info=True) |
209 download_url = self.download_preprocessed_pattern.format(measurement_id) |
262 |
210 self.download_files(measurement_id, 'scc_preprocessed', download_url) |
263 def download_elda(self, measurement_id): |
211 |
|
212 def download_optical(self, measurement_id): |
|
213 """ Download optical files for the measurement id. """ |
264 """ Download optical files for the measurement id. """ |
214 # Construct the download url |
265 # Construct the download url |
215 download_url = self.download_optical_pattern.format(measurement_id) |
266 download_url = self.download_elda_pattern.format(measurement_id) |
216 self.download_files(measurement_id, 'scc_optical', download_url) |
267 try: |
217 |
268 self.download_files(measurement_id, 'scc_optical', download_url) |
218 def download_graphs(self, measurement_id): |
269 except Exception as e: |
|
270 logger.error("Could not download ELDA files. Error message: {}".format(e)) |
|
271 logger.debug('Download exception:', exc_info=True) |
|
272 |
|
273 def download_plots(self, measurement_id): |
219 """ Download profile graphs for the measurement id. """ |
274 """ Download profile graphs for the measurement id. """ |
220 # Construct the download url |
275 # Construct the download url |
221 download_url = self.download_graph_pattern.format(measurement_id) |
276 download_url = self.download_plots_pattern.format(measurement_id) |
222 self.download_files(measurement_id, 'scc_plots', download_url) |
277 try: |
|
278 self.download_files(measurement_id, 'scc_plots', download_url) |
|
279 except Exception as e: |
|
280 logger.error("Could not download ELDA plots. Error message: {}".format(e)) |
|
281 logger.debug('Download exception:', exc_info=True) |
223 |
282 |
224 def download_elic(self, measurement_id): |
283 def download_elic(self, measurement_id): |
225 """ Download profile graphs for the measurement id. """ |
284 """ Download ELIC files for the measurement id. """ |
226 # Construct the download url |
285 # Construct the download url |
227 download_url = self.download_elic_pattern.format(measurement_id) |
286 download_url = self.download_elic_pattern.format(measurement_id) |
228 self.download_files(measurement_id, 'elic', download_url) |
287 try: |
229 |
288 self.download_files(measurement_id, 'scc_elic', download_url) |
230 def rerun_processing(self, measurement_id, monitor=True): |
289 except Exception as e: |
|
290 logger.error("Could not download ELIC files. Error message: {}".format(e)) |
|
291 logger.debug('Download exception:', exc_info=True) |
|
292 |
|
293 def download_eldec(self, measurement_id): |
|
294 """ Download ELDEC files for the measurement id. """ |
|
295 # Construct the download url |
|
296 download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now |
|
297 try: |
|
298 self.download_files(measurement_id, 'scc_eldec', download_url) |
|
299 except Exception as e: |
|
300 logger.error("Could not download EDELC files. Error message: {}".format(e)) |
|
301 logger.debug('Download exception:', exc_info=True) |
|
302 |
|
303 def rerun_elpp(self, measurement_id, monitor=True): |
|
304 logger.debug("Started rerun_elpp procedure.") |
|
305 |
|
306 logger.debug("Getting measurement %s" % measurement_id) |
231 measurement, status = self.get_measurement(measurement_id) |
307 measurement, status = self.get_measurement(measurement_id) |
232 |
308 |
233 if measurement: |
309 if measurement: |
234 request = self.session.get(measurement.rerun_processing_url, stream=True) |
310 logger.debug("Attempting to rerun ElPP through %s." % measurement.rerun_all_url) |
|
311 request = self.session.get(measurement.rerun_elpp_url, stream=True) |
235 |
312 |
236 if request.status_code != 200: |
313 if request.status_code != 200: |
237 logger.error( |
314 logger.error( |
238 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) |
315 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) |
239 return |
316 else: |
|
317 logger.info("Rerun-elpp command submitted successfully for id {}.".format(measurement_id)) |
240 |
318 |
241 if monitor: |
319 if monitor: |
242 self.monitor_processing(measurement_id) |
320 self.monitor_processing(measurement_id) |
243 |
321 |
244 def rerun_all(self, measurement_id, monitor=True): |
322 def rerun_all(self, measurement_id, monitor=True): |
253 request = self.session.get(measurement.rerun_all_url, stream=True) |
331 request = self.session.get(measurement.rerun_all_url, stream=True) |
254 |
332 |
255 if request.status_code != 200: |
333 if request.status_code != 200: |
256 logger.error("Could not rerun pre processing for %s. Status code: %s" % |
334 logger.error("Could not rerun pre processing for %s. Status code: %s" % |
257 (measurement_id, request.status_code)) |
335 (measurement_id, request.status_code)) |
258 return |
336 else: |
|
337 logger.info("Rerun-all command submitted successfully for id {}.".format(measurement_id)) |
259 |
338 |
260 if monitor: |
339 if monitor: |
261 self.monitor_processing(measurement_id) |
340 self.monitor_processing(measurement_id) |
262 |
341 |
263 def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None): |
342 def process(self, filename, system_id, monitor, force_upload, delete_related, rs_filename=None, lr_filename=None, ov_filename=None): |
264 """ Upload a file for processing and wait for the processing to finish. |
343 """ Upload a file for processing and wait for the processing to finish. |
265 If the processing is successful, it will download all produced files. |
344 If the processing is successful, it will download all produced files. |
266 """ |
345 """ |
267 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) |
346 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) |
268 # Upload file |
347 # Upload file |
269 logger.info("--- Uploading file") |
348 logger.info("--- Uploading file") |
270 measurement_id = self.upload_file(filename, system_id, |
349 measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, |
271 rs_filename=rs_filename, |
350 rs_filename=rs_filename, |
272 lr_filename=lr_filename, |
351 lr_filename=lr_filename, |
273 ov_filename=ov_filename) |
352 ov_filename=ov_filename) |
274 |
353 |
275 if measurement_id and monitor: |
354 if measurement_id and monitor: |
300 |
379 |
301 if error_count == error_max: |
380 if error_count == error_max: |
302 logger.critical("Measurement %s doesn't seem to exist", measurement_id) |
381 logger.critical("Measurement %s doesn't seem to exist", measurement_id) |
303 sys.exit(1) |
382 sys.exit(1) |
304 |
383 |
305 logger.info('Measurement %s found', measurement_id) |
384 logger.info('Measurement %s found.', measurement_id) |
306 |
385 |
307 if measurement is not None: |
386 if measurement is not None: |
308 while measurement.is_running: |
387 while measurement.is_running: |
309 logger.info("Measurement is being processed. Please wait.") |
388 logger.info("Measurement is being processed. status: {}, {}, {}, {}, {}, {}). Please wait.".format( |
|
389 measurement.upload, |
|
390 measurement.hirelpp, |
|
391 measurement.cloudmask, |
|
392 measurement.elpp, |
|
393 measurement.elda, |
|
394 measurement.elic)) |
310 time.sleep(10) |
395 time.sleep(10) |
311 measurement, status = self.get_measurement(measurement_id) |
396 measurement, status = self.get_measurement(measurement_id) |
312 |
397 |
313 logger.info("Measurement processing finished.") |
398 logger.info("Measurement processing finished.") |
314 if measurement.hirelpp == 127: |
399 if measurement.hirelpp == 127: |
315 logger.info("Downloading hirelpp files.") |
400 logger.info("Downloading HiRElPP files.") |
316 self.download_hirelpp(measurement_id) |
401 self.download_hirelpp(measurement_id) |
317 if measurement.cloudmask == 127: |
402 if measurement.cloudmask == 127: |
318 logger.info("Downloading cloudmask files.") |
403 logger.info("Downloading cloud screening files.") |
319 self.download_cloudmask(measurement_id) |
404 self.download_cloudmask(measurement_id) |
320 if measurement.elpp == 127: |
405 if measurement.elpp == 127: |
321 logger.info("Downloading preprocessed files.") |
406 logger.info("Downloading ElPP files.") |
322 self.download_preprocessed(measurement_id) |
407 self.download_elpp(measurement_id) |
323 if measurement.elda == 127: |
408 if measurement.elda == 127: |
324 logger.info("Downloading optical files.") |
409 logger.info("Downloading ELDA files.") |
325 self.download_optical(measurement_id) |
410 self.download_elda(measurement_id) |
326 logger.info("Downloading graphs.") |
411 logger.info("Downloading graphs.") |
327 self.download_graphs(measurement_id) |
412 self.download_plots(measurement_id) |
328 if measurement.elic == 127: |
413 if measurement.elic == 127: |
329 logger.info("Downloading preprocessed files.") |
414 logger.info("Downloading ELIC files.") |
330 self.download_elic(measurement_id) |
415 self.download_elic(measurement_id) |
|
416 |
|
417 # TODO: Need to check ELDEC code (when it becomes available in the API) |
|
418 if measurement.is_calibration: |
|
419 logger.info("Downloading ELDEC files.") |
|
420 self.download_eldec(measurement_id) |
331 logger.info("--- Processing finished. ---") |
421 logger.info("--- Processing finished. ---") |
|
422 |
332 return measurement |
423 return measurement |
333 |
424 |
334 def get_measurement(self, measurement_id): |
425 def get_measurement(self, measurement_id): |
|
426 |
|
427 if measurement_id is None: # Is this still required? |
|
428 return None |
|
429 |
335 measurement_url = self.api_measurement_pattern.format(measurement_id) |
430 measurement_url = self.api_measurement_pattern.format(measurement_id) |
336 logger.debug("Measurement API URL: %s" % measurement_url) |
431 logger.debug("Measurement API URL: %s" % measurement_url) |
337 |
432 |
338 response = self.session.get(measurement_url) |
433 response = self.session.get(measurement_url) |
339 |
434 |
340 if not response.ok: |
435 response_dict = None |
|
436 |
|
437 if response.status_code == 200: |
|
438 response_dict = response.json() |
|
439 elif response.status_code == 404: |
|
440 logger.info("No measurement with id %s found on the SCC." % measurement_id) |
|
441 else: |
341 logger.error('Could not access API. Status code %s.' % response.status_code) |
442 logger.error('Could not access API. Status code %s.' % response.status_code) |
342 return None, response.status_code |
|
343 |
|
344 response_dict = response.json() |
|
345 |
443 |
346 if response_dict: |
444 if response_dict: |
347 measurement = Measurement(self.base_url, response_dict) |
445 measurement = Measurement(self.base_url, response_dict) |
348 return measurement, response.status_code |
|
349 else: |
446 else: |
350 logger.error("No measurement with id %s found on the SCC." % measurement_id) |
447 measurement = None |
351 return None, response.status_code |
448 |
352 |
449 return measurement, response.status_code |
353 def delete_measurement(self, measurement_id): |
450 |
|
451 def delete_measurement(self, measurement_id, delete_related): |
354 """ Deletes a measurement with the provided measurement id. The user |
452 """ Deletes a measurement with the provided measurement id. The user |
355 should have the appropriate permissions. |
453 should have the appropriate permissions. |
356 |
454 |
357 The procedures is performed directly through the web interface and |
455 The procedures is performed directly through the web interface and |
358 NOT through the API. |
456 NOT through the API. |
543 |
655 |
544 class Measurement(ApiObject): |
656 class Measurement(ApiObject): |
545 """ This class represents the measurement object as returned in the SCC API. |
657 """ This class represents the measurement object as returned in the SCC API. |
546 """ |
658 """ |
547 |
659 |
|
660 def __init__(self, base_url, dict_response): |
|
661 |
|
662 # Define expected attributes to assist debugging |
|
663 self.cloudmask = None |
|
664 self.elda = None |
|
665 self.elic = None |
|
666 self.elpp = None |
|
667 self.hirelpp = None |
|
668 self.id = None |
|
669 self.is_calibration = None |
|
670 self.is_running = None |
|
671 self.pre_processing_exit_code = None |
|
672 self.processing_exit_code = None |
|
673 self.resource_uri = None |
|
674 self.start = None |
|
675 self.stop = None |
|
676 self.system = None |
|
677 self.upload = None |
|
678 |
|
679 super().__init__(base_url, dict_response) |
|
680 |
548 @property |
681 @property |
549 def rerun_processing_url(self): |
682 def rerun_elda_url(self): |
550 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') |
683 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') |
|
684 return url_pattern.format(self.id) |
|
685 |
|
686 @property |
|
687 def rerun_elpp_url(self): |
|
688 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elpp/') |
551 return url_pattern.format(self.id) |
689 return url_pattern.format(self.id) |
552 |
690 |
553 @property |
691 @property |
554 def rerun_all_url(self): |
692 def rerun_all_url(self): |
555 ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') |
693 ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') |
556 return ulr_pattern.format(self.id) |
694 return ulr_pattern.format(self.id) |
557 |
695 |
558 def __str__(self): |
696 def __str__(self): |
559 return "%s: %s, %s, %s" % (self.id, |
697 return "Measurement {}".format(self.id) |
560 self.upload, |
|
561 self.pre_processing, |
|
562 self.processing) |
|
563 |
698 |
564 |
699 |
565 class AncillaryFile(ApiObject): |
700 class AncillaryFile(ApiObject): |
566 """ This class represents the ancilalry file object as returned in the SCC API. |
701 """ This class represents the ancilalry file object as returned in the SCC API. |
567 """ |
702 """ |
576 return "%s: %s, %s" % (self.id, |
711 return "%s: %s, %s" % (self.id, |
577 self.filename, |
712 self.filename, |
578 self.status) |
713 self.status) |
579 |
714 |
580 |
715 |
581 def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): |
716 def process_file(filename, system_id, settings, force_upload, delete_related, |
|
717 monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): |
582 """ Shortcut function to process a file to the SCC. """ |
718 """ Shortcut function to process a file to the SCC. """ |
583 logger.info("Processing file %s, using system %s" % (filename, system_id)) |
719 logger.info("Processing file %s, using system %s" % (filename, system_id)) |
584 |
720 |
585 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
721 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
586 scc.login(settings['website_credentials']) |
722 scc.login(settings['website_credentials']) |
587 measurement = scc.process(filename, system_id, |
723 measurement = scc.process(filename, system_id, |
588 monitor=monitor, |
724 force_upload=force_upload, |
589 rs_filename=rs_filename, |
725 delete_related=delete_related, |
590 lr_filename=lr_filename, |
726 monitor=monitor, |
591 ov_filename=ov_filename) |
727 rs_filename=rs_filename, |
592 scc.logout() |
728 lr_filename=lr_filename, |
|
729 ov_filename=ov_filename) |
|
730 scc.logout() |
593 return measurement |
731 return measurement |
594 |
732 |
595 |
733 |
596 def delete_measurements(measurement_ids, settings): |
734 def delete_measurements(measurement_ids, delete_related, settings): |
597 """ Shortcut function to delete measurements from the SCC. """ |
735 """ Shortcut function to delete measurements from the SCC. """ |
598 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
736 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
599 scc.login(settings['website_credentials']) |
737 scc.login(settings['website_credentials']) |
600 for m_id in measurement_ids: |
738 for m_id in measurement_ids: |
601 logger.info("Deleting %s" % m_id) |
739 logger.info("Deleting %s" % m_id) |
602 scc.delete_measurement(m_id) |
740 scc.delete_measurement(m_id, delete_related) |
603 scc.logout() |
741 scc.logout() |
604 |
742 |
605 |
743 |
606 def rerun_all(measurement_ids, monitor, settings): |
744 def rerun_all(measurement_ids, monitor, settings): |
607 """ Shortcut function to rerun measurements from the SCC. """ |
745 """ Shortcut function to rerun measurements from the SCC. """ |
608 |
746 |
609 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
747 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
610 scc.login(settings['website_credentials']) |
748 scc.login(settings['website_credentials']) |
611 for m_id in measurement_ids: |
749 for m_id in measurement_ids: |
612 logger.info("Rerunning all products for %s" % m_id) |
750 logger.info("Rerunning all products for %s" % m_id) |
613 scc.rerun_all(m_id, monitor) |
751 scc.rerun_all(m_id, monitor) |
614 scc.logout() |
752 scc.logout() |
615 |
753 |
616 |
754 |
617 def rerun_processing(measurement_ids, monitor, settings): |
755 def rerun_processing(measurement_ids, monitor, settings): |
618 """ Shortcut function to delete a measurement from the SCC. """ |
756 """ Shortcut function to delete a measurement from the SCC. """ |
619 |
757 |
620 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
758 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
621 scc.login(settings['website_credentials']) |
759 scc.login(settings['website_credentials']) |
622 for m_id in measurement_ids: |
760 for m_id in measurement_ids: |
623 logger.info("Rerunning (optical) processing for %s" % m_id) |
761 logger.info("Rerunning (optical) processing for %s" % m_id) |
624 scc.rerun_processing(m_id, monitor) |
762 scc.rerun_elpp(m_id, monitor) |
625 scc.logout() |
763 scc.logout() |
626 |
764 |
627 |
765 |
628 def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None, |
766 def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None, |
629 preprocessing_status=None, |
767 preprocessing_status=None, |
630 optical_processing=None): |
768 optical_processing=None): |
631 """List all available measurements""" |
769 """List all available measurements""" |
632 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
770 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
633 scc.login(settings['website_credentials']) |
771 scc.login(settings['website_credentials']) |
634 ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status, |
772 ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status, |
635 processing_status=preprocessing_status, optical_processing=optical_processing) |
773 processing_status=preprocessing_status, optical_processing=optical_processing) |
636 for entry in ret: |
774 for entry in ret: |
637 print("%s" % entry.id) |
775 print("%s" % entry.id) |
638 scc.logout() |
776 scc.logout() |
639 |
777 |
640 |
778 |
641 def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings): |
779 def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings): |
642 """Download all measurements for the specified IDs""" |
780 """Download all measurements for the specified IDs""" |
643 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) |
781 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
644 scc.login(settings['website_credentials']) |
782 scc.login(settings['website_credentials']) |
645 for m_id in measurement_ids: |
783 for m_id in measurement_ids: |
646 if download_preproc: |
784 if download_preproc: |
647 logger.info("Downloading preprocessed files for '%s'" % m_id) |
785 logger.info("Downloading preprocessed files for '%s'" % m_id) |
648 scc.download_preprocessed(m_id) |
786 scc.download_elpp(m_id) |
649 logger.info("Complete") |
787 logger.info("Complete") |
650 if download_optical: |
788 if download_optical: |
651 logger.info("Downloading optical files for '%s'" % m_id) |
789 logger.info("Downloading optical files for '%s'" % m_id) |
652 scc.download_optical(m_id) |
790 scc.download_elda(m_id) |
653 logger.info("Complete") |
791 logger.info("Complete") |
654 if download_graph: |
792 if download_graph: |
655 logger.info("Downloading profile graph files for '%s'" % m_id) |
793 logger.info("Downloading profile graph files for '%s'" % m_id) |
656 scc.download_graphs(m_id) |
794 scc.download_plots(m_id) |
657 logger.info("Complete") |
795 logger.info("Complete") |
|
796 scc.logout() |
658 |
797 |
659 |
798 |
660 def settings_from_path(config_file_path): |
799 def settings_from_path(config_file_path): |
661 """ Read the configuration file. |
800 """ Read the configuration file. |
662 |
801 |
695 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
836 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
696 action="store_true") |
837 action="store_true") |
697 parser.set_defaults(execute=rerun_all_from_args) |
838 parser.set_defaults(execute=rerun_all_from_args) |
698 |
839 |
699 |
840 |
700 def setup_rerun_processing(parser): |
841 def setup_rerun_elpp(parser): |
701 def rerun_processing_from_args(parsed): |
842 def rerun_processing_from_args(parsed): |
702 rerun_processing(parsed.IDs, parsed.process, parsed.config) |
843 rerun_processing(parsed.IDs, parsed.process, parsed.config) |
703 |
844 |
704 parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.") |
845 parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.") |
705 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
846 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", |
706 action="store_true") |
847 action="store_true") |
707 parser.set_defaults(execute=rerun_processing_from_args) |
848 parser.set_defaults(execute=rerun_processing_from_args) |
708 |
849 |
709 |
850 |
710 def setup_process_file(parser): |
851 def setup_upload_file(parser): |
711 """ Upload and monitor processing progress.""" |
852 """ Upload but do not monitor processing progress. """ |
712 def process_file_from_args(parsed): |
853 def upload_file_from_args(parsed): |
713 process_file(parsed.filename, parsed.system, parsed.config, monitor=True, |
854 process_file(parsed.filename, parsed.system, parsed.config, |
|
855 monitor=parsed.process, |
|
856 force_upload=parsed.force_upload, |
|
857 delete_related=False, # For now, use this as default |
714 rs_filename=parsed.radiosounding, |
858 rs_filename=parsed.radiosounding, |
715 ov_filename=parsed.overlap, |
859 ov_filename=parsed.overlap, |
716 lr_filename=parsed.lidarratio) |
860 lr_filename=parsed.lidarratio) |
717 |
861 |
718 parser.add_argument("filename", help="Measurement file name or path.") |
862 parser.add_argument("filename", help="Measurement file name or path.") |
719 parser.add_argument("system", help="Processing system id.") |
863 parser.add_argument("system", help="Processing system id.") |
|
864 parser.add_argument("-p", "--process", help="Wait for the processing results.", |
|
865 action="store_true") |
|
866 parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.", |
|
867 action="store_true") |
720 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
868 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
721 parser.add_argument("--overlap", default=None, help="Overlap file name or path") |
869 parser.add_argument("--overlap", default=None, help="Overlap file name or path") |
722 parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") |
870 parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") |
723 |
871 |
724 parser.set_defaults(execute=process_file_from_args) |
|
725 |
|
726 |
|
727 def setup_upload_file(parser): |
|
728 """ Upload but do not monitor processing progress. """ |
|
729 def upload_file_from_args(parsed): |
|
730 process_file(parsed.filename, parsed.system, parsed.config, monitor=False, |
|
731 rs_filename=parsed.radiosounding, |
|
732 ov_filename=parsed.overlap, |
|
733 lr_filename=parsed.lidarratio) |
|
734 |
|
735 parser.add_argument("filename", help="Measurement file name or path.") |
|
736 parser.add_argument("system", help="Processing system id.") |
|
737 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
|
738 parser.add_argument("--overlap", default=None, help="Overlap file name or path") |
|
739 parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") |
|
740 |
|
741 parser.set_defaults(execute=upload_file_from_args) |
872 parser.set_defaults(execute=upload_file_from_args) |
742 |
873 |
743 |
874 |
744 def setup_list_measurements(parser): |
875 def setup_list_measurements(parser): |
745 def list_measurements_from_args(parsed): |
876 def list_measurements_from_args(parsed): |
|
877 # TODO: Fix this |
|
878 logger.warning("This method needs to be updated. Cross-chceck any results.") |
|
879 |
746 list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start, |
880 list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start, |
747 stop=parsed.stop, |
881 stop=parsed.stop, |
748 upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status, |
882 upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status, |
749 optical_processing=parsed.optical_processing_status) |
883 optical_processing=parsed.optical_processing_status) |
750 |
884 |
791 # Define the command line arguments. |
928 # Define the command line arguments. |
792 parser = argparse.ArgumentParser() |
929 parser = argparse.ArgumentParser() |
793 subparsers = parser.add_subparsers() |
930 subparsers = parser.add_subparsers() |
794 |
931 |
795 delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.") |
932 delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.") |
796 rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.") |
933 rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun all processing steps for the provided measurement IDs.") |
797 rerun_processing_parser = subparsers.add_parser("rerun-processing", |
934 rerun_processing_parser = subparsers.add_parser("rerun-elpp", |
798 help="Rerun processing routines for a measurement.") |
935 help="Rerun low-resolution processing steps for the provided measurement ID.") |
799 process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download processing results.") |
936 upload_file_parser = subparsers.add_parser("upload-file", help="Submit a file and, optionally, download the output products.") |
800 upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.") |
|
801 list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") |
937 list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") |
802 download_parser = subparsers.add_parser("download", help="Download selected measurements.") |
938 download_parser = subparsers.add_parser("download", help="Download selected measurements.") |
803 |
939 |
804 setup_delete(delete_parser) |
940 setup_delete(delete_parser) |
805 setup_rerun_all(rerun_all_parser) |
941 setup_rerun_all(rerun_all_parser) |
806 setup_rerun_processing(rerun_processing_parser) |
942 setup_rerun_elpp(rerun_processing_parser) |
807 setup_process_file(process_file_parser) |
943 |
808 setup_upload_file(upload_file_parser) |
944 setup_upload_file(upload_file_parser) |
809 setup_list_measurements(list_parser) |
945 setup_list_measurements(list_parser) |
810 setup_download_measurements(download_parser) |
946 setup_download_measurements(download_parser) |
811 |
947 |
812 # Verbosity settings from http://stackoverflow.com/a/20663028 |
948 # Verbosity settings from http://stackoverflow.com/a/20663028 |