scc_access/scc_access.py

changeset 45
6574da43a481
parent 40
8acea12976c4
child 50
1ecbb8de2b39
equal deleted inserted replaced
44:ca290caa7304 45:6574da43a481
1 import sys
2
1 import requests 3 import requests
2 4
3 try: 5 try:
4 import urllib.parse as urlparse # Python 3 6 import urllib.parse as urlparse # Python 3
5 except ImportError: 7 except ImportError:
9 import datetime 11 import datetime
10 import logging 12 import logging
11 import os 13 import os
12 import re 14 import re
13 from io import BytesIO 15 from io import BytesIO
14 import sys 16
15 import time 17 import time
16 18
17 from zipfile import ZipFile 19 from zipfile import ZipFile
18 20
19 import yaml 21 import yaml
22
23 import netCDF4 as netcdf
20 24
21 requests.packages.urllib3.disable_warnings() 25 requests.packages.urllib3.disable_warnings()
22 logger = logging.getLogger(__name__) 26 logger = logging.getLogger(__name__)
23 27
24 # The regex to find the measurement id from the measurement page 28 # The regex to find the measurement id from the measurement page
51 self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, 55 self.download_hirelpp_pattern = urlparse.urljoin(self.base_url,
52 'data_processing/measurements/{0}/download-hirelpp/') 56 'data_processing/measurements/{0}/download-hirelpp/')
53 self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, 57 self.download_cloudmask_pattern = urlparse.urljoin(self.base_url,
54 'data_processing/measurements/{0}/download-cloudmask/') 58 'data_processing/measurements/{0}/download-cloudmask/')
55 59
56 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 60 self.download_elpp_pattern = urlparse.urljoin(self.base_url,
57 'data_processing/measurements/{0}/download-preprocessed/') 61 'data_processing/measurements/{0}/download-preprocessed/')
58 self.download_optical_pattern = urlparse.urljoin(self.base_url, 62 self.download_elda_pattern = urlparse.urljoin(self.base_url,
59 'data_processing/measurements/{0}/download-optical/') 63 'data_processing/measurements/{0}/download-optical/')
60 self.download_graph_pattern = urlparse.urljoin(self.base_url, 64 self.download_plots_pattern = urlparse.urljoin(self.base_url,
61 'data_processing/measurements/{0}/download-plots/') 65 'data_processing/measurements/{0}/download-plots/')
62 self.download_elic_pattern = urlparse.urljoin(self.base_url, 66 self.download_elic_pattern = urlparse.urljoin(self.base_url,
63 'data_processing/measurements/{0}/download-elic/') 67 'data_processing/measurements/{0}/download-elic/')
64 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') 68 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
65 69
94 98
95 def logout(self): 99 def logout(self):
96 """ Logout from SCC """ 100 """ Logout from SCC """
97 return self.session.get(self.logout_url, stream=True) 101 return self.session.get(self.logout_url, stream=True)
98 102
99 def upload_file(self, filename, system_id, rs_filename=None, ov_filename=None, lr_filename=None): 103 def upload_file(self, filename, system_id, force_upload, delete_related, rs_filename=None, ov_filename=None, lr_filename=None):
100 """ Upload a filename for processing with a specific system. If the 104 """ Upload a filename for processing with a specific system. If the
101 upload is successful, it returns the measurement id. """ 105 upload is successful, it returns the measurement id. """
106 measurement_id = self.measurement_id_from_file(filename)
107
108 logger.debug('Checking if a measurement with the same id already exists on the SCC server.')
109 existing_measurement = self.get_measurement(measurement_id)
110
111 if existing_measurement:
112 if force_upload:
113 logger.info(
114 "Measurement with id {} already exists on the SCC. Trying to delete it...".format(measurement_id))
115 self.delete_measurement(measurement_id, delete_related)
116 else:
117 logger.error(
118 "Measurement with id {} already exists on the SCC. Use --force_upload flag to overwrite it.".format(
119 measurement_id))
120 sys.exit(1)
121
102 # Get submit page 122 # Get submit page
103 upload_page = self.session.get(self.upload_url) 123 upload_page = self.session.get(self.upload_url)
104 124
105 # Submit the data 125 # Submit the data
106 upload_data = {'system': system_id} 126 upload_data = {'system': system_id}
151 measurement_id = False 171 measurement_id = False
152 logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.") 172 logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.")
153 else: 173 else:
154 measurement_id = re.findall(regex, upload_submit.text)[0] 174 measurement_id = re.findall(regex, upload_submit.text)[0]
155 logger.info("Successfully uploaded measurement with id %s." % measurement_id) 175 logger.info("Successfully uploaded measurement with id %s." % measurement_id)
176
177 return measurement_id
178
179 @staticmethod
180 def measurement_id_from_file(filename):
181 """ Get the measurement id from the input file. """
182
183 if not os.path.isfile(filename):
184 logger.error("File {} does not exist.".format(filename))
185 sys.exit(1)
186
187 with netcdf.Dataset(filename) as f:
188 try:
189 measurement_id = f.Measurement_ID
190 except AttributeError:
191 logger.error(
192 "Input file {} does not contain a Measurement_ID global attribute. Wrong file format?".format(
193 filename))
194 sys.exit(1)
156 195
157 return measurement_id 196 return measurement_id
158 197
159 def download_files(self, measurement_id, subdir, download_url): 198 def download_files(self, measurement_id, subdir, download_url):
160 """ Downloads some files from the download_url to the specified 199 """ Downloads some files from the download_url to the specified
190 229
191 with open(local_file, 'wb') as f: 230 with open(local_file, 'wb') as f:
192 f.write(zip_file.read(ziped_name)) 231 f.write(zip_file.read(ziped_name))
193 232
194 def download_hirelpp(self, measurement_id): 233 def download_hirelpp(self, measurement_id):
195 """ Download HiRElPP files for the measurement id. """ 234 """ Download hirelpp files for the measurement id. """
196 # Construct the download url 235 # Construct the download url
197 download_url = self.download_hirelpp_pattern.format(measurement_id) 236 download_url = self.download_hirelpp_pattern.format(measurement_id)
198 self.download_files(measurement_id, 'hirelpp', download_url) 237 try:
238 self.download_files(measurement_id, 'scc_hirelpp', download_url)
239 except Exception as e:
240 logger.error("Could not download HiRElPP files. Error message: {}".format(e))
241 logger.debug('Download exception:', exc_info=True)
199 242
200 def download_cloudmask(self, measurement_id): 243 def download_cloudmask(self, measurement_id):
244 """ Download cloudmask files for the measurement id. """
245 # Construct the download url
246 download_url = self.download_cloudmask_pattern.format(measurement_id)
247 try:
248 self.download_files(measurement_id, 'scc_cloudscreen', download_url)
249 except Exception as e:
250 logger.error("Could not download cloudscreen files. Error message: {}".format(e))
251 logger.debug('Download exception:', exc_info=True)
252
253 def download_elpp(self, measurement_id):
201 """ Download preprocessed files for the measurement id. """ 254 """ Download preprocessed files for the measurement id. """
202 # Construct the download url 255 # Construct the download url
203 download_url = self.download_cloudmask_pattern.format(measurement_id) 256 download_url = self.download_elpp_pattern.format(measurement_id)
204 self.download_files(measurement_id, 'cloudmask', download_url) 257 try:
205 258 self.download_files(measurement_id, 'scc_preprocessed', download_url)
206 def download_preprocessed(self, measurement_id): 259 except Exception as e:
207 """ Download preprocessed files for the measurement id. """ 260 logger.error("Could not download ElPP files. Error message: {}".format(e))
208 # Construct the download url 261 logger.debug('Download exception:', exc_info=True)
209 download_url = self.download_preprocessed_pattern.format(measurement_id) 262
210 self.download_files(measurement_id, 'scc_preprocessed', download_url) 263 def download_elda(self, measurement_id):
211
212 def download_optical(self, measurement_id):
213 """ Download optical files for the measurement id. """ 264 """ Download optical files for the measurement id. """
214 # Construct the download url 265 # Construct the download url
215 download_url = self.download_optical_pattern.format(measurement_id) 266 download_url = self.download_elda_pattern.format(measurement_id)
216 self.download_files(measurement_id, 'scc_optical', download_url) 267 try:
217 268 self.download_files(measurement_id, 'scc_optical', download_url)
218 def download_graphs(self, measurement_id): 269 except Exception as e:
270 logger.error("Could not download ELDA files. Error message: {}".format(e))
271 logger.debug('Download exception:', exc_info=True)
272
273 def download_plots(self, measurement_id):
219 """ Download profile graphs for the measurement id. """ 274 """ Download profile graphs for the measurement id. """
220 # Construct the download url 275 # Construct the download url
221 download_url = self.download_graph_pattern.format(measurement_id) 276 download_url = self.download_plots_pattern.format(measurement_id)
222 self.download_files(measurement_id, 'scc_plots', download_url) 277 try:
278 self.download_files(measurement_id, 'scc_plots', download_url)
279 except Exception as e:
280 logger.error("Could not download ELDA plots. Error message: {}".format(e))
281 logger.debug('Download exception:', exc_info=True)
223 282
224 def download_elic(self, measurement_id): 283 def download_elic(self, measurement_id):
225 """ Download profile graphs for the measurement id. """ 284 """ Download ELIC files for the measurement id. """
226 # Construct the download url 285 # Construct the download url
227 download_url = self.download_elic_pattern.format(measurement_id) 286 download_url = self.download_elic_pattern.format(measurement_id)
228 self.download_files(measurement_id, 'elic', download_url) 287 try:
229 288 self.download_files(measurement_id, 'scc_elic', download_url)
230 def rerun_processing(self, measurement_id, monitor=True): 289 except Exception as e:
290 logger.error("Could not download ELIC files. Error message: {}".format(e))
291 logger.debug('Download exception:', exc_info=True)
292
293 def download_eldec(self, measurement_id):
294 """ Download ELDEC files for the measurement id. """
295 # Construct the download url
296 download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now
297 try:
298 self.download_files(measurement_id, 'scc_eldec', download_url)
299 except Exception as e:
300 logger.error("Could not download EDELC files. Error message: {}".format(e))
301 logger.debug('Download exception:', exc_info=True)
302
303 def rerun_elpp(self, measurement_id, monitor=True):
304 logger.debug("Started rerun_elpp procedure.")
305
306 logger.debug("Getting measurement %s" % measurement_id)
231 measurement, status = self.get_measurement(measurement_id) 307 measurement, status = self.get_measurement(measurement_id)
232 308
233 if measurement: 309 if measurement:
234 request = self.session.get(measurement.rerun_processing_url, stream=True) 310 logger.debug("Attempting to rerun ElPP through %s." % measurement.rerun_all_url)
311 request = self.session.get(measurement.rerun_elpp_url, stream=True)
235 312
236 if request.status_code != 200: 313 if request.status_code != 200:
237 logger.error( 314 logger.error(
238 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) 315 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
239 return 316 else:
317 logger.info("Rerun-elpp command submitted successfully for id {}.".format(measurement_id))
240 318
241 if monitor: 319 if monitor:
242 self.monitor_processing(measurement_id) 320 self.monitor_processing(measurement_id)
243 321
244 def rerun_all(self, measurement_id, monitor=True): 322 def rerun_all(self, measurement_id, monitor=True):
253 request = self.session.get(measurement.rerun_all_url, stream=True) 331 request = self.session.get(measurement.rerun_all_url, stream=True)
254 332
255 if request.status_code != 200: 333 if request.status_code != 200:
256 logger.error("Could not rerun pre processing for %s. Status code: %s" % 334 logger.error("Could not rerun pre processing for %s. Status code: %s" %
257 (measurement_id, request.status_code)) 335 (measurement_id, request.status_code))
258 return 336 else:
337 logger.info("Rerun-all command submitted successfully for id {}.".format(measurement_id))
259 338
260 if monitor: 339 if monitor:
261 self.monitor_processing(measurement_id) 340 self.monitor_processing(measurement_id)
262 341
263 def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None): 342 def process(self, filename, system_id, monitor, force_upload, delete_related, rs_filename=None, lr_filename=None, ov_filename=None):
264 """ Upload a file for processing and wait for the processing to finish. 343 """ Upload a file for processing and wait for the processing to finish.
265 If the processing is successful, it will download all produced files. 344 If the processing is successful, it will download all produced files.
266 """ 345 """
267 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) 346 logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
268 # Upload file 347 # Upload file
269 logger.info("--- Uploading file") 348 logger.info("--- Uploading file")
270 measurement_id = self.upload_file(filename, system_id, 349 measurement_id = self.upload_file(filename, system_id, force_upload, delete_related,
271 rs_filename=rs_filename, 350 rs_filename=rs_filename,
272 lr_filename=lr_filename, 351 lr_filename=lr_filename,
273 ov_filename=ov_filename) 352 ov_filename=ov_filename)
274 353
275 if measurement_id and monitor: 354 if measurement_id and monitor:
286 error_max = 6 365 error_max = 6
287 time_sleep = 10 366 time_sleep = 10
288 367
289 # try to wait for measurement to appear in API 368 # try to wait for measurement to appear in API
290 measurement = None 369 measurement = None
291 logger.info("Looking for measurement %s in SCC", measurement_id) 370 logger.info("Looking for measurement %s on SCC", measurement_id)
292 while error_count < error_max: 371 while error_count < error_max:
293 time.sleep(time_sleep) 372 time.sleep(time_sleep)
294 measurement, status = self.get_measurement(measurement_id) 373 measurement, status = self.get_measurement(measurement_id)
295 if status != 200 and error_count < error_max: 374 if status != 200 and error_count < error_max:
296 logger.error("Measurement not found. waiting %ds", time_sleep) 375 logger.error("Measurement not found. waiting %ds", time_sleep)
300 379
301 if error_count == error_max: 380 if error_count == error_max:
302 logger.critical("Measurement %s doesn't seem to exist", measurement_id) 381 logger.critical("Measurement %s doesn't seem to exist", measurement_id)
303 sys.exit(1) 382 sys.exit(1)
304 383
305 logger.info('Measurement %s found', measurement_id) 384 logger.info('Measurement %s found.', measurement_id)
306 385
307 if measurement is not None: 386 if measurement is not None:
308 while measurement.is_running: 387 while measurement.is_running:
309 logger.info("Measurement is being processed. Please wait.") 388 logger.info("Measurement is being processed. status: {}, {}, {}, {}, {}, {}). Please wait.".format(
389 measurement.upload,
390 measurement.hirelpp,
391 measurement.cloudmask,
392 measurement.elpp,
393 measurement.elda,
394 measurement.elic))
310 time.sleep(10) 395 time.sleep(10)
311 measurement, status = self.get_measurement(measurement_id) 396 measurement, status = self.get_measurement(measurement_id)
312 397
313 logger.info("Measurement processing finished.") 398 logger.info("Measurement processing finished.")
314 if measurement.hirelpp == 127: 399 if measurement.hirelpp == 127:
315 logger.info("Downloading hirelpp files.") 400 logger.info("Downloading HiRElPP files.")
316 self.download_hirelpp(measurement_id) 401 self.download_hirelpp(measurement_id)
317 if measurement.cloudmask == 127: 402 if measurement.cloudmask == 127:
318 logger.info("Downloading cloudmask files.") 403 logger.info("Downloading cloud screening files.")
319 self.download_cloudmask(measurement_id) 404 self.download_cloudmask(measurement_id)
320 if measurement.elpp == 127: 405 if measurement.elpp == 127:
321 logger.info("Downloading preprocessed files.") 406 logger.info("Downloading ElPP files.")
322 self.download_preprocessed(measurement_id) 407 self.download_elpp(measurement_id)
323 if measurement.elda == 127: 408 if measurement.elda == 127:
324 logger.info("Downloading optical files.") 409 logger.info("Downloading ELDA files.")
325 self.download_optical(measurement_id) 410 self.download_elda(measurement_id)
326 logger.info("Downloading graphs.") 411 logger.info("Downloading graphs.")
327 self.download_graphs(measurement_id) 412 self.download_plots(measurement_id)
328 if measurement.elic == 127: 413 if measurement.elic == 127:
329 logger.info("Downloading preprocessed files.") 414 logger.info("Downloading ELIC files.")
330 self.download_elic(measurement_id) 415 self.download_elic(measurement_id)
416
417 # TODO: Need to check ELDEC code (when it becomes available in the API)
418 if measurement.is_calibration:
419 logger.info("Downloading ELDEC files.")
420 self.download_eldec(measurement_id)
331 logger.info("--- Processing finished. ---") 421 logger.info("--- Processing finished. ---")
422
332 return measurement 423 return measurement
333 424
334 def get_measurement(self, measurement_id): 425 def get_measurement(self, measurement_id):
426
427 if measurement_id is None: # Is this still required?
428 return None
429
335 measurement_url = self.api_measurement_pattern.format(measurement_id) 430 measurement_url = self.api_measurement_pattern.format(measurement_id)
336 logger.debug("Measurement API URL: %s" % measurement_url) 431 logger.debug("Measurement API URL: %s" % measurement_url)
337 432
338 response = self.session.get(measurement_url) 433 response = self.session.get(measurement_url)
339 434
340 if not response.ok: 435 response_dict = None
436
437 if response.status_code == 200:
438 response_dict = response.json()
439 elif response.status_code == 404:
440 logger.info("No measurement with id %s found on the SCC." % measurement_id)
441 else:
341 logger.error('Could not access API. Status code %s.' % response.status_code) 442 logger.error('Could not access API. Status code %s.' % response.status_code)
342 return None, response.status_code
343
344 response_dict = response.json()
345 443
346 if response_dict: 444 if response_dict:
347 measurement = Measurement(self.base_url, response_dict) 445 measurement = Measurement(self.base_url, response_dict)
348 return measurement, response.status_code
349 else: 446 else:
350 logger.error("No measurement with id %s found on the SCC." % measurement_id) 447 measurement = None
351 return None, response.status_code 448
352 449 return measurement, response.status_code
353 def delete_measurement(self, measurement_id): 450
451 def delete_measurement(self, measurement_id, delete_related):
354 """ Deletes a measurement with the provided measurement id. The user 452 """ Deletes a measurement with the provided measurement id. The user
355 should have the appropriate permissions. 453 should have the appropriate permissions.
356 454
357 The procedures is performed directly through the web interface and 455 The procedures is performed directly through the web interface and
358 NOT through the API. 456 NOT through the API.
373 # Check that the page opened properly 471 # Check that the page opened properly
374 if confirm_page.status_code != 200: 472 if confirm_page.status_code != 200:
375 logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code)) 473 logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code))
376 return None 474 return None
377 475
476 # Get the delete related value
477 if delete_related:
478 delete_related_option = 'delete_related'
479 else:
480 delete_related_option = 'not_delete_related'
481
378 # Delete the measurement 482 # Delete the measurement
379 delete_page = self.session.post(delete_url, 483 delete_page = self.session.post(delete_url,
380 data={'post': 'yes'}, 484 data={'post': 'yes',
485 'select_delete_related_measurements': delete_related_option},
381 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], 486 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
382 'referer': delete_url} 487 'referer': delete_url}
383 ) 488 )
384 if not delete_page.ok: 489 if not delete_page.ok:
385 logger.warning("Something went wrong. Delete page status: {0}".format( 490 logger.warning("Something went wrong. Delete page status: {0}".format(
515 ancillary_file = AncillaryFile(self.api_base_url, object_list[0]) # Assume only one file is returned 620 ancillary_file = AncillaryFile(self.api_base_url, object_list[0]) # Assume only one file is returned
516 else: 621 else:
517 ancillary_file = AncillaryFile(self.api_base_url, None) # Create an empty object 622 ancillary_file = AncillaryFile(self.api_base_url, None) # Create an empty object
518 623
519 return ancillary_file, response.status_code 624 return ancillary_file, response.status_code
625
626 def __enter__(self):
627 return self
628
629 def __exit__(self, *args):
630 logger.debug("Closing SCC connection session.")
631 self.session.close()
520 632
521 class PageNotAccessibleError(RuntimeError): 633 class PageNotAccessibleError(RuntimeError):
522 pass 634 pass
523 635
524 636
543 655
544 class Measurement(ApiObject): 656 class Measurement(ApiObject):
545 """ This class represents the measurement object as returned in the SCC API. 657 """ This class represents the measurement object as returned in the SCC API.
546 """ 658 """
547 659
660 def __init__(self, base_url, dict_response):
661
662 # Define expected attributes to assist debugging
663 self.cloudmask = None
664 self.elda = None
665 self.elic = None
666 self.elpp = None
667 self.hirelpp = None
668 self.id = None
669 self.is_calibration = None
670 self.is_running = None
671 self.pre_processing_exit_code = None
672 self.processing_exit_code = None
673 self.resource_uri = None
674 self.start = None
675 self.stop = None
676 self.system = None
677 self.upload = None
678
679 super().__init__(base_url, dict_response)
680
548 @property 681 @property
549 def rerun_processing_url(self): 682 def rerun_elda_url(self):
550 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') 683 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/')
684 return url_pattern.format(self.id)
685
686 @property
687 def rerun_elpp_url(self):
688 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elpp/')
551 return url_pattern.format(self.id) 689 return url_pattern.format(self.id)
552 690
553 @property 691 @property
554 def rerun_all_url(self): 692 def rerun_all_url(self):
555 ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') 693 ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/')
556 return ulr_pattern.format(self.id) 694 return ulr_pattern.format(self.id)
557 695
558 def __str__(self): 696 def __str__(self):
559 return "%s: %s, %s, %s" % (self.id, 697 return "Measurement {}".format(self.id)
560 self.upload,
561 self.pre_processing,
562 self.processing)
563 698
564 699
565 class AncillaryFile(ApiObject): 700 class AncillaryFile(ApiObject):
566 """ This class represents the ancilalry file object as returned in the SCC API. 701 """ This class represents the ancilalry file object as returned in the SCC API.
567 """ 702 """
576 return "%s: %s, %s" % (self.id, 711 return "%s: %s, %s" % (self.id,
577 self.filename, 712 self.filename,
578 self.status) 713 self.status)
579 714
580 715
581 def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): 716 def process_file(filename, system_id, settings, force_upload, delete_related,
717 monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
582 """ Shortcut function to process a file to the SCC. """ 718 """ Shortcut function to process a file to the SCC. """
583 logger.info("Processing file %s, using system %s" % (filename, system_id)) 719 logger.info("Processing file %s, using system %s" % (filename, system_id))
584 720
585 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 721 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
586 scc.login(settings['website_credentials']) 722 scc.login(settings['website_credentials'])
587 measurement = scc.process(filename, system_id, 723 measurement = scc.process(filename, system_id,
588 monitor=monitor, 724 force_upload=force_upload,
589 rs_filename=rs_filename, 725 delete_related=delete_related,
590 lr_filename=lr_filename, 726 monitor=monitor,
591 ov_filename=ov_filename) 727 rs_filename=rs_filename,
592 scc.logout() 728 lr_filename=lr_filename,
729 ov_filename=ov_filename)
730 scc.logout()
593 return measurement 731 return measurement
594 732
595 733
596 def delete_measurements(measurement_ids, settings): 734 def delete_measurements(measurement_ids, delete_related, settings):
597 """ Shortcut function to delete measurements from the SCC. """ 735 """ Shortcut function to delete measurements from the SCC. """
598 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 736 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
599 scc.login(settings['website_credentials']) 737 scc.login(settings['website_credentials'])
600 for m_id in measurement_ids: 738 for m_id in measurement_ids:
601 logger.info("Deleting %s" % m_id) 739 logger.info("Deleting %s" % m_id)
602 scc.delete_measurement(m_id) 740 scc.delete_measurement(m_id, delete_related)
603 scc.logout() 741 scc.logout()
604 742
605 743
606 def rerun_all(measurement_ids, monitor, settings): 744 def rerun_all(measurement_ids, monitor, settings):
607 """ Shortcut function to rerun measurements from the SCC. """ 745 """ Shortcut function to rerun measurements from the SCC. """
608 746
609 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 747 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
610 scc.login(settings['website_credentials']) 748 scc.login(settings['website_credentials'])
611 for m_id in measurement_ids: 749 for m_id in measurement_ids:
612 logger.info("Rerunning all products for %s" % m_id) 750 logger.info("Rerunning all products for %s" % m_id)
613 scc.rerun_all(m_id, monitor) 751 scc.rerun_all(m_id, monitor)
614 scc.logout() 752 scc.logout()
615 753
616 754
617 def rerun_processing(measurement_ids, monitor, settings): 755 def rerun_processing(measurement_ids, monitor, settings):
618 """ Shortcut function to delete a measurement from the SCC. """ 756 """ Shortcut function to delete a measurement from the SCC. """
619 757
620 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 758 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
621 scc.login(settings['website_credentials']) 759 scc.login(settings['website_credentials'])
622 for m_id in measurement_ids: 760 for m_id in measurement_ids:
623 logger.info("Rerunning (optical) processing for %s" % m_id) 761 logger.info("Rerunning (optical) processing for %s" % m_id)
624 scc.rerun_processing(m_id, monitor) 762 scc.rerun_elpp(m_id, monitor)
625 scc.logout() 763 scc.logout()
626 764
627 765
628 def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None, 766 def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None,
629 preprocessing_status=None, 767 preprocessing_status=None,
630 optical_processing=None): 768 optical_processing=None):
631 """List all available measurements""" 769 """List all available measurements"""
632 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 770 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
633 scc.login(settings['website_credentials']) 771 scc.login(settings['website_credentials'])
634 ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status, 772 ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status,
635 processing_status=preprocessing_status, optical_processing=optical_processing) 773 processing_status=preprocessing_status, optical_processing=optical_processing)
636 for entry in ret: 774 for entry in ret:
637 print("%s" % entry.id) 775 print("%s" % entry.id)
638 scc.logout() 776 scc.logout()
639 777
640 778
641 def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings): 779 def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings):
642 """Download all measurements for the specified IDs""" 780 """Download all measurements for the specified IDs"""
643 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 781 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
644 scc.login(settings['website_credentials']) 782 scc.login(settings['website_credentials'])
645 for m_id in measurement_ids: 783 for m_id in measurement_ids:
646 if download_preproc: 784 if download_preproc:
647 logger.info("Downloading preprocessed files for '%s'" % m_id) 785 logger.info("Downloading preprocessed files for '%s'" % m_id)
648 scc.download_preprocessed(m_id) 786 scc.download_elpp(m_id)
649 logger.info("Complete") 787 logger.info("Complete")
650 if download_optical: 788 if download_optical:
651 logger.info("Downloading optical files for '%s'" % m_id) 789 logger.info("Downloading optical files for '%s'" % m_id)
652 scc.download_optical(m_id) 790 scc.download_elda(m_id)
653 logger.info("Complete") 791 logger.info("Complete")
654 if download_graph: 792 if download_graph:
655 logger.info("Downloading profile graph files for '%s'" % m_id) 793 logger.info("Downloading profile graph files for '%s'" % m_id)
656 scc.download_graphs(m_id) 794 scc.download_plots(m_id)
657 logger.info("Complete") 795 logger.info("Complete")
796 scc.logout()
658 797
659 798
660 def settings_from_path(config_file_path): 799 def settings_from_path(config_file_path):
661 """ Read the configuration file. 800 """ Read the configuration file.
662 801
679 818
680 819
681 # Setup for command specific parsers 820 # Setup for command specific parsers
682 def setup_delete(parser): 821 def setup_delete(parser):
683 def delete_from_args(parsed): 822 def delete_from_args(parsed):
684 delete_measurements(parsed.IDs, parsed.config) 823 delete_measurements(parsed.IDs,
824 delete_related=False,
825 settings=parsed.config)
685 826
686 parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.") 827 parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.")
687 parser.set_defaults(execute=delete_from_args) 828 parser.set_defaults(execute=delete_from_args)
688 829
689 830
695 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", 836 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
696 action="store_true") 837 action="store_true")
697 parser.set_defaults(execute=rerun_all_from_args) 838 parser.set_defaults(execute=rerun_all_from_args)
698 839
699 840
700 def setup_rerun_processing(parser): 841 def setup_rerun_elpp(parser):
701 def rerun_processing_from_args(parsed): 842 def rerun_processing_from_args(parsed):
702 rerun_processing(parsed.IDs, parsed.process, parsed.config) 843 rerun_processing(parsed.IDs, parsed.process, parsed.config)
703 844
704 parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.") 845 parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.")
705 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", 846 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
706 action="store_true") 847 action="store_true")
707 parser.set_defaults(execute=rerun_processing_from_args) 848 parser.set_defaults(execute=rerun_processing_from_args)
708 849
709 850
710 def setup_process_file(parser): 851 def setup_upload_file(parser):
711 """ Upload and monitor processing progress.""" 852 """ Upload but do not monitor processing progress. """
712 def process_file_from_args(parsed): 853 def upload_file_from_args(parsed):
713 process_file(parsed.filename, parsed.system, parsed.config, monitor=True, 854 process_file(parsed.filename, parsed.system, parsed.config,
855 monitor=parsed.process,
856 force_upload=parsed.force_upload,
857 delete_related=False, # For now, use this as default
714 rs_filename=parsed.radiosounding, 858 rs_filename=parsed.radiosounding,
715 ov_filename=parsed.overlap, 859 ov_filename=parsed.overlap,
716 lr_filename=parsed.lidarratio) 860 lr_filename=parsed.lidarratio)
717 861
718 parser.add_argument("filename", help="Measurement file name or path.") 862 parser.add_argument("filename", help="Measurement file name or path.")
719 parser.add_argument("system", help="Processing system id.") 863 parser.add_argument("system", help="Processing system id.")
864 parser.add_argument("-p", "--process", help="Wait for the processing results.",
865 action="store_true")
866 parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.",
867 action="store_true")
720 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") 868 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
721 parser.add_argument("--overlap", default=None, help="Overlap file name or path") 869 parser.add_argument("--overlap", default=None, help="Overlap file name or path")
722 parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") 870 parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
723 871
724 parser.set_defaults(execute=process_file_from_args)
725
726
727 def setup_upload_file(parser):
728 """ Upload but do not monitor processing progress. """
729 def upload_file_from_args(parsed):
730 process_file(parsed.filename, parsed.system, parsed.config, monitor=False,
731 rs_filename=parsed.radiosounding,
732 ov_filename=parsed.overlap,
733 lr_filename=parsed.lidarratio)
734
735 parser.add_argument("filename", help="Measurement file name or path.")
736 parser.add_argument("system", help="Processing system id.")
737 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
738 parser.add_argument("--overlap", default=None, help="Overlap file name or path")
739 parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
740
741 parser.set_defaults(execute=upload_file_from_args) 872 parser.set_defaults(execute=upload_file_from_args)
742 873
743 874
744 def setup_list_measurements(parser): 875 def setup_list_measurements(parser):
745 def list_measurements_from_args(parsed): 876 def list_measurements_from_args(parsed):
877 # TODO: Fix this
878 logger.warning("This method needs to be updated. Cross-chceck any results.")
879
746 list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start, 880 list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start,
747 stop=parsed.stop, 881 stop=parsed.stop,
748 upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status, 882 upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status,
749 optical_processing=parsed.optical_processing_status) 883 optical_processing=parsed.optical_processing_status)
750 884
770 parser.set_defaults(execute=list_measurements_from_args) 904 parser.set_defaults(execute=list_measurements_from_args)
771 905
772 906
773 def setup_download_measurements(parser): 907 def setup_download_measurements(parser):
774 def download_measurements_from_args(parsed): 908 def download_measurements_from_args(parsed):
775 preproc = parsed.download_preprocessed 909 # TODO: Fix this
776 optical = parsed.download_optical 910 logger.warning("This method needs to be updated. Cross-chceck any results.")
911
912 preproc = parsed.download_elpp
913 optical = parsed.download_elda
777 graphs = parsed.download_profile_graphs 914 graphs = parsed.download_profile_graphs
778 if not preproc and not graphs: 915 if not preproc and not graphs:
779 optical = True 916 optical = True
780 download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config) 917 download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config)
781 918
791 # Define the command line arguments. 928 # Define the command line arguments.
792 parser = argparse.ArgumentParser() 929 parser = argparse.ArgumentParser()
793 subparsers = parser.add_subparsers() 930 subparsers = parser.add_subparsers()
794 931
795 delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.") 932 delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
796 rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.") 933 rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun all processing steps for the provided measurement IDs.")
797 rerun_processing_parser = subparsers.add_parser("rerun-processing", 934 rerun_processing_parser = subparsers.add_parser("rerun-elpp",
798 help="Rerun processing routines for a measurement.") 935 help="Rerun low-resolution processing steps for the provided measurement ID.")
799 process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download processing results.") 936 upload_file_parser = subparsers.add_parser("upload-file", help="Submit a file and, optionally, download the output products.")
800 upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.")
801 list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") 937 list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
802 download_parser = subparsers.add_parser("download", help="Download selected measurements.") 938 download_parser = subparsers.add_parser("download", help="Download selected measurements.")
803 939
804 setup_delete(delete_parser) 940 setup_delete(delete_parser)
805 setup_rerun_all(rerun_all_parser) 941 setup_rerun_all(rerun_all_parser)
806 setup_rerun_processing(rerun_processing_parser) 942 setup_rerun_elpp(rerun_processing_parser)
807 setup_process_file(process_file_parser) 943
808 setup_upload_file(upload_file_parser) 944 setup_upload_file(upload_file_parser)
809 setup_list_measurements(list_parser) 945 setup_list_measurements(list_parser)
810 setup_download_measurements(download_parser) 946 setup_download_measurements(download_parser)
811 947
812 # Verbosity settings from http://stackoverflow.com/a/20663028 948 # Verbosity settings from http://stackoverflow.com/a/20663028

mercurial