victor@7: import requests
moritz@29: 
ioannis@17: try:
ioannis@17:     import urllib.parse as urlparse  # Python 3
ioannis@17: except ImportError:
ioannis@17:     import urlparse  # Python 2
victor@7: 
victor@7: import argparse
madrouin@24: import datetime
madrouin@24: import logging
victor@7: import os
victor@7: import re
ioannis@32: from io import BytesIO
madrouin@24: import sys
victor@7: import time
madrouin@24: import urlparse
victor@7: from zipfile import ZipFile
madrouin@24: 
i@14: import yaml
victor@7: 
madrouin@24: requests.packages.urllib3.disable_warnings()
i@14: logger = logging.getLogger(__name__)
victor@7: 
victor@7: # The regex to find the measurement id from the measurement page
victor@7: # This should be read from the uploaded file, but would require an extra NetCDF module.
i@36: regex = "<h3>Measurement (?P<measurement_id>.{12,15}) <small>"  # {12, 15} to handle both old- and new-style measurement ids.
victor@7: 
victor@7: 
victor@7: class SCC:
madrouin@24:     """A simple class that will attempt to upload a file on the SCC server.
i@14: 
victor@7:     The uploading is done by simulating a normal browser session. In the current
madrouin@20:     version no check is performed, and no feedback is given if the upload
madrouin@20:     was successful. If everything is setup correctly, it will work.
victor@7:     """
victor@7: 
i@14:     def __init__(self, auth, output_dir, base_url):
moritz@29: 
victor@7:         self.auth = auth
victor@7:         self.output_dir = output_dir
i@14:         self.base_url = base_url
victor@7:         self.session = requests.Session()
moritz@29:         self.session.auth = auth
moritz@29:         self.session.verify = False
i@31: 
i@14:         self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
i@31:         self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/')
i@31:         self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
i@31: 
i@14:         self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
ioannis@34:         self.download_hirelpp_pattern = urlparse.urljoin(self.base_url,
ioannis@34:                                                              'data_processing/measurements/{0}/download-hirelpp/')
ioannis@34:         self.download_cloudmask_pattern = urlparse.urljoin(self.base_url,
ioannis@34:                                                          'data_processing/measurements/{0}/download-cloudmask/')
ioannis@34: 
moritz@29:         self.download_preprocessed_pattern = urlparse.urljoin(self.base_url,
moritz@29:                                                               'data_processing/measurements/{0}/download-preprocessed/')
moritz@29:         self.download_optical_pattern = urlparse.urljoin(self.base_url,
moritz@29:                                                          'data_processing/measurements/{0}/download-optical/')
moritz@29:         self.download_graph_pattern = urlparse.urljoin(self.base_url,
moritz@29:                                                        'data_processing/measurements/{0}/download-plots/')
ioannis@34:         self.download_elic_pattern = urlparse.urljoin(self.base_url,
ioannis@34:                                                        'data_processing/measurements/{0}/download-elic/')
i@14:         self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
i@31: 
i@14:         self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
i@31:         self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/')
i@31:         self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements')
i@38:         self.api_sounding_search_pattern = urlparse.urljoin(self.api_base_url, 'sounding_files/?filename={0}')
i@38:         self.api_lidarratio_search_pattern = urlparse.urljoin(self.api_base_url, 'lidarratio_files/?filename={0}')
i@38:         self.api_overlap_search_pattern = urlparse.urljoin(self.api_base_url, 'overlap_files/?filename={0}')
i@14: 
i@14:     def login(self, credentials):
i@31:         """ Login to SCC. """
victor@7:         logger.debug("Attempting to login to SCC, username %s." % credentials[0])
moritz@29:         login_credentials = {'username': credentials[0],
moritz@29:                              'password': credentials[1]}
victor@7: 
i@14:         logger.debug("Accessing login page at %s." % self.login_url)
victor@7: 
victor@7:         # Get upload form
moritz@29:         login_page = self.session.get(self.login_url)
victor@7: 
i@31:         if not login_page.ok:
i@31:             raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code)
victor@7: 
i@31:         logger.debug("Submitting credentials.")
victor@7:         # Submit the login data
i@14:         login_submit = self.session.post(self.login_url,
moritz@29:                                          data=login_credentials,
victor@7:                                          headers={'X-CSRFToken': login_page.cookies['csrftoken'],
moritz@29:                                                   'referer': self.login_url})
victor@7:         return login_submit
victor@7: 
victor@7:     def logout(self):
i@31:         """ Logout from SCC """
i@31:         return self.session.get(self.logout_url, stream=True)
victor@7: 
ioannis@32:     def upload_file(self, filename, system_id, rs_filename=None, ov_filename=None, lr_filename=None):
madrouin@20:         """ Upload a filename for processing with a specific system. If the
victor@7:         upload is successful, it returns the measurement id. """
victor@7:         # Get submit page
moritz@29:         upload_page = self.session.get(self.upload_url)
victor@7: 
victor@7:         # Submit the data
victor@7:         upload_data = {'system': system_id}
victor@7:         files = {'data': open(filename, 'rb')}
victor@7: 
madrouin@20:         if rs_filename is not None:
i@38:             ancillary_file, _ = self.get_ancillary(rs_filename, 'sounding')
i@38: 
i@38:             if ancillary_file.already_on_scc:
i@38:                 logger.warning("Sounding file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
i@38:             else:
i@38:                 logger.debug('Adding sounding file %s' % rs_filename)
i@38:                 files['sounding_file'] = open(rs_filename, 'rb')
madrouin@20: 
ioannis@32:         if ov_filename is not None:
i@38:             ancillary_file, _ = self.get_ancillary(ov_filename, 'overlap')
i@38: 
i@38:             if ancillary_file.already_on_scc:
i@38:                 logger.warning("Overlap file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
i@38:             else:
i@38:                 logger.debug('Adding overlap file %s' % ov_filename)
i@38:                 files['overlap_file'] = open(ov_filename, 'rb')
i@31: 
i@31:         if lr_filename is not None:
i@38:             ancillary_file, _ = self.get_ancillary(lr_filename, 'lidarratio')
i@38: 
i@38:             if ancillary_file.already_on_scc:
i@38:                 logger.warning(
i@38:                     "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
i@38:             else:
i@38:                 logger.debug('Adding lidar ratio file %s' % lr_filename)
i@38:                 files['lidar_ratio_file'] = open(lr_filename, 'rb')
i@31: 
ioannis@32:         logger.info("Uploading of file(s) %s started." % filename)
victor@7: 
i@14:         upload_submit = self.session.post(self.upload_url,
victor@7:                                           data=upload_data,
victor@7:                                           files=files,
victor@7:                                           headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
moritz@29:                                                    'referer': self.upload_url})
victor@7: 
victor@7:         if upload_submit.status_code != 200:
i@14:             logger.warning("Connection error. Status code: %s" % upload_submit.status_code)
victor@7:             return False
victor@7: 
victor@7:         # Check if there was a redirect to a new page.
i@14:         if upload_submit.url == self.upload_url:
victor@7:             measurement_id = False
ioannis@32:             logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.")
victor@7:         else:
victor@7:             measurement_id = re.findall(regex, upload_submit.text)[0]
madrouin@23:             logger.info("Successfully uploaded measurement with id %s." % measurement_id)
victor@7: 
victor@7:         return measurement_id
victor@7: 
victor@7:     def download_files(self, measurement_id, subdir, download_url):
victor@7:         """ Downloads some files from the download_url to the specified
madrouin@20:         subdir. This method is used to download preprocessed file, optical
victor@7:         files etc.
victor@7:         """
ioannis@32:         # TODO: Make downloading more robust (e.g. in case that files do not exist on server).
victor@7:         # Get the file
moritz@29:         request = self.session.get(download_url, stream=True)
moritz@29: 
moritz@29:         if not request.ok:
moritz@29:             raise Exception("Could not download files for measurement '%s'" % measurement_id)
victor@7: 
victor@7:         # Create the dir if it does not exist
victor@7:         local_dir = os.path.join(self.output_dir, measurement_id, subdir)
victor@7:         if not os.path.exists(local_dir):
victor@7:             os.makedirs(local_dir)
victor@7: 
victor@7:         # Save the file by chunk, needed if the file is big.
ioannis@32:         memory_file = BytesIO()
victor@7: 
victor@7:         for chunk in request.iter_content(chunk_size=1024):
victor@7:             if chunk:  # filter out keep-alive new chunks
victor@7:                 memory_file.write(chunk)
victor@7:                 memory_file.flush()
victor@7: 
victor@7:         zip_file = ZipFile(memory_file)
victor@7: 
victor@7:         for ziped_name in zip_file.namelist():
victor@7:             basename = os.path.basename(ziped_name)
victor@7: 
victor@7:             local_file = os.path.join(local_dir, basename)
victor@7: 
victor@7:             with open(local_file, 'wb') as f:
victor@7:                 f.write(zip_file.read(ziped_name))
victor@7: 
ioannis@34:     def download_hirelpp(self, measurement_id):
ioannis@34:         """ Download HiRElPP files for the measurement id. """
ioannis@34:         # Construct the download url
ioannis@34:         download_url = self.download_hirelpp_pattern.format(measurement_id)
ioannis@34:         self.download_files(measurement_id, 'hirelpp', download_url)
ioannis@34: 
ioannis@34:     def download_cloudmask(self, measurement_id):
ioannis@34:         """ Download preprocessed files for the measurement id. """
ioannis@34:         # Construct the download url
ioannis@34:         download_url = self.download_cloudmask_pattern.format(measurement_id)
ioannis@34:         self.download_files(measurement_id, 'cloudmask', download_url)
ioannis@34: 
victor@7:     def download_preprocessed(self, measurement_id):
victor@7:         """ Download preprocessed files for the measurement id. """
victor@7:         # Construct the download url
i@14:         download_url = self.download_preprocessed_pattern.format(measurement_id)
victor@7:         self.download_files(measurement_id, 'scc_preprocessed', download_url)
victor@7: 
victor@7:     def download_optical(self, measurement_id):
victor@7:         """ Download optical files for the measurement id. """
victor@7:         # Construct the download url
moritz@29:         download_url = self.download_optical_pattern.format(measurement_id)
victor@7:         self.download_files(measurement_id, 'scc_optical', download_url)
victor@7: 
victor@7:     def download_graphs(self, measurement_id):
victor@7:         """ Download profile graphs for the measurement id. """
victor@7:         # Construct the download url
i@14:         download_url = self.download_graph_pattern.format(measurement_id)
victor@7:         self.download_files(measurement_id, 'scc_plots', download_url)
victor@7: 
ioannis@34:     def download_elic(self, measurement_id):
ioannis@34:         """ Download profile graphs for the measurement id. """
ioannis@34:         # Construct the download url
ioannis@34:         download_url = self.download_elic_pattern.format(measurement_id)
ioannis@34:         self.download_files(measurement_id, 'elic', download_url)
ioannis@32: 
victor@7:     def rerun_processing(self, measurement_id, monitor=True):
madrouin@26:         measurement, status = self.get_measurement(measurement_id)
victor@7: 
victor@7:         if measurement:
moritz@29:             request = self.session.get(measurement.rerun_processing_url, stream=True)
victor@7: 
victor@7:             if request.status_code != 200:
i@14:                 logger.error(
i@14:                     "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
victor@7:                 return
victor@7: 
victor@7:             if monitor:
victor@7:                 self.monitor_processing(measurement_id)
victor@7: 
victor@7:     def rerun_all(self, measurement_id, monitor=True):
victor@7:         logger.debug("Started rerun_all procedure.")
victor@7: 
victor@7:         logger.debug("Getting measurement %s" % measurement_id)
madrouin@26:         measurement, status = self.get_measurement(measurement_id)
victor@7: 
victor@7:         if measurement:
victor@7:             logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
victor@7: 
moritz@29:             request = self.session.get(measurement.rerun_all_url, stream=True)
victor@7: 
victor@7:             if request.status_code != 200:
victor@7:                 logger.error("Could not rerun pre processing for %s. Status code: %s" %
victor@7:                              (measurement_id, request.status_code))
victor@7:                 return
victor@7: 
victor@7:             if monitor:
victor@7:                 self.monitor_processing(measurement_id)
victor@7: 
ioannis@32:     def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None):
victor@7:         """ Upload a file for processing and wait for the processing to finish.
victor@7:         If the processing is successful, it will download all produced files.
victor@7:         """
victor@7:         logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
victor@7:         # Upload file
madrouin@26:         logger.info("--- Uploading file")
ioannis@32:         measurement_id = self.upload_file(filename, system_id,
ioannis@32:                                           rs_filename=rs_filename,
ioannis@32:                                           lr_filename=lr_filename,
ioannis@32:                                           ov_filename=ov_filename)
victor@7: 
ioannis@32:         if measurement_id and monitor:
ioannis@32:             logger.info("--- Monitoring processing")
moritz@29:             return self.monitor_processing(measurement_id)
ioannis@32: 
moritz@29:         return None
victor@7: 
victor@7:     def monitor_processing(self, measurement_id):
victor@7:         """ Monitor the processing progress of a measurement id"""
victor@7: 
madrouin@24:         # try to deal with error 404
madrouin@24:         error_count = 0
madrouin@24:         error_max = 6
madrouin@26:         time_sleep = 10
madrouin@24: 
madrouin@25:         # try to wait for measurement to appear in API
madrouin@24:         measurement = None
ioannis@32:         logger.info("Looking for measurement %s in SCC", measurement_id)
madrouin@26:         while error_count < error_max:
madrouin@26:             time.sleep(time_sleep)
madrouin@26:             measurement, status = self.get_measurement(measurement_id)
madrouin@26:             if status != 200 and error_count < error_max:
ioannis@32:                 logger.error("Measurement not found. waiting %ds", time_sleep)
madrouin@24:                 error_count += 1
madrouin@26:             else:
madrouin@26:                 break
madrouin@24: 
madrouin@24:         if error_count == error_max:
ioannis@32:             logger.critical("Measurement %s doesn't seem to exist", measurement_id)
madrouin@26:             sys.exit(1)
madrouin@26: 
ioannis@32:         logger.info('Measurement %s found', measurement_id)
madrouin@24: 
victor@7:         if measurement is not None:
victor@7:             while measurement.is_running:
ioannis@34:                 logger.info("Measurement is being processed. Please wait.")
victor@7:                 time.sleep(10)
madrouin@26:                 measurement, status = self.get_measurement(measurement_id)
madrouin@24: 
ioannis@34:             logger.info("Measurement processing finished.")
ioannis@34:             if measurement.hirelpp == 127:
ioannis@34:                 logger.info("Downloading hirelpp files.")
ioannis@34:                 self.download_hirelpp(measurement_id)
ioannis@34:             if measurement.cloudmask == 127:
ioannis@34:                 logger.info("Downloading cloudmask files.")
ioannis@34:                 self.download_cloudmask(measurement_id)
ioannis@34:             if measurement.elpp == 127:
victor@7:                 logger.info("Downloading preprocessed files.")
victor@7:                 self.download_preprocessed(measurement_id)
ioannis@34:             if measurement.elda == 127:
victor@7:                 logger.info("Downloading optical files.")
victor@7:                 self.download_optical(measurement_id)
victor@7:                 logger.info("Downloading graphs.")
victor@7:                 self.download_graphs(measurement_id)
ioannis@34:             if measurement.elic == 127:
ioannis@34:                 logger.info("Downloading preprocessed files.")
ioannis@34:                 self.download_elic(measurement_id)
victor@7:             logger.info("--- Processing finished. ---")
victor@7:         return measurement
victor@7: 
victor@7:     def get_measurement(self, measurement_id):
i@31:         measurement_url = self.api_measurement_pattern.format(measurement_id)
i@31:         logger.debug("Measurement API URL: %s" % measurement_url)
victor@7: 
moritz@29:         response = self.session.get(measurement_url)
victor@7: 
moritz@29:         if not response.ok:
i@14:             logger.error('Could not access API. Status code %s.' % response.status_code)
madrouin@26:             return None, response.status_code
i@14: 
victor@7:         response_dict = response.json()
victor@7: 
victor@7:         if response_dict:
moritz@29:             measurement = Measurement(self.base_url, response_dict)
madrouin@26:             return measurement, response.status_code
victor@7:         else:
victor@7:             logger.error("No measurement with id %s found on the SCC." % measurement_id)
madrouin@26:             return None, response.status_code
victor@7: 
victor@7:     def delete_measurement(self, measurement_id):
victor@7:         """ Deletes a measurement with the provided measurement id. The user
madrouin@20:         should have the appropriate permissions.
madrouin@20: 
victor@7:         The procedures is performed directly through the web interface and
victor@7:         NOT through the API.
victor@7:         """
victor@7:         # Get the measurement object
i@31:         measurement, _ = self.get_measurement(measurement_id)
victor@7: 
victor@7:         # Check that it exists
victor@7:         if measurement is None:
victor@7:             logger.warning("Nothing to delete.")
victor@7:             return None
victor@7: 
victor@7:         # Go the the page confirming the deletion
moritz@29:         delete_url = self.delete_measurement_pattern.format(measurement_id)
victor@7: 
moritz@29:         confirm_page = self.session.get(delete_url)
victor@7: 
victor@7:         # Check that the page opened properly
victor@7:         if confirm_page.status_code != 200:
victor@7:             logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code))
victor@7:             return None
victor@7: 
victor@7:         # Delete the measurement
victor@7:         delete_page = self.session.post(delete_url,
victor@7:                                         data={'post': 'yes'},
victor@7:                                         headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
victor@7:                                                  'referer': delete_url}
victor@7:                                         )
i@31:         if not delete_page.ok:
victor@7:             logger.warning("Something went wrong. Delete page status: {0}".format(
victor@7:                 delete_page.status_code))
victor@7:             return None
victor@7: 
victor@7:         logger.info("Deleted measurement {0}".format(measurement_id))
victor@7:         return True
victor@7: 
victor@7:     def available_measurements(self):
victor@7:         """ Get a list of available measurement on the SCC. """
i@31:         response = self.session.get(self.api_measurements_url)
victor@7:         response_dict = response.json()
victor@7: 
victor@7:         if response_dict:
victor@7:             measurement_list = response_dict['objects']
i@14:             measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list]
victor@7:             logger.info("Found %s measurements on the SCC." % len(measurements))
victor@7:         else:
victor@7:             logger.warning("No response received from the SCC when asked for available measurements.")
i@31:             measurements = None
victor@7: 
victor@7:         return measurements
victor@7: 
moritz@29:     def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None,
moritz@29:                           processing_status=None, optical_processing=None):
moritz@29: 
i@31:         # TODO: Change this to work through the API
i@31: 
moritz@29:         # Need to set to empty string if not specified, we won't get any results
moritz@29:         params = {
moritz@29:             "station": station if station is not None else "",
moritz@29:             "system": system if system is not None else "",
moritz@29:             "stop": stop if stop is not None else "",
moritz@29:             "start": start if start is not None else "",
moritz@29:             "upload_status": upload_status if upload_status is not None else "",
moritz@29:             "preprocessing_status": processing_status if processing_status is not None else "",
moritz@29:             "optical_processing_status": optical_processing if optical_processing is not None else ""
moritz@29:         }
i@31: 
i@31:         response_txt = self.session.get(self.list_measurements_url, params=params).text
moritz@29:         tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL)
moritz@29:         entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL)
moritz@29:         measurement_rgx = re.compile(
moritz@29:             r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*',
moritz@29:             re.DOTALL)
i@31:         matches = tbl_rgx.findall(response_txt)
moritz@29:         if len(matches) != 1:
moritz@29:             return []
moritz@29: 
moritz@29:         ret = []
moritz@29:         for entry in entry_rgx.finditer(matches[0]):
moritz@29:             m = measurement_rgx.match(entry.string[entry.start(0):entry.end(0)])
moritz@29:             if m:
moritz@29:                 name, date, upload, preproc, optical = m.groups()
moritz@29:                 ret.append(
moritz@29:                     Measurement(self.base_url, {"id": name, "upload": int(upload), "pre_processing": int(preproc),
moritz@29:                                                 "processing": int(optical)}))
moritz@29: 
moritz@29:         return ret
moritz@29: 
i@31:     def measurement_id_for_date(self, t1, call_sign, base_number=0):
victor@7:         """ Give the first available measurement id on the SCC for the specific
madrouin@20:         date.
victor@7:         """
victor@7:         date_str = t1.strftime('%Y%m%d')
i@31:         base_id = "%s%s" % (date_str, call_sign)
i@31:         search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id)
victor@7: 
moritz@29:         response = self.session.get(search_url)
victor@7: 
victor@7:         response_dict = response.json()
victor@7: 
victor@7:         measurement_id = None
victor@7: 
victor@7:         if response_dict:
victor@7:             measurement_list = response_dict['objects']
i@31: 
i@31:             if len(measurement_list) == 100:
i@31:                 raise ValueError('No available measurement id found.')
i@31: 
victor@7:             existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
victor@7: 
victor@7:             measurement_number = base_number
i@31:             measurement_id = "%s%02i" % (base_id, measurement_number)
victor@7: 
victor@7:             while measurement_id in existing_ids:
victor@7:                 measurement_number = measurement_number + 1
i@31:                 measurement_id = "%s%02i" % (base_id, measurement_number)
victor@7: 
victor@7:         return measurement_id
victor@7: 
i@38:     def get_ancillary(self, filename, file_type):
i@38:         """
i@38:         Try to get the ancillary file data from the SCC API.
i@38: 
i@38:         The result will always be an API object. If the file does not exist, the .exists property is set to False.
i@38: 
i@38:         Parameters
i@38:         ----------
i@38:         filename : str
i@38:            Filename of the uploaded file.
i@38:         file_type : str
i@38:            Type of ancillary file. One of 'sounding', 'overlap', 'lidarratio'.
i@38: 
i@38:         Returns
i@38:         : AncillaryFile
i@38:            The api object.
i@38:         """
i@38:         assert file_type in ['sounding', 'overlap', 'lidarratio']
i@38: 
i@38:         if file_type == 'sounding':
i@38:             file_url = self.api_sounding_search_pattern.format(filename)
i@38:         elif file_type == 'overlap':
i@38:             file_url = self.api_overlap_search_pattern.format(filename)
i@38:         else:
i@38:             file_url = self.api_lidarratio_search_pattern.format(filename)
i@38: 
i@38:         response = self.session.get(file_url)
i@38: 
i@38:         if not response.ok:
i@38:             logger.error('Could not access API. Status code %s.' % response.status_code)
i@38:             return None, response.status_code
i@38: 
i@38:         response_dict = response.json()
i@38:         object_list = response_dict['objects']
i@38: 
i@38:         logger.debug("Ancillary file JSON: {0}".format(object_list))
i@38: 
i@38:         if object_list:
i@38:             ancillary_file = AncillaryFile(self.api_base_url, object_list[0])  # Assume only one file is returned
i@38:         else:
i@38:             ancillary_file = AncillaryFile(self.api_base_url, None)  # Create an empty object
i@38: 
i@38:         return ancillary_file, response.status_code
i@38: 
i@31:     class PageNotAccessibleError(RuntimeError):
i@31:         pass
i@31: 
victor@7: 
ioannis@17: class ApiObject(object):
victor@7:     """ A generic class object. """
victor@7: 
i@14:     def __init__(self, base_url, dict_response):
i@14:         self.base_url = base_url
victor@7: 
victor@7:         if dict_response:
victor@7:             # Add the dictionary key value pairs as object properties
victor@7:             for key, value in dict_response.items():
ioannis@17:                 # logger.debug('Setting key {0} to value {1}'.format(key, value))
ioannis@17:                 try:
ioannis@17:                     setattr(self, key, value)
ioannis@17:                 except:
ioannis@17:                     logger.warning('Could not set attribute {0} to value {1}'.format(key, value))
victor@7:             self.exists = True
victor@7:         else:
victor@7:             self.exists = False
victor@7: 
victor@7: 
victor@7: class Measurement(ApiObject):
victor@7:     """ This class represents the measurement object as returned in the SCC API.
victor@7:     """
victor@7: 
victor@7:     @property
victor@7:     def rerun_processing_url(self):
ioannis@34:         url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/')
i@14:         return url_pattern.format(self.id)
victor@7: 
victor@7:     @property
victor@7:     def rerun_all_url(self):
i@14:         ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/')
i@14:         return ulr_pattern.format(self.id)
victor@7: 
victor@7:     def __str__(self):
victor@7:         return "%s: %s, %s, %s" % (self.id,
victor@7:                                    self.upload,
victor@7:                                    self.pre_processing,
victor@7:                                    self.processing)
victor@7: 
victor@7: 
i@38: class AncillaryFile(ApiObject):
i@38:     """ This class represents the ancilalry file object as returned in the SCC API.
i@38:     """
i@38:     @property
i@38:     def already_on_scc(self):
i@38:         if self.exists is False:
i@38:             return False
i@38: 
i@38:         return not self.status == 'missing'
i@38: 
i@38:     def __str__(self):
i@38:         return "%s: %s, %s" % (self.id,
i@38:                                self.filename,
i@38:                                self.status)
i@38: 
i@38: 
ioannis@32: def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
ioannis@32:     """ Shortcut function to process a file to the SCC. """
ioannis@32:     logger.info("Processing file %s, using system %s" % (filename, system_id))
victor@7: 
i@14:     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
i@14:     scc.login(settings['website_credentials'])
ioannis@32:     measurement = scc.process(filename, system_id,
ioannis@32:                               monitor=monitor,
ioannis@32:                               rs_filename=rs_filename,
ioannis@32:                               lr_filename=lr_filename,
ioannis@32:                               ov_filename=ov_filename)
victor@7:     scc.logout()
victor@7:     return measurement
victor@7: 
victor@7: 
ioannis@32: def delete_measurements(measurement_ids, settings):
moritz@29:     """ Shortcut function to delete measurements from the SCC. """
moritz@29:     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
moritz@29:     scc.login(settings['website_credentials'])
moritz@29:     for m_id in measurement_ids:
moritz@29:         logger.info("Deleting %s" % m_id)
moritz@29:         scc.delete_measurement(m_id)
moritz@29:     scc.logout()
moritz@29: 
moritz@29: 
moritz@29: def rerun_all(measurement_ids, monitor, settings):
moritz@29:     """ Shortcut function to rerun measurements from the SCC. """
i@14: 
i@14:     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
i@14:     scc.login(settings['website_credentials'])
moritz@29:     for m_id in measurement_ids:
moritz@29:         logger.info("Rerunning all products for %s" % m_id)
moritz@29:         scc.rerun_all(m_id, monitor)
victor@7:     scc.logout()
victor@7: 
victor@7: 
moritz@29: def rerun_processing(measurement_ids, monitor, settings):
victor@7:     """ Shortcut function to delete a measurement from the SCC. """
i@14: 
i@14:     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
i@14:     scc.login(settings['website_credentials'])
moritz@29:     for m_id in measurement_ids:
moritz@29:         logger.info("Rerunning (optical) processing for %s" % m_id)
moritz@29:         scc.rerun_processing(m_id, monitor)
moritz@29:     scc.logout()
moritz@29: 
moritz@29: 
moritz@29: def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None,
moritz@29:                       preprocessing_status=None,
moritz@29:                       optical_processing=None):
moritz@29:     """List all available measurements"""
moritz@29:     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
moritz@29:     scc.login(settings['website_credentials'])
moritz@29:     ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status,
moritz@29:                                 processing_status=preprocessing_status, optical_processing=optical_processing)
moritz@29:     for entry in ret:
moritz@29:         print("%s" % entry.id)
victor@7:     scc.logout()
victor@7: 
victor@7: 
moritz@29: def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings):
moritz@29:     """Download all measurements for the specified IDs"""
i@14:     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
i@14:     scc.login(settings['website_credentials'])
moritz@29:     for m_id in measurement_ids:
moritz@29:         if download_preproc:
moritz@29:             logger.info("Downloading preprocessed files for '%s'" % m_id)
moritz@29:             scc.download_preprocessed(m_id)
moritz@29:             logger.info("Complete")
moritz@29:         if download_optical:
moritz@29:             logger.info("Downloading optical files for '%s'" % m_id)
moritz@29:             scc.download_optical(m_id)
moritz@29:             logger.info("Complete")
moritz@29:         if download_graph:
moritz@29:             logger.info("Downloading profile graph files for '%s'" % m_id)
moritz@29:             scc.download_graphs(m_id)
moritz@29:             logger.info("Complete")
i@14: 
i@14: 
moritz@29: def settings_from_path(config_file_path):
i@14:     """ Read the configuration file.
i@14: 
i@14:     The file should be in YAML syntax."""
i@14: 
i@14:     if not os.path.isfile(config_file_path):
moritz@29:         raise argparse.ArgumentTypeError("Wrong path for configuration file (%s)" % config_file_path)
i@14: 
i@14:     with open(config_file_path) as yaml_file:
i@14:         try:
i@14:             settings = yaml.safe_load(yaml_file)
i@14:             logger.debug("Read settings file(%s)" % config_file_path)
moritz@29:         except Exception:
moritz@29:             raise argparse.ArgumentTypeError("Could not parse YAML file (%s)" % config_file_path)
i@14: 
i@14:     # YAML limitation: does not read tuples
i@14:     settings['basic_credentials'] = tuple(settings['basic_credentials'])
i@14:     settings['website_credentials'] = tuple(settings['website_credentials'])
i@14:     return settings
i@14: 
i@14: 
moritz@29: # Setup for command specific parsers
moritz@29: def setup_delete(parser):
moritz@29:     def delete_from_args(parsed):
ioannis@32:         delete_measurements(parsed.IDs, parsed.config)
moritz@29: 
moritz@29:     parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.")
moritz@29:     parser.set_defaults(execute=delete_from_args)
moritz@29: 
moritz@29: 
moritz@29: def setup_rerun_all(parser):
moritz@29:     def rerun_all_from_args(parsed):
moritz@29:         rerun_all(parsed.IDs, parsed.process, parsed.config)
moritz@29: 
moritz@29:     parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun.")
moritz@29:     parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
moritz@29:                         action="store_true")
moritz@29:     parser.set_defaults(execute=rerun_all_from_args)
moritz@29: 
moritz@29: 
moritz@29: def setup_rerun_processing(parser):
moritz@29:     def rerun_processing_from_args(parsed):
moritz@29:         rerun_processing(parsed.IDs, parsed.process, parsed.config)
moritz@29: 
moritz@29:     parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.")
moritz@29:     parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
moritz@29:                         action="store_true")
moritz@29:     parser.set_defaults(execute=rerun_processing_from_args)
moritz@29: 
moritz@29: 
moritz@29: def setup_process_file(parser):
ioannis@32:     """ Upload and monitor processing progress."""
moritz@29:     def process_file_from_args(parsed):
ioannis@32:         process_file(parsed.filename, parsed.system, parsed.config, monitor=True,
ioannis@32:                      rs_filename=parsed.radiosounding,
ioannis@32:                      ov_filename=parsed.overlap,
ioannis@32:                      lr_filename=parsed.lidarratio)
moritz@29: 
moritz@29:     parser.add_argument("filename", help="Measurement file name or path.")
moritz@29:     parser.add_argument("system", help="Processing system id.")
i@30:     parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
ioannis@32:     parser.add_argument("--overlap", default=None, help="Overlap file name or path")
ioannis@32:     parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
ioannis@32: 
moritz@29:     parser.set_defaults(execute=process_file_from_args)
moritz@29: 
moritz@29: 
moritz@29: def setup_upload_file(parser):
ioannis@32:     """ Upload but do not monitor processing progress. """
moritz@29:     def upload_file_from_args(parsed):
ioannis@32:         process_file(parsed.filename, parsed.system, parsed.config, monitor=False,
ioannis@32:                      rs_filename=parsed.radiosounding,
ioannis@32:                      ov_filename=parsed.overlap,
ioannis@32:                      lr_filename=parsed.lidarratio)
moritz@29: 
moritz@29:     parser.add_argument("filename", help="Measurement file name or path.")
moritz@29:     parser.add_argument("system", help="Processing system id.")
i@30:     parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
ioannis@32:     parser.add_argument("--overlap", default=None, help="Overlap file name or path")
ioannis@32:     parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
i@30: 
moritz@29:     parser.set_defaults(execute=upload_file_from_args)
moritz@29: 
moritz@29: 
moritz@29: def setup_list_measurements(parser):
moritz@29:     def list_measurements_from_args(parsed):
moritz@29:         list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start,
moritz@29:                           stop=parsed.stop,
moritz@29:                           upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status,
moritz@29:                           optical_processing=parsed.optical_processing_status)
moritz@29: 
moritz@29:     def status(arg):
moritz@29:         if -127 <= int(arg) <= 127:
moritz@29:             return arg
moritz@29:         else:
moritz@29:             raise argparse.ArgumentTypeError("Status must be between -127 and 127")
moritz@29: 
moritz@29:     def date(arg):
moritz@29:         if re.match(r'\d{4}-\d{2}-\d{2}', arg):
moritz@29:             return arg
moritz@29:         else:
moritz@29:             raise argparse.ArgumentTypeError("Date must be in format 'YYYY-MM-DD'")
moritz@29: 
moritz@29:     parser.add_argument("--station", help="Filter for only the selected station")
moritz@29:     parser.add_argument("--system", help="Filter for only the selected station")
moritz@29:     parser.add_argument("--start", help="Filter for only the selected station", type=date)
moritz@29:     parser.add_argument("--stop", help="Filter for only the selected station", type=date)
moritz@29:     parser.add_argument("--upload-status", help="Filter for only the selected station", type=status)
moritz@29:     parser.add_argument("--preprocessing-status", help="Filter for only the selected station", type=status)
moritz@29:     parser.add_argument("--optical-processing-status", help="Filter for only the selected station", type=status)
moritz@29:     parser.set_defaults(execute=list_measurements_from_args)
moritz@29: 
moritz@29: 
moritz@29: def setup_download_measurements(parser):
moritz@29:     def download_measurements_from_args(parsed):
moritz@29:         preproc = parsed.download_preprocessed
moritz@29:         optical = parsed.download_optical
moritz@29:         graphs = parsed.download_profile_graphs
moritz@29:         if not preproc and not graphs:
moritz@29:             optical = True
moritz@29:         download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config)
moritz@29: 
moritz@29:     parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+")
moritz@29:     parser.add_argument("--download-preprocessed", action="store_true", help="Download preprocessed files.")
moritz@29:     parser.add_argument("--download-optical", action="store_true",
moritz@29:                         help="Download optical files (default if no other download is used).")
moritz@29:     parser.add_argument("--download-profile-graphs", action="store_true", help="Download profile graph files.")
moritz@29:     parser.set_defaults(execute=download_measurements_from_args)
moritz@29: 
moritz@29: 
victor@7: def main():
victor@7:     # Define the command line arguments.
victor@7:     parser = argparse.ArgumentParser()
moritz@29:     subparsers = parser.add_subparsers()
moritz@29: 
moritz@29:     delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
ioannis@32:     rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.")
moritz@29:     rerun_processing_parser = subparsers.add_parser("rerun-processing",
i@36:                                                     help="Rerun processing routines for a measurement.")
i@36:     process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download processing results.")
moritz@29:     upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.")
ioannis@32:     list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
moritz@29:     download_parser = subparsers.add_parser("download", help="Download selected measurements.")
moritz@29: 
moritz@29:     setup_delete(delete_parser)
moritz@29:     setup_rerun_all(rerun_all_parser)
moritz@29:     setup_rerun_processing(rerun_processing_parser)
moritz@29:     setup_process_file(process_file_parser)
moritz@29:     setup_upload_file(upload_file_parser)
moritz@29:     setup_list_measurements(list_parser)
moritz@29:     setup_download_measurements(download_parser)
victor@7: 
victor@7:     # Verbosity settings from http://stackoverflow.com/a/20663028
victor@7:     parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
victor@7:                         dest="loglevel", const=logging.DEBUG, default=logging.INFO,
victor@7:                         )
victor@7:     parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
victor@7:                         dest="loglevel", const=logging.WARNING
victor@7:                         )
victor@7: 
ioannis@32:     # Setup default config location
moritz@29:     home = os.path.expanduser("~")
moritz@29:     default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml"))
moritz@29:     parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path,
moritz@29:                         default=default_config_location)
moritz@29: 
victor@7:     args = parser.parse_args()
victor@7: 
victor@7:     # Get the logger with the appropriate level
victor@7:     logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
victor@7: 
moritz@29:     # Dispatch to appropriate function
moritz@29:     args.execute(args)
moritz@29: 
i@14: 
moritz@29: # When running through terminal
moritz@29: if __name__ == '__main__':
moritz@29:     main()