ioannis@67: """ This is a script that allows interaction with the SCC through the command line.
ioannis@67:
ioannis@67: It is based on the requests module for accessing the server.
ioannis@67:
ioannis@67: Most of the interactions are done through the web interface, i.e. by mimicking user interaction with the
ioannis@67: SCC website (i.e. user login, data submission, etc.). In few cases, the SCC API is also used.
ioannis@67:
ioannis@67: Most of the functionality is included in the SCC class. The class is used to login into the SCC website and automate
ioannis@67: interaction with the site (i.e. upload a file, get measurement status, etc.).
ioannis@67:
ioannis@67: Two other classes (Measurement, AncillaryFile) are used in some cases to handle the output of the SCC API.
ioannis@67:
ioannis@67: Several shortcut functions are defined to perform specific tasks using the SCC class (e.g. process_file, delete_measurements etc).
ioannis@67: """
ioannis@43: import sys
ioannis@43:
victor@7: import requests
victor@7:
ioannis@43: try:
ioannis@43: import urllib.parse as urlparse # Python 3
ioannis@43: except ImportError:
ioannis@17: import urlparse # Python 2
ioannis@43:
victor@7: import argparse
madrouin@24: import datetime
madrouin@24: import logging
victor@7: import os
victor@7: import re
ioannis@32: from io import BytesIO
ioannis@45:
victor@7: import time
i@40:
victor@7: from zipfile import ZipFile
madrouin@24:
i@14: import yaml
victor@7:
ioannis@43: import netCDF4 as netcdf
ioannis@43:
ioannis@43: requests.packages.urllib3.disable_warnings()
i@14: logger = logging.getLogger(__name__)
victor@7:
victor@7: # The regex to find the measurement id from the measurement page
victor@7: # This should be read from the uploaded file, but would require an extra NetCDF module.
i@36: regex = "
Measurement (?P.{12,15}) " # {12, 15} to handle both old- and new-style measurement ids.
victor@7:
victor@7:
victor@7: class SCC:
ioannis@67: """A class that will attempt to interact SCC server.
i@14:
ioannis@67: Most interactions are by simulating a normal browser session. In the current
ioannis@67: version few checks are performed before upload a file, and no feedback is given in case the upload
ioannis@67: fails.
victor@7: """
victor@7:
i@14: def __init__(self, auth, output_dir, base_url):
moritz@29:
victor@7: self.auth = auth
victor@7: self.output_dir = output_dir
i@14: self.base_url = base_url
victor@7: self.session = requests.Session()
moritz@29: self.session.auth = auth
moritz@29: self.session.verify = False
victor@7:
ioannis@67: # Setup SCC server URLS for later use
i@14: self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
i@31: self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/')
i@31: self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
i@31:
i@14: self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
ioannis@54: self.measurement_page_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/')
ioannis@43: self.download_hirelpp_pattern = urlparse.urljoin(self.base_url,
ioannis@67: 'data_processing/measurements/{0}/download-hirelpp/')
ioannis@43: self.download_cloudmask_pattern = urlparse.urljoin(self.base_url,
ioannis@67: 'data_processing/measurements/{0}/download-cloudmask/')
ioannis@34:
ioannis@43: self.download_elpp_pattern = urlparse.urljoin(self.base_url,
ioannis@67: 'data_processing/measurements/{0}/download-preprocessed/')
ioannis@43: self.download_elda_pattern = urlparse.urljoin(self.base_url,
ioannis@67: 'data_processing/measurements/{0}/download-optical/')
ioannis@45: self.download_plots_pattern = urlparse.urljoin(self.base_url,
moritz@29: 'data_processing/measurements/{0}/download-plots/')
ioannis@43: self.download_elic_pattern = urlparse.urljoin(self.base_url,
ioannis@67: 'data_processing/measurements/{0}/download-elic/')
i@14: self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
i@31:
ioannis@67: # Setup API URLs for later use
i@14: self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
i@31: self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/')
i@31: self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements')
i@38: self.api_sounding_search_pattern = urlparse.urljoin(self.api_base_url, 'sounding_files/?filename={0}')
i@38: self.api_lidarratio_search_pattern = urlparse.urljoin(self.api_base_url, 'lidarratio_files/?filename={0}')
i@38: self.api_overlap_search_pattern = urlparse.urljoin(self.api_base_url, 'overlap_files/?filename={0}')
ioannis@43:
i@14: def login(self, credentials):
ioannis@67: """ Login to the SCC.
ioannis@67:
ioannis@67: Parameters
ioannis@67: ----------
ioannis@67: credentials : tuple or list
ioannis@67: A list or tuple in the form (username, password).
ioannis@67: """
victor@7: logger.debug("Attempting to login to SCC, username %s." % credentials[0])
moritz@29: login_credentials = {'username': credentials[0],
moritz@29: 'password': credentials[1]}
victor@7:
i@14: logger.debug("Accessing login page at %s." % self.login_url)
victor@7:
ioannis@67: # Get login form
moritz@29: login_page = self.session.get(self.login_url)
i@14:
i@31: if not login_page.ok:
i@31: raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code)
victor@7:
i@31: logger.debug("Submitting credentials.")
victor@7: # Submit the login data
i@14: login_submit = self.session.post(self.login_url,
moritz@29: data=login_credentials,
victor@7: headers={'X-CSRFToken': login_page.cookies['csrftoken'],
moritz@29: 'referer': self.login_url})
victor@7: return login_submit
victor@7:
victor@7: def logout(self):
ioannis@67: """ Logout from the SCC """
i@31: return self.session.get(self.logout_url, stream=True)
victor@7:
ioannis@67: def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None,
ioannis@67: ov_filename=None, lr_filename=None):
ioannis@67: """ Upload a file for processing.
ioannis@67:
ioannis@67: If the upload is successful, it returns the measurement id.
ioannis@67:
ioannis@67:
ioannis@67: Parameters
ioannis@67: ----------
ioannis@67: filename : str
ioannis@67: File path of the file to upload
ioannis@67: system_id : int
ioannis@67: System id to be used in the processing
ioannis@67: force_upload : bool
ioannis@67: If True, if a measurement with the same ID is found on the server, it will be first deleted and the
ioannis@67: file current file will be uploaded. If False, the file will not be uploaded if the measurement ID is
ioannis@67: already present on the SCC server.
ioannis@67: delete_related : bool
ioannis@67: Answer to delete related question when deleting existing measurements from the SCC.
ioannis@67: rs_filename, ov_filename, lr_filename : str
ioannis@67: Ancillary files pahts to be uploaded.
ioannis@67: """
ioannis@67: # Get the measurement ID from the netcdf file
ioannis@43: measurement_id = self.measurement_id_from_file(filename)
ioannis@43:
ioannis@67: # Handle possible existing measurements with the same ID on the SCC server.
ioannis@43: logger.debug('Checking if a measurement with the same id already exists on the SCC server.')
ioannis@50: existing_measurement, _ = self.get_measurement(measurement_id)
ioannis@43:
ioannis@43: if existing_measurement:
ioannis@43: if force_upload:
ioannis@43: logger.info(
ioannis@43: "Measurement with id {} already exists on the SCC. Trying to delete it...".format(measurement_id))
ioannis@43: self.delete_measurement(measurement_id, delete_related)
ioannis@43: else:
ioannis@43: logger.error(
ioannis@43: "Measurement with id {} already exists on the SCC. Use --force_upload flag to overwrite it.".format(
ioannis@43: measurement_id))
ioannis@67: # TODO: Implement handling at the proper place. Exiting here does not allow the SCC class to be
ioannis@67: # used by external programs. Instead an exception should be raised.
ioannis@43: sys.exit(1)
ioannis@43:
ioannis@67: # Upload the file(s)
ioannis@67:
victor@7: # Get submit page
moritz@29: upload_page = self.session.get(self.upload_url)
victor@7:
victor@7: # Submit the data
ioannis@54: upload_data = {'system': system_id,
ioannis@54: 'delay': delay}
ioannis@54:
ioannis@54: logger.debug("Submitted processing parameters - System: {}, Delay: {}".format(system_id, delay))
ioannis@54:
victor@7: files = {'data': open(filename, 'rb')}
victor@7:
ioannis@67: # Add ancillary files to be uploaded
madrouin@20: if rs_filename is not None:
i@38: ancillary_file, _ = self.get_ancillary(rs_filename, 'sounding')
i@38:
i@38: if ancillary_file.already_on_scc:
ioannis@67: logger.warning(
ioannis@67: "Sounding file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
i@38: else:
i@38: logger.debug('Adding sounding file %s' % rs_filename)
i@38: files['sounding_file'] = open(rs_filename, 'rb')
madrouin@20:
ioannis@32: if ov_filename is not None:
i@38: ancillary_file, _ = self.get_ancillary(ov_filename, 'overlap')
i@38:
i@38: if ancillary_file.already_on_scc:
ioannis@67: logger.warning(
ioannis@67: "Overlap file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
i@38: else:
i@38: logger.debug('Adding overlap file %s' % ov_filename)
i@38: files['overlap_file'] = open(ov_filename, 'rb')
i@31:
i@31: if lr_filename is not None:
i@38: ancillary_file, _ = self.get_ancillary(lr_filename, 'lidarratio')
i@38:
i@38: if ancillary_file.already_on_scc:
i@38: logger.warning(
ioannis@67: "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(
ioannis@67: ancillary_file))
i@38: else:
i@38: logger.debug('Adding lidar ratio file %s' % lr_filename)
i@38: files['lidar_ratio_file'] = open(lr_filename, 'rb')
i@31:
ioannis@67: # Upload the files
i@14: logger.info("Uploading of file %s started." % filename)
victor@7:
i@14: upload_submit = self.session.post(self.upload_url,
victor@7: data=upload_data,
victor@7: files=files,
victor@7: headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
moritz@29: 'referer': self.upload_url})
victor@7:
victor@7: if upload_submit.status_code != 200:
i@14: logger.warning("Connection error. Status code: %s" % upload_submit.status_code)
victor@7: return False
victor@7:
ioannis@67: # Check if there was a redirect to a new page. If not, something went wrong
i@14: if upload_submit.url == self.upload_url:
victor@7: measurement_id = False
ioannis@32: logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.")
victor@7: else:
ioannis@67: # TODO: Check if this is needed. This was used when the measurement ID was not read from the input file.
ioannis@67: measurement_id = re.findall(regex, upload_submit.text)[0] # Get the measurement ID from the output page
ioannis@43: logger.info("Successfully uploaded measurement with id %s." % measurement_id)
ioannis@67: logger.info("You can monitor the processing progress online: {}".format(
ioannis@67: self.measurement_page_pattern.format(measurement_id)))
ioannis@43: return measurement_id
ioannis@43:
ioannis@43: @staticmethod
ioannis@43: def measurement_id_from_file(filename):
ioannis@67: """ Get the measurement id from the input file.
ioannis@67:
ioannis@67: Parameters
ioannis@67: ----------
ioannis@67: filename : str
ioannis@67: File path of the input file.
ioannis@67: """
ioannis@43:
ioannis@43: if not os.path.isfile(filename):
ioannis@43: logger.error("File {} does not exist.".format(filename))
ioannis@43: sys.exit(1)
ioannis@43:
ioannis@43: with netcdf.Dataset(filename) as f:
ioannis@43: try:
ioannis@43: measurement_id = f.Measurement_ID
ioannis@43: except AttributeError:
ioannis@43: logger.error(
ioannis@43: "Input file {} does not contain a Measurement_ID global attribute. Wrong file format?".format(
ioannis@43: filename))
ioannis@43: sys.exit(1)
victor@7:
victor@7: return measurement_id
victor@7:
victor@7: def download_files(self, measurement_id, subdir, download_url):
victor@7: """ Downloads some files from the download_url to the specified
ioannis@67: subdir.
ioannis@67:
ioannis@67: This is a general method used to download preprocessed file, optical
ioannis@67: files by other, file-specific, methods.
victor@7: """
ioannis@32: # TODO: Make downloading more robust (e.g. in case that files do not exist on server).
victor@7: # Get the file
moritz@29: request = self.session.get(download_url, stream=True)
moritz@29:
moritz@29: if not request.ok:
moritz@29: raise Exception("Could not download files for measurement '%s'" % measurement_id)
victor@7:
victor@7: # Create the dir if it does not exist
victor@7: local_dir = os.path.join(self.output_dir, measurement_id, subdir)
victor@7: if not os.path.exists(local_dir):
victor@7: os.makedirs(local_dir)
victor@7:
victor@7: # Save the file by chunk, needed if the file is big.
ioannis@43: memory_file = BytesIO()
victor@7:
victor@7: for chunk in request.iter_content(chunk_size=1024):
victor@7: if chunk: # filter out keep-alive new chunks
victor@7: memory_file.write(chunk)
victor@7: memory_file.flush()
victor@7:
victor@7: zip_file = ZipFile(memory_file)
victor@7:
victor@7: for ziped_name in zip_file.namelist():
victor@7: basename = os.path.basename(ziped_name)
victor@7:
victor@7: local_file = os.path.join(local_dir, basename)
victor@7:
victor@7: with open(local_file, 'wb') as f:
victor@7: f.write(zip_file.read(ziped_name))
victor@7:
ioannis@43: def download_hirelpp(self, measurement_id):
ioannis@43: """ Download hirelpp files for the measurement id. """
ioannis@43: # Construct the download url
ioannis@43: download_url = self.download_hirelpp_pattern.format(measurement_id)
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'hirelpp', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download HiRElPP files. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
ioannis@43:
ioannis@43: def download_cloudmask(self, measurement_id):
ioannis@43: """ Download cloudmask files for the measurement id. """
ioannis@43: # Construct the download url
ioannis@43: download_url = self.download_cloudmask_pattern.format(measurement_id)
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'cloudscreen', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download cloudscreen files. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
ioannis@43:
ioannis@43: def download_elpp(self, measurement_id):
victor@7: """ Download preprocessed files for the measurement id. """
victor@7: # Construct the download url
ioannis@43: download_url = self.download_elpp_pattern.format(measurement_id)
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'elpp', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download ElPP files. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
victor@7:
ioannis@43: def download_elda(self, measurement_id):
victor@7: """ Download optical files for the measurement id. """
victor@7: # Construct the download url
ioannis@43: download_url = self.download_elda_pattern.format(measurement_id)
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'elda', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download ELDA files. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
victor@7:
ioannis@43: def download_plots(self, measurement_id):
victor@7: """ Download profile graphs for the measurement id. """
victor@7: # Construct the download url
ioannis@45: download_url = self.download_plots_pattern.format(measurement_id)
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'elda_plots', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download ELDA plots. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
victor@7:
ioannis@43: def download_elic(self, measurement_id):
ioannis@43: """ Download ELIC files for the measurement id. """
ioannis@43: # Construct the download url
ioannis@43: download_url = self.download_elic_pattern.format(measurement_id)
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'elic', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download ELIC files. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
ioannis@43:
ioannis@43: def download_eldec(self, measurement_id):
ioannis@43: """ Download ELDEC files for the measurement id. """
ioannis@43: # Construct the download url
ioannis@43: download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now
ioannis@43: try:
ioannis@50: self.download_files(measurement_id, 'eldec', download_url)
ioannis@43: except Exception as e:
ioannis@43: logger.error("Could not download EDELC files. Error message: {}".format(e))
ioannis@43: logger.debug('Download exception:', exc_info=True)
ioannis@43:
ioannis@43: def rerun_elpp(self, measurement_id, monitor=True):
ioannis@45: logger.debug("Started rerun_elpp procedure.")
ioannis@45:
ioannis@45: logger.debug("Getting measurement %s" % measurement_id)
madrouin@26: measurement, status = self.get_measurement(measurement_id)
victor@7:
victor@7: if measurement:
ioannis@45: logger.debug("Attempting to rerun ElPP through %s." % measurement.rerun_all_url)
ioannis@45: request = self.session.get(measurement.rerun_elpp_url, stream=True)
victor@7:
victor@7: if request.status_code != 200:
i@14: logger.error(
i@14: "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
ioannis@45: else:
ioannis@45: logger.info("Rerun-elpp command submitted successfully for id {}.".format(measurement_id))
victor@7:
victor@7: if monitor:
victor@7: self.monitor_processing(measurement_id)
victor@7:
victor@7: def rerun_all(self, measurement_id, monitor=True):
victor@7: logger.debug("Started rerun_all procedure.")
victor@7:
victor@7: logger.debug("Getting measurement %s" % measurement_id)
madrouin@26: measurement, status = self.get_measurement(measurement_id)
victor@7:
victor@7: if measurement:
victor@7: logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
victor@7:
moritz@29: request = self.session.get(measurement.rerun_all_url, stream=True)
victor@7:
victor@7: if request.status_code != 200:
victor@7: logger.error("Could not rerun pre processing for %s. Status code: %s" %
victor@7: (measurement_id, request.status_code))
ioannis@45: else:
ioannis@45: logger.info("Rerun-all command submitted successfully for id {}.".format(measurement_id))
victor@7:
victor@7: if monitor:
victor@7: self.monitor_processing(measurement_id)
victor@7:
ioannis@67: def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, rs_filename=None,
ioannis@67: lr_filename=None, ov_filename=None):
victor@7: """ Upload a file for processing and wait for the processing to finish.
victor@7: If the processing is successful, it will download all produced files.
victor@7: """
victor@7: logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
victor@7: # Upload file
ioannis@50: logger.info("Uploading file.")
ioannis@45: measurement_id = self.upload_file(filename, system_id, force_upload, delete_related,
ioannis@54: delay=delay,
ioannis@32: rs_filename=rs_filename,
ioannis@32: lr_filename=lr_filename,
ioannis@32: ov_filename=ov_filename)
ioannis@43:
ioannis@54: if monitor and (delay > 0):
ioannis@54: logger.warning("Will not start monitoring, since a delay was specified: {} hours.".format(delay))
ioannis@54: return None
ioannis@54:
ioannis@32: if measurement_id and monitor:
ioannis@54: logger.info("Monitoring processing.")
moritz@29: return self.monitor_processing(measurement_id)
victor@7:
moritz@29: return None
victor@7:
ioannis@61: def monitor_processing(self, measurement_id, retry_max=2, time_sleep=2, exit_if_missing=True):
victor@7: """ Monitor the processing progress of a measurement id"""
victor@7:
madrouin@24: # try to deal with error 404
ioannis@61: attempts_count = 0
ioannis@61: max_attempts = retry_max + 1
madrouin@24:
ioannis@67: # try to wait for measurement to appear in API. A user has reported that this does not happen immediately.
madrouin@24: measurement = None
ioannis@50: logger.info("Looking for measurement %s on the SCC.", measurement_id)
ioannis@62:
ioannis@61: while attempts_count < max_attempts:
ioannis@61: attempts_count += 1
madrouin@26: measurement, status = self.get_measurement(measurement_id)
ioannis@61: if status != 200:
ioannis@61: logger.warning("Measurement not found.")
ioannis@61: if attempts_count < max_attempts:
ioannis@61: logger.warning("Waiting %ds.", time_sleep)
ioannis@61: time.sleep(time_sleep)
madrouin@26: else:
madrouin@26: break
ioannis@62: print("Measurement: {}".format(measurement))
madrouin@24:
ioannis@62: if measurement is None:
ioannis@61: logger.error("Measurement %s doesn't seem to exist.", measurement_id)
ioannis@61: if exit_if_missing:
ioannis@61: sys.exit(1)
ioannis@61: else:
ioannis@61: return measurement
madrouin@26:
ioannis@45: logger.info('Measurement %s found.', measurement_id)
ioannis@62: while not measurement.has_finished:
ioannis@62: measurement.log_processing_status()
ioannis@62: time.sleep(10)
ioannis@62: measurement, status = self.get_measurement(measurement_id)
madrouin@24:
ioannis@62: logger.info("Measurement processing finished.")
ioannis@62: measurement.log_detailed_status()
ioannis@53:
ioannis@62: if measurement.hirelpp == 127:
ioannis@62: logger.info("Downloading HiRElPP files.")
ioannis@62: self.download_hirelpp(measurement_id)
ioannis@62: if measurement.cloudmask == 127:
ioannis@62: logger.info("Downloading cloud screening files.")
ioannis@62: self.download_cloudmask(measurement_id)
ioannis@62: if measurement.elpp == 127:
ioannis@62: logger.info("Downloading ElPP files.")
ioannis@62: self.download_elpp(measurement_id)
ioannis@62: if measurement.elda == 127:
ioannis@62: logger.info("Downloading ELDA files.")
ioannis@62: self.download_elda(measurement_id)
ioannis@62: logger.info("Downloading ELDA plots.")
ioannis@62: self.download_plots(measurement_id)
ioannis@62: if measurement.elic == 127:
ioannis@62: logger.info("Downloading ELIC files.")
ioannis@62: self.download_elic(measurement_id)
ioannis@67: if measurement.is_calibration and measurement.eldec == 0:
ioannis@62: logger.info("Downloading ELDEC files.")
ioannis@62: self.download_eldec(measurement_id)
ioannis@62: logger.info("--- Processing finished. ---")
ioannis@43:
victor@7: return measurement
victor@7:
ioannis@43: def get_measurement(self, measurement_id):
ioannis@67: """ Get a measurement information from the SCC API.
victor@7:
ioannis@67: Parameters
ioannis@67: ----------
ioannis@67: measurement_id : str
ioannis@67: The measurement ID to search.
ioannis@67:
ioannis@67: Returns
ioannis@67: -------
ioannis@67: : Measurement object or None
ioannis@67: If the measurement is found, a Measurement object is returned. If not, it returns None
ioannis@67: """
ioannis@67: # TODO: Consider to homogenize with get_ancillary method (i.e. always return a Measurement object).
ioannis@67:
ioannis@67: if measurement_id is None: # TODO: Is this still required?
victor@7: return None
victor@7:
ioannis@67: # Access the API
i@31: measurement_url = self.api_measurement_pattern.format(measurement_id)
i@31: logger.debug("Measurement API URL: %s" % measurement_url)
victor@7:
moritz@29: response = self.session.get(measurement_url)
victor@7:
ioannis@43: response_dict = None
ioannis@45:
ioannis@43: if response.status_code == 200:
ioannis@43: response_dict = response.json()
ioannis@45: elif response.status_code == 404:
ioannis@43: logger.info("No measurement with id %s found on the SCC." % measurement_id)
ioannis@45: else:
i@14: logger.error('Could not access API. Status code %s.' % response.status_code)
victor@7:
ioannis@55: # TODO: Implement better handling for status 401.
ioannis@55:
victor@7: if response_dict:
ioannis@43: measurement = Measurement(self.base_url, response_dict)
victor@7: else:
ioannis@45: measurement = None
victor@7:
ioannis@45: return measurement, response.status_code
victor@7:
ioannis@45: def delete_measurement(self, measurement_id, delete_related):
victor@7: """ Deletes a measurement with the provided measurement id. The user
ioannis@67: should have the appropriate permissions (i.e. access to the admin site).
madrouin@20:
victor@7: The procedures is performed directly through the web interface and
victor@7: NOT through the API.
victor@7: """
victor@7: # Get the measurement object
i@31: measurement, _ = self.get_measurement(measurement_id)
victor@7:
victor@7: # Check that it exists
victor@7: if measurement is None:
victor@7: logger.warning("Nothing to delete.")
victor@7: return None
victor@7:
victor@7: # Go the the page confirming the deletion
moritz@29: delete_url = self.delete_measurement_pattern.format(measurement_id)
ioannis@43:
moritz@29: confirm_page = self.session.get(delete_url)
victor@7:
victor@7: # Check that the page opened properly
victor@7: if confirm_page.status_code != 200:
victor@7: logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code))
victor@7: return None
victor@7:
ioannis@43: # Get the delete related value
ioannis@43: if delete_related:
ioannis@43: delete_related_option = 'delete_related'
ioannis@43: else:
ioannis@43: delete_related_option = 'not_delete_related'
ioannis@43:
victor@7: # Delete the measurement
victor@7: delete_page = self.session.post(delete_url,
ioannis@43: data={'post': 'yes',
ioannis@43: 'select_delete_related_measurements': delete_related_option},
victor@7: headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
victor@7: 'referer': delete_url}
victor@7: )
i@31: if not delete_page.ok:
victor@7: logger.warning("Something went wrong. Delete page status: {0}".format(
victor@7: delete_page.status_code))
victor@7: return None
victor@7:
victor@7: logger.info("Deleted measurement {0}".format(measurement_id))
victor@7: return True
victor@7:
victor@7: def available_measurements(self):
ioannis@67: """ Get a list of available measurement on the SCC.
ioannis@67:
ioannis@67: The methods is currently not used, could be merged with list_measurements.
ioannis@67: """
i@31: response = self.session.get(self.api_measurements_url)
victor@7: response_dict = response.json()
victor@7:
victor@7: if response_dict:
victor@7: measurement_list = response_dict['objects']
i@14: measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list]
victor@7: logger.info("Found %s measurements on the SCC." % len(measurements))
victor@7: else:
victor@7: logger.warning("No response received from the SCC when asked for available measurements.")
ioannis@43: measurements = None
victor@7:
victor@7: return measurements
victor@7:
ioannis@65: def list_measurements(self, id_exact=None, id_startswith=None):
ioannis@67: """ Get the response text from the API. """
moritz@29:
ioannis@67: # TODO: Add some error handling, e.g. as per available_measurements method
i@31:
moritz@29: # Need to set to empty string if not specified, we won't get any results
ioannis@65: params = {}
i@31:
ioannis@65: if id_exact is not None:
ioannis@65: params['id__exact'] = id_exact
ioannis@65: else:
ioannis@65: params['id__startswith'] = id_startswith
moritz@29:
ioannis@65: response_json = self.session.get(self.api_measurements_url, params=params).text
moritz@29:
ioannis@65: return response_json
moritz@29:
ioannis@43: def measurement_id_for_date(self, t1, call_sign, base_number=0):
victor@7: """ Give the first available measurement id on the SCC for the specific
madrouin@20: date.
victor@7: """
ioannis@67: # TODO: Check if this method needs updating to handle all measurement_ID formats.
victor@7: date_str = t1.strftime('%Y%m%d')
i@31: base_id = "%s%s" % (date_str, call_sign)
i@31: search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id)
victor@7:
moritz@29: response = self.session.get(search_url)
victor@7:
victor@7: response_dict = response.json()
victor@7:
victor@7: measurement_id = None
victor@7:
victor@7: if response_dict:
victor@7: measurement_list = response_dict['objects']
i@31:
i@31: if len(measurement_list) == 100:
i@31: raise ValueError('No available measurement id found.')
i@31:
victor@7: existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
victor@7:
victor@7: measurement_number = base_number
i@31: measurement_id = "%s%02i" % (base_id, measurement_number)
victor@7:
victor@7: while measurement_id in existing_ids:
victor@7: measurement_number = measurement_number + 1
i@31: measurement_id = "%s%02i" % (base_id, measurement_number)
victor@7:
victor@7: return measurement_id
victor@7:
i@40: def get_ancillary(self, file_path, file_type):
i@38: """
i@38: Try to get the ancillary file data from the SCC API.
i@38:
i@38: The result will always be an API object. If the file does not exist, the .exists property is set to False.
i@38:
i@38: Parameters
i@38: ----------
i@40: file_path : str
i@40: Path of the uploaded file.
i@38: file_type : str
i@38: Type of ancillary file. One of 'sounding', 'overlap', 'lidarratio'.
i@38:
i@38: Returns
i@38: : AncillaryFile
i@38: The api object.
i@38: """
i@38: assert file_type in ['sounding', 'overlap', 'lidarratio']
i@38:
i@40: filename = os.path.basename(file_path)
i@40:
i@38: if file_type == 'sounding':
i@38: file_url = self.api_sounding_search_pattern.format(filename)
i@38: elif file_type == 'overlap':
i@38: file_url = self.api_overlap_search_pattern.format(filename)
i@38: else:
i@38: file_url = self.api_lidarratio_search_pattern.format(filename)
i@38:
i@38: response = self.session.get(file_url)
i@38:
i@38: if not response.ok:
i@38: logger.error('Could not access API. Status code %s.' % response.status_code)
i@38: return None, response.status_code
i@38:
i@38: response_dict = response.json()
i@38: object_list = response_dict['objects']
i@38:
i@38: logger.debug("Ancillary file JSON: {0}".format(object_list))
i@38:
i@38: if object_list:
i@38: ancillary_file = AncillaryFile(self.api_base_url, object_list[0]) # Assume only one file is returned
i@38: else:
i@38: ancillary_file = AncillaryFile(self.api_base_url, None) # Create an empty object
i@38:
i@38: return ancillary_file, response.status_code
i@38:
ioannis@43: def __enter__(self):
ioannis@43: return self
victor@7:
ioannis@43: def __exit__(self, *args):
ioannis@43: logger.debug("Closing SCC connection session.")
ioannis@43: self.session.close()
ioannis@43:
i@31: class PageNotAccessibleError(RuntimeError):
i@31: pass
ioannis@43:
victor@7:
ioannis@17: class ApiObject(object):
victor@7: """ A generic class object. """
victor@7:
i@14: def __init__(self, base_url, dict_response):
i@14: self.base_url = base_url
victor@7:
victor@7: if dict_response:
victor@7: # Add the dictionary key value pairs as object properties
victor@7: for key, value in dict_response.items():
ioannis@17: # logger.debug('Setting key {0} to value {1}'.format(key, value))
ioannis@17: try:
ioannis@17: setattr(self, key, value)
ioannis@17: except:
ioannis@17: logger.warning('Could not set attribute {0} to value {1}'.format(key, value))
victor@7: self.exists = True
victor@7: else:
victor@7: self.exists = False
victor@7:
victor@7:
victor@7: class Measurement(ApiObject):
ioannis@43: """ This class represents the measurement object as returned in the SCC API.
ioannis@43: """
victor@7:
i@14: def __init__(self, base_url, dict_response):
victor@7:
ioannis@45: # Define expected attributes to assist debugging
ioannis@53:
ioannis@53: self.hirelpp = None
ioannis@53: self.hirelpp_exit_code = None
ioannis@43: self.cloudmask = None
ioannis@53: self.cloudmask_exit_code = None
ioannis@53: self.elpp = None
ioannis@53: self.elpp_exit_code = None
ioannis@43: self.elda = None
ioannis@53: self.elda_exit_code = None
ioannis@43: self.elic = None
ioannis@53: self.elic_exit_code = None
ioannis@53: self.eldec = None
ioannis@53: self.eldec_exit_code = None
ioannis@53: self.elquick = None
ioannis@53: self.elquick_exit_code = None
ioannis@53:
ioannis@43: self.id = None
ioannis@43: self.is_calibration = None
ioannis@43: self.is_running = None
ioannis@53:
ioannis@43: self.resource_uri = None
ioannis@43: self.start = None
ioannis@43: self.stop = None
ioannis@43: self.system = None
ioannis@43: self.upload = None
ioannis@43:
ioannis@45: super().__init__(base_url, dict_response)
victor@7:
ioannis@57: @property
ioannis@57: def has_finished(self):
ioannis@57: """ Temporary implementation for SCC version 5.2.0, until the API returns a flag indicating if the
ioannis@57: processing measurement has finished. """
ioannis@57: if (self.is_running is False) and (self.hirelpp != 0 or self.elpp != 0):
ioannis@57: return True
ioannis@57: else:
ioannis@57: return False
ioannis@57:
ioannis@53: def log_processing_status(self):
ioannis@53: """ Log module status. """
ioannis@53: logger.info("Measurement is being processed. Status: {}, {}, {}, {}, {}, {}). Please wait.".format(
ioannis@53: self.upload,
ioannis@53: self.hirelpp,
ioannis@53: self.cloudmask,
ioannis@53: self.elpp,
ioannis@53: self.elda,
ioannis@53: self.elic))
ioannis@53:
ioannis@53: def log_detailed_status(self):
ioannis@53: """ Log module exit and status codes."""
ioannis@53: logger.info("Measurement exit status:".format(self.id))
ioannis@53: if self.is_calibration:
ioannis@53: self._log_module_status('ElPP', self.elpp, self.elpp_exit_code)
ioannis@53: self._log_module_status('ElDEC', self.eldec, self.eldec_exit_code)
ioannis@53: else:
ioannis@53: self._log_module_status('HiRElPP', self.hirelpp, self.hirelpp_exit_code)
ioannis@53: self._log_module_status('CloudScreen', self.cloudmask, self.cloudmask_exit_code)
ioannis@53: self._log_module_status('ElPP', self.elpp, self.elpp_exit_code)
ioannis@53: self._log_module_status('ELDA', self.elda, self.elda_exit_code)
ioannis@53: self._log_module_status('ELIC', self.elic, self.elic_exit_code)
ioannis@53: self._log_module_status('ELQuick', self.elquick, self.elquick_exit_code)
ioannis@53:
ioannis@53: def _log_module_status(self, name, status, exit_code):
ioannis@53: if exit_code:
ioannis@54: if exit_code['exit_code'] > 0:
ioannis@54: logger.warning("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code))
ioannis@54: else:
ioannis@54: logger.info("{0} exit code: {2[exit_code]} - {2[description]}".format(name, status, exit_code))
ioannis@53: else:
ioannis@53: logger.info("{0} exit code: {2}".format(name, status, exit_code))
ioannis@53:
ioannis@43: @property
ioannis@43: def rerun_elda_url(self):
ioannis@43: url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/')
ioannis@43: return url_pattern.format(self.id)
victor@7:
victor@7: @property
ioannis@43: def rerun_elpp_url(self):
ioannis@43: url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elpp/')
i@14: return url_pattern.format(self.id)
victor@7:
victor@7: @property
victor@7: def rerun_all_url(self):
i@14: ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/')
i@14: return ulr_pattern.format(self.id)
victor@7:
victor@7: def __str__(self):
ioannis@43: return "Measurement {}".format(self.id)
victor@7:
victor@7:
i@38: class AncillaryFile(ApiObject):
i@38: """ This class represents the ancilalry file object as returned in the SCC API.
i@38: """
ioannis@67:
i@38: @property
i@38: def already_on_scc(self):
i@38: if self.exists is False:
i@38: return False
i@38:
i@38: return not self.status == 'missing'
i@38:
i@38: def __str__(self):
i@38: return "%s: %s, %s" % (self.id,
i@38: self.filename,
i@38: self.status)
i@38:
i@38:
ioannis@67: # Methods that use the SCC class to perform specific tasks.
ioannis@45: def process_file(filename, system_id, settings, force_upload, delete_related,
ioannis@54: delay=0, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
ioannis@32: """ Shortcut function to process a file to the SCC. """
ioannis@32: logger.info("Processing file %s, using system %s" % (filename, system_id))
victor@7:
ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
ioannis@43: scc.login(settings['website_credentials'])
ioannis@45: measurement = scc.process(filename, system_id,
ioannis@45: force_upload=force_upload,
ioannis@45: delete_related=delete_related,
ioannis@54: delay=delay,
ioannis@45: monitor=monitor,
ioannis@45: rs_filename=rs_filename,
ioannis@45: lr_filename=lr_filename,
ioannis@45: ov_filename=ov_filename)
ioannis@43: scc.logout()
victor@7: return measurement
victor@7:
victor@7:
ioannis@45: def delete_measurements(measurement_ids, delete_related, settings):
moritz@29: """ Shortcut function to delete measurements from the SCC. """
ioannis@45: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
ioannis@45: scc.login(settings['website_credentials'])
ioannis@45: for m_id in measurement_ids:
ioannis@54: logger.info("Deleting %s." % m_id)
ioannis@45: scc.delete_measurement(m_id, delete_related)
ioannis@45: scc.logout()
moritz@29:
moritz@29:
moritz@29: def rerun_all(measurement_ids, monitor, settings):
moritz@29: """ Shortcut function to rerun measurements from the SCC. """
i@14:
ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
ioannis@43: scc.login(settings['website_credentials'])
ioannis@45: for m_id in measurement_ids:
ioannis@54: logger.info("Rerunning all products for %s." % m_id)
ioannis@45: scc.rerun_all(m_id, monitor)
ioannis@43: scc.logout()
victor@7:
victor@7:
moritz@29: def rerun_processing(measurement_ids, monitor, settings):
victor@7: """ Shortcut function to delete a measurement from the SCC. """
i@14:
ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
ioannis@43: scc.login(settings['website_credentials'])
ioannis@45: for m_id in measurement_ids:
ioannis@45: logger.info("Rerunning (optical) processing for %s" % m_id)
ioannis@45: scc.rerun_elpp(m_id, monitor)
ioannis@45: scc.logout()
moritz@29:
moritz@29:
ioannis@65: def list_measurements(settings, id_exact=None, id_startswith=None):
moritz@29: """List all available measurements"""
ioannis@45: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
ioannis@45: scc.login(settings['website_credentials'])
ioannis@65:
ioannis@65: results_json = scc.list_measurements(id_exact=id_exact, id_startswith=id_startswith)
ioannis@65: print(results_json)
ioannis@65:
ioannis@43: scc.logout()
victor@7:
victor@7:
ioannis@61: def download_measurements(measurement_ids, max_retries, exit_if_missing, settings):
moritz@29: """Download all measurements for the specified IDs"""
ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc:
ioannis@43: scc.login(settings['website_credentials'])
ioannis@45: for m_id in measurement_ids:
ioannis@61: scc.monitor_processing(m_id, retry_max=max_retries, time_sleep=3, exit_if_missing=exit_if_missing)
ioannis@61:
ioannis@43: scc.logout()
i@14:
i@14:
moritz@29: def settings_from_path(config_file_path):
i@14: """ Read the configuration file.
i@14:
i@14: The file should be in YAML syntax."""
i@14:
i@14: if not os.path.isfile(config_file_path):
moritz@29: raise argparse.ArgumentTypeError("Wrong path for configuration file (%s)" % config_file_path)
i@14:
i@14: with open(config_file_path) as yaml_file:
i@14: try:
i@14: settings = yaml.safe_load(yaml_file)
i@14: logger.debug("Read settings file(%s)" % config_file_path)
moritz@29: except Exception:
moritz@29: raise argparse.ArgumentTypeError("Could not parse YAML file (%s)" % config_file_path)
i@14:
i@14: # YAML limitation: does not read tuples
i@14: settings['basic_credentials'] = tuple(settings['basic_credentials'])
i@14: settings['website_credentials'] = tuple(settings['website_credentials'])
i@14: return settings
i@14:
i@14:
moritz@29: # Setup for command specific parsers
moritz@29: def setup_delete(parser):
moritz@29: def delete_from_args(parsed):
ioannis@45: delete_measurements(parsed.IDs,
ioannis@45: delete_related=False,
ioannis@45: settings=parsed.config)
moritz@29:
moritz@29: parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.")
moritz@29: parser.set_defaults(execute=delete_from_args)
moritz@29:
moritz@29:
moritz@29: def setup_rerun_all(parser):
moritz@29: def rerun_all_from_args(parsed):
moritz@29: rerun_all(parsed.IDs, parsed.process, parsed.config)
moritz@29:
moritz@29: parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun.")
moritz@29: parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
moritz@29: action="store_true")
moritz@29: parser.set_defaults(execute=rerun_all_from_args)
moritz@29:
moritz@29:
ioannis@45: def setup_rerun_elpp(parser):
moritz@29: def rerun_processing_from_args(parsed):
moritz@29: rerun_processing(parsed.IDs, parsed.process, parsed.config)
moritz@29:
moritz@29: parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.")
moritz@29: parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
moritz@29: action="store_true")
moritz@29: parser.set_defaults(execute=rerun_processing_from_args)
moritz@29:
moritz@29:
ioannis@45: def setup_upload_file(parser):
ioannis@45: """ Upload but do not monitor processing progress. """
ioannis@67:
ioannis@45: def upload_file_from_args(parsed):
ioannis@45: process_file(parsed.filename, parsed.system, parsed.config,
ioannis@54: delay=parsed.delay,
ioannis@45: monitor=parsed.process,
ioannis@45: force_upload=parsed.force_upload,
ioannis@45: delete_related=False, # For now, use this as default
ioannis@32: rs_filename=parsed.radiosounding,
ioannis@32: ov_filename=parsed.overlap,
ioannis@32: lr_filename=parsed.lidarratio)
moritz@29:
ioannis@54: def delay(arg):
ioannis@54: try:
ioannis@54: int_arg = int(arg)
ioannis@54: except ValueError:
ioannis@54: raise argparse.ArgumentTypeError("Could not convert delay argument {} to integer.".format(arg))
ioannis@54:
ioannis@54: if 0 <= int_arg <= 96:
ioannis@54: return int_arg
ioannis@54: else:
ioannis@54: raise argparse.ArgumentTypeError("Delay should be an integer between 0 and 96.")
ioannis@54:
moritz@29: parser.add_argument("filename", help="Measurement file name or path.")
moritz@29: parser.add_argument("system", help="Processing system id.")
ioannis@54: parser.add_argument("--delay", help="Delay processing by the specified number of hours (0 to 96).",
ioannis@54: default=0, type=delay)
ioannis@45: parser.add_argument("-p", "--process", help="Wait for the processing results.",
ioannis@45: action="store_true")
ioannis@45: parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.",
ioannis@45: action="store_true")
i@30: parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
ioannis@32: parser.add_argument("--overlap", default=None, help="Overlap file name or path")
ioannis@32: parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
i@30:
moritz@29: parser.set_defaults(execute=upload_file_from_args)
moritz@29:
moritz@29:
moritz@29: def setup_list_measurements(parser):
moritz@29: def list_measurements_from_args(parsed):
ioannis@65: list_measurements(parsed.config, id_exact=parsed.id_exact, id_startswith=parsed.id_startswith)
moritz@29:
ioannis@65: group = parser.add_mutually_exclusive_group()
ioannis@65: group.add_argument("--id_exact", help="Exact measurement id.")
ioannis@65: group.add_argument("--id_startswith", help="Initial part of measurement id.")
moritz@29:
moritz@29: parser.set_defaults(execute=list_measurements_from_args)
moritz@29:
moritz@29:
moritz@29: def setup_download_measurements(parser):
moritz@29: def download_measurements_from_args(parsed):
ioannis@61: download_measurements(parsed.IDs, parsed.max_retries, parsed.ignore_errors, parsed.config)
moritz@29:
moritz@29: parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+")
ioannis@67: parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0,
ioannis@67: type=int)
ioannis@67: parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.",
ioannis@67: action="store_false")
moritz@29: parser.set_defaults(execute=download_measurements_from_args)
moritz@29:
moritz@29:
victor@7: def main():
victor@7: # Define the command line arguments.
victor@7: parser = argparse.ArgumentParser()
moritz@29: subparsers = parser.add_subparsers()
moritz@29:
moritz@29: delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
ioannis@67: rerun_all_parser = subparsers.add_parser("rerun-all",
ioannis@67: help="Rerun all processing steps for the provided measurement IDs.")
ioannis@45: rerun_processing_parser = subparsers.add_parser("rerun-elpp",
ioannis@45: help="Rerun low-resolution processing steps for the provided measurement ID.")
ioannis@67: upload_file_parser = subparsers.add_parser("upload-file",
ioannis@67: help="Submit a file and, optionally, download the output products.")
ioannis@32: list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
moritz@29: download_parser = subparsers.add_parser("download", help="Download selected measurements.")
moritz@29:
moritz@29: setup_delete(delete_parser)
moritz@29: setup_rerun_all(rerun_all_parser)
ioannis@45: setup_rerun_elpp(rerun_processing_parser)
ioannis@45:
moritz@29: setup_upload_file(upload_file_parser)
moritz@29: setup_list_measurements(list_parser)
moritz@29: setup_download_measurements(download_parser)
victor@7:
victor@7: # Verbosity settings from http://stackoverflow.com/a/20663028
victor@7: parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
victor@7: dest="loglevel", const=logging.DEBUG, default=logging.INFO,
victor@7: )
victor@7: parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
victor@7: dest="loglevel", const=logging.WARNING
victor@7: )
victor@7:
ioannis@32: # Setup default config location
moritz@29: home = os.path.expanduser("~")
moritz@29: default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml"))
moritz@29: parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path,
moritz@29: default=default_config_location)
victor@7:
victor@7: args = parser.parse_args()
ioannis@43:
victor@7: # Get the logger with the appropriate level
victor@7: logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
victor@7:
moritz@29: # Dispatch to appropriate function
moritz@29: args.execute(args)
moritz@29:
i@14:
moritz@29: # When running through terminal
moritz@29: if __name__ == '__main__':
moritz@29: main()