ioannis@43: import sys ioannis@43: victor@7: import requests victor@7: ioannis@43: # Python 2 and 3 support ioannis@43: try: ioannis@43: import urllib.parse as urlparse # Python 3 ioannis@43: except ImportError: ioannis@43: from urlparse import urlparse # Python 2 ioannis@43: victor@7: import argparse victor@7: import os victor@7: import re victor@7: import time ioannis@43: from io import BytesIO victor@7: from zipfile import ZipFile victor@7: import datetime victor@7: import logging i@14: import yaml victor@7: ioannis@43: import netCDF4 as netcdf ioannis@43: ioannis@43: requests.packages.urllib3.disable_warnings() victor@7: i@14: logger = logging.getLogger(__name__) victor@7: victor@7: # The regex to find the measurement id from the measurement page victor@7: # This should be read from the uploaded file, but would require an extra NetCDF module. victor@7: regex = "

Measurement (?P.{12}) " victor@7: victor@7: victor@7: class SCC: victor@7: """ A simple class that will attempt to upload a file on the SCC server. i@14: victor@7: The uploading is done by simulating a normal browser session. In the current victor@7: version no check is performed, and no feedback is given if the upload victor@7: was successful. If everything is setup correctly, it will work. victor@7: """ victor@7: i@14: def __init__(self, auth, output_dir, base_url): victor@7: self.auth = auth victor@7: self.output_dir = output_dir i@14: self.base_url = base_url victor@7: self.session = requests.Session() victor@7: i@14: # Construct the absolute URLs i@14: self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') i@14: self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') ioannis@43: self.download_hirelpp_pattern = urlparse.urljoin(self.base_url, ioannis@43: 'data_processing/measurements/{0}/download-hirelpp/') ioannis@43: self.download_cloudmask_pattern = urlparse.urljoin(self.base_url, ioannis@43: 'data_processing/measurements/{0}/download-cloudmask/') ioannis@43: self.download_elpp_pattern = urlparse.urljoin(self.base_url, ioannis@43: 'data_processing/measurements/{0}/download-preprocessed/') ioannis@43: self.download_elda_pattern = urlparse.urljoin(self.base_url, ioannis@43: 'data_processing/measurements/{0}/download-optical/') ioannis@43: self.download_plot_pattern = urlparse.urljoin(self.base_url, ioannis@43: 'data_processing/measurements/{0}/download-plots/') ioannis@43: self.download_elic_pattern = urlparse.urljoin(self.base_url, ioannis@43: 'data_processing/measurements/{0}/download-elic/') i@14: self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') i@14: self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') i@14: ioannis@43: self.login_credentials = None ioannis@43: i@14: def login(self, credentials): victor@7: """ Login the the website. """ victor@7: logger.debug("Attempting to login to SCC, username %s." % credentials[0]) victor@7: self.login_credentials = {'username': credentials[0], victor@7: 'password': credentials[1]} victor@7: i@14: logger.debug("Accessing login page at %s." % self.login_url) victor@7: victor@7: # Get upload form i@14: login_page = self.session.get(self.login_url, auth=self.auth, verify=False) i@14: i@14: if login_page.status_code != 200: i@14: logger.error('Could not access login pages. Status code %s' % login_page.status_code) i@14: sys.exit(1) victor@7: victor@7: logger.debug("Submiting credentials.") ioannis@43: victor@7: # Submit the login data i@14: login_submit = self.session.post(self.login_url, victor@7: data=self.login_credentials, victor@7: headers={'X-CSRFToken': login_page.cookies['csrftoken'], i@14: 'referer': self.login_url}, victor@7: verify=False, victor@7: auth=self.auth) victor@7: return login_submit victor@7: victor@7: def logout(self): victor@7: pass victor@7: ioannis@43: def upload_file(self, filename, system_id, force_upload, delete_related): victor@7: """ Upload a filename for processing with a specific system. If the victor@7: upload is successful, it returns the measurement id. """ ioannis@43: ioannis@43: measurement_id = self.measurement_id_from_file(filename) ioannis@43: ioannis@43: logger.debug('Checking if a measurement with the same id already exists on the SCC server.') ioannis@43: existing_measurement = self.get_measurement(measurement_id) ioannis@43: ioannis@43: if existing_measurement: ioannis@43: if force_upload: ioannis@43: logger.info( ioannis@43: "Measurement with id {} already exists on the SCC. Trying to delete it...".format(measurement_id)) ioannis@43: self.delete_measurement(measurement_id, delete_related) ioannis@43: else: ioannis@43: logger.error( ioannis@43: "Measurement with id {} already exists on the SCC. Use --force_upload flag to overwrite it.".format( ioannis@43: measurement_id)) ioannis@43: sys.exit(1) ioannis@43: victor@7: # Get submit page i@14: upload_page = self.session.get(self.upload_url, victor@7: auth=self.auth, victor@7: verify=False) victor@7: victor@7: # Submit the data victor@7: upload_data = {'system': system_id} victor@7: files = {'data': open(filename, 'rb')} victor@7: i@14: logger.info("Uploading of file %s started." % filename) ioannis@43: logger.debug("URL: {0}, data: {1}, 'X-CSRFToken': {2}".format(self.upload_url, ioannis@43: upload_data, ioannis@43: upload_page.cookies['csrftoken'])) i@14: upload_submit = self.session.post(self.upload_url, victor@7: data=upload_data, victor@7: files=files, victor@7: headers={'X-CSRFToken': upload_page.cookies['csrftoken'], ioannis@43: 'referer': self.upload_url, }, victor@7: verify=False, victor@7: auth=self.auth) victor@7: victor@7: if upload_submit.status_code != 200: i@14: logger.warning("Connection error. Status code: %s" % upload_submit.status_code) victor@7: return False victor@7: victor@7: # Check if there was a redirect to a new page. i@14: if upload_submit.url == self.upload_url: victor@7: measurement_id = False i@14: logger.error("Uploaded file rejected! Try to upload manually to see the error.") victor@7: else: victor@7: measurement_id = re.findall(regex, upload_submit.text)[0] ioannis@43: logger.info("Successfully uploaded measurement with id %s." % measurement_id) ioannis@43: ioannis@43: return measurement_id ioannis@43: ioannis@43: @staticmethod ioannis@43: def measurement_id_from_file(filename): ioannis@43: """ Get the measurement id from the input file. """ ioannis@43: ioannis@43: if not os.path.isfile(filename): ioannis@43: logger.error("File {} does not exist.".format(filename)) ioannis@43: sys.exit(1) ioannis@43: ioannis@43: with netcdf.Dataset(filename) as f: ioannis@43: try: ioannis@43: measurement_id = f.Measurement_ID ioannis@43: except AttributeError: ioannis@43: logger.error( ioannis@43: "Input file {} does not contain a Measurement_ID global attribute. Wrong file format?".format( ioannis@43: filename)) ioannis@43: sys.exit(1) victor@7: victor@7: return measurement_id victor@7: victor@7: def download_files(self, measurement_id, subdir, download_url): victor@7: """ Downloads some files from the download_url to the specified victor@7: subdir. This method is used to download preprocessed file, optical victor@7: files etc. victor@7: """ victor@7: # Get the file victor@7: request = self.session.get(download_url, auth=self.auth, victor@7: verify=False, victor@7: stream=True) victor@7: victor@7: # Create the dir if it does not exist victor@7: local_dir = os.path.join(self.output_dir, measurement_id, subdir) victor@7: if not os.path.exists(local_dir): victor@7: os.makedirs(local_dir) victor@7: victor@7: # Save the file by chunk, needed if the file is big. ioannis@43: memory_file = BytesIO() victor@7: victor@7: for chunk in request.iter_content(chunk_size=1024): victor@7: if chunk: # filter out keep-alive new chunks victor@7: memory_file.write(chunk) victor@7: memory_file.flush() victor@7: victor@7: zip_file = ZipFile(memory_file) victor@7: victor@7: for ziped_name in zip_file.namelist(): victor@7: basename = os.path.basename(ziped_name) victor@7: victor@7: local_file = os.path.join(local_dir, basename) victor@7: victor@7: with open(local_file, 'wb') as f: victor@7: f.write(zip_file.read(ziped_name)) victor@7: ioannis@43: def download_hirelpp(self, measurement_id): ioannis@43: """ Download hirelpp files for the measurement id. """ ioannis@43: # Construct the download url ioannis@43: download_url = self.download_hirelpp_pattern.format(measurement_id) ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_hirelpp', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download HiRElPP files. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) ioannis@43: ioannis@43: def download_cloudmask(self, measurement_id): ioannis@43: """ Download cloudmask files for the measurement id. """ ioannis@43: # Construct the download url ioannis@43: download_url = self.download_cloudmask_pattern.format(measurement_id) ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_cloudscreen', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download cloudscreen files. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) ioannis@43: ioannis@43: def download_elpp(self, measurement_id): victor@7: """ Download preprocessed files for the measurement id. """ victor@7: # Construct the download url ioannis@43: download_url = self.download_elpp_pattern.format(measurement_id) ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_preprocessed', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download ElPP files. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) victor@7: ioannis@43: def download_elda(self, measurement_id): victor@7: """ Download optical files for the measurement id. """ victor@7: # Construct the download url ioannis@43: download_url = self.download_elda_pattern.format(measurement_id) ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_optical', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download ELDA files. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) victor@7: ioannis@43: def download_plots(self, measurement_id): victor@7: """ Download profile graphs for the measurement id. """ victor@7: # Construct the download url ioannis@43: download_url = self.download_plot_pattern.format(measurement_id) ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_plots', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download ELDA plots. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) victor@7: ioannis@43: def download_elic(self, measurement_id): ioannis@43: """ Download ELIC files for the measurement id. """ ioannis@43: # Construct the download url ioannis@43: download_url = self.download_elic_pattern.format(measurement_id) ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_elic', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download ELIC files. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) ioannis@43: ioannis@43: def download_eldec(self, measurement_id): ioannis@43: """ Download ELDEC files for the measurement id. """ ioannis@43: # Construct the download url ioannis@43: download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now ioannis@43: try: ioannis@43: self.download_files(measurement_id, 'scc_eldec', download_url) ioannis@43: except Exception as e: ioannis@43: logger.error("Could not download EDELC files. Error message: {}".format(e)) ioannis@43: logger.debug('Download exception:', exc_info=True) ioannis@43: ioannis@43: def rerun_elpp(self, measurement_id, monitor=True): victor@7: measurement = self.get_measurement(measurement_id) victor@7: victor@7: if measurement: ioannis@43: request = self.session.get(measurement.rerun_elpp_url, auth=self.auth, victor@7: verify=False, victor@7: stream=True) victor@7: victor@7: if request.status_code != 200: i@14: logger.error( ioannis@43: "Could not rerun ELPP for %s. Status code: %s" % (measurement_id, request.status_code)) victor@7: return victor@7: victor@7: if monitor: victor@7: self.monitor_processing(measurement_id) victor@7: victor@7: def rerun_all(self, measurement_id, monitor=True): victor@7: logger.debug("Started rerun_all procedure.") victor@7: victor@7: logger.debug("Getting measurement %s" % measurement_id) victor@7: measurement = self.get_measurement(measurement_id) victor@7: victor@7: if measurement: victor@7: logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) victor@7: victor@7: request = self.session.get(measurement.rerun_all_url, auth=self.auth, victor@7: verify=False, victor@7: stream=True) victor@7: victor@7: if request.status_code != 200: victor@7: logger.error("Could not rerun pre processing for %s. Status code: %s" % victor@7: (measurement_id, request.status_code)) victor@7: return victor@7: victor@7: if monitor: victor@7: self.monitor_processing(measurement_id) victor@7: ioannis@43: def process(self, filename, system_id, force_upload, delete_related): victor@7: """ Upload a file for processing and wait for the processing to finish. victor@7: If the processing is successful, it will download all produced files. victor@7: """ victor@7: logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) ioannis@43: victor@7: # Upload file ioannis@43: measurement_id = self.upload_file(filename, system_id, force_upload, delete_related) victor@7: victor@7: measurement = self.monitor_processing(measurement_id) victor@7: return measurement victor@7: victor@7: def monitor_processing(self, measurement_id): victor@7: """ Monitor the processing progress of a measurement id""" victor@7: victor@7: measurement = self.get_measurement(measurement_id) victor@7: if measurement is not None: victor@7: while measurement.is_running: ioannis@43: logger.info("Measurement is being processed (status: {}, {}, {}, {}, {}, {}). Please wait.".format( ioannis@43: measurement.upload, ioannis@43: measurement.hirelpp, ioannis@43: measurement.cloudmask, ioannis@43: measurement.elpp, ioannis@43: measurement.elda, ioannis@43: measurement.elic)) victor@7: time.sleep(10) victor@7: measurement = self.get_measurement(measurement_id) ioannis@43: logger.info("Measurement processing finished (status: {}, {}, {}, {}, {}, {}). Please wait.".format( ioannis@43: measurement.upload, ioannis@43: measurement.hirelpp, ioannis@43: measurement.cloudmask, ioannis@43: measurement.elpp, ioannis@43: measurement.elda, ioannis@43: measurement.elic)) ioannis@43: if measurement.hirelpp == 127: ioannis@43: logger.info("Downloading HiRElPP files.") ioannis@43: self.download_hirelpp(measurement_id) ioannis@43: if measurement.cloudmask == 127: ioannis@43: logger.info("Downloading cloud screening files.") ioannis@43: self.download_cloudmask(measurement_id) ioannis@43: if measurement.elpp == 127: ioannis@43: logger.info("Downloading ELPP files.") ioannis@43: self.download_elpp(measurement_id) ioannis@43: if measurement.elda == 127: ioannis@43: logger.info("Downloading ELDA files.") ioannis@43: self.download_elda(measurement_id) victor@7: logger.info("Downloading graphs.") ioannis@43: self.download_plots(measurement_id) ioannis@43: if measurement.elic == 127: ioannis@43: logger.info("Downloading ELIC files.") ioannis@43: self.download_elic(measurement_id) ioannis@43: ioannis@43: # TODO: Need to check ELDEC code (when it becomes available in the API) ioannis@43: if measurement.is_calibration: ioannis@43: logger.info("Downloading ELDEC files.") ioannis@43: self.download_eldec(measurement_id) ioannis@43: victor@7: logger.info("--- Processing finished. ---") victor@7: return measurement victor@7: ioannis@43: def get_measurement(self, measurement_id): victor@7: ioannis@43: if measurement_id is None: victor@7: return None victor@7: i@14: measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id) victor@7: victor@7: response = self.session.get(measurement_url, victor@7: auth=self.auth, victor@7: verify=False) victor@7: ioannis@43: response_dict = None ioannis@43: if response.status_code == 200: ioannis@43: response_dict = response.json() ioannis@43: if response.status_code == 404: ioannis@43: logger.info("No measurement with id %s found on the SCC." % measurement_id) ioannis@43: elif response.status_code != 200: i@14: logger.error('Could not access API. Status code %s.' % response.status_code) i@14: sys.exit(1) i@14: ioannis@43: logger.debug("Response dictionary: {}".format(response_dict)) victor@7: victor@7: if response_dict: ioannis@43: measurement = Measurement(self.base_url, response_dict) victor@7: return measurement victor@7: else: victor@7: return None victor@7: ioannis@43: def delete_measurement(self, measurement_id, delete_related=False): victor@7: """ Deletes a measurement with the provided measurement id. The user victor@7: should have the appropriate permissions. victor@7: victor@7: The procedures is performed directly through the web interface and victor@7: NOT through the API. victor@7: """ victor@7: # Get the measurement object victor@7: measurement = self.get_measurement(measurement_id) victor@7: victor@7: # Check that it exists victor@7: if measurement is None: victor@7: logger.warning("Nothing to delete.") victor@7: return None victor@7: victor@7: # Go the the page confirming the deletion i@14: delete_url = self.delete_measurement_pattern.format(measurement.id) victor@7: ioannis@43: logger.debug("Delete url: {}".format(delete_url)) ioannis@43: victor@7: confirm_page = self.session.get(delete_url, victor@7: auth=self.auth, victor@7: verify=False) victor@7: victor@7: # Check that the page opened properly victor@7: if confirm_page.status_code != 200: victor@7: logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code)) victor@7: return None victor@7: ioannis@43: # Get the delete related value ioannis@43: if delete_related: ioannis@43: delete_related_option = 'delete_related' ioannis@43: else: ioannis@43: delete_related_option = 'not_delete_related' ioannis@43: victor@7: # Delete the measurement victor@7: delete_page = self.session.post(delete_url, victor@7: auth=self.auth, victor@7: verify=False, ioannis@43: data={'post': 'yes', ioannis@43: 'select_delete_related_measurements': delete_related_option}, victor@7: headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], victor@7: 'referer': delete_url} victor@7: ) victor@7: if delete_page.status_code != 200: victor@7: logger.warning("Something went wrong. Delete page status: {0}".format( victor@7: delete_page.status_code)) victor@7: return None victor@7: ioannis@43: logger.info("Deleted measurement {0}.".format(measurement_id)) victor@7: return True victor@7: victor@7: def available_measurements(self): victor@7: """ Get a list of available measurement on the SCC. """ i@14: measurement_url = urlparse.urljoin(self.api_base_url, 'measurements') victor@7: response = self.session.get(measurement_url, victor@7: auth=self.auth, victor@7: verify=False) victor@7: response_dict = response.json() victor@7: victor@7: if response_dict: victor@7: measurement_list = response_dict['objects'] i@14: measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] victor@7: logger.info("Found %s measurements on the SCC." % len(measurements)) victor@7: else: ioannis@43: measurements = None victor@7: logger.warning("No response received from the SCC when asked for available measurements.") victor@7: victor@7: return measurements victor@7: ioannis@43: def measurement_id_for_date(self, t1, call_sign, base_number=0): victor@7: """ Give the first available measurement id on the SCC for the specific victor@7: date. victor@7: """ victor@7: date_str = t1.strftime('%Y%m%d') i@14: search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) victor@7: victor@7: response = self.session.get(search_url, victor@7: auth=self.auth, victor@7: verify=False) victor@7: victor@7: response_dict = response.json() victor@7: victor@7: measurement_id = None victor@7: victor@7: if response_dict: victor@7: measurement_list = response_dict['objects'] victor@7: existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] victor@7: victor@7: measurement_number = base_number ioannis@43: measurement_id = "%s%s%04i" % (date_str, call_sign, measurement_number) victor@7: victor@7: while measurement_id in existing_ids: victor@7: measurement_number = measurement_number + 1 ioannis@43: measurement_id = "%s%s%04i" % (date_str, call_sign, measurement_number) ioannis@43: if measurement_number == 1000: victor@7: raise ValueError('No available measurement id found.') victor@7: victor@7: return measurement_id victor@7: ioannis@43: def __enter__(self): ioannis@43: return self victor@7: ioannis@43: def __exit__(self, *args): ioannis@43: logger.debug("Closing SCC connection session.") ioannis@43: self.session.close() ioannis@43: ioannis@43: ioannis@43: class Measurement: ioannis@43: """ This class represents the measurement object as returned in the SCC API. ioannis@43: """ victor@7: i@14: def __init__(self, base_url, dict_response): i@14: self.base_url = base_url victor@7: ioannis@43: # Define expected attributes to assist debuggin ioannis@43: self.cloudmask = None ioannis@43: self.elda = None ioannis@43: self.elic = None ioannis@43: self.elpp = None ioannis@43: self.hirelpp = None ioannis@43: self.id = None ioannis@43: self.is_calibration = None ioannis@43: self.is_running = None ioannis@43: self.pre_processing_exit_code = None ioannis@43: self.processing_exit_code = None ioannis@43: self.resource_uri = None ioannis@43: self.start = None ioannis@43: self.stop = None ioannis@43: self.system = None ioannis@43: self.upload = None ioannis@43: victor@7: if dict_response: victor@7: # Add the dictionary key value pairs as object properties victor@7: for key, value in dict_response.items(): victor@7: setattr(self, key, value) victor@7: self.exists = True victor@7: else: victor@7: self.exists = False victor@7: ioannis@43: @property ioannis@43: def rerun_elda_url(self): ioannis@43: url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elda/') ioannis@43: return url_pattern.format(self.id) victor@7: victor@7: @property ioannis@43: def rerun_elpp_url(self): ioannis@43: url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-elpp/') i@14: return url_pattern.format(self.id) victor@7: victor@7: @property victor@7: def rerun_all_url(self): i@14: ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') i@14: return ulr_pattern.format(self.id) victor@7: victor@7: def __str__(self): ioannis@43: return "Measurement {}".format(self.id) victor@7: victor@7: ioannis@43: def upload_file(filename, system_id, force_upload, delete_related, settings): victor@7: """ Shortcut function to upload a file to the SCC. """ ioannis@43: logger.info("Uploading file %s, using system %s." % (filename, system_id)) victor@7: ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: ioannis@43: scc.login(settings['website_credentials']) ioannis@43: measurement_id = scc.upload_file(filename, system_id, force_upload, delete_related) ioannis@43: scc.logout() ioannis@43: victor@7: return measurement_id victor@7: victor@7: ioannis@43: def process_file(filename, system_id, force_upload, delete_related, settings): victor@7: """ Shortcut function to process a file to the SCC. """ ioannis@43: logger.info("Processing file %s, using system %s." % (filename, system_id)) victor@7: ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: ioannis@43: scc.login(settings['website_credentials']) ioannis@43: measurement = scc.process(filename, system_id, force_upload, delete_related) ioannis@43: scc.logout() ioannis@43: victor@7: return measurement victor@7: victor@7: ioannis@43: def delete_measurement(measurement_id, settings, delete_related): victor@7: """ Shortcut function to delete a measurement from the SCC. """ ioannis@43: logger.info("Deleting %s." % measurement_id) i@14: ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: ioannis@43: scc.login(settings['website_credentials']) ioannis@43: scc.delete_measurement(measurement_id, delete_related) ioannis@43: scc.logout() victor@7: victor@7: i@14: def rerun_all(measurement_id, monitor, settings): victor@7: """ Shortcut function to delete a measurement from the SCC. """ victor@7: logger.info("Rerunning all products for %s" % measurement_id) i@14: ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: ioannis@43: scc.login(settings['website_credentials']) ioannis@43: scc.rerun_all(measurement_id, monitor) ioannis@43: scc.logout() victor@7: victor@7: ioannis@43: def rerun_elpp(measurement_id, monitor, settings): victor@7: """ Shortcut function to delete a measurement from the SCC. """ victor@7: logger.info("Rerunning (optical) processing for %s" % measurement_id) i@14: ioannis@43: with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: ioannis@43: scc.login(settings['website_credentials']) ioannis@43: scc.rerun_elpp(measurement_id, monitor) ioannis@43: scc.logout() i@14: i@14: i@14: def import_settings(config_file_path): i@14: """ Read the configuration file. i@14: i@14: The file should be in YAML syntax.""" i@14: i@14: if not os.path.isfile(config_file_path): i@14: logger.error("Wrong path for configuration file (%s)" % config_file_path) i@14: sys.exit(1) i@14: i@14: with open(config_file_path) as yaml_file: i@14: try: i@14: settings = yaml.safe_load(yaml_file) i@14: logger.debug("Read settings file(%s)" % config_file_path) ioannis@43: except Exception as e: i@14: logger.error("Could not parse YAML file (%s)" % config_file_path) ioannis@43: logger.debug("Error message: {}".format(e)) i@14: sys.exit(1) i@14: i@14: # YAML limitation: does not read tuples i@14: settings['basic_credentials'] = tuple(settings['basic_credentials']) i@14: settings['website_credentials'] = tuple(settings['website_credentials']) i@14: return settings i@14: i@14: victor@7: def main(): victor@7: # Define the command line arguments. victor@7: parser = argparse.ArgumentParser() i@16: parser.add_argument("config", help="Path to configuration file") victor@7: parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='') victor@7: parser.add_argument("system", nargs='?', help="Processing system id.", default=0) victor@7: parser.add_argument("-p", "--process", help="Wait for the results of the processing.", victor@7: action="store_true") victor@7: parser.add_argument("--delete", help="Measurement ID to delete.") ioannis@43: # parser.add_argument("--delete_related", help= ioannis@43: # "Delete all related measurements. Use only if you know what you are doing!", ioannis@43: # action="store_true") ioannis@43: parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.", ioannis@43: action="store_true") ioannis@43: parser.add_argument("--rerun-all", help="Rerun all processing steps for the provided measurement ID.") ioannis@43: parser.add_argument("--rerun-elpp", help="Rerun low-resolution processing steps for the provided measurement ID.") victor@7: victor@7: # Verbosity settings from http://stackoverflow.com/a/20663028 victor@7: parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", victor@7: dest="loglevel", const=logging.DEBUG, default=logging.INFO, victor@7: ) victor@7: parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", victor@7: dest="loglevel", const=logging.WARNING victor@7: ) victor@7: victor@7: args = parser.parse_args() victor@7: ioannis@43: # For now, don to allow to delete related measurements ioannis@43: delete_related = False ioannis@43: victor@7: # Get the logger with the appropriate level victor@7: logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) victor@7: i@14: settings = import_settings(args.config) i@14: i@14: # If the arguments are OK, try to log-in to SCC and upload. victor@7: if args.delete: victor@7: # If the delete is provided, do nothing else ioannis@43: delete_measurement(args.delete, settings, delete_related) victor@7: elif args.rerun_all: i@14: rerun_all(args.rerun_all, args.process, settings) ioannis@43: elif args.rerun_elpp: ioannis@43: rerun_elpp(args.rerun_elpp, args.process, settings) victor@7: else: victor@7: if (args.filename == '') or (args.system == 0): victor@7: parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') victor@7: victor@7: if args.process: ioannis@43: process_file(args.filename, args.system, args.force_upload, delete_related, settings) victor@7: else: ioannis@43: upload_file(args.filename, args.system, args.force_upload, delete_related, settings)