Mercurial > public > scc_access / changeset
changeset
Restructuring to work with setting file provided as argument form the command line.
Fri, 15 Dec 2017 22:53:17 +0200
- author
- Iannis <i.binietoglou@impworks.gr>
- date
- Fri, 15 Dec 2017 22:53:17 +0200
- changeset 14
- c2020b2fdd05
- parent 13
- 493067e1bfbb
- child 15
- 93b6b945d939
- child 29
- 3e3e5bda6b77
Restructuring to work with setting file provided as argument form the command line.
Settings in YAML format.
Added some error checking (i.e. if reponse!=200 then raise Error).
| scc_access/scc_access.py | file | annotate | diff | comparison | revisions |
--- a/scc_access/scc_access.py Fri Dec 15 22:51:47 2017 +0200 +++ b/scc_access/scc_access.py Fri Dec 15 22:53:17 2017 +0200 @@ -1,41 +1,7 @@ -#!/usr/bin/env python -""" -The MIT License (MIT) - -Copyright (c) 2015, Ioannis Binietoglou - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" - -__version__ = "0.6.0" - -# Try to read the settings from the settings.py file -try: - from settings import * -except: - raise ImportError( - """A settings file (setting.py) is required to run the script. - You can use settings.sample.py as a template.""") - import requests requests.packages.urllib3.disable_warnings() +import sys import urlparse import argparse import os @@ -45,21 +11,10 @@ from zipfile import ZipFile import datetime import logging - -logger = logging.getLogger(__name__) +import yaml -# Construct the absolute URLs -LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/') -UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/') -DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/') -DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/') -DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/') -RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/') -RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/') - -DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/') -API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/') +logger = logging.getLogger(__name__) # The regex to find the measurement id from the measurement page # This should be read from the uploaded file, but would require an extra NetCDF module. @@ -68,34 +23,52 @@ class SCC: """ A simple class that will attempt to upload a file on the SCC server. + The uploading is done by simulating a normal browser session. In the current version no check is performed, and no feedback is given if the upload was successful. If everything is setup correctly, it will work. """ - def __init__(self, auth=BASIC_LOGIN, output_dir=OUTPUT_DIR): + def __init__(self, auth, output_dir, base_url): self.auth = auth self.output_dir = output_dir + self.base_url = base_url self.session = requests.Session() + self.construct_urls() - def login(self, credentials=DJANGO_LOGIN): + def construct_urls(self): + """ Construct all URLs needed for processing. """ + # Construct the absolute URLs + self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') + self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') + self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/') + self.download_optical_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-optical/') + self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/') + self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') + self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') + + def login(self, credentials): """ Login the the website. """ logger.debug("Attempting to login to SCC, username %s." % credentials[0]) self.login_credentials = {'username': credentials[0], 'password': credentials[1]} - logger.debug("Accessing login page at %s." % LOGIN_URL) + logger.debug("Accessing login page at %s." % self.login_url) # Get upload form - login_page = self.session.get(LOGIN_URL, - auth=self.auth, verify=False) + login_page = self.session.get(self.login_url, auth=self.auth, verify=False) + + if login_page.status_code != 200: + logger.error('Could not access login pages. Status code %s' % login_page.status_code) + sys.exit(1) logger.debug("Submiting credentials.") + # Submit the login data - login_submit = self.session.post(LOGIN_URL, + login_submit = self.session.post(self.login_url, data=self.login_credentials, headers={'X-CSRFToken': login_page.cookies['csrftoken'], - 'referer': LOGIN_URL}, + 'referer': self.login_url}, verify=False, auth=self.auth) return login_submit @@ -107,7 +80,7 @@ """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page - upload_page = self.session.get(UPLOAD_URL, + upload_page = self.session.get(self.upload_url, auth=self.auth, verify=False) @@ -115,27 +88,27 @@ upload_data = {'system': system_id} files = {'data': open(filename, 'rb')} - logging.info("Uploading of file %s started." % filename) + logger.info("Uploading of file %s started." % filename) - upload_submit = self.session.post(UPLOAD_URL, + upload_submit = self.session.post(self.upload_url, data=upload_data, files=files, headers={'X-CSRFToken': upload_page.cookies['csrftoken'], - 'referer': UPLOAD_URL}, + 'referer': self.upload_url,}, verify=False, auth=self.auth) if upload_submit.status_code != 200: - logging.warning("Connection error. Status code: %s" % upload_submit.status_code) + logger.warning("Connection error. Status code: %s" % upload_submit.status_code) return False # Check if there was a redirect to a new page. - if upload_submit.url == UPLOAD_URL: + if upload_submit.url == self.upload_url: measurement_id = False - logging.error("Uploaded file rejected! Try to upload manually to see the error.") + logger.error("Uploaded file rejected! Try to upload manually to see the error.") else: measurement_id = re.findall(regex, upload_submit.text)[0] - logging.error("Successfully uploaded measurement with id %s." % measurement_id) + logger.error("Successfully uploaded measurement with id %s." % measurement_id) return measurement_id @@ -175,19 +148,19 @@ def download_preprocessed(self, measurement_id): """ Download preprocessed files for the measurement id. """ # Construct the download url - download_url = DOWNLOAD_PREPROCESSED.format(measurement_id) + download_url = self.download_preprocessed_pattern.format(measurement_id) self.download_files(measurement_id, 'scc_preprocessed', download_url) def download_optical(self, measurement_id): """ Download optical files for the measurement id. """ # Construct the download url - download_url = DOWNLOAD_OPTICAL.format(measurement_id) + download_url = self.download_optical.format(measurement_id) self.download_files(measurement_id, 'scc_optical', download_url) def download_graphs(self, measurement_id): """ Download profile graphs for the measurement id. """ # Construct the download url - download_url = DOWNLOAD_GRAPH.format(measurement_id) + download_url = self.download_graph_pattern.format(measurement_id) self.download_files(measurement_id, 'scc_plots', download_url) def rerun_processing(self, measurement_id, monitor=True): @@ -199,7 +172,8 @@ stream=True) if request.status_code != 200: - logging.error("Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) + logger.error( + "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) return if monitor: @@ -244,13 +218,13 @@ if measurement is not None: while measurement.is_running: logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, - measurement.pre_processing, - measurement.processing)) + measurement.pre_processing, + measurement.processing)) time.sleep(10) measurement = self.get_measurement(measurement_id) logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, - measurement.pre_processing, - measurement.processing)) + measurement.pre_processing, + measurement.processing)) if measurement.pre_processing == 127: logger.info("Downloading preprocessed files.") self.download_preprocessed(measurement_id) @@ -264,7 +238,7 @@ def get_status(self, measurement_id): """ Get the processing status for a measurement id through the API. """ - measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id) + measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id) response = self.session.get(measurement_url, auth=self.auth, @@ -274,23 +248,27 @@ if response_dict['objects']: measurement_list = response_dict['objects'] - measurement = Measurement(measurement_list[0]) - return (measurement.upload, measurement.pre_processing, measurement.processing) + measurement = Measurement(self.base_url, measurement_list[0]) + return measurement.upload, measurement.pre_processing, measurement.processing else: logger.error("No measurement with id %s found on the SCC." % measurement_id) return None def get_measurement(self, measurement_id): - measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id) + measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id) response = self.session.get(measurement_url, auth=self.auth, verify=False) + if response.status_code != 200: + logger.error('Could not access API. Status code %s.' % response.status_code) + sys.exit(1) + response_dict = response.json() if response_dict: - measurement = Measurement(response_dict) + measurement = Measurement(self.base_url,response_dict) return measurement else: logger.error("No measurement with id %s found on the SCC." % measurement_id) @@ -312,7 +290,7 @@ return None # Go the the page confirming the deletion - delete_url = DELETE_MEASUREMENT.format(measurement.id) + delete_url = self.delete_measurement_pattern.format(measurement.id) confirm_page = self.session.get(delete_url, auth=self.auth, @@ -341,7 +319,7 @@ def available_measurements(self): """ Get a list of available measurement on the SCC. """ - measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements') + measurement_url = urlparse.urljoin(self.api_base_url, 'measurements') response = self.session.get(measurement_url, auth=self.auth, verify=False) @@ -350,7 +328,7 @@ measurements = None if response_dict: measurement_list = response_dict['objects'] - measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list] + measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list] logger.info("Found %s measurements on the SCC." % len(measurements)) else: logger.warning("No response received from the SCC when asked for available measurements.") @@ -362,7 +340,7 @@ date. """ date_str = t1.strftime('%Y%m%d') - search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str) + search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) response = self.session.get(search_url, auth=self.auth, @@ -391,7 +369,8 @@ class ApiObject: """ A generic class object. """ - def __init__(self, dict_response): + def __init__(self, base_url, dict_response): + self.base_url = base_url if dict_response: # Add the dictionary key value pairs as object properties @@ -421,11 +400,13 @@ @property def rerun_processing_url(self): - return RERUN_PROCESSING.format(self.id) + url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/') + return url_pattern.format(self.id) @property def rerun_all_url(self): - return RERUN_ALL.format(self.id) + ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/') + return ulr_pattern.format(self.id) def __str__(self): return "%s: %s, %s, %s" % (self.id, @@ -434,64 +415,92 @@ self.processing) -def upload_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): +def upload_file(filename, system_id, settings): """ Shortcut function to upload a file to the SCC. """ logger.info("Uploading file %s, using sytem %s" % (filename, system_id)) - scc = SCC(auth) - scc.login(credential) + scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) + scc.login(settings['website_credentials']) measurement_id = scc.upload_file(filename, system_id) scc.logout() return measurement_id -def process_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): +def process_file(filename, system_id, settings): """ Shortcut function to process a file to the SCC. """ logger.info("Processing file %s, using sytem %s" % (filename, system_id)) - scc = SCC(auth) - scc.login(credential) + scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) + scc.login(settings['website_credentials']) measurement = scc.process(filename, system_id) scc.logout() return measurement -def delete_measurement(measurement_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): +def delete_measurement(measurement_id, settings): """ Shortcut function to delete a measurement from the SCC. """ logger.info("Deleting %s" % measurement_id) - scc = SCC(auth) - scc.login(credential) + + scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) + scc.login(settings['website_credentials']) scc.delete_measurement(measurement_id) scc.logout() -def rerun_all(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): +def rerun_all(measurement_id, monitor, settings): """ Shortcut function to delete a measurement from the SCC. """ logger.info("Rerunning all products for %s" % measurement_id) - scc = SCC(auth) - scc.login(credential) + + scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) + scc.login(settings['website_credentials']) scc.rerun_all(measurement_id, monitor) scc.logout() -def rerun_processing(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): +def rerun_processing(measurement_id, monitor, settings): """ Shortcut function to delete a measurement from the SCC. """ logger.info("Rerunning (optical) processing for %s" % measurement_id) - scc = SCC(auth) - scc.login(credential) + + scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) + scc.login(settings['website_credentials']) scc.rerun_processing(measurement_id, monitor) scc.logout() - + + +def import_settings(config_file_path): + """ Read the configuration file. + + The file should be in YAML syntax.""" + + if not os.path.isfile(config_file_path): + logger.error("Wrong path for configuration file (%s)" % config_file_path) + sys.exit(1) + + with open(config_file_path) as yaml_file: + try: + settings = yaml.safe_load(yaml_file) + logger.debug("Read settings file(%s)" % config_file_path) + except: + logger.error("Could not parse YAML file (%s)" % config_file_path) + sys.exit(1) + + # YAML limitation: does not read tuples + settings['basic_credentials'] = tuple(settings['basic_credentials']) + settings['website_credentials'] = tuple(settings['website_credentials']) + return settings + + def main(): # Define the command line arguments. parser = argparse.ArgumentParser() parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='') parser.add_argument("system", nargs='?', help="Processing system id.", default=0) + parser.add_argument("-c", "--config", nargs='?', help="Path to configuration file") parser.add_argument("-p", "--process", help="Wait for the results of the processing.", action="store_true") parser.add_argument("--delete", help="Measurement ID to delete.") parser.add_argument("--rerun-all", help="Measurement ID to rerun.") - parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.") + parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.") # Verbosity settings from http://stackoverflow.com/a/20663028 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", @@ -506,24 +515,21 @@ # Get the logger with the appropriate level logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) - # If the arguments are OK, try to login on the site and upload. + settings = import_settings(args.config) + + # If the arguments are OK, try to log-in to SCC and upload. if args.delete: # If the delete is provided, do nothing else - delete_measurement(args.delete) + delete_measurement(args.delete, settings) elif args.rerun_all: - rerun_all(args.rerun_all, args.process) + rerun_all(args.rerun_all, args.process, settings) elif args.rerun_processing: - rerun_processing(args.rerun_processing, args.process) + rerun_processing(args.rerun_processing, args.process, settings) else: if (args.filename == '') or (args.system == 0): parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') if args.process: - process_file(args.filename, args.system) + process_file(args.filename, args.system, settings) else: - upload_file(args.filename, args.system) - - -# When running through terminal -if __name__ == '__main__': - main() + upload_file(args.filename, args.system, settings)
