# HG changeset patch # User Iannis # Date 1539252873 -10800 # Node ID 2d90204710af93d356605266c097349ec8ac448f # Parent 8ebbba085eb0e57b50585758fa108b13559cbab2# Parent 0aedb8dd6861d499110e35cf9390f1277d92a4f9 Merging pull request for rs uploading diff -r 8ebbba085eb0 -r 2d90204710af .hgignore --- a/.hgignore Thu Oct 11 12:45:05 2018 +0300 +++ b/.hgignore Thu Oct 11 13:14:33 2018 +0300 @@ -4,4 +4,5 @@ *.pyc re:^settings\.py$ re:^\.idea/ -scc_access.egg-info/ \ No newline at end of file +scc_access.egg-info/ +re:^\.pytest_cache/ \ No newline at end of file diff -r 8ebbba085eb0 -r 2d90204710af scc_access/scc_access.py --- a/scc_access/scc_access.py Thu Oct 11 12:45:05 2018 +0300 +++ b/scc_access/scc_access.py Thu Oct 11 13:14:33 2018 +0300 @@ -9,15 +9,20 @@ import urlparse # Python 2 import argparse +import datetime +import logging import os import re +from io import StringIO +import sys import time -from io import StringIO +import urlparse from zipfile import ZipFile -import datetime -import logging + +import requests import yaml +requests.packages.urllib3.disable_warnings() logger = logging.getLogger(__name__) @@ -27,11 +32,11 @@ class SCC: - """ A simple class that will attempt to upload a file on the SCC server. + """A simple class that will attempt to upload a file on the SCC server. The uploading is done by simulating a normal browser session. In the current - version no check is performed, and no feedback is given if the upload - was successful. If everything is setup correctly, it will work. + version no check is performed, and no feedback is given if the upload + was successful. If everything is setup correctly, it will work. """ def __init__(self, auth, output_dir, base_url): @@ -68,7 +73,7 @@ sys.exit(1) logger.debug("Submiting credentials.") - + # Submit the login data login_submit = self.session.post(self.login_url, data=self.login_credentials, @@ -81,8 +86,8 @@ def logout(self): pass - def upload_file(self, filename, system_id): - """ Upload a filename for processing with a specific system. If the + def upload_file(self, filename, system_id, rs_filename=None): + """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page upload_page = self.session.get(self.upload_url, @@ -93,6 +98,9 @@ upload_data = {'system': system_id} files = {'data': open(filename, 'rb')} + if rs_filename is not None: + files['sounding_file'] = open(rs_filename, 'rb') + logger.info("Uploading of file %s started." % filename) upload_submit = self.session.post(self.upload_url, @@ -119,7 +127,7 @@ def download_files(self, measurement_id, subdir, download_url): """ Downloads some files from the download_url to the specified - subdir. This method is used to download preprocessed file, optical + subdir. This method is used to download preprocessed file, optical files etc. """ # Get the file @@ -169,7 +177,7 @@ self.download_files(measurement_id, 'scc_plots', download_url) def rerun_processing(self, measurement_id, monitor=True): - measurement = self.get_measurement(measurement_id) + measurement, status = self.get_measurement(measurement_id) if measurement: request = self.session.get(measurement.rerun_processing_url, auth=self.auth, @@ -188,7 +196,7 @@ logger.debug("Started rerun_all procedure.") logger.debug("Getting measurement %s" % measurement_id) - measurement = self.get_measurement(measurement_id) + measurement, status = self.get_measurement(measurement_id) if measurement: logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) @@ -205,31 +213,53 @@ if monitor: self.monitor_processing(measurement_id) - def process(self, filename, system_id): + def process(self, filename, system_id, rs_filename=None): """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) # Upload file - measurement_id = self.upload_file(filename, system_id) + logger.info("--- Uploading file") + measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename) + logger.info("--- Monitoring processing") measurement = self.monitor_processing(measurement_id) return measurement def monitor_processing(self, measurement_id): """ Monitor the processing progress of a measurement id""" - measurement = self.get_measurement(measurement_id) + # try to deal with error 404 + error_count = 0 + error_max = 6 + time_sleep = 10 + + # try to wait for measurement to appear in API + measurement = None + logger.info("looking for measurement %s in SCC", measurement_id) + while error_count < error_max: + time.sleep(time_sleep) + measurement, status = self.get_measurement(measurement_id) + if status != 200 and error_count < error_max: + logger.error("measurement not found. waiting %ds", time_sleep) + error_count += 1 + else: + break + + if error_count == error_max: + logger.critical("measurement %s doesn't seem to exist", measurement_id) + sys.exit(1) + + logger.info('measurement %s found', measurement_id) + if measurement is not None: + error_count = 0 while measurement.is_running: - logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, - measurement.pre_processing, - measurement.processing)) + logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing) time.sleep(10) - measurement = self.get_measurement(measurement_id) - logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, - measurement.pre_processing, - measurement.processing)) + measurement, status = self.get_measurement(measurement_id) + + logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing) if measurement.pre_processing == 127: logger.info("Downloading preprocessed files.") self.download_preprocessed(measurement_id) @@ -266,28 +296,32 @@ auth=self.auth, verify=False) + # maybe the measurements isn't already available on the database. + if response.status_code == 404: + return None, 404 + if response.status_code != 200: logger.error('Could not access API. Status code %s.' % response.status_code) - sys.exit(1) + return None, response.status_code response_dict = response.json() if response_dict: - measurement = Measurement(self.base_url,response_dict) - return measurement + measurement = Measurement(self.base_url, response_dict) + return measurement, response.status_code else: logger.error("No measurement with id %s found on the SCC." % measurement_id) - return None + return None, response.status_code def delete_measurement(self, measurement_id): """ Deletes a measurement with the provided measurement id. The user - should have the appropriate permissions. - + should have the appropriate permissions. + The procedures is performed directly through the web interface and NOT through the API. """ # Get the measurement object - measurement = self.get_measurement(measurement_id) + measurement, status = self.get_measurement(measurement_id) # Check that it exists if measurement is None: @@ -342,7 +376,7 @@ def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): """ Give the first available measurement id on the SCC for the specific - date. + date. """ date_str = t1.strftime('%Y%m%d') search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) @@ -424,24 +458,24 @@ self.processing) -def upload_file(filename, system_id, settings): +def upload_file(filename, system_id, settings, rs_filename=None): """ Shortcut function to upload a file to the SCC. """ logger.info("Uploading file %s, using sytem %s" % (filename, system_id)) scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) scc.login(settings['website_credentials']) - measurement_id = scc.upload_file(filename, system_id) + measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename) scc.logout() return measurement_id -def process_file(filename, system_id, settings): +def process_file(filename, system_id, settings, rs_filename=None): """ Shortcut function to process a file to the SCC. """ logger.info("Processing file %s, using sytem %s" % (filename, system_id)) scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) scc.login(settings['website_credentials']) - measurement = scc.process(filename, system_id) + measurement = scc.process(filename, system_id, rs_filename=rs_filename) scc.logout() return measurement @@ -511,6 +545,9 @@ parser.add_argument("--rerun-all", help="Measurement ID to rerun.") parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.") + # others files + parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") + # Verbosity settings from http://stackoverflow.com/a/20663028 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO, @@ -539,6 +576,6 @@ parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') if args.process: - process_file(args.filename, args.system, settings) + process_file(args.filename, args.system, settings, rs_filename=args.radiosounding) else: - upload_file(args.filename, args.system, settings) + upload_file(args.filename, args.system, settings, rs_filename=args.radiosounding)