# HG changeset patch # User Iannis B # Date 1540557953 -10800 # Node ID a7b7e0056fce7ca2bed87cd826e45cae5f708e76 # Parent 020e80de1f645d3d3decb60ffa16474968124745 Support of uploading of all ancillary file types. Cleanup of parsers for processing. diff -r 020e80de1f64 -r a7b7e0056fce scc_access/__init__.py --- a/scc_access/__init__.py Fri Oct 12 16:38:28 2018 +0300 +++ b/scc_access/__init__.py Fri Oct 26 15:45:53 2018 +0300 @@ -1,1 +1,1 @@ -__version__ = "0.6.2" \ No newline at end of file +__version__ = "0.6.3" \ No newline at end of file diff -r 020e80de1f64 -r a7b7e0056fce scc_access/scc_access.py --- a/scc_access/scc_access.py Fri Oct 12 16:38:28 2018 +0300 +++ b/scc_access/scc_access.py Fri Oct 26 15:45:53 2018 +0300 @@ -10,7 +10,7 @@ import logging import os import re -from io import StringIO +from io import BytesIO import sys import time import urlparse @@ -86,7 +86,7 @@ """ Logout from SCC """ return self.session.get(self.logout_url, stream=True) - def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None): + def upload_file(self, filename, system_id, rs_filename=None, ov_filename=None, lr_filename=None): """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page @@ -100,15 +100,15 @@ logger.debug('Adding sounding file %s' % rs_filename) files['sounding_file'] = open(rs_filename, 'rb') - if ol_filename is not None: - logger.debug('Adding overlap file %s' % ol_filename) - files['overlap_file'] = open(ol_filename, 'rb') + if ov_filename is not None: + logger.debug('Adding overlap file %s' % ov_filename) + files['overlap_file'] = open(ov_filename, 'rb') if lr_filename is not None: logger.debug('Adding lidar ratio file %s' % lr_filename) files['lidar_ratio_file'] = open(lr_filename, 'rb') - logger.info("Uploading of file %s started." % filename) + logger.info("Uploading of file(s) %s started." % filename) upload_submit = self.session.post(self.upload_url, data=upload_data, @@ -123,7 +123,7 @@ # Check if there was a redirect to a new page. if upload_submit.url == self.upload_url: measurement_id = False - logger.error("Uploaded file rejected! Try to upload manually to see the error.") + logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.") else: measurement_id = re.findall(regex, upload_submit.text)[0] logger.info("Successfully uploaded measurement with id %s." % measurement_id) @@ -135,6 +135,7 @@ subdir. This method is used to download preprocessed file, optical files etc. """ + # TODO: Make downloading more robust (e.g. in case that files do not exist on server). # Get the file request = self.session.get(download_url, stream=True) @@ -147,7 +148,7 @@ os.makedirs(local_dir) # Save the file by chunk, needed if the file is big. - memory_file = StringIO() + memory_file = BytesIO() for chunk in request.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks @@ -182,6 +183,8 @@ download_url = self.download_graph_pattern.format(measurement_id) self.download_files(measurement_id, 'scc_plots', download_url) + # TODO: Add download method for other types of files. + def rerun_processing(self, measurement_id, monitor=True): measurement, status = self.get_measurement(measurement_id) @@ -215,18 +218,22 @@ if monitor: self.monitor_processing(measurement_id) - def process(self, filename, system_id, monitor, rs_filename=None): + def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None): """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) # Upload file logger.info("--- Uploading file") - measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename) + measurement_id = self.upload_file(filename, system_id, + rs_filename=rs_filename, + lr_filename=lr_filename, + ov_filename=ov_filename) - logger.info("--- Monitoring processing") - if monitor: + if measurement_id and monitor: + logger.info("--- Monitoring processing") return self.monitor_processing(measurement_id) + return None def monitor_processing(self, measurement_id): @@ -239,21 +246,21 @@ # try to wait for measurement to appear in API measurement = None - logger.info("looking for measurement %s in SCC", measurement_id) + logger.info("Looking for measurement %s in SCC", measurement_id) while error_count < error_max: time.sleep(time_sleep) measurement, status = self.get_measurement(measurement_id) if status != 200 and error_count < error_max: - logger.error("measurement not found. waiting %ds", time_sleep) + logger.error("Measurement not found. waiting %ds", time_sleep) error_count += 1 else: break if error_count == error_max: - logger.critical("measurement %s doesn't seem to exist", measurement_id) + logger.critical("Measurement %s doesn't seem to exist", measurement_id) sys.exit(1) - logger.info('measurement %s found', measurement_id) + logger.info('Measurement %s found', measurement_id) if measurement is not None: while measurement.is_running: @@ -458,29 +465,22 @@ self.processing) -def upload_file(filename, system_id, settings, rs_filename=None): - """ Shortcut function to upload a file to the SCC. """ - logger.info("Uploading file %s, using sytem %s" % (filename, system_id)) +def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): + """ Shortcut function to process a file to the SCC. """ + logger.info("Processing file %s, using system %s" % (filename, system_id)) scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) scc.login(settings['website_credentials']) - measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename) - scc.logout() - return measurement_id - - -def process_file(filename, system_id, settings, rs_filename=None): - """ Shortcut function to process a file to the SCC. """ - logger.info("Processing file %s, using sytem %s" % (filename, system_id)) - - scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) - scc.login(settings['website_credentials']) - measurement = scc.process(filename, system_id, rs_filename=rs_filename) + measurement = scc.process(filename, system_id, + monitor=monitor, + rs_filename=rs_filename, + lr_filename=lr_filename, + ov_filename=ov_filename) scc.logout() return measurement -def delete_measurement(measurement_ids, settings): +def delete_measurements(measurement_ids, settings): """ Shortcut function to delete measurements from the SCC. """ scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) scc.login(settings['website_credentials']) @@ -568,7 +568,7 @@ # Setup for command specific parsers def setup_delete(parser): def delete_from_args(parsed): - delete_measurement(parsed.IDs, parsed.config) + delete_measurements(parsed.IDs, parsed.config) parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.") parser.set_defaults(execute=delete_from_args) @@ -595,24 +595,35 @@ def setup_process_file(parser): + """ Upload and monitor processing progress.""" def process_file_from_args(parsed): - process_file(parsed.file, parsed.system, parsed.process, parsed.config, parsed.radiosounding) + process_file(parsed.filename, parsed.system, parsed.config, monitor=True, + rs_filename=parsed.radiosounding, + ov_filename=parsed.overlap, + lr_filename=parsed.lidarratio) parser.add_argument("filename", help="Measurement file name or path.") parser.add_argument("system", help="Processing system id.") parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") - parser.add_argument("-p", "--process", help="Wait for the results of the processing.", - action="store_true") + parser.add_argument("--overlap", default=None, help="Overlap file name or path") + parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") + parser.set_defaults(execute=process_file_from_args) def setup_upload_file(parser): + """ Upload but do not monitor processing progress. """ def upload_file_from_args(parsed): - upload_file(parsed.file, parsed.system, parsed.config, parsed.radiosounding) + process_file(parsed.filename, parsed.system, parsed.config, monitor=False, + rs_filename=parsed.radiosounding, + ov_filename=parsed.overlap, + lr_filename=parsed.lidarratio) parser.add_argument("filename", help="Measurement file name or path.") parser.add_argument("system", help="Processing system id.") parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") + parser.add_argument("--overlap", default=None, help="Overlap file name or path") + parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path") parser.set_defaults(execute=upload_file_from_args) @@ -669,12 +680,12 @@ subparsers = parser.add_subparsers() delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.") - rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun a measurement.") + rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.") rerun_processing_parser = subparsers.add_parser("rerun-processing", help="Rerun processing routings for a measurement.") - process_file_parser = subparsers.add_parser("process-file", help="Process a file.") + process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download procesing results.") upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.") - list_parser = subparsers.add_parser("list", help="List all measurements.") + list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.") download_parser = subparsers.add_parser("download", help="Download selected measurements.") setup_delete(delete_parser) @@ -693,6 +704,7 @@ dest="loglevel", const=logging.WARNING ) + # Setup default config location home = os.path.expanduser("~") default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml")) parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path,