--- a/scc_access/scc_access.py Fri Oct 12 16:38:28 2018 +0300
+++ b/scc_access/scc_access.py Fri Oct 26 15:45:53 2018 +0300
@@ -10,7 +10,7 @@
import logging
import os
import re
-from io import StringIO
+from io import BytesIO
import sys
import time
import urlparse
@@ -86,7 +86,7 @@
""" Logout from SCC """
return self.session.get(self.logout_url, stream=True)
- def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None):
+ def upload_file(self, filename, system_id, rs_filename=None, ov_filename=None, lr_filename=None):
""" Upload a filename for processing with a specific system. If the
upload is successful, it returns the measurement id. """
# Get submit page
@@ -100,15 +100,15 @@
logger.debug('Adding sounding file %s' % rs_filename)
files['sounding_file'] = open(rs_filename, 'rb')
- if ol_filename is not None:
- logger.debug('Adding overlap file %s' % ol_filename)
- files['overlap_file'] = open(ol_filename, 'rb')
+ if ov_filename is not None:
+ logger.debug('Adding overlap file %s' % ov_filename)
+ files['overlap_file'] = open(ov_filename, 'rb')
if lr_filename is not None:
logger.debug('Adding lidar ratio file %s' % lr_filename)
files['lidar_ratio_file'] = open(lr_filename, 'rb')
- logger.info("Uploading of file %s started." % filename)
+ logger.info("Uploading of file(s) %s started." % filename)
upload_submit = self.session.post(self.upload_url,
data=upload_data,
@@ -123,7 +123,7 @@
# Check if there was a redirect to a new page.
if upload_submit.url == self.upload_url:
measurement_id = False
- logger.error("Uploaded file rejected! Try to upload manually to see the error.")
+ logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.")
else:
measurement_id = re.findall(regex, upload_submit.text)[0]
logger.info("Successfully uploaded measurement with id %s." % measurement_id)
@@ -135,6 +135,7 @@
subdir. This method is used to download preprocessed file, optical
files etc.
"""
+ # TODO: Make downloading more robust (e.g. in case that files do not exist on server).
# Get the file
request = self.session.get(download_url, stream=True)
@@ -147,7 +148,7 @@
os.makedirs(local_dir)
# Save the file by chunk, needed if the file is big.
- memory_file = StringIO()
+ memory_file = BytesIO()
for chunk in request.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
@@ -182,6 +183,8 @@
download_url = self.download_graph_pattern.format(measurement_id)
self.download_files(measurement_id, 'scc_plots', download_url)
+ # TODO: Add download method for other types of files.
+
def rerun_processing(self, measurement_id, monitor=True):
measurement, status = self.get_measurement(measurement_id)
@@ -215,18 +218,22 @@
if monitor:
self.monitor_processing(measurement_id)
- def process(self, filename, system_id, monitor, rs_filename=None):
+ def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None):
""" Upload a file for processing and wait for the processing to finish.
If the processing is successful, it will download all produced files.
"""
logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
# Upload file
logger.info("--- Uploading file")
- measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename)
+ measurement_id = self.upload_file(filename, system_id,
+ rs_filename=rs_filename,
+ lr_filename=lr_filename,
+ ov_filename=ov_filename)
- logger.info("--- Monitoring processing")
- if monitor:
+ if measurement_id and monitor:
+ logger.info("--- Monitoring processing")
return self.monitor_processing(measurement_id)
+
return None
def monitor_processing(self, measurement_id):
@@ -239,21 +246,21 @@
# try to wait for measurement to appear in API
measurement = None
- logger.info("looking for measurement %s in SCC", measurement_id)
+ logger.info("Looking for measurement %s in SCC", measurement_id)
while error_count < error_max:
time.sleep(time_sleep)
measurement, status = self.get_measurement(measurement_id)
if status != 200 and error_count < error_max:
- logger.error("measurement not found. waiting %ds", time_sleep)
+ logger.error("Measurement not found. waiting %ds", time_sleep)
error_count += 1
else:
break
if error_count == error_max:
- logger.critical("measurement %s doesn't seem to exist", measurement_id)
+ logger.critical("Measurement %s doesn't seem to exist", measurement_id)
sys.exit(1)
- logger.info('measurement %s found', measurement_id)
+ logger.info('Measurement %s found', measurement_id)
if measurement is not None:
while measurement.is_running:
@@ -458,29 +465,22 @@
self.processing)
-def upload_file(filename, system_id, settings, rs_filename=None):
- """ Shortcut function to upload a file to the SCC. """
- logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
+def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
+ """ Shortcut function to process a file to the SCC. """
+ logger.info("Processing file %s, using system %s" % (filename, system_id))
scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
scc.login(settings['website_credentials'])
- measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename)
- scc.logout()
- return measurement_id
-
-
-def process_file(filename, system_id, settings, rs_filename=None):
- """ Shortcut function to process a file to the SCC. """
- logger.info("Processing file %s, using sytem %s" % (filename, system_id))
-
- scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
- scc.login(settings['website_credentials'])
- measurement = scc.process(filename, system_id, rs_filename=rs_filename)
+ measurement = scc.process(filename, system_id,
+ monitor=monitor,
+ rs_filename=rs_filename,
+ lr_filename=lr_filename,
+ ov_filename=ov_filename)
scc.logout()
return measurement
-def delete_measurement(measurement_ids, settings):
+def delete_measurements(measurement_ids, settings):
""" Shortcut function to delete measurements from the SCC. """
scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
scc.login(settings['website_credentials'])
@@ -568,7 +568,7 @@
# Setup for command specific parsers
def setup_delete(parser):
def delete_from_args(parsed):
- delete_measurement(parsed.IDs, parsed.config)
+ delete_measurements(parsed.IDs, parsed.config)
parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.")
parser.set_defaults(execute=delete_from_args)
@@ -595,24 +595,35 @@
def setup_process_file(parser):
+ """ Upload and monitor processing progress."""
def process_file_from_args(parsed):
- process_file(parsed.file, parsed.system, parsed.process, parsed.config, parsed.radiosounding)
+ process_file(parsed.filename, parsed.system, parsed.config, monitor=True,
+ rs_filename=parsed.radiosounding,
+ ov_filename=parsed.overlap,
+ lr_filename=parsed.lidarratio)
parser.add_argument("filename", help="Measurement file name or path.")
parser.add_argument("system", help="Processing system id.")
parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
- parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
- action="store_true")
+ parser.add_argument("--overlap", default=None, help="Overlap file name or path")
+ parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
+
parser.set_defaults(execute=process_file_from_args)
def setup_upload_file(parser):
+ """ Upload but do not monitor processing progress. """
def upload_file_from_args(parsed):
- upload_file(parsed.file, parsed.system, parsed.config, parsed.radiosounding)
+ process_file(parsed.filename, parsed.system, parsed.config, monitor=False,
+ rs_filename=parsed.radiosounding,
+ ov_filename=parsed.overlap,
+ lr_filename=parsed.lidarratio)
parser.add_argument("filename", help="Measurement file name or path.")
parser.add_argument("system", help="Processing system id.")
parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
+ parser.add_argument("--overlap", default=None, help="Overlap file name or path")
+ parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
parser.set_defaults(execute=upload_file_from_args)
@@ -669,12 +680,12 @@
subparsers = parser.add_subparsers()
delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
- rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun a measurement.")
+ rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.")
rerun_processing_parser = subparsers.add_parser("rerun-processing",
help="Rerun processing routings for a measurement.")
- process_file_parser = subparsers.add_parser("process-file", help="Process a file.")
+ process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download procesing results.")
upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.")
- list_parser = subparsers.add_parser("list", help="List all measurements.")
+ list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
download_parser = subparsers.add_parser("download", help="Download selected measurements.")
setup_delete(delete_parser)
@@ -693,6 +704,7 @@
dest="loglevel", const=logging.WARNING
)
+ # Setup default config location
home = os.path.expanduser("~")
default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml"))
parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path,
