Mercurial > public > scc_access / changeset

--- a/scc_access/__init__.py	Fri Oct 12 16:38:28 2018 +0300
+++ b/scc_access/__init__.py	Fri Oct 26 15:45:53 2018 +0300
@@ -1,1 +1,1 @@
-__version__ = "0.6.2"
\ No newline at end of file
+__version__ = "0.6.3"
\ No newline at end of file

--- a/scc_access/scc_access.py	Fri Oct 12 16:38:28 2018 +0300
+++ b/scc_access/scc_access.py	Fri Oct 26 15:45:53 2018 +0300
@@ -10,7 +10,7 @@
 import logging
 import os
 import re
-from io import StringIO
+from io import BytesIO
 import sys
 import time
 import urlparse
@@ -86,7 +86,7 @@
         """ Logout from SCC """
         return self.session.get(self.logout_url, stream=True)
 
-    def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None):
+    def upload_file(self, filename, system_id, rs_filename=None, ov_filename=None, lr_filename=None):
         """ Upload a filename for processing with a specific system. If the
         upload is successful, it returns the measurement id. """
         # Get submit page
@@ -100,15 +100,15 @@
             logger.debug('Adding sounding file %s' % rs_filename)
             files['sounding_file'] = open(rs_filename, 'rb')
 
-        if ol_filename is not None:
-            logger.debug('Adding overlap file %s' % ol_filename)
-            files['overlap_file'] = open(ol_filename, 'rb')
+        if ov_filename is not None:
+            logger.debug('Adding overlap file %s' % ov_filename)
+            files['overlap_file'] = open(ov_filename, 'rb')
 
         if lr_filename is not None:
             logger.debug('Adding lidar ratio file %s' % lr_filename)
             files['lidar_ratio_file'] = open(lr_filename, 'rb')
 
-        logger.info("Uploading of file %s started." % filename)
+        logger.info("Uploading of file(s) %s started." % filename)
 
         upload_submit = self.session.post(self.upload_url,
                                           data=upload_data,
@@ -123,7 +123,7 @@
         # Check if there was a redirect to a new page.
         if upload_submit.url == self.upload_url:
             measurement_id = False
-            logger.error("Uploaded file rejected! Try to upload manually to see the error.")
+            logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.")
         else:
             measurement_id = re.findall(regex, upload_submit.text)[0]
             logger.info("Successfully uploaded measurement with id %s." % measurement_id)
@@ -135,6 +135,7 @@
         subdir. This method is used to download preprocessed file, optical
         files etc.
         """
+        # TODO: Make downloading more robust (e.g. in case that files do not exist on server).
         # Get the file
         request = self.session.get(download_url, stream=True)
 
@@ -147,7 +148,7 @@
             os.makedirs(local_dir)
 
         # Save the file by chunk, needed if the file is big.
-        memory_file = StringIO()
+        memory_file = BytesIO()
 
         for chunk in request.iter_content(chunk_size=1024):
             if chunk:  # filter out keep-alive new chunks
@@ -182,6 +183,8 @@
         download_url = self.download_graph_pattern.format(measurement_id)
         self.download_files(measurement_id, 'scc_plots', download_url)
 
+    # TODO: Add download method for other types of files.
+
     def rerun_processing(self, measurement_id, monitor=True):
         measurement, status = self.get_measurement(measurement_id)
 
@@ -215,18 +218,22 @@
             if monitor:
                 self.monitor_processing(measurement_id)
 
-    def process(self, filename, system_id, monitor, rs_filename=None):
+    def process(self, filename, system_id, monitor, rs_filename=None, lr_filename=None, ov_filename=None):
         """ Upload a file for processing and wait for the processing to finish.
         If the processing is successful, it will download all produced files.
         """
         logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
         # Upload file
         logger.info("--- Uploading file")
-        measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename)
+        measurement_id = self.upload_file(filename, system_id,
+                                          rs_filename=rs_filename,
+                                          lr_filename=lr_filename,
+                                          ov_filename=ov_filename)
 
-        logger.info("--- Monitoring processing")
-        if monitor:
+        if measurement_id and monitor:
+            logger.info("--- Monitoring processing")
             return self.monitor_processing(measurement_id)
+
         return None
 
     def monitor_processing(self, measurement_id):
@@ -239,21 +246,21 @@
 
         # try to wait for measurement to appear in API
         measurement = None
-        logger.info("looking for measurement %s in SCC", measurement_id)
+        logger.info("Looking for measurement %s in SCC", measurement_id)
         while error_count < error_max:
             time.sleep(time_sleep)
             measurement, status = self.get_measurement(measurement_id)
             if status != 200 and error_count < error_max:
-                logger.error("measurement not found. waiting %ds", time_sleep)
+                logger.error("Measurement not found. waiting %ds", time_sleep)
                 error_count += 1
             else:
                 break
 
         if error_count == error_max:
-            logger.critical("measurement %s doesn't seem to exist", measurement_id)
+            logger.critical("Measurement %s doesn't seem to exist", measurement_id)
             sys.exit(1)
 
-        logger.info('measurement %s found', measurement_id)
+        logger.info('Measurement %s found', measurement_id)
 
         if measurement is not None:
             while measurement.is_running:
@@ -458,29 +465,22 @@
                                    self.processing)
 
 
-def upload_file(filename, system_id, settings, rs_filename=None):
-    """ Shortcut function to upload a file to the SCC. """
-    logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
+def process_file(filename, system_id, settings, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
+    """ Shortcut function to process a file to the SCC. """
+    logger.info("Processing file %s, using system %s" % (filename, system_id))
 
     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
     scc.login(settings['website_credentials'])
-    measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename)
-    scc.logout()
-    return measurement_id
-
-
-def process_file(filename, system_id, settings, rs_filename=None):
-    """ Shortcut function to process a file to the SCC. """
-    logger.info("Processing file %s, using sytem %s" % (filename, system_id))
-
-    scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
-    scc.login(settings['website_credentials'])
-    measurement = scc.process(filename, system_id, rs_filename=rs_filename)
+    measurement = scc.process(filename, system_id,
+                              monitor=monitor,
+                              rs_filename=rs_filename,
+                              lr_filename=lr_filename,
+                              ov_filename=ov_filename)
     scc.logout()
     return measurement
 
 
-def delete_measurement(measurement_ids, settings):
+def delete_measurements(measurement_ids, settings):
     """ Shortcut function to delete measurements from the SCC. """
     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
     scc.login(settings['website_credentials'])
@@ -568,7 +568,7 @@
 # Setup for command specific parsers
 def setup_delete(parser):
     def delete_from_args(parsed):
-        delete_measurement(parsed.IDs, parsed.config)
+        delete_measurements(parsed.IDs, parsed.config)
 
     parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.")
     parser.set_defaults(execute=delete_from_args)
@@ -595,24 +595,35 @@
 
 
 def setup_process_file(parser):
+    """ Upload and monitor processing progress."""
     def process_file_from_args(parsed):
-        process_file(parsed.file, parsed.system, parsed.process, parsed.config, parsed.radiosounding)
+        process_file(parsed.filename, parsed.system, parsed.config, monitor=True,
+                     rs_filename=parsed.radiosounding,
+                     ov_filename=parsed.overlap,
+                     lr_filename=parsed.lidarratio)
 
     parser.add_argument("filename", help="Measurement file name or path.")
     parser.add_argument("system", help="Processing system id.")
     parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
-    parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
-                        action="store_true")
+    parser.add_argument("--overlap", default=None, help="Overlap file name or path")
+    parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
+
     parser.set_defaults(execute=process_file_from_args)
 
 
 def setup_upload_file(parser):
+    """ Upload but do not monitor processing progress. """
     def upload_file_from_args(parsed):
-        upload_file(parsed.file, parsed.system, parsed.config, parsed.radiosounding)
+        process_file(parsed.filename, parsed.system, parsed.config, monitor=False,
+                     rs_filename=parsed.radiosounding,
+                     ov_filename=parsed.overlap,
+                     lr_filename=parsed.lidarratio)
 
     parser.add_argument("filename", help="Measurement file name or path.")
     parser.add_argument("system", help="Processing system id.")
     parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
+    parser.add_argument("--overlap", default=None, help="Overlap file name or path")
+    parser.add_argument("--lidarratio", default=None, help="Lidar ratio file name or path")
 
     parser.set_defaults(execute=upload_file_from_args)
 
@@ -669,12 +680,12 @@
     subparsers = parser.add_subparsers()
 
     delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
-    rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun a measurement.")
+    rerun_all_parser = subparsers.add_parser("rerun-all", help="Reprocess a measurement on the SCC.")
     rerun_processing_parser = subparsers.add_parser("rerun-processing",
                                                     help="Rerun processing routings for a measurement.")
-    process_file_parser = subparsers.add_parser("process-file", help="Process a file.")
+    process_file_parser = subparsers.add_parser("process-file", help="Upload a file and download procesing results.")
     upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.")
-    list_parser = subparsers.add_parser("list", help="List all measurements.")
+    list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
     download_parser = subparsers.add_parser("download", help="Download selected measurements.")
 
     setup_delete(delete_parser)
@@ -693,6 +704,7 @@
                         dest="loglevel", const=logging.WARNING
                         )
 
+    # Setup default config location
     home = os.path.expanduser("~")
     default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml"))
     parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path,