Merging pull request for rs uploading

Thu, 11 Oct 2018 13:14:33 +0300

author
Iannis <i.binietoglou@impworks.gr>
date
Thu, 11 Oct 2018 13:14:33 +0300
changeset 28
2d90204710af
parent 27
8ebbba085eb0 (current diff)
parent 26
0aedb8dd6861 (diff)
child 30
4669876326d4

Merging pull request for rs uploading

.hgignore file | annotate | diff | comparison | revisions
scc_access/scc_access.py file | annotate | diff | comparison | revisions
--- a/.hgignore	Thu Oct 11 12:45:05 2018 +0300
+++ b/.hgignore	Thu Oct 11 13:14:33 2018 +0300
@@ -4,4 +4,5 @@
 *.pyc
 re:^settings\.py$
 re:^\.idea/
-scc_access.egg-info/
\ No newline at end of file
+scc_access.egg-info/
+re:^\.pytest_cache/
\ No newline at end of file
--- a/scc_access/scc_access.py	Thu Oct 11 12:45:05 2018 +0300
+++ b/scc_access/scc_access.py	Thu Oct 11 13:14:33 2018 +0300
@@ -9,15 +9,20 @@
     import urlparse  # Python 2
 
 import argparse
+import datetime
+import logging
 import os
 import re
+from io import StringIO
+import sys
 import time
-from io import StringIO
+import urlparse
 from zipfile import ZipFile
-import datetime
-import logging
+
+import requests
 import yaml
 
+requests.packages.urllib3.disable_warnings()
 
 logger = logging.getLogger(__name__)
 
@@ -27,11 +32,11 @@
 
 
 class SCC:
-    """ A simple class that will attempt to upload a file on the SCC server.
+    """A simple class that will attempt to upload a file on the SCC server.
 
     The uploading is done by simulating a normal browser session. In the current
-    version no check is performed, and no feedback is given if the upload 
-    was successful. If everything is setup correctly, it will work. 
+    version no check is performed, and no feedback is given if the upload
+    was successful. If everything is setup correctly, it will work.
     """
 
     def __init__(self, auth, output_dir, base_url):
@@ -68,7 +73,7 @@
             sys.exit(1)
 
         logger.debug("Submiting credentials.")
-        
+
         # Submit the login data
         login_submit = self.session.post(self.login_url,
                                          data=self.login_credentials,
@@ -81,8 +86,8 @@
     def logout(self):
         pass
 
-    def upload_file(self, filename, system_id):
-        """ Upload a filename for processing with a specific system. If the 
+    def upload_file(self, filename, system_id, rs_filename=None):
+        """ Upload a filename for processing with a specific system. If the
         upload is successful, it returns the measurement id. """
         # Get submit page
         upload_page = self.session.get(self.upload_url,
@@ -93,6 +98,9 @@
         upload_data = {'system': system_id}
         files = {'data': open(filename, 'rb')}
 
+        if rs_filename is not None:
+            files['sounding_file'] = open(rs_filename, 'rb')
+
         logger.info("Uploading of file %s started." % filename)
 
         upload_submit = self.session.post(self.upload_url,
@@ -119,7 +127,7 @@
 
     def download_files(self, measurement_id, subdir, download_url):
         """ Downloads some files from the download_url to the specified
-        subdir. This method is used to download preprocessed file, optical 
+        subdir. This method is used to download preprocessed file, optical
         files etc.
         """
         # Get the file
@@ -169,7 +177,7 @@
         self.download_files(measurement_id, 'scc_plots', download_url)
 
     def rerun_processing(self, measurement_id, monitor=True):
-        measurement = self.get_measurement(measurement_id)
+        measurement, status = self.get_measurement(measurement_id)
 
         if measurement:
             request = self.session.get(measurement.rerun_processing_url, auth=self.auth,
@@ -188,7 +196,7 @@
         logger.debug("Started rerun_all procedure.")
 
         logger.debug("Getting measurement %s" % measurement_id)
-        measurement = self.get_measurement(measurement_id)
+        measurement, status = self.get_measurement(measurement_id)
 
         if measurement:
             logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
@@ -205,31 +213,53 @@
             if monitor:
                 self.monitor_processing(measurement_id)
 
-    def process(self, filename, system_id):
+    def process(self, filename, system_id, rs_filename=None):
         """ Upload a file for processing and wait for the processing to finish.
         If the processing is successful, it will download all produced files.
         """
         logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
         # Upload file
-        measurement_id = self.upload_file(filename, system_id)
+        logger.info("--- Uploading file")
+        measurement_id = self.upload_file(filename, system_id, rs_filename=rs_filename)
 
+        logger.info("--- Monitoring processing")
         measurement = self.monitor_processing(measurement_id)
         return measurement
 
     def monitor_processing(self, measurement_id):
         """ Monitor the processing progress of a measurement id"""
 
-        measurement = self.get_measurement(measurement_id)
+        # try to deal with error 404
+        error_count = 0
+        error_max = 6
+        time_sleep = 10
+
+        # try to wait for measurement to appear in API
+        measurement = None
+        logger.info("looking for measurement %s in SCC", measurement_id)
+        while error_count < error_max:
+            time.sleep(time_sleep)
+            measurement, status = self.get_measurement(measurement_id)
+            if status != 200 and error_count < error_max:
+                logger.error("measurement not found. waiting %ds", time_sleep)
+                error_count += 1
+            else:
+                break
+
+        if error_count == error_max:
+            logger.critical("measurement %s doesn't seem to exist", measurement_id)
+            sys.exit(1)
+
+        logger.info('measurement %s found', measurement_id)
+
         if measurement is not None:
+            error_count = 0
             while measurement.is_running:
-                logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
-                                                                                                   measurement.pre_processing,
-                                                                                                   measurement.processing))
+                logger.info("Measurement is being processed (status: %s, %s, %s). Please wait.", measurement.upload, measurement.pre_processing, measurement.processing)
                 time.sleep(10)
-                measurement = self.get_measurement(measurement_id)
-            logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
-                                                                                   measurement.pre_processing,
-                                                                                   measurement.processing))
+                measurement, status = self.get_measurement(measurement_id)
+
+            logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing)
             if measurement.pre_processing == 127:
                 logger.info("Downloading preprocessed files.")
                 self.download_preprocessed(measurement_id)
@@ -266,28 +296,32 @@
                                     auth=self.auth,
                                     verify=False)
 
+        # maybe the measurements isn't already available on the database.
+        if response.status_code == 404:
+            return None, 404
+
         if response.status_code != 200:
             logger.error('Could not access API. Status code %s.' % response.status_code)
-            sys.exit(1)
+            return None, response.status_code
 
         response_dict = response.json()
 
         if response_dict:
-            measurement = Measurement(self.base_url,response_dict)
-            return measurement
+            measurement = Measurement(self.base_url, response_dict)
+            return measurement, response.status_code
         else:
             logger.error("No measurement with id %s found on the SCC." % measurement_id)
-            return None
+            return None, response.status_code
 
     def delete_measurement(self, measurement_id):
         """ Deletes a measurement with the provided measurement id. The user
-        should have the appropriate permissions. 
-        
+        should have the appropriate permissions.
+
         The procedures is performed directly through the web interface and
         NOT through the API.
         """
         # Get the measurement object
-        measurement = self.get_measurement(measurement_id)
+        measurement, status = self.get_measurement(measurement_id)
 
         # Check that it exists
         if measurement is None:
@@ -342,7 +376,7 @@
 
     def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
         """ Give the first available measurement id on the SCC for the specific
-        date. 
+        date.
         """
         date_str = t1.strftime('%Y%m%d')
         search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
@@ -424,24 +458,24 @@
                                    self.processing)
 
 
-def upload_file(filename, system_id, settings):
+def upload_file(filename, system_id, settings, rs_filename=None):
     """ Shortcut function to upload a file to the SCC. """
     logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
 
     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
     scc.login(settings['website_credentials'])
-    measurement_id = scc.upload_file(filename, system_id)
+    measurement_id = scc.upload_file(filename, system_id, rs_filename=rs_filename)
     scc.logout()
     return measurement_id
 
 
-def process_file(filename, system_id, settings):
+def process_file(filename, system_id, settings, rs_filename=None):
     """ Shortcut function to process a file to the SCC. """
     logger.info("Processing file %s, using sytem %s" % (filename, system_id))
 
     scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
     scc.login(settings['website_credentials'])
-    measurement = scc.process(filename, system_id)
+    measurement = scc.process(filename, system_id, rs_filename=rs_filename)
     scc.logout()
     return measurement
 
@@ -511,6 +545,9 @@
     parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
     parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.")
 
+    # others files
+    parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path")
+
     # Verbosity settings from http://stackoverflow.com/a/20663028
     parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
                         dest="loglevel", const=logging.DEBUG, default=logging.INFO,
@@ -539,6 +576,6 @@
             parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
 
         if args.process:
-            process_file(args.filename, args.system, settings)
+            process_file(args.filename, args.system, settings, rs_filename=args.radiosounding)
         else:
-            upload_file(args.filename, args.system, settings)
+            upload_file(args.filename, args.system, settings, rs_filename=args.radiosounding)

mercurial