# HG changeset patch
# User Iannis <i.binietoglou@impworks.gr>
# Date 1539351508 -10800
# Node ID 020e80de1f645d3d3decb60ffa16474968124745
# Parent  4669876326d4f3fa7f4a4cf9b9a20389d33c48a9
Partial clean-up after merge.

diff -r 4669876326d4 -r 020e80de1f64 scc_access/scc_access.py
--- a/scc_access/scc_access.py	Thu Oct 11 15:25:55 2018 +0300
+++ b/scc_access/scc_access.py	Fri Oct 12 16:38:28 2018 +0300
@@ -42,7 +42,11 @@
         self.session = requests.Session()
         self.session.auth = auth
         self.session.verify = False
+
         self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
+        self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/')
+        self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
+
         self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
         self.download_preprocessed_pattern = urlparse.urljoin(self.base_url,
                                                               'data_processing/measurements/{0}/download-preprocessed/')
@@ -51,11 +55,13 @@
         self.download_graph_pattern = urlparse.urljoin(self.base_url,
                                                        'data_processing/measurements/{0}/download-plots/')
         self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
+
         self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
-        self.list_measurements_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
+        self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/')
+        self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements')
 
     def login(self, credentials):
-        """ Login the the website. """
+        """ Login to SCC. """
         logger.debug("Attempting to login to SCC, username %s." % credentials[0])
         login_credentials = {'username': credentials[0],
                              'password': credentials[1]}
@@ -65,12 +71,10 @@
         # Get upload form
         login_page = self.session.get(self.login_url)
 
-        # TODO: Do we need this? Mortiz removed it.
-        if login_page.status_code != 200:
-            logger.error('Could not access login pages. Status code %s' % login_page.status_code)
-            sys.exit(1)
+        if not login_page.ok:
+            raise self.PageNotAccessibleError('Could not access login pages. Status code %s' % login_page.status_code)
 
-        logger.debug("Submiting credentials.")
+        logger.debug("Submitting credentials.")
         # Submit the login data
         login_submit = self.session.post(self.login_url,
                                          data=login_credentials,
@@ -79,9 +83,10 @@
         return login_submit
 
     def logout(self):
-        pass
+        """ Logout from SCC """
+        return self.session.get(self.logout_url, stream=True)
 
-    def upload_file(self, filename, system_id, rs_filename=None):
+    def upload_file(self, filename, system_id, rs_filename=None, ol_filename=None, lr_filename=None):
         """ Upload a filename for processing with a specific system. If the
         upload is successful, it returns the measurement id. """
         # Get submit page
@@ -92,8 +97,17 @@
         files = {'data': open(filename, 'rb')}
 
         if rs_filename is not None:
+            logger.debug('Adding sounding file %s' % rs_filename)
             files['sounding_file'] = open(rs_filename, 'rb')
 
+        if ol_filename is not None:
+            logger.debug('Adding overlap file %s' % ol_filename)
+            files['overlap_file'] = open(ol_filename, 'rb')
+
+        if lr_filename is not None:
+            logger.debug('Adding lidar ratio file %s' % lr_filename)
+            files['lidar_ratio_file'] = open(lr_filename, 'rb')
+
         logger.info("Uploading of file %s started." % filename)
 
         upload_submit = self.session.post(self.upload_url,
@@ -247,7 +261,7 @@
                 time.sleep(10)
                 measurement, status = self.get_measurement(measurement_id)
 
-            logger.info("Measurement processing finished (status: %s, %s, %s).",measurement.upload, measurement.pre_processing, measurement.processing)
+            logger.info("Measurement processing finished (status: %s, %s, %s).", measurement.upload, measurement.pre_processing, measurement.processing)
             if measurement.pre_processing == 127:
                 logger.info("Downloading preprocessed files.")
                 self.download_preprocessed(measurement_id)
@@ -259,24 +273,9 @@
             logger.info("--- Processing finished. ---")
         return measurement
 
-    def get_status(self, measurement_id):
-        """ Get the processing status for a measurement id through the API. """
-        measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id)
-
-        response = self.session.get(measurement_url)
-
-        response_dict = response.json()
-
-        if response_dict['objects']:
-            measurement_list = response_dict['objects']
-            measurement = Measurement(self.base_url, measurement_list[0])
-            return measurement.upload, measurement.pre_processing, measurement.processing
-        else:
-            logger.error("No measurement with id %s found on the SCC." % measurement_id)
-            return None
-
     def get_measurement(self, measurement_id):
-        measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id)
+        measurement_url = self.api_measurement_pattern.format(measurement_id)
+        logger.debug("Measurement API URL: %s" % measurement_url)
 
         response = self.session.get(measurement_url)
 
@@ -301,7 +300,7 @@
         NOT through the API.
         """
         # Get the measurement object
-        measurement, status = self.get_measurement(measurement_id)
+        measurement, _ = self.get_measurement(measurement_id)
 
         # Check that it exists
         if measurement is None:
@@ -324,7 +323,7 @@
                                         headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
                                                  'referer': delete_url}
                                         )
-        if delete_page.status_code != 200:
+        if not delete_page.ok:
             logger.warning("Something went wrong. Delete page status: {0}".format(
                 delete_page.status_code))
             return None
@@ -334,23 +333,24 @@
 
     def available_measurements(self):
         """ Get a list of available measurement on the SCC. """
-        measurement_url = urlparse.urljoin(self.api_base_url, 'measurements')
-        response = self.session.get(measurement_url)
+        response = self.session.get(self.api_measurements_url)
         response_dict = response.json()
 
-        measurements = None
         if response_dict:
             measurement_list = response_dict['objects']
             measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list]
             logger.info("Found %s measurements on the SCC." % len(measurements))
         else:
             logger.warning("No response received from the SCC when asked for available measurements.")
+            measurements = None
 
         return measurements
 
     def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None,
                           processing_status=None, optical_processing=None):
 
+        # TODO: Change this to work through the API
+
         # Need to set to empty string if not specified, we won't get any results
         params = {
             "station": station if station is not None else "",
@@ -361,13 +361,14 @@
             "preprocessing_status": processing_status if processing_status is not None else "",
             "optical_processing_status": optical_processing if optical_processing is not None else ""
         }
-        resp = self.session.get(self.list_measurements_pattern, params=params).text
+
+        response_txt = self.session.get(self.list_measurements_url, params=params).text
         tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL)
         entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL)
         measurement_rgx = re.compile(
             r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*',
             re.DOTALL)
-        matches = tbl_rgx.findall(resp)
+        matches = tbl_rgx.findall(response_txt)
         if len(matches) != 1:
             return []
 
@@ -382,12 +383,13 @@
 
         return ret
 
-    def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
+    def measurement_id_for_date(self, t1, call_sign, base_number=0):
         """ Give the first available measurement id on the SCC for the specific
         date.
         """
         date_str = t1.strftime('%Y%m%d')
-        search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
+        base_id = "%s%s" % (date_str, call_sign)
+        search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id)
 
         response = self.session.get(search_url)
 
@@ -397,19 +399,24 @@
 
         if response_dict:
             measurement_list = response_dict['objects']
+
+            if len(measurement_list) == 100:
+                raise ValueError('No available measurement id found.')
+
             existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
 
             measurement_number = base_number
-            measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
+            measurement_id = "%s%02i" % (base_id, measurement_number)
 
             while measurement_id in existing_ids:
                 measurement_number = measurement_number + 1
-                measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
-                if measurement_number == 100:
-                    raise ValueError('No available measurement id found.')
+                measurement_id = "%s%02i" % (base_id, measurement_number)
 
         return measurement_id
 
+    class PageNotAccessibleError(RuntimeError):
+        pass
+
 
 class ApiObject(object):
     """ A generic class object. """
@@ -435,19 +442,6 @@
     """
 
     @property
-    def is_running(self):
-        """ Returns True if the processing has not finished.
-        """
-        if self.upload == 0:
-            return False
-        if self.pre_processing == -127:
-            return False
-        if self.pre_processing == 127:
-            if self.processing in [127, -127]:
-                return False
-        return True
-
-    @property
     def rerun_processing_url(self):
         url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/')
         return url_pattern.format(self.id)