Slightly improved documentation. default tip

Mon, 24 May 2021 11:43:31 +0300

author
ioannis@ioannis-VirtualBox
date
Mon, 24 May 2021 11:43:31 +0300
changeset 67
0106aeed80d8
parent 66
58006f895f8a

Slightly improved documentation.

scc_access/scc_access.py file | annotate | diff | comparison | revisions
--- a/scc_access/scc_access.py	Fri May 21 16:27:08 2021 +0300
+++ b/scc_access/scc_access.py	Mon May 24 11:43:31 2021 +0300
@@ -1,3 +1,17 @@
+""" This is a script that allows interaction with the SCC through the command line.
+
+It is based on the requests module for accessing the server.
+
+Most of the interactions are done through the web interface, i.e. by mimicking user interaction with the
+SCC website (i.e. user login, data submission, etc.). In few cases, the SCC API is also used.
+
+Most of the functionality is included in the SCC class. The class is used to login into the SCC website and automate
+interaction with the site (i.e. upload a file, get measurement status, etc.).
+
+Two other classes (Measurement, AncillaryFile) are used in some cases to handle the output of the SCC API.
+
+Several shortcut functions are defined to perform specific tasks using the SCC class (e.g. process_file, delete_measurements etc).
+"""
 import sys
 
 import requests
@@ -31,11 +45,11 @@
 
 
 class SCC:
-    """A simple class that will attempt to upload a file on the SCC server.
+    """A  class that will attempt to interact SCC server.
 
-    The uploading is done by simulating a normal browser session. In the current
-    version no check is performed, and no feedback is given if the upload
-    was successful. If everything is setup correctly, it will work.
+    Most interactions are by simulating a normal browser session. In the current
+    version few checks are performed before upload a file, and no feedback is given in case the upload
+    fails.
     """
 
     def __init__(self, auth, output_dir, base_url):
@@ -47,6 +61,7 @@
         self.session.auth = auth
         self.session.verify = False
 
+        # Setup SCC server URLS for later use
         self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
         self.logout_url = urlparse.urljoin(self.base_url, 'accounts/logout/')
         self.list_measurements_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
@@ -54,20 +69,21 @@
         self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
         self.measurement_page_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/')
         self.download_hirelpp_pattern = urlparse.urljoin(self.base_url,
-                                                             'data_processing/measurements/{0}/download-hirelpp/')
+                                                         'data_processing/measurements/{0}/download-hirelpp/')
         self.download_cloudmask_pattern = urlparse.urljoin(self.base_url,
-                                                         'data_processing/measurements/{0}/download-cloudmask/')
+                                                           'data_processing/measurements/{0}/download-cloudmask/')
 
         self.download_elpp_pattern = urlparse.urljoin(self.base_url,
-                                                              'data_processing/measurements/{0}/download-preprocessed/')
+                                                      'data_processing/measurements/{0}/download-preprocessed/')
         self.download_elda_pattern = urlparse.urljoin(self.base_url,
-                                                         'data_processing/measurements/{0}/download-optical/')
+                                                      'data_processing/measurements/{0}/download-optical/')
         self.download_plots_pattern = urlparse.urljoin(self.base_url,
                                                        'data_processing/measurements/{0}/download-plots/')
         self.download_elic_pattern = urlparse.urljoin(self.base_url,
-                                                       'data_processing/measurements/{0}/download-elic/')
+                                                      'data_processing/measurements/{0}/download-elic/')
         self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
 
+        # Setup API URLs for later use
         self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
         self.api_measurement_pattern = urlparse.urljoin(self.api_base_url, 'measurements/{0}/')
         self.api_measurements_url = urlparse.urljoin(self.api_base_url, 'measurements')
@@ -76,14 +92,20 @@
         self.api_overlap_search_pattern = urlparse.urljoin(self.api_base_url, 'overlap_files/?filename={0}')
 
     def login(self, credentials):
-        """ Login to SCC. """
+        """ Login to the SCC.
+
+        Parameters
+        ----------
+        credentials : tuple or list
+           A list or tuple in the form (username, password).
+        """
         logger.debug("Attempting to login to SCC, username %s." % credentials[0])
         login_credentials = {'username': credentials[0],
                              'password': credentials[1]}
 
         logger.debug("Accessing login page at %s." % self.login_url)
 
-        # Get upload form
+        # Get login form
         login_page = self.session.get(self.login_url)
 
         if not login_page.ok:
@@ -98,14 +120,35 @@
         return login_submit
 
     def logout(self):
-        """ Logout from SCC """
+        """ Logout from the SCC """
         return self.session.get(self.logout_url, stream=True)
 
-    def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None, ov_filename=None, lr_filename=None):
-        """ Upload a filename for processing with a specific system. If the
-        upload is successful, it returns the measurement id. """
+    def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None,
+                    ov_filename=None, lr_filename=None):
+        """ Upload a file for processing.
+
+        If the upload is successful, it returns the measurement id.
+
+
+        Parameters
+        ----------
+        filename : str
+           File path of the file to upload
+        system_id : int
+           System id to be used in the processing
+        force_upload : bool
+           If True, if a measurement with the same ID is found on the server, it will be first deleted and the
+           file current file will be uploaded. If False, the file will not be uploaded if the measurement ID is
+           already present on the SCC server.
+        delete_related : bool
+           Answer to delete related question when deleting existing measurements from the SCC.
+        rs_filename, ov_filename, lr_filename : str
+           Ancillary files pahts to be uploaded.
+        """
+        # Get the measurement ID from the netcdf file
         measurement_id = self.measurement_id_from_file(filename)
 
+        # Handle possible existing measurements with the same ID on the SCC server.
         logger.debug('Checking if a measurement with the same id already exists on the SCC server.')
         existing_measurement, _ = self.get_measurement(measurement_id)
 
@@ -118,9 +161,12 @@
                 logger.error(
                     "Measurement with id {} already exists on the SCC. Use --force_upload flag to overwrite it.".format(
                         measurement_id))
-                # TODO: Implement handling at the proper place. This does not allow the SCC class to be used by external programs.
+                # TODO: Implement handling at the proper place. Exiting here does not allow the SCC class to be
+                # used by external programs. Instead an exception should be raised.
                 sys.exit(1)
 
+        # Upload the file(s)
+
         # Get submit page
         upload_page = self.session.get(self.upload_url)
 
@@ -132,11 +178,13 @@
 
         files = {'data': open(filename, 'rb')}
 
+        # Add ancillary files to be uploaded
         if rs_filename is not None:
             ancillary_file, _ = self.get_ancillary(rs_filename, 'sounding')
 
             if ancillary_file.already_on_scc:
-                logger.warning("Sounding file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
+                logger.warning(
+                    "Sounding file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
             else:
                 logger.debug('Adding sounding file %s' % rs_filename)
                 files['sounding_file'] = open(rs_filename, 'rb')
@@ -145,7 +193,8 @@
             ancillary_file, _ = self.get_ancillary(ov_filename, 'overlap')
 
             if ancillary_file.already_on_scc:
-                logger.warning("Overlap file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
+                logger.warning(
+                    "Overlap file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
             else:
                 logger.debug('Adding overlap file %s' % ov_filename)
                 files['overlap_file'] = open(ov_filename, 'rb')
@@ -155,11 +204,13 @@
 
             if ancillary_file.already_on_scc:
                 logger.warning(
-                    "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file))
+                    "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(
+                        ancillary_file))
             else:
                 logger.debug('Adding lidar ratio file %s' % lr_filename)
                 files['lidar_ratio_file'] = open(lr_filename, 'rb')
 
+        # Upload the files
         logger.info("Uploading of file %s started." % filename)
 
         upload_submit = self.session.post(self.upload_url,
@@ -172,19 +223,27 @@
             logger.warning("Connection error. Status code: %s" % upload_submit.status_code)
             return False
 
-        # Check if there was a redirect to a new page.
+        # Check if there was a redirect to a new page. If not, something went wrong
         if upload_submit.url == self.upload_url:
             measurement_id = False
             logger.error("Uploaded file(s) rejected! Try to upload manually to see the error.")
         else:
-            measurement_id = re.findall(regex, upload_submit.text)[0]
+            # TODO: Check if this is needed. This was used when the measurement ID was not read from the input file.
+            measurement_id = re.findall(regex, upload_submit.text)[0]  # Get the measurement ID from the output page
             logger.info("Successfully uploaded measurement with id %s." % measurement_id)
-            logger.info("You can monitor the processing progress online: {}".format(self.measurement_page_pattern.format(measurement_id)))
+            logger.info("You can monitor the processing progress online: {}".format(
+                self.measurement_page_pattern.format(measurement_id)))
         return measurement_id
 
     @staticmethod
     def measurement_id_from_file(filename):
-        """ Get the measurement id from the input file. """
+        """ Get the measurement id from the input file.
+
+        Parameters
+        ----------
+        filename : str
+           File path of the input file.
+        """
 
         if not os.path.isfile(filename):
             logger.error("File {} does not exist.".format(filename))
@@ -203,8 +262,10 @@
 
     def download_files(self, measurement_id, subdir, download_url):
         """ Downloads some files from the download_url to the specified
-        subdir. This method is used to download preprocessed file, optical
-        files etc.
+        subdir.
+
+        This is a general method used to download preprocessed file, optical
+        files by other, file-specific, methods.
         """
         # TODO: Make downloading more robust (e.g. in case that files do not exist on server).
         # Get the file
@@ -345,7 +406,8 @@
             if monitor:
                 self.monitor_processing(measurement_id)
 
-    def process(self, filename, system_id, monitor,  force_upload, delete_related, delay=0, rs_filename=None, lr_filename=None, ov_filename=None):
+    def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, rs_filename=None,
+                lr_filename=None, ov_filename=None):
         """ Upload a file for processing and wait for the processing to finish.
         If the processing is successful, it will download all produced files.
         """
@@ -375,7 +437,7 @@
         attempts_count = 0
         max_attempts = retry_max + 1
 
-        # try to wait for measurement to appear in API
+        # try to wait for measurement to appear in API. A user has reported that this does not happen immediately.
         measurement = None
         logger.info("Looking for measurement %s on the SCC.", measurement_id)
 
@@ -424,7 +486,7 @@
         if measurement.elic == 127:
             logger.info("Downloading ELIC files.")
             self.download_elic(measurement_id)
-        if measurement.is_calibration and measurement.eldec==0:
+        if measurement.is_calibration and measurement.eldec == 0:
             logger.info("Downloading ELDEC files.")
             self.download_eldec(measurement_id)
         logger.info("--- Processing finished. ---")
@@ -432,10 +494,24 @@
         return measurement
 
     def get_measurement(self, measurement_id):
+        """ Get a measurement information from the SCC API.
 
-        if measurement_id is None:  # Is this still required?
+        Parameters
+        ----------
+        measurement_id : str
+           The measurement ID to search.
+
+        Returns
+        -------
+        : Measurement object or None
+           If the measurement is found, a Measurement object is returned. If not, it returns None
+        """
+        # TODO: Consider to homogenize with get_ancillary method (i.e. always return a Measurement object).
+
+        if measurement_id is None:  # TODO: Is this still required?
             return None
 
+        # Access the API
         measurement_url = self.api_measurement_pattern.format(measurement_id)
         logger.debug("Measurement API URL: %s" % measurement_url)
 
@@ -461,7 +537,7 @@
 
     def delete_measurement(self, measurement_id, delete_related):
         """ Deletes a measurement with the provided measurement id. The user
-        should have the appropriate permissions.
+        should have the appropriate permissions (i.e. access to the admin site).
 
         The procedures is performed directly through the web interface and
         NOT through the API.
@@ -506,7 +582,10 @@
         return True
 
     def available_measurements(self):
-        """ Get a list of available measurement on the SCC. """
+        """ Get a list of available measurement on the SCC.
+
+        The methods is currently not used, could be merged with list_measurements.
+        """
         response = self.session.get(self.api_measurements_url)
         response_dict = response.json()
 
@@ -521,8 +600,9 @@
         return measurements
 
     def list_measurements(self, id_exact=None, id_startswith=None):
+        """ Get the response text from the API. """
 
-        # TODO: Change this to work through the API
+        # TODO: Add some error handling, e.g. as per available_measurements method
 
         # Need to set to empty string if not specified, we won't get any results
         params = {}
@@ -540,6 +620,7 @@
         """ Give the first available measurement id on the SCC for the specific
         date.
         """
+        # TODO: Check if this method needs updating to handle all measurement_ID formats.
         date_str = t1.strftime('%Y%m%d')
         base_id = "%s%s" % (date_str, call_sign)
         search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % base_id)
@@ -742,6 +823,7 @@
 class AncillaryFile(ApiObject):
     """ This class represents the ancilalry file object as returned in the SCC API.
     """
+
     @property
     def already_on_scc(self):
         if self.exists is False:
@@ -755,6 +837,7 @@
                                self.status)
 
 
+# Methods that use the SCC class to perform specific tasks.
 def process_file(filename, system_id, settings, force_upload, delete_related,
                  delay=0, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None):
     """ Shortcut function to process a file to the SCC. """
@@ -881,6 +964,7 @@
 
 def setup_upload_file(parser):
     """ Upload but do not monitor processing progress. """
+
     def upload_file_from_args(parsed):
         process_file(parsed.filename, parsed.system, parsed.config,
                      delay=parsed.delay,
@@ -933,8 +1017,10 @@
         download_measurements(parsed.IDs, parsed.max_retries, parsed.ignore_errors, parsed.config)
 
     parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+")
-    parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0, type=int)
-    parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.", action="store_false")
+    parser.add_argument("--max_retries", help="Number of times to retry in cases of missing measurement id.", default=0,
+                        type=int)
+    parser.add_argument("--ignore_errors", help="Ignore errors when downloading multiple measurements.",
+                        action="store_false")
     parser.set_defaults(execute=download_measurements_from_args)
 
 
@@ -944,10 +1030,12 @@
     subparsers = parser.add_subparsers()
 
     delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
-    rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun all processing steps for the provided measurement IDs.")
+    rerun_all_parser = subparsers.add_parser("rerun-all",
+                                             help="Rerun all processing steps for the provided measurement IDs.")
     rerun_processing_parser = subparsers.add_parser("rerun-elpp",
                                                     help="Rerun low-resolution processing steps for the provided measurement ID.")
-    upload_file_parser = subparsers.add_parser("upload-file", help="Submit a file and, optionally, download the output products.")
+    upload_file_parser = subparsers.add_parser("upload-file",
+                                               help="Submit a file and, optionally, download the output products.")
     list_parser = subparsers.add_parser("list", help="List measurements registered on the SCC.")
     download_parser = subparsers.add_parser("download", help="Download selected measurements.")
 

mercurial