ioannis@0: #!/usr/bin/env python
ioannis@0: """
ioannis@0: The MIT License (MIT)
ioannis@0: 
ioannis@0: Copyright (c) 2015, Ioannis Binietoglou
ioannis@0: 
ioannis@0: Permission is hereby granted, free of charge, to any person obtaining a copy
ioannis@0: of this software and associated documentation files (the "Software"), to deal
ioannis@0: in the Software without restriction, including without limitation the rights
ioannis@0: to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
ioannis@0: copies of the Software, and to permit persons to whom the Software is
ioannis@0: furnished to do so, subject to the following conditions:
ioannis@0: 
ioannis@0: The above copyright notice and this permission notice shall be included in
ioannis@0: all copies or substantial portions of the Software.
ioannis@0: 
ioannis@0: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
ioannis@0: IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
ioannis@0: FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
ioannis@0: AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
ioannis@0: LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
ioannis@0: OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
ioannis@0: THE SOFTWARE.
ioannis@0: """
ioannis@0: 
ioannis@0: # Try to read the settings from the settings.py file
ioannis@0: try:
ioannis@0:     from settings import *
ioannis@0: except:
ioannis@0:     raise ImportError(
ioannis@0:             """A settings file (setting.py) is required to run the script. 
ioannis@0:              You can use settings.sample.py for a template.""")
ioannis@0: 
ioannis@0: 
ioannis@0: import requests
ioannis@0: import urlparse
ioannis@0: import argparse
ioannis@0: import os
ioannis@0: import re
ioannis@0: import time
ioannis@0: import StringIO
ioannis@0: from zipfile import ZipFile
ioannis@0: import datetime
ioannis@0: 
ioannis@0: 
ioannis@0: # Construct the absolute URLs
ioannis@0: LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/')
ioannis@0: UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/')
ioannis@0: DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/')
ioannis@0: DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/')
ioannis@0: DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/')
ioannis@0: DELETE_MEASUREMENT =  urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
ioannis@0: API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
ioannis@0: 
ioannis@0: # The regex to find the measurement id from the measurement page
ioannis@0: # This should be read from the uploaded file, but would require an extra module
ioannis@0: regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
ioannis@0: 
ioannis@0: 
ioannis@0: class SCC:
ioannis@0:     """ A simple class that will attempt to upload a file on the SCC server.
ioannis@0:     The uploading is done by simulation a normal browser session. In the current
ioannis@0:     version no check is performed, and no feedback is given if the upload 
ioannis@0:     was successful. If everything is setup correctly, it will work. 
ioannis@0:     """
ioannis@0:     def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR):
ioannis@0:         self.auth = auth
ioannis@0:         self.output_dir = OUTPUT_DIR
ioannis@0:         self.session = requests.Session()
ioannis@0:         
ioannis@0:     def login(self, credential = DJANGO_LOGIN):
ioannis@0:         """ Login the the website. """
ioannis@0:         self.login_credentials = {'username': credential[0], 
ioannis@0:                                   'password': credential[1]}
ioannis@0:         
ioannis@0:         # Get upload form
ioannis@0:         login_page = self.session.get(LOGIN_URL, 
ioannis@0:                                         auth = self.auth, verify = False)
ioannis@0:         
ioannis@0:         # Submit the login data
ioannis@0:         login_submit = self.session.post(LOGIN_URL, 
ioannis@0:                                           data = self.login_credentials, 
ioannis@0:                                           headers = {'X-CSRFToken': login_page.cookies['csrftoken'], 
ioannis@0:                                                      'referer': LOGIN_URL}, 
ioannis@0:                                           verify = False, 
ioannis@0:                                           auth = self.auth)
ioannis@0:         return login_submit
ioannis@0: 
ioannis@0:     def logout(self):
ioannis@0:         pass
ioannis@0:     
ioannis@0:     def upload_file(self, filename, system_id):
ioannis@0:         """ Upload a filename for processing with a specific system. If the 
ioannis@0:         upload is successful, it returns the measurement id. """
ioannis@0:         # Get submit page
ioannis@0:         upload_page = self.session.get(UPLOAD_URL, 
ioannis@0:                                        auth = self.auth, 
ioannis@0:                                        verify = False)
ioannis@0:        
ioannis@0:        # Submit the data
ioannis@0:         upload_data = {'system': system_id}
ioannis@0:         files  = {'data': open(filename, 'rb')}
ioannis@0:         
ioannis@0:         print "Uploading of file %s started." % filename
ioannis@0:         
ioannis@0:         upload_submit = self.session.post(UPLOAD_URL, 
ioannis@0:                                           data = upload_data, 
ioannis@0:                                           files = files, 
ioannis@0:                                           headers = {'X-CSRFToken': upload_page.cookies['csrftoken'], 
ioannis@0:                                                      'referer': UPLOAD_URL}, 
ioannis@0:                                           verify = False, 
ioannis@0:                                           auth = self.auth)
ioannis@0:         
ioannis@0:         if upload_submit.status_code != 200:
ioannis@0:             print "Connection error. Status code: %s" % upload_submit.status_code
ioannis@0:             return False
ioannis@0:         
ioannis@0:         measurement_id = True    
ioannis@0:         
ioannis@0:         # Check if there was a redirect to a new page.
ioannis@0:         if upload_submit.url == UPLOAD_URL:
ioannis@0:             measurement_id = False
ioannis@0:             print "Uploaded file rejected! Try to upload manually to see the error."
ioannis@0:         else:
ioannis@0:             measurement_id = re.findall(regex, upload_submit.text)[0]
ioannis@0:             print "Successfully uploaded measurement with id %s." % measurement_id
ioannis@0: 
ioannis@0:         return measurement_id
ioannis@0:     
ioannis@0:     def download_files(self, measurement_id, subdir, download_url):
ioannis@0:         """ Downloads some files from the download_url to the specified
ioannis@0:         subdir. This method is used to download preprocessed file, optical 
ioannis@0:         files etc.
ioannis@0:         """        
ioannis@0:         # Get the file
ioannis@0:         request = self.session.get(download_url, auth = self.auth, 
ioannis@0:                                                  verify = False,
ioannis@0:                                                  stream=True)
ioannis@0:         
ioannis@0:         # Create the dir if it does not exist
ioannis@0:         local_dir = os.path.join(self.output_dir, measurement_id, subdir)
ioannis@0:         if not os.path.exists(local_dir):
ioannis@0:             os.makedirs(local_dir)
ioannis@0:         
ioannis@0:         
ioannis@0:         # Save the file by chunk, needed if the file is big.
ioannis@0:         memory_file = StringIO.StringIO()
ioannis@0:         
ioannis@0:         for chunk in request.iter_content(chunk_size=1024): 
ioannis@0:             if chunk: # filter out keep-alive new chunks
ioannis@0:                 memory_file.write(chunk)
ioannis@0:                 memory_file.flush()
ioannis@0:         
ioannis@0:         zip_file = ZipFile(memory_file)
ioannis@0:         
ioannis@0:         for ziped_name in zip_file.namelist():
ioannis@0:             basename = os.path.basename(ziped_name)
ioannis@0:            
ioannis@0:             local_file = os.path.join(local_dir, basename)
ioannis@0:         
ioannis@0:             with open(local_file, 'wb') as f:
ioannis@0:                 f.write(zip_file.read(ziped_name))
ioannis@0: 
ioannis@0:     def download_preprocessed(self, measurement_id):
ioannis@0:         """ Download preprocessed files for the measurement id. """
ioannis@0:         # Construct the download url
ioannis@0:         download_url = DOWNLOAD_PREPROCESSED.format(measurement_id)
ioannis@0:         self.download_files(measurement_id, 'scc_preprocessed', download_url)
ioannis@0:     
ioannis@0:     def download_optical(self, measurement_id):
ioannis@0:         """ Download optical files for the measurement id. """
ioannis@0:         # Construct the download url
ioannis@0:         download_url = DOWNLOAD_OPTICAL.format(measurement_id)
ioannis@0:         self.download_files(measurement_id, 'scc_optical', download_url)
ioannis@0:     
ioannis@0:     def download_graphs(self, measurement_id):
ioannis@0:         """ Download profile graphs for the measurement id. """
ioannis@0:         # Construct the download url
ioannis@0:         download_url = DOWNLOAD_GRAPH.format(measurement_id)
ioannis@0:         self.download_files(measurement_id, 'scc_plots', download_url)
ioannis@0:     
ioannis@0:     def process(self, filename, system_id):
ioannis@0:         """ Upload a file for processing and wait for the processing to finish.
ioannis@0:         If the processing is successful, it will download all produced files.
ioannis@0:         """
ioannis@0:         print "--- Processing started on %s. ---" % datetime.datetime.now()
ioannis@0:         # Upload file
ioannis@0:         measurement_id = self.upload_file(filename, system_id) 
ioannis@0:         
ioannis@0:         measurement = None
ioannis@0:         if measurement_id:
ioannis@0:             measurement = self.get_measurement(measurement_id)
ioannis@0:             while measurement.is_running:
ioannis@0:                 print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, 
ioannis@0:                                                                                              measurement.pre_processing, 
ioannis@0:                                                                                              measurement.opt_retrievals)
ioannis@0:                 time.sleep(10)
ioannis@0:                 measurement = self.get_measurement(measurement_id)
ioannis@0:             print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, 
ioannis@0:                                                                              measurement.pre_processing, 
ioannis@0:                                                                              measurement.opt_retrievals)
ioannis@0:             if measurement.pre_processing == 127:
ioannis@0:                 print "Downloading preprocessed files."
ioannis@0:                 self.download_preprocessed(measurement_id)
ioannis@0:             if measurement.opt_retrievals == 127:
ioannis@0:                 print "Downloading optical files."
ioannis@0:                 self.download_optical(measurement_id)
ioannis@0:                 print "Downloading graphs."
ioannis@0:                 self.download_graphs(measurement_id)
ioannis@0:         print "--- Processing finished. ---"
ioannis@0:         return measurement
ioannis@0:         
ioannis@0:     def get_status(self, measurement_id):
ioannis@0:         """ Get the processing status for a measurement id through the API. """
ioannis@0:         measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id)
ioannis@0:         
ioannis@0:         response = self.session.get(measurement_url, 
ioannis@0:                                     auth = self.auth, 
ioannis@0:                                     verify = False)
ioannis@0:         
ioannis@0:         response_dict = response.json()
ioannis@0:         
ioannis@0:         if response_dict['objects']:
ioannis@0:             measurement_list = response_dict['objects']
ioannis@0:             measurement = Measurement(measurement_list[0])
ioannis@0:             return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals)
ioannis@0:         else:
ioannis@0:             print "No measurement with id %s found on the SCC." % measurement_id
ioannis@0:             return None
ioannis@0:     
ioannis@0:     def get_measurement(self, measurement_id):
ioannis@0:         measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id)
ioannis@0:         
ioannis@0:         response = self.session.get(measurement_url, 
ioannis@0:                                     auth = self.auth, 
ioannis@0:                                     verify = False)
ioannis@0:         
ioannis@0:         response_dict = response.json()
ioannis@0:         
ioannis@0:         if response_dict:
ioannis@0:             measurement = Measurement(response_dict)
ioannis@0:             return measurement
ioannis@0:         else:
ioannis@0:             print "No measurement with id %s found on the SCC." % measurement_id
ioannis@0:             return None
ioannis@0:     
ioannis@0:     def delete_measurement(self, measurement_id):
ioannis@0:         """ Deletes a measurement with the provided measurement id. The user
ioannis@0:         should have the appropriate permissions. 
ioannis@0:         
ioannis@0:         The procedures is performed directly through the web interface and
ioannis@0:         NOT through the API.
ioannis@0:         """
ioannis@0:         # Get the measurement object
ioannis@0:         measurement = self.get_measurement(measurement_id)
ioannis@0:         
ioannis@0:         # Check that it exists
ioannis@0:         if measurement is None:
ioannis@0:             print "Nothing to delete."
ioannis@0:             return None
ioannis@0:         
ioannis@0:         # Go the the page confirming the deletion
ioannis@0:         delete_url = DELETE_MEASUREMENT.format(measurement.id)
ioannis@0:         
ioannis@0:         confirm_page = self.session.get(delete_url, 
ioannis@0:                                     auth = self.auth, 
ioannis@0:                                     verify = False)
ioannis@0:          
ioannis@0:         # Check that the page opened properly
ioannis@0:         if confirm_page.status_code != 200:
ioannis@0:             print "Could not open delete page. Status: {0}".format(confirm_page.status_code)
ioannis@0:             return None
ioannis@0:         
ioannis@0:         # Delete the measurement
ioannis@0:         delete_page = self.session.post(delete_url, 
ioannis@0:                                        auth=self.auth, 
ioannis@0:                                        verify=False,
ioannis@0:                                        data={'post':'yes'}, 
ioannis@0:                                        headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
ioannis@0:                                                  'referer': delete_url}
ioannis@0:                                                  )
ioannis@0:         if delete_page.status_code != 200:
ioannis@0:             print "Something went wrong. Delete page status: {0}".format(
ioannis@0:                                                         delete_page.status_code)
ioannis@0:             return None
ioannis@0:         
ioannis@0:         print "Deleted measurement {0}".format(measurement_id)
ioannis@0:         return True    
ioannis@0: 
ioannis@0:     def available_measurements(self):
ioannis@0:         """ Get a list of available measurement on the SCC. """
ioannis@0:         measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements')
ioannis@0:         response = self.session.get(measurement_url, 
ioannis@0:                                     auth = self.auth, 
ioannis@0:                                     verify = False)
ioannis@0:         response_dict = response.json()
ioannis@0:         
ioannis@0:         measurements = None
ioannis@0:         if response_dict:
ioannis@0:             measurement_list = response_dict['objects']
ioannis@0:             measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list]
ioannis@0:             print "Found %s measurements on the SCC." % len(measurements)
ioannis@0:         else:
ioannis@0:             print "No response received from the SCC when asked for available measurements."
ioannis@0: 
ioannis@0:         return measurements
ioannis@0:     
ioannis@0:     def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0):
ioannis@0:         """ Give the first available measurement id on the SCC for the specific
ioannis@0:         date. 
ioannis@0:         """
ioannis@0:         date_str = t1.strftime('%Y%m%d')
ioannis@0:         search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str)
ioannis@0:         
ioannis@0:         response = self.session.get(search_url, 
ioannis@0:                                     auth = self.auth, 
ioannis@0:                                     verify = False)
ioannis@0:         
ioannis@0:         response_dict = response.json()
ioannis@0:         
ioannis@0:         measurement_id = None
ioannis@0:         
ioannis@0:         if response_dict:
ioannis@0:             measurement_list = response_dict['objects']
ioannis@0:             existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
ioannis@0:             
ioannis@0:             measurement_number = base_number
ioannis@0:             measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
ioannis@0:            
ioannis@0:             while measurement_id in existing_ids:
ioannis@0:                 measurement_number = measurement_number + 1
ioannis@0:                 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
ioannis@0:                 if measurement_number == 100:
ioannis@0:                     raise ValueError('No available measurement id found.')
ioannis@0: 
ioannis@0:         return measurement_id     
ioannis@0:         
ioannis@0:         
ioannis@0: class ApiObject:
ioannis@0:     """ A generic class object. """
ioannis@0:     
ioannis@0:     def __init__(self, dict_response):
ioannis@0:         
ioannis@0:         if dict_response:
ioannis@0:             # Add the dictionary key value pairs as object properties
ioannis@0:             for key, value in dict_response.items():
ioannis@0:                 setattr(self, key, value)
ioannis@0:             self.exists = True
ioannis@0:         else:
ioannis@0:             self.exists = False
ioannis@0: 
ioannis@0: 
ioannis@0: class Measurement(ApiObject):
ioannis@0:     """ This class represents the measurement object as returned in the SCC API.
ioannis@0:     """
ioannis@0:     @property
ioannis@0:     def is_running(self):
ioannis@0:         """ Returns True if the processing has not finished.
ioannis@0:         """
ioannis@0:         if self.upload == 0:
ioannis@0:             return False
ioannis@0:         if self.pre_processing == -127:
ioannis@0:             return False
ioannis@0:         if self.pre_processing == 127:
ioannis@0:             if self.opt_retrievals in [127, -127]:
ioannis@0:                 return False
ioannis@0:         return True
ioannis@0:     
ioannis@0:     def delete(self):
ioannis@0:         """ Delete the entry from the SCC database. """
ioannis@0:         
ioannis@0:         
ioannis@0:     def __str__(self):
ioannis@0:         return "%s: %s, %s, %s" % (self.id, 
ioannis@0:                                    self.upload, 
ioannis@0:                                    self.pre_processing, 
ioannis@0:                                    self.opt_retrievals)
ioannis@0:                                    
ioannis@0:                                    
ioannis@0: def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
ioannis@0:     """ Shortcut function to upload a file to the SCC. """
ioannis@0:     scc = SCC(auth)
ioannis@0:     scc.login(credential)
ioannis@0:     measurement_id = scc.upload_file(filename, system_id)
ioannis@0:     scc.logout()
ioannis@0:     return measurement_id
ioannis@0: 
ioannis@0: def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
ioannis@0:     """ Shortcut function to process a file to the SCC. """
ioannis@0:     scc = SCC(auth)
ioannis@0:     scc.login(credential)
ioannis@0:     measurement = scc.process(filename, system_id)
ioannis@0:     scc.logout()
ioannis@0:     return measurement   
ioannis@0: 
ioannis@0: def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
ioannis@0:     """ Shortcut function to delete a measurement from the SCC. """
ioannis@0:     scc = SCC(auth)
ioannis@0:     scc.login(credential)
ioannis@0:     scc.delete_measurement(measurement_id)
ioannis@0:     scc.logout()
ioannis@0: 
ioannis@0: # When running through terminal
ioannis@0: if __name__ == '__main__':
ioannis@0:     
ioannis@0:     # Define the command line arguments.
ioannis@0:     parser = argparse.ArgumentParser()
ioannis@0:     parser.add_argument("filename",  nargs='?', help = "Measurement file name or path.", default='')
ioannis@0:     parser.add_argument("system", nargs='?', help = "Processing system id.", default=0)
ioannis@0:     parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
ioannis@0:                     action="store_true")
ioannis@0:     parser.add_argument("-d", "--delete", help="Measurement ID to delete.")
ioannis@0:     args = parser.parse_args()
ioannis@0:     
ioannis@0:     # If the arguments are OK, try to login on the site and upload.
ioannis@0:     if args.delete:
ioannis@0:         # If the delete is provided, do nothing else
ioannis@0:         delete_measurement(args.delete)
ioannis@0:     else:
ioannis@0:         if (args.filename == '') or  (args.system == 0):
ioannis@0:             parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
ioannis@0:             
ioannis@0:         if args.process:
ioannis@0:             process_file(args.filename, args.system)
ioannis@0:         else:
ioannis@0:             upload_file(args.filename, args.system)