Mercurial > public > scc_access / changeset
changeset
Updated script according to new database/api changes.
Fri, 24 Nov 2017 18:50:29 +0200
- author
- Iannis <ulalume3@yahoo.com>
- date
- Fri, 24 Nov 2017 18:50:29 +0200
- changeset 6
- c02712d2ab9e
- parent 5
- 1c170a1ae099
- child 7
- 415d034b0864
Updated script according to new database/api changes.
Added basic logging functions, instead of print statements.
| scc_access.py | file | annotate | diff | comparison | revisions |
--- a/scc_access.py Fri Nov 24 17:32:29 2017 +0200 +++ b/scc_access.py Fri Nov 24 18:50:29 2017 +0200 @@ -25,17 +25,17 @@ __version__ = "0.6.0" - # Try to read the settings from the settings.py file try: from settings import * except: raise ImportError( - """A settings file (setting.py) is required to run the script. - You can use settings.sample.py as a template.""") - + """A settings file (setting.py) is required to run the script. + You can use settings.sample.py as a template.""") import requests +requests.packages.urllib3.disable_warnings() + import urlparse import argparse import os @@ -44,6 +44,9 @@ import StringIO from zipfile import ZipFile import datetime +import logging + +logger = logging.getLogger(__name__) # Construct the absolute URLs @@ -55,115 +58,117 @@ RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/') RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/') -DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/') +DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/') API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/') # The regex to find the measurement id from the measurement page -# This should be read from the uploaded file, but would require an extra module +# This should be read from the uploaded file, but would require an extra NetCDF module. regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>" class SCC: """ A simple class that will attempt to upload a file on the SCC server. - The uploading is done by simulation a normal browser session. In the current + The uploading is done by simulating a normal browser session. In the current version no check is performed, and no feedback is given if the upload was successful. If everything is setup correctly, it will work. """ - def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR): + + def __init__(self, auth=BASIC_LOGIN, output_dir=OUTPUT_DIR): self.auth = auth - self.output_dir = OUTPUT_DIR + self.output_dir = output_dir self.session = requests.Session() - - def login(self, credential = DJANGO_LOGIN): + + def login(self, credentials=DJANGO_LOGIN): """ Login the the website. """ - self.login_credentials = {'username': credential[0], - 'password': credential[1]} - + logger.debug("Attempting to login to SCC, username %s." % credentials[0]) + self.login_credentials = {'username': credentials[0], + 'password': credentials[1]} + + logger.debug("Accessing login page at %s." % LOGIN_URL) + # Get upload form - login_page = self.session.get(LOGIN_URL, - auth = self.auth, verify = False) - + login_page = self.session.get(LOGIN_URL, + auth=self.auth, verify=False) + + logger.debug("Submiting credentials.") # Submit the login data - login_submit = self.session.post(LOGIN_URL, - data = self.login_credentials, - headers = {'X-CSRFToken': login_page.cookies['csrftoken'], - 'referer': LOGIN_URL}, - verify = False, - auth = self.auth) + login_submit = self.session.post(LOGIN_URL, + data=self.login_credentials, + headers={'X-CSRFToken': login_page.cookies['csrftoken'], + 'referer': LOGIN_URL}, + verify=False, + auth=self.auth) return login_submit def logout(self): pass - + def upload_file(self, filename, system_id): """ Upload a filename for processing with a specific system. If the upload is successful, it returns the measurement id. """ # Get submit page - upload_page = self.session.get(UPLOAD_URL, - auth = self.auth, - verify = False) - - # Submit the data + upload_page = self.session.get(UPLOAD_URL, + auth=self.auth, + verify=False) + + # Submit the data upload_data = {'system': system_id} - files = {'data': open(filename, 'rb')} - - print "Uploading of file %s started." % filename - - upload_submit = self.session.post(UPLOAD_URL, - data = upload_data, - files = files, - headers = {'X-CSRFToken': upload_page.cookies['csrftoken'], - 'referer': UPLOAD_URL}, - verify = False, - auth = self.auth) - + files = {'data': open(filename, 'rb')} + + logging.info("Uploading of file %s started." % filename) + + upload_submit = self.session.post(UPLOAD_URL, + data=upload_data, + files=files, + headers={'X-CSRFToken': upload_page.cookies['csrftoken'], + 'referer': UPLOAD_URL}, + verify=False, + auth=self.auth) + if upload_submit.status_code != 200: - print "Connection error. Status code: %s" % upload_submit.status_code + logging.warning("Connection error. Status code: %s" % upload_submit.status_code) return False - - measurement_id = True - + # Check if there was a redirect to a new page. if upload_submit.url == UPLOAD_URL: measurement_id = False - print "Uploaded file rejected! Try to upload manually to see the error." + logging.error("Uploaded file rejected! Try to upload manually to see the error.") else: measurement_id = re.findall(regex, upload_submit.text)[0] - print "Successfully uploaded measurement with id %s." % measurement_id + logging.error("Successfully uploaded measurement with id %s." % measurement_id) return measurement_id - + def download_files(self, measurement_id, subdir, download_url): """ Downloads some files from the download_url to the specified subdir. This method is used to download preprocessed file, optical files etc. - """ + """ # Get the file - request = self.session.get(download_url, auth = self.auth, - verify = False, - stream=True) - + request = self.session.get(download_url, auth=self.auth, + verify=False, + stream=True) + # Create the dir if it does not exist local_dir = os.path.join(self.output_dir, measurement_id, subdir) if not os.path.exists(local_dir): os.makedirs(local_dir) - - + # Save the file by chunk, needed if the file is big. memory_file = StringIO.StringIO() - - for chunk in request.iter_content(chunk_size=1024): - if chunk: # filter out keep-alive new chunks + + for chunk in request.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks memory_file.write(chunk) memory_file.flush() - + zip_file = ZipFile(memory_file) - + for ziped_name in zip_file.namelist(): basename = os.path.basename(ziped_name) - + local_file = os.path.join(local_dir, basename) - + with open(local_file, 'wb') as f: f.write(zip_file.read(ziped_name)) @@ -172,13 +177,13 @@ # Construct the download url download_url = DOWNLOAD_PREPROCESSED.format(measurement_id) self.download_files(measurement_id, 'scc_preprocessed', download_url) - + def download_optical(self, measurement_id): """ Download optical files for the measurement id. """ # Construct the download url download_url = DOWNLOAD_OPTICAL.format(measurement_id) self.download_files(measurement_id, 'scc_optical', download_url) - + def download_graphs(self, measurement_id): """ Download profile graphs for the measurement id. """ # Construct the download url @@ -194,22 +199,28 @@ stream=True) if request.status_code != 200: - print "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code) + logging.error("Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) return if monitor: self.monitor_processing(measurement_id) def rerun_all(self, measurement_id, monitor=True): + logger.debug("Started rerun_all procedure.") + + logger.debug("Getting measurement %s" % measurement_id) measurement = self.get_measurement(measurement_id) if measurement: + logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) + request = self.session.get(measurement.rerun_all_url, auth=self.auth, verify=False, stream=True) if request.status_code != 200: - print "Could not rerun pre processing for %s. Status code: %s" % (measurement_id, request.status_code) + logger.error("Could not rerun pre processing for %s. Status code: %s" % + (measurement_id, request.status_code)) return if monitor: @@ -219,7 +230,7 @@ """ Upload a file for processing and wait for the processing to finish. If the processing is successful, it will download all produced files. """ - print "--- Processing started on %s. ---" % datetime.datetime.now() + logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) # Upload file measurement_id = self.upload_file(filename, system_id) @@ -232,59 +243,59 @@ measurement = self.get_measurement(measurement_id) if measurement is not None: while measurement.is_running: - print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, + logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, measurement.pre_processing, - measurement.opt_retrievals) + measurement.processing)) time.sleep(10) measurement = self.get_measurement(measurement_id) - print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, + logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, measurement.pre_processing, - measurement.opt_retrievals) + measurement.processing)) if measurement.pre_processing == 127: - print "Downloading preprocessed files." + logger.info("Downloading preprocessed files.") self.download_preprocessed(measurement_id) - if measurement.opt_retrievals == 127: - print "Downloading optical files." + if measurement.processing == 127: + logger.info("Downloading optical files.") self.download_optical(measurement_id) - print "Downloading graphs." + logger.info("Downloading graphs.") self.download_graphs(measurement_id) - print "--- Processing finished. ---" + logger.info("--- Processing finished. ---") return measurement def get_status(self, measurement_id): """ Get the processing status for a measurement id through the API. """ measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id) - - response = self.session.get(measurement_url, - auth = self.auth, - verify = False) - + + response = self.session.get(measurement_url, + auth=self.auth, + verify=False) + response_dict = response.json() - + if response_dict['objects']: measurement_list = response_dict['objects'] measurement = Measurement(measurement_list[0]) - return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals) + return (measurement.upload, measurement.pre_processing, measurement.processing) else: - print "No measurement with id %s found on the SCC." % measurement_id + logger.error("No measurement with id %s found on the SCC." % measurement_id) return None - + def get_measurement(self, measurement_id): measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id) - - response = self.session.get(measurement_url, - auth = self.auth, - verify = False) - + + response = self.session.get(measurement_url, + auth=self.auth, + verify=False) + response_dict = response.json() - + if response_dict: measurement = Measurement(response_dict) return measurement else: - print "No measurement with id %s found on the SCC." % measurement_id + logger.error("No measurement with id %s found on the SCC." % measurement_id) return None - + def delete_measurement(self, measurement_id): """ Deletes a measurement with the provided measurement id. The user should have the appropriate permissions. @@ -294,94 +305,94 @@ """ # Get the measurement object measurement = self.get_measurement(measurement_id) - + # Check that it exists if measurement is None: - print "Nothing to delete." + logger.warning("Nothing to delete.") return None - + # Go the the page confirming the deletion delete_url = DELETE_MEASUREMENT.format(measurement.id) - - confirm_page = self.session.get(delete_url, - auth = self.auth, - verify = False) - + + confirm_page = self.session.get(delete_url, + auth=self.auth, + verify=False) + # Check that the page opened properly if confirm_page.status_code != 200: - print "Could not open delete page. Status: {0}".format(confirm_page.status_code) + logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code)) return None - + # Delete the measurement - delete_page = self.session.post(delete_url, - auth=self.auth, - verify=False, - data={'post':'yes'}, - headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], + delete_page = self.session.post(delete_url, + auth=self.auth, + verify=False, + data={'post': 'yes'}, + headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], 'referer': delete_url} - ) + ) if delete_page.status_code != 200: - print "Something went wrong. Delete page status: {0}".format( - delete_page.status_code) + logger.warning("Something went wrong. Delete page status: {0}".format( + delete_page.status_code)) return None - - print "Deleted measurement {0}".format(measurement_id) - return True + + logger.info("Deleted measurement {0}".format(measurement_id)) + return True def available_measurements(self): """ Get a list of available measurement on the SCC. """ measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements') - response = self.session.get(measurement_url, - auth = self.auth, - verify = False) + response = self.session.get(measurement_url, + auth=self.auth, + verify=False) response_dict = response.json() - + measurements = None if response_dict: measurement_list = response_dict['objects'] measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list] - print "Found %s measurements on the SCC." % len(measurements) + logger.info("Found %s measurements on the SCC." % len(measurements)) else: - print "No response received from the SCC when asked for available measurements." + logger.warning("No response received from the SCC when asked for available measurements.") return measurements - - def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0): + + def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): """ Give the first available measurement id on the SCC for the specific date. """ date_str = t1.strftime('%Y%m%d') search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str) - - response = self.session.get(search_url, - auth = self.auth, - verify = False) - + + response = self.session.get(search_url, + auth=self.auth, + verify=False) + response_dict = response.json() - + measurement_id = None - + if response_dict: measurement_list = response_dict['objects'] existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] - + measurement_number = base_number measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) - + while measurement_id in existing_ids: measurement_number = measurement_number + 1 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) if measurement_number == 100: raise ValueError('No available measurement id found.') - return measurement_id - - + return measurement_id + + class ApiObject: """ A generic class object. """ - + def __init__(self, dict_response): - + if dict_response: # Add the dictionary key value pairs as object properties for key, value in dict_response.items(): @@ -394,6 +405,7 @@ class Measurement(ApiObject): """ This class represents the measurement object as returned in the SCC API. """ + @property def is_running(self): """ Returns True if the processing has not finished. @@ -403,74 +415,100 @@ if self.pre_processing == -127: return False if self.pre_processing == 127: - if self.opt_retrievals in [127, -127]: + if self.processing in [127, -127]: return False return True - + + @property def rerun_processing_url(self): return RERUN_PROCESSING.format(self.id) + @property def rerun_all_url(self): return RERUN_ALL.format(self.id) def __str__(self): - return "%s: %s, %s, %s" % (self.id, - self.upload, - self.pre_processing, - self.opt_retrievals) - - -def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + return "%s: %s, %s, %s" % (self.id, + self.upload, + self.pre_processing, + self.processing) + + +def upload_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): """ Shortcut function to upload a file to the SCC. """ + logger.info("Uploading file %s, using sytem %s" % (filename, system_id)) + scc = SCC(auth) scc.login(credential) measurement_id = scc.upload_file(filename, system_id) scc.logout() return measurement_id -def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + +def process_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): """ Shortcut function to process a file to the SCC. """ + logger.info("Processing file %s, using sytem %s" % (filename, system_id)) + scc = SCC(auth) scc.login(credential) measurement = scc.process(filename, system_id) scc.logout() - return measurement + return measurement + -def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): +def delete_measurement(measurement_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): """ Shortcut function to delete a measurement from the SCC. """ + logger.info("Deleting %s" % measurement_id) scc = SCC(auth) scc.login(credential) scc.delete_measurement(measurement_id) scc.logout() -def rerun_all(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + +def rerun_all(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): """ Shortcut function to delete a measurement from the SCC. """ + logger.info("Rerunning all products for %s" % measurement_id) scc = SCC(auth) scc.login(credential) scc.rerun_all(measurement_id, monitor) scc.logout() -def rerun_processing(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + +def rerun_processing(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): """ Shortcut function to delete a measurement from the SCC. """ + logger.info("Rerunning (optical) processing for %s" % measurement_id) scc = SCC(auth) scc.login(credential) scc.rerun_processing(measurement_id, monitor) scc.logout() + # When running through terminal if __name__ == '__main__': - + # Define the command line arguments. parser = argparse.ArgumentParser() - parser.add_argument("filename", nargs='?', help = "Measurement file name or path.", default='') - parser.add_argument("system", nargs='?', help = "Processing system id.", default=0) + parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='') + parser.add_argument("system", nargs='?', help="Processing system id.", default=0) parser.add_argument("-p", "--process", help="Wait for the results of the processing.", - action="store_true") - parser.add_argument("-d", "--delete", help="Measurement ID to delete.") + action="store_true") + parser.add_argument("--delete", help="Measurement ID to delete.") parser.add_argument("--rerun-all", help="Measurement ID to rerun.") parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.") + + # Verbosity settings from http://stackoverflow.com/a/20663028 + parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", + dest="loglevel", const=logging.DEBUG, default=logging.INFO, + ) + parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", + dest="loglevel", const=logging.WARNING + ) + args = parser.parse_args() - + + # Get the logger with the appropriate level + logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) + # If the arguments are OK, try to login on the site and upload. if args.delete: # If the delete is provided, do nothing else @@ -480,9 +518,9 @@ elif args.rerun_processing: rerun_processing(args.rerun_processing, args.process) else: - if (args.filename == '') or (args.system == 0): + if (args.filename == '') or (args.system == 0): parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') - + if args.process: process_file(args.filename, args.system) else:
