Mercurial > public > scc_access / changeset
changeset
Initial commit.
Tue, 23 Jun 2015 10:30:08 +0300
- author
- Iannis <ioannis@inoe.ro>
- date
- Tue, 23 Jun 2015 10:30:08 +0300
- changeset 0
- a172f26a566a
- child 1
- 783c8a0db76f
Initial commit.
| .hgignore | file | annotate | diff | comparison | revisions | |
| readme.rst | file | annotate | diff | comparison | revisions | |
| scc_access.py | file | annotate | diff | comparison | revisions | |
| settings.sample.py | file | annotate | diff | comparison | revisions |
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgignore Tue Jun 23 10:30:08 2015 +0300 @@ -0,0 +1,5 @@ +syntax: glob +*.rst~ +*.py~ +*.pyc +re:^settings\.py$
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/readme.rst Tue Jun 23 10:30:08 2015 +0300 @@ -0,0 +1,71 @@ +SCC access script +================= + +This small scripts permits interacting with the Single Calculus Chain through +the command line. Specifically, with the script you can: + +* Upload a file to the SCC for processing +* Download the processed files and graphs +* Delete an existing measurement from the SCC (with appropriate privileges) + +The main functions are implemented in a class (SCC) that you can also import +and use in your custom scripts. + +The script does not provide any feedback if a file upload fails. Before using +the script, you will need to upload some files manually and be confident that +your SCC file format and processing settings are correct. + +Please note that this is not part of the "official" SCC tools. I will update +it when I have time but I cannot guarantee to maintain it in the long run. + +Any suggestions for improvements and new features are more than welcome. + +Installation +------------ + +The only requirement for running this program is the requests python module. +If you have pip installed you can install it by:: + + pip install requests. + +See http://docs.python-requests.org/en/latest/user/install/ for more details. + + +Settings +-------- +You will need to change some user-defined settings in a settings.py file. You +can rename the settings.sample.py file to settings.py and follow the instructions +there. + +Specifically, you will need to: + +1. Change the BASIC_LOGIN and DJANGO_LOGIN to your credentials. +2. Change the OUTPUT_DIR to the location were the results will be stored. + +Please not that it's not a good idea to store your own credentials in the settings +file. The standard user has "Station Management" privileges and if the credentials +are stolen, someone could change/delete the stations settings from the SCC database. +For this, it is better to use a used account with minimum access settings, that +can only upload files and measurements. + + +Usage +----- + +You can upload a file specifying the username and the system id:: + + python scc_access.py 20110101po01.nc 125 + +If you want to wait for the processing to finish and download the resulting files +you need to define the -p flag:: + + python scc_access.py 20110101po01.nc 125 -p + +If you want to delete an existing measurement id from the database use the -d flag and +the measurement id:: + + python scc_access.py -d 20110101po01 + +For more information on the syntax type:: + + python scc_access.py -h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scc_access.py Tue Jun 23 10:30:08 2015 +0300 @@ -0,0 +1,426 @@ +#!/usr/bin/env python +""" +The MIT License (MIT) + +Copyright (c) 2015, Ioannis Binietoglou + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +# Try to read the settings from the settings.py file +try: + from settings import * +except: + raise ImportError( + """A settings file (setting.py) is required to run the script. + You can use settings.sample.py for a template.""") + + +import requests +import urlparse +import argparse +import os +import re +import time +import StringIO +from zipfile import ZipFile +import datetime + + +# Construct the absolute URLs +LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/') +UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/') +DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/') +DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/') +DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/') +DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/') +API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/') + +# The regex to find the measurement id from the measurement page +# This should be read from the uploaded file, but would require an extra module +regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>" + + +class SCC: + """ A simple class that will attempt to upload a file on the SCC server. + The uploading is done by simulation a normal browser session. In the current + version no check is performed, and no feedback is given if the upload + was successful. If everything is setup correctly, it will work. + """ + def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR): + self.auth = auth + self.output_dir = OUTPUT_DIR + self.session = requests.Session() + + def login(self, credential = DJANGO_LOGIN): + """ Login the the website. """ + self.login_credentials = {'username': credential[0], + 'password': credential[1]} + + # Get upload form + login_page = self.session.get(LOGIN_URL, + auth = self.auth, verify = False) + + # Submit the login data + login_submit = self.session.post(LOGIN_URL, + data = self.login_credentials, + headers = {'X-CSRFToken': login_page.cookies['csrftoken'], + 'referer': LOGIN_URL}, + verify = False, + auth = self.auth) + return login_submit + + def logout(self): + pass + + def upload_file(self, filename, system_id): + """ Upload a filename for processing with a specific system. If the + upload is successful, it returns the measurement id. """ + # Get submit page + upload_page = self.session.get(UPLOAD_URL, + auth = self.auth, + verify = False) + + # Submit the data + upload_data = {'system': system_id} + files = {'data': open(filename, 'rb')} + + print "Uploading of file %s started." % filename + + upload_submit = self.session.post(UPLOAD_URL, + data = upload_data, + files = files, + headers = {'X-CSRFToken': upload_page.cookies['csrftoken'], + 'referer': UPLOAD_URL}, + verify = False, + auth = self.auth) + + if upload_submit.status_code != 200: + print "Connection error. Status code: %s" % upload_submit.status_code + return False + + measurement_id = True + + # Check if there was a redirect to a new page. + if upload_submit.url == UPLOAD_URL: + measurement_id = False + print "Uploaded file rejected! Try to upload manually to see the error." + else: + measurement_id = re.findall(regex, upload_submit.text)[0] + print "Successfully uploaded measurement with id %s." % measurement_id + + return measurement_id + + def download_files(self, measurement_id, subdir, download_url): + """ Downloads some files from the download_url to the specified + subdir. This method is used to download preprocessed file, optical + files etc. + """ + # Get the file + request = self.session.get(download_url, auth = self.auth, + verify = False, + stream=True) + + # Create the dir if it does not exist + local_dir = os.path.join(self.output_dir, measurement_id, subdir) + if not os.path.exists(local_dir): + os.makedirs(local_dir) + + + # Save the file by chunk, needed if the file is big. + memory_file = StringIO.StringIO() + + for chunk in request.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + memory_file.write(chunk) + memory_file.flush() + + zip_file = ZipFile(memory_file) + + for ziped_name in zip_file.namelist(): + basename = os.path.basename(ziped_name) + + local_file = os.path.join(local_dir, basename) + + with open(local_file, 'wb') as f: + f.write(zip_file.read(ziped_name)) + + def download_preprocessed(self, measurement_id): + """ Download preprocessed files for the measurement id. """ + # Construct the download url + download_url = DOWNLOAD_PREPROCESSED.format(measurement_id) + self.download_files(measurement_id, 'scc_preprocessed', download_url) + + def download_optical(self, measurement_id): + """ Download optical files for the measurement id. """ + # Construct the download url + download_url = DOWNLOAD_OPTICAL.format(measurement_id) + self.download_files(measurement_id, 'scc_optical', download_url) + + def download_graphs(self, measurement_id): + """ Download profile graphs for the measurement id. """ + # Construct the download url + download_url = DOWNLOAD_GRAPH.format(measurement_id) + self.download_files(measurement_id, 'scc_plots', download_url) + + def process(self, filename, system_id): + """ Upload a file for processing and wait for the processing to finish. + If the processing is successful, it will download all produced files. + """ + print "--- Processing started on %s. ---" % datetime.datetime.now() + # Upload file + measurement_id = self.upload_file(filename, system_id) + + measurement = None + if measurement_id: + measurement = self.get_measurement(measurement_id) + while measurement.is_running: + print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, + measurement.pre_processing, + measurement.opt_retrievals) + time.sleep(10) + measurement = self.get_measurement(measurement_id) + print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, + measurement.pre_processing, + measurement.opt_retrievals) + if measurement.pre_processing == 127: + print "Downloading preprocessed files." + self.download_preprocessed(measurement_id) + if measurement.opt_retrievals == 127: + print "Downloading optical files." + self.download_optical(measurement_id) + print "Downloading graphs." + self.download_graphs(measurement_id) + print "--- Processing finished. ---" + return measurement + + def get_status(self, measurement_id): + """ Get the processing status for a measurement id through the API. """ + measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id) + + response = self.session.get(measurement_url, + auth = self.auth, + verify = False) + + response_dict = response.json() + + if response_dict['objects']: + measurement_list = response_dict['objects'] + measurement = Measurement(measurement_list[0]) + return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals) + else: + print "No measurement with id %s found on the SCC." % measurement_id + return None + + def get_measurement(self, measurement_id): + measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id) + + response = self.session.get(measurement_url, + auth = self.auth, + verify = False) + + response_dict = response.json() + + if response_dict: + measurement = Measurement(response_dict) + return measurement + else: + print "No measurement with id %s found on the SCC." % measurement_id + return None + + def delete_measurement(self, measurement_id): + """ Deletes a measurement with the provided measurement id. The user + should have the appropriate permissions. + + The procedures is performed directly through the web interface and + NOT through the API. + """ + # Get the measurement object + measurement = self.get_measurement(measurement_id) + + # Check that it exists + if measurement is None: + print "Nothing to delete." + return None + + # Go the the page confirming the deletion + delete_url = DELETE_MEASUREMENT.format(measurement.id) + + confirm_page = self.session.get(delete_url, + auth = self.auth, + verify = False) + + # Check that the page opened properly + if confirm_page.status_code != 200: + print "Could not open delete page. Status: {0}".format(confirm_page.status_code) + return None + + # Delete the measurement + delete_page = self.session.post(delete_url, + auth=self.auth, + verify=False, + data={'post':'yes'}, + headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], + 'referer': delete_url} + ) + if delete_page.status_code != 200: + print "Something went wrong. Delete page status: {0}".format( + delete_page.status_code) + return None + + print "Deleted measurement {0}".format(measurement_id) + return True + + def available_measurements(self): + """ Get a list of available measurement on the SCC. """ + measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements') + response = self.session.get(measurement_url, + auth = self.auth, + verify = False) + response_dict = response.json() + + measurements = None + if response_dict: + measurement_list = response_dict['objects'] + measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list] + print "Found %s measurements on the SCC." % len(measurements) + else: + print "No response received from the SCC when asked for available measurements." + + return measurements + + def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0): + """ Give the first available measurement id on the SCC for the specific + date. + """ + date_str = t1.strftime('%Y%m%d') + search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str) + + response = self.session.get(search_url, + auth = self.auth, + verify = False) + + response_dict = response.json() + + measurement_id = None + + if response_dict: + measurement_list = response_dict['objects'] + existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] + + measurement_number = base_number + measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) + + while measurement_id in existing_ids: + measurement_number = measurement_number + 1 + measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) + if measurement_number == 100: + raise ValueError('No available measurement id found.') + + return measurement_id + + +class ApiObject: + """ A generic class object. """ + + def __init__(self, dict_response): + + if dict_response: + # Add the dictionary key value pairs as object properties + for key, value in dict_response.items(): + setattr(self, key, value) + self.exists = True + else: + self.exists = False + + +class Measurement(ApiObject): + """ This class represents the measurement object as returned in the SCC API. + """ + @property + def is_running(self): + """ Returns True if the processing has not finished. + """ + if self.upload == 0: + return False + if self.pre_processing == -127: + return False + if self.pre_processing == 127: + if self.opt_retrievals in [127, -127]: + return False + return True + + def delete(self): + """ Delete the entry from the SCC database. """ + + + def __str__(self): + return "%s: %s, %s, %s" % (self.id, + self.upload, + self.pre_processing, + self.opt_retrievals) + + +def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + """ Shortcut function to upload a file to the SCC. """ + scc = SCC(auth) + scc.login(credential) + measurement_id = scc.upload_file(filename, system_id) + scc.logout() + return measurement_id + +def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + """ Shortcut function to process a file to the SCC. """ + scc = SCC(auth) + scc.login(credential) + measurement = scc.process(filename, system_id) + scc.logout() + return measurement + +def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): + """ Shortcut function to delete a measurement from the SCC. """ + scc = SCC(auth) + scc.login(credential) + scc.delete_measurement(measurement_id) + scc.logout() + +# When running through terminal +if __name__ == '__main__': + + # Define the command line arguments. + parser = argparse.ArgumentParser() + parser.add_argument("filename", nargs='?', help = "Measurement file name or path.", default='') + parser.add_argument("system", nargs='?', help = "Processing system id.", default=0) + parser.add_argument("-p", "--process", help="Wait for the results of the processing.", + action="store_true") + parser.add_argument("-d", "--delete", help="Measurement ID to delete.") + args = parser.parse_args() + + # If the arguments are OK, try to login on the site and upload. + if args.delete: + # If the delete is provided, do nothing else + delete_measurement(args.delete) + else: + if (args.filename == '') or (args.system == 0): + parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') + + if args.process: + process_file(args.filename, args.system) + else: + upload_file(args.filename, args.system)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/settings.sample.py Tue Jun 23 10:30:08 2015 +0300 @@ -0,0 +1,18 @@ +""" This file contains the user-specific settings for the scc_access script. + +You should rename the file settings.sample.py to setting.py. +""" + +# Add here the HTTP user name and password that is needed to access the SCC site. +BASIC_LOGIN = ('username', 'password') + +# Add here the user-name and password that is needed to log in to the SCC site. +# It is recommended to use credential for a user without management privileges +DJANGO_LOGIN = ('username', 'password') + +# Change this to the directory to download the files +OUTPUT_DIR = '/path/to/files/scc_output/' + +# SCC base URL. Normally you shouldn't need to change that. You should change +# this only if you want to use a different SCC installation e.g. a local version. +BASE_URL = 'https://scc.imaa.cnr.it/'
