scc_access.py

Tue, 23 Jun 2015 11:13:04 +0300

author
Iannis <ioannis@inoe.ro>
date
Tue, 23 Jun 2015 11:13:04 +0300
changeset 3
2f7cde6f836e
parent 1
783c8a0db76f
child 4
809c63be8a40
permissions
-rw-r--r--

Added changelog

ioannis@0 1 #!/usr/bin/env python
ioannis@0 2 """
ioannis@0 3 The MIT License (MIT)
ioannis@0 4
ioannis@0 5 Copyright (c) 2015, Ioannis Binietoglou
ioannis@0 6
ioannis@0 7 Permission is hereby granted, free of charge, to any person obtaining a copy
ioannis@0 8 of this software and associated documentation files (the "Software"), to deal
ioannis@0 9 in the Software without restriction, including without limitation the rights
ioannis@0 10 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
ioannis@0 11 copies of the Software, and to permit persons to whom the Software is
ioannis@0 12 furnished to do so, subject to the following conditions:
ioannis@0 13
ioannis@0 14 The above copyright notice and this permission notice shall be included in
ioannis@0 15 all copies or substantial portions of the Software.
ioannis@0 16
ioannis@0 17 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
ioannis@0 18 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
ioannis@0 19 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
ioannis@0 20 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
ioannis@0 21 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
ioannis@0 22 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
ioannis@0 23 THE SOFTWARE.
ioannis@0 24 """
ioannis@0 25
ioannis@1 26 __version__ = "0.5.0"
ioannis@1 27
ioannis@1 28
ioannis@0 29 # Try to read the settings from the settings.py file
ioannis@0 30 try:
ioannis@0 31 from settings import *
ioannis@0 32 except:
ioannis@0 33 raise ImportError(
ioannis@0 34 """A settings file (setting.py) is required to run the script.
ioannis@0 35 You can use settings.sample.py for a template.""")
ioannis@0 36
ioannis@0 37
ioannis@0 38 import requests
ioannis@0 39 import urlparse
ioannis@0 40 import argparse
ioannis@0 41 import os
ioannis@0 42 import re
ioannis@0 43 import time
ioannis@0 44 import StringIO
ioannis@0 45 from zipfile import ZipFile
ioannis@0 46 import datetime
ioannis@0 47
ioannis@0 48
ioannis@0 49 # Construct the absolute URLs
ioannis@0 50 LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/')
ioannis@0 51 UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/')
ioannis@0 52 DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/')
ioannis@0 53 DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/')
ioannis@0 54 DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/')
ioannis@0 55 DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
ioannis@0 56 API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
ioannis@0 57
ioannis@0 58 # The regex to find the measurement id from the measurement page
ioannis@0 59 # This should be read from the uploaded file, but would require an extra module
ioannis@0 60 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
ioannis@0 61
ioannis@0 62
ioannis@0 63 class SCC:
ioannis@0 64 """ A simple class that will attempt to upload a file on the SCC server.
ioannis@0 65 The uploading is done by simulation a normal browser session. In the current
ioannis@0 66 version no check is performed, and no feedback is given if the upload
ioannis@0 67 was successful. If everything is setup correctly, it will work.
ioannis@0 68 """
ioannis@0 69 def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR):
ioannis@0 70 self.auth = auth
ioannis@0 71 self.output_dir = OUTPUT_DIR
ioannis@0 72 self.session = requests.Session()
ioannis@0 73
ioannis@0 74 def login(self, credential = DJANGO_LOGIN):
ioannis@0 75 """ Login the the website. """
ioannis@0 76 self.login_credentials = {'username': credential[0],
ioannis@0 77 'password': credential[1]}
ioannis@0 78
ioannis@0 79 # Get upload form
ioannis@0 80 login_page = self.session.get(LOGIN_URL,
ioannis@0 81 auth = self.auth, verify = False)
ioannis@0 82
ioannis@0 83 # Submit the login data
ioannis@0 84 login_submit = self.session.post(LOGIN_URL,
ioannis@0 85 data = self.login_credentials,
ioannis@0 86 headers = {'X-CSRFToken': login_page.cookies['csrftoken'],
ioannis@0 87 'referer': LOGIN_URL},
ioannis@0 88 verify = False,
ioannis@0 89 auth = self.auth)
ioannis@0 90 return login_submit
ioannis@0 91
ioannis@0 92 def logout(self):
ioannis@0 93 pass
ioannis@0 94
ioannis@0 95 def upload_file(self, filename, system_id):
ioannis@0 96 """ Upload a filename for processing with a specific system. If the
ioannis@0 97 upload is successful, it returns the measurement id. """
ioannis@0 98 # Get submit page
ioannis@0 99 upload_page = self.session.get(UPLOAD_URL,
ioannis@0 100 auth = self.auth,
ioannis@0 101 verify = False)
ioannis@0 102
ioannis@0 103 # Submit the data
ioannis@0 104 upload_data = {'system': system_id}
ioannis@0 105 files = {'data': open(filename, 'rb')}
ioannis@0 106
ioannis@0 107 print "Uploading of file %s started." % filename
ioannis@0 108
ioannis@0 109 upload_submit = self.session.post(UPLOAD_URL,
ioannis@0 110 data = upload_data,
ioannis@0 111 files = files,
ioannis@0 112 headers = {'X-CSRFToken': upload_page.cookies['csrftoken'],
ioannis@0 113 'referer': UPLOAD_URL},
ioannis@0 114 verify = False,
ioannis@0 115 auth = self.auth)
ioannis@0 116
ioannis@0 117 if upload_submit.status_code != 200:
ioannis@0 118 print "Connection error. Status code: %s" % upload_submit.status_code
ioannis@0 119 return False
ioannis@0 120
ioannis@0 121 measurement_id = True
ioannis@0 122
ioannis@0 123 # Check if there was a redirect to a new page.
ioannis@0 124 if upload_submit.url == UPLOAD_URL:
ioannis@0 125 measurement_id = False
ioannis@0 126 print "Uploaded file rejected! Try to upload manually to see the error."
ioannis@0 127 else:
ioannis@0 128 measurement_id = re.findall(regex, upload_submit.text)[0]
ioannis@0 129 print "Successfully uploaded measurement with id %s." % measurement_id
ioannis@0 130
ioannis@0 131 return measurement_id
ioannis@0 132
ioannis@0 133 def download_files(self, measurement_id, subdir, download_url):
ioannis@0 134 """ Downloads some files from the download_url to the specified
ioannis@0 135 subdir. This method is used to download preprocessed file, optical
ioannis@0 136 files etc.
ioannis@0 137 """
ioannis@0 138 # Get the file
ioannis@0 139 request = self.session.get(download_url, auth = self.auth,
ioannis@0 140 verify = False,
ioannis@0 141 stream=True)
ioannis@0 142
ioannis@0 143 # Create the dir if it does not exist
ioannis@0 144 local_dir = os.path.join(self.output_dir, measurement_id, subdir)
ioannis@0 145 if not os.path.exists(local_dir):
ioannis@0 146 os.makedirs(local_dir)
ioannis@0 147
ioannis@0 148
ioannis@0 149 # Save the file by chunk, needed if the file is big.
ioannis@0 150 memory_file = StringIO.StringIO()
ioannis@0 151
ioannis@0 152 for chunk in request.iter_content(chunk_size=1024):
ioannis@0 153 if chunk: # filter out keep-alive new chunks
ioannis@0 154 memory_file.write(chunk)
ioannis@0 155 memory_file.flush()
ioannis@0 156
ioannis@0 157 zip_file = ZipFile(memory_file)
ioannis@0 158
ioannis@0 159 for ziped_name in zip_file.namelist():
ioannis@0 160 basename = os.path.basename(ziped_name)
ioannis@0 161
ioannis@0 162 local_file = os.path.join(local_dir, basename)
ioannis@0 163
ioannis@0 164 with open(local_file, 'wb') as f:
ioannis@0 165 f.write(zip_file.read(ziped_name))
ioannis@0 166
ioannis@0 167 def download_preprocessed(self, measurement_id):
ioannis@0 168 """ Download preprocessed files for the measurement id. """
ioannis@0 169 # Construct the download url
ioannis@0 170 download_url = DOWNLOAD_PREPROCESSED.format(measurement_id)
ioannis@0 171 self.download_files(measurement_id, 'scc_preprocessed', download_url)
ioannis@0 172
ioannis@0 173 def download_optical(self, measurement_id):
ioannis@0 174 """ Download optical files for the measurement id. """
ioannis@0 175 # Construct the download url
ioannis@0 176 download_url = DOWNLOAD_OPTICAL.format(measurement_id)
ioannis@0 177 self.download_files(measurement_id, 'scc_optical', download_url)
ioannis@0 178
ioannis@0 179 def download_graphs(self, measurement_id):
ioannis@0 180 """ Download profile graphs for the measurement id. """
ioannis@0 181 # Construct the download url
ioannis@0 182 download_url = DOWNLOAD_GRAPH.format(measurement_id)
ioannis@0 183 self.download_files(measurement_id, 'scc_plots', download_url)
ioannis@0 184
ioannis@0 185 def process(self, filename, system_id):
ioannis@0 186 """ Upload a file for processing and wait for the processing to finish.
ioannis@0 187 If the processing is successful, it will download all produced files.
ioannis@0 188 """
ioannis@0 189 print "--- Processing started on %s. ---" % datetime.datetime.now()
ioannis@0 190 # Upload file
ioannis@0 191 measurement_id = self.upload_file(filename, system_id)
ioannis@0 192
ioannis@0 193 measurement = None
ioannis@0 194 if measurement_id:
ioannis@0 195 measurement = self.get_measurement(measurement_id)
ioannis@0 196 while measurement.is_running:
ioannis@0 197 print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
ioannis@0 198 measurement.pre_processing,
ioannis@0 199 measurement.opt_retrievals)
ioannis@0 200 time.sleep(10)
ioannis@0 201 measurement = self.get_measurement(measurement_id)
ioannis@0 202 print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
ioannis@0 203 measurement.pre_processing,
ioannis@0 204 measurement.opt_retrievals)
ioannis@0 205 if measurement.pre_processing == 127:
ioannis@0 206 print "Downloading preprocessed files."
ioannis@0 207 self.download_preprocessed(measurement_id)
ioannis@0 208 if measurement.opt_retrievals == 127:
ioannis@0 209 print "Downloading optical files."
ioannis@0 210 self.download_optical(measurement_id)
ioannis@0 211 print "Downloading graphs."
ioannis@0 212 self.download_graphs(measurement_id)
ioannis@0 213 print "--- Processing finished. ---"
ioannis@0 214 return measurement
ioannis@0 215
ioannis@0 216 def get_status(self, measurement_id):
ioannis@0 217 """ Get the processing status for a measurement id through the API. """
ioannis@0 218 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id)
ioannis@0 219
ioannis@0 220 response = self.session.get(measurement_url,
ioannis@0 221 auth = self.auth,
ioannis@0 222 verify = False)
ioannis@0 223
ioannis@0 224 response_dict = response.json()
ioannis@0 225
ioannis@0 226 if response_dict['objects']:
ioannis@0 227 measurement_list = response_dict['objects']
ioannis@0 228 measurement = Measurement(measurement_list[0])
ioannis@0 229 return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals)
ioannis@0 230 else:
ioannis@0 231 print "No measurement with id %s found on the SCC." % measurement_id
ioannis@0 232 return None
ioannis@0 233
ioannis@0 234 def get_measurement(self, measurement_id):
ioannis@0 235 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id)
ioannis@0 236
ioannis@0 237 response = self.session.get(measurement_url,
ioannis@0 238 auth = self.auth,
ioannis@0 239 verify = False)
ioannis@0 240
ioannis@0 241 response_dict = response.json()
ioannis@0 242
ioannis@0 243 if response_dict:
ioannis@0 244 measurement = Measurement(response_dict)
ioannis@0 245 return measurement
ioannis@0 246 else:
ioannis@0 247 print "No measurement with id %s found on the SCC." % measurement_id
ioannis@0 248 return None
ioannis@0 249
ioannis@0 250 def delete_measurement(self, measurement_id):
ioannis@0 251 """ Deletes a measurement with the provided measurement id. The user
ioannis@0 252 should have the appropriate permissions.
ioannis@0 253
ioannis@0 254 The procedures is performed directly through the web interface and
ioannis@0 255 NOT through the API.
ioannis@0 256 """
ioannis@0 257 # Get the measurement object
ioannis@0 258 measurement = self.get_measurement(measurement_id)
ioannis@0 259
ioannis@0 260 # Check that it exists
ioannis@0 261 if measurement is None:
ioannis@0 262 print "Nothing to delete."
ioannis@0 263 return None
ioannis@0 264
ioannis@0 265 # Go the the page confirming the deletion
ioannis@0 266 delete_url = DELETE_MEASUREMENT.format(measurement.id)
ioannis@0 267
ioannis@0 268 confirm_page = self.session.get(delete_url,
ioannis@0 269 auth = self.auth,
ioannis@0 270 verify = False)
ioannis@0 271
ioannis@0 272 # Check that the page opened properly
ioannis@0 273 if confirm_page.status_code != 200:
ioannis@0 274 print "Could not open delete page. Status: {0}".format(confirm_page.status_code)
ioannis@0 275 return None
ioannis@0 276
ioannis@0 277 # Delete the measurement
ioannis@0 278 delete_page = self.session.post(delete_url,
ioannis@0 279 auth=self.auth,
ioannis@0 280 verify=False,
ioannis@0 281 data={'post':'yes'},
ioannis@0 282 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
ioannis@0 283 'referer': delete_url}
ioannis@0 284 )
ioannis@0 285 if delete_page.status_code != 200:
ioannis@0 286 print "Something went wrong. Delete page status: {0}".format(
ioannis@0 287 delete_page.status_code)
ioannis@0 288 return None
ioannis@0 289
ioannis@0 290 print "Deleted measurement {0}".format(measurement_id)
ioannis@0 291 return True
ioannis@0 292
ioannis@0 293 def available_measurements(self):
ioannis@0 294 """ Get a list of available measurement on the SCC. """
ioannis@0 295 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements')
ioannis@0 296 response = self.session.get(measurement_url,
ioannis@0 297 auth = self.auth,
ioannis@0 298 verify = False)
ioannis@0 299 response_dict = response.json()
ioannis@0 300
ioannis@0 301 measurements = None
ioannis@0 302 if response_dict:
ioannis@0 303 measurement_list = response_dict['objects']
ioannis@0 304 measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list]
ioannis@0 305 print "Found %s measurements on the SCC." % len(measurements)
ioannis@0 306 else:
ioannis@0 307 print "No response received from the SCC when asked for available measurements."
ioannis@0 308
ioannis@0 309 return measurements
ioannis@0 310
ioannis@0 311 def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0):
ioannis@0 312 """ Give the first available measurement id on the SCC for the specific
ioannis@0 313 date.
ioannis@0 314 """
ioannis@0 315 date_str = t1.strftime('%Y%m%d')
ioannis@0 316 search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str)
ioannis@0 317
ioannis@0 318 response = self.session.get(search_url,
ioannis@0 319 auth = self.auth,
ioannis@0 320 verify = False)
ioannis@0 321
ioannis@0 322 response_dict = response.json()
ioannis@0 323
ioannis@0 324 measurement_id = None
ioannis@0 325
ioannis@0 326 if response_dict:
ioannis@0 327 measurement_list = response_dict['objects']
ioannis@0 328 existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
ioannis@0 329
ioannis@0 330 measurement_number = base_number
ioannis@0 331 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
ioannis@0 332
ioannis@0 333 while measurement_id in existing_ids:
ioannis@0 334 measurement_number = measurement_number + 1
ioannis@0 335 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
ioannis@0 336 if measurement_number == 100:
ioannis@0 337 raise ValueError('No available measurement id found.')
ioannis@0 338
ioannis@0 339 return measurement_id
ioannis@0 340
ioannis@0 341
ioannis@0 342 class ApiObject:
ioannis@0 343 """ A generic class object. """
ioannis@0 344
ioannis@0 345 def __init__(self, dict_response):
ioannis@0 346
ioannis@0 347 if dict_response:
ioannis@0 348 # Add the dictionary key value pairs as object properties
ioannis@0 349 for key, value in dict_response.items():
ioannis@0 350 setattr(self, key, value)
ioannis@0 351 self.exists = True
ioannis@0 352 else:
ioannis@0 353 self.exists = False
ioannis@0 354
ioannis@0 355
ioannis@0 356 class Measurement(ApiObject):
ioannis@0 357 """ This class represents the measurement object as returned in the SCC API.
ioannis@0 358 """
ioannis@0 359 @property
ioannis@0 360 def is_running(self):
ioannis@0 361 """ Returns True if the processing has not finished.
ioannis@0 362 """
ioannis@0 363 if self.upload == 0:
ioannis@0 364 return False
ioannis@0 365 if self.pre_processing == -127:
ioannis@0 366 return False
ioannis@0 367 if self.pre_processing == 127:
ioannis@0 368 if self.opt_retrievals in [127, -127]:
ioannis@0 369 return False
ioannis@0 370 return True
ioannis@0 371
ioannis@0 372 def delete(self):
ioannis@0 373 """ Delete the entry from the SCC database. """
ioannis@0 374
ioannis@0 375
ioannis@0 376 def __str__(self):
ioannis@0 377 return "%s: %s, %s, %s" % (self.id,
ioannis@0 378 self.upload,
ioannis@0 379 self.pre_processing,
ioannis@0 380 self.opt_retrievals)
ioannis@0 381
ioannis@0 382
ioannis@0 383 def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
ioannis@0 384 """ Shortcut function to upload a file to the SCC. """
ioannis@0 385 scc = SCC(auth)
ioannis@0 386 scc.login(credential)
ioannis@0 387 measurement_id = scc.upload_file(filename, system_id)
ioannis@0 388 scc.logout()
ioannis@0 389 return measurement_id
ioannis@0 390
ioannis@0 391 def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
ioannis@0 392 """ Shortcut function to process a file to the SCC. """
ioannis@0 393 scc = SCC(auth)
ioannis@0 394 scc.login(credential)
ioannis@0 395 measurement = scc.process(filename, system_id)
ioannis@0 396 scc.logout()
ioannis@0 397 return measurement
ioannis@0 398
ioannis@0 399 def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
ioannis@0 400 """ Shortcut function to delete a measurement from the SCC. """
ioannis@0 401 scc = SCC(auth)
ioannis@0 402 scc.login(credential)
ioannis@0 403 scc.delete_measurement(measurement_id)
ioannis@0 404 scc.logout()
ioannis@0 405
ioannis@0 406 # When running through terminal
ioannis@0 407 if __name__ == '__main__':
ioannis@0 408
ioannis@0 409 # Define the command line arguments.
ioannis@0 410 parser = argparse.ArgumentParser()
ioannis@0 411 parser.add_argument("filename", nargs='?', help = "Measurement file name or path.", default='')
ioannis@0 412 parser.add_argument("system", nargs='?', help = "Processing system id.", default=0)
ioannis@0 413 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
ioannis@0 414 action="store_true")
ioannis@0 415 parser.add_argument("-d", "--delete", help="Measurement ID to delete.")
ioannis@0 416 args = parser.parse_args()
ioannis@0 417
ioannis@0 418 # If the arguments are OK, try to login on the site and upload.
ioannis@0 419 if args.delete:
ioannis@0 420 # If the delete is provided, do nothing else
ioannis@0 421 delete_measurement(args.delete)
ioannis@0 422 else:
ioannis@0 423 if (args.filename == '') or (args.system == 0):
ioannis@0 424 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
ioannis@0 425
ioannis@0 426 if args.process:
ioannis@0 427 process_file(args.filename, args.system)
ioannis@0 428 else:
ioannis@0 429 upload_file(args.filename, args.system)

mercurial