Mercurial > public > scc_access / file comparison

comparison: scc_access/scc_access.py

scc_access/scc_access.py

changeset 14
c2020b2fdd05
parent 7
415d034b0864
child 15
93b6b945d939
child 29
3e3e5bda6b77
equal deleted inserted replaced
13:493067e1bfbb 14:c2020b2fdd05
1 #!/usr/bin/env python
2 """
3 The MIT License (MIT)
4
5 Copyright (c) 2015, Ioannis Binietoglou
6
7 Permission is hereby granted, free of charge, to any person obtaining a copy
8 of this software and associated documentation files (the "Software"), to deal
9 in the Software without restriction, including without limitation the rights
10 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 copies of the Software, and to permit persons to whom the Software is
12 furnished to do so, subject to the following conditions:
13
14 The above copyright notice and this permission notice shall be included in
15 all copies or substantial portions of the Software.
16
17 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 THE SOFTWARE.
24 """
25
26 __version__ = "0.6.0"
27
28 # Try to read the settings from the settings.py file
29 try:
30 from settings import *
31 except:
32 raise ImportError(
33 """A settings file (setting.py) is required to run the script.
34 You can use settings.sample.py as a template.""")
35
36 import requests 1 import requests
37 requests.packages.urllib3.disable_warnings() 2 requests.packages.urllib3.disable_warnings()
38 3
4 import sys
39 import urlparse 5 import urlparse
40 import argparse 6 import argparse
41 import os 7 import os
42 import re 8 import re
43 import time 9 import time
44 import StringIO 10 import StringIO
45 from zipfile import ZipFile 11 from zipfile import ZipFile
46 import datetime 12 import datetime
47 import logging 13 import logging
14 import yaml
15
48 16
49 logger = logging.getLogger(__name__) 17 logger = logging.getLogger(__name__)
50
51
52 # Construct the absolute URLs
53 LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/')
54 UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/')
55 DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/')
56 DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/')
57 DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/')
58 RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/')
59 RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/')
60
61 DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
62 API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
63 18
64 # The regex to find the measurement id from the measurement page 19 # The regex to find the measurement id from the measurement page
65 # This should be read from the uploaded file, but would require an extra NetCDF module. 20 # This should be read from the uploaded file, but would require an extra NetCDF module.
66 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>" 21 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
67 22
68 23
69 class SCC: 24 class SCC:
70 """ A simple class that will attempt to upload a file on the SCC server. 25 """ A simple class that will attempt to upload a file on the SCC server.
26
71 The uploading is done by simulating a normal browser session. In the current 27 The uploading is done by simulating a normal browser session. In the current
72 version no check is performed, and no feedback is given if the upload 28 version no check is performed, and no feedback is given if the upload
73 was successful. If everything is setup correctly, it will work. 29 was successful. If everything is setup correctly, it will work.
74 """ 30 """
75 31
76 def __init__(self, auth=BASIC_LOGIN, output_dir=OUTPUT_DIR): 32 def __init__(self, auth, output_dir, base_url):
77 self.auth = auth 33 self.auth = auth
78 self.output_dir = output_dir 34 self.output_dir = output_dir
35 self.base_url = base_url
79 self.session = requests.Session() 36 self.session = requests.Session()
80 37 self.construct_urls()
81 def login(self, credentials=DJANGO_LOGIN): 38
39 def construct_urls(self):
40 """ Construct all URLs needed for processing. """
41 # Construct the absolute URLs
42 self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
43 self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
44 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/')
45 self.download_optical_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-optical/')
46 self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/')
47 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
48 self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
49
50 def login(self, credentials):
82 """ Login the the website. """ 51 """ Login the the website. """
83 logger.debug("Attempting to login to SCC, username %s." % credentials[0]) 52 logger.debug("Attempting to login to SCC, username %s." % credentials[0])
84 self.login_credentials = {'username': credentials[0], 53 self.login_credentials = {'username': credentials[0],
85 'password': credentials[1]} 54 'password': credentials[1]}
86 55
87 logger.debug("Accessing login page at %s." % LOGIN_URL) 56 logger.debug("Accessing login page at %s." % self.login_url)
88 57
89 # Get upload form 58 # Get upload form
90 login_page = self.session.get(LOGIN_URL, 59 login_page = self.session.get(self.login_url, auth=self.auth, verify=False)
91 auth=self.auth, verify=False) 60
61 if login_page.status_code != 200:
62 logger.error('Could not access login pages. Status code %s' % login_page.status_code)
63 sys.exit(1)
92 64
93 logger.debug("Submiting credentials.") 65 logger.debug("Submiting credentials.")
66
94 # Submit the login data 67 # Submit the login data
95 login_submit = self.session.post(LOGIN_URL, 68 login_submit = self.session.post(self.login_url,
96 data=self.login_credentials, 69 data=self.login_credentials,
97 headers={'X-CSRFToken': login_page.cookies['csrftoken'], 70 headers={'X-CSRFToken': login_page.cookies['csrftoken'],
98 'referer': LOGIN_URL}, 71 'referer': self.login_url},
99 verify=False, 72 verify=False,
100 auth=self.auth) 73 auth=self.auth)
101 return login_submit 74 return login_submit
102 75
103 def logout(self): 76 def logout(self):
105 78
106 def upload_file(self, filename, system_id): 79 def upload_file(self, filename, system_id):
107 """ Upload a filename for processing with a specific system. If the 80 """ Upload a filename for processing with a specific system. If the
108 upload is successful, it returns the measurement id. """ 81 upload is successful, it returns the measurement id. """
109 # Get submit page 82 # Get submit page
110 upload_page = self.session.get(UPLOAD_URL, 83 upload_page = self.session.get(self.upload_url,
111 auth=self.auth, 84 auth=self.auth,
112 verify=False) 85 verify=False)
113 86
114 # Submit the data 87 # Submit the data
115 upload_data = {'system': system_id} 88 upload_data = {'system': system_id}
116 files = {'data': open(filename, 'rb')} 89 files = {'data': open(filename, 'rb')}
117 90
118 logging.info("Uploading of file %s started." % filename) 91 logger.info("Uploading of file %s started." % filename)
119 92
120 upload_submit = self.session.post(UPLOAD_URL, 93 upload_submit = self.session.post(self.upload_url,
121 data=upload_data, 94 data=upload_data,
122 files=files, 95 files=files,
123 headers={'X-CSRFToken': upload_page.cookies['csrftoken'], 96 headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
124 'referer': UPLOAD_URL}, 97 'referer': self.upload_url,},
125 verify=False, 98 verify=False,
126 auth=self.auth) 99 auth=self.auth)
127 100
128 if upload_submit.status_code != 200: 101 if upload_submit.status_code != 200:
129 logging.warning("Connection error. Status code: %s" % upload_submit.status_code) 102 logger.warning("Connection error. Status code: %s" % upload_submit.status_code)
130 return False 103 return False
131 104
132 # Check if there was a redirect to a new page. 105 # Check if there was a redirect to a new page.
133 if upload_submit.url == UPLOAD_URL: 106 if upload_submit.url == self.upload_url:
134 measurement_id = False 107 measurement_id = False
135 logging.error("Uploaded file rejected! Try to upload manually to see the error.") 108 logger.error("Uploaded file rejected! Try to upload manually to see the error.")
136 else: 109 else:
137 measurement_id = re.findall(regex, upload_submit.text)[0] 110 measurement_id = re.findall(regex, upload_submit.text)[0]
138 logging.error("Successfully uploaded measurement with id %s." % measurement_id) 111 logger.error("Successfully uploaded measurement with id %s." % measurement_id)
139 112
140 return measurement_id 113 return measurement_id
141 114
142 def download_files(self, measurement_id, subdir, download_url): 115 def download_files(self, measurement_id, subdir, download_url):
143 """ Downloads some files from the download_url to the specified 116 """ Downloads some files from the download_url to the specified
173 f.write(zip_file.read(ziped_name)) 146 f.write(zip_file.read(ziped_name))
174 147
175 def download_preprocessed(self, measurement_id): 148 def download_preprocessed(self, measurement_id):
176 """ Download preprocessed files for the measurement id. """ 149 """ Download preprocessed files for the measurement id. """
177 # Construct the download url 150 # Construct the download url
178 download_url = DOWNLOAD_PREPROCESSED.format(measurement_id) 151 download_url = self.download_preprocessed_pattern.format(measurement_id)
179 self.download_files(measurement_id, 'scc_preprocessed', download_url) 152 self.download_files(measurement_id, 'scc_preprocessed', download_url)
180 153
181 def download_optical(self, measurement_id): 154 def download_optical(self, measurement_id):
182 """ Download optical files for the measurement id. """ 155 """ Download optical files for the measurement id. """
183 # Construct the download url 156 # Construct the download url
184 download_url = DOWNLOAD_OPTICAL.format(measurement_id) 157 download_url = self.download_optical.format(measurement_id)
185 self.download_files(measurement_id, 'scc_optical', download_url) 158 self.download_files(measurement_id, 'scc_optical', download_url)
186 159
187 def download_graphs(self, measurement_id): 160 def download_graphs(self, measurement_id):
188 """ Download profile graphs for the measurement id. """ 161 """ Download profile graphs for the measurement id. """
189 # Construct the download url 162 # Construct the download url
190 download_url = DOWNLOAD_GRAPH.format(measurement_id) 163 download_url = self.download_graph_pattern.format(measurement_id)
191 self.download_files(measurement_id, 'scc_plots', download_url) 164 self.download_files(measurement_id, 'scc_plots', download_url)
192 165
193 def rerun_processing(self, measurement_id, monitor=True): 166 def rerun_processing(self, measurement_id, monitor=True):
194 measurement = self.get_measurement(measurement_id) 167 measurement = self.get_measurement(measurement_id)
195 168
197 request = self.session.get(measurement.rerun_processing_url, auth=self.auth, 170 request = self.session.get(measurement.rerun_processing_url, auth=self.auth,
198 verify=False, 171 verify=False,
199 stream=True) 172 stream=True)
200 173
201 if request.status_code != 200: 174 if request.status_code != 200:
202 logging.error("Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) 175 logger.error(
176 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
203 return 177 return
204 178
205 if monitor: 179 if monitor:
206 self.monitor_processing(measurement_id) 180 self.monitor_processing(measurement_id)
207 181
242 216
243 measurement = self.get_measurement(measurement_id) 217 measurement = self.get_measurement(measurement_id)
244 if measurement is not None: 218 if measurement is not None:
245 while measurement.is_running: 219 while measurement.is_running:
246 logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, 220 logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
247 measurement.pre_processing, 221 measurement.pre_processing,
248 measurement.processing)) 222 measurement.processing))
249 time.sleep(10) 223 time.sleep(10)
250 measurement = self.get_measurement(measurement_id) 224 measurement = self.get_measurement(measurement_id)
251 logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, 225 logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
252 measurement.pre_processing, 226 measurement.pre_processing,
253 measurement.processing)) 227 measurement.processing))
254 if measurement.pre_processing == 127: 228 if measurement.pre_processing == 127:
255 logger.info("Downloading preprocessed files.") 229 logger.info("Downloading preprocessed files.")
256 self.download_preprocessed(measurement_id) 230 self.download_preprocessed(measurement_id)
257 if measurement.processing == 127: 231 if measurement.processing == 127:
258 logger.info("Downloading optical files.") 232 logger.info("Downloading optical files.")
262 logger.info("--- Processing finished. ---") 236 logger.info("--- Processing finished. ---")
263 return measurement 237 return measurement
264 238
265 def get_status(self, measurement_id): 239 def get_status(self, measurement_id):
266 """ Get the processing status for a measurement id through the API. """ 240 """ Get the processing status for a measurement id through the API. """
267 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id) 241 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id)
268 242
269 response = self.session.get(measurement_url, 243 response = self.session.get(measurement_url,
270 auth=self.auth, 244 auth=self.auth,
271 verify=False) 245 verify=False)
272 246
273 response_dict = response.json() 247 response_dict = response.json()
274 248
275 if response_dict['objects']: 249 if response_dict['objects']:
276 measurement_list = response_dict['objects'] 250 measurement_list = response_dict['objects']
277 measurement = Measurement(measurement_list[0]) 251 measurement = Measurement(self.base_url, measurement_list[0])
278 return (measurement.upload, measurement.pre_processing, measurement.processing) 252 return measurement.upload, measurement.pre_processing, measurement.processing
279 else: 253 else:
280 logger.error("No measurement with id %s found on the SCC." % measurement_id) 254 logger.error("No measurement with id %s found on the SCC." % measurement_id)
281 return None 255 return None
282 256
283 def get_measurement(self, measurement_id): 257 def get_measurement(self, measurement_id):
284 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id) 258 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id)
285 259
286 response = self.session.get(measurement_url, 260 response = self.session.get(measurement_url,
287 auth=self.auth, 261 auth=self.auth,
288 verify=False) 262 verify=False)
289 263
264 if response.status_code != 200:
265 logger.error('Could not access API. Status code %s.' % response.status_code)
266 sys.exit(1)
267
290 response_dict = response.json() 268 response_dict = response.json()
291 269
292 if response_dict: 270 if response_dict:
293 measurement = Measurement(response_dict) 271 measurement = Measurement(self.base_url,response_dict)
294 return measurement 272 return measurement
295 else: 273 else:
296 logger.error("No measurement with id %s found on the SCC." % measurement_id) 274 logger.error("No measurement with id %s found on the SCC." % measurement_id)
297 return None 275 return None
298 276
310 if measurement is None: 288 if measurement is None:
311 logger.warning("Nothing to delete.") 289 logger.warning("Nothing to delete.")
312 return None 290 return None
313 291
314 # Go the the page confirming the deletion 292 # Go the the page confirming the deletion
315 delete_url = DELETE_MEASUREMENT.format(measurement.id) 293 delete_url = self.delete_measurement_pattern.format(measurement.id)
316 294
317 confirm_page = self.session.get(delete_url, 295 confirm_page = self.session.get(delete_url,
318 auth=self.auth, 296 auth=self.auth,
319 verify=False) 297 verify=False)
320 298
339 logger.info("Deleted measurement {0}".format(measurement_id)) 317 logger.info("Deleted measurement {0}".format(measurement_id))
340 return True 318 return True
341 319
342 def available_measurements(self): 320 def available_measurements(self):
343 """ Get a list of available measurement on the SCC. """ 321 """ Get a list of available measurement on the SCC. """
344 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements') 322 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements')
345 response = self.session.get(measurement_url, 323 response = self.session.get(measurement_url,
346 auth=self.auth, 324 auth=self.auth,
347 verify=False) 325 verify=False)
348 response_dict = response.json() 326 response_dict = response.json()
349 327
350 measurements = None 328 measurements = None
351 if response_dict: 329 if response_dict:
352 measurement_list = response_dict['objects'] 330 measurement_list = response_dict['objects']
353 measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list] 331 measurements = [Measurement(self.base_url, measurement_dict) for measurement_dict in measurement_list]
354 logger.info("Found %s measurements on the SCC." % len(measurements)) 332 logger.info("Found %s measurements on the SCC." % len(measurements))
355 else: 333 else:
356 logger.warning("No response received from the SCC when asked for available measurements.") 334 logger.warning("No response received from the SCC when asked for available measurements.")
357 335
358 return measurements 336 return measurements
360 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): 338 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
361 """ Give the first available measurement id on the SCC for the specific 339 """ Give the first available measurement id on the SCC for the specific
362 date. 340 date.
363 """ 341 """
364 date_str = t1.strftime('%Y%m%d') 342 date_str = t1.strftime('%Y%m%d')
365 search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str) 343 search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
366 344
367 response = self.session.get(search_url, 345 response = self.session.get(search_url,
368 auth=self.auth, 346 auth=self.auth,
369 verify=False) 347 verify=False)
370 348
389 367
390 368
391 class ApiObject: 369 class ApiObject:
392 """ A generic class object. """ 370 """ A generic class object. """
393 371
394 def __init__(self, dict_response): 372 def __init__(self, base_url, dict_response):
373 self.base_url = base_url
395 374
396 if dict_response: 375 if dict_response:
397 # Add the dictionary key value pairs as object properties 376 # Add the dictionary key value pairs as object properties
398 for key, value in dict_response.items(): 377 for key, value in dict_response.items():
399 setattr(self, key, value) 378 setattr(self, key, value)
419 return False 398 return False
420 return True 399 return True
421 400
422 @property 401 @property
423 def rerun_processing_url(self): 402 def rerun_processing_url(self):
424 return RERUN_PROCESSING.format(self.id) 403 url_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-optical/')
404 return url_pattern.format(self.id)
425 405
426 @property 406 @property
427 def rerun_all_url(self): 407 def rerun_all_url(self):
428 return RERUN_ALL.format(self.id) 408 ulr_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/rerun-all/')
409 return ulr_pattern.format(self.id)
429 410
430 def __str__(self): 411 def __str__(self):
431 return "%s: %s, %s, %s" % (self.id, 412 return "%s: %s, %s, %s" % (self.id,
432 self.upload, 413 self.upload,
433 self.pre_processing, 414 self.pre_processing,
434 self.processing) 415 self.processing)
435 416
436 417
437 def upload_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): 418 def upload_file(filename, system_id, settings):
438 """ Shortcut function to upload a file to the SCC. """ 419 """ Shortcut function to upload a file to the SCC. """
439 logger.info("Uploading file %s, using sytem %s" % (filename, system_id)) 420 logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
440 421
441 scc = SCC(auth) 422 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
442 scc.login(credential) 423 scc.login(settings['website_credentials'])
443 measurement_id = scc.upload_file(filename, system_id) 424 measurement_id = scc.upload_file(filename, system_id)
444 scc.logout() 425 scc.logout()
445 return measurement_id 426 return measurement_id
446 427
447 428
448 def process_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): 429 def process_file(filename, system_id, settings):
449 """ Shortcut function to process a file to the SCC. """ 430 """ Shortcut function to process a file to the SCC. """
450 logger.info("Processing file %s, using sytem %s" % (filename, system_id)) 431 logger.info("Processing file %s, using sytem %s" % (filename, system_id))
451 432
452 scc = SCC(auth) 433 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
453 scc.login(credential) 434 scc.login(settings['website_credentials'])
454 measurement = scc.process(filename, system_id) 435 measurement = scc.process(filename, system_id)
455 scc.logout() 436 scc.logout()
456 return measurement 437 return measurement
457 438
458 439
459 def delete_measurement(measurement_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): 440 def delete_measurement(measurement_id, settings):
460 """ Shortcut function to delete a measurement from the SCC. """ 441 """ Shortcut function to delete a measurement from the SCC. """
461 logger.info("Deleting %s" % measurement_id) 442 logger.info("Deleting %s" % measurement_id)
462 scc = SCC(auth) 443
463 scc.login(credential) 444 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
445 scc.login(settings['website_credentials'])
464 scc.delete_measurement(measurement_id) 446 scc.delete_measurement(measurement_id)
465 scc.logout() 447 scc.logout()
466 448
467 449
468 def rerun_all(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): 450 def rerun_all(measurement_id, monitor, settings):
469 """ Shortcut function to delete a measurement from the SCC. """ 451 """ Shortcut function to delete a measurement from the SCC. """
470 logger.info("Rerunning all products for %s" % measurement_id) 452 logger.info("Rerunning all products for %s" % measurement_id)
471 scc = SCC(auth) 453
472 scc.login(credential) 454 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
455 scc.login(settings['website_credentials'])
473 scc.rerun_all(measurement_id, monitor) 456 scc.rerun_all(measurement_id, monitor)
474 scc.logout() 457 scc.logout()
475 458
476 459
477 def rerun_processing(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN): 460 def rerun_processing(measurement_id, monitor, settings):
478 """ Shortcut function to delete a measurement from the SCC. """ 461 """ Shortcut function to delete a measurement from the SCC. """
479 logger.info("Rerunning (optical) processing for %s" % measurement_id) 462 logger.info("Rerunning (optical) processing for %s" % measurement_id)
480 scc = SCC(auth) 463
481 scc.login(credential) 464 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
465 scc.login(settings['website_credentials'])
482 scc.rerun_processing(measurement_id, monitor) 466 scc.rerun_processing(measurement_id, monitor)
483 scc.logout() 467 scc.logout()
484 468
469
470 def import_settings(config_file_path):
471 """ Read the configuration file.
472
473 The file should be in YAML syntax."""
474
475 if not os.path.isfile(config_file_path):
476 logger.error("Wrong path for configuration file (%s)" % config_file_path)
477 sys.exit(1)
478
479 with open(config_file_path) as yaml_file:
480 try:
481 settings = yaml.safe_load(yaml_file)
482 logger.debug("Read settings file(%s)" % config_file_path)
483 except:
484 logger.error("Could not parse YAML file (%s)" % config_file_path)
485 sys.exit(1)
486
487 # YAML limitation: does not read tuples
488 settings['basic_credentials'] = tuple(settings['basic_credentials'])
489 settings['website_credentials'] = tuple(settings['website_credentials'])
490 return settings
491
492
485 def main(): 493 def main():
486 # Define the command line arguments. 494 # Define the command line arguments.
487 parser = argparse.ArgumentParser() 495 parser = argparse.ArgumentParser()
488 parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='') 496 parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='')
489 parser.add_argument("system", nargs='?', help="Processing system id.", default=0) 497 parser.add_argument("system", nargs='?', help="Processing system id.", default=0)
498 parser.add_argument("-c", "--config", nargs='?', help="Path to configuration file")
490 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", 499 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
491 action="store_true") 500 action="store_true")
492 parser.add_argument("--delete", help="Measurement ID to delete.") 501 parser.add_argument("--delete", help="Measurement ID to delete.")
493 parser.add_argument("--rerun-all", help="Measurement ID to rerun.") 502 parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
494 parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.") 503 parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.")
495 504
496 # Verbosity settings from http://stackoverflow.com/a/20663028 505 # Verbosity settings from http://stackoverflow.com/a/20663028
497 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", 506 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
498 dest="loglevel", const=logging.DEBUG, default=logging.INFO, 507 dest="loglevel", const=logging.DEBUG, default=logging.INFO,
499 ) 508 )
504 args = parser.parse_args() 513 args = parser.parse_args()
505 514
506 # Get the logger with the appropriate level 515 # Get the logger with the appropriate level
507 logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) 516 logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
508 517
509 # If the arguments are OK, try to login on the site and upload. 518 settings = import_settings(args.config)
519
520 # If the arguments are OK, try to log-in to SCC and upload.
510 if args.delete: 521 if args.delete:
511 # If the delete is provided, do nothing else 522 # If the delete is provided, do nothing else
512 delete_measurement(args.delete) 523 delete_measurement(args.delete, settings)
513 elif args.rerun_all: 524 elif args.rerun_all:
514 rerun_all(args.rerun_all, args.process) 525 rerun_all(args.rerun_all, args.process, settings)
515 elif args.rerun_processing: 526 elif args.rerun_processing:
516 rerun_processing(args.rerun_processing, args.process) 527 rerun_processing(args.rerun_processing, args.process, settings)
517 else: 528 else:
518 if (args.filename == '') or (args.system == 0): 529 if (args.filename == '') or (args.system == 0):
519 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') 530 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
520 531
521 if args.process: 532 if args.process:
522 process_file(args.filename, args.system) 533 process_file(args.filename, args.system, settings)
523 else: 534 else:
524 upload_file(args.filename, args.system) 535 upload_file(args.filename, args.system, settings)
525
526
527 # When running through terminal
528 if __name__ == '__main__':
529 main()

EARLINET Repository: SCC access

mercurial