Mercurial > public > scc_access / file comparison

comparison: scc_access/scc_access.py

scc_access/scc_access.py

changeset 29
3e3e5bda6b77
parent 14
c2020b2fdd05
child 30
4669876326d4
equal deleted inserted replaced
14:c2020b2fdd05 29:3e3e5bda6b77
1 import requests 1 import requests
2
2 requests.packages.urllib3.disable_warnings() 3 requests.packages.urllib3.disable_warnings()
3 4
4 import sys
5 import urlparse 5 import urlparse
6 import argparse 6 import argparse
7 import os 7 import os
8 import re 8 import re
9 import time 9 import time
11 from zipfile import ZipFile 11 from zipfile import ZipFile
12 import datetime 12 import datetime
13 import logging 13 import logging
14 import yaml 14 import yaml
15 15
16
17 logger = logging.getLogger(__name__) 16 logger = logging.getLogger(__name__)
18 17
19 # The regex to find the measurement id from the measurement page 18 # The regex to find the measurement id from the measurement page
20 # This should be read from the uploaded file, but would require an extra NetCDF module. 19 # This should be read from the uploaded file, but would require an extra NetCDF module.
21 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>" 20 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
28 version no check is performed, and no feedback is given if the upload 27 version no check is performed, and no feedback is given if the upload
29 was successful. If everything is setup correctly, it will work. 28 was successful. If everything is setup correctly, it will work.
30 """ 29 """
31 30
32 def __init__(self, auth, output_dir, base_url): 31 def __init__(self, auth, output_dir, base_url):
32
33 self.auth = auth 33 self.auth = auth
34 self.output_dir = output_dir 34 self.output_dir = output_dir
35 self.base_url = base_url 35 self.base_url = base_url
36 self.session = requests.Session() 36 self.session = requests.Session()
37 self.construct_urls() 37 self.session.auth = auth
38 38 self.session.verify = False
39 def construct_urls(self):
40 """ Construct all URLs needed for processing. """
41 # Construct the absolute URLs
42 self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/') 39 self.login_url = urlparse.urljoin(self.base_url, 'accounts/login/')
43 self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/') 40 self.upload_url = urlparse.urljoin(self.base_url, 'data_processing/measurements/quick/')
44 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-preprocessed/') 41 self.download_preprocessed_pattern = urlparse.urljoin(self.base_url,
45 self.download_optical_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-optical/') 42 'data_processing/measurements/{0}/download-preprocessed/')
46 self.download_graph_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/{0}/download-plots/') 43 self.download_optical_pattern = urlparse.urljoin(self.base_url,
44 'data_processing/measurements/{0}/download-optical/')
45 self.download_graph_pattern = urlparse.urljoin(self.base_url,
46 'data_processing/measurements/{0}/download-plots/')
47 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/') 47 self.delete_measurement_pattern = urlparse.urljoin(self.base_url, 'admin/database/measurements/{0}/delete/')
48 self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/') 48 self.api_base_url = urlparse.urljoin(self.base_url, 'api/v1/')
49 self.list_measurements_pattern = urlparse.urljoin(self.base_url, 'data_processing/measurements/')
49 50
50 def login(self, credentials): 51 def login(self, credentials):
51 """ Login the the website. """ 52 """ Login the the website. """
52 logger.debug("Attempting to login to SCC, username %s." % credentials[0]) 53 logger.debug("Attempting to login to SCC, username %s." % credentials[0])
53 self.login_credentials = {'username': credentials[0], 54 login_credentials = {'username': credentials[0],
54 'password': credentials[1]} 55 'password': credentials[1]}
55 56
56 logger.debug("Accessing login page at %s." % self.login_url) 57 logger.debug("Accessing login page at %s." % self.login_url)
57 58
58 # Get upload form 59 # Get upload form
59 login_page = self.session.get(self.login_url, auth=self.auth, verify=False) 60 login_page = self.session.get(self.login_url)
60
61 if login_page.status_code != 200:
62 logger.error('Could not access login pages. Status code %s' % login_page.status_code)
63 sys.exit(1)
64 61
65 logger.debug("Submiting credentials.") 62 logger.debug("Submiting credentials.")
66
67 # Submit the login data 63 # Submit the login data
68 login_submit = self.session.post(self.login_url, 64 login_submit = self.session.post(self.login_url,
69 data=self.login_credentials, 65 data=login_credentials,
70 headers={'X-CSRFToken': login_page.cookies['csrftoken'], 66 headers={'X-CSRFToken': login_page.cookies['csrftoken'],
71 'referer': self.login_url}, 67 'referer': self.login_url})
72 verify=False,
73 auth=self.auth)
74 return login_submit 68 return login_submit
75 69
76 def logout(self): 70 def logout(self):
77 pass 71 pass
78 72
79 def upload_file(self, filename, system_id): 73 def upload_file(self, filename, system_id):
80 """ Upload a filename for processing with a specific system. If the 74 """ Upload a filename for processing with a specific system. If the
81 upload is successful, it returns the measurement id. """ 75 upload is successful, it returns the measurement id. """
82 # Get submit page 76 # Get submit page
83 upload_page = self.session.get(self.upload_url, 77 upload_page = self.session.get(self.upload_url)
84 auth=self.auth,
85 verify=False)
86 78
87 # Submit the data 79 # Submit the data
88 upload_data = {'system': system_id} 80 upload_data = {'system': system_id}
89 files = {'data': open(filename, 'rb')} 81 files = {'data': open(filename, 'rb')}
90 82
92 84
93 upload_submit = self.session.post(self.upload_url, 85 upload_submit = self.session.post(self.upload_url,
94 data=upload_data, 86 data=upload_data,
95 files=files, 87 files=files,
96 headers={'X-CSRFToken': upload_page.cookies['csrftoken'], 88 headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
97 'referer': self.upload_url,}, 89 'referer': self.upload_url})
98 verify=False,
99 auth=self.auth)
100 90
101 if upload_submit.status_code != 200: 91 if upload_submit.status_code != 200:
102 logger.warning("Connection error. Status code: %s" % upload_submit.status_code) 92 logger.warning("Connection error. Status code: %s" % upload_submit.status_code)
103 return False 93 return False
104 94
116 """ Downloads some files from the download_url to the specified 106 """ Downloads some files from the download_url to the specified
117 subdir. This method is used to download preprocessed file, optical 107 subdir. This method is used to download preprocessed file, optical
118 files etc. 108 files etc.
119 """ 109 """
120 # Get the file 110 # Get the file
121 request = self.session.get(download_url, auth=self.auth, 111 request = self.session.get(download_url, stream=True)
122 verify=False, 112
123 stream=True) 113 if not request.ok:
114 raise Exception("Could not download files for measurement '%s'" % measurement_id)
124 115
125 # Create the dir if it does not exist 116 # Create the dir if it does not exist
126 local_dir = os.path.join(self.output_dir, measurement_id, subdir) 117 local_dir = os.path.join(self.output_dir, measurement_id, subdir)
127 if not os.path.exists(local_dir): 118 if not os.path.exists(local_dir):
128 os.makedirs(local_dir) 119 os.makedirs(local_dir)
152 self.download_files(measurement_id, 'scc_preprocessed', download_url) 143 self.download_files(measurement_id, 'scc_preprocessed', download_url)
153 144
154 def download_optical(self, measurement_id): 145 def download_optical(self, measurement_id):
155 """ Download optical files for the measurement id. """ 146 """ Download optical files for the measurement id. """
156 # Construct the download url 147 # Construct the download url
157 download_url = self.download_optical.format(measurement_id) 148 download_url = self.download_optical_pattern.format(measurement_id)
158 self.download_files(measurement_id, 'scc_optical', download_url) 149 self.download_files(measurement_id, 'scc_optical', download_url)
159 150
160 def download_graphs(self, measurement_id): 151 def download_graphs(self, measurement_id):
161 """ Download profile graphs for the measurement id. """ 152 """ Download profile graphs for the measurement id. """
162 # Construct the download url 153 # Construct the download url
165 156
166 def rerun_processing(self, measurement_id, monitor=True): 157 def rerun_processing(self, measurement_id, monitor=True):
167 measurement = self.get_measurement(measurement_id) 158 measurement = self.get_measurement(measurement_id)
168 159
169 if measurement: 160 if measurement:
170 request = self.session.get(measurement.rerun_processing_url, auth=self.auth, 161 request = self.session.get(measurement.rerun_processing_url, stream=True)
171 verify=False,
172 stream=True)
173 162
174 if request.status_code != 200: 163 if request.status_code != 200:
175 logger.error( 164 logger.error(
176 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code)) 165 "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
177 return 166 return
186 measurement = self.get_measurement(measurement_id) 175 measurement = self.get_measurement(measurement_id)
187 176
188 if measurement: 177 if measurement:
189 logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url) 178 logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
190 179
191 request = self.session.get(measurement.rerun_all_url, auth=self.auth, 180 request = self.session.get(measurement.rerun_all_url, stream=True)
192 verify=False,
193 stream=True)
194 181
195 if request.status_code != 200: 182 if request.status_code != 200:
196 logger.error("Could not rerun pre processing for %s. Status code: %s" % 183 logger.error("Could not rerun pre processing for %s. Status code: %s" %
197 (measurement_id, request.status_code)) 184 (measurement_id, request.status_code))
198 return 185 return
199 186
200 if monitor: 187 if monitor:
201 self.monitor_processing(measurement_id) 188 self.monitor_processing(measurement_id)
202 189
203 def process(self, filename, system_id): 190 def process(self, filename, system_id, monitor):
204 """ Upload a file for processing and wait for the processing to finish. 191 """ Upload a file for processing and wait for the processing to finish.
205 If the processing is successful, it will download all produced files. 192 If the processing is successful, it will download all produced files.
206 """ 193 """
207 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) 194 logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
208 # Upload file 195 # Upload file
209 measurement_id = self.upload_file(filename, system_id) 196 measurement_id = self.upload_file(filename, system_id)
210 197
211 measurement = self.monitor_processing(measurement_id) 198 if monitor:
212 return measurement 199 return self.monitor_processing(measurement_id)
200 return None
213 201
214 def monitor_processing(self, measurement_id): 202 def monitor_processing(self, measurement_id):
215 """ Monitor the processing progress of a measurement id""" 203 """ Monitor the processing progress of a measurement id"""
216 204
217 measurement = self.get_measurement(measurement_id) 205 measurement = self.get_measurement(measurement_id)
238 226
239 def get_status(self, measurement_id): 227 def get_status(self, measurement_id):
240 """ Get the processing status for a measurement id through the API. """ 228 """ Get the processing status for a measurement id through the API. """
241 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id) 229 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__exact=%s' % measurement_id)
242 230
243 response = self.session.get(measurement_url, 231 response = self.session.get(measurement_url)
244 auth=self.auth,
245 verify=False)
246 232
247 response_dict = response.json() 233 response_dict = response.json()
248 234
249 if response_dict['objects']: 235 if response_dict['objects']:
250 measurement_list = response_dict['objects'] 236 measurement_list = response_dict['objects']
255 return None 241 return None
256 242
257 def get_measurement(self, measurement_id): 243 def get_measurement(self, measurement_id):
258 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id) 244 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements/%s/' % measurement_id)
259 245
260 response = self.session.get(measurement_url, 246 response = self.session.get(measurement_url)
261 auth=self.auth, 247
262 verify=False) 248 if not response.ok:
263
264 if response.status_code != 200:
265 logger.error('Could not access API. Status code %s.' % response.status_code) 249 logger.error('Could not access API. Status code %s.' % response.status_code)
266 sys.exit(1) 250 return None
267 251
268 response_dict = response.json() 252 response_dict = response.json()
269 253
270 if response_dict: 254 if response_dict:
271 measurement = Measurement(self.base_url,response_dict) 255 measurement = Measurement(self.base_url, response_dict)
272 return measurement 256 return measurement
273 else: 257 else:
274 logger.error("No measurement with id %s found on the SCC." % measurement_id) 258 logger.error("No measurement with id %s found on the SCC." % measurement_id)
275 return None 259 return None
276 260
288 if measurement is None: 272 if measurement is None:
289 logger.warning("Nothing to delete.") 273 logger.warning("Nothing to delete.")
290 return None 274 return None
291 275
292 # Go the the page confirming the deletion 276 # Go the the page confirming the deletion
293 delete_url = self.delete_measurement_pattern.format(measurement.id) 277 delete_url = self.delete_measurement_pattern.format(measurement_id)
294 278
295 confirm_page = self.session.get(delete_url, 279 confirm_page = self.session.get(delete_url)
296 auth=self.auth,
297 verify=False)
298 280
299 # Check that the page opened properly 281 # Check that the page opened properly
300 if confirm_page.status_code != 200: 282 if confirm_page.status_code != 200:
301 logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code)) 283 logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code))
302 return None 284 return None
303 285
304 # Delete the measurement 286 # Delete the measurement
305 delete_page = self.session.post(delete_url, 287 delete_page = self.session.post(delete_url,
306 auth=self.auth,
307 verify=False,
308 data={'post': 'yes'}, 288 data={'post': 'yes'},
309 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], 289 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
310 'referer': delete_url} 290 'referer': delete_url}
311 ) 291 )
312 if delete_page.status_code != 200: 292 if delete_page.status_code != 200:
318 return True 298 return True
319 299
320 def available_measurements(self): 300 def available_measurements(self):
321 """ Get a list of available measurement on the SCC. """ 301 """ Get a list of available measurement on the SCC. """
322 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements') 302 measurement_url = urlparse.urljoin(self.api_base_url, 'measurements')
323 response = self.session.get(measurement_url, 303 response = self.session.get(measurement_url)
324 auth=self.auth,
325 verify=False)
326 response_dict = response.json() 304 response_dict = response.json()
327 305
328 measurements = None 306 measurements = None
329 if response_dict: 307 if response_dict:
330 measurement_list = response_dict['objects'] 308 measurement_list = response_dict['objects']
333 else: 311 else:
334 logger.warning("No response received from the SCC when asked for available measurements.") 312 logger.warning("No response received from the SCC when asked for available measurements.")
335 313
336 return measurements 314 return measurements
337 315
316 def list_measurements(self, station=None, system=None, start=None, stop=None, upload_status=None,
317 processing_status=None, optical_processing=None):
318
319 # Need to set to empty string if not specified, we won't get any results
320 params = {
321 "station": station if station is not None else "",
322 "system": system if system is not None else "",
323 "stop": stop if stop is not None else "",
324 "start": start if start is not None else "",
325 "upload_status": upload_status if upload_status is not None else "",
326 "preprocessing_status": processing_status if processing_status is not None else "",
327 "optical_processing_status": optical_processing if optical_processing is not None else ""
328 }
329 resp = self.session.get(self.list_measurements_pattern, params=params).text
330 tbl_rgx = re.compile(r'<table id="measurements">(.*?)</table>', re.DOTALL)
331 entry_rgx = re.compile(r'<tr>(.*?)</tr>', re.DOTALL)
332 measurement_rgx = re.compile(
333 r'.*?<td><a[^>]*>(\w+)</a>.*?<td>.*?<td>([\w-]+ [\w:]+)</td>.*<td data-order="([-]?\d+),([-]?\d+),([-]?\d+)".*',
334 re.DOTALL)
335 matches = tbl_rgx.findall(resp)
336 if len(matches) != 1:
337 return []
338
339 ret = []
340 for entry in entry_rgx.finditer(matches[0]):
341 m = measurement_rgx.match(entry.string[entry.start(0):entry.end(0)])
342 if m:
343 name, date, upload, preproc, optical = m.groups()
344 ret.append(
345 Measurement(self.base_url, {"id": name, "upload": int(upload), "pre_processing": int(preproc),
346 "processing": int(optical)}))
347
348 return ret
349
338 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0): 350 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
339 """ Give the first available measurement id on the SCC for the specific 351 """ Give the first available measurement id on the SCC for the specific
340 date. 352 date.
341 """ 353 """
342 date_str = t1.strftime('%Y%m%d') 354 date_str = t1.strftime('%Y%m%d')
343 search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str) 355 search_url = urlparse.urljoin(self.api_base_url, 'measurements/?id__startswith=%s' % date_str)
344 356
345 response = self.session.get(search_url, 357 response = self.session.get(search_url)
346 auth=self.auth,
347 verify=False)
348 358
349 response_dict = response.json() 359 response_dict = response.json()
350 360
351 measurement_id = None 361 measurement_id = None
352 362
424 measurement_id = scc.upload_file(filename, system_id) 434 measurement_id = scc.upload_file(filename, system_id)
425 scc.logout() 435 scc.logout()
426 return measurement_id 436 return measurement_id
427 437
428 438
429 def process_file(filename, system_id, settings): 439 def process_file(filename, system_id, monitor, settings):
430 """ Shortcut function to process a file to the SCC. """ 440 """ Shortcut function to process a file to the SCC. """
431 logger.info("Processing file %s, using sytem %s" % (filename, system_id)) 441 logger.info("Processing file %s, using sytem %s" % (filename, system_id))
432
433 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 442 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
434 scc.login(settings['website_credentials']) 443 scc.login(settings['website_credentials'])
435 measurement = scc.process(filename, system_id) 444 measurement = scc.process(filename, system_id, monitor)
436 scc.logout() 445 scc.logout()
437 return measurement 446 return measurement
438 447
439 448
440 def delete_measurement(measurement_id, settings): 449 def delete_measurement(measurement_ids, settings):
441 """ Shortcut function to delete a measurement from the SCC. """ 450 """ Shortcut function to delete measurements from the SCC. """
442 logger.info("Deleting %s" % measurement_id)
443
444 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 451 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
445 scc.login(settings['website_credentials']) 452 scc.login(settings['website_credentials'])
446 scc.delete_measurement(measurement_id) 453 for m_id in measurement_ids:
454 logger.info("Deleting %s" % m_id)
455 scc.delete_measurement(m_id)
447 scc.logout() 456 scc.logout()
448 457
449 458
450 def rerun_all(measurement_id, monitor, settings): 459 def rerun_all(measurement_ids, monitor, settings):
451 """ Shortcut function to delete a measurement from the SCC. """ 460 """ Shortcut function to rerun measurements from the SCC. """
452 logger.info("Rerunning all products for %s" % measurement_id)
453 461
454 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 462 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
455 scc.login(settings['website_credentials']) 463 scc.login(settings['website_credentials'])
456 scc.rerun_all(measurement_id, monitor) 464 for m_id in measurement_ids:
465 logger.info("Rerunning all products for %s" % m_id)
466 scc.rerun_all(m_id, monitor)
457 scc.logout() 467 scc.logout()
458 468
459 469
460 def rerun_processing(measurement_id, monitor, settings): 470 def rerun_processing(measurement_ids, monitor, settings):
461 """ Shortcut function to delete a measurement from the SCC. """ 471 """ Shortcut function to delete a measurement from the SCC. """
462 logger.info("Rerunning (optical) processing for %s" % measurement_id)
463 472
464 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) 473 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
465 scc.login(settings['website_credentials']) 474 scc.login(settings['website_credentials'])
466 scc.rerun_processing(measurement_id, monitor) 475 for m_id in measurement_ids:
476 logger.info("Rerunning (optical) processing for %s" % m_id)
477 scc.rerun_processing(m_id, monitor)
467 scc.logout() 478 scc.logout()
468 479
469 480
470 def import_settings(config_file_path): 481 def list_measurements(settings, station=None, system=None, start=None, stop=None, upload_status=None,
482 preprocessing_status=None,
483 optical_processing=None):
484 """List all available measurements"""
485 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
486 scc.login(settings['website_credentials'])
487 ret = scc.list_measurements(station=station, system=system, start=start, stop=stop, upload_status=upload_status,
488 processing_status=preprocessing_status, optical_processing=optical_processing)
489 for entry in ret:
490 print("%s" % entry.id)
491 scc.logout()
492
493
494 def download_measurements(measurement_ids, download_preproc, download_optical, download_graph, settings):
495 """Download all measurements for the specified IDs"""
496 scc = SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url'])
497 scc.login(settings['website_credentials'])
498 for m_id in measurement_ids:
499 if download_preproc:
500 logger.info("Downloading preprocessed files for '%s'" % m_id)
501 scc.download_preprocessed(m_id)
502 logger.info("Complete")
503 if download_optical:
504 logger.info("Downloading optical files for '%s'" % m_id)
505 scc.download_optical(m_id)
506 logger.info("Complete")
507 if download_graph:
508 logger.info("Downloading profile graph files for '%s'" % m_id)
509 scc.download_graphs(m_id)
510 logger.info("Complete")
511
512
513 def settings_from_path(config_file_path):
471 """ Read the configuration file. 514 """ Read the configuration file.
472 515
473 The file should be in YAML syntax.""" 516 The file should be in YAML syntax."""
474 517
475 if not os.path.isfile(config_file_path): 518 if not os.path.isfile(config_file_path):
476 logger.error("Wrong path for configuration file (%s)" % config_file_path) 519 raise argparse.ArgumentTypeError("Wrong path for configuration file (%s)" % config_file_path)
477 sys.exit(1)
478 520
479 with open(config_file_path) as yaml_file: 521 with open(config_file_path) as yaml_file:
480 try: 522 try:
481 settings = yaml.safe_load(yaml_file) 523 settings = yaml.safe_load(yaml_file)
482 logger.debug("Read settings file(%s)" % config_file_path) 524 logger.debug("Read settings file(%s)" % config_file_path)
483 except: 525 except Exception:
484 logger.error("Could not parse YAML file (%s)" % config_file_path) 526 raise argparse.ArgumentTypeError("Could not parse YAML file (%s)" % config_file_path)
485 sys.exit(1)
486 527
487 # YAML limitation: does not read tuples 528 # YAML limitation: does not read tuples
488 settings['basic_credentials'] = tuple(settings['basic_credentials']) 529 settings['basic_credentials'] = tuple(settings['basic_credentials'])
489 settings['website_credentials'] = tuple(settings['website_credentials']) 530 settings['website_credentials'] = tuple(settings['website_credentials'])
490 return settings 531 return settings
491 532
492 533
534 # Setup for command specific parsers
535 def setup_delete(parser):
536 def delete_from_args(parsed):
537 delete_measurement(parsed.IDs, parsed.config)
538
539 parser.add_argument("IDs", nargs="+", help="measurement IDs to delete.")
540 parser.set_defaults(execute=delete_from_args)
541
542
543 def setup_rerun_all(parser):
544 def rerun_all_from_args(parsed):
545 rerun_all(parsed.IDs, parsed.process, parsed.config)
546
547 parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun.")
548 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
549 action="store_true")
550 parser.set_defaults(execute=rerun_all_from_args)
551
552
553 def setup_rerun_processing(parser):
554 def rerun_processing_from_args(parsed):
555 rerun_processing(parsed.IDs, parsed.process, parsed.config)
556
557 parser.add_argument("IDs", nargs="+", help="Measurement IDs to rerun the processing on.")
558 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
559 action="store_true")
560 parser.set_defaults(execute=rerun_processing_from_args)
561
562
563 def setup_process_file(parser):
564 def process_file_from_args(parsed):
565 process_file(parsed.file, parsed.system, parsed.process, parsed.config)
566
567 parser.add_argument("filename", help="Measurement file name or path.")
568 parser.add_argument("system", help="Processing system id.")
569 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
570 action="store_true")
571 parser.set_defaults(execute=process_file_from_args)
572
573
574 def setup_upload_file(parser):
575 def upload_file_from_args(parsed):
576 upload_file(parsed.file, parsed.system, parsed.config)
577
578 parser.add_argument("filename", help="Measurement file name or path.")
579 parser.add_argument("system", help="Processing system id.")
580 parser.set_defaults(execute=upload_file_from_args)
581
582
583 def setup_list_measurements(parser):
584 def list_measurements_from_args(parsed):
585 list_measurements(parsed.config, station=parsed.station, system=parsed.system, start=parsed.start,
586 stop=parsed.stop,
587 upload_status=parsed.upload_status, preprocessing_status=parsed.preprocessing_status,
588 optical_processing=parsed.optical_processing_status)
589
590 def status(arg):
591 if -127 <= int(arg) <= 127:
592 return arg
593 else:
594 raise argparse.ArgumentTypeError("Status must be between -127 and 127")
595
596 def date(arg):
597 if re.match(r'\d{4}-\d{2}-\d{2}', arg):
598 return arg
599 else:
600 raise argparse.ArgumentTypeError("Date must be in format 'YYYY-MM-DD'")
601
602 parser.add_argument("--station", help="Filter for only the selected station")
603 parser.add_argument("--system", help="Filter for only the selected station")
604 parser.add_argument("--start", help="Filter for only the selected station", type=date)
605 parser.add_argument("--stop", help="Filter for only the selected station", type=date)
606 parser.add_argument("--upload-status", help="Filter for only the selected station", type=status)
607 parser.add_argument("--preprocessing-status", help="Filter for only the selected station", type=status)
608 parser.add_argument("--optical-processing-status", help="Filter for only the selected station", type=status)
609 parser.set_defaults(execute=list_measurements_from_args)
610
611
612 def setup_download_measurements(parser):
613 def download_measurements_from_args(parsed):
614 preproc = parsed.download_preprocessed
615 optical = parsed.download_optical
616 graphs = parsed.download_profile_graphs
617 if not preproc and not graphs:
618 optical = True
619 download_measurements(parsed.IDs, preproc, optical, graphs, parsed.config)
620
621 parser.add_argument("IDs", help="Measurement IDs that should be downloaded.", nargs="+")
622 parser.add_argument("--download-preprocessed", action="store_true", help="Download preprocessed files.")
623 parser.add_argument("--download-optical", action="store_true",
624 help="Download optical files (default if no other download is used).")
625 parser.add_argument("--download-profile-graphs", action="store_true", help="Download profile graph files.")
626 parser.set_defaults(execute=download_measurements_from_args)
627
628
493 def main(): 629 def main():
494 # Define the command line arguments. 630 # Define the command line arguments.
495 parser = argparse.ArgumentParser() 631 parser = argparse.ArgumentParser()
496 parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='') 632 subparsers = parser.add_subparsers()
497 parser.add_argument("system", nargs='?', help="Processing system id.", default=0) 633
498 parser.add_argument("-c", "--config", nargs='?', help="Path to configuration file") 634 delete_parser = subparsers.add_parser("delete", help="Deletes a measurement.")
499 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", 635 rerun_all_parser = subparsers.add_parser("rerun-all", help="Rerun a measurement.")
500 action="store_true") 636 rerun_processing_parser = subparsers.add_parser("rerun-processing",
501 parser.add_argument("--delete", help="Measurement ID to delete.") 637 help="Rerun processing routings for a measurement.")
502 parser.add_argument("--rerun-all", help="Measurement ID to rerun.") 638 process_file_parser = subparsers.add_parser("process-file", help="Process a file.")
503 parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routines.") 639 upload_file_parser = subparsers.add_parser("upload-file", help="Upload a file.")
640 list_parser = subparsers.add_parser("list", help="List all measurements.")
641 download_parser = subparsers.add_parser("download", help="Download selected measurements.")
642
643 setup_delete(delete_parser)
644 setup_rerun_all(rerun_all_parser)
645 setup_rerun_processing(rerun_processing_parser)
646 setup_process_file(process_file_parser)
647 setup_upload_file(upload_file_parser)
648 setup_list_measurements(list_parser)
649 setup_download_measurements(download_parser)
504 650
505 # Verbosity settings from http://stackoverflow.com/a/20663028 651 # Verbosity settings from http://stackoverflow.com/a/20663028
506 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const", 652 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
507 dest="loglevel", const=logging.DEBUG, default=logging.INFO, 653 dest="loglevel", const=logging.DEBUG, default=logging.INFO,
508 ) 654 )
509 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const", 655 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
510 dest="loglevel", const=logging.WARNING 656 dest="loglevel", const=logging.WARNING
511 ) 657 )
512 658
659 home = os.path.expanduser("~")
660 default_config_location = os.path.abspath(os.path.join(home, ".scc_access.yaml"))
661 parser.add_argument("-c", "--config", help="Path to the config file.", type=settings_from_path,
662 default=default_config_location)
663
513 args = parser.parse_args() 664 args = parser.parse_args()
514 665
515 # Get the logger with the appropriate level 666 # Get the logger with the appropriate level
516 logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel) 667 logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
517 668
518 settings = import_settings(args.config) 669 # Dispatch to appropriate function
519 670 args.execute(args)
520 # If the arguments are OK, try to log-in to SCC and upload. 671
521 if args.delete: 672
522 # If the delete is provided, do nothing else 673 # When running through terminal
523 delete_measurement(args.delete, settings) 674 if __name__ == '__main__':
524 elif args.rerun_all: 675 main()
525 rerun_all(args.rerun_all, args.process, settings)
526 elif args.rerun_processing:
527 rerun_processing(args.rerun_processing, args.process, settings)
528 else:
529 if (args.filename == '') or (args.system == 0):
530 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
531
532 if args.process:
533 process_file(args.filename, args.system, settings)
534 else:
535 upload_file(args.filename, args.system, settings)

EARLINET Repository: SCC access

mercurial