Mercurial > public > scc_access / file comparison

comparison: scc_access.py

scc_access.py

changeset 6
c02712d2ab9e
parent 4
809c63be8a40
equal deleted inserted replaced
5:1c170a1ae099 6:c02712d2ab9e
23 THE SOFTWARE. 23 THE SOFTWARE.
24 """ 24 """
25 25
26 __version__ = "0.6.0" 26 __version__ = "0.6.0"
27 27
28
29 # Try to read the settings from the settings.py file 28 # Try to read the settings from the settings.py file
30 try: 29 try:
31 from settings import * 30 from settings import *
32 except: 31 except:
33 raise ImportError( 32 raise ImportError(
34 """A settings file (setting.py) is required to run the script. 33 """A settings file (setting.py) is required to run the script.
35 You can use settings.sample.py as a template.""") 34 You can use settings.sample.py as a template.""")
36
37 35
38 import requests 36 import requests
37 requests.packages.urllib3.disable_warnings()
38
39 import urlparse 39 import urlparse
40 import argparse 40 import argparse
41 import os 41 import os
42 import re 42 import re
43 import time 43 import time
44 import StringIO 44 import StringIO
45 from zipfile import ZipFile 45 from zipfile import ZipFile
46 import datetime 46 import datetime
47 import logging
48
49 logger = logging.getLogger(__name__)
47 50
48 51
49 # Construct the absolute URLs 52 # Construct the absolute URLs
50 LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/') 53 LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/')
51 UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/') 54 UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/')
53 DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/') 56 DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/')
54 DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/') 57 DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/')
55 RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/') 58 RERUN_ALL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-all/')
56 RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/') 59 RERUN_PROCESSING = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/rerun-optical/')
57 60
58 DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/') 61 DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
59 API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/') 62 API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
60 63
61 # The regex to find the measurement id from the measurement page 64 # The regex to find the measurement id from the measurement page
62 # This should be read from the uploaded file, but would require an extra module 65 # This should be read from the uploaded file, but would require an extra NetCDF module.
63 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>" 66 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
64 67
65 68
66 class SCC: 69 class SCC:
67 """ A simple class that will attempt to upload a file on the SCC server. 70 """ A simple class that will attempt to upload a file on the SCC server.
68 The uploading is done by simulation a normal browser session. In the current 71 The uploading is done by simulating a normal browser session. In the current
69 version no check is performed, and no feedback is given if the upload 72 version no check is performed, and no feedback is given if the upload
70 was successful. If everything is setup correctly, it will work. 73 was successful. If everything is setup correctly, it will work.
71 """ 74 """
72 def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR): 75
76 def __init__(self, auth=BASIC_LOGIN, output_dir=OUTPUT_DIR):
73 self.auth = auth 77 self.auth = auth
74 self.output_dir = OUTPUT_DIR 78 self.output_dir = output_dir
75 self.session = requests.Session() 79 self.session = requests.Session()
76 80
77 def login(self, credential = DJANGO_LOGIN): 81 def login(self, credentials=DJANGO_LOGIN):
78 """ Login the the website. """ 82 """ Login the the website. """
79 self.login_credentials = {'username': credential[0], 83 logger.debug("Attempting to login to SCC, username %s." % credentials[0])
80 'password': credential[1]} 84 self.login_credentials = {'username': credentials[0],
81 85 'password': credentials[1]}
86
87 logger.debug("Accessing login page at %s." % LOGIN_URL)
88
82 # Get upload form 89 # Get upload form
83 login_page = self.session.get(LOGIN_URL, 90 login_page = self.session.get(LOGIN_URL,
84 auth = self.auth, verify = False) 91 auth=self.auth, verify=False)
85 92
93 logger.debug("Submiting credentials.")
86 # Submit the login data 94 # Submit the login data
87 login_submit = self.session.post(LOGIN_URL, 95 login_submit = self.session.post(LOGIN_URL,
88 data = self.login_credentials, 96 data=self.login_credentials,
89 headers = {'X-CSRFToken': login_page.cookies['csrftoken'], 97 headers={'X-CSRFToken': login_page.cookies['csrftoken'],
90 'referer': LOGIN_URL}, 98 'referer': LOGIN_URL},
91 verify = False, 99 verify=False,
92 auth = self.auth) 100 auth=self.auth)
93 return login_submit 101 return login_submit
94 102
95 def logout(self): 103 def logout(self):
96 pass 104 pass
97 105
98 def upload_file(self, filename, system_id): 106 def upload_file(self, filename, system_id):
99 """ Upload a filename for processing with a specific system. If the 107 """ Upload a filename for processing with a specific system. If the
100 upload is successful, it returns the measurement id. """ 108 upload is successful, it returns the measurement id. """
101 # Get submit page 109 # Get submit page
102 upload_page = self.session.get(UPLOAD_URL, 110 upload_page = self.session.get(UPLOAD_URL,
103 auth = self.auth, 111 auth=self.auth,
104 verify = False) 112 verify=False)
105 113
106 # Submit the data 114 # Submit the data
107 upload_data = {'system': system_id} 115 upload_data = {'system': system_id}
108 files = {'data': open(filename, 'rb')} 116 files = {'data': open(filename, 'rb')}
109 117
110 print "Uploading of file %s started." % filename 118 logging.info("Uploading of file %s started." % filename)
111 119
112 upload_submit = self.session.post(UPLOAD_URL, 120 upload_submit = self.session.post(UPLOAD_URL,
113 data = upload_data, 121 data=upload_data,
114 files = files, 122 files=files,
115 headers = {'X-CSRFToken': upload_page.cookies['csrftoken'], 123 headers={'X-CSRFToken': upload_page.cookies['csrftoken'],
116 'referer': UPLOAD_URL}, 124 'referer': UPLOAD_URL},
117 verify = False, 125 verify=False,
118 auth = self.auth) 126 auth=self.auth)
119 127
120 if upload_submit.status_code != 200: 128 if upload_submit.status_code != 200:
121 print "Connection error. Status code: %s" % upload_submit.status_code 129 logging.warning("Connection error. Status code: %s" % upload_submit.status_code)
122 return False 130 return False
123 131
124 measurement_id = True
125
126 # Check if there was a redirect to a new page. 132 # Check if there was a redirect to a new page.
127 if upload_submit.url == UPLOAD_URL: 133 if upload_submit.url == UPLOAD_URL:
128 measurement_id = False 134 measurement_id = False
129 print "Uploaded file rejected! Try to upload manually to see the error." 135 logging.error("Uploaded file rejected! Try to upload manually to see the error.")
130 else: 136 else:
131 measurement_id = re.findall(regex, upload_submit.text)[0] 137 measurement_id = re.findall(regex, upload_submit.text)[0]
132 print "Successfully uploaded measurement with id %s." % measurement_id 138 logging.error("Successfully uploaded measurement with id %s." % measurement_id)
133 139
134 return measurement_id 140 return measurement_id
135 141
136 def download_files(self, measurement_id, subdir, download_url): 142 def download_files(self, measurement_id, subdir, download_url):
137 """ Downloads some files from the download_url to the specified 143 """ Downloads some files from the download_url to the specified
138 subdir. This method is used to download preprocessed file, optical 144 subdir. This method is used to download preprocessed file, optical
139 files etc. 145 files etc.
140 """ 146 """
141 # Get the file 147 # Get the file
142 request = self.session.get(download_url, auth = self.auth, 148 request = self.session.get(download_url, auth=self.auth,
143 verify = False, 149 verify=False,
144 stream=True) 150 stream=True)
145 151
146 # Create the dir if it does not exist 152 # Create the dir if it does not exist
147 local_dir = os.path.join(self.output_dir, measurement_id, subdir) 153 local_dir = os.path.join(self.output_dir, measurement_id, subdir)
148 if not os.path.exists(local_dir): 154 if not os.path.exists(local_dir):
149 os.makedirs(local_dir) 155 os.makedirs(local_dir)
150 156
151
152 # Save the file by chunk, needed if the file is big. 157 # Save the file by chunk, needed if the file is big.
153 memory_file = StringIO.StringIO() 158 memory_file = StringIO.StringIO()
154 159
155 for chunk in request.iter_content(chunk_size=1024): 160 for chunk in request.iter_content(chunk_size=1024):
156 if chunk: # filter out keep-alive new chunks 161 if chunk: # filter out keep-alive new chunks
157 memory_file.write(chunk) 162 memory_file.write(chunk)
158 memory_file.flush() 163 memory_file.flush()
159 164
160 zip_file = ZipFile(memory_file) 165 zip_file = ZipFile(memory_file)
161 166
162 for ziped_name in zip_file.namelist(): 167 for ziped_name in zip_file.namelist():
163 basename = os.path.basename(ziped_name) 168 basename = os.path.basename(ziped_name)
164 169
165 local_file = os.path.join(local_dir, basename) 170 local_file = os.path.join(local_dir, basename)
166 171
167 with open(local_file, 'wb') as f: 172 with open(local_file, 'wb') as f:
168 f.write(zip_file.read(ziped_name)) 173 f.write(zip_file.read(ziped_name))
169 174
170 def download_preprocessed(self, measurement_id): 175 def download_preprocessed(self, measurement_id):
171 """ Download preprocessed files for the measurement id. """ 176 """ Download preprocessed files for the measurement id. """
172 # Construct the download url 177 # Construct the download url
173 download_url = DOWNLOAD_PREPROCESSED.format(measurement_id) 178 download_url = DOWNLOAD_PREPROCESSED.format(measurement_id)
174 self.download_files(measurement_id, 'scc_preprocessed', download_url) 179 self.download_files(measurement_id, 'scc_preprocessed', download_url)
175 180
176 def download_optical(self, measurement_id): 181 def download_optical(self, measurement_id):
177 """ Download optical files for the measurement id. """ 182 """ Download optical files for the measurement id. """
178 # Construct the download url 183 # Construct the download url
179 download_url = DOWNLOAD_OPTICAL.format(measurement_id) 184 download_url = DOWNLOAD_OPTICAL.format(measurement_id)
180 self.download_files(measurement_id, 'scc_optical', download_url) 185 self.download_files(measurement_id, 'scc_optical', download_url)
181 186
182 def download_graphs(self, measurement_id): 187 def download_graphs(self, measurement_id):
183 """ Download profile graphs for the measurement id. """ 188 """ Download profile graphs for the measurement id. """
184 # Construct the download url 189 # Construct the download url
185 download_url = DOWNLOAD_GRAPH.format(measurement_id) 190 download_url = DOWNLOAD_GRAPH.format(measurement_id)
186 self.download_files(measurement_id, 'scc_plots', download_url) 191 self.download_files(measurement_id, 'scc_plots', download_url)
192 request = self.session.get(measurement.rerun_processing_url, auth=self.auth, 197 request = self.session.get(measurement.rerun_processing_url, auth=self.auth,
193 verify=False, 198 verify=False,
194 stream=True) 199 stream=True)
195 200
196 if request.status_code != 200: 201 if request.status_code != 200:
197 print "Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code) 202 logging.error("Could not rerun processing for %s. Status code: %s" % (measurement_id, request.status_code))
198 return 203 return
199 204
200 if monitor: 205 if monitor:
201 self.monitor_processing(measurement_id) 206 self.monitor_processing(measurement_id)
202 207
203 def rerun_all(self, measurement_id, monitor=True): 208 def rerun_all(self, measurement_id, monitor=True):
209 logger.debug("Started rerun_all procedure.")
210
211 logger.debug("Getting measurement %s" % measurement_id)
204 measurement = self.get_measurement(measurement_id) 212 measurement = self.get_measurement(measurement_id)
205 213
206 if measurement: 214 if measurement:
215 logger.debug("Attempting to rerun all processing through %s." % measurement.rerun_all_url)
216
207 request = self.session.get(measurement.rerun_all_url, auth=self.auth, 217 request = self.session.get(measurement.rerun_all_url, auth=self.auth,
208 verify=False, 218 verify=False,
209 stream=True) 219 stream=True)
210 220
211 if request.status_code != 200: 221 if request.status_code != 200:
212 print "Could not rerun pre processing for %s. Status code: %s" % (measurement_id, request.status_code) 222 logger.error("Could not rerun pre processing for %s. Status code: %s" %
223 (measurement_id, request.status_code))
213 return 224 return
214 225
215 if monitor: 226 if monitor:
216 self.monitor_processing(measurement_id) 227 self.monitor_processing(measurement_id)
217 228
218 def process(self, filename, system_id): 229 def process(self, filename, system_id):
219 """ Upload a file for processing and wait for the processing to finish. 230 """ Upload a file for processing and wait for the processing to finish.
220 If the processing is successful, it will download all produced files. 231 If the processing is successful, it will download all produced files.
221 """ 232 """
222 print "--- Processing started on %s. ---" % datetime.datetime.now() 233 logger.info("--- Processing started on %s. ---" % datetime.datetime.now())
223 # Upload file 234 # Upload file
224 measurement_id = self.upload_file(filename, system_id) 235 measurement_id = self.upload_file(filename, system_id)
225 236
226 measurement = self.monitor_processing(measurement_id) 237 measurement = self.monitor_processing(measurement_id)
227 return measurement 238 return measurement
230 """ Monitor the processing progress of a measurement id""" 241 """ Monitor the processing progress of a measurement id"""
231 242
232 measurement = self.get_measurement(measurement_id) 243 measurement = self.get_measurement(measurement_id)
233 if measurement is not None: 244 if measurement is not None:
234 while measurement.is_running: 245 while measurement.is_running:
235 print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload, 246 logger.info("Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
236 measurement.pre_processing, 247 measurement.pre_processing,
237 measurement.opt_retrievals) 248 measurement.processing))
238 time.sleep(10) 249 time.sleep(10)
239 measurement = self.get_measurement(measurement_id) 250 measurement = self.get_measurement(measurement_id)
240 print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload, 251 logger.info("Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
241 measurement.pre_processing, 252 measurement.pre_processing,
242 measurement.opt_retrievals) 253 measurement.processing))
243 if measurement.pre_processing == 127: 254 if measurement.pre_processing == 127:
244 print "Downloading preprocessed files." 255 logger.info("Downloading preprocessed files.")
245 self.download_preprocessed(measurement_id) 256 self.download_preprocessed(measurement_id)
246 if measurement.opt_retrievals == 127: 257 if measurement.processing == 127:
247 print "Downloading optical files." 258 logger.info("Downloading optical files.")
248 self.download_optical(measurement_id) 259 self.download_optical(measurement_id)
249 print "Downloading graphs." 260 logger.info("Downloading graphs.")
250 self.download_graphs(measurement_id) 261 self.download_graphs(measurement_id)
251 print "--- Processing finished. ---" 262 logger.info("--- Processing finished. ---")
252 return measurement 263 return measurement
253 264
254 def get_status(self, measurement_id): 265 def get_status(self, measurement_id):
255 """ Get the processing status for a measurement id through the API. """ 266 """ Get the processing status for a measurement id through the API. """
256 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id) 267 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id)
257 268
258 response = self.session.get(measurement_url, 269 response = self.session.get(measurement_url,
259 auth = self.auth, 270 auth=self.auth,
260 verify = False) 271 verify=False)
261 272
262 response_dict = response.json() 273 response_dict = response.json()
263 274
264 if response_dict['objects']: 275 if response_dict['objects']:
265 measurement_list = response_dict['objects'] 276 measurement_list = response_dict['objects']
266 measurement = Measurement(measurement_list[0]) 277 measurement = Measurement(measurement_list[0])
267 return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals) 278 return (measurement.upload, measurement.pre_processing, measurement.processing)
268 else: 279 else:
269 print "No measurement with id %s found on the SCC." % measurement_id 280 logger.error("No measurement with id %s found on the SCC." % measurement_id)
270 return None 281 return None
271 282
272 def get_measurement(self, measurement_id): 283 def get_measurement(self, measurement_id):
273 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id) 284 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id)
274 285
275 response = self.session.get(measurement_url, 286 response = self.session.get(measurement_url,
276 auth = self.auth, 287 auth=self.auth,
277 verify = False) 288 verify=False)
278 289
279 response_dict = response.json() 290 response_dict = response.json()
280 291
281 if response_dict: 292 if response_dict:
282 measurement = Measurement(response_dict) 293 measurement = Measurement(response_dict)
283 return measurement 294 return measurement
284 else: 295 else:
285 print "No measurement with id %s found on the SCC." % measurement_id 296 logger.error("No measurement with id %s found on the SCC." % measurement_id)
286 return None 297 return None
287 298
288 def delete_measurement(self, measurement_id): 299 def delete_measurement(self, measurement_id):
289 """ Deletes a measurement with the provided measurement id. The user 300 """ Deletes a measurement with the provided measurement id. The user
290 should have the appropriate permissions. 301 should have the appropriate permissions.
291 302
292 The procedures is performed directly through the web interface and 303 The procedures is performed directly through the web interface and
293 NOT through the API. 304 NOT through the API.
294 """ 305 """
295 # Get the measurement object 306 # Get the measurement object
296 measurement = self.get_measurement(measurement_id) 307 measurement = self.get_measurement(measurement_id)
297 308
298 # Check that it exists 309 # Check that it exists
299 if measurement is None: 310 if measurement is None:
300 print "Nothing to delete." 311 logger.warning("Nothing to delete.")
301 return None 312 return None
302 313
303 # Go the the page confirming the deletion 314 # Go the the page confirming the deletion
304 delete_url = DELETE_MEASUREMENT.format(measurement.id) 315 delete_url = DELETE_MEASUREMENT.format(measurement.id)
305 316
306 confirm_page = self.session.get(delete_url, 317 confirm_page = self.session.get(delete_url,
307 auth = self.auth, 318 auth=self.auth,
308 verify = False) 319 verify=False)
309 320
310 # Check that the page opened properly 321 # Check that the page opened properly
311 if confirm_page.status_code != 200: 322 if confirm_page.status_code != 200:
312 print "Could not open delete page. Status: {0}".format(confirm_page.status_code) 323 logger.warning("Could not open delete page. Status: {0}".format(confirm_page.status_code))
313 return None 324 return None
314 325
315 # Delete the measurement 326 # Delete the measurement
316 delete_page = self.session.post(delete_url, 327 delete_page = self.session.post(delete_url,
317 auth=self.auth, 328 auth=self.auth,
318 verify=False, 329 verify=False,
319 data={'post':'yes'}, 330 data={'post': 'yes'},
320 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'], 331 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
321 'referer': delete_url} 332 'referer': delete_url}
322 ) 333 )
323 if delete_page.status_code != 200: 334 if delete_page.status_code != 200:
324 print "Something went wrong. Delete page status: {0}".format( 335 logger.warning("Something went wrong. Delete page status: {0}".format(
325 delete_page.status_code) 336 delete_page.status_code))
326 return None 337 return None
327 338
328 print "Deleted measurement {0}".format(measurement_id) 339 logger.info("Deleted measurement {0}".format(measurement_id))
329 return True 340 return True
330 341
331 def available_measurements(self): 342 def available_measurements(self):
332 """ Get a list of available measurement on the SCC. """ 343 """ Get a list of available measurement on the SCC. """
333 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements') 344 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements')
334 response = self.session.get(measurement_url, 345 response = self.session.get(measurement_url,
335 auth = self.auth, 346 auth=self.auth,
336 verify = False) 347 verify=False)
337 response_dict = response.json() 348 response_dict = response.json()
338 349
339 measurements = None 350 measurements = None
340 if response_dict: 351 if response_dict:
341 measurement_list = response_dict['objects'] 352 measurement_list = response_dict['objects']
342 measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list] 353 measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list]
343 print "Found %s measurements on the SCC." % len(measurements) 354 logger.info("Found %s measurements on the SCC." % len(measurements))
344 else: 355 else:
345 print "No response received from the SCC when asked for available measurements." 356 logger.warning("No response received from the SCC when asked for available measurements.")
346 357
347 return measurements 358 return measurements
348 359
349 def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0): 360 def measurement_id_for_date(self, t1, call_sign='bu', base_number=0):
350 """ Give the first available measurement id on the SCC for the specific 361 """ Give the first available measurement id on the SCC for the specific
351 date. 362 date.
352 """ 363 """
353 date_str = t1.strftime('%Y%m%d') 364 date_str = t1.strftime('%Y%m%d')
354 search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str) 365 search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str)
355 366
356 response = self.session.get(search_url, 367 response = self.session.get(search_url,
357 auth = self.auth, 368 auth=self.auth,
358 verify = False) 369 verify=False)
359 370
360 response_dict = response.json() 371 response_dict = response.json()
361 372
362 measurement_id = None 373 measurement_id = None
363 374
364 if response_dict: 375 if response_dict:
365 measurement_list = response_dict['objects'] 376 measurement_list = response_dict['objects']
366 existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list] 377 existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
367 378
368 measurement_number = base_number 379 measurement_number = base_number
369 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) 380 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
370 381
371 while measurement_id in existing_ids: 382 while measurement_id in existing_ids:
372 measurement_number = measurement_number + 1 383 measurement_number = measurement_number + 1
373 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number) 384 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
374 if measurement_number == 100: 385 if measurement_number == 100:
375 raise ValueError('No available measurement id found.') 386 raise ValueError('No available measurement id found.')
376 387
377 return measurement_id 388 return measurement_id
378 389
379 390
380 class ApiObject: 391 class ApiObject:
381 """ A generic class object. """ 392 """ A generic class object. """
382 393
383 def __init__(self, dict_response): 394 def __init__(self, dict_response):
384 395
385 if dict_response: 396 if dict_response:
386 # Add the dictionary key value pairs as object properties 397 # Add the dictionary key value pairs as object properties
387 for key, value in dict_response.items(): 398 for key, value in dict_response.items():
388 setattr(self, key, value) 399 setattr(self, key, value)
389 self.exists = True 400 self.exists = True
392 403
393 404
394 class Measurement(ApiObject): 405 class Measurement(ApiObject):
395 """ This class represents the measurement object as returned in the SCC API. 406 """ This class represents the measurement object as returned in the SCC API.
396 """ 407 """
408
397 @property 409 @property
398 def is_running(self): 410 def is_running(self):
399 """ Returns True if the processing has not finished. 411 """ Returns True if the processing has not finished.
400 """ 412 """
401 if self.upload == 0: 413 if self.upload == 0:
402 return False 414 return False
403 if self.pre_processing == -127: 415 if self.pre_processing == -127:
404 return False 416 return False
405 if self.pre_processing == 127: 417 if self.pre_processing == 127:
406 if self.opt_retrievals in [127, -127]: 418 if self.processing in [127, -127]:
407 return False 419 return False
408 return True 420 return True
409 421
422 @property
410 def rerun_processing_url(self): 423 def rerun_processing_url(self):
411 return RERUN_PROCESSING.format(self.id) 424 return RERUN_PROCESSING.format(self.id)
412 425
426 @property
413 def rerun_all_url(self): 427 def rerun_all_url(self):
414 return RERUN_ALL.format(self.id) 428 return RERUN_ALL.format(self.id)
415 429
416 def __str__(self): 430 def __str__(self):
417 return "%s: %s, %s, %s" % (self.id, 431 return "%s: %s, %s, %s" % (self.id,
418 self.upload, 432 self.upload,
419 self.pre_processing, 433 self.pre_processing,
420 self.opt_retrievals) 434 self.processing)
421 435
422 436
423 def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): 437 def upload_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
424 """ Shortcut function to upload a file to the SCC. """ 438 """ Shortcut function to upload a file to the SCC. """
439 logger.info("Uploading file %s, using sytem %s" % (filename, system_id))
440
425 scc = SCC(auth) 441 scc = SCC(auth)
426 scc.login(credential) 442 scc.login(credential)
427 measurement_id = scc.upload_file(filename, system_id) 443 measurement_id = scc.upload_file(filename, system_id)
428 scc.logout() 444 scc.logout()
429 return measurement_id 445 return measurement_id
430 446
431 def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): 447
448 def process_file(filename, system_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
432 """ Shortcut function to process a file to the SCC. """ 449 """ Shortcut function to process a file to the SCC. """
450 logger.info("Processing file %s, using sytem %s" % (filename, system_id))
451
433 scc = SCC(auth) 452 scc = SCC(auth)
434 scc.login(credential) 453 scc.login(credential)
435 measurement = scc.process(filename, system_id) 454 measurement = scc.process(filename, system_id)
436 scc.logout() 455 scc.logout()
437 return measurement 456 return measurement
438 457
439 def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): 458
459 def delete_measurement(measurement_id, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
440 """ Shortcut function to delete a measurement from the SCC. """ 460 """ Shortcut function to delete a measurement from the SCC. """
461 logger.info("Deleting %s" % measurement_id)
441 scc = SCC(auth) 462 scc = SCC(auth)
442 scc.login(credential) 463 scc.login(credential)
443 scc.delete_measurement(measurement_id) 464 scc.delete_measurement(measurement_id)
444 scc.logout() 465 scc.logout()
445 466
446 def rerun_all(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): 467
468 def rerun_all(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
447 """ Shortcut function to delete a measurement from the SCC. """ 469 """ Shortcut function to delete a measurement from the SCC. """
470 logger.info("Rerunning all products for %s" % measurement_id)
448 scc = SCC(auth) 471 scc = SCC(auth)
449 scc.login(credential) 472 scc.login(credential)
450 scc.rerun_all(measurement_id, monitor) 473 scc.rerun_all(measurement_id, monitor)
451 scc.logout() 474 scc.logout()
452 475
453 def rerun_processing(measurement_id, monitor, auth = BASIC_LOGIN, credential = DJANGO_LOGIN): 476
477 def rerun_processing(measurement_id, monitor, auth=BASIC_LOGIN, credential=DJANGO_LOGIN):
454 """ Shortcut function to delete a measurement from the SCC. """ 478 """ Shortcut function to delete a measurement from the SCC. """
479 logger.info("Rerunning (optical) processing for %s" % measurement_id)
455 scc = SCC(auth) 480 scc = SCC(auth)
456 scc.login(credential) 481 scc.login(credential)
457 scc.rerun_processing(measurement_id, monitor) 482 scc.rerun_processing(measurement_id, monitor)
458 scc.logout() 483 scc.logout()
459 484
485
460 # When running through terminal 486 # When running through terminal
461 if __name__ == '__main__': 487 if __name__ == '__main__':
462 488
463 # Define the command line arguments. 489 # Define the command line arguments.
464 parser = argparse.ArgumentParser() 490 parser = argparse.ArgumentParser()
465 parser.add_argument("filename", nargs='?', help = "Measurement file name or path.", default='') 491 parser.add_argument("filename", nargs='?', help="Measurement file name or path.", default='')
466 parser.add_argument("system", nargs='?', help = "Processing system id.", default=0) 492 parser.add_argument("system", nargs='?', help="Processing system id.", default=0)
467 parser.add_argument("-p", "--process", help="Wait for the results of the processing.", 493 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
468 action="store_true") 494 action="store_true")
469 parser.add_argument("-d", "--delete", help="Measurement ID to delete.") 495 parser.add_argument("--delete", help="Measurement ID to delete.")
470 parser.add_argument("--rerun-all", help="Measurement ID to rerun.") 496 parser.add_argument("--rerun-all", help="Measurement ID to rerun.")
471 parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.") 497 parser.add_argument("--rerun-processing", help="Measurement ID to rerun processing routings.")
498
499 # Verbosity settings from http://stackoverflow.com/a/20663028
500 parser.add_argument('-d', '--debug', help="Print debugging information.", action="store_const",
501 dest="loglevel", const=logging.DEBUG, default=logging.INFO,
502 )
503 parser.add_argument('-s', '--silent', help="Show only warning and error messages.", action="store_const",
504 dest="loglevel", const=logging.WARNING
505 )
506
472 args = parser.parse_args() 507 args = parser.parse_args()
473 508
509 # Get the logger with the appropriate level
510 logging.basicConfig(format='%(levelname)s: %(message)s', level=args.loglevel)
511
474 # If the arguments are OK, try to login on the site and upload. 512 # If the arguments are OK, try to login on the site and upload.
475 if args.delete: 513 if args.delete:
476 # If the delete is provided, do nothing else 514 # If the delete is provided, do nothing else
477 delete_measurement(args.delete) 515 delete_measurement(args.delete)
478 elif args.rerun_all: 516 elif args.rerun_all:
479 rerun_all(args.rerun_all, args.process) 517 rerun_all(args.rerun_all, args.process)
480 elif args.rerun_processing: 518 elif args.rerun_processing:
481 rerun_processing(args.rerun_processing, args.process) 519 rerun_processing(args.rerun_processing, args.process)
482 else: 520 else:
483 if (args.filename == '') or (args.system == 0): 521 if (args.filename == '') or (args.system == 0):
484 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n') 522 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
485 523
486 if args.process: 524 if args.process:
487 process_file(args.filename, args.system) 525 process_file(args.filename, args.system)
488 else: 526 else:
489 upload_file(args.filename, args.system) 527 upload_file(args.filename, args.system)

EARLINET Repository: SCC access

mercurial