Mercurial > public > scc_access / file comparison

comparison: scc_access.py

scc_access.py

changeset 0
a172f26a566a
child 1
783c8a0db76f
equal deleted inserted replaced
-1:000000000000 0:a172f26a566a
1 #!/usr/bin/env python
2 """
3 The MIT License (MIT)
4
5 Copyright (c) 2015, Ioannis Binietoglou
6
7 Permission is hereby granted, free of charge, to any person obtaining a copy
8 of this software and associated documentation files (the "Software"), to deal
9 in the Software without restriction, including without limitation the rights
10 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 copies of the Software, and to permit persons to whom the Software is
12 furnished to do so, subject to the following conditions:
13
14 The above copyright notice and this permission notice shall be included in
15 all copies or substantial portions of the Software.
16
17 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 THE SOFTWARE.
24 """
25
26 # Try to read the settings from the settings.py file
27 try:
28 from settings import *
29 except:
30 raise ImportError(
31 """A settings file (setting.py) is required to run the script.
32 You can use settings.sample.py for a template.""")
33
34
35 import requests
36 import urlparse
37 import argparse
38 import os
39 import re
40 import time
41 import StringIO
42 from zipfile import ZipFile
43 import datetime
44
45
46 # Construct the absolute URLs
47 LOGIN_URL = urlparse.urljoin(BASE_URL, 'accounts/login/')
48 UPLOAD_URL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/quick/')
49 DOWNLOAD_PREPROCESSED = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-preprocessed/')
50 DOWNLOAD_OPTICAL = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-optical/')
51 DOWNLOAD_GRAPH = urlparse.urljoin(BASE_URL, 'data_processing/measurements/{0}/download-plots/')
52 DELETE_MEASUREMENT = urlparse.urljoin(BASE_URL, 'admin/database/measurements/{0}/delete/')
53 API_BASE_URL = urlparse.urljoin(BASE_URL, 'api/v1/')
54
55 # The regex to find the measurement id from the measurement page
56 # This should be read from the uploaded file, but would require an extra module
57 regex = "<h3>Measurement (?P<measurement_id>.{12}) <small>"
58
59
60 class SCC:
61 """ A simple class that will attempt to upload a file on the SCC server.
62 The uploading is done by simulation a normal browser session. In the current
63 version no check is performed, and no feedback is given if the upload
64 was successful. If everything is setup correctly, it will work.
65 """
66 def __init__(self, auth = BASIC_LOGIN, output_dir = OUTPUT_DIR):
67 self.auth = auth
68 self.output_dir = OUTPUT_DIR
69 self.session = requests.Session()
70
71 def login(self, credential = DJANGO_LOGIN):
72 """ Login the the website. """
73 self.login_credentials = {'username': credential[0],
74 'password': credential[1]}
75
76 # Get upload form
77 login_page = self.session.get(LOGIN_URL,
78 auth = self.auth, verify = False)
79
80 # Submit the login data
81 login_submit = self.session.post(LOGIN_URL,
82 data = self.login_credentials,
83 headers = {'X-CSRFToken': login_page.cookies['csrftoken'],
84 'referer': LOGIN_URL},
85 verify = False,
86 auth = self.auth)
87 return login_submit
88
89 def logout(self):
90 pass
91
92 def upload_file(self, filename, system_id):
93 """ Upload a filename for processing with a specific system. If the
94 upload is successful, it returns the measurement id. """
95 # Get submit page
96 upload_page = self.session.get(UPLOAD_URL,
97 auth = self.auth,
98 verify = False)
99
100 # Submit the data
101 upload_data = {'system': system_id}
102 files = {'data': open(filename, 'rb')}
103
104 print "Uploading of file %s started." % filename
105
106 upload_submit = self.session.post(UPLOAD_URL,
107 data = upload_data,
108 files = files,
109 headers = {'X-CSRFToken': upload_page.cookies['csrftoken'],
110 'referer': UPLOAD_URL},
111 verify = False,
112 auth = self.auth)
113
114 if upload_submit.status_code != 200:
115 print "Connection error. Status code: %s" % upload_submit.status_code
116 return False
117
118 measurement_id = True
119
120 # Check if there was a redirect to a new page.
121 if upload_submit.url == UPLOAD_URL:
122 measurement_id = False
123 print "Uploaded file rejected! Try to upload manually to see the error."
124 else:
125 measurement_id = re.findall(regex, upload_submit.text)[0]
126 print "Successfully uploaded measurement with id %s." % measurement_id
127
128 return measurement_id
129
130 def download_files(self, measurement_id, subdir, download_url):
131 """ Downloads some files from the download_url to the specified
132 subdir. This method is used to download preprocessed file, optical
133 files etc.
134 """
135 # Get the file
136 request = self.session.get(download_url, auth = self.auth,
137 verify = False,
138 stream=True)
139
140 # Create the dir if it does not exist
141 local_dir = os.path.join(self.output_dir, measurement_id, subdir)
142 if not os.path.exists(local_dir):
143 os.makedirs(local_dir)
144
145
146 # Save the file by chunk, needed if the file is big.
147 memory_file = StringIO.StringIO()
148
149 for chunk in request.iter_content(chunk_size=1024):
150 if chunk: # filter out keep-alive new chunks
151 memory_file.write(chunk)
152 memory_file.flush()
153
154 zip_file = ZipFile(memory_file)
155
156 for ziped_name in zip_file.namelist():
157 basename = os.path.basename(ziped_name)
158
159 local_file = os.path.join(local_dir, basename)
160
161 with open(local_file, 'wb') as f:
162 f.write(zip_file.read(ziped_name))
163
164 def download_preprocessed(self, measurement_id):
165 """ Download preprocessed files for the measurement id. """
166 # Construct the download url
167 download_url = DOWNLOAD_PREPROCESSED.format(measurement_id)
168 self.download_files(measurement_id, 'scc_preprocessed', download_url)
169
170 def download_optical(self, measurement_id):
171 """ Download optical files for the measurement id. """
172 # Construct the download url
173 download_url = DOWNLOAD_OPTICAL.format(measurement_id)
174 self.download_files(measurement_id, 'scc_optical', download_url)
175
176 def download_graphs(self, measurement_id):
177 """ Download profile graphs for the measurement id. """
178 # Construct the download url
179 download_url = DOWNLOAD_GRAPH.format(measurement_id)
180 self.download_files(measurement_id, 'scc_plots', download_url)
181
182 def process(self, filename, system_id):
183 """ Upload a file for processing and wait for the processing to finish.
184 If the processing is successful, it will download all produced files.
185 """
186 print "--- Processing started on %s. ---" % datetime.datetime.now()
187 # Upload file
188 measurement_id = self.upload_file(filename, system_id)
189
190 measurement = None
191 if measurement_id:
192 measurement = self.get_measurement(measurement_id)
193 while measurement.is_running:
194 print "Measurement is being processed (status: %s, %s, %s). Please wait." % (measurement.upload,
195 measurement.pre_processing,
196 measurement.opt_retrievals)
197 time.sleep(10)
198 measurement = self.get_measurement(measurement_id)
199 print "Measurement processing finished (status: %s, %s, %s)." % (measurement.upload,
200 measurement.pre_processing,
201 measurement.opt_retrievals)
202 if measurement.pre_processing == 127:
203 print "Downloading preprocessed files."
204 self.download_preprocessed(measurement_id)
205 if measurement.opt_retrievals == 127:
206 print "Downloading optical files."
207 self.download_optical(measurement_id)
208 print "Downloading graphs."
209 self.download_graphs(measurement_id)
210 print "--- Processing finished. ---"
211 return measurement
212
213 def get_status(self, measurement_id):
214 """ Get the processing status for a measurement id through the API. """
215 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__exact=%s' % measurement_id)
216
217 response = self.session.get(measurement_url,
218 auth = self.auth,
219 verify = False)
220
221 response_dict = response.json()
222
223 if response_dict['objects']:
224 measurement_list = response_dict['objects']
225 measurement = Measurement(measurement_list[0])
226 return (measurement.upload, measurement.pre_processing, measurement.opt_retrievals)
227 else:
228 print "No measurement with id %s found on the SCC." % measurement_id
229 return None
230
231 def get_measurement(self, measurement_id):
232 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements/%s/' % measurement_id)
233
234 response = self.session.get(measurement_url,
235 auth = self.auth,
236 verify = False)
237
238 response_dict = response.json()
239
240 if response_dict:
241 measurement = Measurement(response_dict)
242 return measurement
243 else:
244 print "No measurement with id %s found on the SCC." % measurement_id
245 return None
246
247 def delete_measurement(self, measurement_id):
248 """ Deletes a measurement with the provided measurement id. The user
249 should have the appropriate permissions.
250
251 The procedures is performed directly through the web interface and
252 NOT through the API.
253 """
254 # Get the measurement object
255 measurement = self.get_measurement(measurement_id)
256
257 # Check that it exists
258 if measurement is None:
259 print "Nothing to delete."
260 return None
261
262 # Go the the page confirming the deletion
263 delete_url = DELETE_MEASUREMENT.format(measurement.id)
264
265 confirm_page = self.session.get(delete_url,
266 auth = self.auth,
267 verify = False)
268
269 # Check that the page opened properly
270 if confirm_page.status_code != 200:
271 print "Could not open delete page. Status: {0}".format(confirm_page.status_code)
272 return None
273
274 # Delete the measurement
275 delete_page = self.session.post(delete_url,
276 auth=self.auth,
277 verify=False,
278 data={'post':'yes'},
279 headers={'X-CSRFToken': confirm_page.cookies['csrftoken'],
280 'referer': delete_url}
281 )
282 if delete_page.status_code != 200:
283 print "Something went wrong. Delete page status: {0}".format(
284 delete_page.status_code)
285 return None
286
287 print "Deleted measurement {0}".format(measurement_id)
288 return True
289
290 def available_measurements(self):
291 """ Get a list of available measurement on the SCC. """
292 measurement_url = urlparse.urljoin(API_BASE_URL, 'measurements')
293 response = self.session.get(measurement_url,
294 auth = self.auth,
295 verify = False)
296 response_dict = response.json()
297
298 measurements = None
299 if response_dict:
300 measurement_list = response_dict['objects']
301 measurements = [Measurement(measurement_dict) for measurement_dict in measurement_list]
302 print "Found %s measurements on the SCC." % len(measurements)
303 else:
304 print "No response received from the SCC when asked for available measurements."
305
306 return measurements
307
308 def measurement_id_for_date(self, t1, call_sign = 'bu', base_number = 0):
309 """ Give the first available measurement id on the SCC for the specific
310 date.
311 """
312 date_str = t1.strftime('%Y%m%d')
313 search_url = urlparse.urljoin(API_BASE_URL, 'measurements/?id__startswith=%s' % date_str)
314
315 response = self.session.get(search_url,
316 auth = self.auth,
317 verify = False)
318
319 response_dict = response.json()
320
321 measurement_id = None
322
323 if response_dict:
324 measurement_list = response_dict['objects']
325 existing_ids = [measurement_dict['id'] for measurement_dict in measurement_list]
326
327 measurement_number = base_number
328 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
329
330 while measurement_id in existing_ids:
331 measurement_number = measurement_number + 1
332 measurement_id = "%s%s%02i" % (date_str, call_sign, measurement_number)
333 if measurement_number == 100:
334 raise ValueError('No available measurement id found.')
335
336 return measurement_id
337
338
339 class ApiObject:
340 """ A generic class object. """
341
342 def __init__(self, dict_response):
343
344 if dict_response:
345 # Add the dictionary key value pairs as object properties
346 for key, value in dict_response.items():
347 setattr(self, key, value)
348 self.exists = True
349 else:
350 self.exists = False
351
352
353 class Measurement(ApiObject):
354 """ This class represents the measurement object as returned in the SCC API.
355 """
356 @property
357 def is_running(self):
358 """ Returns True if the processing has not finished.
359 """
360 if self.upload == 0:
361 return False
362 if self.pre_processing == -127:
363 return False
364 if self.pre_processing == 127:
365 if self.opt_retrievals in [127, -127]:
366 return False
367 return True
368
369 def delete(self):
370 """ Delete the entry from the SCC database. """
371
372
373 def __str__(self):
374 return "%s: %s, %s, %s" % (self.id,
375 self.upload,
376 self.pre_processing,
377 self.opt_retrievals)
378
379
380 def upload_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
381 """ Shortcut function to upload a file to the SCC. """
382 scc = SCC(auth)
383 scc.login(credential)
384 measurement_id = scc.upload_file(filename, system_id)
385 scc.logout()
386 return measurement_id
387
388 def process_file(filename, system_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
389 """ Shortcut function to process a file to the SCC. """
390 scc = SCC(auth)
391 scc.login(credential)
392 measurement = scc.process(filename, system_id)
393 scc.logout()
394 return measurement
395
396 def delete_measurement(measurement_id, auth = BASIC_LOGIN, credential = DJANGO_LOGIN):
397 """ Shortcut function to delete a measurement from the SCC. """
398 scc = SCC(auth)
399 scc.login(credential)
400 scc.delete_measurement(measurement_id)
401 scc.logout()
402
403 # When running through terminal
404 if __name__ == '__main__':
405
406 # Define the command line arguments.
407 parser = argparse.ArgumentParser()
408 parser.add_argument("filename", nargs='?', help = "Measurement file name or path.", default='')
409 parser.add_argument("system", nargs='?', help = "Processing system id.", default=0)
410 parser.add_argument("-p", "--process", help="Wait for the results of the processing.",
411 action="store_true")
412 parser.add_argument("-d", "--delete", help="Measurement ID to delete.")
413 args = parser.parse_args()
414
415 # If the arguments are OK, try to login on the site and upload.
416 if args.delete:
417 # If the delete is provided, do nothing else
418 delete_measurement(args.delete)
419 else:
420 if (args.filename == '') or (args.system == 0):
421 parser.error('Provide a valid filename and system parameters.\nRun with -h for help.\n')
422
423 if args.process:
424 process_file(args.filename, args.system)
425 else:
426 upload_file(args.filename, args.system)

EARLINET Repository: SCC access

mercurial