Mercurial > public > scc_access / file comparison
comparison: scc_access/scc_access.py
scc_access/scc_access.py
- changeset 50
- 1ecbb8de2b39
- parent 45
- 6574da43a481
- child 51
- a4ca2b6d67f5
equal
deleted
inserted
replaced
| 49:3c0b9e97b442 | 50:1ecbb8de2b39 |
|---|---|
| 104 """ Upload a filename for processing with a specific system. If the | 104 """ Upload a filename for processing with a specific system. If the |
| 105 upload is successful, it returns the measurement id. """ | 105 upload is successful, it returns the measurement id. """ |
| 106 measurement_id = self.measurement_id_from_file(filename) | 106 measurement_id = self.measurement_id_from_file(filename) |
| 107 | 107 |
| 108 logger.debug('Checking if a measurement with the same id already exists on the SCC server.') | 108 logger.debug('Checking if a measurement with the same id already exists on the SCC server.') |
| 109 existing_measurement = self.get_measurement(measurement_id) | 109 existing_measurement, _ = self.get_measurement(measurement_id) |
| 110 | 110 |
| 111 if existing_measurement: | 111 if existing_measurement: |
| 112 if force_upload: | 112 if force_upload: |
| 113 logger.info( | 113 logger.info( |
| 114 "Measurement with id {} already exists on the SCC. Trying to delete it...".format(measurement_id)) | 114 "Measurement with id {} already exists on the SCC. Trying to delete it...".format(measurement_id)) |
| 152 "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) | 152 "Lidar ratio file {0.filename} already on the SCC with id {0.id}. Ignoring it.".format(ancillary_file)) |
| 153 else: | 153 else: |
| 154 logger.debug('Adding lidar ratio file %s' % lr_filename) | 154 logger.debug('Adding lidar ratio file %s' % lr_filename) |
| 155 files['lidar_ratio_file'] = open(lr_filename, 'rb') | 155 files['lidar_ratio_file'] = open(lr_filename, 'rb') |
| 156 | 156 |
| 157 logger.info("Uploading of file(s) %s started." % filename) | 157 logger.info("Uploading of file %s started." % filename) |
| 158 | 158 |
| 159 upload_submit = self.session.post(self.upload_url, | 159 upload_submit = self.session.post(self.upload_url, |
| 160 data=upload_data, | 160 data=upload_data, |
| 161 files=files, | 161 files=files, |
| 162 headers={'X-CSRFToken': upload_page.cookies['csrftoken'], | 162 headers={'X-CSRFToken': upload_page.cookies['csrftoken'], |
| 233 def download_hirelpp(self, measurement_id): | 233 def download_hirelpp(self, measurement_id): |
| 234 """ Download hirelpp files for the measurement id. """ | 234 """ Download hirelpp files for the measurement id. """ |
| 235 # Construct the download url | 235 # Construct the download url |
| 236 download_url = self.download_hirelpp_pattern.format(measurement_id) | 236 download_url = self.download_hirelpp_pattern.format(measurement_id) |
| 237 try: | 237 try: |
| 238 self.download_files(measurement_id, 'scc_hirelpp', download_url) | 238 self.download_files(measurement_id, 'hirelpp', download_url) |
| 239 except Exception as e: | 239 except Exception as e: |
| 240 logger.error("Could not download HiRElPP files. Error message: {}".format(e)) | 240 logger.error("Could not download HiRElPP files. Error message: {}".format(e)) |
| 241 logger.debug('Download exception:', exc_info=True) | 241 logger.debug('Download exception:', exc_info=True) |
| 242 | 242 |
| 243 def download_cloudmask(self, measurement_id): | 243 def download_cloudmask(self, measurement_id): |
| 244 """ Download cloudmask files for the measurement id. """ | 244 """ Download cloudmask files for the measurement id. """ |
| 245 # Construct the download url | 245 # Construct the download url |
| 246 download_url = self.download_cloudmask_pattern.format(measurement_id) | 246 download_url = self.download_cloudmask_pattern.format(measurement_id) |
| 247 try: | 247 try: |
| 248 self.download_files(measurement_id, 'scc_cloudscreen', download_url) | 248 self.download_files(measurement_id, 'cloudscreen', download_url) |
| 249 except Exception as e: | 249 except Exception as e: |
| 250 logger.error("Could not download cloudscreen files. Error message: {}".format(e)) | 250 logger.error("Could not download cloudscreen files. Error message: {}".format(e)) |
| 251 logger.debug('Download exception:', exc_info=True) | 251 logger.debug('Download exception:', exc_info=True) |
| 252 | 252 |
| 253 def download_elpp(self, measurement_id): | 253 def download_elpp(self, measurement_id): |
| 254 """ Download preprocessed files for the measurement id. """ | 254 """ Download preprocessed files for the measurement id. """ |
| 255 # Construct the download url | 255 # Construct the download url |
| 256 download_url = self.download_elpp_pattern.format(measurement_id) | 256 download_url = self.download_elpp_pattern.format(measurement_id) |
| 257 try: | 257 try: |
| 258 self.download_files(measurement_id, 'scc_preprocessed', download_url) | 258 self.download_files(measurement_id, 'elpp', download_url) |
| 259 except Exception as e: | 259 except Exception as e: |
| 260 logger.error("Could not download ElPP files. Error message: {}".format(e)) | 260 logger.error("Could not download ElPP files. Error message: {}".format(e)) |
| 261 logger.debug('Download exception:', exc_info=True) | 261 logger.debug('Download exception:', exc_info=True) |
| 262 | 262 |
| 263 def download_elda(self, measurement_id): | 263 def download_elda(self, measurement_id): |
| 264 """ Download optical files for the measurement id. """ | 264 """ Download optical files for the measurement id. """ |
| 265 # Construct the download url | 265 # Construct the download url |
| 266 download_url = self.download_elda_pattern.format(measurement_id) | 266 download_url = self.download_elda_pattern.format(measurement_id) |
| 267 try: | 267 try: |
| 268 self.download_files(measurement_id, 'scc_optical', download_url) | 268 self.download_files(measurement_id, 'elda', download_url) |
| 269 except Exception as e: | 269 except Exception as e: |
| 270 logger.error("Could not download ELDA files. Error message: {}".format(e)) | 270 logger.error("Could not download ELDA files. Error message: {}".format(e)) |
| 271 logger.debug('Download exception:', exc_info=True) | 271 logger.debug('Download exception:', exc_info=True) |
| 272 | 272 |
| 273 def download_plots(self, measurement_id): | 273 def download_plots(self, measurement_id): |
| 274 """ Download profile graphs for the measurement id. """ | 274 """ Download profile graphs for the measurement id. """ |
| 275 # Construct the download url | 275 # Construct the download url |
| 276 download_url = self.download_plots_pattern.format(measurement_id) | 276 download_url = self.download_plots_pattern.format(measurement_id) |
| 277 try: | 277 try: |
| 278 self.download_files(measurement_id, 'scc_plots', download_url) | 278 self.download_files(measurement_id, 'elda_plots', download_url) |
| 279 except Exception as e: | 279 except Exception as e: |
| 280 logger.error("Could not download ELDA plots. Error message: {}".format(e)) | 280 logger.error("Could not download ELDA plots. Error message: {}".format(e)) |
| 281 logger.debug('Download exception:', exc_info=True) | 281 logger.debug('Download exception:', exc_info=True) |
| 282 | 282 |
| 283 def download_elic(self, measurement_id): | 283 def download_elic(self, measurement_id): |
| 284 """ Download ELIC files for the measurement id. """ | 284 """ Download ELIC files for the measurement id. """ |
| 285 # Construct the download url | 285 # Construct the download url |
| 286 download_url = self.download_elic_pattern.format(measurement_id) | 286 download_url = self.download_elic_pattern.format(measurement_id) |
| 287 try: | 287 try: |
| 288 self.download_files(measurement_id, 'scc_elic', download_url) | 288 self.download_files(measurement_id, 'elic', download_url) |
| 289 except Exception as e: | 289 except Exception as e: |
| 290 logger.error("Could not download ELIC files. Error message: {}".format(e)) | 290 logger.error("Could not download ELIC files. Error message: {}".format(e)) |
| 291 logger.debug('Download exception:', exc_info=True) | 291 logger.debug('Download exception:', exc_info=True) |
| 292 | 292 |
| 293 def download_eldec(self, measurement_id): | 293 def download_eldec(self, measurement_id): |
| 294 """ Download ELDEC files for the measurement id. """ | 294 """ Download ELDEC files for the measurement id. """ |
| 295 # Construct the download url | 295 # Construct the download url |
| 296 download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now | 296 download_url = self.download_elda_pattern.format(measurement_id) # ELDA patter is used for now |
| 297 try: | 297 try: |
| 298 self.download_files(measurement_id, 'scc_eldec', download_url) | 298 self.download_files(measurement_id, 'eldec', download_url) |
| 299 except Exception as e: | 299 except Exception as e: |
| 300 logger.error("Could not download EDELC files. Error message: {}".format(e)) | 300 logger.error("Could not download EDELC files. Error message: {}".format(e)) |
| 301 logger.debug('Download exception:', exc_info=True) | 301 logger.debug('Download exception:', exc_info=True) |
| 302 | 302 |
| 303 def rerun_elpp(self, measurement_id, monitor=True): | 303 def rerun_elpp(self, measurement_id, monitor=True): |
| 343 """ Upload a file for processing and wait for the processing to finish. | 343 """ Upload a file for processing and wait for the processing to finish. |
| 344 If the processing is successful, it will download all produced files. | 344 If the processing is successful, it will download all produced files. |
| 345 """ | 345 """ |
| 346 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) | 346 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) |
| 347 # Upload file | 347 # Upload file |
| 348 logger.info("--- Uploading file") | 348 logger.info("Uploading file.") |
| 349 measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, | 349 measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, |
| 350 rs_filename=rs_filename, | 350 rs_filename=rs_filename, |
| 351 lr_filename=lr_filename, | 351 lr_filename=lr_filename, |
| 352 ov_filename=ov_filename) | 352 ov_filename=ov_filename) |
| 353 | 353 |
| 354 if measurement_id and monitor: | 354 if measurement_id and monitor: |
| 355 logger.info("--- Monitoring processing") | 355 logger.info("Monitoring processing") |
| 356 return self.monitor_processing(measurement_id) | 356 return self.monitor_processing(measurement_id) |
| 357 | 357 |
| 358 return None | 358 return None |
| 359 | 359 |
| 360 def monitor_processing(self, measurement_id): | 360 def monitor_processing(self, measurement_id): |
| 365 error_max = 6 | 365 error_max = 6 |
| 366 time_sleep = 10 | 366 time_sleep = 10 |
| 367 | 367 |
| 368 # try to wait for measurement to appear in API | 368 # try to wait for measurement to appear in API |
| 369 measurement = None | 369 measurement = None |
| 370 logger.info("Looking for measurement %s on SCC", measurement_id) | 370 logger.info("Looking for measurement %s on the SCC.", measurement_id) |
| 371 while error_count < error_max: | 371 while error_count < error_max: |
| 372 time.sleep(time_sleep) | 372 time.sleep(time_sleep) |
| 373 measurement, status = self.get_measurement(measurement_id) | 373 measurement, status = self.get_measurement(measurement_id) |
| 374 if status != 200 and error_count < error_max: | 374 if status != 200 and error_count < error_max: |
| 375 logger.error("Measurement not found. waiting %ds", time_sleep) | 375 logger.error("Measurement not found. waiting %ds", time_sleep) |
| 406 logger.info("Downloading ElPP files.") | 406 logger.info("Downloading ElPP files.") |
| 407 self.download_elpp(measurement_id) | 407 self.download_elpp(measurement_id) |
| 408 if measurement.elda == 127: | 408 if measurement.elda == 127: |
| 409 logger.info("Downloading ELDA files.") | 409 logger.info("Downloading ELDA files.") |
| 410 self.download_elda(measurement_id) | 410 self.download_elda(measurement_id) |
| 411 logger.info("Downloading graphs.") | 411 logger.info("Downloading ELDA plots.") |
| 412 self.download_plots(measurement_id) | 412 self.download_plots(measurement_id) |
| 413 if measurement.elic == 127: | 413 if measurement.elic == 127: |
| 414 logger.info("Downloading ELIC files.") | 414 logger.info("Downloading ELIC files.") |
| 415 self.download_elic(measurement_id) | 415 self.download_elic(measurement_id) |
| 416 | 416 |
