Mercurial > public > scc_access / file comparison
comparison: scc_access/scc_access.py
scc_access/scc_access.py
- changeset 69
- cd8bc07f8419
- parent 68
- 70c869fa3242
- child 70
- 4ef6f2102a61
equal
deleted
inserted
replaced
| 68:70c869fa3242 | 69:cd8bc07f8419 |
|---|---|
| 121 | 121 |
| 122 def logout(self): | 122 def logout(self): |
| 123 """ Logout from the SCC """ | 123 """ Logout from the SCC """ |
| 124 return self.session.get(self.logout_url, stream=True) | 124 return self.session.get(self.logout_url, stream=True) |
| 125 | 125 |
| 126 def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, rs_filename=None, | 126 def upload_file(self, filename, system_id, force_upload, delete_related, delay=0, cloudfree=False, |
| 127 ov_filename=None, lr_filename=None): | 127 rs_filename=None, ov_filename=None, lr_filename=None): |
| 128 """ Upload a file for processing. | 128 """ Upload a file for processing. |
| 129 | 129 |
| 130 If the upload is successful, it returns the measurement id. | 130 If the upload is successful, it returns the measurement id. |
| 131 | 131 |
| 132 | 132 |
| 140 If True, if a measurement with the same ID is found on the server, it will be first deleted and the | 140 If True, if a measurement with the same ID is found on the server, it will be first deleted and the |
| 141 file current file will be uploaded. If False, the file will not be uploaded if the measurement ID is | 141 file current file will be uploaded. If False, the file will not be uploaded if the measurement ID is |
| 142 already present on the SCC server. | 142 already present on the SCC server. |
| 143 delete_related : bool | 143 delete_related : bool |
| 144 Answer to delete related question when deleting existing measurements from the SCC. | 144 Answer to delete related question when deleting existing measurements from the SCC. |
| 145 cloudfree : bool | |
| 146 Manually set the measurement as cloud free | |
| 145 rs_filename, ov_filename, lr_filename : str | 147 rs_filename, ov_filename, lr_filename : str |
| 146 Ancillary files pahts to be uploaded. | 148 Ancillary files pahts to be uploaded. |
| 147 """ | 149 """ |
| 148 # Get the measurement ID from the netcdf file | 150 # Get the measurement ID from the netcdf file |
| 149 measurement_id = self.measurement_id_from_file(filename) | 151 measurement_id = self.measurement_id_from_file(filename) |
| 170 # Get submit page | 172 # Get submit page |
| 171 upload_page = self.session.get(self.upload_url) | 173 upload_page = self.session.get(self.upload_url) |
| 172 | 174 |
| 173 # Submit the data | 175 # Submit the data |
| 174 upload_data = {'system': system_id, | 176 upload_data = {'system': system_id, |
| 175 'delay': delay} | 177 'delay': delay, |
| 176 | 178 'manually_cloud_free': cloudfree, } |
| 177 logger.debug("Submitted processing parameters - System: {}, Delay: {}".format(system_id, delay)) | 179 |
| 180 logger.debug("Submitted processing parameters - System: {}, Delay: {}, Manually Cloud Free: {}".format(system_id, delay, cloudfree)) | |
| 178 | 181 |
| 179 files = {'data': open(filename, 'rb')} | 182 files = {'data': open(filename, 'rb')} |
| 180 | 183 |
| 181 # Add ancillary files to be uploaded | 184 # Add ancillary files to be uploaded |
| 182 if rs_filename is not None: | 185 if rs_filename is not None: |
| 404 logger.info("Rerun-all command submitted successfully for id {}.".format(measurement_id)) | 407 logger.info("Rerun-all command submitted successfully for id {}.".format(measurement_id)) |
| 405 | 408 |
| 406 if monitor: | 409 if monitor: |
| 407 self.monitor_processing(measurement_id) | 410 self.monitor_processing(measurement_id) |
| 408 | 411 |
| 409 def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, rs_filename=None, | 412 def process(self, filename, system_id, monitor, force_upload, delete_related, delay=0, cloudfree=False, |
| 410 lr_filename=None, ov_filename=None): | 413 rs_filename=None, lr_filename=None, ov_filename=None): |
| 411 """ Upload a file for processing and wait for the processing to finish. | 414 """ Upload a file for processing and wait for the processing to finish. |
| 412 If the processing is successful, it will download all produced files. | 415 If the processing is successful, it will download all produced files. |
| 413 """ | 416 """ |
| 414 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) | 417 logger.info("--- Processing started on %s. ---" % datetime.datetime.now()) |
| 415 # Upload file | 418 # Upload file |
| 416 logger.info("Uploading file.") | 419 logger.info("Uploading file.") |
| 417 measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, | 420 measurement_id = self.upload_file(filename, system_id, force_upload, delete_related, |
| 418 delay=delay, | 421 delay=delay, |
| 422 cloudfree=cloudfree, | |
| 419 rs_filename=rs_filename, | 423 rs_filename=rs_filename, |
| 420 lr_filename=lr_filename, | 424 lr_filename=lr_filename, |
| 421 ov_filename=ov_filename) | 425 ov_filename=ov_filename) |
| 422 | 426 |
| 423 if monitor and (delay > 0): | 427 if monitor and (delay > 0): |
| 903 self.status) | 907 self.status) |
| 904 | 908 |
| 905 | 909 |
| 906 # Methods that use the SCC class to perform specific tasks. | 910 # Methods that use the SCC class to perform specific tasks. |
| 907 def process_file(filename, system_id, settings, force_upload, delete_related, | 911 def process_file(filename, system_id, settings, force_upload, delete_related, |
| 908 delay=0, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): | 912 delay=0, cloudfree=False, monitor=True, rs_filename=None, lr_filename=None, ov_filename=None): |
| 909 """ Shortcut function to process a file to the SCC. """ | 913 """ Shortcut function to process a file to the SCC. """ |
| 910 logger.info("Processing file %s, using system %s" % (filename, system_id)) | 914 logger.info("Processing file %s, using system %s" % (filename, system_id)) |
| 911 | 915 |
| 912 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: | 916 with SCC(settings['basic_credentials'], settings['output_dir'], settings['base_url']) as scc: |
| 913 scc.login(settings['website_credentials']) | 917 scc.login(settings['website_credentials']) |
| 914 measurement = scc.process(filename, system_id, | 918 measurement = scc.process(filename, system_id, |
| 915 force_upload=force_upload, | 919 force_upload=force_upload, |
| 916 delete_related=delete_related, | 920 delete_related=delete_related, |
| 917 delay=delay, | 921 delay=delay, |
| 922 cloudfree=cloudfree, | |
| 918 monitor=monitor, | 923 monitor=monitor, |
| 919 rs_filename=rs_filename, | 924 rs_filename=rs_filename, |
| 920 lr_filename=lr_filename, | 925 lr_filename=lr_filename, |
| 921 ov_filename=ov_filename) | 926 ov_filename=ov_filename) |
| 922 scc.logout() | 927 scc.logout() |
| 1063 """ Upload but do not monitor processing progress. """ | 1068 """ Upload but do not monitor processing progress. """ |
| 1064 | 1069 |
| 1065 def upload_file_from_args(parsed): | 1070 def upload_file_from_args(parsed): |
| 1066 process_file(parsed.filename, parsed.system, parsed.config, | 1071 process_file(parsed.filename, parsed.system, parsed.config, |
| 1067 delay=parsed.delay, | 1072 delay=parsed.delay, |
| 1073 cloudfree=parsed.cloudfree, | |
| 1068 monitor=parsed.process, | 1074 monitor=parsed.process, |
| 1069 force_upload=parsed.force_upload, | 1075 force_upload=parsed.force_upload, |
| 1070 delete_related=False, # For now, use this as default | 1076 delete_related=False, # For now, use this as default |
| 1071 rs_filename=parsed.radiosounding, | 1077 rs_filename=parsed.radiosounding, |
| 1072 ov_filename=parsed.overlap, | 1078 ov_filename=parsed.overlap, |
| 1085 | 1091 |
| 1086 parser.add_argument("filename", help="Measurement file name or path.") | 1092 parser.add_argument("filename", help="Measurement file name or path.") |
| 1087 parser.add_argument("system", help="Processing system id.") | 1093 parser.add_argument("system", help="Processing system id.") |
| 1088 parser.add_argument("--delay", help="Delay processing by the specified number of hours (0 to 96).", | 1094 parser.add_argument("--delay", help="Delay processing by the specified number of hours (0 to 96).", |
| 1089 default=0, type=delay) | 1095 default=0, type=delay) |
| 1096 parser.add_argument("--cloudfree", help="Manually assume this measurement as cloud free.", | |
| 1097 action="store_true", default=False) | |
| 1090 parser.add_argument("-p", "--process", help="Wait for the processing results.", | 1098 parser.add_argument("-p", "--process", help="Wait for the processing results.", |
| 1091 action="store_true") | 1099 action="store_true") |
| 1092 parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.", | 1100 parser.add_argument("--force_upload", help="If measurement ID exists on SCC, delete before uploading.", |
| 1093 action="store_true") | 1101 action="store_true") |
| 1094 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") | 1102 parser.add_argument("--radiosounding", default=None, help="Radiosounding file name or path") |
