Mercurial > public > scc-netcdf-checker / file comparison

comparison: netcdf_checker.py

netcdf_checker.py

changeset 4
5a18d437cd96
parent 2
dcc02b7b6c52
equal deleted inserted replaced
3:5fa09a129470 4:5a18d437cd96
1 import fnmatch
2 import netCDF4 as netcdf
1 import os 3 import os
2 import fnmatch 4
3 import numpy as np 5 import numpy as np
4 import netCDF4 as netcdf
5
6 6
7 ERROR_ORDER = {'notification': 1, 7 ERROR_ORDER = {'notification': 1,
8 'warning': 2, 8 'warning': 2,
9 'error': 3, 9 'error': 3,
10 } 10 }
13 header_template = """\n----- Report for file {0.filename} ----- 13 header_template = """\n----- Report for file {0.filename} -----
14 Checked against specification: {0.specs[name]}. 14 Checked against specification: {0.specs[name]}.
15 15
16 Output summary: 16 Output summary:
17 Errors: {1[error]}, Warnings: {1[warning]}, Notifications: {1[notification]}""" 17 Errors: {1[error]}, Warnings: {1[warning]}, Notifications: {1[notification]}"""
18
19 18
20 19
21 # We firstly define the possible specification elements 20 # We firstly define the possible specification elements
22 # In the end of this file we use them to define complete specifications 21 # In the end of this file we use them to define complete specifications
23 22
24 23
25 class SpecGenericError: 24 class SpecGenericError:
26 def __repr__(self): 25 def __repr__(self):
27 return "{0}: {1}".format(self.level.title(), self.message) 26 return "{0}: {1}".format(self.level.title(), self.message)
28 27
29 28
30 class SpecError(SpecGenericError): 29 class SpecError(SpecGenericError):
31 def __init__(self, message): 30 def __init__(self, message):
32 self.message = message 31 self.message = message
33 self.level = 'error' 32 self.level = 'error'
34 33
47 46
48 class GenericSpecification: 47 class GenericSpecification:
49 @property 48 @property
50 def continue_check(self): 49 def continue_check(self):
51 return True 50 return True
52 51
52
53 class DimensionMandatory(GenericSpecification): 53 class DimensionMandatory(GenericSpecification):
54 def __init__(self, is_mandatory = True): 54 def __init__(self, is_mandatory=True):
55 self.block_next = True # if true the next checks for this dimension will not be performed. 55 self.block_next = True # if true the next checks for this dimension will not be performed.
56 self.is_mandatory = is_mandatory 56 self.is_mandatory = is_mandatory
57 57
58 def check(self, netcdf_file, dimension_name): 58 def check(self, netcdf_file, dimension_name):
59 the_dimension = netcdf_file.dimensions.get(dimension_name, None) 59 the_dimension = netcdf_file.dimensions.get(dimension_name, None)
60 60
61 error = [] 61 error = []
62 if the_dimension: 62 if the_dimension:
63 # If the dimension is found in the file 63 # If the dimension is found in the file
64 self.dimension_exists = True 64 self.dimension_exists = True
65 check_passed = True 65 check_passed = True
66 66
67 else: 67 else:
68 self.dimension_exists = False 68 self.dimension_exists = False
69 if self.is_mandatory: 69 if self.is_mandatory:
70 # If a mandatory dimension is not found in the file 70 # If a mandatory dimension is not found in the file
71 check_passed = False 71 check_passed = False
72 error.append(SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name))) 72 error.append(
73 SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name)))
73 else: 74 else:
74 check_passed = True 75 check_passed = True
75 error.append(SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name))) 76 error.append(
76 77 SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name)))
78
77 self.check_passed = check_passed 79 self.check_passed = check_passed
78 self.error = error 80 self.error = error
79 81
80 return check_passed, error 82 return check_passed, error
81 83
82 @property 84 @property
83 def continue_check(self): 85 def continue_check(self):
84 if (not self.dimension_exists) and (self.block_next): 86 if (not self.dimension_exists) and (self.block_next):
85 return False 87 return False
86 else: 88 else:
87 return True 89 return True
88 90
89 91
90 class DimensionUnlimited(GenericSpecification): 92 class DimensionUnlimited(GenericSpecification):
91 def __init__(self, is_unlimited): 93 def __init__(self, is_unlimited):
92 self.block_next = False 94 self.block_next = False
93 self.is_unlimited = is_unlimited 95 self.is_unlimited = is_unlimited
94 96
95 def check(self, netcdf_file, dimension_name): 97 def check(self, netcdf_file, dimension_name):
96 the_dimension = netcdf_file.dimensions.get(dimension_name, None) 98 the_dimension = netcdf_file.dimensions.get(dimension_name, None)
97 error = [] 99 error = []
98 100
99 if the_dimension: 101 if the_dimension:
100 if the_dimension.isunlimited() == True and self.is_unlimited == False: 102 if the_dimension.isunlimited() == True and self.is_unlimited == False:
101 check_passed = False 103 check_passed = False
102 error.append(SpecWarning('Dimension {0} should not be unlimited but is.'.format(dimension_name))) 104 error.append(SpecWarning('Dimension {0} should not be unlimited but is.'.format(dimension_name)))
103 elif the_dimension.isunlimited() == False and self.is_unlimited == True: 105 elif the_dimension.isunlimited() == False and self.is_unlimited == True:
105 error.append(SpecWarning('Dimension {0} should be unlimited but it is not.'.format(dimension_name))) 107 error.append(SpecWarning('Dimension {0} should be unlimited but it is not.'.format(dimension_name)))
106 else: 108 else:
107 check_passed = True 109 check_passed = True
108 else: 110 else:
109 check_passed = True 111 check_passed = True
110 error.append(SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name))) 112 error.append(
111 113 SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name)))
114
112 self.check_passed = check_passed 115 self.check_passed = check_passed
113 self.error = error 116 self.error = error
114 117
115 return check_passed, error 118 return check_passed, error
116 119
117 120
118 class VariableMandatory(GenericSpecification): 121 class VariableMandatory(GenericSpecification):
119 def __init__(self, is_mandatory = True): 122 def __init__(self, is_mandatory=True):
120 self.block_next = True # if true the next checks for this variable will not be performed. 123 self.block_next = True # if true the next checks for this variable will not be performed.
121 self.is_mandatory = is_mandatory 124 self.is_mandatory = is_mandatory
122 125
123 def check(self, netcdf_file, variable_name): 126 def check(self, netcdf_file, variable_name):
124 the_variable = netcdf_file.variables.get(variable_name, None) 127 the_variable = netcdf_file.variables.get(variable_name, None)
125 error = [] 128 error = []
126 129
127 if the_variable != None: 130 if the_variable != None:
128 # If the variable is found in the file 131 # If the variable is found in the file
129 self.variable_exists = True 132 self.variable_exists = True
130 check_passed = True 133 check_passed = True
131 134
132 else: 135 else:
133 self.variable_exists = False 136 self.variable_exists = False
134 if self.is_mandatory: 137 if self.is_mandatory:
135 # If a mandatory variable is not found in the file 138 # If a mandatory variable is not found in the file
136 check_passed = False 139 check_passed = False
137 error.append(SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name))) 140 error.append(
141 SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name)))
138 else: 142 else:
139 check_passed = True 143 check_passed = True
140 error.append(SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name))) 144 error.append(
141 145 SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name)))
146
142 self.check_passed = check_passed 147 self.check_passed = check_passed
143 self.error = error 148 self.error = error
144 149
145 return check_passed, error 150 return check_passed, error
146 151
147 @property 152 @property
148 def continue_check(self): 153 def continue_check(self):
149 if (not self.variable_exists) and (self.block_next): 154 if (not self.variable_exists) and (self.block_next):
150 return False 155 return False
151 else: 156 else:
152 return True 157 return True
153 158
159
154 class VariableDimensions(GenericSpecification): 160 class VariableDimensions(GenericSpecification):
155 def __init__(self, dimensions): 161 def __init__(self, dimensions):
156 self.dimensions = dimensions 162 self.dimensions = dimensions
157 163
158 def check(self, netcdf_file, variable_name): 164 def check(self, netcdf_file, variable_name):
159 the_variable = netcdf_file.variables.get(variable_name, None) 165 the_variable = netcdf_file.variables.get(variable_name, None)
160 166
161 if the_variable != None: 167 if the_variable != None:
162 variable_dimensions = list(the_variable.dimensions) 168 variable_dimensions = list(the_variable.dimensions)
163 error = [] 169 error = []
164 check_passed = True 170 check_passed = True
165 for dimension in self.dimensions: 171 for dimension in self.dimensions:
166 if not (dimension in variable_dimensions): 172 if not (dimension in variable_dimensions):
167 check_passed = False 173 check_passed = False
168 error.append(SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension))) 174 error.append(
169 175 SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension)))
176
170 # If all dimensions are present, check if the variables are in the 177 # If all dimensions are present, check if the variables are in the
171 # correct order. 178 # correct order.
172 if check_passed: 179 if check_passed:
173 if list(self.dimensions) != variable_dimensions: 180 if list(self.dimensions) != variable_dimensions:
174 check_passed = False 181 check_passed = False
175 error.append(SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name, 182 error.append(
176 variable_dimensions, 183 SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name,
177 list(self.dimensions)))) 184 variable_dimensions,
185 list(
186 self.dimensions))))
178 for dimension in variable_dimensions: 187 for dimension in variable_dimensions:
179 if dimension not in self.dimensions: 188 if dimension not in self.dimensions:
180 error.append(SpecWarning('Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension, variable_name))) 189 error.append(SpecWarning(
181 190 'Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension,
191 variable_name)))
192
182 else: 193 else:
183 check_passed = False 194 check_passed = False
184 error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format(variable_name)),] 195 error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format(
185 196 variable_name)), ]
197
186 self.check_passed = check_passed 198 self.check_passed = check_passed
187 self.error = error 199 self.error = error
188 200
189 return check_passed, error 201 return check_passed, error
190 202
191 203
192 class VariableType(GenericSpecification): 204 class VariableType(GenericSpecification):
193 def __init__(self, dtype): 205 def __init__(self, dtype):
194 self.dtype = dtype 206 self.dtype = dtype
195 207
196 def check(self, netcdf_file, variable_name): 208 def check(self, netcdf_file, variable_name):
197 the_variable = netcdf_file.variables.get(variable_name, None) 209 the_variable = netcdf_file.variables.get(variable_name, None)
198 error = [] 210 error = []
199 211
200 if the_variable != None: 212 if the_variable != None:
201 # Get the internal python type and not the numpy.dtype. 213 # Get the internal python type and not the numpy.dtype.
202 # The conversions guarantee (?) that a single element is always returned 214 # The conversions guarantee (?) that a single element is always returned
203 variable_type_python = type(np.asscalar(np.asarray(np.asarray(the_variable[:]).item(0)))) 215 variable_type_python = type(np.asscalar(np.asarray(np.asarray(the_variable[:]).item(0))))
204 216
205 if not (variable_type_python == self.dtype): 217 if not (variable_type_python == self.dtype):
206 check_passed = False 218 check_passed = False
207 error.append(SpecError('Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype, self.dtype))) 219 error.append(SpecError(
220 'Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype,
221 self.dtype)))
208 else: 222 else:
209 check_passed = True 223 check_passed = True
210 else: 224 else:
211 check_passed = False 225 check_passed = False
212 error.append(SpecError('Variable {0} should be checked for type, but was not found in the file.'.format(variable_name))) 226 error.append(SpecError(
213 227 'Variable {0} should be checked for type, but was not found in the file.'.format(variable_name)))
228
214 self.check_passed = check_passed 229 self.check_passed = check_passed
215 self.error = error 230 self.error = error
216 231
217 return check_passed, error 232 return check_passed, error
218 233
219 234
220 class AttributeMandatory(GenericSpecification): 235 class AttributeMandatory(GenericSpecification):
221 def __init__(self, is_mandatory = True): 236 def __init__(self, is_mandatory=True):
222 self.block_next = True # if true the next checks for this variable will not be performed. 237 self.block_next = True # if true the next checks for this variable will not be performed.
223 self.is_mandatory = is_mandatory 238 self.is_mandatory = is_mandatory
224 239
225 def check(self, netcdf_file, attribute_name): 240 def check(self, netcdf_file, attribute_name):
226 the_attribute = getattr(netcdf_file, attribute_name, None) 241 the_attribute = getattr(netcdf_file, attribute_name, None)
227 error = [] 242 error = []
228 243
229 if the_attribute: 244 if the_attribute:
230 # If the variable is found in the file 245 # If the variable is found in the file
231 self.attribute_exists = True 246 self.attribute_exists = True
232 check_passed = True 247 check_passed = True
233 248
234 else: 249 else:
235 self.attribute_exists = False 250 self.attribute_exists = False
236 if self.is_mandatory: 251 if self.is_mandatory:
237 # If a mandatory variable is not found in the file 252 # If a mandatory variable is not found in the file
238 check_passed = False 253 check_passed = False
239 error.append(SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name))) 254 error.append(
255 SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name)))
240 else: 256 else:
241 check_passed = True 257 check_passed = True
242 error.append(SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name))) 258 error.append(
243 259 SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name)))
260
244 self.check_passed = check_passed 261 self.check_passed = check_passed
245 self.error = error 262 self.error = error
246 263
247 return check_passed, error 264 return check_passed, error
248 265
249 @property 266 @property
250 def continue_check(self): 267 def continue_check(self):
251 if (not self.attribute_exists) and (self.block_next): 268 if (not self.attribute_exists) and (self.block_next):
252 return False 269 return False
253 else: 270 else:
254 return True 271 return True
255 272
256 273
257 class AttributeType(GenericSpecification): 274 class AttributeType(GenericSpecification):
258 def __init__(self, dtype, block_next = True): 275 def __init__(self, dtype, block_next=True):
259 self.block_next = block_next 276 self.block_next = block_next
260 self.dtype = dtype 277 self.dtype = dtype
261 278
262 def check(self, netcdf_file, attribute_name): 279 def check(self, netcdf_file, attribute_name):
263 the_attribute = getattr(netcdf_file, attribute_name, None) 280 the_attribute = getattr(netcdf_file, attribute_name, None)
264 error = [] 281 error = []
265 282
266 if the_attribute: 283 if the_attribute:
267 # Get the internal python type and not the numpy.dtype. 284 # Get the internal python type and not the numpy.dtype.
268 # The conversions guarantee (?) that a single element is always returned 285 # The conversions guarantee (?) that a single element is always returned
269 try: 286 try:
270 attribute_type_python = type(np.asscalar(np.asarray(np.asarray(the_attribute[:]).item(0)))) 287 attribute_type_python = type(np.asscalar(np.asarray(np.asarray(the_attribute[:]).item(0))))
271 except: 288 except:
272 attribute_type_python = type(np.asscalar(the_attribute)) 289 attribute_type_python = type(np.asscalar(the_attribute))
273 290
274 if not (attribute_type_python == self.dtype): 291 if not (attribute_type_python == self.dtype):
275 check_passed = False 292 check_passed = False
276 error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name, type(the_attribute).__name__, self.dtype.__name__))) 293 error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name,
294 type(
295 the_attribute).__name__,
296 self.dtype.__name__)))
277 else: 297 else:
278 error = None 298 error = None
279 check_passed = True 299 check_passed = True
280 else: 300 else:
281 check_passed = False 301 check_passed = False
282 error.append(SpecError('Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name))) 302 error.append(SpecError(
283 303 'Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name)))
304
284 self.check_passed = check_passed 305 self.check_passed = check_passed
285 self.error = error 306 self.error = error
286 307
287 return check_passed, error 308 return check_passed, error
288 309
289 @property 310 @property
290 def continue_check(self): 311 def continue_check(self):
291 if (not self.check_passed) and (self.block_next): 312 if (not self.check_passed) and (self.block_next):
293 else: 314 else:
294 return True 315 return True
295 316
296 317
297 class AttributeStrLength(GenericSpecification): 318 class AttributeStrLength(GenericSpecification):
298
299 def __init__(self, length): 319 def __init__(self, length):
300 self.length = length 320 self.length = length
301 321
302
303 def check(self, netcdf_file, attribute_name): 322 def check(self, netcdf_file, attribute_name):
304 the_attribute = getattr(netcdf_file, attribute_name, None) 323 the_attribute = getattr(netcdf_file, attribute_name, None)
305 error = [] 324 error = []
306 325
307 if the_attribute: 326 if the_attribute:
308 if len(the_attribute) != self.length: 327 if len(the_attribute) != self.length:
309 check_passed = False 328 check_passed = False
310 error.append(SpecError('Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length, len(the_attribute)))) 329 error.append(SpecError(
330 'Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length,
331 len(the_attribute))))
311 else: 332 else:
312 check_passed = True 333 check_passed = True
313 else: 334 else:
314 check_passed = False 335 check_passed = False
315 error.append(SpecError('Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name))) 336 error.append(SpecError(
316 337 'Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name)))
338
317 self.check_passed = check_passed 339 self.check_passed = check_passed
318 self.error = error 340 self.error = error
319 341
320 return check_passed, error 342 return check_passed, error
321 343
322 344
323 class FilenameShellPattern(GenericSpecification): 345 class FilenameShellPattern(GenericSpecification):
324 def __init__(self, shell_pattern): 346 def __init__(self, shell_pattern):
325 self.pattern = shell_pattern 347 self.pattern = shell_pattern
326 348
327 def check(self, netcdf_file, filename): 349 def check(self, netcdf_file, filename):
328 error = [] 350 error = []
329 351
330 if fnmatch.fnmatch(filename, self.pattern): 352 if fnmatch.fnmatch(filename, self.pattern):
331 check_passed = True 353 check_passed = True
332 else: 354 else:
333 check_passed = False 355 check_passed = False
334 error.append(SpecError('Filename {0} does not match patter {1}'.format(filename, self.pattern))) 356 error.append(SpecError('Filename {0} does not match patter {1}'.format(filename, self.pattern)))
335 357
336 self.check_passed = check_passed 358 self.check_passed = check_passed
337 self.error = error 359 self.error = error
338 360
339 return check_passed, error 361 return check_passed, error
340 362
341 363
342 # This is the main class of the script. 364 # This is the main class of the script.
343 class FileChecker: 365 class FileChecker:
344 """ It uses the provided specifications to check the a file. 366 """ It uses the provided specifications to check the a file.
347 with FileChecker(filename, specs) as file_checker: 369 with FileChecker(filename, specs) as file_checker:
348 file_checker.run_checks() 370 file_checker.run_checks()
349 file_checker.print_report('error') 371 file_checker.print_report('error')
350 372
351 """ 373 """
352 374
353
354 def __init__(self, filepath, specs): 375 def __init__(self, filepath, specs):
355 self.file = None 376 self.file = None
356 self.checks_run = False 377 self.checks_run = False
357 self.filepath = filepath 378 self.filepath = filepath
358 self.filename = os.path.basename(filepath) 379 self.filename = os.path.basename(filepath)
359 self.specs = specs 380 self.specs = specs
360 self.check_results = {} 381 self.check_results = {}
361 self.check_results['general'] = [] 382 self.check_results['general'] = []
362 383
363 def __enter__(self): 384 def __enter__(self):
364 self.open_file() 385 self.open_file()
365 return self 386 return self
366 387
367 def __exit__(self, type, value, traceback): 388 def __exit__(self, type, value, traceback):
368 if self.file: 389 if self.file:
369 self.file.close() 390 self.file.close()
370 391
371 def open_file(self): 392 def open_file(self):
372 try: 393 try:
373 self.file = netcdf.Dataset(self.filepath) 394 self.file = netcdf.Dataset(self.filepath)
374 except: 395 except:
375 self.check_results['general'].append(SpecError('Could not open file {0}.'.format(self.filename))) 396 self.check_results['general'].append(SpecError('Could not open file {0}.'.format(self.filename)))
376 397
377 def close_file(self): 398 def close_file(self):
378 self.file.close() 399 self.file.close()
379 400
380 def run_checks(self): 401 def run_checks(self):
381 if self.file: 402 if self.file:
382 self.check_file() 403 self.check_file()
383 self.check_attributes() 404 self.check_attributes()
384 self.check_dimensions() 405 self.check_dimensions()
385 self.check_variables() 406 self.check_variables()
386 self.checks_run = True 407 self.checks_run = True
387 408
388 def check_file(self): 409 def check_file(self):
389 self.check_results['file'] = [] 410 self.check_results['file'] = []
390 411
391 try: 412 try:
392 specs_file = self.specs['file'] 413 specs_file = self.specs['file']
393 except: 414 except:
394 specs_file = [] 415 specs_file = []
395 416
396 for file_spec in specs_file: 417 for file_spec in specs_file:
397 check_passed, error = file_spec.check(self.file, self.filename) 418 check_passed, error = file_spec.check(self.file, self.filename)
398 419
399 if error: 420 if error:
400 self.check_results['file'].extend(list(error)) 421 self.check_results['file'].extend(list(error))
401 422
402 if not file_spec.continue_check: 423 if not file_spec.continue_check:
403 break 424 break
404 425
405 def check_attributes(self): 426 def check_attributes(self):
406 """ Check if attributes are according to specs """ 427 """ Check if attributes are according to specs """
407 428
408 self.check_results['attributes'] = [] 429 self.check_results['attributes'] = []
409 430
410 try: 431 try:
411 spec_attributes = self.specs['attributes'].keys() 432 spec_attributes = self.specs['attributes'].keys()
412 except: 433 except:
413 spec_attributes = [] 434 spec_attributes = []
414 435
415 for attribute_name in spec_attributes: 436 for attribute_name in spec_attributes:
416 attribute_specs = self.specs['attributes'][attribute_name] 437 attribute_specs = self.specs['attributes'][attribute_name]
417 for attribute_spec in attribute_specs: 438 for attribute_spec in attribute_specs:
418 check_passed, error = attribute_spec.check(self.file, attribute_name) 439 check_passed, error = attribute_spec.check(self.file, attribute_name)
419 440
420 if error: 441 if error:
421 self.check_results['attributes'].extend(list(error)) 442 self.check_results['attributes'].extend(list(error))
422 443
423 if not attribute_spec.continue_check: 444 if not attribute_spec.continue_check:
424 break # Don't continue checking specifications if a blocking check failed. 445 break # Don't continue checking specifications if a blocking check failed.
425 446
426 for attribute_name in self.file.ncattrs(): 447 for attribute_name in self.file.ncattrs():
427 if attribute_name not in spec_attributes: 448 if attribute_name not in spec_attributes:
428 self.check_results['attributes'].append(SpecWarning('Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name))) 449 self.check_results['attributes'].append(SpecWarning(
429 450 'Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name)))
451
430 def check_dimensions(self): 452 def check_dimensions(self):
431 """ Check if dimension are according to specs """ 453 """ Check if dimension are according to specs """
432 self.check_results['dimensions'] = [] 454 self.check_results['dimensions'] = []
433 455
434 try: 456 try:
435 spec_dimensions = self.specs['dimensions'].keys() 457 spec_dimensions = self.specs['dimensions'].keys()
436 except: 458 except:
437 spec_dimensions = [] 459 spec_dimensions = []
438 460
439
440 for dimension_name in spec_dimensions: 461 for dimension_name in spec_dimensions:
441 dimension_specs = self.specs['dimensions'][dimension_name] 462 dimension_specs = self.specs['dimensions'][dimension_name]
442 for dimension_spec in dimension_specs: 463 for dimension_spec in dimension_specs:
443 check_passed, error = dimension_spec.check(self.file, dimension_name) 464 check_passed, error = dimension_spec.check(self.file, dimension_name)
444 465
445 if error: 466 if error:
446 self.check_results['dimensions'].extend(list(error)) 467 self.check_results['dimensions'].extend(list(error))
447 468
448 if not dimension_spec.continue_check: 469 if not dimension_spec.continue_check:
449 break # Don't continue checking specifications if a blocking check failed. 470 break # Don't continue checking specifications if a blocking check failed.
450 471
451 for dimension in self.file.dimensions: 472 for dimension in self.file.dimensions:
452 if dimension not in spec_dimensions: 473 if dimension not in spec_dimensions:
453 self.check_results['dimensions'].append(SpecWarning('Dimension {0} found in the file but is not defined in the specifications'.format(dimension))) 474 self.check_results['dimensions'].append(SpecWarning(
454 475 'Dimension {0} found in the file but is not defined in the specifications'.format(dimension)))
476
455 def check_variables(self): 477 def check_variables(self):
456 """ Check if variables are according to specs """ 478 """ Check if variables are according to specs """
457 479
458 self.check_results['variables'] = [] 480 self.check_results['variables'] = []
459 481
460 try: 482 try:
461 spec_variables = self.specs['variables'].keys() 483 spec_variables = self.specs['variables'].keys()
462 except: 484 except:
463 spec_variables = [] 485 spec_variables = []
464 486
465 for variable_name in spec_variables: 487 for variable_name in spec_variables:
466 variable_specs = self.specs['variables'][variable_name] 488 variable_specs = self.specs['variables'][variable_name]
467 for variable_spec in variable_specs: 489 for variable_spec in variable_specs:
468 check_passed, error = variable_spec.check(self.file, variable_name) 490 check_passed, error = variable_spec.check(self.file, variable_name)
469 491
470 if error: 492 if error:
471 self.check_results['variables'].extend(list(error)) 493 self.check_results['variables'].extend(list(error))
472 494
473 if not variable_spec.continue_check: 495 if not variable_spec.continue_check:
474 break # Don't continue checking specifications if a blocking check failed. 496 break # Don't continue checking specifications if a blocking check failed.
475 497
476 for variable_name in self.file.variables: 498 for variable_name in self.file.variables:
477 if variable_name not in spec_variables: 499 if variable_name not in spec_variables:
478 self.check_results['variables'].append(SpecWarning('Variable {0} found in the file but is not defined in the specifications'.format(variable_name))) 500 self.check_results['variables'].append(SpecWarning(
479 501 'Variable {0} found in the file but is not defined in the specifications'.format(variable_name)))
480 def file_ok(self, level = 'error'): 502
503 def file_ok(self, level='error'):
481 """ Check if the file checked is ok. What ok means is defined by the level variable """ 504 """ Check if the file checked is ok. What ok means is defined by the level variable """
482 505
483 status = None 506 status = None
484 if self.checks_run: 507 if self.checks_run:
485 status = True 508 status = True
486 for category, result_list in self.check_results.items(): 509 for category, result_list in self.check_results.items():
487 for result in result_list: 510 for result in result_list:
488 if ERROR_ORDER[result.level] >= ERROR_ORDER[level]: 511 if ERROR_ORDER[result.level] >= ERROR_ORDER[level]:
489 status = False 512 status = False
490 513
491 return status 514 return status
492 515
493 def results_for_level(self, level): 516 def results_for_level(self, level):
494 """ Returns all the results of a specific level """ 517 """ Returns all the results of a specific level """
495 results = None 518 results = None
496 if self.checks_run: 519 if self.checks_run:
497 results = [] 520 results = []
498 for category, result_list in self.check_results.items(): 521 for category, result_list in self.check_results.items():
499 for result in result_list: 522 for result in result_list:
500 if ERROR_ORDER[result.level] == ERROR_ORDER[level]: 523 if ERROR_ORDER[result.level] == ERROR_ORDER[level]:
501 results.append(result) 524 results.append(result)
502 525
503 return results 526 return results
504 527
505 def results_by_level(self): 528 def results_by_level(self):
506 """ Returns a dictionary with the results by level. """ 529 """ Returns a dictionary with the results by level. """
507 530
508 results = {} 531 results = {}
509 for level, order in ERROR_ORDER.items(): 532 for level, order in ERROR_ORDER.items():
510 results[level] = self.results_for_level(level) 533 results[level] = self.results_for_level(level)
511 534
512 return results 535 return results
513 536
514 def result_count(self): 537 def result_count(self):
515 """ Returns a dictionary with the number of results per category. """ 538 """ Returns a dictionary with the number of results per category. """
516 539
517 result_number = {} 540 result_number = {}
518 results = self.results_by_level() 541 results = self.results_by_level()
519 542
520 for category, error_list in results.items(): 543 for category, error_list in results.items():
521 if error_list is None: 544 if error_list is None:
522 result_number[category] = 0 545 result_number[category] = 0
523 else: 546 else:
524 result_number[category] = len(error_list) 547 result_number[category] = len(error_list)
525 return result_number 548 return result_number
526 549
527 def print_report(self, level): 550 def print_report(self, level):
528 """ Print a report for the given level. """ 551 """ Print a report for the given level. """
529 552
530 print header_template.format(self, self.result_count()) 553 print header_template.format(self, self.result_count())
531 554
532 results = self.results_by_level() 555 results = self.results_by_level()
533 556
534 for result_level in ['error', 'warning', 'notification']: 557 for result_level in ['error', 'warning', 'notification']:
535 if ERROR_ORDER[result_level] >= ERROR_ORDER[level]: 558 if ERROR_ORDER[result_level] >= ERROR_ORDER[level]:
536 print "\n{0} details".format(result_level.capitalize()) 559 print "\n{0} details".format(result_level.capitalize())
537 print "----------------" 560 print "----------------"
538 for result in results[result_level]: 561 for result in results[result_level]:
539 print result 562 print result
540 563
541 564
542 # Sounding file specifications 565 # Sounding file specifications
543 sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'),], 566
544 'dimensions': {'points': [DimensionMandatory(True), 567
545 DimensionUnlimited(False),], 568 sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'), ],
569 'dimensions': {'points': [DimensionMandatory(True),
570 DimensionUnlimited(False), ],
571 },
572 'variables': {'Altitude': [VariableMandatory(True),
573 VariableDimensions(['points', ]),
574 VariableType(float)],
575 'Temperature': [VariableMandatory(True),
576 VariableDimensions(['points', ]),
577 VariableType(float)],
578 'Pressure': [VariableMandatory(True),
579 VariableDimensions(['points', ]),
580 VariableType(float)],
581 'RelativeHumidity': [VariableMandatory(False),
582 VariableDimensions(['points', ]),
583 VariableType(float)],
584 },
585 'attributes': {'Latitude_degrees_north': [AttributeMandatory(True),
586 AttributeType(float), ],
587 'Longitude_degrees_east': [AttributeMandatory(True),
588 AttributeType(float), ],
589 'Altitude_meter_asl': [AttributeMandatory(True),
590 AttributeType(float), ],
591 'Location': [AttributeMandatory(False),
592 AttributeType(unicode), ],
593 'Sounding_Station_Name': [AttributeMandatory(False),
594 AttributeType(unicode), ],
595 'WMO_Station_Number': [AttributeMandatory(False),
596 AttributeType(unicode), ],
597 'WBAN_Station_Number': [AttributeMandatory(False),
598 AttributeType(unicode), ],
599 'Sounding_Start_Date': [AttributeMandatory(True),
600 AttributeType(unicode, block_next=True),
601 AttributeStrLength(8)],
602 'Sounding_Start_Time_UT': [AttributeMandatory(True),
603 AttributeType(unicode, block_next=True),
604 AttributeStrLength(6)],
605 'Sounding_Stop_Time_UT': [AttributeMandatory(False),
606 AttributeType(unicode, block_next=True),
607 AttributeStrLength(6)],
608 },
609 'name': "SCC Sounding file"
610 }
611
612 # Lidar ratio file specifications
613 lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'), ],
614 'dimensions': {'points': [DimensionMandatory(True),
615 DimensionUnlimited(False), ],
616 'products': [DimensionMandatory(True),
617 DimensionUnlimited(False), ],
618 },
619 'variables': {'Altitude': [VariableMandatory(True),
620 VariableDimensions(['points', ]),
621 VariableType(float)],
622 'Lidar_Ratio': [VariableMandatory(True),
623 VariableDimensions(['products', 'points']),
624 VariableType(float)],
625 'product_ID': [VariableMandatory(True),
626 VariableDimensions(['products', ]),
627 VariableType(int)],
628 },
629 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
630 AttributeType(unicode), ],
631 },
632 'name': "SCC Lidar ratio file"
633 }
634
635 # Overlap file specifications
636 overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'), ],
637 'dimensions': {'points': [DimensionMandatory(True),
638 DimensionUnlimited(False), ],
639 'channels': [DimensionMandatory(True),
640 DimensionUnlimited(False), ],
641 },
642 'variables': {'Altitude': [VariableMandatory(True),
643 VariableDimensions(['points', ]),
644 VariableType(float)],
645 'Overlap_Function': [VariableMandatory(True),
646 VariableDimensions(['channels', 'points']),
647 VariableType(float)],
648 'channel_ID': [VariableMandatory(True),
649 VariableDimensions(['channels', ]),
650 VariableType(int)],
546 }, 651 },
547 'variables': {'Altitude': [VariableMandatory(True),
548 VariableDimensions(['points',]),
549 VariableType(float)],
550 'Temperature': [VariableMandatory(True),
551 VariableDimensions(['points',]),
552 VariableType(float)],
553 'Pressure': [VariableMandatory(True),
554 VariableDimensions(['points',]),
555 VariableType(float)],
556 'RelativeHumidity': [VariableMandatory(False),
557 VariableDimensions(['points',]),
558 VariableType(float)],
559 },
560 'attributes': {'Latitude_degrees_north': [AttributeMandatory(True),
561 AttributeType(float),],
562 'Longitude_degrees_east': [AttributeMandatory(True),
563 AttributeType(float),],
564 'Altitude_meter_asl': [AttributeMandatory(True),
565 AttributeType(float),],
566 'Location': [AttributeMandatory(False),
567 AttributeType(unicode),],
568 'Sounding_Station_Name': [AttributeMandatory(False),
569 AttributeType(unicode),],
570 'WMO_Station_Number': [AttributeMandatory(False),
571 AttributeType(unicode),],
572 'WBAN_Station_Number':[AttributeMandatory(False),
573 AttributeType(unicode),],
574 'Sounding_Start_Date':[AttributeMandatory(True),
575 AttributeType(unicode, block_next = True),
576 AttributeStrLength(8)],
577 'Sounding_Start_Time_UT':[AttributeMandatory(True),
578 AttributeType(unicode, block_next = True),
579 AttributeStrLength(6)],
580 'Sounding_Stop_Time_UT':[AttributeMandatory(False),
581 AttributeType(unicode, block_next = True),
582 AttributeStrLength(6)],
583 },
584 'name': "SCC Sounding file"
585 }
586
587 # Lidar ratio file specifications
588 lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'),],
589 'dimensions': {'points': [DimensionMandatory(True),
590 DimensionUnlimited(False),],
591 'products': [DimensionMandatory(True),
592 DimensionUnlimited(False),],
593 },
594 'variables': {'Altitude': [VariableMandatory(True),
595 VariableDimensions(['points',]),
596 VariableType(float)],
597 'Lidar_Ratio': [VariableMandatory(True),
598 VariableDimensions(['products', 'points']),
599 VariableType(float)],
600 'product_ID': [VariableMandatory(True),
601 VariableDimensions(['products',]),
602 VariableType(int)],
603 },
604 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
605 AttributeType(unicode),],
606 },
607 'name': "SCC Lidar ratio file"
608 }
609
610 # Overlap file specifications
611 overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'),],
612 'dimensions': {'points': [DimensionMandatory(True),
613 DimensionUnlimited(False),],
614 'channels': [DimensionMandatory(True),
615 DimensionUnlimited(False),],
616 },
617 'variables': {'Altitude': [VariableMandatory(True),
618 VariableDimensions(['points',]),
619 VariableType(float)],
620 'Overlap_Function': [VariableMandatory(True),
621 VariableDimensions(['channels', 'points']),
622 VariableType(float)],
623 'channel_ID': [VariableMandatory(True),
624 VariableDimensions(['channels',]),
625 VariableType(int)],
626 },
627 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True), 652 'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
628 AttributeType(unicode, block_next = True), 653 AttributeType(unicode, block_next=True),
629 AttributeStrLength(2)], 654 AttributeStrLength(2)],
630 'Overlap_Measurement_Date': [AttributeMandatory(True), 655 'Overlap_Measurement_Date': [AttributeMandatory(True),
631 AttributeType(unicode, block_next = True), 656 AttributeType(unicode, block_next=True),
632 AttributeStrLength(8)], 657 AttributeStrLength(8)],
633 }, 658 },
634 'name': "SCC Overlap file" 659 'name': "SCC Overlap file"
635 } 660 }
636 661
637 # Raw data file specifications 662 # Raw data file specifications
638 data_specs = {'file': [FilenameShellPattern('*.nc'),], 663 data_specs = {'file': [FilenameShellPattern('*.nc'), ],
639 'dimensions': {'points': [DimensionMandatory(True), 664 'dimensions': {'points': [DimensionMandatory(True),
640 DimensionUnlimited(False),], 665 DimensionUnlimited(False), ],
641 'channels': [DimensionMandatory(True), 666 'channels': [DimensionMandatory(True),
642 DimensionUnlimited(False),], 667 DimensionUnlimited(False), ],
643 'nb_of_time_scales': [DimensionMandatory(True), 668 'nb_of_time_scales': [DimensionMandatory(True),
644 DimensionUnlimited(False),], 669 DimensionUnlimited(False), ],
645 'time': [DimensionMandatory(True), 670 'time': [DimensionMandatory(True),
646 DimensionUnlimited(True),], 671 DimensionUnlimited(True), ],
647 'time_bck': [DimensionMandatory(False), 672 'time_bck': [DimensionMandatory(False),
648 DimensionUnlimited(False),], 673 DimensionUnlimited(False), ],
649 'scan_angles': [DimensionMandatory(True), 674 'scan_angles': [DimensionMandatory(True),
650 DimensionUnlimited(False),], 675 DimensionUnlimited(False), ],
651 }, 676 },
652 'variables': {'channel_ID': [VariableMandatory(True), 677 'variables': {'channel_ID': [VariableMandatory(True),
653 VariableDimensions(['channels',]), 678 VariableDimensions(['channels', ]),
654 VariableType(int)], 679 VariableType(int)],
655 'Laser_Repetition_Rate': [VariableMandatory(False), 680 'Laser_Repetition_Rate': [VariableMandatory(False),
656 VariableDimensions(['channels',]), 681 VariableDimensions(['channels', ]),
657 VariableType(int)], 682 VariableType(int)],
658 'Laser_Pointing_Angle': [VariableMandatory(True), 683 'Laser_Pointing_Angle': [VariableMandatory(True),
659 VariableDimensions(['scan_angles',]), 684 VariableDimensions(['scan_angles', ]),
685 VariableType(float)],
686 'ID_Range': [VariableMandatory(False),
687 VariableDimensions(['channels', ]),
688 VariableType(int)],
689 'Scattering_Mechanism': [VariableMandatory(False),
690 VariableDimensions(['channels', ]),
691 VariableType(int)],
692 'Emitted_Wavelength': [VariableMandatory(False),
693 VariableDimensions(['channels', ]),
694 VariableType(float)],
695 'Detected_Wavelength': [VariableMandatory(False),
696 VariableDimensions(['channels', ]),
697 VariableType(float)],
698 'Raw_Data_Range_Resolution': [VariableMandatory(False),
699 VariableDimensions(['channels', ]),
700 VariableType(float)],
701 'Background_Mode': [VariableMandatory(False),
702 VariableDimensions(['channels', ]),
703 VariableType(int)],
704 'Background_Low': [VariableMandatory(True),
705 VariableDimensions(['channels', ]),
706 VariableType(float)],
707 'Background_High': [VariableMandatory(True),
708 VariableDimensions(['channels', ]),
709 VariableType(float)],
710 'Molecular_Calc': [VariableMandatory(True),
711 VariableDimensions([]),
712 VariableType(int)],
713 'id_timescale': [VariableMandatory(True),
714 VariableDimensions(['channels', ]),
715 VariableType(int)],
716 'Dead_Time_Corr_Type': [VariableMandatory(False),
717 VariableDimensions(['channels', ]),
718 VariableType(int)],
719 'Dead_Time': [VariableMandatory(False),
720 VariableDimensions(['channels', ]),
721 VariableType(float)],
722 'Acquisition_Mode': [VariableMandatory(False),
723 VariableDimensions(['channels', ]),
724 VariableType(int)],
725 'Trigger_Delay': [VariableMandatory(False),
726 VariableDimensions(['channels', ]),
727 VariableType(float)],
728 'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True),
729 VariableDimensions(['time', 'nb_of_time_scales', ]),
730 VariableType(int)],
731 'Raw_Data_Start_Time': [VariableMandatory(True),
732 VariableDimensions(['time', 'nb_of_time_scales', ]),
733 VariableType(int)],
734 'Raw_Data_Stop_Time': [VariableMandatory(True),
735 VariableDimensions(['time', 'nb_of_time_scales', ]),
736 VariableType(int)],
737 'Laser_Shots': [VariableMandatory(True),
738 VariableDimensions(['time', 'channels', ]),
739 VariableType(int)],
740 'Raw_Lidar_Data': [VariableMandatory(False),
741 VariableDimensions(['time', 'channels', 'points']),
742 VariableType(float)],
743 'Depolarization_Factor': [VariableMandatory(False),
744 VariableDimensions(['channels', ]),
745 VariableType(float)],
746 'LR_Input': [VariableMandatory(False),
747 VariableDimensions(['channels', ]),
748 VariableType(int)],
749 'DAQ_Range': [VariableMandatory(False),
750 VariableDimensions(['channels', ]),
751 VariableType(float)],
752 'Pressure_at_Lidar_Station': [VariableMandatory(False),
753 VariableDimensions([]),
754 VariableType(float)],
755 'Temperature_at_Lidar_Station': [VariableMandatory(False),
756 VariableDimensions([]),
757 VariableType(float)],
758 'Background_Profile': [VariableMandatory(False),
759 VariableDimensions(['time_bck', 'channels', 'points']),
760 VariableType(float)],
761 'Raw_Bck_Start_Time': [VariableMandatory(False),
762 VariableDimensions(['time_bck', 'nb_of_time_scales', ]),
763 VariableType(int)],
764 'Raw_Bck_Stop_Time': [VariableMandatory(False),
765 VariableDimensions(['time_bck', 'nb_of_time_scales', ]),
766 VariableType(int)],
767 'Error_On_Raw_Lidar_Data': [VariableMandatory(False),
768 VariableDimensions(['time', 'channels', 'points']),
660 VariableType(float)], 769 VariableType(float)],
661 'ID_Range': [VariableMandatory(False), 770 'First_Signal_Rangebin': [VariableMandatory(False),
662 VariableDimensions(['channels',]), 771 VariableDimensions(['channels', ]),
663 VariableType(int)],
664 'Scattering_Mechanism': [VariableMandatory(False),
665 VariableDimensions(['channels',]),
666 VariableType(int)],
667 'Emitted_Wavelength': [VariableMandatory(False),
668 VariableDimensions(['channels',]),
669 VariableType(float)],
670 'Detected_Wavelength': [VariableMandatory(False),
671 VariableDimensions(['channels',]),
672 VariableType(float)],
673 'Raw_Data_Range_Resolution': [VariableMandatory(False),
674 VariableDimensions(['channels',]),
675 VariableType(float)],
676 'Background_Mode': [VariableMandatory(False),
677 VariableDimensions(['channels',]),
678 VariableType(int)],
679 'Background_Low': [VariableMandatory(True),
680 VariableDimensions(['channels',]),
681 VariableType(float)],
682 'Background_High': [VariableMandatory(True),
683 VariableDimensions(['channels',]),
684 VariableType(float)],
685 'Molecular_Calc': [VariableMandatory(True),
686 VariableDimensions([]),
687 VariableType(int)],
688 'id_timescale': [VariableMandatory(True),
689 VariableDimensions(['channels',]),
690 VariableType(int)],
691 'Dead_Time_Corr_Type': [VariableMandatory(False),
692 VariableDimensions(['channels',]),
693 VariableType(int)],
694 'Dead_Time': [VariableMandatory(False),
695 VariableDimensions(['channels',]),
696 VariableType(float)],
697 'Acquisition_Mode': [VariableMandatory(False),
698 VariableDimensions(['channels',]),
699 VariableType(int)],
700 'Trigger_Delay': [VariableMandatory(False),
701 VariableDimensions(['channels',]),
702 VariableType(float)],
703 'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True),
704 VariableDimensions(['time','nb_of_time_scales',]),
705 VariableType(int)],
706 'Raw_Data_Start_Time': [VariableMandatory(True),
707 VariableDimensions(['time','nb_of_time_scales',]),
708 VariableType(int)],
709 'Raw_Data_Stop_Time': [VariableMandatory(True),
710 VariableDimensions(['time','nb_of_time_scales',]),
711 VariableType(int)], 772 VariableType(int)],
712 'Laser_Shots': [VariableMandatory(True), 773 },
713 VariableDimensions(['time','channels',]), 774 'attributes': {'Measurement_ID': [AttributeMandatory(True),
714 VariableType(int)], 775 AttributeType(unicode, block_next=True),
715 'Raw_Lidar_Data': [VariableMandatory(False), 776 AttributeStrLength(12)],
716 VariableDimensions(['time', 'channels', 'points']), 777 'RawData_Start_Date': [AttributeMandatory(True),
717 VariableType(float)], 778 AttributeType(unicode, block_next=True),
718 'Depolarization_Factor': [VariableMandatory(False), 779 AttributeStrLength(8)],
719 VariableDimensions(['channels',]), 780 'RawData_Start_Time_UT': [AttributeMandatory(True),
720 VariableType(float)], 781 AttributeType(unicode, block_next=True),
721 'LR_Input': [VariableMandatory(False), 782 AttributeStrLength(6)],
722 VariableDimensions(['channels',]), 783 'RawData_Stop_Time_UT': [AttributeMandatory(True),
723 VariableType(int)], 784 AttributeType(unicode, block_next=True),
724 'DAQ_Range': [VariableMandatory(False), 785 AttributeStrLength(6)],
725 VariableDimensions(['channels',]), 786 'RawBck_Start_Date': [AttributeMandatory(False),
726 VariableType(float)], 787 AttributeType(unicode, block_next=True),
727 'Pressure_at_Lidar_Station': [VariableMandatory(False), 788 AttributeStrLength(8)],
728 VariableDimensions([]), 789 'RawBck_Start_Time_UT': [AttributeMandatory(False),
729 VariableType(float)], 790 AttributeType(unicode, block_next=True),
730 'Temperature_at_Lidar_Station': [VariableMandatory(False), 791 AttributeStrLength(6)],
731 VariableDimensions([]), 792 'RawBck_Stop_Time_UT': [AttributeMandatory(False),
732 VariableType(float)], 793 AttributeType(unicode, block_next=True),
733 'Background_Profile': [VariableMandatory(False), 794 AttributeStrLength(6)],
734 VariableDimensions(['time_bck', 'channels', 'points']), 795 'Sounding_File_Name': [AttributeMandatory(False),
735 VariableType(float)], 796 AttributeType(unicode), ],
736 'Raw_Bck_Start_Time': [VariableMandatory(False), 797 'LR_File_Name': [AttributeMandatory(False),
737 VariableDimensions(['time_bck','nb_of_time_scales',]), 798 AttributeType(unicode), ],
738 VariableType(int)], 799 'Overlap_File_Name': [AttributeMandatory(False),
739 'Raw_Bck_Stop_Time': [VariableMandatory(False), 800 AttributeType(unicode), ],
740 VariableDimensions(['time_bck','nb_of_time_scales',]), 801 'Location': [AttributeMandatory(False),
741 VariableType(int)], 802 AttributeType(unicode), ],
742 'Error_On_Raw_Lidar_Data': [VariableMandatory(False), 803 'System': [AttributeMandatory(False),
743 VariableDimensions(['time','channels', 'points']), 804 AttributeType(unicode), ],
744 VariableType(float)], 805 'Latitude_degrees_north': [AttributeMandatory(False),
745 'First_Signal_Rangebin': [VariableMandatory(False), 806 AttributeType(float), ],
746 VariableDimensions(['channels',]), 807 'Longitude_degrees_east': [AttributeMandatory(False),
747 VariableType(int)], 808 AttributeType(float), ],
809 'Altitude_meter_asl': [AttributeMandatory(False),
810 AttributeType(float), ],
748 }, 811 },
749 'attributes': {'Measurement_ID': [AttributeMandatory(True), 812 'name': "SCC Raw input file"
750 AttributeType(unicode, block_next = True), 813 }
751 AttributeStrLength(12)],
752 'RawData_Start_Date': [AttributeMandatory(True),
753 AttributeType(unicode, block_next = True),
754 AttributeStrLength(8)],
755 'RawData_Start_Time_UT': [AttributeMandatory(True),
756 AttributeType(unicode, block_next = True),
757 AttributeStrLength(6)],
758 'RawData_Stop_Time_UT': [AttributeMandatory(True),
759 AttributeType(unicode, block_next = True),
760 AttributeStrLength(6)],
761 'RawBck_Start_Date': [AttributeMandatory(False),
762 AttributeType(unicode, block_next = True),
763 AttributeStrLength(8)],
764 'RawBck_Start_Time_UT': [AttributeMandatory(False),
765 AttributeType(unicode, block_next = True),
766 AttributeStrLength(6)],
767 'RawBck_Stop_Time_UT': [AttributeMandatory(False),
768 AttributeType(unicode, block_next = True),
769 AttributeStrLength(6)],
770 'Sounding_File_Name': [AttributeMandatory(False),
771 AttributeType(unicode),],
772 'LR_File_Name': [AttributeMandatory(False),
773 AttributeType(unicode),],
774 'Overlap_File_Name': [AttributeMandatory(False),
775 AttributeType(unicode),],
776 'Location': [AttributeMandatory(False),
777 AttributeType(unicode),],
778 'System': [AttributeMandatory(False),
779 AttributeType(unicode),],
780 'Latitude_degrees_north': [AttributeMandatory(False),
781 AttributeType(float),],
782 'Longitude_degrees_east': [AttributeMandatory(False),
783 AttributeType(float),],
784 'Altitude_meter_asl': [AttributeMandatory(False),
785 AttributeType(float),],
786 },
787 'name': "SCC Raw input file"
788 }
789 814
790 # Used for the command line arguments 815 # Used for the command line arguments
791 spec_shorthands = {'sounding': sounding_specs, 816 spec_shorthands = {'sounding': sounding_specs,
792 'lidar_ratio': lidar_ratio_specs, 817 'lidar_ratio': lidar_ratio_specs,
793 'overlap': overlap_specs, 818 'overlap': overlap_specs,
794 'data': data_specs,} 819 'data': data_specs,}
795 820
796
797 if __name__ == "__main__": 821 if __name__ == "__main__":
798
799 # For use from a terminal 822 # For use from a terminal
800 import argparse 823 import argparse
801 824
802 parser = argparse.ArgumentParser() 825 parser = argparse.ArgumentParser()
803 parser.add_argument("file", help = "The path of the file to be checked") 826 parser.add_argument("file", help="The path of the file to be checked")
804 parser.add_argument("-s", "--specs", default = 'data', 827 parser.add_argument("-s", "--specs", default='data',
805 help = "The specificiations to use", 828 help="The specificiations to use",
806 choices = ['data', 'overlap', 'lidar_ratio', 'sounding']) 829 choices=['data', 'overlap', 'lidar_ratio', 'sounding'])
807 parser.add_argument("-l", "--level", default = 'warning', 830 parser.add_argument("-l", "--level", default='warning',
808 help = "The output level", 831 help="The output level",
809 choices = ['error', 'warning', 'notification']) 832 choices=['error', 'warning', 'notification'])
810 833
811 # Check the arguments 834 # Check the arguments
812 args = parser.parse_args() 835 args = parser.parse_args()
813 836
814 specs = spec_shorthands[args.specs] 837 specs = spec_shorthands[args.specs]
815 838
816 with FileChecker(args.file, specs) as file_checker: 839 with FileChecker(args.file, specs) as file_checker:
817 file_checker.run_checks() 840 file_checker.run_checks()
818 file_checker.print_report(args.level) 841 file_checker.print_report(args.level)
819

EARLINET Repository: SCC NetCDF checker

mercurial