• file
• latest
• revisions
• annotate
• diff
• comparison
• raw

netcdf_checker.py@fa814cbe01f9 (annotated)

netcdf_checker.py

Fri, 10 Oct 2014 00:16:55 +0200

author

Iannis <ulalume3@yahoo.com>

date

Fri, 10 Oct 2014 00:16:55 +0200

changeset 0

fa814cbe01f9

child 2

dcc02b7b6c52

permissions

-rw-r--r--

Initial commit.

ulalume3@0	1	import os
ulalume3@0	2	import fnmatch
ulalume3@0	3	import numpy as np
ulalume3@0	4	import netCDF4 as netcdf
ulalume3@0	5
ulalume3@0	6
ulalume3@0	7	ERROR_ORDER = {'notification': 1,
ulalume3@0	8	'warning': 2,
ulalume3@0	9	'error': 3,
ulalume3@0	10	}
ulalume3@0	11
ulalume3@0	12	# Used when printing the report
ulalume3@0	13	header_template = """\n----- Report for file {0.filename} -----
ulalume3@0	14	Checked against specification: {0.specs[name]}.
ulalume3@0	15
ulalume3@0	16	Output summary:
ulalume3@0	17	Errors: {1[error]}, Warnings: {1[warning]}, Notifications: {1[notification]}"""
ulalume3@0	18
ulalume3@0	19
ulalume3@0	20
ulalume3@0	21	# We firstly define the possible specification elements
ulalume3@0	22	# In the end of this file we use them to define complete specifications
ulalume3@0	23
ulalume3@0	24
ulalume3@0	25	class SpecGenericError:
ulalume3@0	26	def __repr__(self):
ulalume3@0	27	return "{0}: {1}".format(self.level.title(), self.message)
ulalume3@0	28
ulalume3@0	29
ulalume3@0	30	class SpecError(SpecGenericError):
ulalume3@0	31	def __init__(self, message):
ulalume3@0	32	self.message = message
ulalume3@0	33	self.level = 'error'
ulalume3@0	34
ulalume3@0	35
ulalume3@0	36	class SpecWarning(SpecGenericError):
ulalume3@0	37	def __init__(self, message):
ulalume3@0	38	self.message = message
ulalume3@0	39	self.level = 'warning'
ulalume3@0	40
ulalume3@0	41
ulalume3@0	42	class SpecNotification(SpecGenericError):
ulalume3@0	43	def __init__(self, message):
ulalume3@0	44	self.message = message
ulalume3@0	45	self.level = 'notification'
ulalume3@0	46
ulalume3@0	47
ulalume3@0	48	class GenericSpecification:
ulalume3@0	49	@property
ulalume3@0	50	def continue_check(self):
ulalume3@0	51	return True
ulalume3@0	52
ulalume3@0	53	class DimensionMandatory(GenericSpecification):
ulalume3@0	54	def __init__(self, is_mandatory = True):
ulalume3@0	55	self.block_next = True # if true the next checks for this dimension will not be performed.
ulalume3@0	56	self.is_mandatory = is_mandatory
ulalume3@0	57
ulalume3@0	58	def check(self, netcdf_file, dimension_name):
ulalume3@0	59	the_dimension = netcdf_file.dimensions.get(dimension_name, None)
ulalume3@0	60
ulalume3@0	61	error = []
ulalume3@0	62	if the_dimension:
ulalume3@0	63	# If the dimension is found in the file
ulalume3@0	64	self.dimension_exists = True
ulalume3@0	65	check_passed = True
ulalume3@0	66
ulalume3@0	67	else:
ulalume3@0	68	self.dimension_exists = False
ulalume3@0	69	if self.is_mandatory:
ulalume3@0	70	# If a mandatory dimension is not found in the file
ulalume3@0	71	check_passed = False
ulalume3@0	72	error.append(SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name)))
ulalume3@0	73	else:
ulalume3@0	74	check_passed = True
ulalume3@0	75	error.append(SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name)))
ulalume3@0	76
ulalume3@0	77	self.check_passed = check_passed
ulalume3@0	78	self.error = error
ulalume3@0	79
ulalume3@0	80	return check_passed, error
ulalume3@0	81
ulalume3@0	82	@property
ulalume3@0	83	def continue_check(self):
ulalume3@0	84	if (not self.dimension_exists) and (self.block_next):
ulalume3@0	85	return False
ulalume3@0	86	else:
ulalume3@0	87	return True
ulalume3@0	88
ulalume3@0	89
ulalume3@0	90	class DimensionUnlimited(GenericSpecification):
ulalume3@0	91	def __init__(self, is_unlimited):
ulalume3@0	92	self.block_next = False
ulalume3@0	93	self.is_unlimited = is_unlimited
ulalume3@0	94
ulalume3@0	95	def check(self, netcdf_file, dimension_name):
ulalume3@0	96	the_dimension = netcdf_file.dimensions.get(dimension_name, None)
ulalume3@0	97	error = []
ulalume3@0	98
ulalume3@0	99	if the_dimension:
ulalume3@0	100	if the_dimension.isunlimited() == True and self.is_unlimited == False:
ulalume3@0	101	check_passed = False
ulalume3@0	102	error.append(SpecWarning('Dimension {0} should not be unlimited but is.'.format(dimension_name)))
ulalume3@0	103	elif the_dimension.isunlimited() == False and self.is_unlimited == True:
ulalume3@0	104	check_passed = False
ulalume3@0	105	error.append(SpecWarning('Dimension {0} should be unlimited but it is not.'.format(dimension_name)))
ulalume3@0	106	else:
ulalume3@0	107	check_passed = True
ulalume3@0	108	else:
ulalume3@0	109	check_passed = True
ulalume3@0	110	error.append(SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name)))
ulalume3@0	111
ulalume3@0	112	self.check_passed = check_passed
ulalume3@0	113	self.error = error
ulalume3@0	114
ulalume3@0	115	return check_passed, error
ulalume3@0	116
ulalume3@0	117
ulalume3@0	118	class VariableMandatory(GenericSpecification):
ulalume3@0	119	def __init__(self, is_mandatory = True):
ulalume3@0	120	self.block_next = True # if true the next checks for this variable will not be performed.
ulalume3@0	121	self.is_mandatory = is_mandatory
ulalume3@0	122
ulalume3@0	123	def check(self, netcdf_file, variable_name):
ulalume3@0	124	the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0	125	error = []
ulalume3@0	126
ulalume3@0	127	if the_variable != None:
ulalume3@0	128	# If the variable is found in the file
ulalume3@0	129	self.variable_exists = True
ulalume3@0	130	check_passed = True
ulalume3@0	131
ulalume3@0	132	else:
ulalume3@0	133	self.variable_exists = False
ulalume3@0	134	if self.is_mandatory:
ulalume3@0	135	# If a mandatory variable is not found in the file
ulalume3@0	136	check_passed = False
ulalume3@0	137	error.append(SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name)))
ulalume3@0	138	else:
ulalume3@0	139	check_passed = True
ulalume3@0	140	error.append(SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name)))
ulalume3@0	141
ulalume3@0	142	self.check_passed = check_passed
ulalume3@0	143	self.error = error
ulalume3@0	144
ulalume3@0	145	return check_passed, error
ulalume3@0	146
ulalume3@0	147	@property
ulalume3@0	148	def continue_check(self):
ulalume3@0	149	if (not self.variable_exists) and (self.block_next):
ulalume3@0	150	return False
ulalume3@0	151	else:
ulalume3@0	152	return True
ulalume3@0	153
ulalume3@0	154	class VariableDimensions(GenericSpecification):
ulalume3@0	155	def __init__(self, dimensions):
ulalume3@0	156	self.dimensions = dimensions
ulalume3@0	157
ulalume3@0	158	def check(self, netcdf_file, variable_name):
ulalume3@0	159	the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0	160
ulalume3@0	161	if the_variable != None:
ulalume3@0	162	variable_dimensions = list(the_variable.dimensions)
ulalume3@0	163	error = []
ulalume3@0	164	check_passed = True
ulalume3@0	165	for dimension in self.dimensions:
ulalume3@0	166	if not (dimension in variable_dimensions):
ulalume3@0	167	check_passed = False
ulalume3@0	168	error.append(SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension)))
ulalume3@0	169
ulalume3@0	170	# If all dimensions are present, check if the variables are in the
ulalume3@0	171	# correct order.
ulalume3@0	172	if check_passed:
ulalume3@0	173	if list(self.dimensions) != variable_dimensions:
ulalume3@0	174	check_passed = False
ulalume3@0	175	error.append(SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name,
ulalume3@0	176	variable_dimensions,
ulalume3@0	177	list(self.dimensions))))
ulalume3@0	178	for dimension in variable_dimensions:
ulalume3@0	179	if dimension not in self.dimensions:
ulalume3@0	180	error.append(SpecWarning('Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension, variable_name)))
ulalume3@0	181
ulalume3@0	182	else:
ulalume3@0	183	check_passed = False
ulalume3@0	184	error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format(variable_name)),]
ulalume3@0	185
ulalume3@0	186	self.check_passed = check_passed
ulalume3@0	187	self.error = error
ulalume3@0	188
ulalume3@0	189	return check_passed, error
ulalume3@0	190
ulalume3@0	191
ulalume3@0	192	class VariableType(GenericSpecification):
ulalume3@0	193	def __init__(self, dtype):
ulalume3@0	194	self.dtype = dtype
ulalume3@0	195
ulalume3@0	196	def check(self, netcdf_file, variable_name):
ulalume3@0	197	the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0	198	error = []
ulalume3@0	199
ulalume3@0	200	if the_variable != None:
ulalume3@0	201	# Get the internal python type and not the numpy.dtype.
ulalume3@0	202	# The conversions guarantee (?) that a single element is always returned
ulalume3@0	203	variable_type_python = type(np.asscalar(np.asarray(np.asarray(the_variable[:]).item(0))))
ulalume3@0	204
ulalume3@0	205	if not (variable_type_python == self.dtype):
ulalume3@0	206	check_passed = False
ulalume3@0	207	error.append(SpecError('Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype, self.dtype)))
ulalume3@0	208	else:
ulalume3@0	209	check_passed = True
ulalume3@0	210	else:
ulalume3@0	211	check_passed = False
ulalume3@0	212	error.append(SpecError('Variable {0} should be checked for type, but was not found in the file.'.format(variable_name)))
ulalume3@0	213
ulalume3@0	214	self.check_passed = check_passed
ulalume3@0	215	self.error = error
ulalume3@0	216
ulalume3@0	217	return check_passed, error
ulalume3@0	218
ulalume3@0	219
ulalume3@0	220	class AttributeMandatory(GenericSpecification):
ulalume3@0	221	def __init__(self, is_mandatory = True):
ulalume3@0	222	self.block_next = True # if true the next checks for this variable will not be performed.
ulalume3@0	223	self.is_mandatory = is_mandatory
ulalume3@0	224
ulalume3@0	225	def check(self, netcdf_file, attribute_name):
ulalume3@0	226	the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0	227	error = []
ulalume3@0	228
ulalume3@0	229	if the_attribute:
ulalume3@0	230	# If the variable is found in the file
ulalume3@0	231	self.attribute_exists = True
ulalume3@0	232	check_passed = True
ulalume3@0	233
ulalume3@0	234	else:
ulalume3@0	235	self.attribute_exists = False
ulalume3@0	236	if self.is_mandatory:
ulalume3@0	237	# If a mandatory variable is not found in the file
ulalume3@0	238	check_passed = False
ulalume3@0	239	error.append(SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name)))
ulalume3@0	240	else:
ulalume3@0	241	check_passed = True
ulalume3@0	242	error.append(SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name)))
ulalume3@0	243
ulalume3@0	244	self.check_passed = check_passed
ulalume3@0	245	self.error = error
ulalume3@0	246
ulalume3@0	247	return check_passed, error
ulalume3@0	248
ulalume3@0	249	@property
ulalume3@0	250	def continue_check(self):
ulalume3@0	251	if (not self.attribute_exists) and (self.block_next):
ulalume3@0	252	return False
ulalume3@0	253	else:
ulalume3@0	254	return True
ulalume3@0	255
ulalume3@0	256
ulalume3@0	257	class AttributeType(GenericSpecification):
ulalume3@0	258	def __init__(self, dtype, block_next = True):
ulalume3@0	259	self.block_next = block_next
ulalume3@0	260	self.dtype = dtype
ulalume3@0	261
ulalume3@0	262	def check(self, netcdf_file, attribute_name):
ulalume3@0	263	the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0	264	error = []
ulalume3@0	265
ulalume3@0	266	if the_attribute:
ulalume3@0	267	# Get the internal python type and not the numpy.dtype.
ulalume3@0	268	# The conversions guarantee (?) that a single element is always returned
ulalume3@0	269	try:
ulalume3@0	270	attribute_type_python = type(np.asscalar(np.asarray(np.asarray(the_attribute[:]).item(0))))
ulalume3@0	271	except:
ulalume3@0	272	attribute_type_python = type(np.asscalar(the_attribute))
ulalume3@0	273
ulalume3@0	274	if not (attribute_type_python == self.dtype):
ulalume3@0	275	check_passed = False
ulalume3@0	276	error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name, type(the_attribute).__name__, self.dtype.__name__)))
ulalume3@0	277	else:
ulalume3@0	278	error = None
ulalume3@0	279	check_passed = True
ulalume3@0	280	else:
ulalume3@0	281	check_passed = False
ulalume3@0	282	error.append(SpecError('Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name)))
ulalume3@0	283
ulalume3@0	284	self.check_passed = check_passed
ulalume3@0	285	self.error = error
ulalume3@0	286
ulalume3@0	287	return check_passed, error
ulalume3@0	288
ulalume3@0	289	@property
ulalume3@0	290	def continue_check(self):
ulalume3@0	291	if (not self.check_passed) and (self.block_next):
ulalume3@0	292	return False
ulalume3@0	293	else:
ulalume3@0	294	return True
ulalume3@0	295
ulalume3@0	296
ulalume3@0	297	class AttributeStrLength(GenericSpecification):
ulalume3@0	298
ulalume3@0	299	def __init__(self, length):
ulalume3@0	300	self.length = length
ulalume3@0	301
ulalume3@0	302
ulalume3@0	303	def check(self, netcdf_file, attribute_name):
ulalume3@0	304	the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0	305	error = []
ulalume3@0	306
ulalume3@0	307	if the_attribute:
ulalume3@0	308	if len(the_attribute) != self.length:
ulalume3@0	309	check_passed = False
ulalume3@0	310	error.append(SpecError('Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length, len(the_attribute))))
ulalume3@0	311	else:
ulalume3@0	312	check_passed = True
ulalume3@0	313	else:
ulalume3@0	314	check_passed = False
ulalume3@0	315	error.append(SpecError('Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name)))
ulalume3@0	316
ulalume3@0	317	self.check_passed = check_passed
ulalume3@0	318	self.error = error
ulalume3@0	319
ulalume3@0	320	return check_passed, error
ulalume3@0	321
ulalume3@0	322
ulalume3@0	323	class FilenameShellPattern(GenericSpecification):
ulalume3@0	324	def __init__(self, shell_pattern):
ulalume3@0	325	self.pattern = shell_pattern
ulalume3@0	326
ulalume3@0	327	def check(self, netcdf_file, filename):
ulalume3@0	328	error = []
ulalume3@0	329
ulalume3@0	330	if fnmatch.fnmatch(filename, self.pattern):
ulalume3@0	331	check_passed = True
ulalume3@0	332	else:
ulalume3@0	333	check_passed = False
ulalume3@0	334	error.append(SpecError('Filename {0} does not match patter {1}'.format(filename, self.pattern)))
ulalume3@0	335
ulalume3@0	336	self.check_passed = check_passed
ulalume3@0	337	self.error = error
ulalume3@0	338
ulalume3@0	339	return check_passed, error
ulalume3@0	340
ulalume3@0	341
ulalume3@0	342	# This is the main class of the script.
ulalume3@0	343	class FileChecker:
ulalume3@0	344	""" It uses the provided specifications to check the a file.
ulalume3@0	345	It can be used with the 'with' statement. For example:
ulalume3@0	346
ulalume3@0	347	with FileChecker(filename, specs) as file_checker:
ulalume3@0	348	file_checker.run_checks()
ulalume3@0	349	file_checker.print_report('error')
ulalume3@0	350
ulalume3@0	351	"""
ulalume3@0	352
ulalume3@0	353
ulalume3@0	354	def __init__(self, filepath, specs):
ulalume3@0	355	self.file = None
ulalume3@0	356	self.checks_run = False
ulalume3@0	357	self.filepath = filepath
ulalume3@0	358	self.filename = os.path.basename(filepath)
ulalume3@0	359	self.specs = specs
ulalume3@0	360	self.check_results = {}
ulalume3@0	361	self.check_results['general'] = []
ulalume3@0	362
ulalume3@0	363	def __enter__(self):
ulalume3@0	364	self.open_file()
ulalume3@0	365	return self
ulalume3@0	366
ulalume3@0	367	def __exit__(self, type, value, traceback):
ulalume3@0	368	if self.file:
ulalume3@0	369	self.file.close()
ulalume3@0	370
ulalume3@0	371	def open_file(self):
ulalume3@0	372	try:
ulalume3@0	373	self.file = netcdf.Dataset(self.filepath)
ulalume3@0	374	except:
ulalume3@0	375	self.check_results['general'].append(SpecError('Could not open file {0}.'.format(self.filename)))
ulalume3@0	376
ulalume3@0	377	def close_file(self):
ulalume3@0	378	self.file.close()
ulalume3@0	379
ulalume3@0	380	def run_checks(self):
ulalume3@0	381	if self.file:
ulalume3@0	382	self.check_file()
ulalume3@0	383	self.check_attributes()
ulalume3@0	384	self.check_dimensions()
ulalume3@0	385	self.check_variables()
ulalume3@0	386	self.checks_run = True
ulalume3@0	387
ulalume3@0	388	def check_file(self):
ulalume3@0	389	self.check_results['file'] = []
ulalume3@0	390
ulalume3@0	391	try:
ulalume3@0	392	specs_file = self.specs['file']
ulalume3@0	393	except:
ulalume3@0	394	specs_file = []
ulalume3@0	395
ulalume3@0	396	for file_spec in specs_file:
ulalume3@0	397	check_passed, error = file_spec.check(self.file, self.filename)
ulalume3@0	398
ulalume3@0	399	if error:
ulalume3@0	400	self.check_results['file'].extend(list(error))
ulalume3@0	401
ulalume3@0	402	if not file_spec.continue_check:
ulalume3@0	403	break
ulalume3@0	404
ulalume3@0	405	def check_attributes(self):
ulalume3@0	406	""" Check if attributes are according to specs """
ulalume3@0	407
ulalume3@0	408	self.check_results['attributes'] = []
ulalume3@0	409
ulalume3@0	410	try:
ulalume3@0	411	spec_attributes = self.specs['attributes'].keys()
ulalume3@0	412	except:
ulalume3@0	413	spec_attributes = []
ulalume3@0	414
ulalume3@0	415	for attribute_name in spec_attributes:
ulalume3@0	416	attribute_specs = self.specs['attributes'][attribute_name]
ulalume3@0	417	for attribute_spec in attribute_specs:
ulalume3@0	418	check_passed, error = attribute_spec.check(self.file, attribute_name)
ulalume3@0	419
ulalume3@0	420	if error:
ulalume3@0	421	self.check_results['attributes'].extend(list(error))
ulalume3@0	422
ulalume3@0	423	if not attribute_spec.continue_check:
ulalume3@0	424	break # Don't continue checking specifications if a blocking check failed.
ulalume3@0	425
ulalume3@0	426	for attribute_name in self.file.ncattrs():
ulalume3@0	427	if attribute_name not in spec_attributes:
ulalume3@0	428	self.check_results['attributes'].append(SpecWarning('Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name)))
ulalume3@0	429
ulalume3@0	430	def check_dimensions(self):
ulalume3@0	431	""" Check if dimension are according to specs """
ulalume3@0	432	self.check_results['dimensions'] = []
ulalume3@0	433
ulalume3@0	434	try:
ulalume3@0	435	spec_dimensions = self.specs['dimensions'].keys()
ulalume3@0	436	except:
ulalume3@0	437	spec_dimensions = []
ulalume3@0	438
ulalume3@0	439
ulalume3@0	440	for dimension_name in spec_dimensions:
ulalume3@0	441	dimension_specs = self.specs['dimensions'][dimension_name]
ulalume3@0	442	for dimension_spec in dimension_specs:
ulalume3@0	443	check_passed, error = dimension_spec.check(self.file, dimension_name)
ulalume3@0	444
ulalume3@0	445	if error:
ulalume3@0	446	self.check_results['dimensions'].extend(list(error))
ulalume3@0	447
ulalume3@0	448	if not dimension_spec.continue_check:
ulalume3@0	449	break # Don't continue checking specifications if a blocking check failed.
ulalume3@0	450
ulalume3@0	451	for dimension in self.file.dimensions:
ulalume3@0	452	if dimension not in spec_dimensions:
ulalume3@0	453	self.check_results['dimensions'].append(SpecWarning('Dimension {0} found in the file but is not defined in the specifications'.format(dimension)))
ulalume3@0	454
ulalume3@0	455	def check_variables(self):
ulalume3@0	456	""" Check if variables are according to specs """
ulalume3@0	457
ulalume3@0	458	self.check_results['variables'] = []
ulalume3@0	459
ulalume3@0	460	try:
ulalume3@0	461	spec_variables = self.specs['variables'].keys()
ulalume3@0	462	except:
ulalume3@0	463	spec_variables = []
ulalume3@0	464
ulalume3@0	465	for variable_name in spec_variables:
ulalume3@0	466	variable_specs = self.specs['variables'][variable_name]
ulalume3@0	467	for variable_spec in variable_specs:
ulalume3@0	468	check_passed, error = variable_spec.check(self.file, variable_name)
ulalume3@0	469
ulalume3@0	470	if error:
ulalume3@0	471	self.check_results['variables'].extend(list(error))
ulalume3@0	472
ulalume3@0	473	if not variable_spec.continue_check:
ulalume3@0	474	break # Don't continue checking specifications if a blocking check failed.
ulalume3@0	475
ulalume3@0	476	for variable_name in self.file.variables:
ulalume3@0	477	if variable_name not in spec_variables:
ulalume3@0	478	self.check_results['variables'].append(SpecWarning('Variable {0} found in the file but is not defined in the specifications'.format(variable_name)))
ulalume3@0	479
ulalume3@0	480	def file_ok(self, level = 'error'):
ulalume3@0	481	""" Check if the file checked is ok. What ok means is defined by the level variable """
ulalume3@0	482
ulalume3@0	483	status = None
ulalume3@0	484	if self.checks_run:
ulalume3@0	485	status = True
ulalume3@0	486	for category, result_list in self.check_results.items():
ulalume3@0	487	for result in result_list:
ulalume3@0	488	if ERROR_ORDER[result.level] >= ERROR_ORDER[level]:
ulalume3@0	489	status = False
ulalume3@0	490
ulalume3@0	491	return status
ulalume3@0	492
ulalume3@0	493	def results_for_level(self, level):
ulalume3@0	494	""" Returns all the results of a specific level """
ulalume3@0	495	results = None
ulalume3@0	496	if self.checks_run:
ulalume3@0	497	results = []
ulalume3@0	498	for category, result_list in self.check_results.items():
ulalume3@0	499	for result in result_list:
ulalume3@0	500	if ERROR_ORDER[result.level] == ERROR_ORDER[level]:
ulalume3@0	501	results.append(result)
ulalume3@0	502
ulalume3@0	503	return results
ulalume3@0	504
ulalume3@0	505	def results_by_level(self):
ulalume3@0	506	""" Returns a dictionary with the results by level. """
ulalume3@0	507
ulalume3@0	508	results = {}
ulalume3@0	509	for level, order in ERROR_ORDER.items():
ulalume3@0	510	results[level] = self.results_for_level(level)
ulalume3@0	511
ulalume3@0	512	return results
ulalume3@0	513
ulalume3@0	514	def result_count(self):
ulalume3@0	515	""" Returns a dictionary with the number of results per category. """
ulalume3@0	516
ulalume3@0	517	result_number = {}
ulalume3@0	518	results = self.results_by_level()
ulalume3@0	519
ulalume3@0	520	for category, error_list in results.items():
ulalume3@0	521	if error_list is None:
ulalume3@0	522	result_number[category] = 0
ulalume3@0	523	else:
ulalume3@0	524	result_number[category] = len(error_list)
ulalume3@0	525	return result_number
ulalume3@0	526
ulalume3@0	527	def print_report(self, level):
ulalume3@0	528	""" Print a report for the given level. """
ulalume3@0	529
ulalume3@0	530	print header_template.format(self, self.result_count())
ulalume3@0	531
ulalume3@0	532	results = self.results_by_level()
ulalume3@0	533
ulalume3@0	534	for result_level in ['error', 'warning', 'notification']:
ulalume3@0	535	if ERROR_ORDER[result_level] >= ERROR_ORDER[level]:
ulalume3@0	536	print "\n{0} details".format(result_level.capitalize())
ulalume3@0	537	print "----------------"
ulalume3@0	538	for result in results[result_level]:
ulalume3@0	539	print result
ulalume3@0	540
ulalume3@0	541
ulalume3@0	542	# Sounding file specifications
ulalume3@0	543	sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'),],
ulalume3@0	544	'dimensions': {'points': [DimensionMandatory(True),
ulalume3@0	545	DimensionUnlimited(False),],
ulalume3@0	546	},
ulalume3@0	547	'variables': {'Altitude': [VariableMandatory(True),
ulalume3@0	548	VariableDimensions(['points',]),
ulalume3@0	549	VariableType(float)],
ulalume3@0	550	'Temperature': [VariableMandatory(True),
ulalume3@0	551	VariableDimensions(['points',]),
ulalume3@0	552	VariableType(float)],
ulalume3@0	553	'Pressure': [VariableMandatory(True),
ulalume3@0	554	VariableDimensions(['points',]),
ulalume3@0	555	VariableType(float)],
ulalume3@0	556	'RelativeHumidity': [VariableMandatory(False),
ulalume3@0	557	VariableDimensions(['points',]),
ulalume3@0	558	VariableType(float)],
ulalume3@0	559	},
ulalume3@0	560	'attributes': {'Latitude_degrees_north': [AttributeMandatory(True),
ulalume3@0	561	AttributeType(float),],
ulalume3@0	562	'Longitude_degrees_east': [AttributeMandatory(True),
ulalume3@0	563	AttributeType(float),],
ulalume3@0	564	'Altitude_meter_asl': [AttributeMandatory(True),
ulalume3@0	565	AttributeType(float),],
ulalume3@0	566	'Location': [AttributeMandatory(False),
ulalume3@0	567	AttributeType(unicode),],
ulalume3@0	568	'Sounding_Station_Name': [AttributeMandatory(False),
ulalume3@0	569	AttributeType(unicode),],
ulalume3@0	570	'WMO_Station_Number': [AttributeMandatory(False),
ulalume3@0	571	AttributeType(unicode),],
ulalume3@0	572	'WBAN_Station_Number':[AttributeMandatory(False),
ulalume3@0	573	AttributeType(unicode),],
ulalume3@0	574	'Sounding_Start_Date':[AttributeMandatory(True),
ulalume3@0	575	AttributeType(unicode, block_next = True),
ulalume3@0	576	AttributeStrLength(8)],
ulalume3@0	577	'Sounding_Start_Time_UT':[AttributeMandatory(True),
ulalume3@0	578	AttributeType(unicode, block_next = True),
ulalume3@0	579	AttributeStrLength(6)],
ulalume3@0	580	'Sounding_Stop_Time_UT':[AttributeMandatory(False),
ulalume3@0	581	AttributeType(unicode, block_next = True),
ulalume3@0	582	AttributeStrLength(6)],
ulalume3@0	583	},
ulalume3@0	584	'name': "SCC Sounding file"
ulalume3@0	585	}
ulalume3@0	586
ulalume3@0	587	# Lidar ratio file specifications
ulalume3@0	588	lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'),],
ulalume3@0	589	'dimensions': {'points': [DimensionMandatory(True),
ulalume3@0	590	DimensionUnlimited(False),],
ulalume3@0	591	'products': [DimensionMandatory(True),
ulalume3@0	592	DimensionUnlimited(False),],
ulalume3@0	593	},
ulalume3@0	594	'variables': {'Altitude': [VariableMandatory(True),
ulalume3@0	595	VariableDimensions(['points',]),
ulalume3@0	596	VariableType(float)],
ulalume3@0	597	'Lidar_Ratio': [VariableMandatory(True),
ulalume3@0	598	VariableDimensions(['points', 'products']),
ulalume3@0	599	VariableType(float)],
ulalume3@0	600	'product_ID': [VariableMandatory(True),
ulalume3@0	601	VariableDimensions(['products',]),
ulalume3@0	602	VariableType(int)],
ulalume3@0	603	},
ulalume3@0	604	'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
ulalume3@0	605	AttributeType(unicode),],
ulalume3@0	606	},
ulalume3@0	607	'name': "SCC Lidar ratio file"
ulalume3@0	608	}
ulalume3@0	609
ulalume3@0	610	# Overlap file specifications
ulalume3@0	611	overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'),],
ulalume3@0	612	'dimensions': {'points': [DimensionMandatory(True),
ulalume3@0	613	DimensionUnlimited(False),],
ulalume3@0	614	'channels': [DimensionMandatory(True),
ulalume3@0	615	DimensionUnlimited(False),],
ulalume3@0	616	},
ulalume3@0	617	'variables': {'Altitude': [VariableMandatory(True),
ulalume3@0	618	VariableDimensions(['points',]),
ulalume3@0	619	VariableType(float)],
ulalume3@0	620	'Overlap_Function': [VariableMandatory(True),
ulalume3@0	621	VariableDimensions(['points', 'channels']),
ulalume3@0	622	VariableType(float)],
ulalume3@0	623	'channel_ID': [VariableMandatory(True),
ulalume3@0	624	VariableDimensions(['channels',]),
ulalume3@0	625	VariableType(int)],
ulalume3@0	626	},
ulalume3@0	627	'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
ulalume3@0	628	AttributeType(unicode, block_next = True),
ulalume3@0	629	AttributeStrLength(2)],
ulalume3@0	630	'Overlap_Measurement_Date': [AttributeMandatory(True),
ulalume3@0	631	AttributeType(unicode, block_next = True),
ulalume3@0	632	AttributeStrLength(8)],
ulalume3@0	633	},
ulalume3@0	634	'name': "SCC Overlap file"
ulalume3@0	635	}
ulalume3@0	636
ulalume3@0	637	# Raw data file specifications
ulalume3@0	638	data_specs = {'file': [FilenameShellPattern('*.nc'),],
ulalume3@0	639	'dimensions': {'points': [DimensionMandatory(True),
ulalume3@0	640	DimensionUnlimited(False),],
ulalume3@0	641	'channels': [DimensionMandatory(True),
ulalume3@0	642	DimensionUnlimited(False),],
ulalume3@0	643	'nb_of_time_scales': [DimensionMandatory(True),
ulalume3@0	644	DimensionUnlimited(False),],
ulalume3@0	645	'time': [DimensionMandatory(True),
ulalume3@0	646	DimensionUnlimited(True),],
ulalume3@0	647	'time_bck': [DimensionMandatory(False),
ulalume3@0	648	DimensionUnlimited(False),],
ulalume3@0	649	'scan_angles': [DimensionMandatory(True),
ulalume3@0	650	DimensionUnlimited(False),],
ulalume3@0	651	},
ulalume3@0	652	'variables': {'channel_ID': [VariableMandatory(True),
ulalume3@0	653	VariableDimensions(['channels',]),
ulalume3@0	654	VariableType(int)],
ulalume3@0	655	'Laser_Repetition_Rate': [VariableMandatory(False),
ulalume3@0	656	VariableDimensions(['channels',]),
ulalume3@0	657	VariableType(int)],
ulalume3@0	658	'Laser_Pointing_Angle': [VariableMandatory(True),
ulalume3@0	659	VariableDimensions(['scan_angles',]),
ulalume3@0	660	VariableType(float)],
ulalume3@0	661	'ID_Range': [VariableMandatory(False),
ulalume3@0	662	VariableDimensions(['channels',]),
ulalume3@0	663	VariableType(int)],
ulalume3@0	664	'Scattering_Mechanism': [VariableMandatory(False),
ulalume3@0	665	VariableDimensions(['channels',]),
ulalume3@0	666	VariableType(int)],
ulalume3@0	667	'Emitted_Wavelength': [VariableMandatory(False),
ulalume3@0	668	VariableDimensions(['channels',]),
ulalume3@0	669	VariableType(float)],
ulalume3@0	670	'Detected_Wavelength': [VariableMandatory(False),
ulalume3@0	671	VariableDimensions(['channels',]),
ulalume3@0	672	VariableType(float)],
ulalume3@0	673	'Raw_Data_Range_Resolution': [VariableMandatory(False),
ulalume3@0	674	VariableDimensions(['channels',]),
ulalume3@0	675	VariableType(float)],
ulalume3@0	676	'Background_Mode': [VariableMandatory(False),
ulalume3@0	677	VariableDimensions(['channels',]),
ulalume3@0	678	VariableType(int)],
ulalume3@0	679	'Background_Low': [VariableMandatory(True),
ulalume3@0	680	VariableDimensions(['channels',]),
ulalume3@0	681	VariableType(float)],
ulalume3@0	682	'Background_High': [VariableMandatory(True),
ulalume3@0	683	VariableDimensions(['channels',]),
ulalume3@0	684	VariableType(float)],
ulalume3@0	685	'Molecular_Calc': [VariableMandatory(True),
ulalume3@0	686	VariableDimensions([]),
ulalume3@0	687	VariableType(int)],
ulalume3@0	688	'id_timescale': [VariableMandatory(True),
ulalume3@0	689	VariableDimensions(['channels',]),
ulalume3@0	690	VariableType(int)],
ulalume3@0	691	'Dead_Time_Corr_Type': [VariableMandatory(False),
ulalume3@0	692	VariableDimensions(['channels',]),
ulalume3@0	693	VariableType(int)],
ulalume3@0	694	'Dead_Time': [VariableMandatory(False),
ulalume3@0	695	VariableDimensions(['channels',]),
ulalume3@0	696	VariableType(float)],
ulalume3@0	697	'Acquisition_Mode': [VariableMandatory(False),
ulalume3@0	698	VariableDimensions(['channels',]),
ulalume3@0	699	VariableType(int)],
ulalume3@0	700	'Trigger_Delay': [VariableMandatory(False),
ulalume3@0	701	VariableDimensions(['channels',]),
ulalume3@0	702	VariableType(float)],
ulalume3@0	703	'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True),
ulalume3@0	704	VariableDimensions(['time','nb_of_time_scales',]),
ulalume3@0	705	VariableType(int)],
ulalume3@0	706	'Raw_Data_Start_Time': [VariableMandatory(True),
ulalume3@0	707	VariableDimensions(['time','nb_of_time_scales',]),
ulalume3@0	708	VariableType(int)],
ulalume3@0	709	'Raw_Data_Stop_Time': [VariableMandatory(True),
ulalume3@0	710	VariableDimensions(['time','nb_of_time_scales',]),
ulalume3@0	711	VariableType(int)],
ulalume3@0	712	'Laser_Shots': [VariableMandatory(True),
ulalume3@0	713	VariableDimensions(['time','channels',]),
ulalume3@0	714	VariableType(int)],
ulalume3@0	715	'Raw_Lidar_Data': [VariableMandatory(False),
ulalume3@0	716	VariableDimensions(['time', 'channels', 'points']),
ulalume3@0	717	VariableType(float)],
ulalume3@0	718	'Depolarization_Factor': [VariableMandatory(False),
ulalume3@0	719	VariableDimensions(['channels',]),
ulalume3@0	720	VariableType(float)],
ulalume3@0	721	'LR_Input': [VariableMandatory(False),
ulalume3@0	722	VariableDimensions(['channels',]),
ulalume3@0	723	VariableType(int)],
ulalume3@0	724	'DAQ_Range': [VariableMandatory(False),
ulalume3@0	725	VariableDimensions(['channels',]),
ulalume3@0	726	VariableType(float)],
ulalume3@0	727	'Pressure_at_Lidar_Station': [VariableMandatory(False),
ulalume3@0	728	VariableDimensions([]),
ulalume3@0	729	VariableType(float)],
ulalume3@0	730	'Temperature_at_Lidar_Station': [VariableMandatory(False),
ulalume3@0	731	VariableDimensions([]),
ulalume3@0	732	VariableType(float)],
ulalume3@0	733	'Background_Profile': [VariableMandatory(False),
ulalume3@0	734	VariableDimensions(['time_bck', 'channels', 'points']),
ulalume3@0	735	VariableType(float)],
ulalume3@0	736	'Raw_Bck_Start_Time': [VariableMandatory(False),
ulalume3@0	737	VariableDimensions(['time_bck','nb_of_time_scales',]),
ulalume3@0	738	VariableType(int)],
ulalume3@0	739	'Raw_Bck_Stop_Time': [VariableMandatory(False),
ulalume3@0	740	VariableDimensions(['time_bck','nb_of_time_scales',]),
ulalume3@0	741	VariableType(int)],
ulalume3@0	742	'Error_On_Raw_Lidar_Data': [VariableMandatory(False),
ulalume3@0	743	VariableDimensions(['time','channels', 'points']),
ulalume3@0	744	VariableType(float)],
ulalume3@0	745	'First_Signal_Rangebin': [VariableMandatory(False),
ulalume3@0	746	VariableDimensions(['channels',]),
ulalume3@0	747	VariableType(int)],
ulalume3@0	748	},
ulalume3@0	749	'attributes': {'Measurement_ID': [AttributeMandatory(True),
ulalume3@0	750	AttributeType(unicode, block_next = True),
ulalume3@0	751	AttributeStrLength(12)],
ulalume3@0	752	'RawData_Start_Date': [AttributeMandatory(True),
ulalume3@0	753	AttributeType(unicode, block_next = True),
ulalume3@0	754	AttributeStrLength(8)],
ulalume3@0	755	'RawData_Start_Time_UT': [AttributeMandatory(True),
ulalume3@0	756	AttributeType(unicode, block_next = True),
ulalume3@0	757	AttributeStrLength(6)],
ulalume3@0	758	'RawData_Stop_Time_UT': [AttributeMandatory(True),
ulalume3@0	759	AttributeType(unicode, block_next = True),
ulalume3@0	760	AttributeStrLength(6)],
ulalume3@0	761	'RawBck_Start_Date': [AttributeMandatory(False),
ulalume3@0	762	AttributeType(unicode, block_next = True),
ulalume3@0	763	AttributeStrLength(8)],
ulalume3@0	764	'RawBck_Start_Time_UT': [AttributeMandatory(False),
ulalume3@0	765	AttributeType(unicode, block_next = True),
ulalume3@0	766	AttributeStrLength(6)],
ulalume3@0	767	'RawBck_Stop_Time_UT': [AttributeMandatory(False),
ulalume3@0	768	AttributeType(unicode, block_next = True),
ulalume3@0	769	AttributeStrLength(6)],
ulalume3@0	770	'Sounding_File_Name': [AttributeMandatory(False),
ulalume3@0	771	AttributeType(unicode),],
ulalume3@0	772	'LR_File_Name': [AttributeMandatory(False),
ulalume3@0	773	AttributeType(unicode),],
ulalume3@0	774	'Overlap_File_Name': [AttributeMandatory(False),
ulalume3@0	775	AttributeType(unicode),],
ulalume3@0	776	'Location': [AttributeMandatory(False),
ulalume3@0	777	AttributeType(unicode),],
ulalume3@0	778	'System': [AttributeMandatory(False),
ulalume3@0	779	AttributeType(unicode),],
ulalume3@0	780	'Latitude_degrees_north': [AttributeMandatory(False),
ulalume3@0	781	AttributeType(float),],
ulalume3@0	782	'Longitude_degrees_east': [AttributeMandatory(False),
ulalume3@0	783	AttributeType(float),],
ulalume3@0	784	'Altitude_meter_asl': [AttributeMandatory(False),
ulalume3@0	785	AttributeType(float),],
ulalume3@0	786	},
ulalume3@0	787	'name': "SCC Raw input file"
ulalume3@0	788	}
ulalume3@0	789
ulalume3@0	790	# Used for the command line arguments
ulalume3@0	791	spec_shorthands = {'sounding': sounding_specs,
ulalume3@0	792	'lidar_ratio': lidar_ratio_specs,
ulalume3@0	793	'overlap': overlap_specs,
ulalume3@0	794	'data': data_specs,}
ulalume3@0	795
ulalume3@0	796
ulalume3@0	797	if __name__ == "__main__":
ulalume3@0	798
ulalume3@0	799	# For use from a terminal
ulalume3@0	800	import argparse
ulalume3@0	801
ulalume3@0	802	parser = argparse.ArgumentParser()
ulalume3@0	803	parser.add_argument("file", help = "The path of the file to be checked")
ulalume3@0	804	parser.add_argument("-s", "--specs", default = 'data',
ulalume3@0	805	help = "The specificiations to use",
ulalume3@0	806	choices = ['data', 'overlap', 'lidar_ratio', 'sounding'])
ulalume3@0	807	parser.add_argument("-l", "--level", default = 'warning',
ulalume3@0	808	help = "The output level",
ulalume3@0	809	choices = ['error', 'warning', 'notification'])
ulalume3@0	810
ulalume3@0	811	# Check the arguments
ulalume3@0	812	args = parser.parse_args()
ulalume3@0	813
ulalume3@0	814	specs = spec_shorthands[args.specs]
ulalume3@0	815
ulalume3@0	816	with FileChecker(args.file, specs) as file_checker:
ulalume3@0	817	file_checker.run_checks()
ulalume3@0	818	file_checker.print_report(args.level)
ulalume3@0	819