• file
• latest
• revisions
• annotate
• diff
• comparison
• raw

netcdf_checker.py@5a18d437cd96 (annotated)

netcdf_checker.py

Thu, 13 Oct 2016 13:20:12 +0300

author

Ioannis Binietoglou <binietoglou@noa.gr>

date

Thu, 13 Oct 2016 13:20:12 +0300

changeset 4

5a18d437cd96

parent 2

dcc02b7b6c52

permissions

-rwxr-xr-x

Small changes and formating.

binietoglou@4	1	import fnmatch
binietoglou@4	2	import netCDF4 as netcdf
ulalume3@0	3	import os
binietoglou@4	4
ulalume3@0	5	import numpy as np
ulalume3@0	6
ulalume3@0	7	ERROR_ORDER = {'notification': 1,
ulalume3@0	8	'warning': 2,
ulalume3@0	9	'error': 3,
ulalume3@0	10	}
ulalume3@0	11
ulalume3@0	12	# Used when printing the report
ulalume3@0	13	header_template = """\n----- Report for file {0.filename} -----
ulalume3@0	14	Checked against specification: {0.specs[name]}.
ulalume3@0	15
ulalume3@0	16	Output summary:
ulalume3@0	17	Errors: {1[error]}, Warnings: {1[warning]}, Notifications: {1[notification]}"""
ulalume3@0	18
ulalume3@0	19
ulalume3@0	20	# We firstly define the possible specification elements
ulalume3@0	21	# In the end of this file we use them to define complete specifications
ulalume3@0	22
ulalume3@0	23
ulalume3@0	24	class SpecGenericError:
ulalume3@0	25	def __repr__(self):
ulalume3@0	26	return "{0}: {1}".format(self.level.title(), self.message)
ulalume3@0	27
binietoglou@4	28
ulalume3@0	29	class SpecError(SpecGenericError):
ulalume3@0	30	def __init__(self, message):
ulalume3@0	31	self.message = message
ulalume3@0	32	self.level = 'error'
ulalume3@0	33
ulalume3@0	34
ulalume3@0	35	class SpecWarning(SpecGenericError):
ulalume3@0	36	def __init__(self, message):
ulalume3@0	37	self.message = message
ulalume3@0	38	self.level = 'warning'
ulalume3@0	39
ulalume3@0	40
ulalume3@0	41	class SpecNotification(SpecGenericError):
ulalume3@0	42	def __init__(self, message):
ulalume3@0	43	self.message = message
ulalume3@0	44	self.level = 'notification'
ulalume3@0	45
ulalume3@0	46
ulalume3@0	47	class GenericSpecification:
ulalume3@0	48	@property
ulalume3@0	49	def continue_check(self):
ulalume3@0	50	return True
binietoglou@4	51
binietoglou@4	52
ulalume3@0	53	class DimensionMandatory(GenericSpecification):
binietoglou@4	54	def __init__(self, is_mandatory=True):
binietoglou@4	55	self.block_next = True # if true the next checks for this dimension will not be performed.
ulalume3@0	56	self.is_mandatory = is_mandatory
binietoglou@4	57
ulalume3@0	58	def check(self, netcdf_file, dimension_name):
ulalume3@0	59	the_dimension = netcdf_file.dimensions.get(dimension_name, None)
binietoglou@4	60
ulalume3@0	61	error = []
ulalume3@0	62	if the_dimension:
ulalume3@0	63	# If the dimension is found in the file
ulalume3@0	64	self.dimension_exists = True
ulalume3@0	65	check_passed = True
binietoglou@4	66
ulalume3@0	67	else:
ulalume3@0	68	self.dimension_exists = False
ulalume3@0	69	if self.is_mandatory:
ulalume3@0	70	# If a mandatory dimension is not found in the file
ulalume3@0	71	check_passed = False
binietoglou@4	72	error.append(
binietoglou@4	73	SpecError('The dimension {0} is obligatory but was not found in the file.'.format(dimension_name)))
ulalume3@0	74	else:
ulalume3@0	75	check_passed = True
binietoglou@4	76	error.append(
binietoglou@4	77	SpecNotification('The optional dimension {0} was not found in the file.'.format(dimension_name)))
binietoglou@4	78
ulalume3@0	79	self.check_passed = check_passed
ulalume3@0	80	self.error = error
binietoglou@4	81
ulalume3@0	82	return check_passed, error
binietoglou@4	83
ulalume3@0	84	@property
ulalume3@0	85	def continue_check(self):
ulalume3@0	86	if (not self.dimension_exists) and (self.block_next):
ulalume3@0	87	return False
ulalume3@0	88	else:
ulalume3@0	89	return True
binietoglou@4	90
ulalume3@0	91
ulalume3@0	92	class DimensionUnlimited(GenericSpecification):
ulalume3@0	93	def __init__(self, is_unlimited):
ulalume3@0	94	self.block_next = False
ulalume3@0	95	self.is_unlimited = is_unlimited
binietoglou@4	96
ulalume3@0	97	def check(self, netcdf_file, dimension_name):
ulalume3@0	98	the_dimension = netcdf_file.dimensions.get(dimension_name, None)
ulalume3@0	99	error = []
binietoglou@4	100
ulalume3@0	101	if the_dimension:
ulalume3@0	102	if the_dimension.isunlimited() == True and self.is_unlimited == False:
ulalume3@0	103	check_passed = False
ulalume3@0	104	error.append(SpecWarning('Dimension {0} should not be unlimited but is.'.format(dimension_name)))
ulalume3@0	105	elif the_dimension.isunlimited() == False and self.is_unlimited == True:
ulalume3@0	106	check_passed = False
ulalume3@0	107	error.append(SpecWarning('Dimension {0} should be unlimited but it is not.'.format(dimension_name)))
ulalume3@0	108	else:
ulalume3@0	109	check_passed = True
ulalume3@0	110	else:
ulalume3@0	111	check_passed = True
binietoglou@4	112	error.append(
binietoglou@4	113	SpecError('Dimension {0} should be unlimited, but was not found in the file.'.format(dimension_name)))
binietoglou@4	114
ulalume3@0	115	self.check_passed = check_passed
ulalume3@0	116	self.error = error
binietoglou@4	117
binietoglou@4	118	return check_passed, error
ulalume3@0	119
ulalume3@0	120
ulalume3@0	121	class VariableMandatory(GenericSpecification):
binietoglou@4	122	def __init__(self, is_mandatory=True):
binietoglou@4	123	self.block_next = True # if true the next checks for this variable will not be performed.
ulalume3@0	124	self.is_mandatory = is_mandatory
binietoglou@4	125
ulalume3@0	126	def check(self, netcdf_file, variable_name):
ulalume3@0	127	the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0	128	error = []
binietoglou@4	129
ulalume3@0	130	if the_variable != None:
ulalume3@0	131	# If the variable is found in the file
ulalume3@0	132	self.variable_exists = True
ulalume3@0	133	check_passed = True
binietoglou@4	134
ulalume3@0	135	else:
ulalume3@0	136	self.variable_exists = False
ulalume3@0	137	if self.is_mandatory:
ulalume3@0	138	# If a mandatory variable is not found in the file
ulalume3@0	139	check_passed = False
binietoglou@4	140	error.append(
binietoglou@4	141	SpecError('The variable {0} is obligatory but was not found in the file.'.format(variable_name)))
ulalume3@0	142	else:
ulalume3@0	143	check_passed = True
binietoglou@4	144	error.append(
binietoglou@4	145	SpecNotification('The optional variable {0} was not found in the file.'.format(variable_name)))
binietoglou@4	146
ulalume3@0	147	self.check_passed = check_passed
ulalume3@0	148	self.error = error
binietoglou@4	149
ulalume3@0	150	return check_passed, error
binietoglou@4	151
ulalume3@0	152	@property
ulalume3@0	153	def continue_check(self):
ulalume3@0	154	if (not self.variable_exists) and (self.block_next):
ulalume3@0	155	return False
ulalume3@0	156	else:
ulalume3@0	157	return True
ulalume3@0	158
binietoglou@4	159
ulalume3@0	160	class VariableDimensions(GenericSpecification):
ulalume3@0	161	def __init__(self, dimensions):
ulalume3@0	162	self.dimensions = dimensions
binietoglou@4	163
ulalume3@0	164	def check(self, netcdf_file, variable_name):
ulalume3@0	165	the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0	166
ulalume3@0	167	if the_variable != None:
ulalume3@0	168	variable_dimensions = list(the_variable.dimensions)
ulalume3@0	169	error = []
ulalume3@0	170	check_passed = True
ulalume3@0	171	for dimension in self.dimensions:
ulalume3@0	172	if not (dimension in variable_dimensions):
ulalume3@0	173	check_passed = False
binietoglou@4	174	error.append(
binietoglou@4	175	SpecError("Variable {0} does not have dimension {1}.".format(variable_name, dimension)))
binietoglou@4	176
ulalume3@0	177	# If all dimensions are present, check if the variables are in the
ulalume3@0	178	# correct order.
ulalume3@0	179	if check_passed:
ulalume3@0	180	if list(self.dimensions) != variable_dimensions:
ulalume3@0	181	check_passed = False
binietoglou@4	182	error.append(
binietoglou@4	183	SpecError("Variable {0} has wrong dimension order: {1} instead of {2}.".format(variable_name,
binietoglou@4	184	variable_dimensions,
binietoglou@4	185	list(
binietoglou@4	186	self.dimensions))))
ulalume3@0	187	for dimension in variable_dimensions:
ulalume3@0	188	if dimension not in self.dimensions:
binietoglou@4	189	error.append(SpecWarning(
binietoglou@4	190	'Dimension {0} found in variable {1} but is not defined in the specifications'.format(dimension,
binietoglou@4	191	variable_name)))
binietoglou@4	192
ulalume3@0	193	else:
ulalume3@0	194	check_passed = False
binietoglou@4	195	error = [SpecError('Variable {0} should be checked for dimensions, but was not found in the file.'.format(
binietoglou@4	196	variable_name)), ]
binietoglou@4	197
ulalume3@0	198	self.check_passed = check_passed
ulalume3@0	199	self.error = error
binietoglou@4	200
ulalume3@0	201	return check_passed, error
ulalume3@0	202
ulalume3@0	203
ulalume3@0	204	class VariableType(GenericSpecification):
ulalume3@0	205	def __init__(self, dtype):
ulalume3@0	206	self.dtype = dtype
binietoglou@4	207
ulalume3@0	208	def check(self, netcdf_file, variable_name):
ulalume3@0	209	the_variable = netcdf_file.variables.get(variable_name, None)
ulalume3@0	210	error = []
binietoglou@4	211
ulalume3@0	212	if the_variable != None:
ulalume3@0	213	# Get the internal python type and not the numpy.dtype.
ulalume3@0	214	# The conversions guarantee (?) that a single element is always returned
ulalume3@0	215	variable_type_python = type(np.asscalar(np.asarray(np.asarray(the_variable[:]).item(0))))
binietoglou@4	216
ulalume3@0	217	if not (variable_type_python == self.dtype):
ulalume3@0	218	check_passed = False
binietoglou@4	219	error.append(SpecError(
binietoglou@4	220	'Variable {0} is of type {1} while it should be {2}'.format(variable_name, the_variable.dtype,
binietoglou@4	221	self.dtype)))
ulalume3@0	222	else:
ulalume3@0	223	check_passed = True
ulalume3@0	224	else:
ulalume3@0	225	check_passed = False
binietoglou@4	226	error.append(SpecError(
binietoglou@4	227	'Variable {0} should be checked for type, but was not found in the file.'.format(variable_name)))
binietoglou@4	228
ulalume3@0	229	self.check_passed = check_passed
ulalume3@0	230	self.error = error
binietoglou@4	231
ulalume3@0	232	return check_passed, error
ulalume3@0	233
ulalume3@0	234
ulalume3@0	235	class AttributeMandatory(GenericSpecification):
binietoglou@4	236	def __init__(self, is_mandatory=True):
binietoglou@4	237	self.block_next = True # if true the next checks for this variable will not be performed.
ulalume3@0	238	self.is_mandatory = is_mandatory
binietoglou@4	239
ulalume3@0	240	def check(self, netcdf_file, attribute_name):
ulalume3@0	241	the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0	242	error = []
binietoglou@4	243
ulalume3@0	244	if the_attribute:
ulalume3@0	245	# If the variable is found in the file
ulalume3@0	246	self.attribute_exists = True
ulalume3@0	247	check_passed = True
ulalume3@0	248
ulalume3@0	249	else:
ulalume3@0	250	self.attribute_exists = False
ulalume3@0	251	if self.is_mandatory:
ulalume3@0	252	# If a mandatory variable is not found in the file
ulalume3@0	253	check_passed = False
binietoglou@4	254	error.append(
binietoglou@4	255	SpecError('The attribute {0} is obligatory but was not found in the file.'.format(attribute_name)))
ulalume3@0	256	else:
ulalume3@0	257	check_passed = True
binietoglou@4	258	error.append(
binietoglou@4	259	SpecNotification('The optional attribute {0} was not found in the file.'.format(attribute_name)))
binietoglou@4	260
ulalume3@0	261	self.check_passed = check_passed
ulalume3@0	262	self.error = error
binietoglou@4	263
ulalume3@0	264	return check_passed, error
binietoglou@4	265
ulalume3@0	266	@property
ulalume3@0	267	def continue_check(self):
ulalume3@0	268	if (not self.attribute_exists) and (self.block_next):
ulalume3@0	269	return False
ulalume3@0	270	else:
ulalume3@0	271	return True
ulalume3@0	272
binietoglou@4	273
ulalume3@0	274	class AttributeType(GenericSpecification):
binietoglou@4	275	def __init__(self, dtype, block_next=True):
ulalume3@0	276	self.block_next = block_next
ulalume3@0	277	self.dtype = dtype
binietoglou@4	278
ulalume3@0	279	def check(self, netcdf_file, attribute_name):
ulalume3@0	280	the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0	281	error = []
binietoglou@4	282
ulalume3@0	283	if the_attribute:
ulalume3@0	284	# Get the internal python type and not the numpy.dtype.
ulalume3@0	285	# The conversions guarantee (?) that a single element is always returned
ulalume3@0	286	try:
ulalume3@0	287	attribute_type_python = type(np.asscalar(np.asarray(np.asarray(the_attribute[:]).item(0))))
ulalume3@0	288	except:
ulalume3@0	289	attribute_type_python = type(np.asscalar(the_attribute))
binietoglou@4	290
ulalume3@0	291	if not (attribute_type_python == self.dtype):
ulalume3@0	292	check_passed = False
binietoglou@4	293	error.append(SpecError('Attribute {0} is of type {1} while it should be {2}'.format(attribute_name,
binietoglou@4	294	type(
binietoglou@4	295	the_attribute).__name__,
binietoglou@4	296	self.dtype.__name__)))
ulalume3@0	297	else:
ulalume3@0	298	error = None
ulalume3@0	299	check_passed = True
ulalume3@0	300	else:
ulalume3@0	301	check_passed = False
binietoglou@4	302	error.append(SpecError(
binietoglou@4	303	'Attribute {0} should be checked for type, but was not found in the file.'.format(attribute_name)))
binietoglou@4	304
ulalume3@0	305	self.check_passed = check_passed
ulalume3@0	306	self.error = error
binietoglou@4	307
ulalume3@0	308	return check_passed, error
ulalume3@0	309
ulalume3@0	310	@property
ulalume3@0	311	def continue_check(self):
ulalume3@0	312	if (not self.check_passed) and (self.block_next):
ulalume3@0	313	return False
ulalume3@0	314	else:
ulalume3@0	315	return True
ulalume3@0	316
ulalume3@0	317
ulalume3@0	318	class AttributeStrLength(GenericSpecification):
ulalume3@0	319	def __init__(self, length):
ulalume3@0	320	self.length = length
binietoglou@4	321
ulalume3@0	322	def check(self, netcdf_file, attribute_name):
ulalume3@0	323	the_attribute = getattr(netcdf_file, attribute_name, None)
ulalume3@0	324	error = []
binietoglou@4	325
ulalume3@0	326	if the_attribute:
ulalume3@0	327	if len(the_attribute) != self.length:
ulalume3@0	328	check_passed = False
binietoglou@4	329	error.append(SpecError(
binietoglou@4	330	'Attribute {0} should be of length {1} while it has length {2}'.format(attribute_name, self.length,
binietoglou@4	331	len(the_attribute))))
ulalume3@0	332	else:
ulalume3@0	333	check_passed = True
ulalume3@0	334	else:
ulalume3@0	335	check_passed = False
binietoglou@4	336	error.append(SpecError(
binietoglou@4	337	'Attribute {0} should be checked for length, but was not found in the file.'.format(attribute_name)))
binietoglou@4	338
ulalume3@0	339	self.check_passed = check_passed
ulalume3@0	340	self.error = error
binietoglou@4	341
binietoglou@4	342	return check_passed, error
ulalume3@0	343
ulalume3@0	344
ulalume3@0	345	class FilenameShellPattern(GenericSpecification):
ulalume3@0	346	def __init__(self, shell_pattern):
ulalume3@0	347	self.pattern = shell_pattern
binietoglou@4	348
ulalume3@0	349	def check(self, netcdf_file, filename):
ulalume3@0	350	error = []
binietoglou@4	351
ulalume3@0	352	if fnmatch.fnmatch(filename, self.pattern):
ulalume3@0	353	check_passed = True
ulalume3@0	354	else:
ulalume3@0	355	check_passed = False
ulalume3@0	356	error.append(SpecError('Filename {0} does not match patter {1}'.format(filename, self.pattern)))
binietoglou@4	357
ulalume3@0	358	self.check_passed = check_passed
ulalume3@0	359	self.error = error
binietoglou@4	360
binietoglou@4	361	return check_passed, error
ulalume3@0	362
ulalume3@0	363
ulalume3@0	364	# This is the main class of the script.
ulalume3@0	365	class FileChecker:
ulalume3@0	366	""" It uses the provided specifications to check the a file.
ulalume3@0	367	It can be used with the 'with' statement. For example:
ulalume3@0	368
ulalume3@0	369	with FileChecker(filename, specs) as file_checker:
ulalume3@0	370	file_checker.run_checks()
ulalume3@0	371	file_checker.print_report('error')
ulalume3@0	372
ulalume3@0	373	"""
binietoglou@4	374
ulalume3@0	375	def __init__(self, filepath, specs):
ulalume3@0	376	self.file = None
ulalume3@0	377	self.checks_run = False
ulalume3@0	378	self.filepath = filepath
ulalume3@0	379	self.filename = os.path.basename(filepath)
ulalume3@0	380	self.specs = specs
ulalume3@0	381	self.check_results = {}
ulalume3@0	382	self.check_results['general'] = []
binietoglou@4	383
ulalume3@0	384	def __enter__(self):
ulalume3@0	385	self.open_file()
ulalume3@0	386	return self
binietoglou@4	387
ulalume3@0	388	def __exit__(self, type, value, traceback):
ulalume3@0	389	if self.file:
ulalume3@0	390	self.file.close()
binietoglou@4	391
ulalume3@0	392	def open_file(self):
ulalume3@0	393	try:
ulalume3@0	394	self.file = netcdf.Dataset(self.filepath)
ulalume3@0	395	except:
ulalume3@0	396	self.check_results['general'].append(SpecError('Could not open file {0}.'.format(self.filename)))
binietoglou@4	397
ulalume3@0	398	def close_file(self):
ulalume3@0	399	self.file.close()
binietoglou@4	400
ulalume3@0	401	def run_checks(self):
binietoglou@4	402	if self.file:
ulalume3@0	403	self.check_file()
ulalume3@0	404	self.check_attributes()
ulalume3@0	405	self.check_dimensions()
ulalume3@0	406	self.check_variables()
ulalume3@0	407	self.checks_run = True
ulalume3@0	408
ulalume3@0	409	def check_file(self):
ulalume3@0	410	self.check_results['file'] = []
binietoglou@4	411
ulalume3@0	412	try:
ulalume3@0	413	specs_file = self.specs['file']
ulalume3@0	414	except:
ulalume3@0	415	specs_file = []
binietoglou@4	416
ulalume3@0	417	for file_spec in specs_file:
binietoglou@4	418	check_passed, error = file_spec.check(self.file, self.filename)
binietoglou@4	419
ulalume3@0	420	if error:
ulalume3@0	421	self.check_results['file'].extend(list(error))
binietoglou@4	422
ulalume3@0	423	if not file_spec.continue_check:
binietoglou@4	424	break
binietoglou@4	425
ulalume3@0	426	def check_attributes(self):
ulalume3@0	427	""" Check if attributes are according to specs """
binietoglou@4	428
ulalume3@0	429	self.check_results['attributes'] = []
binietoglou@4	430
ulalume3@0	431	try:
ulalume3@0	432	spec_attributes = self.specs['attributes'].keys()
ulalume3@0	433	except:
ulalume3@0	434	spec_attributes = []
binietoglou@4	435
ulalume3@0	436	for attribute_name in spec_attributes:
ulalume3@0	437	attribute_specs = self.specs['attributes'][attribute_name]
ulalume3@0	438	for attribute_spec in attribute_specs:
ulalume3@0	439	check_passed, error = attribute_spec.check(self.file, attribute_name)
ulalume3@0	440
ulalume3@0	441	if error:
binietoglou@4	442	self.check_results['attributes'].extend(list(error))
ulalume3@0	443
ulalume3@0	444	if not attribute_spec.continue_check:
binietoglou@4	445	break # Don't continue checking specifications if a blocking check failed.
binietoglou@4	446
ulalume3@0	447	for attribute_name in self.file.ncattrs():
ulalume3@0	448	if attribute_name not in spec_attributes:
binietoglou@4	449	self.check_results['attributes'].append(SpecWarning(
binietoglou@4	450	'Attribute {0} found in the file but is not defined in the specifications'.format(attribute_name)))
binietoglou@4	451
ulalume3@0	452	def check_dimensions(self):
ulalume3@0	453	""" Check if dimension are according to specs """
ulalume3@0	454	self.check_results['dimensions'] = []
binietoglou@4	455
ulalume3@0	456	try:
ulalume3@0	457	spec_dimensions = self.specs['dimensions'].keys()
ulalume3@0	458	except:
ulalume3@0	459	spec_dimensions = []
binietoglou@4	460
ulalume3@0	461	for dimension_name in spec_dimensions:
ulalume3@0	462	dimension_specs = self.specs['dimensions'][dimension_name]
ulalume3@0	463	for dimension_spec in dimension_specs:
ulalume3@0	464	check_passed, error = dimension_spec.check(self.file, dimension_name)
binietoglou@4	465
ulalume3@0	466	if error:
binietoglou@4	467	self.check_results['dimensions'].extend(list(error))
binietoglou@4	468
ulalume3@0	469	if not dimension_spec.continue_check:
binietoglou@4	470	break # Don't continue checking specifications if a blocking check failed.
binietoglou@4	471
ulalume3@0	472	for dimension in self.file.dimensions:
ulalume3@0	473	if dimension not in spec_dimensions:
binietoglou@4	474	self.check_results['dimensions'].append(SpecWarning(
binietoglou@4	475	'Dimension {0} found in the file but is not defined in the specifications'.format(dimension)))
binietoglou@4	476
ulalume3@0	477	def check_variables(self):
ulalume3@0	478	""" Check if variables are according to specs """
binietoglou@4	479
ulalume3@0	480	self.check_results['variables'] = []
binietoglou@4	481
ulalume3@0	482	try:
ulalume3@0	483	spec_variables = self.specs['variables'].keys()
ulalume3@0	484	except:
ulalume3@0	485	spec_variables = []
binietoglou@4	486
ulalume3@0	487	for variable_name in spec_variables:
ulalume3@0	488	variable_specs = self.specs['variables'][variable_name]
ulalume3@0	489	for variable_spec in variable_specs:
ulalume3@0	490	check_passed, error = variable_spec.check(self.file, variable_name)
ulalume3@0	491
ulalume3@0	492	if error:
binietoglou@4	493	self.check_results['variables'].extend(list(error))
ulalume3@0	494
ulalume3@0	495	if not variable_spec.continue_check:
binietoglou@4	496	break # Don't continue checking specifications if a blocking check failed.
binietoglou@4	497
ulalume3@0	498	for variable_name in self.file.variables:
ulalume3@0	499	if variable_name not in spec_variables:
binietoglou@4	500	self.check_results['variables'].append(SpecWarning(
binietoglou@4	501	'Variable {0} found in the file but is not defined in the specifications'.format(variable_name)))
ulalume3@0	502
binietoglou@4	503	def file_ok(self, level='error'):
ulalume3@0	504	""" Check if the file checked is ok. What ok means is defined by the level variable """
binietoglou@4	505
binietoglou@4	506	status = None
ulalume3@0	507	if self.checks_run:
ulalume3@0	508	status = True
ulalume3@0	509	for category, result_list in self.check_results.items():
ulalume3@0	510	for result in result_list:
ulalume3@0	511	if ERROR_ORDER[result.level] >= ERROR_ORDER[level]:
ulalume3@0	512	status = False
binietoglou@4	513
ulalume3@0	514	return status
binietoglou@4	515
ulalume3@0	516	def results_for_level(self, level):
ulalume3@0	517	""" Returns all the results of a specific level """
ulalume3@0	518	results = None
ulalume3@0	519	if self.checks_run:
ulalume3@0	520	results = []
ulalume3@0	521	for category, result_list in self.check_results.items():
ulalume3@0	522	for result in result_list:
ulalume3@0	523	if ERROR_ORDER[result.level] == ERROR_ORDER[level]:
ulalume3@0	524	results.append(result)
binietoglou@4	525
ulalume3@0	526	return results
binietoglou@4	527
ulalume3@0	528	def results_by_level(self):
ulalume3@0	529	""" Returns a dictionary with the results by level. """
binietoglou@4	530
ulalume3@0	531	results = {}
ulalume3@0	532	for level, order in ERROR_ORDER.items():
ulalume3@0	533	results[level] = self.results_for_level(level)
binietoglou@4	534
ulalume3@0	535	return results
binietoglou@4	536
ulalume3@0	537	def result_count(self):
ulalume3@0	538	""" Returns a dictionary with the number of results per category. """
binietoglou@4	539
ulalume3@0	540	result_number = {}
ulalume3@0	541	results = self.results_by_level()
binietoglou@4	542
ulalume3@0	543	for category, error_list in results.items():
ulalume3@0	544	if error_list is None:
ulalume3@0	545	result_number[category] = 0
ulalume3@0	546	else:
ulalume3@0	547	result_number[category] = len(error_list)
ulalume3@0	548	return result_number
binietoglou@4	549
ulalume3@0	550	def print_report(self, level):
ulalume3@0	551	""" Print a report for the given level. """
binietoglou@4	552
ulalume3@0	553	print header_template.format(self, self.result_count())
binietoglou@4	554
ulalume3@0	555	results = self.results_by_level()
binietoglou@4	556
ulalume3@0	557	for result_level in ['error', 'warning', 'notification']:
ulalume3@0	558	if ERROR_ORDER[result_level] >= ERROR_ORDER[level]:
ulalume3@0	559	print "\n{0} details".format(result_level.capitalize())
ulalume3@0	560	print "----------------"
ulalume3@0	561	for result in results[result_level]:
binietoglou@4	562	print result
binietoglou@4	563
binietoglou@4	564
binietoglou@4	565	# Sounding file specifications
binietoglou@4	566
ulalume3@0	567
binietoglou@4	568	sounding_specs = {'file': [FilenameShellPattern('rs_*.nc'), ],
binietoglou@4	569	'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4	570	DimensionUnlimited(False), ],
binietoglou@4	571	},
binietoglou@4	572	'variables': {'Altitude': [VariableMandatory(True),
binietoglou@4	573	VariableDimensions(['points', ]),
binietoglou@4	574	VariableType(float)],
binietoglou@4	575	'Temperature': [VariableMandatory(True),
binietoglou@4	576	VariableDimensions(['points', ]),
binietoglou@4	577	VariableType(float)],
binietoglou@4	578	'Pressure': [VariableMandatory(True),
binietoglou@4	579	VariableDimensions(['points', ]),
binietoglou@4	580	VariableType(float)],
binietoglou@4	581	'RelativeHumidity': [VariableMandatory(False),
binietoglou@4	582	VariableDimensions(['points', ]),
binietoglou@4	583	VariableType(float)],
binietoglou@4	584	},
binietoglou@4	585	'attributes': {'Latitude_degrees_north': [AttributeMandatory(True),
binietoglou@4	586	AttributeType(float), ],
binietoglou@4	587	'Longitude_degrees_east': [AttributeMandatory(True),
binietoglou@4	588	AttributeType(float), ],
binietoglou@4	589	'Altitude_meter_asl': [AttributeMandatory(True),
binietoglou@4	590	AttributeType(float), ],
binietoglou@4	591	'Location': [AttributeMandatory(False),
binietoglou@4	592	AttributeType(unicode), ],
binietoglou@4	593	'Sounding_Station_Name': [AttributeMandatory(False),
binietoglou@4	594	AttributeType(unicode), ],
binietoglou@4	595	'WMO_Station_Number': [AttributeMandatory(False),
binietoglou@4	596	AttributeType(unicode), ],
binietoglou@4	597	'WBAN_Station_Number': [AttributeMandatory(False),
binietoglou@4	598	AttributeType(unicode), ],
binietoglou@4	599	'Sounding_Start_Date': [AttributeMandatory(True),
binietoglou@4	600	AttributeType(unicode, block_next=True),
binietoglou@4	601	AttributeStrLength(8)],
binietoglou@4	602	'Sounding_Start_Time_UT': [AttributeMandatory(True),
binietoglou@4	603	AttributeType(unicode, block_next=True),
binietoglou@4	604	AttributeStrLength(6)],
binietoglou@4	605	'Sounding_Stop_Time_UT': [AttributeMandatory(False),
binietoglou@4	606	AttributeType(unicode, block_next=True),
binietoglou@4	607	AttributeStrLength(6)],
binietoglou@4	608	},
binietoglou@4	609	'name': "SCC Sounding file"
binietoglou@4	610	}
ulalume3@0	611
ulalume3@0	612	# Lidar ratio file specifications
binietoglou@4	613	lidar_ratio_specs = {'file': [FilenameShellPattern('*.nc'), ],
binietoglou@4	614	'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4	615	DimensionUnlimited(False), ],
binietoglou@4	616	'products': [DimensionMandatory(True),
binietoglou@4	617	DimensionUnlimited(False), ],
binietoglou@4	618	},
binietoglou@4	619	'variables': {'Altitude': [VariableMandatory(True),
binietoglou@4	620	VariableDimensions(['points', ]),
binietoglou@4	621	VariableType(float)],
binietoglou@4	622	'Lidar_Ratio': [VariableMandatory(True),
binietoglou@4	623	VariableDimensions(['products', 'points']),
ulalume3@0	624	VariableType(float)],
binietoglou@4	625	'product_ID': [VariableMandatory(True),
binietoglou@4	626	VariableDimensions(['products', ]),
binietoglou@4	627	VariableType(int)],
binietoglou@4	628	},
binietoglou@4	629	'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
binietoglou@4	630	AttributeType(unicode), ],
binietoglou@4	631	},
binietoglou@4	632	'name': "SCC Lidar ratio file"
binietoglou@4	633	}
ulalume3@0	634
ulalume3@0	635	# Overlap file specifications
binietoglou@4	636	overlap_specs = {'file': [FilenameShellPattern('ov_*.nc'), ],
binietoglou@4	637	'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4	638	DimensionUnlimited(False), ],
binietoglou@4	639	'channels': [DimensionMandatory(True),
binietoglou@4	640	DimensionUnlimited(False), ],
binietoglou@4	641	},
binietoglou@4	642	'variables': {'Altitude': [VariableMandatory(True),
binietoglou@4	643	VariableDimensions(['points', ]),
ulalume3@0	644	VariableType(float)],
binietoglou@4	645	'Overlap_Function': [VariableMandatory(True),
binietoglou@4	646	VariableDimensions(['channels', 'points']),
ulalume3@0	647	VariableType(float)],
binietoglou@4	648	'channel_ID': [VariableMandatory(True),
binietoglou@4	649	VariableDimensions(['channels', ]),
ulalume3@0	650	VariableType(int)],
binietoglou@4	651	},
ulalume3@0	652	'attributes': {'Lidar_Station_Name': [AttributeMandatory(True),
binietoglou@4	653	AttributeType(unicode, block_next=True),
ulalume3@0	654	AttributeStrLength(2)],
ulalume3@0	655	'Overlap_Measurement_Date': [AttributeMandatory(True),
binietoglou@4	656	AttributeType(unicode, block_next=True),
ulalume3@0	657	AttributeStrLength(8)],
binietoglou@4	658	},
ulalume3@0	659	'name': "SCC Overlap file"
binietoglou@4	660	}
ulalume3@0	661
ulalume3@0	662	# Raw data file specifications
binietoglou@4	663	data_specs = {'file': [FilenameShellPattern('*.nc'), ],
binietoglou@4	664	'dimensions': {'points': [DimensionMandatory(True),
binietoglou@4	665	DimensionUnlimited(False), ],
binietoglou@4	666	'channels': [DimensionMandatory(True),
binietoglou@4	667	DimensionUnlimited(False), ],
binietoglou@4	668	'nb_of_time_scales': [DimensionMandatory(True),
binietoglou@4	669	DimensionUnlimited(False), ],
binietoglou@4	670	'time': [DimensionMandatory(True),
binietoglou@4	671	DimensionUnlimited(True), ],
binietoglou@4	672	'time_bck': [DimensionMandatory(False),
binietoglou@4	673	DimensionUnlimited(False), ],
binietoglou@4	674	'scan_angles': [DimensionMandatory(True),
binietoglou@4	675	DimensionUnlimited(False), ],
binietoglou@4	676	},
binietoglou@4	677	'variables': {'channel_ID': [VariableMandatory(True),
binietoglou@4	678	VariableDimensions(['channels', ]),
binietoglou@4	679	VariableType(int)],
binietoglou@4	680	'Laser_Repetition_Rate': [VariableMandatory(False),
binietoglou@4	681	VariableDimensions(['channels', ]),
ulalume3@0	682	VariableType(int)],
binietoglou@4	683	'Laser_Pointing_Angle': [VariableMandatory(True),
binietoglou@4	684	VariableDimensions(['scan_angles', ]),
binietoglou@4	685	VariableType(float)],
binietoglou@4	686	'ID_Range': [VariableMandatory(False),
binietoglou@4	687	VariableDimensions(['channels', ]),
binietoglou@4	688	VariableType(int)],
binietoglou@4	689	'Scattering_Mechanism': [VariableMandatory(False),
binietoglou@4	690	VariableDimensions(['channels', ]),
binietoglou@4	691	VariableType(int)],
binietoglou@4	692	'Emitted_Wavelength': [VariableMandatory(False),
binietoglou@4	693	VariableDimensions(['channels', ]),
binietoglou@4	694	VariableType(float)],
binietoglou@4	695	'Detected_Wavelength': [VariableMandatory(False),
binietoglou@4	696	VariableDimensions(['channels', ]),
binietoglou@4	697	VariableType(float)],
binietoglou@4	698	'Raw_Data_Range_Resolution': [VariableMandatory(False),
binietoglou@4	699	VariableDimensions(['channels', ]),
binietoglou@4	700	VariableType(float)],
binietoglou@4	701	'Background_Mode': [VariableMandatory(False),
binietoglou@4	702	VariableDimensions(['channels', ]),
binietoglou@4	703	VariableType(int)],
binietoglou@4	704	'Background_Low': [VariableMandatory(True),
binietoglou@4	705	VariableDimensions(['channels', ]),
binietoglou@4	706	VariableType(float)],
binietoglou@4	707	'Background_High': [VariableMandatory(True),
binietoglou@4	708	VariableDimensions(['channels', ]),
binietoglou@4	709	VariableType(float)],
binietoglou@4	710	'Molecular_Calc': [VariableMandatory(True),
binietoglou@4	711	VariableDimensions([]),
ulalume3@0	712	VariableType(int)],
binietoglou@4	713	'id_timescale': [VariableMandatory(True),
binietoglou@4	714	VariableDimensions(['channels', ]),
binietoglou@4	715	VariableType(int)],
binietoglou@4	716	'Dead_Time_Corr_Type': [VariableMandatory(False),
binietoglou@4	717	VariableDimensions(['channels', ]),
binietoglou@4	718	VariableType(int)],
binietoglou@4	719	'Dead_Time': [VariableMandatory(False),
binietoglou@4	720	VariableDimensions(['channels', ]),
binietoglou@4	721	VariableType(float)],
binietoglou@4	722	'Acquisition_Mode': [VariableMandatory(False),
binietoglou@4	723	VariableDimensions(['channels', ]),
binietoglou@4	724	VariableType(int)],
binietoglou@4	725	'Trigger_Delay': [VariableMandatory(False),
binietoglou@4	726	VariableDimensions(['channels', ]),
binietoglou@4	727	VariableType(float)],
binietoglou@4	728	'Laser_Pointing_Angle_of_Profiles': [VariableMandatory(True),
binietoglou@4	729	VariableDimensions(['time', 'nb_of_time_scales', ]),
binietoglou@4	730	VariableType(int)],
binietoglou@4	731	'Raw_Data_Start_Time': [VariableMandatory(True),
binietoglou@4	732	VariableDimensions(['time', 'nb_of_time_scales', ]),
binietoglou@4	733	VariableType(int)],
binietoglou@4	734	'Raw_Data_Stop_Time': [VariableMandatory(True),
binietoglou@4	735	VariableDimensions(['time', 'nb_of_time_scales', ]),
binietoglou@4	736	VariableType(int)],
binietoglou@4	737	'Laser_Shots': [VariableMandatory(True),
binietoglou@4	738	VariableDimensions(['time', 'channels', ]),
ulalume3@0	739	VariableType(int)],
binietoglou@4	740	'Raw_Lidar_Data': [VariableMandatory(False),
binietoglou@4	741	VariableDimensions(['time', 'channels', 'points']),
binietoglou@4	742	VariableType(float)],
binietoglou@4	743	'Depolarization_Factor': [VariableMandatory(False),
binietoglou@4	744	VariableDimensions(['channels', ]),
binietoglou@4	745	VariableType(float)],
binietoglou@4	746	'LR_Input': [VariableMandatory(False),
binietoglou@4	747	VariableDimensions(['channels', ]),
binietoglou@4	748	VariableType(int)],
binietoglou@4	749	'DAQ_Range': [VariableMandatory(False),
binietoglou@4	750	VariableDimensions(['channels', ]),
binietoglou@4	751	VariableType(float)],
binietoglou@4	752	'Pressure_at_Lidar_Station': [VariableMandatory(False),
binietoglou@4	753	VariableDimensions([]),
binietoglou@4	754	VariableType(float)],
binietoglou@4	755	'Temperature_at_Lidar_Station': [VariableMandatory(False),
binietoglou@4	756	VariableDimensions([]),
ulalume3@0	757	VariableType(float)],
binietoglou@4	758	'Background_Profile': [VariableMandatory(False),
binietoglou@4	759	VariableDimensions(['time_bck', 'channels', 'points']),
binietoglou@4	760	VariableType(float)],
binietoglou@4	761	'Raw_Bck_Start_Time': [VariableMandatory(False),
binietoglou@4	762	VariableDimensions(['time_bck', 'nb_of_time_scales', ]),
binietoglou@4	763	VariableType(int)],
binietoglou@4	764	'Raw_Bck_Stop_Time': [VariableMandatory(False),
binietoglou@4	765	VariableDimensions(['time_bck', 'nb_of_time_scales', ]),
binietoglou@4	766	VariableType(int)],
binietoglou@4	767	'Error_On_Raw_Lidar_Data': [VariableMandatory(False),
binietoglou@4	768	VariableDimensions(['time', 'channels', 'points']),
binietoglou@4	769	VariableType(float)],
binietoglou@4	770	'First_Signal_Rangebin': [VariableMandatory(False),
binietoglou@4	771	VariableDimensions(['channels', ]),
ulalume3@0	772	VariableType(int)],
binietoglou@4	773	},
binietoglou@4	774	'attributes': {'Measurement_ID': [AttributeMandatory(True),
binietoglou@4	775	AttributeType(unicode, block_next=True),
binietoglou@4	776	AttributeStrLength(12)],
binietoglou@4	777	'RawData_Start_Date': [AttributeMandatory(True),
binietoglou@4	778	AttributeType(unicode, block_next=True),
binietoglou@4	779	AttributeStrLength(8)],
binietoglou@4	780	'RawData_Start_Time_UT': [AttributeMandatory(True),
binietoglou@4	781	AttributeType(unicode, block_next=True),
binietoglou@4	782	AttributeStrLength(6)],
binietoglou@4	783	'RawData_Stop_Time_UT': [AttributeMandatory(True),
binietoglou@4	784	AttributeType(unicode, block_next=True),
binietoglou@4	785	AttributeStrLength(6)],
binietoglou@4	786	'RawBck_Start_Date': [AttributeMandatory(False),
binietoglou@4	787	AttributeType(unicode, block_next=True),
binietoglou@4	788	AttributeStrLength(8)],
binietoglou@4	789	'RawBck_Start_Time_UT': [AttributeMandatory(False),
binietoglou@4	790	AttributeType(unicode, block_next=True),
binietoglou@4	791	AttributeStrLength(6)],
binietoglou@4	792	'RawBck_Stop_Time_UT': [AttributeMandatory(False),
binietoglou@4	793	AttributeType(unicode, block_next=True),
binietoglou@4	794	AttributeStrLength(6)],
binietoglou@4	795	'Sounding_File_Name': [AttributeMandatory(False),
binietoglou@4	796	AttributeType(unicode), ],
binietoglou@4	797	'LR_File_Name': [AttributeMandatory(False),
binietoglou@4	798	AttributeType(unicode), ],
binietoglou@4	799	'Overlap_File_Name': [AttributeMandatory(False),
binietoglou@4	800	AttributeType(unicode), ],
binietoglou@4	801	'Location': [AttributeMandatory(False),
binietoglou@4	802	AttributeType(unicode), ],
binietoglou@4	803	'System': [AttributeMandatory(False),
binietoglou@4	804	AttributeType(unicode), ],
binietoglou@4	805	'Latitude_degrees_north': [AttributeMandatory(False),
binietoglou@4	806	AttributeType(float), ],
binietoglou@4	807	'Longitude_degrees_east': [AttributeMandatory(False),
binietoglou@4	808	AttributeType(float), ],
binietoglou@4	809	'Altitude_meter_asl': [AttributeMandatory(False),
binietoglou@4	810	AttributeType(float), ],
ulalume3@0	811	},
binietoglou@4	812	'name': "SCC Raw input file"
binietoglou@4	813	}
ulalume3@0	814
ulalume3@0	815	# Used for the command line arguments
ulalume3@0	816	spec_shorthands = {'sounding': sounding_specs,
ulalume3@0	817	'lidar_ratio': lidar_ratio_specs,
ulalume3@0	818	'overlap': overlap_specs,
ulalume3@0	819	'data': data_specs,}
ulalume3@0	820
ulalume3@0	821	if __name__ == "__main__":
ulalume3@0	822	# For use from a terminal
ulalume3@0	823	import argparse
binietoglou@4	824
ulalume3@0	825	parser = argparse.ArgumentParser()
binietoglou@4	826	parser.add_argument("file", help="The path of the file to be checked")
binietoglou@4	827	parser.add_argument("-s", "--specs", default='data',
binietoglou@4	828	help="The specificiations to use",
binietoglou@4	829	choices=['data', 'overlap', 'lidar_ratio', 'sounding'])
binietoglou@4	830	parser.add_argument("-l", "--level", default='warning',
binietoglou@4	831	help="The output level",
binietoglou@4	832	choices=['error', 'warning', 'notification'])
binietoglou@4	833
ulalume3@0	834	# Check the arguments
ulalume3@0	835	args = parser.parse_args()
binietoglou@4	836
ulalume3@0	837	specs = spec_shorthands[args.specs]
binietoglou@4	838
binietoglou@4	839	with FileChecker(args.file, specs) as file_checker:
ulalume3@0	840	file_checker.run_checks()
ulalume3@0	841	file_checker.print_report(args.level)