1 import numpy as np |
|
2 import datetime |
|
3 from generic import BaseLidarMeasurement, LidarChannel |
|
4 import musa_netcdf_parameters |
|
5 import musa_2009_netcdf_parameters |
|
6 |
|
7 licel_file_header_format = ['Filename', |
|
8 'StartDate StartTime EndDate EndTime Altitude Longtitude Latitude ZenithAngle', # Appart from Site that is read manually |
|
9 'LS1 Rate1 LS2 Rate2 DataSets', ] |
|
10 licel_file_channel_format = 'Active AnalogPhoton LaserUsed DataPoints 1 HV BinW Wavelength d1 d2 d3 d4 ADCbits NShots Discriminator ID' |
|
11 |
|
12 |
|
13 class LicelFile: |
|
14 |
|
15 def __init__(self, filename): |
|
16 self.start_time = None |
|
17 self.stop_time = None |
|
18 self.import_file(filename) |
|
19 self.calculate_physical() |
|
20 self.filename = filename |
|
21 |
|
22 def calculate_physical(self): |
|
23 for channel in self.channels.itervalues(): |
|
24 channel.calculate_physical() |
|
25 |
|
26 def import_file(self, filename): |
|
27 """Imports a licel file. |
|
28 Input: filename |
|
29 Output: object """ |
|
30 |
|
31 channels = {} |
|
32 |
|
33 with open(filename, 'rb') as f: |
|
34 |
|
35 self.read_header(f) |
|
36 |
|
37 # Check the complete header is read |
|
38 a = f.readline() |
|
39 |
|
40 # Import the data |
|
41 for current_channel_info in self.channel_info: |
|
42 raw_data = np.fromfile(f, 'i4', int(current_channel_info['DataPoints'])) |
|
43 a = np.fromfile(f, 'b', 1) |
|
44 b = np.fromfile(f, 'b', 1) |
|
45 |
|
46 if (a[0] != 13) | (b[0] != 10): |
|
47 print "Warning: No end of line found after record. File could be corrupt" |
|
48 channel = LicelFileChannel(current_channel_info, raw_data, self.duration()) |
|
49 |
|
50 channel_name = channel.channel_name |
|
51 |
|
52 if channel_name in channels.keys(): |
|
53 # If the analog/photon naming scheme is not enough, find a new one! |
|
54 raise IOError('Trying to import two channels with the same name') |
|
55 |
|
56 channels[channel_name] = channel |
|
57 |
|
58 self.channels = channels |
|
59 |
|
60 |
|
61 def read_header(self, f): |
|
62 """ Read the header of a open file f. |
|
63 |
|
64 Returns raw_info and channel_info. Updates some object properties. """ |
|
65 |
|
66 #Read the first 3 lines of the header |
|
67 raw_info = {} |
|
68 channel_info = [] |
|
69 |
|
70 # Read first line |
|
71 raw_info['Filename'] = f.readline().strip() |
|
72 |
|
73 # Read second line |
|
74 second_line = f.readline() |
|
75 |
|
76 # Many licel files don't follow the licel standard. Specifically, the |
|
77 # measurement site is not always 8 characters, and can include white |
|
78 # spaces. For this, the site name is detect everything before the first |
|
79 # date. For efficiency, the first date is found by the first '/'. |
|
80 # e.g. assuming a string like 'Site name 01/01/2010 ...' |
|
81 |
|
82 site_name = second_line.split('/')[0][:-2] |
|
83 clean_site_name = site_name.strip() |
|
84 raw_info['Site'] = clean_site_name |
|
85 raw_info.update(match_lines(second_line[len(clean_site_name) + 1:], licel_file_header_format[1])) |
|
86 |
|
87 # Read third line |
|
88 third_line = f.readline() |
|
89 raw_info.update(match_lines(third_line, licel_file_header_format[2])) |
|
90 |
|
91 # Update the object properties based on the raw info |
|
92 start_string = '%s %s' % (raw_info['StartDate'], raw_info['StartTime']) |
|
93 stop_string = '%s %s' % (raw_info['EndDate'], raw_info['EndTime']) |
|
94 date_format = '%d/%m/%Y %H:%M:%S' |
|
95 |
|
96 self.start_time = datetime.datetime.strptime(start_string, date_format) |
|
97 self.stop_time = datetime.datetime.strptime(stop_string, date_format) |
|
98 self.latitude = float(raw_info['Latitude']) |
|
99 self.longitude = float(raw_info['Longtitude']) |
|
100 |
|
101 # Read the rest of the header. |
|
102 for c1 in range(int(raw_info['DataSets'])): |
|
103 channel_info.append(match_lines(f.readline(), licel_file_channel_format)) |
|
104 |
|
105 self.raw_info = raw_info |
|
106 self.channel_info = channel_info |
|
107 |
|
108 def duration(self): |
|
109 """ Return the duration of the file. """ |
|
110 dt = self.stop_time - self.start_time |
|
111 return dt.seconds |
|
112 |
|
113 |
|
114 class LicelFileChannel: |
|
115 |
|
116 def __init__(self, raw_info = None, raw_data = None, duration = None): |
|
117 self.raw_info = raw_info |
|
118 self.raw_data = raw_data |
|
119 self.duration = duration |
|
120 |
|
121 @property |
|
122 def wavelength(self): |
|
123 if self.raw_info is not None: |
|
124 wave_str = self.raw_info['Wavelength'] |
|
125 wavelength = wave_str.split('.')[0] |
|
126 return int(wavelength) |
|
127 else: |
|
128 return None |
|
129 |
|
130 @property |
|
131 def channel_name(self): |
|
132 ''' |
|
133 Construct the channel name adding analog photon info to avoid duplicates |
|
134 ''' |
|
135 acquisition_type = self.analog_photon_string(self.raw_info['AnalogPhoton']) |
|
136 channel_name = "%s_%s" % (self.raw_info['Wavelength'], acquisition_type) |
|
137 return channel_name |
|
138 |
|
139 def analog_photon_string(self, analog_photon_number): |
|
140 if analog_photon_number == '0': |
|
141 string = 'an' |
|
142 else: |
|
143 string = 'ph' |
|
144 return string |
|
145 |
|
146 def calculate_physical(self): |
|
147 data = self.raw_data |
|
148 |
|
149 number_of_shots = float(self.raw_info['NShots']) |
|
150 norm = data / number_of_shots |
|
151 dz = float(self.raw_info['BinW']) |
|
152 |
|
153 if self.raw_info['AnalogPhoton']=='0': |
|
154 # If the channel is in analog mode |
|
155 ADCb = int(self.raw_info['ADCbits']) |
|
156 ADCrange = float(self.raw_info['Discriminator'])*1000 # Value in mV |
|
157 channel_data = norm*ADCrange/((2**ADCb)-1) |
|
158 |
|
159 # print ADCb, ADCRange,cdata,norm |
|
160 else: |
|
161 # If the channel is in photoncounting mode |
|
162 # Frequency deduced from range resolution! (is this ok?) |
|
163 # c = 300 # The result will be in MHZ |
|
164 # SR = c/(2*dz) # To account for pulse folding |
|
165 # channel_data = norm*SR |
|
166 # CHANGE: |
|
167 # For the SCC the data are needed in photons |
|
168 channel_data = norm *number_of_shots |
|
169 #print res,c,cdata,norm |
|
170 |
|
171 #Calculate Z |
|
172 number_of_bins = int(self.raw_info['DataPoints']) |
|
173 self.z = np.array([dz*bin_number + dz/2.0 for bin_number in range(number_of_bins)]) |
|
174 self.dz = dz |
|
175 self.number_of_bins = number_of_bins |
|
176 self.data = channel_data |
|
177 |
|
178 |
|
179 class LicelLidarMeasurement(BaseLidarMeasurement): |
|
180 ''' |
|
181 |
|
182 ''' |
|
183 extra_netcdf_parameters = musa_netcdf_parameters |
|
184 raw_info = {} # Keep the raw info from the files |
|
185 durations = {} # Keep the duration of the files |
|
186 |
|
187 def import_file(self, filename): |
|
188 if filename in self.files: |
|
189 print "File has been imported already:" + filename |
|
190 else: |
|
191 current_file = LicelFile(filename) |
|
192 self.raw_info[current_file.filename] = current_file.raw_info |
|
193 self.durations[current_file.filename] = current_file.duration() |
|
194 |
|
195 for channel_name, channel in current_file.channels.items(): |
|
196 if channel_name not in self.channels: |
|
197 self.channels[channel_name] = LicelChannel(channel) |
|
198 self.channels[channel_name].data[current_file.start_time] = channel.data |
|
199 self.files.append(current_file.filename) |
|
200 |
|
201 def append(self, other): |
|
202 |
|
203 self.start_times.extend(other.start_times) |
|
204 self.stop_times.extend(other.stop_times) |
|
205 |
|
206 for channel_name, channel in self.channels.items(): |
|
207 channel.append(other.channels[channel_name]) |
|
208 |
|
209 def _get_duration(self, raw_start_in_seconds): |
|
210 ''' Return the duration for a given time scale. If only a single |
|
211 file is imported, then this cannot be guessed from the time difference |
|
212 and the raw_info of the file are checked. |
|
213 ''' |
|
214 |
|
215 if len(raw_start_in_seconds) == 1: # If only one file imported |
|
216 duration = self.durations.itervalues().next() # Get the first (and only) raw_info |
|
217 duration_sec = duration |
|
218 else: |
|
219 duration_sec = np.diff(raw_start_in_seconds)[0] |
|
220 |
|
221 return duration_sec |
|
222 |
|
223 |
|
224 class LicelChannel(LidarChannel): |
|
225 |
|
226 def __init__(self, channel_file): |
|
227 c = 299792458.0 # Speed of light |
|
228 self.wavelength = channel_file.wavelength |
|
229 self.name = channel_file.channel_name |
|
230 self.binwidth = channel_file.dz * 2 / c # in seconds |
|
231 self.data = {} |
|
232 self.resolution = channel_file.dz |
|
233 self.z = np.arange(channel_file.number_of_bins) * self.resolution + self.resolution/2.0 #Change: add half bin in the z |
|
234 self.points = channel_file.number_of_bins |
|
235 self.rc = [] |
|
236 self.duration = channel_file.duration |
|
237 |
|
238 def append(self, other): |
|
239 if self.info != other.info: |
|
240 raise ValueError('Channel info are different. Data can not be combined.') |
|
241 |
|
242 self.data = np.vstack([self.data, other.data]) |
|
243 |
|
244 def __unicode__(self): |
|
245 return "<Licel channel: %s>" % self.name |
|
246 |
|
247 def __str__(self): |
|
248 return unicode(self).encode('utf-8') |
|
249 |
|
250 |
|
251 class Licel2009LidarMeasurement(LicelLidarMeasurement): |
|
252 extra_netcdf_parameters = musa_2009_netcdf_parameters |
|
253 |
|
254 |
|
255 def match_lines(f1,f2): |
|
256 list1 = f1.split() |
|
257 list2 = f2.split() |
|
258 |
|
259 if len(list1) != len(list2): |
|
260 print "Warning: Combining lists of different lengths." |
|
261 print "List 1: %s" % list1 |
|
262 print "List 2: %s" % list2 |
|
263 combined = zip(list2,list1) |
|
264 combined = dict(combined) |
|
265 return combined |
|
266 |
|