001 /**
002 *
003 */
004 package org.wdssii.ncingest;
005
006 import java.io.File;
007 import java.io.IOException;
008 import java.util.Calendar;
009 import java.util.Date;
010 import java.util.GregorianCalendar;
011
012 import org.apache.commons.logging.Log;
013 import org.apache.commons.logging.LogFactory;
014 import org.wdssii.core.DataEncoder;
015 import org.wdssii.core.Ingestor;
016 import org.wdssii.core.LatLonGrid;
017 import org.wdssii.core.Location;
018
019 import ucar.ma2.Array;
020 import ucar.ma2.Index;
021 import ucar.nc2.NetcdfFile;
022 import ucar.nc2.Variable;
023
024 /**
025 * A general-purpose ingestor for netcdf files. Reads netcdf files and writes
026 * out WDSS-II format files and notification records.
027 *
028 * @author lakshman
029 *
030 */
031 public class NetcdfIngest extends Ingestor {
032 private static Log log = LogFactory.getLog(NetcdfIngest.class);
033
034 private String typeNameGlobalAttribute = "";
035 private String defaultTypeName = "";
036 private String cornerLatGlobalAttribute = "";
037 private double defaultCornerLat = 35;
038 private String cornerLonGlobalAttribute = "";
039 private double defaultCornerLon = -97;
040 private String heightGlobalAttribute = "";
041 private double defaultHeight = 0;
042 private String timeGlobalAttribute = "";
043 private String julianDateVariable = "";
044 private String timeOfDayVariable = "";
045 private String variableGlobalAttribute = "";
046 private String defaultVariableName = "data";
047 private String latSpacingGlobalAttribute = "";
048 private double defaultLatSpacing = 0.01;
049 private String lonSpacingGlobalAttribute = "";
050 private double defaultLonSpacing = 0.01;
051 private boolean cornerIsNorthwest = true; // otherwise, southwest
052 private String subTypeGlobalAttribute = "";
053 private String defaultSubType = "";
054 private String defaultUnit = "dimensionless";
055
056 public String getTypeNameGlobalAttribute() {
057 return typeNameGlobalAttribute;
058 }
059
060 public void setTypeNameGlobalAttribute(String typeNameGlobalAttribute) {
061 this.typeNameGlobalAttribute = typeNameGlobalAttribute;
062 }
063
064 public String getDefaultTypeName() {
065 return defaultTypeName;
066 }
067
068 public void setDefaultTypeName(String defaultTypeName) {
069 this.defaultTypeName = defaultTypeName;
070 }
071
072 public String getCornerLatGlobalAttribute() {
073 return cornerLatGlobalAttribute;
074 }
075
076 public void setCornerLatGlobalAttribute(String cornerLatGlobalAttribute) {
077 this.cornerLatGlobalAttribute = cornerLatGlobalAttribute;
078 }
079
080 public double getDefaultCornerLat() {
081 return defaultCornerLat;
082 }
083
084 public void setDefaultCornerLat(double defaultCornerLat) {
085 this.defaultCornerLat = defaultCornerLat;
086 }
087
088 public String getCornerLonGlobalAttribute() {
089 return cornerLonGlobalAttribute;
090 }
091
092 public void setCornerLonGlobalAttribute(String cornerLonGlobalAttribute) {
093 this.cornerLonGlobalAttribute = cornerLonGlobalAttribute;
094 }
095
096 public double getDefaultCornerLon() {
097 return defaultCornerLon;
098 }
099
100 public void setDefaultCornerLon(double defaultCornerLon) {
101 this.defaultCornerLon = defaultCornerLon;
102 }
103
104 public String getHeightGlobalAttribute() {
105 return heightGlobalAttribute;
106 }
107
108 public void setHeightGlobalAttribute(String heightGlobalAttribute) {
109 this.heightGlobalAttribute = heightGlobalAttribute;
110 }
111
112 public double getDefaultHeight() {
113 return defaultHeight;
114 }
115
116 public void setDefaultHeight(double defaultHeight) {
117 this.defaultHeight = defaultHeight;
118 }
119
120 public String getTimeGlobalAttribute() {
121 return timeGlobalAttribute;
122 }
123
124 public void setTimeGlobalAttribute(String timeGlobalAttribute) {
125 this.timeGlobalAttribute = timeGlobalAttribute;
126 }
127
128 public String getVariableGlobalAttribute() {
129 return variableGlobalAttribute;
130 }
131
132 public void setVariableGlobalAttribute(String variableGlobalAttribute) {
133 this.variableGlobalAttribute = variableGlobalAttribute;
134 }
135
136 public String getDefaultVariableName() {
137 return defaultVariableName;
138 }
139
140 public void setDefaultVariableName(String defaultVariableName) {
141 this.defaultVariableName = defaultVariableName;
142 }
143
144 public String getLatSpacingGlobalAttribute() {
145 return latSpacingGlobalAttribute;
146 }
147
148 public void setLatSpacingGlobalAttribute(String latSpacingGlobalAttribute) {
149 this.latSpacingGlobalAttribute = latSpacingGlobalAttribute;
150 }
151
152 public double getDefaultLatSpacing() {
153 return defaultLatSpacing;
154 }
155
156 public void setDefaultLatSpacing(double defaultLatSpacing) {
157 this.defaultLatSpacing = defaultLatSpacing;
158 }
159
160 public String getLonSpacingGlobalAttribute() {
161 return lonSpacingGlobalAttribute;
162 }
163
164 public void setLonSpacingGlobalAttribute(String lonSpacingGlobalAttribute) {
165 this.lonSpacingGlobalAttribute = lonSpacingGlobalAttribute;
166 }
167
168 public double getDefaultLonSpacing() {
169 return defaultLonSpacing;
170 }
171
172 public void setDefaultLonSpacing(double defaultLonSpacing) {
173 this.defaultLonSpacing = defaultLonSpacing;
174 }
175
176 public boolean isCornerIsNorthwest() {
177 return cornerIsNorthwest;
178 }
179
180 public void setCornerIsNorthwest(boolean cornerIsNorthwest) {
181 this.cornerIsNorthwest = cornerIsNorthwest;
182 }
183
184 public String getSubTypeGlobalAttribute() {
185 return subTypeGlobalAttribute;
186 }
187
188 public void setSubTypeGlobalAttribute(String subTypeGlobalAttribute) {
189 this.subTypeGlobalAttribute = subTypeGlobalAttribute;
190 }
191
192 public String getDefaultSubType() {
193 return defaultSubType;
194 }
195
196 public void setDefaultSubType(String defaultSubType) {
197 this.defaultSubType = defaultSubType;
198 }
199
200 public String getJulianDateVariable() {
201 return julianDateVariable;
202 }
203
204 public void setJulianDateVariable(String julianDateVariable) {
205 this.julianDateVariable = julianDateVariable;
206 }
207
208 public String getTimeOfDayVariable() {
209 return timeOfDayVariable;
210 }
211
212 public void setTimeOfDayVariable(String timeOfDayVariable) {
213 this.timeOfDayVariable = timeOfDayVariable;
214 }
215
216 public String getDefaultUnit() {
217 return defaultUnit;
218 }
219
220 public void setDefaultUnit(String defaultUnit) {
221 this.defaultUnit = defaultUnit;
222 }
223
224 private String getValue(NetcdfFile ncfile, String what, String attrName,
225 String value) {
226 if (attrName.length() != 0) {
227 return ncfile.findGlobalAttribute(attrName).getStringValue();
228 } else if (value.length() != 0) {
229 return value;
230 } else {
231 throw new IllegalStateException("Need to specify " + what);
232 }
233 }
234
235 private long getValue(NetcdfFile ncfile, String what, String attrName) {
236 if (attrName.length() != 0) {
237 return ncfile.findGlobalAttribute(attrName).getNumericValue()
238 .longValue();
239 } else {
240 throw new IllegalStateException("Need to specify " + what);
241 }
242 }
243
244 private double getValue(NetcdfFile ncfile, String what, String attrName,
245 double value) {
246 if (attrName.length() != 0) {
247 return ncfile.findGlobalAttribute(attrName).getNumericValue()
248 .doubleValue();
249 } else {
250 return value;
251 }
252 }
253
254 private float[][] readData2D(NetcdfFile ncfile) throws IOException {
255 String variableName = getValue(ncfile, "VariableName",
256 variableGlobalAttribute, defaultVariableName);
257 Variable data = ncfile.findVariable(variableName);
258 int numDims = data.getDimensions().size();
259 if (numDims < 2) {
260 throw new IllegalArgumentException(data.getName()
261 + " should have at least 2 dimensions");
262 }
263 Array gate_values = data.read();
264 int num_radials = data.getDimension(numDims - 2).getLength();
265 int num_gates = data.getDimension(numDims - 1).getLength();
266 float[][] values = new float[num_radials][num_gates];
267 Index gate_index = gate_values.getIndex();
268 int[] index = new int[numDims];
269 for (int i = 0; i < num_radials; ++i) {
270 for (int j = 0; j < num_gates; ++j) {
271 index[index.length - 2] = i;
272 index[index.length - 1] = j;
273 gate_index.set(index);
274 values[i][j] = gate_values.getFloat(gate_index);
275 }
276 }
277 return values;
278 }
279
280 @Override
281 protected void doIngest(File path) {
282 NetcdfFile ncfile = null;
283
284 try {
285 log.info("Opening " + path + " for reading");
286 ncfile = NetcdfFile.open(path.getAbsolutePath());
287
288 // expected global variables
289 String typeName = getValue(ncfile, "typeName",
290 typeNameGlobalAttribute, defaultTypeName);
291 // String dataType = "LatLonGrid";
292
293 // location
294 double lat = getValue(ncfile, "Latitude", cornerLatGlobalAttribute,
295 defaultCornerLat);
296 double lon = getValue(ncfile, "Longitude",
297 cornerLonGlobalAttribute, defaultCornerLon);
298 double ht = getValue(ncfile, "Height", heightGlobalAttribute,
299 defaultHeight);
300
301 // time
302 Date dt;
303 if (timeGlobalAttribute.length() != 0) {
304 long tm = getValue(ncfile, "TimeUTC", timeGlobalAttribute);
305 long tm_long = 1000 * tm;
306 dt = new Date(tm_long);
307 } else if (julianDateVariable.length() != 0
308 && timeOfDayVariable.length() != 0) {
309 int julianDate = ncfile.findVariable(julianDateVariable)
310 .readScalarInt();
311 int timeOfDay = ncfile.findVariable(timeOfDayVariable)
312 .readScalarInt();
313 Calendar c = GregorianCalendar.getInstance();
314 c.set(julianDate / 1000 + 1900, 0, julianDate % 1000, 0, 0,
315 timeOfDay);
316 dt = c.getTime();
317 } else {
318 throw new IllegalStateException(
319 "Need to provide either the timeGlobalAttribute or the julianDateVariable/timeOfDayVariable");
320 }
321 // spacing
322 double deltaLat = Math.abs(getValue(ncfile, "LatSpacing",
323 latSpacingGlobalAttribute, defaultLatSpacing));
324 double deltaLon = Math.abs(getValue(ncfile, "LonSpacing",
325 lonSpacingGlobalAttribute, defaultLonSpacing));
326
327 // values
328 float[][] values = readData2D(ncfile);
329 int numLat = values.length;
330 // int numLon = values[0].length;
331
332 // northwest corner; flip rows if necessary
333 Location nwcorner;
334 if (cornerIsNorthwest) {
335 nwcorner = new Location(lat, lon, ht / 1000);
336 // no need to flip
337 } else {
338 nwcorner = new Location(lat + (numLat + 1) * deltaLat, lon,
339 ht / 1000);
340 float[][] tmp = new float[numLat][];
341 for (int i = 0; i < numLat; ++i) {
342 tmp[i] = values[numLat - i];
343 }
344 values = tmp;
345 }
346
347 // unit
348 String unit = getValue(ncfile, "Unit", "", defaultUnit);
349
350 // create latlongrid, write and notify
351 LatLonGrid llgrid = new LatLonGrid(nwcorner, dt, typeName,
352 (float) deltaLat, (float) deltaLon, values);
353 llgrid.setAttribute("Unit", unit);
354
355 String subtype = null;
356 try {
357 subtype = getValue(ncfile, "SubType", subTypeGlobalAttribute,
358 defaultSubType);
359 } catch (Exception e) {
360 // ok if not there
361 }
362 DataEncoder.writeDataAndNotify(llgrid, super.getOutputDir(),
363 subtype);
364
365 } catch (Exception e) {
366 log.warn(e);
367 } finally {
368 try {
369 if (ncfile != null)
370 ncfile.close();
371 } catch (Exception e) {
372 }
373 }
374 }
375
376 public static void main(String[] args) {
377 final NetcdfIngest alg = new NetcdfIngest();
378 alg.setupAndExecute(args);
379 }
380
381 }