001 /**
002 *
003 */
004 package org.wdssii.core;
005
006 import java.io.File;
007 import java.io.FileWriter;
008 import java.io.IOException;
009 import java.io.PrintWriter;
010 import java.text.SimpleDateFormat;
011 import java.util.Date;
012 import java.util.Iterator;
013 import java.util.Map.Entry;
014
015 import org.apache.commons.logging.Log;
016 import org.apache.commons.logging.LogFactory;
017
018 import ucar.ma2.ArrayFloat;
019 import ucar.nc2.Attribute;
020 import ucar.nc2.Dimension;
021 import ucar.nc2.NetcdfFileWriteable;
022
023 /**
024 * @author lakshman
025 *
026 */
027 public class NetcdfDataEncoder extends DataEncoder {
028 private static Log log = LogFactory.getLog(NetcdfDataEncoder.class);
029
030 protected NetcdfDataEncoder(){
031 }
032
033 private String compress(String fileName, String relativeFileName) throws IOException {
034 if ( this.isCompressionEnabled() ){
035 try{
036 new File( fileName + ".gz" ).delete();
037 } catch (Exception e){
038 // ok if file doesn't exist
039 }
040 Runtime.getRuntime().exec("gzip " + fileName);
041 relativeFileName += ".gz";
042 }
043 return relativeFileName;
044 }
045
046 @Override
047 protected String[] getParams(String relativeFileName){
048 return new String[] { "netcdf", "{indexlocation}", relativeFileName };
049 }
050
051 @Override
052 protected void writeAndNotify(PolarGrid pg, String outputDir, String[] subtypes) {
053 boolean sparse = shouldWriteSparse(pg.getValues(), DataType.MissingData);
054 NetcdfFileWriteable ncfile = null;
055 try {
056 // filename
057 String formattedDate = getFormattedDate(pg);
058 String relativeFileName = getRelativeFileName(pg, outputDir,
059 formattedDate, subtypes);
060 String fileName = outputDir + '/' + relativeFileName;
061 if (log.isDebugEnabled()) {
062 log.debug("Writing to " + fileName);
063 }
064 // create sparse/regular grid
065 NetcdfSparseGrid sparseGrid = null;
066 NetcdfRegularGrid netcdfGrid = null;
067 if ( sparse ){
068 sparseGrid = new NetcdfSparseGrid(pg.getValues(), DataType.MissingData);
069 } else {
070 netcdfGrid = new NetcdfRegularGrid(pg.getValues());
071 }
072 int numRadials = pg.getNumRadials();
073 int numGates = pg.getNumGates();
074
075 // create netcdf file and set up dimensions
076 ncfile = NetcdfFileWriteable.createNew(fileName, false);
077 Dimension azdim = ncfile.addDimension("Azimuth", numRadials);
078 Dimension rndim = ncfile.addDimension("Gate", numGates);
079
080 ncfile.addVariable("Azimuth", ucar.ma2.DataType.FLOAT,
081 new Dimension[] { azdim });
082 ncfile.addVariableAttribute("Azimuth", new Attribute("Units","Degrees"));
083 ncfile.addVariable("BeamWidth", ucar.ma2.DataType.FLOAT,
084 new Dimension[] { azdim });
085 ncfile.addVariableAttribute("BeamWidth", new Attribute("Units","Degrees"));
086 ncfile.addVariable("AzimuthalSpacing", ucar.ma2.DataType.FLOAT,
087 new Dimension[] { azdim });
088 ncfile.addVariableAttribute("AzimuthalSpacing", new Attribute("Units","Degrees"));
089 ncfile.addVariable("GateWidth", ucar.ma2.DataType.FLOAT,
090 new Dimension[] { azdim });
091 ncfile.addVariableAttribute("GateWidth", new Attribute("Units","Meters"));
092 ncfile.addVariable("NyquistVelocity", ucar.ma2.DataType.FLOAT,
093 new Dimension[] { azdim });
094 ncfile.addVariableAttribute("NyquistVelocity", new Attribute("Units","MetersPerSecond"));
095
096 Dimension pxdim = null;
097 if ( sparse ){
098 int numPixels = (int) (sparseGrid.getValues().getSize());
099 pxdim = ncfile.addDimension("pixel", numPixels);
100 ncfile.addVariable(pg.getTypeName(), ucar.ma2.DataType.FLOAT,
101 new Dimension[] { pxdim });
102 ncfile.addVariable("pixel_x", ucar.ma2.DataType.SHORT,
103 new Dimension[] { pxdim });
104 ncfile.addVariable("pixel_y", ucar.ma2.DataType.SHORT,
105 new Dimension[] { pxdim });
106 ncfile.addVariable("pixel_count", ucar.ma2.DataType.INT,
107 new Dimension[] { pxdim });
108 ncfile.addVariableAttribute(pg.getTypeName(),new Attribute("BackgroundValue", DataType.MissingData));
109 } else {
110 ncfile.addVariable(pg.getTypeName(), ucar.ma2.DataType.FLOAT,
111 new Dimension[] { azdim,rndim });
112 }
113
114 ncfile.addVariableAttribute(pg.getTypeName(), new Attribute("Units",pg.getUnit()));
115 // follow netcdf best practices convention
116 ncfile.addVariableAttribute(pg.getTypeName(), new Attribute("units",pg.getUnit()));
117 ncfile.addVariableAttribute(pg.getTypeName(), new Attribute("missing_value",DataType.MissingData));
118
119 ncfile.addGlobalAttribute("Elevation", pg.getElevation());
120 ncfile.addGlobalAttribute("ElevationUnits", "Degrees");
121 ncfile.addGlobalAttribute("RangeToFirstGate", pg.getRangeToFirstGateKms() * 1000 );
122 ncfile.addGlobalAttribute("RangeToFirstGateUnits", "Meters" );
123
124 if (sparse){
125 addGlobalAttributes(pg, ncfile, "SparseRadialSet");
126 } else {
127 addGlobalAttributes(pg, ncfile, "RadialSet");
128 }
129 ncfile.create();
130
131 // write the variables
132 ArrayFloat az = new ArrayFloat(new int[] { numRadials });
133 ArrayFloat bw = new ArrayFloat(new int[] { numRadials });
134 ArrayFloat as = new ArrayFloat(new int[] { numRadials });
135 ArrayFloat gw = new ArrayFloat(new int[] { numRadials });
136 ArrayFloat ny = new ArrayFloat(new int[] { numRadials });
137 ucar.ma2.Index azindex = az.getIndex();
138 for (int i = 0; i < numRadials; ++i) {
139 azindex.set(i);
140 az.set(azindex, pg.getAngularResolution() * i);
141 bw.set(azindex, pg.getBeamWidth());
142 as.set(azindex, pg.getAngularResolution());
143 gw.set(azindex, pg.getGateWidthKms() * 1000); // meters in netcdf file
144 ny.set(azindex, pg.getNyquist());
145 }
146 ncfile.write("Azimuth", az);
147 ncfile.write("BeamWidth", bw);
148 ncfile.write("AzimuthalSpacing", as);
149 ncfile.write("GateWidth", gw);
150 ncfile.write("NyquistVelocity", ny);
151
152 if (sparse){
153 ncfile.write(pg.getTypeName(), sparseGrid.getValues());
154 ncfile.write("pixel_x", sparseGrid.getPixelx());
155 ncfile.write("pixel_y", sparseGrid.getPixely());
156 ncfile.write("pixel_count", sparseGrid.getPixelcount());
157 } else {
158 ncfile.write(pg.getTypeName(), netcdfGrid.getValues());
159 }
160 // close the file. Set to null so that finally doesn't try to close
161 // again
162 ncfile.close();
163 ncfile = null;
164
165 // invoke gzip on the file
166 relativeFileName = compress(fileName, relativeFileName);
167
168 // create index
169 notifyRecord(pg, outputDir, subtypes, formattedDate,
170 relativeFileName);
171
172 if (log.isInfoEnabled()){
173 log.info(relativeFileName + " written");
174 }
175 } catch (Exception e) {
176 log.error("Writing failed: ", e);
177 } finally {
178 if (ncfile != null) {
179 try {
180 if (ncfile != null){
181 ncfile.close();
182 }
183 } catch (IOException e) {
184 }
185 }
186 }
187 }
188
189 protected void writeAndNotify(LatLonGrid llg, String outputDir, String[] subtypes) {
190 boolean sparse = shouldWriteSparse(llg.getValues(), DataType.MissingData);
191 NetcdfFileWriteable ncfile = null;
192 try {
193 // filename
194 String formattedDate = getFormattedDate(llg);
195 String relativeFileName = getRelativeFileName(llg, outputDir,
196 formattedDate, subtypes);
197 String fileName = outputDir + '/' + relativeFileName;
198 if (log.isDebugEnabled()) {
199 log.debug("Writing to " + fileName);
200 }
201 // create sparse/regular grid
202 NetcdfSparseGrid sparseGrid = null;
203 NetcdfRegularGrid netcdfGrid = null;
204 if ( sparse ){
205 sparseGrid = new NetcdfSparseGrid(llg.getValues(), DataType.MissingData);
206 } else {
207 netcdfGrid = new NetcdfRegularGrid(llg.getValues());
208 }
209 int numLat = llg.getNumLat();
210 int numLon = llg.getNumLon();
211
212 // create netcdf file and set up dimensions
213 ncfile = NetcdfFileWriteable.createNew(fileName, false);
214 Dimension azdim = ncfile.addDimension("Lat", numLat);
215 Dimension rndim = ncfile.addDimension("Lon", numLon);
216
217 Dimension pxdim = null;
218 if ( sparse ){
219 int numPixels = (int) (sparseGrid.getValues().getSize());
220 pxdim = ncfile.addDimension("pixel", numPixels);
221 ncfile.addVariable(llg.getTypeName(), ucar.ma2.DataType.FLOAT,
222 new Dimension[] { pxdim });
223 ncfile.addVariable("pixel_x", ucar.ma2.DataType.SHORT,
224 new Dimension[] { pxdim });
225 ncfile.addVariable("pixel_y", ucar.ma2.DataType.SHORT,
226 new Dimension[] { pxdim });
227 ncfile.addVariable("pixel_count", ucar.ma2.DataType.INT,
228 new Dimension[] { pxdim });
229 ncfile.addVariableAttribute(llg.getTypeName(),new Attribute("BackgroundValue", DataType.MissingData));
230 } else {
231 ncfile.addVariable(llg.getTypeName(), ucar.ma2.DataType.FLOAT,
232 new Dimension[] { azdim,rndim });
233 }
234
235 ncfile.addVariableAttribute(llg.getTypeName(), new Attribute("Units",llg.getUnit()));
236 // follow netcdf best practices convention
237 ncfile.addVariableAttribute(llg.getTypeName(), new Attribute("units",llg.getUnit()));
238 ncfile.addVariableAttribute(llg.getTypeName(), new Attribute("missing_value",DataType.MissingData));
239
240 ncfile.addGlobalAttribute("LatGridSpacing", llg.getDeltaLat() );
241 ncfile.addGlobalAttribute("LatGridSpacingUnits", "Degrees");
242 ncfile.addGlobalAttribute("LonGridSpacing", llg.getDeltaLon() );
243 ncfile.addGlobalAttribute("LonGridSpacingUnits", "Degrees");
244
245 if (sparse){
246 addGlobalAttributes(llg, ncfile, "SparseLatLonGrid");
247 } else {
248 addGlobalAttributes(llg, ncfile, "LatLonGrid");
249 }
250 ncfile.create();
251
252 // write the variables
253 if (sparse){
254 ncfile.write(llg.getTypeName(), sparseGrid.getValues());
255 ncfile.write("pixel_x", sparseGrid.getPixelx());
256 ncfile.write("pixel_y", sparseGrid.getPixely());
257 ncfile.write("pixel_count", sparseGrid.getPixelcount());
258 } else {
259 ncfile.write(llg.getTypeName(), netcdfGrid.getValues());
260 }
261 // close the file. Set to null so that finally doesn't try to close
262 // again
263 ncfile.close();
264 ncfile = null;
265
266 // invoke gzip on the file
267 relativeFileName = compress(fileName, relativeFileName);
268
269 // create index
270 notifyRecord(llg, outputDir, subtypes, formattedDate,
271 relativeFileName);
272
273 if (log.isInfoEnabled()){
274 log.info(relativeFileName + " written");
275 }
276 } catch (Exception e) {
277 log.error("Writing failed: ", e);
278 } finally {
279 if (ncfile != null) {
280 try {
281 if (ncfile != null){
282 ncfile.close();
283 }
284 } catch (IOException e) {
285 }
286 }
287 }
288 }
289
290 private static void addGlobalAttributes(DataType dt,
291 NetcdfFileWriteable ncfile, String dataType) {
292 ncfile.addGlobalAttribute("TypeName", dt.getTypeName());
293 ncfile.addGlobalAttribute("DataType", dataType);
294 ncfile.addGlobalAttribute("Latitude", dt.getLocation().getLatitude());
295 ncfile.addGlobalAttribute("Longitude", dt.getLocation().getLongitude());
296 ncfile.addGlobalAttribute("Height", dt.getLocation().getHeightKms() * 1000);
297 ncfile.addGlobalAttribute("MissingData", DataType.MissingData);
298 ncfile.addGlobalAttribute("RangeFolded", DataType.RangeFolded);
299
300 long tm_long = dt.getTime().getTime();
301 long tm = tm_long / 1000;
302 double frac = (tm_long - tm * 1000) / 1000.0;
303 ncfile.addGlobalAttribute("Time", (int) tm); // FIXME: netcdf 3 does
304 // not support long
305 ncfile.addGlobalAttribute("FractionalTime", frac);
306 Iterator<Entry<String, String>> attributes = dt.getAttributes()
307 .entrySet().iterator();
308 Iterator<String> attributeKeys = dt.getAttributes().keySet().iterator();
309 StringBuilder attributeNames = new StringBuilder();
310 while (attributeKeys.hasNext()) {
311 attributeNames.append(attributeKeys.next()).append(' ');
312 }
313 ncfile.addGlobalAttribute("attributes", attributeNames.toString());
314 while (attributes.hasNext()) {
315 Entry<String, String> entry = attributes.next();
316 ncfile.addGlobalAttribute(entry.getKey() + "-value", entry
317 .getValue());
318 }
319 }
320
321 private static String getRelativeFileName(DataType dt, String outputDir,
322 String formattedDate, String[] subtypes) throws IOException {
323 String subtype = (subtypes.length > 0) ? subtypes[0] : "";
324
325 StringBuilder dirName = new StringBuilder();
326 dirName.append(dt.getTypeName()).append("/");
327 dirName.append(subtype).append("/");
328
329 new File(outputDir + '/' + dirName.toString()).mkdirs();
330
331 String fileName = dirName.append(formattedDate).append(".netcdf")
332 .toString();
333 return fileName;
334 }
335
336 @Override
337 protected void writeAndNotify(DataTable dt, String outputDir,
338 String[] subtypes) {
339 throw new UnsupportedOperationException("DataTable netcdf format not defined");
340 }
341 }