001    /**
002     * 
003     */
004    package org.wdssii.core;
005    
006    /**
007     * @author lakshman
008     * 
009     */
010    import java.io.IOException;
011    import java.util.Arrays;
012    import java.util.Date;
013    import java.util.HashMap;
014    import java.util.List;
015    import java.util.Map;
016    
017    import org.apache.commons.logging.Log;
018    import org.apache.commons.logging.LogFactory;
019    
020    import ucar.ma2.Array;
021    import ucar.ma2.Index;
022    import ucar.nc2.Dimension;
023    import ucar.nc2.NetcdfFile;
024    import ucar.nc2.Variable;
025    
026    /**
027     * Reads a local/remote netcdf file (gzipped or not) and builds a DataType
028     * 
029     * @author Lakshman
030     * @version $Id: NetcdfBuilder.java,v 1.8 2007/11/20 21:03:59 lakshman Exp $
031     */
032    public class NetcdfBuilder implements Builder {
033            private static Log log = LogFactory.getLog(NetcdfBuilder.class);
034    
035            public NetcdfBuilder() {
036            }
037    
038            /** create object if the record params are netcdf ones. */
039            public DataType createObject(IndexRecord rec) {
040                    String[] params = rec.getParams();
041                    StringBuilder path = new StringBuilder(params[1]);
042                    for (int i = 2; i < params.length; ++i) {
043                            path.append('/').append(params[i]);
044                    }
045                    return createObject(path.toString());
046            }
047    
048            /** pass in the file name and obtain an object back. */
049            public DataType createObject(String path) {
050                    int colon = path.indexOf(':');
051                    if (colon < 0) {
052                            return fromNetcdfFile(path);
053                    } else {
054                            throw new FormatException("Can not read remote files");
055                    }
056            }
057    
058            @SuppressWarnings("serial")
059            private static class FormatException extends DataUnavailableException {
060                    FormatException(String msg) {
061                            super(msg);
062                    }
063            }
064    
065            private DataType fromNetcdfFile(String path) {
066                    NetcdfFile ncfile = null;
067                    DataType obj = null;
068    
069                    try {
070                            log.info("Opening " + path + " for reading");
071                            ncfile = NetcdfFile.open(path);
072    
073                            // expected global variables
074                            String typeName, dataType;
075                            Location loc;
076                            Date dt;
077                            try {
078                                    typeName = ncfile.findGlobalAttribute("TypeName")
079                                                    .getStringValue();
080                                    dataType = ncfile.findGlobalAttribute("DataType")
081                                                    .getStringValue();
082    
083                                    // location and time
084                                    double lat = ncfile.findGlobalAttribute("Latitude")
085                                                    .getNumericValue().doubleValue();
086                                    double lon = ncfile.findGlobalAttribute("Longitude")
087                                                    .getNumericValue().doubleValue();
088                                    double ht = ncfile.findGlobalAttribute("Height")
089                                                    .getNumericValue().doubleValue();
090                                    long tm = ncfile.findGlobalAttribute("Time").getNumericValue()
091                                                    .longValue();
092    
093                                    loc = new Location(lat, lon, ht / 1000);
094                                    long tm_long = 1000 * tm;
095    
096                                    try {
097                                            double ftm = ncfile.findGlobalAttribute("FractionalTime")
098                                                            .getNumericValue().doubleValue();
099                                            tm_long += (int) Math.round(1000 * ftm);
100                                    } catch (Exception e) {
101                                    } // okay if no fractional
102    
103                                    dt = new Date(tm_long);
104                            } catch (Exception e) {
105                                    throw new FormatException("Missing required global attribute.");
106                            }
107    
108                            // get the global attribute attributes and use it ...
109                            Map<String, String> attr = new HashMap<String, String>();
110                            try {
111                                    List<String> attrNames = StringUtil.split(ncfile
112                                                    .findGlobalAttribute("attributes").getStringValue());
113                                    for (int i = 0; i < attrNames.size(); ++i) {
114                                            String name = (String) attrNames.get(i);
115                                            if (name.equals("") == false) {
116                                                    String val = ncfile
117                                                                    .findGlobalAttribute(name + "-value")
118                                                                    .getStringValue();
119                                                    attr.put(name, val);
120                                            }
121                                    }
122                            } catch (Exception e) {
123                                    log.warn("While extracting attributes.", e);
124                            }
125    
126                            boolean sparse = dataType.contains("Sparse");
127                            float[][] values;
128                            if (sparse) {
129                                    values = readSparse2D(ncfile, typeName);
130                            } else {
131                                    values = readData2D(ncfile, typeName);
132                            }
133    
134                            // now get the variable of interest
135                            if (dataType.endsWith("RadialSet")) {
136                                    obj = readRadialSet(ncfile, loc, dt, attr, typeName, values);
137                            } else if (dataType.endsWith("LatLonGrid")) {
138                                    obj = readLatLonGrid(ncfile, loc, dt, attr, typeName, values);
139                            } else {
140                                    throw new FormatException("unsupported datatype: " + dataType);
141                            }
142                            obj.setAttributes(attr);
143                    } catch (Exception e) {
144                            log.warn(e);
145                    } finally {
146                            try {
147                                    if (ncfile != null)
148                                            ncfile.close();
149                            } catch (Exception e) {
150                            }
151                    }
152                    return obj;
153            }
154    
155            private Variable getVariable(NetcdfFile ncfile, String name)
156                            throws FormatException {
157                    Variable data = ncfile.findVariable(name);
158                    if (data == null) {
159                            throw new FormatException("missing variable " + name);
160                    }
161                    return data;
162            }
163    
164            private float[][] readData2D(NetcdfFile ncfile, String typeName)
165                            throws IOException {
166                    Variable data = getVariable(ncfile, typeName);
167                    Array gate_values = data.read();
168                    int num_radials = data.getDimension(0).getLength();
169                    int num_gates = data.getDimension(1).getLength();
170                    float[][] values = new float[num_radials][num_gates];
171                    Index gate_index = gate_values.getIndex();
172                    for (int i = 0; i < num_radials; ++i) {
173                            for (int j = 0; j < num_gates; ++j) {
174                                    gate_index.set(i, j);
175                                    values[i][j] = gate_values.getFloat(gate_index);
176                            }
177                    }
178                    return values;
179            }
180    
181            private float[][] readSparse2D(NetcdfFile ncfile, String typeName)
182                            throws IOException {
183                    int num_radials = ((Dimension) ncfile.getDimensions().get(0))
184                                    .getLength();
185                    int num_gates = ((Dimension) ncfile.getDimensions().get(1)).getLength();
186    
187                    Variable data = getVariable(ncfile, typeName);
188                    float backgroundValue = data.findAttribute("BackgroundValue")
189                                    .getNumericValue().floatValue();
190                    Variable pixelxVariable = getVariable(ncfile, "pixel_x");
191                    Variable pixelyVariable = getVariable(ncfile, "pixel_y");
192                    Variable countVariable = null;
193                    try {
194                            countVariable = getVariable(ncfile, "pixel_count");
195                    } catch (Exception e) {
196                            // ok
197                    }
198                    Array data_values = data.read();
199                    Array pixelx_values = pixelxVariable.read();
200                    Array pixely_values = pixelyVariable.read();
201                    Array count_values = (countVariable == null) ? null : countVariable
202                                    .read();
203                    int num_pixels = pixelxVariable.getDimension(0).getLength();
204                    Index pixel_index = data_values.getIndex();
205    
206                    float[][] values = new float[num_radials][num_gates];
207                    for (int i = 0; i < num_radials; ++i) {
208                            Arrays.fill(values[i], backgroundValue);
209                    }
210                    for (int i = 0; i < num_pixels; ++i) {
211                            pixel_index.set(i);
212                            float value = data_values.getFloat(pixel_index);
213                            short x = pixelx_values.getShort(pixel_index);
214                            short y = pixely_values.getShort(pixel_index);
215                            int count = (count_values == null) ? 1 : count_values
216                                            .getInt(pixel_index);
217                            for (int j = 0; j < count; ++j, ++y) {
218                                    if (y == num_gates) {
219                                            ++x;
220                                            y = 0;
221                                    }
222                                    values[x][y] = value;
223                            }
224                    }
225                    return values;
226            }
227    
228            private RadialSet readRadialSet(NetcdfFile ncfile, Location radarLoc,
229                            Date scanTime, Map<String, String> attr, String typeName,
230                            final float[][] values) throws FormatException, IOException {
231    
232                    Variable v_az = ncfile.findVariable("Azimuth");
233                    Variable v_bw = ncfile.findVariable("BeamWidth");
234                    Variable v_as = ncfile.findVariable("AzimuthalSpacing");
235                    Variable v_gw = ncfile.findVariable("GateWidth");
236                    Variable v_ny = ncfile.findVariable("NyquistVelocity");
237                    float elev = ncfile.findGlobalAttribute("Elevation").getNumericValue()
238                                    .floatValue();
239                    float distToFirstGate = ncfile.findGlobalAttribute("RangeToFirstGate")
240                                    .getNumericValue().floatValue();
241                    float nyquist = DataType.MissingData;
242                    if (attr.containsKey("Nyquist_Vel")) {
243                            nyquist = Float.parseFloat(attr.get("Nyquist_Vel"));
244                    }
245                    Array az_values = v_az.read();
246                    Array bw_values = v_bw.read();
247                    Array gw_values = v_gw.read();
248    
249                    // optional
250                    Array as_values = null;
251                    if (v_as != null) {
252                            as_values = v_as.read();
253                    }
254                    Array ny_values = null;
255                    if (v_ny != null) {
256                            ny_values = v_ny.read();
257                    }
258    
259                    int num_radials = values.length;
260                    Radial[] radials = new Radial[num_radials];
261                    Index radial_index = az_values.getIndex();
262                    for (int i = 0; i < num_radials; ++i) {
263                            radial_index.set(i);
264                            float az = az_values.getFloat(radial_index);
265                            float bw = bw_values.getFloat(radial_index);
266                            float as = (as_values == null) ? bw : as_values
267                                            .getFloat(radial_index);
268                            float gw = gw_values.getFloat(radial_index) / 1000; // meters to kms
269                            float ny = (ny_values == null) ? nyquist : ny_values
270                                            .getFloat(radial_index);
271                            radials[i] = new Radial(az, bw, as, gw, ny, values[i]);
272                    }
273                    // construct RadialSet
274                    RadialSet result = new RadialSet(elev, radarLoc, scanTime,
275                                    distToFirstGate / 1000, typeName, radials);
276                    return result;
277            }
278    
279            private LatLonGrid readLatLonGrid(NetcdfFile ncfile, Location nwCorner,
280                            Date gridTime, Map<String, String> attr, String typeName,
281                            final float[][] values) throws FormatException, IOException {
282                    float latres = ncfile.findGlobalAttribute("LatGridSpacing")
283                                    .getNumericValue().floatValue();
284                    float lonres = ncfile.findGlobalAttribute("LonGridSpacing")
285                                    .getNumericValue().floatValue();
286    
287                    LatLonGrid result = new LatLonGrid(nwCorner, gridTime, typeName, latres, lonres, values);
288                    return result;
289            }
290    }