001 /**
002 *
003 */
004 package org.wdssii.dualpol;
005
006 import java.io.File;
007 import java.io.IOException;
008 import java.text.ParsePosition;
009 import java.text.SimpleDateFormat;
010 import java.util.Date;
011 import java.util.List;
012
013 import org.apache.commons.logging.Log;
014 import org.apache.commons.logging.LogFactory;
015 import org.wdssii.core.DataEncoder;
016 import org.wdssii.core.DataType;
017 import org.wdssii.core.DataUnavailableException;
018 import org.wdssii.core.LatLonGrid;
019 import org.wdssii.core.Location;
020
021 import ucar.ma2.Array;
022 import ucar.ma2.Index;
023 import ucar.nc2.NetcdfFile;
024 import ucar.nc2.Variable;
025
026 /**
027 * Reads IDL-generated 3D grids
028 *
029 * @author lakshman
030 *
031 */
032 public class IDLNetcdfGridReader {
033 private static Log log = LogFactory.getLog(IDLNetcdfGridReader.class);
034
035 public IDLNetcdfGridReader() {
036 }
037
038 @SuppressWarnings("serial")
039 private static class FormatException extends DataUnavailableException {
040 public FormatException(String msg) {
041 super(msg);
042 }
043 }
044
045 @SuppressWarnings("unchecked")
046 /**
047 * @return an array of LatLonGrid where the first dimension is the gridType
048 * (DZ, DR, etc.) and the second dimension is the height
049 */
050 public LatLonGrid[][] fromNetcdfFile(File path) {
051 NetcdfFile ncfile = null;
052
053 try {
054 Date time = getTimeFromFileName(path.getName());
055 ncfile = NetcdfFile.open(path.getAbsolutePath());
056
057 IDLGridProperties idl = new IDLGridProperties(
058 ncfile.findGlobalAttribute("radarlat"),
059 ncfile.findGlobalAttribute("radarlon"),
060 ncfile.findGlobalAttribute("ident"),
061 ncfile.findGlobalAttribute("xdelta"),
062 ncfile.findGlobalAttribute("ydelta"),
063 ncfile.findGlobalAttribute("zdelta"));
064
065 List<Variable> variables = ncfile.getVariables();
066 int numVariables = variables.size();
067 int numHt = -1;
068 LatLonGrid[][] grids = new LatLonGrid[numVariables][];
069 for (int i = 0; i < numVariables; ++i) {
070 String variableName = variables.get(i).getName();
071 float[][][] values = readData3D(ncfile, variableName);
072 numHt = values.length;
073 int numLat = values[0].length;
074 int numLon = values[0][0].length;
075 grids[i] = new LatLonGrid[numHt];
076 for (int ht = 0; ht < numHt; ++ht) {
077 Location nwCorner = new Location(idl.getRadarLat() + 0.5
078 * numLat * idl.getDeltaLat(), idl.getRadarLon() - 0.5
079 * numLon * idl.getDeltaLon(), idl.getRadarHtKms() + ht
080 * idl.getDeltaHeightKms());
081 grids[i][ht] = new LatLonGrid(nwCorner, time, idl.getTypeName(variableName),
082 idl.getDeltaLat(), idl.getDeltaLon(), values[ht]);
083 }
084 }
085
086 // Threshold based on RhoHV
087 int rhohv = -1;
088 for (int i=0; i < numVariables; ++i){
089 if ( numHt > 0 && grids[i][0].getTypeName().equals("RhoHV") ){
090 rhohv = i;
091 }
092 }
093 if (rhohv >= 0){
094 for (int ht=0; ht < numHt; ++ht){
095 for (int i=0; i < numVariables; ++i){
096 if ( i != rhohv ){
097 threshold( grids[i][ht], grids[rhohv][ht] , 0.5f );
098 }
099 }
100 }
101 }
102
103 return grids;
104 } catch (Exception e) {
105 log.warn(e);
106 } finally {
107 try {
108 if (ncfile != null)
109 ncfile.close();
110 } catch (Exception e) {
111 }
112 }
113 return null;
114 }
115
116 private void threshold(LatLonGrid grid, LatLonGrid rhohv, float thresh) {
117 int numLat = grid.getNumLat();
118 int numLon = grid.getNumLon();
119 float[][] values = grid.getValues();
120 float[][] corr = rhohv.getValues();
121 int numRemoved = 0;
122 for (int i=0; i < numLat; ++i){
123 for (int j=0; j < numLon; ++j){
124 if ( corr[i][j] < thresh ){
125 values[i][j] = DataType.MissingData;
126 ++numRemoved;
127 }
128 }
129 }
130 log.info("Thresholding based on RhoHV removed " + numRemoved + " pixels from " +
131 grid.getTypeName() + " at ht=" + grid.getLocation().getHeightKms());
132 }
133
134 private Variable getVariable(NetcdfFile ncfile, String name)
135 throws FormatException {
136 Variable data = ncfile.findVariable(name);
137 if (data == null) {
138 throw new FormatException("missing variable " + name);
139 }
140 return data;
141 }
142
143
144 private boolean isValid(float value){
145 if ( Math.abs(value-999) < 0.001 || Math.abs(value+999) < 0.001 ){
146 return false;
147 }
148 return true;
149 }
150
151 private float[][][] readData3D(NetcdfFile ncfile, String typeName)
152 throws IOException {
153 Variable data = getVariable(ncfile, typeName);
154 Array values = data.read();
155 int numz = data.getDimension(0).getLength();
156 int numy = data.getDimension(1).getLength();
157 int numx = data.getDimension(2).getLength();
158
159 float[][][] result = new float[numz][numy][numx];
160 Index index = values.getIndex();
161 for (int z = 0; z < numz; ++z) {
162 for (int y = 0; y < numy; ++y) {
163 for (int x = 0; x < numx; ++x) {
164 index.set(z, y, x);
165 float value = values.getFloat(index);
166 if ( !isValid(value) ) {
167 value = DataType.MissingData;
168 }
169 result[z][numy-y-1][x] = value;
170 }
171 }
172 }
173 return result;
174 }
175
176 public static Date getTimeFromFileName(String path) {
177 SimpleDateFormat sdf = new SimpleDateFormat(IDLGridProperties.getFileNameDateFormat());
178 Date result = sdf.parse(path, new ParsePosition(5));
179 if (result == null) {
180 throw new FormatException(
181 "The name of the file: "
182 + path
183 + " has to be of the form KOUN_031505_172326_anything.nc.gz");
184 } else {
185 return result;
186 }
187 }
188
189 /**
190 * Simple program that reads IDL-generated files and writes out WDSS-II
191 * netcdf files
192 */
193 public static void main(String[] args) {
194 if (args.length < 2) {
195 System.out.println("Usage: java "
196 + IDLNetcdfGridReader.class.getCanonicalName()
197 + " icingInDir icingOutDir");
198 return;
199 }
200 IDLNetcdfGridReader reader = new IDLNetcdfGridReader();
201 String outputDir = args[1];
202 new File(outputDir).mkdirs();
203 File[] inFiles = new File(args[0]).listFiles();
204 for (File file : inFiles) {
205 if (file.getName().contains(".nc")
206 || file.getName().contains(".netcdf")) {
207 log.info("Reading " + file);
208 LatLonGrid[][] grids = reader.fromNetcdfFile(file);
209 for (LatLonGrid[] gridarray : grids) {
210 for (LatLonGrid grid : gridarray) {
211 DataEncoder.writeDataAndNotify(grid, outputDir);
212 }
213 }
214 } else {
215 System.out.println("Ignoring " + file
216 + " since it does not have .nc in name");
217 }
218 }
219 }
220 }