001 /**
002 *
003 */
004 package org.wdssii.ltgingest;
005
006 import java.io.BufferedReader;
007 import java.io.File;
008 import java.io.FileReader;
009 import java.io.IOException;
010 import java.text.ParseException;
011 import java.text.SimpleDateFormat;
012 import java.util.Date;
013
014 import org.apache.commons.logging.Log;
015 import org.apache.commons.logging.LogFactory;
016 import org.wdssii.core.DataEncoder;
017 import org.wdssii.core.DataTable;
018 import org.wdssii.core.Ingestor;
019 import org.wdssii.core.Location;
020 import org.wdssii.core.DataTable.Column;
021
022 /**
023 * Base class of ingestors for ASCII lightning data
024 *
025 * @author lakshman
026 *
027 */
028 public abstract class AbstractAsciiLightningIngest extends Ingestor {
029 private static Log log = LogFactory.getLog(AbstractAsciiLightningIngest.class);
030
031 private String fieldSeparatorRegex = ",?\\s+";
032 private int everyNseconds = 30;
033 private String subtype = "";
034 private String dateFormatInFileName = "yyyyMMdd-HHmmss";
035 private String dateFormatInData = "yyyyMMdd HHmmss";
036
037 public String getFieldSeparatorRegex() {
038 return fieldSeparatorRegex;
039 }
040
041 public void setFieldSeparatorRegex(String fieldSeparatorRegex) {
042 this.fieldSeparatorRegex = fieldSeparatorRegex;
043 }
044
045 public int getEveryNseconds() {
046 return everyNseconds;
047 }
048
049 public void setEveryNseconds(int everyNseconds) {
050 this.everyNseconds = everyNseconds;
051 }
052
053 public String getSubtype() {
054 return subtype;
055 }
056
057 public void setSubtype(String subtype) {
058 this.subtype = subtype;
059 }
060
061 public String getDateFormatInFileName() {
062 return dateFormatInFileName;
063 }
064
065 public void setDateFormatInFileName(String dateFormatInFileName) {
066 this.dateFormatInFileName = dateFormatInFileName;
067 fileNameTimeParser = new SimpleDateFormat(dateFormatInFileName);
068 }
069
070 public String getDateFormatInData() {
071 return dateFormatInData;
072 }
073
074 public void setDateFormatInData(String dateFormatInData) {
075 this.dateFormatInData = dateFormatInData;
076 fileContentTimeParser = new SimpleDateFormat(dateFormatInData);
077 }
078
079 private SimpleDateFormat fileNameTimeParser = new SimpleDateFormat(dateFormatInFileName);
080 private SimpleDateFormat fileContentTimeParser = new SimpleDateFormat(dateFormatInData);
081
082 public AbstractAsciiLightningIngest(){
083 setFilenamePatterns(".txt ltg");
084 }
085
086 @Override
087 protected void doIngest(File f) {
088 BufferedReader reader = null;
089 try {
090 log.info("Reading " + f);
091 // Create DataTable for output
092 DataTable table = newDataTable(getTimeFromFileName(f.getAbsolutePath()));
093 reader = new BufferedReader(new FileReader(f));
094 String line;
095
096 while ( (line = reader.readLine()) != null){
097 String[] fields = line.split(fieldSeparatorRegex);
098
099 // get the time from the line
100 Date ltgTime = fileContentTimeParser.parse(line.substring(0,dateFormatInData.length()));
101
102 // if > allowable time ... write out XML
103 long elapsed;
104 while ( (elapsed = ltgTime.getTime() - table.getTime().getTime()) > (everyNseconds*1000) ){
105 // write current table
106 DataEncoder.writeDataAndNotify(table, getOutputDir(), subtype);
107
108 // start new table at current time + 30s
109 Date newTime = new Date(table.getTime().getTime() + everyNseconds*1000 );
110 if (log.isInfoEnabled()){
111 log.info("Starting new table at " + newTime + " elapsed=" + elapsed + " need to catch up to " + ltgTime);
112 }
113 table = newDataTable(newTime);
114 }
115
116 // add record to table
117 addRow(table, ltgTime, fields);
118
119 }
120 // write current table
121 log.info("Writing table up to " + table.getTime() + " because input file ended");
122 DataEncoder.writeDataAndNotify(table, getOutputDir(), subtype);
123
124 } catch (Exception e){
125 log.error("Unable to ingest " + f, e);
126 } finally {
127 if (reader != null){
128 try {
129 reader.close();
130 } catch (IOException e) {
131 // ok
132 }
133 }
134 }
135 }
136
137 protected abstract void addRow(DataTable table, Date ltgTime, String[] fields);
138
139 private DataTable newDataTable(Date time){
140 Location loc = new Location(35,-97,0); // arbitrary anyway
141 Column[] columns = new Column[]{
142 new Column("RowName", "dimensionless"),
143 new Column("Date", "dimensionless"),
144 new Column("Latitude", "Degrees"),
145 new Column("Longitude", "Degrees"),
146 new Column("PeakCurrent", "kA"),
147 new Column("Sign", "dimensionless"),
148 new Column("Count", "dimensionless"),
149 new Column("DischargeType", "dimensionless"),
150 new Column("Rank", "dimensionless"),
151 };
152 DataTable table = new DataTable(loc, time, "Lightning", columns);
153 if (subtype.length() > 0){
154 table.setAttribute("SubType", subtype);
155 }
156 table.setAttribute("interval", Integer.toString(everyNseconds));
157 return table;
158 }
159
160 /** Assumes that the time is the last part of the file name with the exception of an optional filename extension */
161 private Date getTimeFromFileName(String fileName) throws ParseException {
162 int startOfSuffix = fileName.lastIndexOf('.');
163 if ( startOfSuffix == -1 ){
164 startOfSuffix = fileName.length();
165 }
166
167 String digits = fileName.substring(startOfSuffix - dateFormatInFileName.length(), startOfSuffix);
168
169 return fileNameTimeParser.parse(digits);
170 }
171
172 }