001 //$HeadURL: svn+ssh://jwilden@svn.wald.intevation.org/deegree/base/branches/2.5_testing/src/org/deegree/io/csv/CSVReader.java $
002 /*----------------------------------------------------------------------------
003 This file is part of deegree, http://deegree.org/
004 Copyright (C) 2001-2009 by:
005 Department of Geography, University of Bonn
006 and
007 lat/lon GmbH
008
009 This library is free software; you can redistribute it and/or modify it under
010 the terms of the GNU Lesser General Public License as published by the Free
011 Software Foundation; either version 2.1 of the License, or (at your option)
012 any later version.
013 This library is distributed in the hope that it will be useful, but WITHOUT
014 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
015 FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
016 details.
017 You should have received a copy of the GNU Lesser General Public License
018 along with this library; if not, write to the Free Software Foundation, Inc.,
019 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
020
021 Contact information:
022
023 lat/lon GmbH
024 Aennchenstr. 19, 53177 Bonn
025 Germany
026 http://lat-lon.de/
027
028 Department of Geography, University of Bonn
029 Prof. Dr. Klaus Greve
030 Postfach 1147, 53001 Bonn
031 Germany
032 http://www.geographie.uni-bonn.de/deegree/
033
034 e-mail: info@deegree.org
035 ----------------------------------------------------------------------------*/
036
037 package org.deegree.io.csv;
038
039 import static java.io.StreamTokenizer.TT_EOF;
040 import static java.lang.Double.parseDouble;
041 import static java.util.Collections.unmodifiableList;
042 import static org.deegree.datatypes.Types.VARCHAR;
043 import static org.deegree.framework.log.LoggerFactory.getLogger;
044 import static org.deegree.io.mapinfoapi.MapInfoReader.whitespaceChars;
045 import static org.deegree.io.mapinfoapi.MapInfoReader.wordChars;
046 import static org.deegree.model.feature.FeatureFactory.createFeature;
047 import static org.deegree.model.feature.FeatureFactory.createFeatureCollection;
048 import static org.deegree.model.feature.FeatureFactory.createFeatureProperty;
049 import static org.deegree.model.feature.FeatureFactory.createFeatureType;
050 import static org.deegree.model.feature.FeatureFactory.createGeometryPropertyType;
051 import static org.deegree.model.feature.FeatureFactory.createSimplePropertyType;
052 import static org.deegree.model.spatialschema.GeometryFactory.createPoint;
053 import static org.deegree.model.spatialschema.WKTAdapter.wrap;
054
055 import java.io.BufferedReader;
056 import java.io.File;
057 import java.io.FileReader;
058 import java.io.IOException;
059 import java.io.StreamTokenizer;
060 import java.io.StringReader;
061 import java.net.URI;
062 import java.net.URISyntaxException;
063 import java.util.ArrayList;
064 import java.util.LinkedList;
065 import java.util.List;
066
067 import org.deegree.datatypes.QualifiedName;
068 import org.deegree.framework.log.ILogger;
069 import org.deegree.model.feature.FeatureCollection;
070 import org.deegree.model.feature.FeatureProperty;
071 import org.deegree.model.feature.schema.FeatureType;
072 import org.deegree.model.feature.schema.PropertyType;
073 import org.deegree.model.spatialschema.Geometry;
074 import org.deegree.model.spatialschema.GeometryException;
075
076 /**
077 * <code>CSVReader</code>
078 *
079 * @author <a href="mailto:schmitz@lat-lon.de">Andreas Schmitz</a>
080 * @author last edited by: $Author: mschneider $
081 *
082 * @version $Revision: 18195 $, $Date: 2009-06-18 17:55:39 +0200 (Do, 18 Jun 2009) $
083 */
084 public class CSVReader {
085
086 private static final ILogger LOG = getLogger( CSVReader.class );
087
088 private File fileName;
089
090 private int xcol = 0, ycol = 1, wkt = -1;
091
092 private static URI APPNS;
093
094 private List<String[]> header;
095
096 private boolean ignoreFirstLine, parseGeometryProperty = true;
097
098 static {
099 try {
100 APPNS = new URI( "http://www.deegree.org/app" );
101 } catch ( URISyntaxException e ) {
102 // yes, cannot happen
103 }
104 }
105
106 /**
107 * @param name
108 * @param ignoreFirstLine
109 * @throws IOException
110 */
111 public CSVReader( String name, boolean ignoreFirstLine ) throws IOException {
112 this.ignoreFirstLine = ignoreFirstLine;
113
114 fileName = new File( name ).getAbsoluteFile();
115
116 header = new ArrayList<String[]>( 3 );
117
118 BufferedReader in = new BufferedReader( new FileReader( name ) );
119 String str = in.readLine();
120 char separat = determineSeparator( str );
121 do {
122 List<String> lst = parseLine( str, separat );
123 header.add( lst.toArray( new String[lst.size()] ) );
124 } while ( ( ( str = in.readLine() ) != null ) && header.size() < 3 );
125 in.close();
126 }
127
128 /**
129 * @return max. the first three lines of the file (if there are three)
130 */
131 public List<String[]> getHeader() {
132 return unmodifiableList( header );
133 }
134
135 /**
136 * By default, a geometry property will be parsed. Set this to false to get "simple property only" features.
137 *
138 * @param parseGeometryProperty
139 */
140 public void setParseGeometryProperty( boolean parseGeometryProperty ) {
141 this.parseGeometryProperty = parseGeometryProperty;
142 }
143
144 private static char determineSeparator( String s ) {
145 // determine most likely separator
146 int ccount = countChars( s, ',' );
147 int scount = countChars( s, ';' );
148 int tcount = countChars( s, '\t' );
149 if ( ccount >= scount && ccount >= tcount ) {
150 return ',';
151 }
152 if ( tcount >= ccount && tcount >= scount ) {
153 return '\t';
154 }
155 if ( scount >= ccount && scount >= tcount ) {
156 return ';';
157 }
158 return ',';
159 }
160
161 private static List<String> parseLine( String line, char separator )
162 throws IOException {
163 String seps = ",;\t";
164 for ( int i = 0; i < seps.length(); ++i ) {
165 if ( line.startsWith( "" + seps.charAt( i ) ) ) {
166 line = "\"\"" + line;
167 }
168 String dseps = "" + seps.charAt( i ) + seps.charAt( i );
169 while ( line.indexOf( dseps ) != -1 ) {
170 line = line.replace( dseps, seps.charAt( i ) + "\"\"" + seps.charAt( i ) );
171 }
172 }
173 StreamTokenizer tok = getCSVFromStringTokenizer( line, separator );
174
175 LinkedList<String> list = new LinkedList<String>();
176
177 tok.nextToken();
178 if ( tok.ttype == TT_EOF ) {
179 return list;
180 }
181 while ( tok.ttype != TT_EOF ) {
182 list.add( tok.sval );
183 tok.nextToken();
184 }
185
186 return list;
187 }
188
189 /**
190 * Also sets wkt to -1.
191 *
192 * @param x
193 * @param y
194 */
195 public void setPointColumns( int x, int y ) {
196 xcol = x;
197 ycol = y;
198 wkt = -1;
199 }
200
201 /**
202 * @param wkt
203 * if -1, x/y will be used instead
204 */
205 public void setWKTColumn( int wkt ) {
206 this.wkt = wkt;
207 }
208
209 /**
210 * @param input
211 * @param separator
212 * @return a tokenizer with a stringreader as data input
213 */
214 public static StreamTokenizer getCSVFromStringTokenizer( String input, char separator ) {
215 StreamTokenizer tok = new StreamTokenizer( new StringReader( input ) );
216
217 tok.resetSyntax();
218 tok.eolIsSignificant( true );
219 tok.lowerCaseMode( true );
220 tok.slashSlashComments( false );
221 tok.slashStarComments( false );
222 tok.wordChars( 'a', 'z' );
223 tok.wordChars( 'A', 'Z' );
224 tok.wordChars( '\u00a0', '\u00ff' );
225 tok.wordChars( '0', '9' );
226 wordChars( tok, ',', '\t', ';' );
227 wordChars( tok, '.', '-', '_', ' ', '+', '/', '\\', '(', ')', '^' );
228 tok.quoteChar( '"' );
229 whitespaceChars( tok, '\n', '\r', '\f' );
230
231 // reset separator
232 whitespaceChars( tok, separator );
233
234 return tok;
235 }
236
237 private static int countChars( String s, char c ) {
238 int count = 0;
239 for ( int i = 0; i < s.length(); ++i ) {
240 if ( s.charAt( i ) == c ) {
241 ++count;
242 }
243 }
244 return count;
245 }
246
247 /**
248 * @return a new feature collection
249 * @throws IOException
250 */
251 public FeatureCollection parseFeatureCollection()
252 throws IOException {
253 FeatureCollection fc = createFeatureCollection( "uniquemy_", 512 );
254 QualifiedName geomName = new QualifiedName( "app:geometry", APPNS );
255 QualifiedName featureName = new QualifiedName( "app:feature", APPNS );
256
257 int counter = 0;
258
259 BufferedReader in = new BufferedReader( new FileReader( fileName ) );
260 String str = in.readLine();
261 List<String> colNames = null;
262
263 char separator = determineSeparator( str );
264 if ( ignoreFirstLine ) {
265 colNames = parseLine( str, separator );
266 str = in.readLine();
267 }
268 outer: do {
269 LOG.logDebug( "Trying to parse line ", str );
270 List<String> vals = parseLine( str, separator );
271
272 double x = 0, y = 0;
273 Geometry wktGeom = null;
274 LinkedList<FeatureProperty> fps = new LinkedList<FeatureProperty>();
275 LinkedList<PropertyType> fpt = new LinkedList<PropertyType>();
276
277 for ( int i = 0; i < vals.size(); ++i ) {
278
279 if ( parseGeometryProperty && wkt == -1 && i == xcol ) {
280 try {
281 x = parseDouble( vals.get( i ) );
282 } catch ( NumberFormatException nfe ) {
283 // puh, CSV is an easy format? I think not...
284 try {
285 x = parseDouble( vals.get( i ).replace( ",", "." ) );
286 } catch ( NumberFormatException nfe2 ) {
287 LOG.logWarning( "Skipping line " + str );
288 continue outer;
289 }
290 }
291 continue;
292 }
293 if ( parseGeometryProperty && wkt == -1 && i == ycol ) {
294 if ( vals.get( i ).equals( "" ) ) {
295 y = 0; // this seems to be a sensible (Java-like) default
296 } else {
297 try {
298 y = parseDouble( vals.get( i ) );
299 } catch ( NumberFormatException nfe ) {
300 // puh, CSV is an easy format? I think not...
301 try {
302 y = parseDouble( vals.get( i ).replace( ",", "." ) );
303 } catch ( NumberFormatException nfe2 ) {
304 LOG.logWarning( "Skipping line " + str );
305 continue outer;
306 }
307 }
308 }
309 continue;
310 }
311 if ( parseGeometryProperty && wkt != -1 && i == wkt ) {
312 try {
313 wktGeom = wrap( vals.get( i ), null );
314 } catch ( GeometryException e ) {
315 LOG.logError( "Invalid WKT geometry", e );
316 }
317 if ( wktGeom == null ) {
318 LOG.logError( "Could not parse WKT geometry: " + vals.get( i ) );
319 }
320 continue;
321 }
322
323 String n;
324 if ( ignoreFirstLine ) {
325 String coln = colNames.get( i );
326 n = "app:" + ( coln.trim().equals( "" ) ? "property" + i : coln );
327 } else {
328 n = "app:property" + i;
329 }
330 n = n.replace( ' ', '_' );
331 QualifiedName name = new QualifiedName( n, APPNS );
332 fps.add( createFeatureProperty( name, vals.get( i ) ) );
333 fpt.add( createSimplePropertyType( name, VARCHAR, true ) );
334 }
335
336 if ( parseGeometryProperty ) {
337 if ( wkt != -1 && wktGeom != null ) {
338 fps.add( createFeatureProperty( geomName, wktGeom ) );
339 } else {
340 fps.add( createFeatureProperty( geomName, createPoint( x, y, null ) ) );
341 }
342 fpt.add( createGeometryPropertyType( geomName, null, 1, 1 ) );
343 }
344
345 FeatureType tp = createFeatureType( featureName, false, fpt.toArray( new PropertyType[fpt.size()] ) );
346 fc.add( createFeature( ++counter + "", tp, fps ) );
347 } while ( ( ( str = in.readLine() ) != null ) );
348
349 in.close();
350
351 // makes sense (?)
352 if ( fc.size() > 0 ) {
353 fc.setFeatureType( fc.getFeature( 0 ).getFeatureType() );
354 }
355
356 return fc;
357 }
358 }