001    //$HeadURL: svn+ssh://jwilden@svn.wald.intevation.org/deegree/base/branches/2.5_testing/src/org/deegree/io/datastore/shape/ShapeDatastore.java $
002    /*----------------------------------------------------------------------------
003     This file is part of deegree, http://deegree.org/
004     Copyright (C) 2001-2009 by:
005       Department of Geography, University of Bonn
006     and
007       lat/lon GmbH
008    
009     This library is free software; you can redistribute it and/or modify it under
010     the terms of the GNU Lesser General Public License as published by the Free
011     Software Foundation; either version 2.1 of the License, or (at your option)
012     any later version.
013     This library is distributed in the hope that it will be useful, but WITHOUT
014     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
015     FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
016     details.
017     You should have received a copy of the GNU Lesser General Public License
018     along with this library; if not, write to the Free Software Foundation, Inc.,
019     59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
020    
021     Contact information:
022    
023     lat/lon GmbH
024     Aennchenstr. 19, 53177 Bonn
025     Germany
026     http://lat-lon.de/
027    
028     Department of Geography, University of Bonn
029     Prof. Dr. Klaus Greve
030     Postfach 1147, 53001 Bonn
031     Germany
032     http://www.geographie.uni-bonn.de/deegree/
033    
034     e-mail: info@deegree.org
035    ----------------------------------------------------------------------------*/
036    
037    package org.deegree.io.datastore.shape;
038    
039    import java.io.IOException;
040    import java.net.URL;
041    import java.util.Collection;
042    import java.util.HashMap;
043    import java.util.HashSet;
044    import java.util.Map;
045    import java.util.Set;
046    
047    import org.deegree.datatypes.QualifiedName;
048    import org.deegree.framework.log.ILogger;
049    import org.deegree.framework.log.LoggerFactory;
050    import org.deegree.framework.util.IDGenerator;
051    import org.deegree.framework.util.StringTools;
052    import org.deegree.framework.xml.NamespaceContext;
053    import org.deegree.framework.xml.XMLTools;
054    import org.deegree.i18n.Messages;
055    import org.deegree.io.datastore.AnnotationDocument;
056    import org.deegree.io.datastore.Datastore;
057    import org.deegree.io.datastore.DatastoreException;
058    import org.deegree.io.datastore.DatastoreTransaction;
059    import org.deegree.io.datastore.PropertyPathResolver;
060    import org.deegree.io.datastore.PropertyPathResolvingException;
061    import org.deegree.io.datastore.schema.MappedFeatureType;
062    import org.deegree.io.datastore.schema.MappedGMLSchema;
063    import org.deegree.io.datastore.schema.MappedGeometryPropertyType;
064    import org.deegree.io.datastore.schema.MappedPropertyType;
065    import org.deegree.io.datastore.schema.MappedSimplePropertyType;
066    import org.deegree.io.datastore.schema.content.MappingField;
067    import org.deegree.io.datastore.schema.content.SimpleContent;
068    import org.deegree.io.dbaseapi.DBaseException;
069    import org.deegree.io.dbaseapi.DBaseFile;
070    import org.deegree.io.shpapi.HasNoDBaseFileException;
071    import org.deegree.io.shpapi.ShapeFile;
072    import org.deegree.model.crs.CRSFactory;
073    import org.deegree.model.crs.CoordinateSystem;
074    import org.deegree.model.feature.Feature;
075    import org.deegree.model.feature.FeatureCollection;
076    import org.deegree.model.feature.FeatureFactory;
077    import org.deegree.model.feature.schema.FeatureType;
078    import org.deegree.model.feature.schema.PropertyType;
079    import org.deegree.model.filterencoding.ComparisonOperation;
080    import org.deegree.model.filterencoding.ComplexFilter;
081    import org.deegree.model.filterencoding.Expression;
082    import org.deegree.model.filterencoding.Filter;
083    import org.deegree.model.filterencoding.FilterEvaluationException;
084    import org.deegree.model.filterencoding.FilterTools;
085    import org.deegree.model.filterencoding.LogicalOperation;
086    import org.deegree.model.filterencoding.Operation;
087    import org.deegree.model.filterencoding.PropertyIsBetweenOperation;
088    import org.deegree.model.filterencoding.PropertyIsCOMPOperation;
089    import org.deegree.model.filterencoding.PropertyIsInstanceOfOperation;
090    import org.deegree.model.filterencoding.PropertyIsLikeOperation;
091    import org.deegree.model.filterencoding.PropertyIsNullOperation;
092    import org.deegree.model.filterencoding.PropertyName;
093    import org.deegree.model.filterencoding.SpatialOperation;
094    import org.deegree.model.spatialschema.Envelope;
095    import org.deegree.model.spatialschema.GeometryImpl;
096    import org.deegree.ogcbase.CommonNamespaces;
097    import org.deegree.ogcbase.PropertyPath;
098    import org.deegree.ogcwebservices.wfs.operation.Query;
099    import org.w3c.dom.Element;
100    
101    /**
102     * {@link Datastore} implementation that allows (read-only) access to ESRI shape files.
103     *
104     * @author <a href="mailto:deshmukh@lat-lon.de">Anup Deshmukh</a>
105     * @author <a href="mailto:schneider@lat-lon.de">Markus Schneider </a>
106     * @author last edited by: $Author: mschneider $
107     *
108     * @version $Revision: 18195 $, $Date: 2009-06-18 17:55:39 +0200 (Do, 18 Jun 2009) $
109     */
110    public class ShapeDatastore extends Datastore {
111    
112        private static final ILogger LOG = LoggerFactory.getLogger( ShapeDatastore.class );
113    
114        private static final NamespaceContext nsContext = CommonNamespaces.getNamespaceContext();
115    
116        // keys: FeatureTypes, values: Property-To-column mappings
117        private Map<FeatureType, Map<PropertyType, String>> ftMappings = new HashMap<FeatureType, Map<PropertyType, String>>();
118    
119        // NOTE: this is equal for all bound schemas
120        private URL shapeFileURL;
121    
122        private String srsName;
123    
124        @Override
125        public AnnotationDocument getAnnotationParser() {
126            return new ShapeAnnotationDocument();
127        }
128    
129        @Override
130        public void bindSchema( MappedGMLSchema schema )
131                                throws DatastoreException {
132            super.bindSchema( schema );
133            validate( schema );
134            srsName = schema.getDefaultSRS().toString();
135        }
136    
137        @Override
138        public FeatureCollection performQuery( Query query, MappedFeatureType[] rootFts )
139                                throws DatastoreException {
140    
141            if ( rootFts.length > 1 ) {
142                String msg = Messages.getMessage( "DATASTORE_SHAPE_DOES_NOT_SUPPORT_JOINS" );
143                throw new DatastoreException( msg );
144            }
145    
146            MappedFeatureType ft = rootFts[0];
147    
148            // perform CRS transformation (if necessary)
149            Query transformedQuery = transformQuery( query );
150    
151            // determine which properties must be contained in the returned features
152            Map<PropertyType, String> fetchPropsToColumns = determineSelectedProps( ft, transformedQuery );
153            if ( LOG.getLevel() == ILogger.LOG_DEBUG ) {
154                LOG.logDebug( "Selected properties / columns from the shapefile:" );
155                for ( PropertyType pt : fetchPropsToColumns.keySet() ) {
156                    LOG.logDebug( "- " + pt.getName() + " / " + fetchPropsToColumns.get( pt ) );
157                }
158            }
159    
160            // determine the properties that have to be removed after filter evaluation (because they
161            // are not requested, but used inside the filter expression)
162            Set<PropertyType> filterProps = determineFilterProps( ft, transformedQuery.getFilter() );
163            Set<PropertyType> removeProps = new HashSet<PropertyType>( filterProps );
164            for ( PropertyType pt : fetchPropsToColumns.keySet() ) {
165                removeProps.remove( pt );
166            }
167    
168            // add all property-to-column mappings for properties needed for the filter evaluation
169            Map<PropertyType, String> allPropsToCols = this.ftMappings.get( ft );
170            for ( PropertyType pt : filterProps ) {
171                fetchPropsToColumns.put( pt, allPropsToCols.get( pt ) );
172            }
173    
174            FeatureCollection result = null;
175            ShapeFile shapeFile = null;
176            int startPosition = -1;
177            int maxFeatures = -1;
178    
179            int record = -1;
180            try {
181                LOG.logDebug( "Opening shapefile '" + shapeFileURL.getFile() + ".shp'." );
182                shapeFile = new ShapeFile( shapeFileURL.getFile() );
183                startPosition = transformedQuery.getStartPosition()-1;
184                maxFeatures = transformedQuery.getMaxFeatures();
185                Filter filter = transformedQuery.getFilter();
186                Envelope bbox = null;
187                if ( filter instanceof ComplexFilter ) {
188                    bbox = FilterTools.firstBBOX( (ComplexFilter) filter );
189                }
190                if ( bbox == null ) {
191                    bbox = shapeFile.getFileMBR();
192                }
193    
194                shapeFile.setFeatureType( ft, fetchPropsToColumns );
195    
196                int[] idx = shapeFile.getGeoNumbersByRect( bbox );
197                // id=identity required
198                IDGenerator idg = IDGenerator.getInstance();
199                String id = ft.getName().getLocalName();
200                id += idg.generateUniqueID();
201                if ( idx != null ) {
202                    // check parameters for sanity
203                    if ( startPosition < 0 ) {
204                        startPosition = 0;
205                    }
206                    maxFeatures = maxFeatures + startPosition;
207                    if ( ( maxFeatures < 0 ) || ( maxFeatures >= idx.length ) ) {
208                        maxFeatures = idx.length;
209                    }
210                    LOG.logDebug( "Generating ID '" + id + "' for the FeatureCollection." );
211                    result = FeatureFactory.createFeatureCollection( id, idx.length );
212    
213                    // TODO: respect startposition
214    
215                    CoordinateSystem crs = CRSFactory.create( srsName );
216                    for ( int i = startPosition; i < maxFeatures; i++ ) {
217                        record = idx[i];
218                        Feature feat = shapeFile.getFeatureByRecNo( idx[i] );
219                        if ( filter == null || filter.evaluate( feat ) ) {
220                            String msg = StringTools.concat( 200, "Adding feature '", feat.getId(),
221                                                             "' to FeatureCollection (with CRS ", srsName, ")." );
222                            LOG.logDebug( msg );
223                            for ( PropertyType unrequestedPt : removeProps ) {
224                                msg = StringTools.concat( 200, "Removing unrequested property '", unrequestedPt.getName(),
225                                                          "' from feature: filter expression used it." );
226                                LOG.logDebug( msg );
227                                feat.removeProperty( unrequestedPt.getName() );
228                            }
229                            GeometryImpl geom = (GeometryImpl) feat.getDefaultGeometryPropertyValue();
230                            geom.setCoordinateSystem( crs );
231                            feat.setEnvelopesUpdated();
232                            result.add( feat );
233                        }
234                    }
235    
236                    // update the envelopes
237                    result.setEnvelopesUpdated();
238                    result.getBoundedBy();
239                } else {
240                    result = FeatureFactory.createFeatureCollection( id, 1 );
241                }
242            } catch ( IOException e ) {
243                LOG.logError( e.getMessage(), e );
244                String msg = Messages.getMessage( "DATASTORE_READINGFROMDBF", record );
245                throw new DatastoreException( msg, e );
246            } catch ( DBaseException e ) {
247                LOG.logError( e.getMessage(), e );
248                String msg = Messages.getMessage( "DATASTORE_READINGFROMDBF", record );
249                throw new DatastoreException( msg, e );
250            } catch ( HasNoDBaseFileException e ) {
251                LOG.logError( e.getMessage(), e );
252                String msg = Messages.getMessage( "DATASTORE_NODBASEFILE", record );
253                throw new DatastoreException( msg, e );
254            } catch ( FilterEvaluationException e ) {
255                throw new DatastoreException( e.getMessage(), e );
256            } catch ( Exception e ) {
257                LOG.logError( e.getMessage(), e );
258                String msg = Messages.getMessage( "DATASTORE_READINGFROMDBF", record );
259                throw new DatastoreException( msg, e );
260            } finally {
261                LOG.logDebug( "Closing shapefile." );
262                try {
263                    shapeFile.close();
264                } catch ( Exception e ) {
265                    String msg = Messages.getMessage( "DATASTORE_ERROR_CLOSING_SHAPEFILE", this.shapeFileURL.getFile() );
266                    throw new DatastoreException( msg );
267                }
268            }
269    
270            // transform result to queried srs if necessary
271            String targetSrsName = transformedQuery.getSrsName();
272            if ( targetSrsName != null && !targetSrsName.equals( this.srsName ) ) {
273                result = transformResult( result, transformedQuery.getSrsName() );
274            }
275    
276            return result;
277        }
278    
279        /**
280         * Determines the {@link PropertyType}s of the given feature type that are selected by the given {@link Query}
281         * implicitly and explicitly, i.e that are either listed or that have a <code>minOccurs</code> value greater than
282         * one. *
283         *
284         * @param ft
285         *            feature type
286         * @param query
287         * @return all properties that need to be fetched, mapped to the shapefile columns that store them
288         * @throws PropertyPathResolvingException
289         *             if a selected property does not denote a property of the feature type
290         */
291        private Map<PropertyType, String> determineSelectedProps( MappedFeatureType ft, Query query )
292                                throws PropertyPathResolvingException {
293    
294            Map<PropertyType, String> allPropsToCols = this.ftMappings.get( ft );
295            Map<PropertyType, String> fetchPropsToCols = new HashMap<PropertyType, String>();
296            // TODO: respect aliases
297            PropertyPath[] selectedPaths = PropertyPathResolver.normalizePropertyPaths( ft, null, query.getPropertyNames() );
298            // TODO respect alias
299            Map<MappedPropertyType, Collection<PropertyPath>> fetchProps = PropertyPathResolver.determineFetchProperties(
300                                                                                                                          ft,
301                                                                                                                          null,
302                                                                                                                          selectedPaths );
303            for ( MappedPropertyType pt : fetchProps.keySet() ) {
304                fetchPropsToCols.put( pt, allPropsToCols.get( pt ) );
305            }
306            return fetchPropsToCols;
307        }
308    
309        /**
310         * Determines the {@link PropertyType}s that are necessary to apply the given {@link Filter} expression, i.e. the
311         * <code>PropertyNames</code> that occur in it.
312         *
313         * @see PropertyPathResolver#determineFetchProperties(MappedFeatureType, String, PropertyPath[])
314         * @param ft
315         *            feature type on which the filter shall be applicable
316         * @param filter
317         *            filter expression
318         * @return all <code>PropertyType</code>s that are referenced inside the filter
319         * @throws PropertyPathResolvingException
320         */
321        private Set<PropertyType> determineFilterProps( MappedFeatureType ft, Filter filter )
322                                throws PropertyPathResolvingException {
323    
324            Set<PropertyType> filterPts = new HashSet<PropertyType>();
325            if ( filter != null && filter instanceof ComplexFilter ) {
326                ComplexFilter complexFilter = (ComplexFilter) filter;
327                Operation operation = complexFilter.getOperation();
328                addFilterProps( ft, filterPts, operation );
329            }
330            return filterPts;
331        }
332    
333        private void addFilterProps( MappedFeatureType ft, Set<PropertyType> filterPts, Operation operation )
334                                throws PropertyPathResolvingException {
335    
336            if ( operation instanceof ComparisonOperation ) {
337                if ( operation instanceof PropertyIsBetweenOperation ) {
338                    PropertyIsBetweenOperation betweenOperation = (PropertyIsBetweenOperation) operation;
339                    filterPts.add( getFilterProperty( ft, betweenOperation.getPropertyName() ) );
340                } else if ( operation instanceof PropertyIsCOMPOperation ) {
341                    PropertyIsCOMPOperation compOperation = (PropertyIsCOMPOperation) operation;
342                    Expression firstExpression = compOperation.getFirstExpression();
343                    Expression secondExpression = compOperation.getSecondExpression();
344                    if ( firstExpression instanceof PropertyName ) {
345                        filterPts.add( getFilterProperty( ft, (PropertyName) firstExpression ) );
346                    }
347                    if ( secondExpression instanceof PropertyName ) {
348                        filterPts.add( getFilterProperty( ft, (PropertyName) secondExpression ) );
349                    }
350                } else if ( operation instanceof PropertyIsInstanceOfOperation ) {
351                    PropertyIsInstanceOfOperation instanceOfOperation = (PropertyIsInstanceOfOperation) operation;
352                    filterPts.add( getFilterProperty( ft, instanceOfOperation.getPropertyName() ) );
353                } else if ( operation instanceof PropertyIsLikeOperation ) {
354                    PropertyIsLikeOperation likeOperation = (PropertyIsLikeOperation) operation;
355                    filterPts.add( getFilterProperty( ft, likeOperation.getPropertyName() ) );
356                } else if ( operation instanceof PropertyIsNullOperation ) {
357                    PropertyIsNullOperation nullOperation = (PropertyIsNullOperation) operation;
358                    filterPts.add( getFilterProperty( ft, nullOperation.getPropertyName() ) );
359                } else {
360                    assert false;
361                }
362            } else if ( operation instanceof LogicalOperation ) {
363                LogicalOperation logicalOperation = (LogicalOperation) operation;
364                for ( Operation subOperation : logicalOperation.getArguments() ) {
365                    addFilterProps( ft, filterPts, subOperation );
366                }
367            } else if ( operation instanceof SpatialOperation ) {
368                SpatialOperation spatialOperation = (SpatialOperation) operation;
369                filterPts.add( getFilterProperty( ft, spatialOperation.getPropertyName() ) );
370            } else {
371                assert false;
372            }
373        }
374    
375        private PropertyType getFilterProperty( MappedFeatureType ft, PropertyName propName )
376                                throws PropertyPathResolvingException {
377    
378            // TODO respect aliases
379            PropertyPath path = PropertyPathResolver.normalizePropertyPath( ft, null, propName.getValue() );
380    
381            QualifiedName propStep = path.getStep( 1 ).getPropertyName();
382            PropertyType pt = ft.getProperty( propStep );
383            if ( pt == null ) {
384                String msg = Messages.getMessage( "DATASTORE_PROPERTY_PATH_RESOLVE4", path, 2, propStep, ft.getName(),
385                                                  propName );
386                throw new PropertyPathResolvingException( msg );
387            }
388            return pt;
389        }
390    
391        @Override
392        public FeatureCollection performQuery( final Query query, final MappedFeatureType[] rootFts,
393                                               final DatastoreTransaction context )
394                                throws DatastoreException {
395            return performQuery( query, rootFts );
396        }
397    
398        /**
399         * Validates the given {@link MappedGMLSchema} against the available columns in the referenced shape file.
400         *
401         * @param schema
402         * @throws DatastoreException
403         */
404        private void validate( MappedGMLSchema schema )
405                                throws DatastoreException {
406    
407            Set<String> columnNames = determineShapeFileColumns( schema );
408    
409            FeatureType[] featureTypes = schema.getFeatureTypes();
410            for ( int i = 0; i < featureTypes.length; i++ ) {
411                Map<PropertyType, String> ftMapping = getFTMapping( featureTypes[i], columnNames );
412                ftMappings.put( featureTypes[i], ftMapping );
413            }
414        }
415    
416        private Map<PropertyType, String> getFTMapping( FeatureType ft, Set<String> columnNames )
417                                throws DatastoreException {
418            Map<PropertyType, String> ftMapping = new HashMap<PropertyType, String>();
419            PropertyType[] properties = ft.getProperties();
420            for ( int i = 0; i < properties.length; i++ ) {
421                MappedPropertyType pt = (MappedPropertyType) properties[i];
422                if ( pt instanceof MappedSimplePropertyType ) {
423                    SimpleContent content = ( (MappedSimplePropertyType) pt ).getContent();
424                    if ( !( content instanceof MappingField ) ) {
425                        String msg = Messages.getMessage( "DATASTORE_UNSUPPORTED_CONTENT", pt.getName() );
426                        throw new DatastoreException( msg );
427                    }
428                    // ensure that field name is in uppercase
429                    String field = ( (MappingField) content ).getField().toUpperCase();
430                    if ( !columnNames.contains( field ) ) {
431                        String msg = Messages.getMessage( "DATASTORE_FIELDNOTFOUND", field, pt.getName(),
432                                                          shapeFileURL.getFile(), columnNames );
433                        throw new DatastoreException( msg );
434                    }
435                    ftMapping.put( pt, field );
436                } else if ( pt instanceof MappedGeometryPropertyType ) {
437                    // nothing to do
438                } else {
439                    String msg = Messages.getMessage( "DATASTORE_NO_NESTED_FEATURE_TYPES", pt.getName() );
440                    throw new DatastoreException( msg );
441                }
442            }
443            return ftMapping;
444        }
445    
446        /**
447         * Determines the column names (in uppercase) of the shape file that is referenced by the given schema.
448         *
449         * @param schema
450         * @return column names (in uppercase)
451         * @throws DatastoreException
452         */
453        private Set<String> determineShapeFileColumns( MappedGMLSchema schema )
454                                throws DatastoreException {
455    
456            Set<String> columnNames = new HashSet<String>();
457            DBaseFile dbfFile = null;
458    
459            try {
460                Element schemaRoot = schema.getDocument().getRootElement();
461                String shapePath = XMLTools.getNodeAsString( schemaRoot, "xs:annotation/xs:appinfo/deegreewfs:File/text()",
462                                                             nsContext, null );
463                shapeFileURL = schema.getDocument().resolve( shapePath );
464                LOG.logDebug( "Opening dbf file '" + shapeFileURL + "'." );
465                dbfFile = new DBaseFile( shapeFileURL.getFile() );
466                String[] columns = dbfFile.getProperties();
467                for ( int i = 0; i < columns.length; i++ ) {
468                    columnNames.add( columns[i].toUpperCase() );
469                }
470                String s = "Successfully opened dbf file '" + shapeFileURL.getFile()
471                           + "' and retrieved the property columns.";
472                LOG.logDebug( s );
473            } catch ( Exception e ) {
474                LOG.logError( e.getMessage(), e );
475                throw new DatastoreException( Messages.getMessage( "DATASTORE_DBACCESSERROR" ) );
476            } finally {
477                if ( dbfFile != null ) {
478                    dbfFile.close();
479                }
480            }
481    
482            return columnNames;
483        }
484    
485        /**
486         * Closes the datastore so it can free dependent resources.
487         *
488         * @throws DatastoreException
489         */
490        @Override
491        public void close()
492                                throws DatastoreException {
493            // TODO
494        }
495    }