001    //$HeadURL: svn+ssh://jwilden@svn.wald.intevation.org/deegree/base/branches/2.5_testing/src/org/deegree/tools/wms/Paver.java $
002    /*----------------------------------------------------------------------------
003     This file is part of deegree, http://deegree.org/
004     Copyright (C) 2001-2009 by:
005       Department of Geography, University of Bonn
006     and
007       lat/lon GmbH
008    
009     This library is free software; you can redistribute it and/or modify it under
010     the terms of the GNU Lesser General Public License as published by the Free
011     Software Foundation; either version 2.1 of the License, or (at your option)
012     any later version.
013     This library is distributed in the hope that it will be useful, but WITHOUT
014     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
015     FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
016     details.
017     You should have received a copy of the GNU Lesser General Public License
018     along with this library; if not, write to the Free Software Foundation, Inc.,
019     59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
020    
021     Contact information:
022    
023     lat/lon GmbH
024     Aennchenstr. 19, 53177 Bonn
025     Germany
026     http://lat-lon.de/
027    
028     Department of Geography, University of Bonn
029     Prof. Dr. Klaus Greve
030     Postfach 1147, 53001 Bonn
031     Germany
032     http://www.geographie.uni-bonn.de/deegree/
033    
034     e-mail: info@deegree.org
035    ----------------------------------------------------------------------------*/
036    
037    package org.deegree.tools.wms;
038    
039    import static java.lang.Integer.parseInt;
040    import static javax.imageio.ImageIO.read;
041    import static javax.imageio.ImageIO.write;
042    import static org.deegree.framework.log.LoggerFactory.getLogger;
043    import static org.deegree.model.coverage.grid.WorldFile.readWorldFile;
044    import static org.deegree.model.coverage.grid.WorldFile.TYPE.CENTER;
045    import static org.deegree.model.spatialschema.GeometryFactory.createEnvelope;
046    
047    import java.awt.image.BufferedImage;
048    import java.io.BufferedReader;
049    import java.io.File;
050    import java.io.FileNotFoundException;
051    import java.io.FileReader;
052    import java.io.IOException;
053    import java.io.InputStream;
054    import java.net.MalformedURLException;
055    import java.net.URL;
056    import java.net.URLConnection;
057    import java.util.LinkedList;
058    import java.util.Map;
059    import java.util.TreeMap;
060    
061    import org.deegree.framework.log.ILogger;
062    import org.deegree.framework.util.StringPair;
063    import org.deegree.model.coverage.grid.WorldFile;
064    import org.deegree.model.spatialschema.Envelope;
065    
066    /**
067     * <code>Paver</code> should really be run with Java6, since Java5 has severe problems with opening many (temporary)
068     * files when using ImageIO, which are LEFT OPEN. Result is that you run out of file handles real fast, and all stops.
069     *
070     * @author <a href="mailto:schmitz@lat-lon.de">Andreas Schmitz</a>
071     * @author last edited by: $Author: mschneider $
072     *
073     * @version $Revision: 18195 $, $Date: 2009-06-18 17:55:39 +0200 (Do, 18 Jun 2009) $
074     */
075    public class Paver implements Runnable {
076    
077        static final ILogger LOG = getLogger( Paver.class );
078    
079        private int number = 0;
080    
081        final Config config;
082    
083        final LinkedList<StringPair> requests = new LinkedList<StringPair>();
084    
085        boolean finished = false;
086    
087        private Paver( Config config ) {
088            this.config = config;
089            prepareRequests();
090        }
091    
092        private void prepareRequests() {
093            new Thread( new Runnable() {
094                public void run() {
095                    if ( config.logfile != null ) {
096                        LOG.logInfo( "Scanning logfile for errors..." );
097                        int count = 0;
098                        try {
099                            BufferedReader in = new BufferedReader( new FileReader( config.logfile ) );
100                            String s;
101                            while ( ( s = in.readLine() ) != null ) {
102                                if ( s.indexOf( "ERROR" ) != -1 ) {
103                                    String[] ss = s.split( "'" );
104                                    requests.addFirst( new StringPair( ss[1], ss[3] ) );
105                                    ++count;
106                                }
107                            }
108                            in.close();
109                        } catch ( FileNotFoundException e ) {
110                            LOG.logError( "Logfile was not found." );
111                            System.exit( 1 );
112                        } catch ( IOException e ) {
113                            LOG.logError( "Logfile could not be read: " + e.getLocalizedMessage() );
114                            System.exit( 1 );
115                        }
116    
117                        LOG.logInfo( "Found " + count + " errors." );
118                        finished = true;
119                        return;
120                    }
121    
122                    int count = 0;
123                    for ( String name : config.dirsToLayers.keySet() ) {
124                        String layers = config.dirsToLayers.get( name );
125                        LOG.logInfo( "Preparing requests for directory " + name + " (layers '" + layers + "')" );
126    
127                        File dir = new File( name );
128                        File[] dirs = dir.listFiles();
129    
130                        for ( File f : dirs ) {
131                            if ( f.isDirectory() ) {
132                                for ( File file : f.listFiles() ) {
133                                    if ( file.getName().endsWith( ".wld" ) ) {
134                                        ++count;
135                                        if ( count % 1000 == 0 ) {
136                                            LOG.logInfo( "Read " + count + " world files." );
137                                        }
138                                        String base = file.toString().substring( 0, file.toString().length() - 4 );
139                                        try {
140                                            WorldFile wf = readWorldFile( base + ".png", CENTER );
141                                            Envelope e = wf.getEnvelope();
142                                            if ( config.bbox != null && !e.intersects( config.bbox ) ) {
143                                                continue;
144                                            }
145                                            int width = (int) ( ( e.getWidth() + wf.getResx() ) / wf.getResx() );
146                                            int height = (int) ( ( e.getHeight() + wf.getResy() ) / wf.getResy() );
147                                            String req = config.request + "BBOX=" + e.getMin().getX() + ","
148                                                         + e.getMin().getY() + ",";
149                                            req += e.getMax().getX() + "," + e.getMax().getY() + "&WIDTH=" + width
150                                                   + "&HEIGHT=" + height + "&LAYERS=" + layers;
151                                            requests.addFirst( new StringPair( base, req ) );
152                                        } catch ( IOException e1 ) {
153                                            LOG.logError( "Cannot read a world file: " + e1.getLocalizedMessage() + "("
154                                                          + base + ")" );
155                                        }
156                                    }
157                                }
158                            }
159                        }
160                    }
161    
162                    finished = true;
163    
164                    LOG.logInfo( "Total number of requests is " + count );
165                }
166            } ).start();
167        }
168    
169        private void sendRequests() {
170            LOG.logInfo( "Sending unknown number of requests." );
171            for ( int i = 0; i < config.numThreads; ++i ) {
172                new Thread( this ).start();
173            }
174    
175            while ( true ) {
176                if ( finished && requests.isEmpty() ) {
177                    System.exit( 0 );
178                }
179    
180                try {
181                    Thread.sleep( 1000 );
182                } catch ( InterruptedException e ) {
183                    // shutting down.
184                }
185            }
186        }
187    
188        public void run() {
189            outer: while ( true ) {
190                StringPair pair = null;
191                synchronized ( requests ) {
192                    if ( !requests.isEmpty() ) {
193                        pair = requests.poll();
194                    }
195                }
196                if ( pair != null ) {
197                    try {
198                        LOG.logDebug( "Sending ", pair.second );
199                        LOG.logDebug( "Storing at ", pair.first );
200                        BufferedImage img = null;
201    
202                        while ( img == null ) {
203                            try {
204                                URLConnection conn = new URL( pair.second ).openConnection();
205                                conn.setReadTimeout( 60000 );
206                                conn.setConnectTimeout( 60000 );
207                                conn.setUseCaches( false );
208                                InputStream stream = conn.getInputStream();
209                                img = read( stream );
210                                stream.close();
211                                if ( img == null ) {
212                                    LOG.logError( "Cannot parse map for file '" + pair.first + "' using request '"
213                                                  + pair.second + "'" );
214                                    ++number;
215                                    continue outer;
216                                }
217                            } catch ( IOException e ) {
218                                LOG.logError( "Unable to retrieve map for '" + pair.first + "' from '" + pair.second
219                                              + "': " + e.getLocalizedMessage() );
220                                System.gc(); // maybe a file or two will get cleaned up...
221                                try {
222                                    Thread.sleep( 60000 );
223                                } catch ( InterruptedException e1 ) {
224                                    // then we're gone anyway
225                                }
226                            }
227                        }
228                        int num = ++number;
229                        if ( num % 100 == 0 ) {
230                            LOG.logInfo( "Currently processing number " + num );
231                        }
232                        write( img, "png", new File( pair.first + ".png" ) );
233                    } catch ( MalformedURLException e ) {
234                        LOG.logError( "Unknown error", e );
235                    } catch ( IOException e ) {
236                        LOG.logError( "Unable to retrieve map from " + pair.second, e );
237                    }
238                } else {
239                    try {
240                        Thread.sleep( 100 );
241                        LOG.logInfo( "Waiting for requests list..." );
242                    } catch ( InterruptedException e ) {
243                        // ok, so we shutdown
244                    }
245                }
246            }
247        }
248    
249        private static void printUsage( String missing ) {
250            System.out.println( "The " + missing + " parameter is missing." );
251            System.out.println( "--request,-q: the basic request, without LAYERS, BBOX, WIDTH and HEIGHT parameters. Mandatory." );
252            System.out.println( "--layers,-l: the layers to request plus tiling directory. Can occur multiple times. Mandatory." );
253            System.out.println( "--bbox,-b: the bounding box to update. Optional, default is bbox of the raster tree." );
254            System.out.println( "--num-threads,-n: the number of requests to send simultaneously. Optional, default is 1." );
255            System.out.println( "--directory,-d: the directory where the raster trees reside. Optional, default is current directory." );
256            System.out.println( "--fix, -f: a log file to scan for requests that failed. Specifying this causes the program to go into a different mode." );
257            System.out.println( "Example: " );
258            System.out.println( "Paver -q \"http://demo.deegree.org/deegree-wms/services?REQUEST=GetMap&SERVICE=WMS&VERSION=1.1.1&TRANSPARENT=TRUE&FORMAT=image/png&SRS=EPSG:26912&STYLES=\" -l dir1:Vegetation -l dir2:Lake,Roads" );
259            System.out.println();
260            System.exit( 1 );
261        }
262    
263        /**
264         * @param args
265         * @throws IOException
266         */
267        public static void main( String[] args )
268                                throws IOException {
269            Config config = new Config();
270            for ( int i = 0; i < args.length; ++i ) {
271                if ( args[i].equals( "--bbox" ) || args[i].equals( "-b" ) ) {
272                    if ( i != args.length - 1 ) {
273                        config.bbox = createEnvelope( args[++i], null );
274                    }
275                } else if ( args[i].equals( "--request" ) || args[i].equals( "-q" ) ) {
276                    if ( i != args.length - 1 ) {
277                        config.request = args[++i];
278                        if ( !config.request.endsWith( "&" ) ) {
279                            config.request += "&";
280                        }
281                    }
282                } else if ( args[i].equals( "--num-threads" ) || args[i].equals( "-n" ) ) {
283                    if ( i != args.length - 1 ) {
284                        config.numThreads = parseInt( args[++i] );
285                    }
286                } else if ( args[i].equals( "--directory" ) || args[i].equals( "-d" ) ) {
287                    if ( i != args.length - 1 ) {
288                        config.dir = new File( args[++i] );
289                        if ( !config.dir.exists() ) {
290                            config.dir.mkdirs();
291                        }
292                    }
293                } else if ( args[i].equals( "--layers" ) || args[i].equals( "-l" ) ) {
294                    if ( i != args.length - 1 ) {
295                        String[] param = args[++i].split( ":" );
296                        config.dirsToLayers.put( param[0], param[1] );
297                    }
298                } else if ( args[i].equals( "--fix" ) || args[i].equals( "-f" ) ) {
299                    if ( i != args.length - 1 ) {
300                        config.logfile = args[++i];
301                    }
302                }
303            }
304    
305            if ( config.request == null && config.logfile == null ) {
306                printUsage( "request" );
307            }
308    
309            Paver paver = new Paver( config );
310            paver.sendRequests();
311        }
312    
313        static class Config {
314            Envelope bbox;
315    
316            String request, logfile;
317    
318            int numThreads = 1;
319    
320            File dir = new File( "." );
321    
322            Map<String, String> dirsToLayers = new TreeMap<String, String>();
323        }
324    
325    }