001 //$HeadURL: svn+ssh://jwilden@svn.wald.intevation.org/deegree/base/branches/2.5_testing/src/org/deegree/tools/wms/Paver.java $
002 /*----------------------------------------------------------------------------
003 This file is part of deegree, http://deegree.org/
004 Copyright (C) 2001-2009 by:
005 Department of Geography, University of Bonn
006 and
007 lat/lon GmbH
008
009 This library is free software; you can redistribute it and/or modify it under
010 the terms of the GNU Lesser General Public License as published by the Free
011 Software Foundation; either version 2.1 of the License, or (at your option)
012 any later version.
013 This library is distributed in the hope that it will be useful, but WITHOUT
014 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
015 FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
016 details.
017 You should have received a copy of the GNU Lesser General Public License
018 along with this library; if not, write to the Free Software Foundation, Inc.,
019 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
020
021 Contact information:
022
023 lat/lon GmbH
024 Aennchenstr. 19, 53177 Bonn
025 Germany
026 http://lat-lon.de/
027
028 Department of Geography, University of Bonn
029 Prof. Dr. Klaus Greve
030 Postfach 1147, 53001 Bonn
031 Germany
032 http://www.geographie.uni-bonn.de/deegree/
033
034 e-mail: info@deegree.org
035 ----------------------------------------------------------------------------*/
036
037 package org.deegree.tools.wms;
038
039 import static java.lang.Integer.parseInt;
040 import static javax.imageio.ImageIO.read;
041 import static javax.imageio.ImageIO.write;
042 import static org.deegree.framework.log.LoggerFactory.getLogger;
043 import static org.deegree.model.coverage.grid.WorldFile.readWorldFile;
044 import static org.deegree.model.coverage.grid.WorldFile.TYPE.CENTER;
045 import static org.deegree.model.spatialschema.GeometryFactory.createEnvelope;
046
047 import java.awt.image.BufferedImage;
048 import java.io.BufferedReader;
049 import java.io.File;
050 import java.io.FileNotFoundException;
051 import java.io.FileReader;
052 import java.io.IOException;
053 import java.io.InputStream;
054 import java.net.MalformedURLException;
055 import java.net.URL;
056 import java.net.URLConnection;
057 import java.util.LinkedList;
058 import java.util.Map;
059 import java.util.TreeMap;
060
061 import org.deegree.framework.log.ILogger;
062 import org.deegree.framework.util.StringPair;
063 import org.deegree.model.coverage.grid.WorldFile;
064 import org.deegree.model.spatialschema.Envelope;
065
066 /**
067 * <code>Paver</code> should really be run with Java6, since Java5 has severe problems with opening many (temporary)
068 * files when using ImageIO, which are LEFT OPEN. Result is that you run out of file handles real fast, and all stops.
069 *
070 * @author <a href="mailto:schmitz@lat-lon.de">Andreas Schmitz</a>
071 * @author last edited by: $Author: mschneider $
072 *
073 * @version $Revision: 18195 $, $Date: 2009-06-18 17:55:39 +0200 (Do, 18 Jun 2009) $
074 */
075 public class Paver implements Runnable {
076
077 static final ILogger LOG = getLogger( Paver.class );
078
079 private int number = 0;
080
081 final Config config;
082
083 final LinkedList<StringPair> requests = new LinkedList<StringPair>();
084
085 boolean finished = false;
086
087 private Paver( Config config ) {
088 this.config = config;
089 prepareRequests();
090 }
091
092 private void prepareRequests() {
093 new Thread( new Runnable() {
094 public void run() {
095 if ( config.logfile != null ) {
096 LOG.logInfo( "Scanning logfile for errors..." );
097 int count = 0;
098 try {
099 BufferedReader in = new BufferedReader( new FileReader( config.logfile ) );
100 String s;
101 while ( ( s = in.readLine() ) != null ) {
102 if ( s.indexOf( "ERROR" ) != -1 ) {
103 String[] ss = s.split( "'" );
104 requests.addFirst( new StringPair( ss[1], ss[3] ) );
105 ++count;
106 }
107 }
108 in.close();
109 } catch ( FileNotFoundException e ) {
110 LOG.logError( "Logfile was not found." );
111 System.exit( 1 );
112 } catch ( IOException e ) {
113 LOG.logError( "Logfile could not be read: " + e.getLocalizedMessage() );
114 System.exit( 1 );
115 }
116
117 LOG.logInfo( "Found " + count + " errors." );
118 finished = true;
119 return;
120 }
121
122 int count = 0;
123 for ( String name : config.dirsToLayers.keySet() ) {
124 String layers = config.dirsToLayers.get( name );
125 LOG.logInfo( "Preparing requests for directory " + name + " (layers '" + layers + "')" );
126
127 File dir = new File( name );
128 File[] dirs = dir.listFiles();
129
130 for ( File f : dirs ) {
131 if ( f.isDirectory() ) {
132 for ( File file : f.listFiles() ) {
133 if ( file.getName().endsWith( ".wld" ) ) {
134 ++count;
135 if ( count % 1000 == 0 ) {
136 LOG.logInfo( "Read " + count + " world files." );
137 }
138 String base = file.toString().substring( 0, file.toString().length() - 4 );
139 try {
140 WorldFile wf = readWorldFile( base + ".png", CENTER );
141 Envelope e = wf.getEnvelope();
142 if ( config.bbox != null && !e.intersects( config.bbox ) ) {
143 continue;
144 }
145 int width = (int) ( ( e.getWidth() + wf.getResx() ) / wf.getResx() );
146 int height = (int) ( ( e.getHeight() + wf.getResy() ) / wf.getResy() );
147 String req = config.request + "BBOX=" + e.getMin().getX() + ","
148 + e.getMin().getY() + ",";
149 req += e.getMax().getX() + "," + e.getMax().getY() + "&WIDTH=" + width
150 + "&HEIGHT=" + height + "&LAYERS=" + layers;
151 requests.addFirst( new StringPair( base, req ) );
152 } catch ( IOException e1 ) {
153 LOG.logError( "Cannot read a world file: " + e1.getLocalizedMessage() + "("
154 + base + ")" );
155 }
156 }
157 }
158 }
159 }
160 }
161
162 finished = true;
163
164 LOG.logInfo( "Total number of requests is " + count );
165 }
166 } ).start();
167 }
168
169 private void sendRequests() {
170 LOG.logInfo( "Sending unknown number of requests." );
171 for ( int i = 0; i < config.numThreads; ++i ) {
172 new Thread( this ).start();
173 }
174
175 while ( true ) {
176 if ( finished && requests.isEmpty() ) {
177 System.exit( 0 );
178 }
179
180 try {
181 Thread.sleep( 1000 );
182 } catch ( InterruptedException e ) {
183 // shutting down.
184 }
185 }
186 }
187
188 public void run() {
189 outer: while ( true ) {
190 StringPair pair = null;
191 synchronized ( requests ) {
192 if ( !requests.isEmpty() ) {
193 pair = requests.poll();
194 }
195 }
196 if ( pair != null ) {
197 try {
198 LOG.logDebug( "Sending ", pair.second );
199 LOG.logDebug( "Storing at ", pair.first );
200 BufferedImage img = null;
201
202 while ( img == null ) {
203 try {
204 URLConnection conn = new URL( pair.second ).openConnection();
205 conn.setReadTimeout( 60000 );
206 conn.setConnectTimeout( 60000 );
207 conn.setUseCaches( false );
208 InputStream stream = conn.getInputStream();
209 img = read( stream );
210 stream.close();
211 if ( img == null ) {
212 LOG.logError( "Cannot parse map for file '" + pair.first + "' using request '"
213 + pair.second + "'" );
214 ++number;
215 continue outer;
216 }
217 } catch ( IOException e ) {
218 LOG.logError( "Unable to retrieve map for '" + pair.first + "' from '" + pair.second
219 + "': " + e.getLocalizedMessage() );
220 System.gc(); // maybe a file or two will get cleaned up...
221 try {
222 Thread.sleep( 60000 );
223 } catch ( InterruptedException e1 ) {
224 // then we're gone anyway
225 }
226 }
227 }
228 int num = ++number;
229 if ( num % 100 == 0 ) {
230 LOG.logInfo( "Currently processing number " + num );
231 }
232 write( img, "png", new File( pair.first + ".png" ) );
233 } catch ( MalformedURLException e ) {
234 LOG.logError( "Unknown error", e );
235 } catch ( IOException e ) {
236 LOG.logError( "Unable to retrieve map from " + pair.second, e );
237 }
238 } else {
239 try {
240 Thread.sleep( 100 );
241 LOG.logInfo( "Waiting for requests list..." );
242 } catch ( InterruptedException e ) {
243 // ok, so we shutdown
244 }
245 }
246 }
247 }
248
249 private static void printUsage( String missing ) {
250 System.out.println( "The " + missing + " parameter is missing." );
251 System.out.println( "--request,-q: the basic request, without LAYERS, BBOX, WIDTH and HEIGHT parameters. Mandatory." );
252 System.out.println( "--layers,-l: the layers to request plus tiling directory. Can occur multiple times. Mandatory." );
253 System.out.println( "--bbox,-b: the bounding box to update. Optional, default is bbox of the raster tree." );
254 System.out.println( "--num-threads,-n: the number of requests to send simultaneously. Optional, default is 1." );
255 System.out.println( "--directory,-d: the directory where the raster trees reside. Optional, default is current directory." );
256 System.out.println( "--fix, -f: a log file to scan for requests that failed. Specifying this causes the program to go into a different mode." );
257 System.out.println( "Example: " );
258 System.out.println( "Paver -q \"http://demo.deegree.org/deegree-wms/services?REQUEST=GetMap&SERVICE=WMS&VERSION=1.1.1&TRANSPARENT=TRUE&FORMAT=image/png&SRS=EPSG:26912&STYLES=\" -l dir1:Vegetation -l dir2:Lake,Roads" );
259 System.out.println();
260 System.exit( 1 );
261 }
262
263 /**
264 * @param args
265 * @throws IOException
266 */
267 public static void main( String[] args )
268 throws IOException {
269 Config config = new Config();
270 for ( int i = 0; i < args.length; ++i ) {
271 if ( args[i].equals( "--bbox" ) || args[i].equals( "-b" ) ) {
272 if ( i != args.length - 1 ) {
273 config.bbox = createEnvelope( args[++i], null );
274 }
275 } else if ( args[i].equals( "--request" ) || args[i].equals( "-q" ) ) {
276 if ( i != args.length - 1 ) {
277 config.request = args[++i];
278 if ( !config.request.endsWith( "&" ) ) {
279 config.request += "&";
280 }
281 }
282 } else if ( args[i].equals( "--num-threads" ) || args[i].equals( "-n" ) ) {
283 if ( i != args.length - 1 ) {
284 config.numThreads = parseInt( args[++i] );
285 }
286 } else if ( args[i].equals( "--directory" ) || args[i].equals( "-d" ) ) {
287 if ( i != args.length - 1 ) {
288 config.dir = new File( args[++i] );
289 if ( !config.dir.exists() ) {
290 config.dir.mkdirs();
291 }
292 }
293 } else if ( args[i].equals( "--layers" ) || args[i].equals( "-l" ) ) {
294 if ( i != args.length - 1 ) {
295 String[] param = args[++i].split( ":" );
296 config.dirsToLayers.put( param[0], param[1] );
297 }
298 } else if ( args[i].equals( "--fix" ) || args[i].equals( "-f" ) ) {
299 if ( i != args.length - 1 ) {
300 config.logfile = args[++i];
301 }
302 }
303 }
304
305 if ( config.request == null && config.logfile == null ) {
306 printUsage( "request" );
307 }
308
309 Paver paver = new Paver( config );
310 paver.sendRequests();
311 }
312
313 static class Config {
314 Envelope bbox;
315
316 String request, logfile;
317
318 int numThreads = 1;
319
320 File dir = new File( "." );
321
322 Map<String, String> dirsToLayers = new TreeMap<String, String>();
323 }
324
325 }