    001    //$HeadURL: https://svn.wald.intevation.org/svn/deegree/base/branches/2.4_testing/src/org/deegree/tools/wms/Paver.java $
    002    /*----------------------------------------------------------------------------
    003     This file is part of deegree, http://deegree.org/
    004     Copyright (C) 2001-2009 by:
    005       Department of Geography, University of Bonn
    006     and
    007       lat/lon GmbH
    008
    009     This library is free software; you can redistribute it and/or modify it under
    010     the terms of the GNU Lesser General Public License as published by the Free
    011     Software Foundation; either version 2.1 of the License, or (at your option)
    012     any later version.
    013     This library is distributed in the hope that it will be useful, but WITHOUT
    014     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
    015     FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
    016     details.
    017     You should have received a copy of the GNU Lesser General Public License
    018     along with this library; if not, write to the Free Software Foundation, Inc.,
    019     59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
    020
    021     Contact information:
    022
    023     lat/lon GmbH
    024     Aennchenstr. 19, 53177 Bonn
    025     Germany
    026     http://lat-lon.de/
    027
    028     Department of Geography, University of Bonn
    029     Prof. Dr. Klaus Greve
    030     Postfach 1147, 53001 Bonn
    031     Germany
    032     http://www.geographie.uni-bonn.de/deegree/
    033
    034     e-mail: info@deegree.org
    035    ----------------------------------------------------------------------------*/
    036
    037    package org.deegree.tools.wms;
    038
    039    import static java.lang.Integer.parseInt;
    040    import static javax.imageio.ImageIO.read;
    041    import static javax.imageio.ImageIO.write;
    042    import static org.deegree.framework.log.LoggerFactory.getLogger;
    043    import static org.deegree.model.coverage.grid.WorldFile.readWorldFile;
    044    import static org.deegree.model.coverage.grid.WorldFile.TYPE.CENTER;
    045    import static org.deegree.model.spatialschema.GeometryFactory.createEnvelope;
    046
    047    import java.awt.image.BufferedImage;
    048    import java.io.BufferedReader;
    049    import java.io.File;
    050    import java.io.FileNotFoundException;
    051    import java.io.FileReader;
    052    import java.io.IOException;
    053    import java.io.InputStream;
    054    import java.net.MalformedURLException;
    055    import java.net.URL;
    056    import java.net.URLConnection;
    057    import java.util.LinkedList;
    058    import java.util.Map;
    059    import java.util.TreeMap;
    060
    061    import org.deegree.framework.log.ILogger;
    062    import org.deegree.framework.util.StringPair;
    063    import org.deegree.model.coverage.grid.WorldFile;
    064    import org.deegree.model.spatialschema.Envelope;
    065
    066    /**
    067     * <code>Paver</code> should really be run with Java6, since Java5 has severe problems with opening many (temporary)
    068     * files when using ImageIO, which are LEFT OPEN. Result is that you run out of file handles real fast, and all stops.
    069     *
    070     * @author <a href="mailto:schmitz@lat-lon.de">Andreas Schmitz</a>
    071     * @author last edited by: $Author: mschneider $
    072     *
    073     * @version $Revision: 18195 $, $Date: 2009-06-18 17:55:39 +0200 (Do, 18. Jun 2009) $
    074     */
    075    public class Paver implements Runnable {
    076
    077        static final ILogger LOG = getLogger( Paver.class );
    078
    079        private int number = 0;
    080
    081        final Config config;
    082
    083        final LinkedList<StringPair> requests = new LinkedList<StringPair>();
    084
    085        boolean finished = false;
    086
    087        private Paver( Config config ) {
    088            this.config = config;
    089            prepareRequests();
    090        }
    091
    092        private void prepareRequests() {
    093            new Thread( new Runnable() {
    094                public void run() {
    095                    if ( config.logfile != null ) {
    096                        LOG.logInfo( "Scanning logfile for errors..." );
    097                        int count = 0;
    098                        try {
    099                            BufferedReader in = new BufferedReader( new FileReader( config.logfile ) );
    100                            String s;
    101                            while ( ( s = in.readLine() ) != null ) {
    102                                if ( s.indexOf( "ERROR" ) != -1 ) {
    103                                    String[] ss = s.split( "'" );
    104                                    requests.addFirst( new StringPair( ss[1], ss[3] ) );
    105                                    ++count;
    106                                }
    107                            }
    108                            in.close();
    109                        } catch ( FileNotFoundException e ) {
    110                            LOG.logError( "Logfile was not found." );
    111                            System.exit( 1 );
    112                        } catch ( IOException e ) {
    113                            LOG.logError( "Logfile could not be read: " + e.getLocalizedMessage() );
    114                            System.exit( 1 );
    115                        }
    116
    117                        LOG.logInfo( "Found " + count + " errors." );
    118                        finished = true;
    119                        return;
    120                    }
    121
    122                    int count = 0;
    123                    for ( String name : config.dirsToLayers.keySet() ) {
    124                        String layers = config.dirsToLayers.get( name );
    125                        LOG.logInfo( "Preparing requests for directory " + name + " (layers '" + layers + "')" );
    126
    127                        File dir = new File( name );
    128                        File[] dirs = dir.listFiles();
    129
    130                        for ( File f : dirs ) {
    131                            if ( f.isDirectory() ) {
    132                                for ( File file : f.listFiles() ) {
    133                                    if ( file.getName().endsWith( ".wld" ) ) {
    134                                        ++count;
    135                                        if ( count % 1000 == 0 ) {
    136                                            LOG.logInfo( "Read " + count + " world files." );
    137                                        }
    138                                        String base = file.toString().substring( 0, file.toString().length() - 4 );
    139                                        try {
    140                                            WorldFile wf = readWorldFile( base + ".png", CENTER );
    141                                            Envelope e = wf.getEnvelope();
    142                                            if ( config.bbox != null && !e.intersects( config.bbox ) ) {
    143                                                continue;
    144                                            }
    145                                            int width = (int) ( ( e.getWidth() + wf.getResx() ) / wf.getResx() );
    146                                            int height = (int) ( ( e.getHeight() + wf.getResy() ) / wf.getResy() );
    147                                            String req = config.request + "BBOX=" + e.getMin().getX() + ","
    148                                                         + e.getMin().getY() + ",";
    149                                            req += e.getMax().getX() + "," + e.getMax().getY() + "&WIDTH=" + width
    150                                                   + "&HEIGHT=" + height + "&LAYERS=" + layers;
    151                                            requests.addFirst( new StringPair( base, req ) );
    152                                        } catch ( IOException e1 ) {
    153                                            LOG.logError( "Cannot read a world file: " + e1.getLocalizedMessage() + "("
    154                                                          + base + ")" );
    155                                        }
    156                                    }
    157                                }
    158                            }
    159                        }
    160                    }
    161
    162                    finished = true;
    163
    164                    LOG.logInfo( "Total number of requests is " + count );
    165                }
    166            } ).start();
    167        }
    168
    169        private void sendRequests() {
    170            LOG.logInfo( "Sending unknown number of requests." );
    171            for ( int i = 0; i < config.numThreads; ++i ) {
    172                new Thread( this ).start();
    173            }
    174
    175            while ( true ) {
    176                if ( finished && requests.isEmpty() ) {
    177                    System.exit( 0 );
    178                }
    179
    180                try {
    181                    Thread.sleep( 1000 );
    182                } catch ( InterruptedException e ) {
    183                    // shutting down.
    184                }
    185            }
    186        }
    187
    188        public void run() {
    189            outer: while ( true ) {
    190                StringPair pair = null;
    191                synchronized ( requests ) {
    192                    if ( !requests.isEmpty() ) {
    193                        pair = requests.poll();
    194                    }
    195                }
    196                if ( pair != null ) {
    197                    try {
    198                        LOG.logDebug( "Sending ", pair.second );
    199                        LOG.logDebug( "Storing at ", pair.first );
    200                        BufferedImage img = null;
    201
    202                        while ( img == null ) {
    203                            try {
    204                                URLConnection conn = new URL( pair.second ).openConnection();
    205                                conn.setReadTimeout( 60000 );
    206                                conn.setConnectTimeout( 60000 );
    207                                conn.setUseCaches( false );
    208                                InputStream stream = conn.getInputStream();
    209                                img = read( stream );
    210                                stream.close();
    211                                if ( img == null ) {
    212                                    LOG.logError( "Cannot parse map for file '" + pair.first + "' using request '"
    213                                                  + pair.second + "'" );
    214                                    ++number;
    215                                    continue outer;
    216                                }
    217                            } catch ( IOException e ) {
    218                                LOG.logError( "Unable to retrieve map for '" + pair.first + "' from '" + pair.second
    219                                              + "': " + e.getLocalizedMessage() );
    220                                System.gc(); // maybe a file or two will get cleaned up...
    221                                try {
    222                                    Thread.sleep( 60000 );
    223                                } catch ( InterruptedException e1 ) {
    224                                    // then we're gone anyway
    225                                }
    226                            }
    227                        }
    228                        int num = ++number;
    229                        if ( num % 100 == 0 ) {
    230                            LOG.logInfo( "Currently processing number " + num );
    231                        }
    232                        write( img, "png", new File( pair.first + ".png" ) );
    233                    } catch ( MalformedURLException e ) {
    234                        LOG.logError( "Unknown error", e );
    235                    } catch ( IOException e ) {
    236                        LOG.logError( "Unable to retrieve map from " + pair.second, e );
    237                    }
    238                } else {
    239                    try {
    240                        Thread.sleep( 100 );
    241                        LOG.logInfo( "Waiting for requests list..." );
    242                    } catch ( InterruptedException e ) {
    243                        // ok, so we shutdown
    244                    }
    245                }
    246            }
    247        }
    248
    249        private static void printUsage( String missing ) {
    250            System.out.println( "The " + missing + " parameter is missing." );
    251            System.out.println( "--request,-q: the basic request, without LAYERS, BBOX, WIDTH and HEIGHT parameters. Mandatory." );
    252            System.out.println( "--layers,-l: the layers to request plus tiling directory. Can occur multiple times. Mandatory." );
    253            System.out.println( "--bbox,-b: the bounding box to update. Optional, default is bbox of the raster tree." );
    254            System.out.println( "--num-threads,-n: the number of requests to send simultaneously. Optional, default is 1." );
    255            System.out.println( "--directory,-d: the directory where the raster trees reside. Optional, default is current directory." );
    256            System.out.println( "--fix, -f: a log file to scan for requests that failed. Specifying this causes the program to go into a different mode." );
    257            System.out.println( "Example: " );
    258            System.out.println( "Paver -q \"http://demo.deegree.org/deegree-wms/services?REQUEST=GetMap&SERVICE=WMS&VERSION=1.1.1&TRANSPARENT=TRUE&FORMAT=image/png&SRS=EPSG:26912&STYLES=\" -l dir1:Vegetation -l dir2:Lake,Roads" );
    259            System.out.println();
    260            System.exit( 1 );
    261        }
    262
    263        /**
    264         * @param args
    265         * @throws IOException
    266         */
    267        public static void main( String[] args )
    268                                throws IOException {
    269            Config config = new Config();
    270            for ( int i = 0; i < args.length; ++i ) {
    271                if ( args[i].equals( "--bbox" ) || args[i].equals( "-b" ) ) {
    272                    if ( i != args.length - 1 ) {
    273                        config.bbox = createEnvelope( args[++i], null );
    274                    }
    275                } else if ( args[i].equals( "--request" ) || args[i].equals( "-q" ) ) {
    276                    if ( i != args.length - 1 ) {
    277                        config.request = args[++i];
    278                        if ( !config.request.endsWith( "&" ) ) {
    279                            config.request += "&";
    280                        }
    281                    }
    282                } else if ( args[i].equals( "--num-threads" ) || args[i].equals( "-n" ) ) {
    283                    if ( i != args.length - 1 ) {
    284                        config.numThreads = parseInt( args[++i] );
    285                    }
    286                } else if ( args[i].equals( "--directory" ) || args[i].equals( "-d" ) ) {
    287                    if ( i != args.length - 1 ) {
    288                        config.dir = new File( args[++i] );
    289                        if ( !config.dir.exists() ) {
    290                            config.dir.mkdirs();
    291                        }
    292                    }
    293                } else if ( args[i].equals( "--layers" ) || args[i].equals( "-l" ) ) {
    294                    if ( i != args.length - 1 ) {
    295                        String[] param = args[++i].split( ":" );
    296                        config.dirsToLayers.put( param[0], param[1] );
    297                    }
    298                } else if ( args[i].equals( "--fix" ) || args[i].equals( "-f" ) ) {
    299                    if ( i != args.length - 1 ) {
    300                        config.logfile = args[++i];
    301                    }
    302                }
    303            }
    304
    305            if ( config.request == null && config.logfile == null ) {
    306                printUsage( "request" );
    307            }
    308
    309            Paver paver = new Paver( config );
    310            paver.sendRequests();
    311        }
    312
    313        static class Config {
    314            Envelope bbox;
    315
    316            String request, logfile;
    317
    318            int numThreads = 1;
    319
    320            File dir = new File( "." );
    321
    322            Map<String, String> dirsToLayers = new TreeMap<String, String>();
    323        }
    324
    325    }