import java.awt.image.BufferedImage;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.LinkedList;

import javax.imageio.ImageIO;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.DefaultHttpClient;


public class Clawler {
    private static final int END_PAGE = 6764;
    private static final String PREFIX = "http://www./wbcx/index5.asp?page=";
    private static final String CODE_SAVE_PATH = "D:\\Wubi\\WubiCode.txt";
    private static final String IMG_SAVE_PATH_PREFIX = "D:\\Wubi\\img\\";
    
    private static LinkedList queue = new LinkedList();
    
    private static String m_imguri;
    
    public static void main(String[] args) throws IOException {
        HttpClient httpClient = new DefaultHttpClient();
        
        FileWriter fw = null;
        fw = new FileWriter(CODE_SAVE_PATH);
        
        for(int i = 1; i <= END_PAGE; ++i) {
            HttpUriRequest request = new HttpGet(PREFIX + i);
            
            try {
                HttpResponse response = httpClient.execute(request);
                                
                HttpEntity entity = response.getEntity();
                StringBuilder builder = new StringBuilder();
                if( entity != null ) {
                    InputStream is = entity.getContent();
                    byte[] tmp = new byte[2048];
                    while( is.read(tmp) != -1 ) {
                        builder.append(new String(tmp));
                    }
                    
                    fw.write( getWubiCode(builder.toString(), i) );    
                    downloadImg(m_imguri, IMG_SAVE_PATH_PREFIX + i + ".gif", i);
                }

            }
            catch( Exception e ) {
                queue.addLast((Integer)i);
                e.printStackTrace();        
            }    
            
            if( i%100==0 ) {
                fw.flush();
            }
        }    
        
        System.out.println("\n missing Code");
        while( !queue.isEmpty() ) {   //下載失敗的頁(yè)面
            System.out.println(queue.element());
            queue.removeFirst();
        }
        
        System.out.print("all done");
        fw.close();
        httpClient.getConnectionManager().shutdown();
    }
    
    public static String getWubiCode(String page, int number) { //提取五筆碼，字根圖的URL
        StringBuilder save = new StringBuilder();
        page = page.substring(page.indexOf("86五筆編碼"));    
        
        int index = 7;
        while( page.charAt(index)!= '<' ) save.append(page.charAt(index++));
        save.append(System.getProperty("line.separator"));
                
        index = 0;
        StringBuilder imgpath = new StringBuilder();
        page = page.substring(page.indexOf("http://www./GIF-82"));    
        while( page.charAt(index) != '\"' ) imgpath.append(page.charAt(index++));
        m_imguri = imgpath.toString();
                
        save.insert(0, imgpath.charAt(imgpath.length() - 5));
        save.insert(1, ' ');
                                
        return save.toString();        
    }
    
    public static void downloadImg(String url, String path, int number) { //下載圖片
        try {
            File out = new File(path);
            BufferedImage buffer = ImageIO.read(new URL(url));
            if( buffer == null ) {
                queue.addLast(number);
                System.out.println(number + "  " + url);
            }
            else {
                ImageIO.write(buffer, "gif", out);
            }
        }
        catch( IOException e ) {
            queue.addLast(number);
            System.out.println(url);
            System.out.println(e.getMessage());
        }
    }
}