用多线程实现的Java爬虫程序
以下是一个Java爬虫程序，它能从指定主页开始，按照指定的深度抓取该站点域名下的网页并维护简单索引。
参数：private static int webDepth = 2;//爬虫深度。主页的深度为1，设置深度后超过该深度的网页不会抓取。
private int intThreadNum = 10;//线程数。开启的线程数。
抓取时也会在程序源文件目录下生成一个report.txt文件记录爬虫的运行情况，并在抓取结束后生成一个fileindex.txt文件维护网页文件索引。
本程序用到了多线程(静态变量和同步)，泛型，文件操作，URL类和连接，Hashtable类关联数组，正则表达式及其相关类。运行时需使用命令行参数，第一个参数应使用http://开头的有效URL字符串作为爬虫的主页，第二个参数（可选）应输入可转换为int型的字符串（用Integer.parseInt(String s)静态
方法
快递客服问题件处理详细方法山木方法pdf计算方法pdf华与华方法下载八字理论方法下载
可以转换的字符串，如3）作为爬虫深度，如果没有，则默认深度为2。
本程序的不足之处是：只考虑了href= href=' href="后加绝对url的这三种情况(由于url地址在网页源文件中情况比较复杂，有时处理也会出现错误)，还有相对url和window.open('的情况没有考虑。异常处理程序也只是简单处理。如果读者有改进办法可以把源代码帖出，不胜感激。
附上源代码如下（保存名为GetWeb.java）：
import java.io.File;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Hashtable;
public class GetWeb {
private int webDepth = 2;//爬虫深度
private int intThreadNum = 10;//线程数
private String strHomePage = "";//主页地址
private String myDomain;//域名
private String fPath = "web";//储存网页文件的目录名
private ArrayList
arrUrls = new ArrayList();//存储未处理URL
private ArrayList arrUrl = new ArrayList();//存储所有URL供建立索引
private Hashtable allUrls = new Hashtable();//存储所有URL的网页号
private Hashtable deepUrls = new Hashtable();//存储所有URL深度
private int intWebIndex = 0;//网页对应文件下标，从0开始
private String charset = "GB2312";
private String report = "";
private long startTime;
private int webSuccessed = 0;
private int webFailed = 0;
public GetWeb(String s)
{
this.strHomePage = s;
}
public GetWeb(String s,int i)
{
this.strHomePage = s;
this.webDepth = i;
}
public synchronized void addWebSuccessed()
{
webSuccessed++;
}
public synchronized void addWebFailed()
{
webFailed++;
}
public synchronized void addReport(String s)
{
try
{
report += s;
PrintWriter pwReport = new PrintWriter(new FileOutputStream("report.txt"));
pwReport.println(report);
pwReport.close();
}
catch(Exception e)
{
System.out.println("生成报告文件失败!");
}
}
public synchronized String getAUrl()
{
String tmpAUrl = arrUrls.get(0);
arrUrls.remove(0);
return tmpAUrl;
}
public synchronized String getUrl()
{
String tmpUrl = arrUrl.get(0);
arrUrl.remove(0);
return tmpUrl;
}
public synchronized Integer getIntWebIndex()
{
intWebIndex++;
return intWebIndex;
}
/**
* @param args
*/
public static void main(String[] args)
{
if (args.length == 0 || args[0].equals(""))
{
System.out.println("No input!");
System.exit(1);
}
else if(args.length == 1)
{
GetWeb gw = new GetWeb(args[0]);
gw.getWebByHomePage();
}
else
{
GetWeb gw = new GetWeb(args[0],Integer.parseInt(args[1]));
gw.getWebByHomePage();
}
}
public void getWebByHomePage()
{
startTime = System.currentTimeMillis();
this.myDomain = getDomain();
if (myDomain == null)
{
System.out.println("Wrong input!");
//System.exit(1);
return;
}
System.out.println("Homepage = " + strHomePage);
addReport("Homepage = " + strHomePage + "!\n");
System.out.println("Domain = " + myDomain);
addReport("Domain = " + myDomain + "!\n");
arrUrls.add(strHomePage);
arrUrl.add(strHomePage);
allUrls.put(strHomePage,0);
deepUrls.put(strHomePage,1);
File fDir = new File(fPath);
if(!fDir.exists())
{
fDir.mkdir();
}
System.out.println("Start!");
this.addReport("Start!\n");
String tmp = getAUrl();
this.getWebByUrl(tmp,charset,allUrls.get(tmp)+"");
int i = 0;
for (i=0;i 0)
{
sb.append("\n" + tmp_rLine);
pw.println(tmp_rLine);
pw.flush();
if (deepUrls.get(strUrl) < webDepth)getUrlByString(tmp_rLine,strUrl);
}
tmp_rLine = null;
}
is.close();
pw.close();
System.out.println("Get web successfully! " + strUrl);
addReport("Get web successfully! " + strUrl + "\n");
addWebSuccessed();
}
catch (Exception e)
{
System.out.println("Get web failed! " + strUrl);
addReport("Get web failed! " + strUrl + "\n");
addWebFailed();
}
}
public String getDomain()
{
String reg = "(?<=http\\://[a-zA-Z0-9]{0,100}[.]{0,1})[^.\\s]*?\\.(com|cn|net|org|biz|info|cc|tv)";
Pattern p = Pattern.compile(reg,Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(strHomePage);
boolean blnp = m.find();
if (blnp == true)
{
return m.group(0);
}
return null;
}
public void getUrlByString(String inputArgs,String strUrl)
{
String tmpStr = inputArgs;
String regUrl = "(?<=(href=)[\"]?[\']?)[http://][^\\s\"\'\\?]*(" + myDomain + ")[^\\s\"\'>]*";
Pattern p = Pattern.compile(regUrl,Pattern.CASE_INSENSITIVE);
Matcher m = p.matcher(tmpStr);
boolean blnp = m.find();
//int i = 0;
while (blnp == true)
{
if (!allUrls.containsKey(m.group(0)))
{
System.out.println("Find a new url,depth:" + (deepUrls.get(strUrl)+1) + " "+ m.group(0));
addReport("Find a new url,depth:" + (deepUrls.get(strUrl)+1) + " "+ m.group(0) + "\n");
arrUrls.add(m.group(0));
arrUrl.add(m.group(0));
allUrls.put(m.group(0),getIntWebIndex());
deepUrls.put(m.group(0),(deepUrls.get(strUrl)+1));
}
tmpStr = tmpStr.substring(m.end(),tmpStr.length());
m = p.matcher(tmpStr);
blnp = m.find();
}
}
class Processer implements Runnable
{
GetWeb gw;
public Processer(GetWeb g)
{
this.gw = g;
}
public void run()
{
//Thread.sleep(5000);
while (!arrUrls.isEmpty())
{
String tmp = getAUrl();
getWebByUrl(tmp,charset,allUrls.get(tmp)+"");
}
}
}
}
浙公网安备 33021202002483