Heritrix架构学习笔记（三）

3、Frontier链接制造工厂

在heritrix-1.12.1/docs/articles/developer_manual/frontier.html下可找到Heritrix的官方文档的一个Frontier例子：

/** * A simple Frontier implementation for tutorial purposes */ public class MyFrontier extends ModuleType implements Frontier, FetchStatusCodes { // A list of the discovered URIs that should be crawled. // 列表用来保存还未抓取的链接 List pendingURIs = new ArrayList(); // A list of prerequisites that needs to be met before any other URI is // allowed to be crawled, e.g. DNS-lookups //这个列表中保存了一系列的链接，它们的优先级要高于pendingURIs那个List中的任何一个链接， //表中的链接表示一些需要被满足的先决条件 List prerequisites = new ArrayList(); // A hash of already crawled URIs so that every URI is crawled only once. //一个HashMap，用于存储那些已经抓取过的链接 Map alreadyIncluded = new HashMap(); // Reference to the CrawlController. // CrawlController对象 CrawlController controller; // Flag to note if a URI is being processed. //用于标识是否一个链接正在被处理 boolean uriInProcess = false; // top-level stats //成功下载的数量 long successCount = 0; //失败的数量 long failedCount = 0; //抛弃掉链接的数量 long disregardedCount = 0; //总共下载的字节数 long totalProcessedBytes = 0; public MyFrontier(String name) { super(Frontier.ATTR_NAME, "A simple frontier."); } public void initialize(CrawlController controller) throws FatalConfigurationException, IOException { //注入 this.controller = controller; // Initialize the pending queue with the seeds //把种子文件中的链接加入到pengdingURIs中去 this.controller.getScope().refreshSeeds(); List seeds = this.controller.getScope().getSeedlist(); synchronized(seeds) { for (Iterator i = seeds.iterator(); i.hasNext();) { UURI u = (UURI) i.next(); CandidateURI caUri = new CandidateURI(u); caUri.setSeed(); schedule(caUri); } } } //该方法是给线程池中的线程调用的，用以取出下一个准备处理的链接 public synchronized CrawlURI next(int timeout) throws InterruptedException { if (!uriInProcess && !isEmpty()) { uriInProcess = true; CrawlURI curi; /* 先看prerequistes队列中是否有要处理的链接，如果有，就先处理，如果没有，再看pengdingURIs队列中是否有链接。每次在处理的时候，总是取出队列中的第一个链接 */ if (!prerequisites.isEmpty()) { curi = CrawlURI.from((CandidateURI) prerequisites.remove(0)); } else { curi = CrawlURI.from((CandidateURI) pendingURIs.remove(0)); } curi.setServer(controller.getServerCache().getServerFor(curi)); return curi; } else { wait(timeout); return null; } } public boolean isEmpty() { return pendingURIs.isEmpty() && prerequisites.isEmpty(); } //该方法用于将新链接加入到pengdingURIs队列中，等待处理 public synchronized void schedule(CandidateURI caURI) { // Schedule a uri for crawling if it is not already crawled /* 首先判断要加入的链接是否已经被抓取过，如果已经包含在alreadyIncluded这个HashMap中则说明处理过了，就可以放弃处理 */ if (!alreadyIncluded.containsKey(caURI.getURIString())) { if(caURI.needsImmediateScheduling()) { prerequisites.add(caURI); } else { pendingURIs.add(caURI); } //HashMap中使用url的字符串来作为key，而将实际的CadidateURI对象作为value alreadyIncluded.put(caURI.getURIString(), caURI); } } public void batchSchedule(CandidateURI caURI) { schedule(caURI); } public void batchFlush() { } //一次抓取结束后所执行的操作，该操作由线程池中的线程来进行调用 public synchronized void finished(CrawlURI cURI) { uriInProcess = false; //成功下载 if (cURI.isSuccess()) { successCount++; //统计下载总数 totalProcessedBytes += cURI.getContentSize(); //如果成功，则触发一个成功事件，比如将Extractor解析出来的新URL加入队列中 controller.fireCrawledURISuccessfulEvent(cURI); cURI.stripToMinimal(); } //需要推迟下载 else if (cURI.getFetchStatus() == S_DEFERRED) { cURI.processingCleanup(); alreadyIncluded.remove(cURI.getURIString()); schedule(cURI); } //其他状态 else if (cURI.getFetchStatus() == S_ROBOTS_PRECLUDED || cURI.getFetchStatus() == S_OUT_OF_SCOPE || cURI.getFetchStatus() == S_BLOCKED_BY_USER || cURI.getFetchStatus() == S_TOO_MANY_EMBED_HOPS || cURI.getFetchStatus() == S_TOO_MANY_LINK_HOPS || cURI.getFetchStatus() == S_DELETED_BY_USER) { //抛弃当前URI controller.fireCrawledURIDisregardEvent(cURI); disregardedCount++; cURI.stripToMinimal(); } else { controller.fireCrawledURIFailureEvent(cURI); failedCount++; cURI.stripToMinimal(); } cURI.processingCleanup(); } //返回所有已经处理过的链接数量 public long discoveredUriCount() { return alreadyIncluded.size(); } //返回所有等待处理的链接数量 public long queuedUriCount() { return pendingURIs.size() + prerequisites.size(); } //返回所有已经完成的链接数量 public long finishedUriCount() { return successCount + failedCount + disregardedCount; } //返回所有成功处理的链接数量 public long successfullyFetchedCount() { return successCount; } //返回所有失败的链接数量 public long failedFetchCount() { return failedCount; } //返回所有抛弃的链接数量 public long disregardedFetchCount() { return disregardedCount; } //返回总共下载的字节数 public long totalBytesWritten() { return totalProcessedBytes; } public String report() { return "This frontier does not return a report."; } public void importRecoverLog(String pathToLog) throws IOException { throw new UnsupportedOperationException(); } public FrontierMarker getInitialMarker(String regexpr, boolean inCacheOnly) { return null; } public ArrayList getURIsList(FrontierMarker marker, int numberOfMatches, boolean verbose) throws InvalidFrontierMarkerException { return null; } public long deleteURIs(String match) { return 0; } }

注意：上面仅仅是一个最基础的代码，从结构上揭示一个Frontier的作用