基本信息
源码名称：C# 抓取网页数据示例代码(爬虫)
源码大小：0.78M
文件格式：.zip
开发语言：C#
更新时间：2017-06-13
友情提示：（无需注册或充值，赞助后即可获取资源下载链接）
嘿，亲！知识可是无价之宝呢，但咱这精心整理的资料也耗费了不少心血呀。小小地破费一下，绝对物超所值哦！如有下载和支付问题，请联系我们QQ(微信同号)：78630559
本次赞助数额为： 2 元×
微信扫码支付：2 元
×
请留下您的邮箱，我们将在2小时内将文件发到您的邮箱
源码介绍
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Text.RegularExpressions;
using System.Collections;
using System.Data;
using System.Net;
using System.Runtime.InteropServices;
using Wesley.Crawler.SimpleCrawler.Models;
using System.Diagnostics;
namespace Wesley.Crawler.SimpleCrawler
{
class Program
{
static void Main1(string[] args)
{
//测试代理IP是否生效：http://1212.ip138.com/ic.asp
//测试当前爬虫的User-Agent：http://www.whatismyuseragent.net
//1.抓取城市
//CityCrawler();
//2.抓取酒店
Stopwatch sw = new Stopwatch();
sw.Start();
HotelCrawler();
sw.Stop();
TimeSpan ts2 = sw.Elapsed;
string a = string.Format("Stopwatch总共花费{0}ms.", ts2.TotalMilliseconds);
Console.WriteLine("携程北京耗时：" ts2.TotalMilliseconds * 1000/60 "秒");
//3.并发抓取示例
//ConcurrentCrawler();
Console.ReadKey();
}
/// <summary>
/// 抓取城市列表
/// </summary>
public static void CityCrawler() {
var cityUrl = "http://hotels.ctrip.com/citylist";//定义爬虫入口URL
var cityList = new List<City>();//定义泛型列表存放城市名称及对应的酒店URL
var cityCrawler = new SimpleCrawler();//调用刚才写的爬虫程序
cityCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
cityCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
cityCrawler.OnCompleted = (s, e) =>
{
//使用正则表达式清洗网页源代码中的数据
var links = Regex.Matches(e.PageSource, @"<a[^>] href=""*(?<href>/hotel/[^>\s] )""\s*[^>]*>(?<text>(?!.*img).*?)</a>", RegexOptions.IgnoreCase);
foreach (Match match in links)
{
var city = new City
{
CityName = match.Groups["text"].Value,
Uri = new Uri("http://hotels.ctrip.com" match.Groups["href"].Value
)
};
if (!cityList.Contains(city)) cityList.Add(city);//将数据加入到泛型列表
Console.WriteLine(city.CityName "|" city.Uri);//将城市名称及URL显示到控制台
}
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！合计 " links.Count " 个城市。");
Console.WriteLine("耗时：" e.Milliseconds "毫秒");
Console.WriteLine("线程：" e.ThreadId);
Console.WriteLine("地址：" e.Uri.ToString());
};
cityCrawler.Start(new Uri(cityUrl)).Wait();//没被封锁就别使用代理：60.221.50.118:8090
}
/// <summary>
/// 抓取酒店列表
/// </summary>
public static void HotelCrawler() {
//var hotelUrl = "http://hotels.ctrip.com/hotel/beijing1#ctm_ref=hod_hp_sb_lst";
var hotelUrl = "http://hotels.ctrip.com/hotel/beijing1#ctm_ref=hod_hp_sb_lst";
var hotelList = new List<Hotel>();
var hotelCrawler = new SimpleCrawler();
hotelCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
hotelCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
hotelCrawler.OnCompleted = (s, e) =>
{
#region 获取最大页数
int max_num = 1;
try
{
var max_num_str = HtmlTag.GetBetween(e.PageSource, "data-pagecount=", "name=\"\"");
max_num = int.Parse(max_num_str);
}
catch (Exception)
{
max_num = 1;
}
#endregion
var hotelLists = new List<string>();
for (int i = 1; i < max_num; i )
{
hotelLists.Add(string.Format("http://hotels.ctrip.com/hotel/beijing1/p{0}", i));
//hotelLists.Add(string.Format("http://hotels.ctrip.com/hotel/beijing1/p{0}", i));
}
var hotelList1 = hotelLists;
var hotelCrawler1 = new SimpleCrawler();
hotelCrawler1.OnStart = (a, b) =>
{
Console.WriteLine("爬虫开始抓取地址：" b.Uri.ToString());
};
hotelCrawler1.OnError = (a, b) =>
{
Console.WriteLine("爬虫抓取出现错误：" b.Uri.ToString() "，异常消息：" b.Exception.Message);
};
hotelCrawler1.OnCompleted = (a, b) =>
{
var links = Regex.Matches(b.PageSource, @"""><a[^>] href=""*(?<href>/hotel/[^>\s] )""\s*data-dopost[^>]*><span[^>] >.*?</span>(?<text>.*?)</a>", RegexOptions.IgnoreCase);
foreach (Match match in links)
{
var hotel = new Hotel
{
HotelName = match.Groups["text"].Value,
Uri = new Uri("http://hotels.ctrip.com" match.Groups["href"].Value
)
};
if (!hotelList.Contains(hotel)) hotelList.Add(hotel);//将数据加入到泛型列表
Console.WriteLine(hotel.HotelName "|" hotel.Uri);//将酒店名称及详细页URL显示到控制台
}
Console.WriteLine();
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！");
Console.WriteLine("耗时：" b.Milliseconds "毫秒");
Console.WriteLine("线程：" b.ThreadId);
Console.WriteLine("地址：" b.Uri.ToString());
};
Parallel.For(0, hotelList1.Count, (i) =>
{
var hotel = hotelList1[i];
hotelCrawler1.Start(new Uri(hotel)).Wait();
});
};
hotelCrawler.Start(new Uri(hotelUrl)).Wait();//没被封锁就别使用代理：60.221.50.118:8090
ConcurrentCrawler(hotelList);
}
/// <summary>
/// 并发抓取示例
/// </summary>
public static void ConcurrentCrawler(List<Hotel> hotelLists)
{
var hotelList = hotelLists;
var hotelCrawler = new SimpleCrawler();
hotelCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
hotelCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
hotelCrawler.OnCompleted = (s, e) =>
{
Hotel h=new Hotel();
h.HotelName= HtmlTag.GetBetween(e.PageSource, "<h1>", "</h1>");
h.Price =0;
h.Uri = e.Uri;
DbHelperOra.ExecuteSql(string.Format("insert into t_ctrip(HOTEL_NAME,PRICE,URL) values('{0}','{1}','{2}')",
h.HotelName, h.Price, h.Uri));
Console.WriteLine();
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！");
Console.WriteLine("耗时：" e.Milliseconds "毫秒");
Console.WriteLine("线程：" e.ThreadId);
Console.WriteLine("地址：" e.Uri.ToString());
};
Parallel.For(0, hotelList.Count, (i) =>
{
var hotel = hotelList[i];
hotelCrawler.Start(hotel.Uri).Wait();
});
}
/// <summary>
/// 并发抓取示例
/// </summary>
public static void ConcurrentCrawler() {
var hotelList = new List<Hotel>() {
new Hotel { HotelName="遵义浙商酒店", Uri=new Uri("http://hotels.ctrip.com/hotel/4983680.html?isFull=F") },
new Hotel { HotelName="遵义森林大酒店", Uri=new Uri("http://hotels.ctrip.com/hotel/1665124.html?isFull=F") },
};
var hotelCrawler = new SimpleCrawler();
hotelCrawler.OnStart = (s, e) =>
{
Console.WriteLine("爬虫开始抓取地址：" e.Uri.ToString());
};
hotelCrawler.OnError = (s, e) =>
{
Console.WriteLine("爬虫抓取出现错误：" e.Uri.ToString() "，异常消息：" e.Exception.Message);
};
hotelCrawler.OnCompleted = (s, e) =>
{
Console.WriteLine();
Console.WriteLine("===============================================");
Console.WriteLine("爬虫抓取任务完成！");
Console.WriteLine("耗时：" e.Milliseconds "毫秒");
Console.WriteLine("线程：" e.ThreadId);
Console.WriteLine("地址：" e.Uri.ToString());
};
Parallel.For(0, 2, (i) =>
{
var hotel = hotelList[i];
hotelCrawler.Start(hotel.Uri).Wait();
});
}
}
}