爬虫(动态)代理·使用案例 下载爬虫(动态)代理IP使用DEMO项目
using System;
namespace TestDynamicIp
{
class Program
{
static string order = "这里需要改为您自己的IP提取码";
static bool run = true;
static string ipApi = "http://api.ip.data5u.com/dynamic/get.html?order=" + order;
static string targetUrl = "http://pv.sohu.com/cityjson?ie=utf-8";
public static void Main(string[] args)
{
// 抓取HTTPS网址时,需要接受证书,否则会报错:基础链接已经关闭,未能为SSL/TLS 安全通道建立信任关系
System.Net.ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback((object sender, System.Security.Cryptography.X509Certificates.X509Certificate certificate, System.Security.Cryptography.X509Certificates.X509Chain chain, System.Net.Security.SslPolicyErrors errors) =>
{
return true; //总是接受证书
});
System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Ssl3 | System.Net.SecurityProtocolType.Tls;
Console.WriteLine("\n********************\nC#测试爬虫(动态)代理IP\n********************\n");
Console.WriteLine("请求网址为:" + targetUrl);
StartCrawl();
Console.WriteLine("\n********************\n测试结束,按任意键退出程序\n********************\n");
Console.ReadKey(true);
}
public static void StartCrawl()
{
for (int i = 0; i < 10; i++)
{
Console.WriteLine("\n>>>>>>>>>>>>>>>>>>>>>>>第" + i + "次请求测试");
// 获取动态IP
Uri uri = new Uri(ipApi);
System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ipApi);
// 设置超时时间为20秒
request.Timeout = 20000;
System.Net.HttpWebResponse response = (System.Net.HttpWebResponse)request.GetResponse();
System.IO.Stream stream = response.GetResponseStream();
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
String iptxt = reader.ReadToEnd();
reader.Dispose();
reader.Close();
string[] ipports = iptxt.Split(new String[] { "\n" }, StringSplitOptions.None);
System.Threading.Thread.Sleep(1000);
try
{
// 定义client
string ipport = ipports[(int)(ipports.Length * new Random().Next(0, 1))];
System.Net.WebClient client = new System.Net.WebClient();
client.Encoding = System.Text.Encoding.GetEncoding("GB2312");
// 设置代理
System.Net.WebProxy proxy = new System.Net.WebProxy();
proxy.Address = new Uri("http://" + ipports[0] + "/");
client.Proxy = proxy;
// 获取网页内容
byte[] byResponse = client.DownloadData(targetUrl);
String txt = System.Text.Encoding.GetEncoding("GB2312").GetString(byResponse);
Console.WriteLine("使用代理" + ipports[0] + "得到如下内容:\n" + txt);
client.Dispose();
}
catch (Exception e) {
Console.WriteLine("出错了:" + e.Message);
}
}
}
}
}
普通代理·使用HttpWebRequest发送请求
HttpWebRequest httpRequest = (HttpWebRequest)HttpWebRequest.Create("http://www.baidu.com");
httpRequest.Method = "GET";
httpRequest.Credentials = CredentialCache.DefaultCredentials;
// 设置代理属性WebProxy
WebProxy proxy = new WebProxy();
proxy.Address = new Uri("http://58.22.62.163:888/");
// 在发起HTTP请求前将proxy赋值给HttpWebRequest的Proxy 属性
httpRequest.Proxy = proxy;
HttpWebResponse res = (HttpWebResponse)httpRequest.GetResponse();
StreamReader reader = new StreamReader(res.GetResponseStream(), System.Text.Encoding.UTF8);
string content = reader.ReadToEnd();
reader.Close();
普通代理·设置和取消全局代理IP
// 设置代理IP,参数ip的格式为是 ip:port
private void setProxy(string ip) {
Microsoft.Win32.RegistryKey rk = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(@"Software/Microsoft/Windows/CurrentVersion/Internet Settings", true);
//设置代理可用
rk.SetValue("ProxyEnable", 1);
//设置代理IP和端口
rk.SetValue("ProxyServer", ip);
rk.Close();
}
// 取消本地的代理IP
private void disProxy() {
Microsoft.Win32.RegistryKey rk = Microsoft.Win32.Registry.CurrentUser.OpenSubKey(@"Software/Microsoft/Windows/CurrentVersion/Internet Settings", true);
//设置代理可用
rk.SetValue("ProxyEnable", 0);
//设置代理IP和端口
rk.SetValue("ProxyServer", "");
rk.Close();
}