當麻許的碎念筆記

/// <summary>

/// 將可能為相對路徑轉成絕對路徑

/// </summary>

/// <param name="baseURL">網站</param>

/// <param name="checkURL">相對路徑/絕對路徑</param>

/// <returns></returns>

string  ConvertToAbsoluteURL(string baseURL,string checkURL)

{

    Uri result;

    var res = Uri.TryCreate(new Uri(baseURL, UriKind.Absolute), new Uri(checkURL, UriKind.RelativeOrAbsolute), out  result); 

    if(res)

    {

        return result.AbsoluteUri; 

    }else

    {

        return checkURL; 

    }

}

/// <summary>

/// 輸入網址原始碼，透過refex 將所有網址找出來

/// </summary>

/// <returns> 所有抓取到的網址 </returns>

public static string[] GetLinkUrlsFromPath(string htmlSource)

{

    List<string> res = new List<string>();

    var regex = new Regex(

        @"(?:href\s*=)(?:[\s""']*)(?!#|mailto|location.|javascript)(?<PARAM1>.*?)(?:[\s>""'])",

        RegexOptions.IgnoreCase);

    MatchCollection matches = regex.Matches(htmlSource);

    foreach (Match match in matches)

    {

        //將href=" 濾掉

        // res.Add(match.Groups[0].Value.Substring(6, match.Groups[0].Value.Length-7));

        res.Add(match.Groups["PARAM1"].Value);

    }

    return res.ToArray();

}

/// <summary>

/// 從網路上取得原始碼

/// </summary>

/// <param name="url"></param>

/// <returns></returns>

public static string GetSourceFromUrl(string url)

{

    WebClient client = new WebClient();

    //以防萬一 模擬自己為瀏覽器

    client.Headers.Add("User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.56 Safari/536.5");

    client.Headers.Add("Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");

    client.Headers.Add("Accept-Encoding: identity");

    client.Headers.Add("Accept-Language: zh-TW,en;q=0.8");

    client.Headers.Add("Accept-Charset: utf-8;q=0.7,*;q=0.3");

    client.Headers.Add("ContentType", "application/x-www-form-urlencoded");

    return client.DownloadString(url);

}

protected void btnTest_Click(object sender, EventArgs e)

{

  ltlResult.Text = "";

  foreach (var str in GetLinkUrlsFromPath(GetSourceFromUrl(txtLink.Text)))

  {

      ltlResult.Text += "<a href='" + ConvertToAbsoluteURL(txtLink.Text, str) + "' >" + ConvertToAbsoluteURL(txtLink.Text, str) + "</a><br />";    

     // ltlResult.Text += "<a href='" +str+ "' >" +str + "</a><br />";    

  }

}