-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathScraper.cs
58 lines (51 loc) · 2.13 KB
/
Scraper.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Net;
using HtmlAgilityPack;
namespace DigikeyApi
{
public class Scraper
{
public static Product scrapePart(string partId)
{
try
{
WebClient wc = new WebClient();
string pageContents = wc.DownloadString("http://search.digikey.com/scripts/DkSearch/dksus.dll?Detail&name=" + System.Web.HttpUtility.UrlEncode(partId));
pageContents = pageContents.Replace("<CS=0><RF=141>", "");
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(pageContents);
HtmlNode partNumberNode = doc.DocumentNode.SelectSingleNode("//td[@id='reportpartnumber']");
HtmlNode quantityNode = doc.DocumentNode.SelectSingleNode("//td[@id='quantityavailable']");
HtmlNodeCollection pricingNodes = doc.DocumentNode.SelectNodes("//table[@id='pricing']/tr");
Product p = new Product();
p.partNumber = partNumberNode.InnerText;
try
{
p.quantityAvailiable = Convert.ToInt32(Regex.Replace(quantityNode.InnerText, "[^0-9]", ""));
}
catch
{
p.quantityAvailiable = 0;
p.warning += "Quantity might not be availiable\n";
}
p.pricing = new SortedDictionary<int, decimal>();
for (int i = 1; i < pricingNodes.Count(); i++)
{
int quantity = Convert.ToInt32(Regex.Replace(pricingNodes[i].ChildNodes[0].InnerText, "[^0-9]", ""));
decimal unitPrice = Convert.ToDecimal(pricingNodes[i].ChildNodes[1].InnerText);
p.pricing.Add(quantity, unitPrice);
}
return p;
}
catch (Exception ex)
{
Console.WriteLine("Exception on part {0}: {1}", partId, ex.Message);
return null;
}
}
}
}