diff --git a/xgoogle/browser.py b/xgoogle/browser.py index c7d2618..5eaae47 100755 --- a/xgoogle/browser.py +++ b/xgoogle/browser.py @@ -20,6 +20,7 @@ # awk -F\" '{B[$6]++} END { for (b in B) { print B[b] ": " b } }' | # sort -rn | # head -20 + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-US; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)', @@ -32,7 +33,6 @@ 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.5) Gecko/2008121621 Ubuntu/8.04 (hardy) Firefox/3.0.5', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)', - 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' ) diff --git a/xgoogle/search.py b/xgoogle/search.py index 98b681e..86d22fd 100755 --- a/xgoogle/search.py +++ b/xgoogle/search.py @@ -250,7 +250,10 @@ def _extract_title_url(self, result): title = ''.join(title_a.findAll(text=True)) title = self._html_unescape(title) url = title_a['href'] - match = re.match(r'/url\?q=(http[^&]+)&', url) + match = re.match(r'/url\?q=((http|ftp|https)[^&]+)&', url) + if match: + url = urllib.unquote(match.group(1)) + match = re.match(r'/interstitial\?url=((http|ftp|https)[^&]+)&', url) if match: url = urllib.unquote(match.group(1)) return title, url @@ -260,6 +263,10 @@ def _extract_description(self, result): if not desc_div: self._maybe_raise(ParseError, "Description tag in Google search result was not found", result) return None + desc_span = desc_div.find('span', {'class': 'st'}) + if not desc_span: + self._maybe_raise(ParseError, "Description tag in Google search result was not found", result) + return None desc_strs = [] def looper(tag): @@ -275,8 +282,8 @@ def looper(tag): except AttributeError: desc_strs.append(t) - looper(desc_div) - looper(desc_div.find('wbr')) # BeautifulSoup does not self-close + looper(desc_span) + looper(desc_span.find('wbr')) # BeautifulSoup does not self-close desc = ''.join(s for s in desc_strs if s) return self._html_unescape(desc)