| #!/usr/bin/env python |
| |
| """ I honestly don't even know how the hell this works, just use it. """ |
| __author__ = "Scott Stamp <scott@hypermine.com>" |
| |
| from HTMLParser import HTMLParser |
| from urlparse import urljoin |
| from sys import setrecursionlimit |
| import re |
| import requests |
| |
| setrecursionlimit(10000) |
| root = 'http://localhost:8000' |
| |
| |
| class DataHolder: |
| |
| def __init__(self, value=None, attr_name='value'): |
| self._attr_name = attr_name |
| self.set(value) |
| |
| def __call__(self, value): |
| return self.set(value) |
| |
| def set(self, value): |
| setattr(self, self._attr_name, value) |
| return value |
| |
| def get(self): |
| return getattr(self, self._attr_name) |
| |
| |
| class Parser(HTMLParser): |
| global root |
| |
| ids = set() |
| crawled = set() |
| anchors = {} |
| pages = set() |
| save_match = DataHolder(attr_name='match') |
| |
| def __init__(self, origin): |
| self.origin = origin |
| HTMLParser.__init__(self) |
| |
| def handle_starttag(self, tag, attrs): |
| attrs = dict(attrs) |
| if 'href' in attrs: |
| href = attrs['href'] |
| |
| if re.match('^{0}|\/|\#[\S]{{1,}}'.format(root), href): |
| if self.save_match(re.search('.*\#(.*?)$', href)): |
| if self.origin not in self.anchors: |
| self.anchors[self.origin] = set() |
| self.anchors[self.origin].add( |
| self.save_match.match.groups(1)[0]) |
| |
| url = urljoin(root, href) |
| |
| if url not in self.crawled and not re.match('^\#', href): |
| self.crawled.add(url) |
| Parser(url).feed(requests.get(url).content) |
| |
| if 'id' in attrs: |
| self.ids.add(attrs['id']) |
| # explicit <a name=""></a> references |
| if 'name' in attrs: |
| self.ids.add(attrs['name']) |
| |
| |
| r = requests.get(root) |
| parser = Parser(root) |
| parser.feed(r.content) |
| for anchor in sorted(parser.anchors): |
| if not re.match('.*/\#.*', anchor): |
| for anchor_name in parser.anchors[anchor]: |
| if anchor_name not in parser.ids: |
| print 'Missing - ({0}): #{1}'.format( |
| anchor.replace(root, ''), anchor_name) |
