#! /usr/bin/env python | |
# Original code by Guido van Rossum; extensive changes by Sam Bayer, | |
# including code to check URL fragments. | |
"""Web tree checker. | |
This utility is handy to check a subweb of the world-wide web for | |
errors. A subweb is specified by giving one or more ``root URLs''; a | |
page belongs to the subweb if one of the root URLs is an initial | |
prefix of it. | |
File URL extension: | |
In order to easy the checking of subwebs via the local file system, | |
the interpretation of ``file:'' URLs is extended to mimic the behavior | |
of your average HTTP daemon: if a directory pathname is given, the | |
file index.html in that directory is returned if it exists, otherwise | |
a directory listing is returned. Now, you can point webchecker to the | |
document tree in the local file system of your HTTP daemon, and have | |
most of it checked. In fact the default works this way if your local | |
web tree is located at /usr/local/etc/httpd/htdpcs (the default for | |
the NCSA HTTP daemon and probably others). | |
Report printed: | |
When done, it reports pages with bad links within the subweb. When | |
interrupted, it reports for the pages that it has checked already. | |
In verbose mode, additional messages are printed during the | |
information gathering phase. By default, it prints a summary of its | |
work status every 50 URLs (adjustable with the -r option), and it | |
reports errors as they are encountered. Use the -q option to disable | |
this output. | |
Checkpoint feature: | |
Whether interrupted or not, it dumps its state (a Python pickle) to a | |
checkpoint file and the -R option allows it to restart from the | |
checkpoint (assuming that the pages on the subweb that were already | |
processed haven't changed). Even when it has run till completion, -R | |
can still be useful -- it will print the reports again, and -Rq prints | |
the errors only. In this case, the checkpoint file is not written | |
again. The checkpoint file can be set with the -d option. | |
The checkpoint file is written as a Python pickle. Remember that | |
Python's pickle module is currently quite slow. Give it the time it | |
needs to load and save the checkpoint file. When interrupted while | |
writing the checkpoint file, the old checkpoint file is not | |
overwritten, but all work done in the current run is lost. | |
Miscellaneous: | |
- You may find the (Tk-based) GUI version easier to use. See wcgui.py. | |
- Webchecker honors the "robots.txt" convention. Thanks to Skip | |
Montanaro for his robotparser.py module (included in this directory)! | |
The agent name is hardwired to "webchecker". URLs that are disallowed | |
by the robots.txt file are reported as external URLs. | |
- Because the SGML parser is a bit slow, very large SGML files are | |
skipped. The size limit can be set with the -m option. | |
- When the server or protocol does not tell us a file's type, we guess | |
it based on the URL's suffix. The mimetypes.py module (also in this | |
directory) has a built-in table mapping most currently known suffixes, | |
and in addition attempts to read the mime.types configuration files in | |
the default locations of Netscape and the NCSA HTTP daemon. | |
- We follow links indicated by <A>, <FRAME> and <IMG> tags. We also | |
honor the <BASE> tag. | |
- We now check internal NAME anchor links, as well as toplevel links. | |
- Checking external links is now done by default; use -x to *disable* | |
this feature. External links are now checked during normal | |
processing. (XXX The status of a checked link could be categorized | |
better. Later...) | |
- If external links are not checked, you can use the -t flag to | |
provide specific overrides to -x. | |
Usage: webchecker.py [option] ... [rooturl] ... | |
Options: | |
-R -- restart from checkpoint file | |
-d file -- checkpoint filename (default %(DUMPFILE)s) | |
-m bytes -- skip HTML pages larger than this size (default %(MAXPAGE)d) | |
-n -- reports only, no checking (use with -R) | |
-q -- quiet operation (also suppresses external links report) | |
-r number -- number of links processed per round (default %(ROUNDSIZE)d) | |
-t root -- specify root dir which should be treated as internal (can repeat) | |
-v -- verbose operation; repeating -v will increase verbosity | |
-x -- don't check external links (these are often slow to check) | |
-a -- don't check name anchors | |
Arguments: | |
rooturl -- URL to start checking | |
(default %(DEFROOT)s) | |
""" | |
__version__ = "$Revision$" | |
import sys | |
import os | |
from types import * | |
import StringIO | |
import getopt | |
import pickle | |
import urllib | |
import urlparse | |
import sgmllib | |
import cgi | |
import mimetypes | |
import robotparser | |
# Extract real version number if necessary | |
if __version__[0] == '$': | |
_v = __version__.split() | |
if len(_v) == 3: | |
__version__ = _v[1] | |
# Tunable parameters | |
DEFROOT = "file:/usr/local/etc/httpd/htdocs/" # Default root URL | |
CHECKEXT = 1 # Check external references (1 deep) | |
VERBOSE = 1 # Verbosity level (0-3) | |
MAXPAGE = 150000 # Ignore files bigger than this | |
ROUNDSIZE = 50 # Number of links processed per round | |
DUMPFILE = "@webchecker.pickle" # Pickled checkpoint | |
AGENTNAME = "webchecker" # Agent name for robots.txt parser | |
NONAMES = 0 # Force name anchor checking | |
# Global variables | |
def main(): | |
checkext = CHECKEXT | |
verbose = VERBOSE | |
maxpage = MAXPAGE | |
roundsize = ROUNDSIZE | |
dumpfile = DUMPFILE | |
restart = 0 | |
norun = 0 | |
try: | |
opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:t:vxa') | |
except getopt.error, msg: | |
sys.stdout = sys.stderr | |
print msg | |
print __doc__%globals() | |
sys.exit(2) | |
# The extra_roots variable collects extra roots. | |
extra_roots = [] | |
nonames = NONAMES | |
for o, a in opts: | |
if o == '-R': | |
restart = 1 | |
if o == '-d': | |
dumpfile = a | |
if o == '-m': | |
maxpage = int(a) | |
if o == '-n': | |
norun = 1 | |
if o == '-q': | |
verbose = 0 | |
if o == '-r': | |
roundsize = int(a) | |
if o == '-t': | |
extra_roots.append(a) | |
if o == '-a': | |
nonames = not nonames | |
if o == '-v': | |
verbose = verbose + 1 | |
if o == '-x': | |
checkext = not checkext | |
if verbose > 0: | |
print AGENTNAME, "version", __version__ | |
if restart: | |
c = load_pickle(dumpfile=dumpfile, verbose=verbose) | |
else: | |
c = Checker() | |
c.setflags(checkext=checkext, verbose=verbose, | |
maxpage=maxpage, roundsize=roundsize, | |
nonames=nonames | |
) | |
if not restart and not args: | |
args.append(DEFROOT) | |
for arg in args: | |
c.addroot(arg) | |
# The -t flag is only needed if external links are not to be | |
# checked. So -t values are ignored unless -x was specified. | |
if not checkext: | |
for root in extra_roots: | |
# Make sure it's terminated by a slash, | |
# so that addroot doesn't discard the last | |
# directory component. | |
if root[-1] != "/": | |
root = root + "/" | |
c.addroot(root, add_to_do = 0) | |
try: | |
if not norun: | |
try: | |
c.run() | |
except KeyboardInterrupt: | |
if verbose > 0: | |
print "[run interrupted]" | |
try: | |
c.report() | |
except KeyboardInterrupt: | |
if verbose > 0: | |
print "[report interrupted]" | |
finally: | |
if c.save_pickle(dumpfile): | |
if dumpfile == DUMPFILE: | |
print "Use ``%s -R'' to restart." % sys.argv[0] | |
else: | |
print "Use ``%s -R -d %s'' to restart." % (sys.argv[0], | |
dumpfile) | |
def load_pickle(dumpfile=DUMPFILE, verbose=VERBOSE): | |
if verbose > 0: | |
print "Loading checkpoint from %s ..." % dumpfile | |
f = open(dumpfile, "rb") | |
c = pickle.load(f) | |
f.close() | |
if verbose > 0: | |
print "Done." | |
print "Root:", "\n ".join(c.roots) | |
return c | |
class Checker: | |
checkext = CHECKEXT | |
verbose = VERBOSE | |
maxpage = MAXPAGE | |
roundsize = ROUNDSIZE | |
nonames = NONAMES | |
validflags = tuple(dir()) | |
def __init__(self): | |
self.reset() | |
def setflags(self, **kw): | |
for key in kw.keys(): | |
if key not in self.validflags: | |
raise NameError, "invalid keyword argument: %s" % str(key) | |
for key, value in kw.items(): | |
setattr(self, key, value) | |
def reset(self): | |
self.roots = [] | |
self.todo = {} | |
self.done = {} | |
self.bad = {} | |
# Add a name table, so that the name URLs can be checked. Also | |
# serves as an implicit cache for which URLs are done. | |
self.name_table = {} | |
self.round = 0 | |
# The following are not pickled: | |
self.robots = {} | |
self.errors = {} | |
self.urlopener = MyURLopener() | |
self.changed = 0 | |
def note(self, level, format, *args): | |
if self.verbose > level: | |
if args: | |
format = format%args | |
self.message(format) | |
def message(self, format, *args): | |
if args: | |
format = format%args | |
print format | |
def __getstate__(self): | |
return (self.roots, self.todo, self.done, self.bad, self.round) | |
def __setstate__(self, state): | |
self.reset() | |
(self.roots, self.todo, self.done, self.bad, self.round) = state | |
for root in self.roots: | |
self.addrobot(root) | |
for url in self.bad.keys(): | |
self.markerror(url) | |
def addroot(self, root, add_to_do = 1): | |
if root not in self.roots: | |
troot = root | |
scheme, netloc, path, params, query, fragment = \ | |
urlparse.urlparse(root) | |
i = path.rfind("/") + 1 | |
if 0 < i < len(path): | |
path = path[:i] | |
troot = urlparse.urlunparse((scheme, netloc, path, | |
params, query, fragment)) | |
self.roots.append(troot) | |
self.addrobot(root) | |
if add_to_do: | |
self.newlink((root, ""), ("<root>", root)) | |
def addrobot(self, root): | |
root = urlparse.urljoin(root, "/") | |
if self.robots.has_key(root): return | |
url = urlparse.urljoin(root, "/robots.txt") | |
self.robots[root] = rp = robotparser.RobotFileParser() | |
self.note(2, "Parsing %s", url) | |
rp.debug = self.verbose > 3 | |
rp.set_url(url) | |
try: | |
rp.read() | |
except (OSError, IOError), msg: | |
self.note(1, "I/O error parsing %s: %s", url, msg) | |
def run(self): | |
while self.todo: | |
self.round = self.round + 1 | |
self.note(0, "\nRound %d (%s)\n", self.round, self.status()) | |
urls = self.todo.keys() | |
urls.sort() | |
del urls[self.roundsize:] | |
for url in urls: | |
self.dopage(url) | |
def status(self): | |
return "%d total, %d to do, %d done, %d bad" % ( | |
len(self.todo)+len(self.done), | |
len(self.todo), len(self.done), | |
len(self.bad)) | |
def report(self): | |
self.message("") | |
if not self.todo: s = "Final" | |
else: s = "Interim" | |
self.message("%s Report (%s)", s, self.status()) | |
self.report_errors() | |
def report_errors(self): | |
if not self.bad: | |
self.message("\nNo errors") | |
return | |
self.message("\nError Report:") | |
sources = self.errors.keys() | |
sources.sort() | |
for source in sources: | |
triples = self.errors[source] | |
self.message("") | |
if len(triples) > 1: | |
self.message("%d Errors in %s", len(triples), source) | |
else: | |
self.message("Error in %s", source) | |
# Call self.format_url() instead of referring | |
# to the URL directly, since the URLs in these | |
# triples is now a (URL, fragment) pair. The value | |
# of the "source" variable comes from the list of | |
# origins, and is a URL, not a pair. | |
for url, rawlink, msg in triples: | |
if rawlink != self.format_url(url): s = " (%s)" % rawlink | |
else: s = "" | |
self.message(" HREF %s%s\n msg %s", | |
self.format_url(url), s, msg) | |
def dopage(self, url_pair): | |
# All printing of URLs uses format_url(); argument changed to | |
# url_pair for clarity. | |
if self.verbose > 1: | |
if self.verbose > 2: | |
self.show("Check ", self.format_url(url_pair), | |
" from", self.todo[url_pair]) | |
else: | |
self.message("Check %s", self.format_url(url_pair)) | |
url, local_fragment = url_pair | |
if local_fragment and self.nonames: | |
self.markdone(url_pair) | |
return | |
try: | |
page = self.getpage(url_pair) | |
except sgmllib.SGMLParseError, msg: | |
msg = self.sanitize(msg) | |
self.note(0, "Error parsing %s: %s", | |
self.format_url(url_pair), msg) | |
# Dont actually mark the URL as bad - it exists, just | |
# we can't parse it! | |
page = None | |
if page: | |
# Store the page which corresponds to this URL. | |
self.name_table[url] = page | |
# If there is a fragment in this url_pair, and it's not | |
# in the list of names for the page, call setbad(), since | |
# it's a missing anchor. | |
if local_fragment and local_fragment not in page.getnames(): | |
self.setbad(url_pair, ("Missing name anchor `%s'" % local_fragment)) | |
for info in page.getlinkinfos(): | |
# getlinkinfos() now returns the fragment as well, | |
# and we store that fragment here in the "todo" dictionary. | |
link, rawlink, fragment = info | |
# However, we don't want the fragment as the origin, since | |
# the origin is logically a page. | |
origin = url, rawlink | |
self.newlink((link, fragment), origin) | |
else: | |
# If no page has been created yet, we want to | |
# record that fact. | |
self.name_table[url_pair[0]] = None | |
self.markdone(url_pair) | |
def newlink(self, url, origin): | |
if self.done.has_key(url): | |
self.newdonelink(url, origin) | |
else: | |
self.newtodolink(url, origin) | |
def newdonelink(self, url, origin): | |
if origin not in self.done[url]: | |
self.done[url].append(origin) | |
# Call self.format_url(), since the URL here | |
# is now a (URL, fragment) pair. | |
self.note(3, " Done link %s", self.format_url(url)) | |
# Make sure that if it's bad, that the origin gets added. | |
if self.bad.has_key(url): | |
source, rawlink = origin | |
triple = url, rawlink, self.bad[url] | |
self.seterror(source, triple) | |
def newtodolink(self, url, origin): | |
# Call self.format_url(), since the URL here | |
# is now a (URL, fragment) pair. | |
if self.todo.has_key(url): | |
if origin not in self.todo[url]: | |
self.todo[url].append(origin) | |
self.note(3, " Seen todo link %s", self.format_url(url)) | |
else: | |
self.todo[url] = [origin] | |
self.note(3, " New todo link %s", self.format_url(url)) | |
def format_url(self, url): | |
link, fragment = url | |
if fragment: return link + "#" + fragment | |
else: return link | |
def markdone(self, url): | |
self.done[url] = self.todo[url] | |
del self.todo[url] | |
self.changed = 1 | |
def inroots(self, url): | |
for root in self.roots: | |
if url[:len(root)] == root: | |
return self.isallowed(root, url) | |
return 0 | |
def isallowed(self, root, url): | |
root = urlparse.urljoin(root, "/") | |
return self.robots[root].can_fetch(AGENTNAME, url) | |
def getpage(self, url_pair): | |
# Incoming argument name is a (URL, fragment) pair. | |
# The page may have been cached in the name_table variable. | |
url, fragment = url_pair | |
if self.name_table.has_key(url): | |
return self.name_table[url] | |
scheme, path = urllib.splittype(url) | |
if scheme in ('mailto', 'news', 'javascript', 'telnet'): | |
self.note(1, " Not checking %s URL" % scheme) | |
return None | |
isint = self.inroots(url) | |
# Ensure that openpage gets the URL pair to | |
# print out its error message and record the error pair | |
# correctly. | |
if not isint: | |
if not self.checkext: | |
self.note(1, " Not checking ext link") | |
return None | |
f = self.openpage(url_pair) | |
if f: | |
self.safeclose(f) | |
return None | |
text, nurl = self.readhtml(url_pair) | |
if nurl != url: | |
self.note(1, " Redirected to %s", nurl) | |
url = nurl | |
if text: | |
return Page(text, url, maxpage=self.maxpage, checker=self) | |
# These next three functions take (URL, fragment) pairs as | |
# arguments, so that openpage() receives the appropriate tuple to | |
# record error messages. | |
def readhtml(self, url_pair): | |
url, fragment = url_pair | |
text = None | |
f, url = self.openhtml(url_pair) | |
if f: | |
text = f.read() | |
f.close() | |
return text, url | |
def openhtml(self, url_pair): | |
url, fragment = url_pair | |
f = self.openpage(url_pair) | |
if f: | |
url = f.geturl() | |
info = f.info() | |
if not self.checkforhtml(info, url): | |
self.safeclose(f) | |
f = None | |
return f, url | |
def openpage(self, url_pair): | |
url, fragment = url_pair | |
try: | |
return self.urlopener.open(url) | |
except (OSError, IOError), msg: | |
msg = self.sanitize(msg) | |
self.note(0, "Error %s", msg) | |
if self.verbose > 0: | |
self.show(" HREF ", url, " from", self.todo[url_pair]) | |
self.setbad(url_pair, msg) | |
return None | |
def checkforhtml(self, info, url): | |
if info.has_key('content-type'): | |
ctype = cgi.parse_header(info['content-type'])[0].lower() | |
if ';' in ctype: | |
# handle content-type: text/html; charset=iso8859-1 : | |
ctype = ctype.split(';', 1)[0].strip() | |
else: | |
if url[-1:] == "/": | |
return 1 | |
ctype, encoding = mimetypes.guess_type(url) | |
if ctype == 'text/html': | |
return 1 | |
else: | |
self.note(1, " Not HTML, mime type %s", ctype) | |
return 0 | |
def setgood(self, url): | |
if self.bad.has_key(url): | |
del self.bad[url] | |
self.changed = 1 | |
self.note(0, "(Clear previously seen error)") | |
def setbad(self, url, msg): | |
if self.bad.has_key(url) and self.bad[url] == msg: | |
self.note(0, "(Seen this error before)") | |
return | |
self.bad[url] = msg | |
self.changed = 1 | |
self.markerror(url) | |
def markerror(self, url): | |
try: | |
origins = self.todo[url] | |
except KeyError: | |
origins = self.done[url] | |
for source, rawlink in origins: | |
triple = url, rawlink, self.bad[url] | |
self.seterror(source, triple) | |
def seterror(self, url, triple): | |
try: | |
# Because of the way the URLs are now processed, I need to | |
# check to make sure the URL hasn't been entered in the | |
# error list. The first element of the triple here is a | |
# (URL, fragment) pair, but the URL key is not, since it's | |
# from the list of origins. | |
if triple not in self.errors[url]: | |
self.errors[url].append(triple) | |
except KeyError: | |
self.errors[url] = [triple] | |
# The following used to be toplevel functions; they have been | |
# changed into methods so they can be overridden in subclasses. | |
def show(self, p1, link, p2, origins): | |
self.message("%s %s", p1, link) | |
i = 0 | |
for source, rawlink in origins: | |
i = i+1 | |
if i == 2: | |
p2 = ' '*len(p2) | |
if rawlink != link: s = " (%s)" % rawlink | |
else: s = "" | |
self.message("%s %s%s", p2, source, s) | |
def sanitize(self, msg): | |
if isinstance(IOError, ClassType) and isinstance(msg, IOError): | |
# Do the other branch recursively | |
msg.args = self.sanitize(msg.args) | |
elif isinstance(msg, TupleType): | |
if len(msg) >= 4 and msg[0] == 'http error' and \ | |
isinstance(msg[3], InstanceType): | |
# Remove the Message instance -- it may contain | |
# a file object which prevents pickling. | |
msg = msg[:3] + msg[4:] | |
return msg | |
def safeclose(self, f): | |
try: | |
url = f.geturl() | |
except AttributeError: | |
pass | |
else: | |
if url[:4] == 'ftp:' or url[:7] == 'file://': | |
# Apparently ftp connections don't like to be closed | |
# prematurely... | |
text = f.read() | |
f.close() | |
def save_pickle(self, dumpfile=DUMPFILE): | |
if not self.changed: | |
self.note(0, "\nNo need to save checkpoint") | |
elif not dumpfile: | |
self.note(0, "No dumpfile, won't save checkpoint") | |
else: | |
self.note(0, "\nSaving checkpoint to %s ...", dumpfile) | |
newfile = dumpfile + ".new" | |
f = open(newfile, "wb") | |
pickle.dump(self, f) | |
f.close() | |
try: | |
os.unlink(dumpfile) | |
except os.error: | |
pass | |
os.rename(newfile, dumpfile) | |
self.note(0, "Done.") | |
return 1 | |
class Page: | |
def __init__(self, text, url, verbose=VERBOSE, maxpage=MAXPAGE, checker=None): | |
self.text = text | |
self.url = url | |
self.verbose = verbose | |
self.maxpage = maxpage | |
self.checker = checker | |
# The parsing of the page is done in the __init__() routine in | |
# order to initialize the list of names the file | |
# contains. Stored the parser in an instance variable. Passed | |
# the URL to MyHTMLParser(). | |
size = len(self.text) | |
if size > self.maxpage: | |
self.note(0, "Skip huge file %s (%.0f Kbytes)", self.url, (size*0.001)) | |
self.parser = None | |
return | |
self.checker.note(2, " Parsing %s (%d bytes)", self.url, size) | |
self.parser = MyHTMLParser(url, verbose=self.verbose, | |
checker=self.checker) | |
self.parser.feed(self.text) | |
self.parser.close() | |
def note(self, level, msg, *args): | |
if self.checker: | |
apply(self.checker.note, (level, msg) + args) | |
else: | |
if self.verbose >= level: | |
if args: | |
msg = msg%args | |
print msg | |
# Method to retrieve names. | |
def getnames(self): | |
if self.parser: | |
return self.parser.names | |
else: | |
return [] | |
def getlinkinfos(self): | |
# File reading is done in __init__() routine. Store parser in | |
# local variable to indicate success of parsing. | |
# If no parser was stored, fail. | |
if not self.parser: return [] | |
rawlinks = self.parser.getlinks() | |
base = urlparse.urljoin(self.url, self.parser.getbase() or "") | |
infos = [] | |
for rawlink in rawlinks: | |
t = urlparse.urlparse(rawlink) | |
# DON'T DISCARD THE FRAGMENT! Instead, include | |
# it in the tuples which are returned. See Checker.dopage(). | |
fragment = t[-1] | |
t = t[:-1] + ('',) | |
rawlink = urlparse.urlunparse(t) | |
link = urlparse.urljoin(base, rawlink) | |
infos.append((link, rawlink, fragment)) | |
return infos | |
class MyStringIO(StringIO.StringIO): | |
def __init__(self, url, info): | |
self.__url = url | |
self.__info = info | |
StringIO.StringIO.__init__(self) | |
def info(self): | |
return self.__info | |
def geturl(self): | |
return self.__url | |
class MyURLopener(urllib.FancyURLopener): | |
http_error_default = urllib.URLopener.http_error_default | |
def __init__(*args): | |
self = args[0] | |
apply(urllib.FancyURLopener.__init__, args) | |
self.addheaders = [ | |
('User-agent', 'Python-webchecker/%s' % __version__), | |
] | |
def http_error_401(self, url, fp, errcode, errmsg, headers): | |
return None | |
def open_file(self, url): | |
path = urllib.url2pathname(urllib.unquote(url)) | |
if os.path.isdir(path): | |
if path[-1] != os.sep: | |
url = url + '/' | |
indexpath = os.path.join(path, "index.html") | |
if os.path.exists(indexpath): | |
return self.open_file(url + "index.html") | |
try: | |
names = os.listdir(path) | |
except os.error, msg: | |
exc_type, exc_value, exc_tb = sys.exc_info() | |
raise IOError, msg, exc_tb | |
names.sort() | |
s = MyStringIO("file:"+url, {'content-type': 'text/html'}) | |
s.write('<BASE HREF="file:%s">\n' % | |
urllib.quote(os.path.join(path, ""))) | |
for name in names: | |
q = urllib.quote(name) | |
s.write('<A HREF="%s">%s</A>\n' % (q, q)) | |
s.seek(0) | |
return s | |
return urllib.FancyURLopener.open_file(self, url) | |
class MyHTMLParser(sgmllib.SGMLParser): | |
def __init__(self, url, verbose=VERBOSE, checker=None): | |
self.myverbose = verbose # now unused | |
self.checker = checker | |
self.base = None | |
self.links = {} | |
self.names = [] | |
self.url = url | |
sgmllib.SGMLParser.__init__(self) | |
def check_name_id(self, attributes): | |
""" Check the name or id attributes on an element. | |
""" | |
# We must rescue the NAME or id (name is deprecated in XHTML) | |
# attributes from the anchor, in order to | |
# cache the internal anchors which are made | |
# available in the page. | |
for name, value in attributes: | |
if name == "name" or name == "id": | |
if value in self.names: | |
self.checker.message("WARNING: duplicate ID name %s in %s", | |
value, self.url) | |
else: self.names.append(value) | |
break | |
def unknown_starttag(self, tag, attributes): | |
""" In XHTML, you can have id attributes on any element. | |
""" | |
self.check_name_id(attributes) | |
def start_a(self, attributes): | |
self.link_attr(attributes, 'href') | |
self.check_name_id(attributes) | |
def end_a(self): pass | |
def do_area(self, attributes): | |
self.link_attr(attributes, 'href') | |
self.check_name_id(attributes) | |
def do_body(self, attributes): | |
self.link_attr(attributes, 'background', 'bgsound') | |
self.check_name_id(attributes) | |
def do_img(self, attributes): | |
self.link_attr(attributes, 'src', 'lowsrc') | |
self.check_name_id(attributes) | |
def do_frame(self, attributes): | |
self.link_attr(attributes, 'src', 'longdesc') | |
self.check_name_id(attributes) | |
def do_iframe(self, attributes): | |
self.link_attr(attributes, 'src', 'longdesc') | |
self.check_name_id(attributes) | |
def do_link(self, attributes): | |
for name, value in attributes: | |
if name == "rel": | |
parts = value.lower().split() | |
if ( parts == ["stylesheet"] | |
or parts == ["alternate", "stylesheet"]): | |
self.link_attr(attributes, "href") | |
break | |
self.check_name_id(attributes) | |
def do_object(self, attributes): | |
self.link_attr(attributes, 'data', 'usemap') | |
self.check_name_id(attributes) | |
def do_script(self, attributes): | |
self.link_attr(attributes, 'src') | |
self.check_name_id(attributes) | |
def do_table(self, attributes): | |
self.link_attr(attributes, 'background') | |
self.check_name_id(attributes) | |
def do_td(self, attributes): | |
self.link_attr(attributes, 'background') | |
self.check_name_id(attributes) | |
def do_th(self, attributes): | |
self.link_attr(attributes, 'background') | |
self.check_name_id(attributes) | |
def do_tr(self, attributes): | |
self.link_attr(attributes, 'background') | |
self.check_name_id(attributes) | |
def link_attr(self, attributes, *args): | |
for name, value in attributes: | |
if name in args: | |
if value: value = value.strip() | |
if value: self.links[value] = None | |
def do_base(self, attributes): | |
for name, value in attributes: | |
if name == 'href': | |
if value: value = value.strip() | |
if value: | |
if self.checker: | |
self.checker.note(1, " Base %s", value) | |
self.base = value | |
self.check_name_id(attributes) | |
def getlinks(self): | |
return self.links.keys() | |
def getbase(self): | |
return self.base | |
if __name__ == '__main__': | |
main() |