(See: MachineCodeBlocks)
The code relies on webcache.py.
import re
import webcache
re_start = re.compile(r'(.*)<machinecode>\s*')
re_data_template = r'%s(.*?): (.*)'
re_end_template = r'%s</machinecode>\s*'
class DataBlockStore:
def __init__(self, directory = "."):
self.cache = webcache.WebCache(directory + "/pages.db",
directory + "/times.db",
24*60*60)
self.blocks = {}
def get_block(self, url):
if url in self.blocks:
return self.blocks[url]
else:
self.blocks[url] = DataBlock(url, self)
return self.blocks[url]
class DataBlock:
def __init__(self, url, store):
self.url = url
self.store = store
self.head = None
self.data = None
self.tail = None
self.indentation = None
def __str__(self):
if self.data == None:
self.read()
return str(self.data)
def __repr__(self):
return 'DataBlock("%s", %s)' % (self.url, repr(self.store))
def read(self, url = None):
url = url or self.url
page = self.store.cache.get_page(url)
lines = page.splitlines()
start, end = None, None
for (i, line) in enumerate(lines):
if start is None:
mo = re_start.match(line)
if mo is not None:
self.indentation = mo.group(1)
self.data = {}
start = i
re_data = re.compile(re_data_template %
self.indentation)
re_end = re.compile(re_end_template %
self.indentation)
else:
data_mo = re_data.match(line)
end_mo = re_end.match(line)
if data_mo is not None:
(k, v) = data_mo.groups()
if k not in self.data:
self.data[k] = self._retype(v)
else:
if isinstance(self.data[k], list):
self.data[k].append(self._retype(v))
else:
self.data[k] = [self.data[k],
self._retype(v)]
elif end_mo is not None:
end = i
break
if start is not None:
self.head = lines[:start]
if end is not None:
self.tail = lines[end:]
return self.data
def _retype(self, arg):
try:
return int(arg)
except ValueError:
pass
if arg.startswith("http://"):
return self.store.get_block(arg)
return arg
def __getitem__(self, k):
if self.data is None:
self.read()
return self.data[k]
def keys(self):
if self.data is None:
self.read()
return self.data.keys()
def values(self, *args):
if self.data is None:
self.read()
return self.data.values(*args)
def items(self, *args):
if self.data is None:
self.read()
return self.data.items(*args)
if __name__ == "__main__":
store = DataBlockStore()
mc = store.get_block("http://www.emacswiki.org/cw?action=browse;id=WikiNode;raw=1")
print mc.read()The code relies on webcache.py.
This is the one that works on the basis of: keys are bold, values are italic-link.
"""MachineCode interpreter.
Interprets and caches MachineCode from XHTML pages.
DataBlockStore -- Create DataBlock instances.
DataBlock -- A MachineCode block on a website.
"""
import re
import pprint
import HTMLParser
import webcache
class DataBlockStore:
"""Creates and caches data blocks.
When you want a DataBlock, call "get_block". It will return a cached
block, or a new block.
The DataBlockStore also maintains a shared webpage cache.
get_block -- Obtain a DataBlock
"""
def __init__(self, directory = "."):
self.cache = webcache.WebCache(directory + "/pages.db",
directory + "/times.db",
24*60*60) # Shared webpage cache
self.blocks = {} # Blocks cache
def get_block(self, url):
"""Return a DataBlock representing a website's MachineCode.
If the DataBlock has already been made, return it. If not,
construct it, cache it, and then return it.
"""
if url in self.blocks:
return self.blocks[url]
else:
self.blocks[url] = DataBlock(url, self)
return self.blocks[url]
class DataBlock:
"""Proxy for a MachineCode block on a website.
Represents the MachineCode block found on a website. You can't
write to it, yet, but you can read from it.
MachineCode isn't retrieved from the web (or cache) until you
request it, by calling the "read" method.
MachineCode blocks are networked. If the value of a key is a URL,
then you can leap straight to the MachineCode block found at that
target URL.
read -- Interpret the MachineCode from a given URL.
"""
def __init__(self, url, store):
self.url = url
self.store = store
self.data = None # Call "read" to interpret.
def __str__(self):
if self.data == None:
self.read()
return str(self.data)
def __repr__(self):
return 'DataBlock("%s", %s)' % (self.url, repr(self.store))
def read(self, url = None):
"""Interpret MachineCode out from the given URL.
The page is loaded from the web if it's not found in the
DataBlockStore's cache.
The interpreted data is stored in self.data, and available by
using Python Dictionary accessors on this object.
"""
self.url = url or self.url
page = self.store.cache.get_page(self.url)
self.data = {}
parser = MachineCodeHtmlParser(self)
parser.feed(page)
parser.close()
return self.data
def _retype(self, arg):
try:
return int(arg)
except ValueError:
pass
if arg.startswith("http://"):
return self.store.get_block(arg)
return arg
def __getitem__(self, k):
if self.data is None:
self.read()
return self.data[k]
def keys(self):
if self.data is None:
self.read()
return self.data.keys()
def values(self, *args):
if self.data is None:
self.read()
return self.data.values(*args)
def items(self, *args):
if self.data is None:
self.read()
return self.data.items(*args)
class MachineCodeHtmlParser(HTMLParser.HTMLParser):
"""Interpret XHTML tag events and store the results in a DataBlock.
A SAX-like handler for XHTML events.
The handler waits for the text "MACHINECODE" in bold.
Then it reads bold-tagged text as dictionary keys, and italic-tagged
text or anchored text as dictionary values.
A final "MACHINECODE" in bold seals the interpretation.
"""
def __init__(self, data_block):
HTMLParser.HTMLParser.__init__(self)
self.data_block = data_block
self.reading_machinecode = False
self.last_text = ""
self.last_key = None
def handle_starttag(self, tag, attrs):
self.last_text = ""
if self.reading_machinecode:
if tag == "a":
attrs = dict(attrs)
if "href" in attrs:
self._store_value(attrs["href"])
def handle_endtag(self, tag):
if not self.reading_machinecode:
if tag in ("strong", "b") and \
self.last_text.startswith("MACHINECODE"):
self.reading_machinecode = True
else:
if tag in ("em", "i"):
self._store_value(self.last_text)
elif tag in ("strong", "b"):
if self.last_text.startswith("MACHINECODE"):
self.reading_machinecode = False
else:
self.last_key = self.last_text
self.last_text = ""
def handle_data(self, data):
self.last_text += data
def _store_value(self, value):
if self.last_key in self.data_block.data:
old_val = self.data_block.data[self.last_key]
if isinstance(old_val, list):
old_val.append(value)
else:
self.data_block.data[self.last_key] = [old_val, value]
else:
self.data_block.data[self.last_key] = value
self.last_key = None
if __name__ == "__main__":
store = DataBlockStore()
mc = store.get_block("http://www.emacswiki.org/cw/CommunityWiki")
pprint.pprint(mc.read())Output:
{'default-licence-title': 'GFDL 1.2 and/or CC-SA',
'description': 'This is a jam-session. We do not know where we are going. This is open. It has no mission.',
'engine': 'http://www.emacswiki.org/cgi-bin/latin-1.pl?url=http://www.usemod.com/cgi-bin/mb.pl?OddMuse',
'engine-title': 'Oddmuse',
'front-page': 'http://www.emacswiki.org/cw/SiteMap',
'language': ['en', 'fr'],
'maintainer': 'http://www.emacswiki.org/cw/AlexSchroeder',
'participant': ['http://www.emacswiki.org/cw/MurrayAltheim',
'http://www.emacswiki.org/cw/DavidCary',
'http://www.emacswiki.org/cw/ChristopheDucamp',
'http://www.emacswiki.org/cw/LionKimbro',
'http://www.emacswiki.org/cw/MattisManzel',
'http://www.emacswiki.org/cw/AlexSchroeder',
'http://www.emacswiki.org/cw/BayleShanks'],
'title': 'Community Wiki',
'wikinode': 'http://www.emacswiki.org/cw/WikiNode'}