2009 206
Toward a navigable text
Acquiring
Today we are working with the text of 10 poems by Edgar Allen Poe, from Project Gutenberg.
Processing
import sys, re
wc = {}
for line in sys.stdin:
line = line.rstrip()
words = re.split("[^a-zA-Z]*", line)
for word in words:
word=word.lower()
if word:
wc[word]=wc.get(word, 0)+1
allwords = wc.keys()
allwords.sort()
for word in allwords:
print word, wc[word]
Now we make a function that takes a file and turns it into a "word count dictionary". Then we can use this function on different poems.
import sys, re
def countwords(file):
wc = {}
for line in file:
line = line.rstrip()
words = re.split("[^a-zA-Z]*", line)
for word in words:
word=word.lower()
if word:
wc[word]=wc.get(word, 0)+1
return wc
def dump(wc):
allwords = wc.keys()
allwords.sort()
for word in allwords:
print word, wc[word]
def describe(wc):
allwords = wc.keys()
allwords.sort()
n = len(allwords)
return str(n) +" words"+" from "+ allwords[0]+" to "+ allwords[-1]
for filename in sys.argv[1:]:
print filename
wc = countwords(open(filename))
print " " + describe(wc)
Word subtraction
def subtract(wc1, wc2):
"""
return a new dictionary with the contents of wc1, minus all the words appearing in wc2 """
...