User:Laurier Rochon/work/tragicnine/soft
news.cgi
#!/usr/bin/python2.6
#-*- coding:utf-8 -*-
#print "Content-Type: text/html"
#print
from __future__ import division
import urllib2
import json
import html5lib, lxml, lxml.cssselect
from BeautifulSoup import BeautifulSoup, Comment
import random
import datetime
import narrator
import filters
entries = []
us = []
titles = []
search = True
maxcrawl = 64
terms = ['tragedy']
c = 0
for term in terms:
start=0
while start<maxcrawl:
url = ('https://ajax.googleapis.com/ajax/services/search/news?v=1.0&q='+term+'&start='+ str (start)+'&rsz=large')
f = urllib2.urlopen(url)
data = json.load(f)
nb = data['responseData']['cursor']['estimatedResultCount']
if nb>maxcrawl:
nb=maxcrawl
for r in data['responseData']['results']:
entry = r['unescapedUrl']
if entry not in us:
us.append(entry)
entries.append(entry)
c = c+1
percent = int (round((c/maxcrawl)*100))
print 'harvesting links...'+str (percent)+'% of possible maximum'
start += 8
print titles
print '----------------------------------------------------------------------'
print 'done getting links - crawling them to find titles and content'
print '----------------------------------------------------------------------'
random.shuffle(entries)
#entries = ['http://www.thelantern.com/sports/commentary-sports-give-hope-escape-to-those-affected-by-tragedy-1.2218560']
for b in entries:
if search:
print '----------------------------------------------------------------------'
print 'crawling : '+b
request = urllib2.Request(b)
request.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux x86_64;fr; rv:1.9.1.5) Gecko/20091109 Ubuntu/9.10 (karmic) Firefox/3.5.5")
try:
f=urllib2.urlopen(request)
q = f.read()
except Exception, e:
print e
try:
cc=''
for line in q:
cc=cc+line
a = BeautifulSoup(''.join(BeautifulSoup(cc).findAll(text=lambda text:text.parent.name != "script" and text.parent.name != "style")))
except Exception, e:
print e
try:
a = str (a)
body = a.split("\n")
#check min size of our chunk
article = filters.minsize(body)
sentences = article.split('. ')
print sentences
cleartext = ''
cleartext = cleartext +'\n\n'+ str (filters.cleanup(sentences))
clrstr = BeautifulSoup(cleartext,convertEntities=BeautifulSoup.HTML_ENTITIES).contents[0]
listed = clrstr.split('.')
if len(listed)>10:
search = False
l = ''
for v in listed[1:10]:
l = l + v + '. '
print
t = narrator.narrate(l.encode('utf-8'))
print t
fname = 'public_html/talk_files/episodes/'+str (datetime.date.today())
f = open(fname, 'w')
f.write(t)
f.close()
except Exception, e:
print e
narrator.py
import random
import re
import urllib
newd = []
activeppl = []
people = ['Mandy','Jules','Winnie']
def narrate(d):
c=0
script = ''
data = d.split('.')
occ = rnd(3)
lined = rnd(10)
inserttime = False
for l in data:
if l and len(l)>10:
ol = len(l)
person = people[rnd(3)]
activeppl.append(person)
l = l.replace('"','')
#replace 2 non-ascii quotation marks by nothing
v = urllib.quote(l)
v = v.replace("%E2%80%99","%27")
l = urllib.unquote(v)
l = l.replace("''","")
#make sure the 1st character is always a letter
while not l[0].isalpha():
l = l[1:]
try:
if not l[0].isupper():
l = l.replace(l[0],l[0].upper(),1)
except Exception, e:
print e
#take out the 'he said' 'she said' blablabla
l = re.sub(r',\s\bsaid\b.*','',l)
l = re.sub(r',.*\bsaid\b','',l)
#small sentence? make it a question!
if ol < 30:
l = l + '?'
else:
l = l + '.'
#check if it's the same person talking
if c>0 and person != activeppl[c-1]:
same = True
else:
same = False
if c==0:
same = True
before = rnd(2)
if before==0:
l = '"'+l.strip(' \t\n\r')+'"' +str (postverb(person,same))
else:
if same:
add = str (person) + ' : '
else:
add = ''
l = add+'"'+l.strip(' \t\n\r')+'"'
#group many lines by same person together
if same:
l = '\n' + l
if c>3 and not same and not inserttime:
l = '\n'+str (givemetime())
inserttime = True
#give a bit of context...
#if occ!=0 and lined==c:
script = str (script) + str (l)+'\n'
#script = script.replace('\n\n\n','\n\n')
c = c + 1
z = situate('')
script = z +'\n'+script
return script
def postverb(person,same):
if same:
order = rnd(3)
verbs = ['said','offered','affirmed','mumbled','voiced','said','declared','suggested']
if order==0:
return ' '+str (person) +' '+str (verbs[rnd(len(verbs))])
elif order==1:
return ' '+str (verbs[rnd(len(verbs))])+' '+str (person)
else:
return ' - '+str (person)
else:
return ''
def givemetime():
time = ['(a few minutes later)','long hours pass, eventually they agree to continue the conversation...','the sun goes down slowly, as they finish their conversation','the air gets slghtly heavier, weighing on everyone present','a distant ringing is heard in the distance...','half an hour later','the next day...','suddenly something springs up...']
return time[rnd(len(time))]
def context(param):
group = []
place = []
action = []
if param==0:
g1 = str (people[0]) + ' and '+ str(people[1])
g2 = str (people[0]) + ' and '+ str(people[2])
g3 = str (people[1]) + ' and '+ str(people[2])
group = ['they','the gang','the friends','all of them',g1,g2,g3]
return group[rnd(len(group))]
if param==1:
place = ['downstairs','upstairs','outside','to the lake','towards the frosted bay window','in the kitchen','into the hallway','in the office', 'on the porch','towards the bathroom','sit on the sofa in front of the tv','take a seat in the kitchen','to the table','grab a drink','']
return place[rnd(len(place))]
if param==2:
action = ['ponder life for a short moment',' stare at each other, like they had never met before','wait for someone to break the silence',str (people[rnd(3)])+' sighs, looking at the other two','act as if somehow, they did could not comprehend what had just been said']
return action[rnd(len(action))]
def rnd(nm):
no = random.randrange(0,nm)
return no
def situate(line):
sentence = str (context(0).capitalize()) + ' go ' + str (context(1)) + ' and ' + str (context(2))
return sentence
Filters.py
this file in particular needs massive reworking...sigh.
from __future__ import division
import nltk.util
import re
def minsize(body):
article = ''
#go line by line
for line in body:
#are you at least 100 chars? (sentence)
if len(line)>100:
#line = line.replace('','')
line = line.replace("\xe2\x80\x9c","")
line = line.replace("\xe2\x80\x94","")
line = line.replace("\xe2\x80\x9d","")
line = line.replace("Mrs.","Mrs")
line = line.replace("Ms.","Ms")
line = line.replace("Mr.","Mr")
line = line.replace("Dr.","Dr")
line = line.replace('"','')
line = line.replace(' ',' ')
line = nltk.util.clean_html(line)
line = line.replace("Jan.","January")
line = line.replace("Feb.","February")
line = line.replace("Mar.","March")
line = line.replace("Apr.","April")
line = line.replace("Jun.","June")
line = line.replace("Jul.","July")
line = line.replace("Aug.","August")
line = line.replace("Sep.","Septembre")
line = line.replace("Oct.","October")
line = line.replace("Nov.","November")
line = line.replace("Dec.","December")
article = article + line
return article
def cleanup(sentences):
sents = ''
blacklist = ['|','Internet Options','IE','comment','ENCODING','rss','Login','article','.com','Favorites','Advertisement','PM','AM','ET' 'DOCTYPE','login','password','loading','Loading','form','Stories','/','Buy','buy','Comment','News','Feed']
for sent in sentences:
found = False
sent = re.sub('\.(?![a-zA-Z]{2})','',sent)
#do you contain blacklist elements (passed the pre-html filter)?
for x in blacklist:
if x in sent:
found = True
break
#
#too many capital words (over 40%) = probably up to no good
capitals = 0
words = sent.split(' ')
if len(words)>0:
for ww in words:
#and words that are too long (over 25 chars?)
if len(ww)>25:
found = True
else:
if ww:
if ww[0].isupper():
capitals = capitals + 1
if round(capitals/len(words)*100) > 40:
found = True
#
#if filters above are passed...guess you're ok to join the bunch
if not found:
sents = sents + str (sent) +'. '
return sents