User:Birgit Bachler/openevents/code
< User:Birgit Bachler | openevents
Revision as of 21:34, 23 September 2010 by Migratebot (talk | contribs) (Created page with "
== ah04.py: ==
<source lang="text"> #!/usr/local/bin/python
import html5lib, urllib2, urlparse
import os, glob
from os import system, sys
import subprocess
import time...")
ah04.py:
#!/usr/local/bin/python
import html5lib, urllib2, urlparse
import os, glob
from os import system, sys
import subprocess
import time, random
cardnumber=""
def runBash(cmd):
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
out = p.stdout.read().strip()
return out #This is the stdout from the shell command
def absolutize (href, base):
if not href.lower().startswith("http://"):
return urlparse.urljoin(base, href)
return href
def openURL (url, data):
"""
returns (page, actualurl)
sets user_agent and resolves possible redirection
realurl maybe different than url in the case of a redirect
"""
request = urllib2.Request(url)
user_agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14"
request.add_header("User-Agent", user_agent)
if "?" in url:
(url, data) = url.split("?")
if data:
# print "posting"
pagefile=urllib2.urlopen(request, data)
else:
pagefile=urllib2.urlopen(request)
realurl = pagefile.geturl()
return (pagefile, realurl)
def downloadURL (url, foldername):
"""
returns (page, actualurl)
sets user_agent and resolves possible redirection
realurl maybe different than url in the case of a redirect
"""
request = urllib2.Request(url)
user_agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.14) Gecko/20080418 Ubuntu/7.10 (gutsy) Firefox/2.0.0.14"
request.add_header("User-Agent", user_agent)
pagefile=urllib2.urlopen(request)
realurl = pagefile.geturl()
data = pagefile.read()
urlpath = urlparse.urlparse(url)[2]
(path, filename) = os.path.split(urlpath)
filename = os.path.join(foldername, filename)
out = open(filename, "wb")
out.write(data)
def topProducts():
from string import punctuation
N = 20
words = {}
words_gen = (word.strip(punctuation).lower() for line in open("alle.txt")
for word in line.split())
for word in words_gen:
words[word] = words.get(word, 0) + 1
top_words = sorted(words.iteritems(),
key=lambda(word, count): (-count, word))[:N]
for word, frequency in top_words:
alle = "%s: %d" % (word, frequency)+"\n"
print >> html2, alle+"<br>"
print "*"*50
print "*"*50
print "New Bonuskaartfriend? Otherwise just press enter. Example:"
print "2601234567890 YourName"
newCard=raw_input()
print "*"*50
print "*"*50
if newCard != "":
# read the current contents of the file
f = open("cardnumbers.txt")
text = f.read()
f.close()
# open the file again for writing
f = open("cardnumbers.txt", "w")
f.write(str(newCard)+"\n")
# write the original contents
f.write(text)
f.close()
print "*"*50
print "START"
print "*"*50
txx = open("cardnumbers.txt", "r")
for line in txx:
print "*"*50
print "Busy with Bonuskaartfriend", line
print "*"*50
url = "http://www.ah.nl/previouslybought/PreviouslyBought.do"
line = line.rstrip("\n")
line2 = line[0:13]
print line, line2
cardnumber = line2
system("mkdir "+str(cardnumber)+"")
system("mkdir "+str(cardnumber)+"/output")
#print type(cardnumber)
#f, url2 = openURL(url, "cardNumber=2620481956139")
#cardnumber = "2620474055641"
f, url = openURL(url, "cardNumber="+cardnumber)
parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("BeautifulSoup"))
firstPage = parser.parse(f)
# print type(firstPage)
f.close()
#fillet stands for pythonnumber of item pages
loadedURL = {}
import codecs
txt = codecs.open(str(cardnumber)+"/"+str(cardnumber)+".txt", "w", encoding="utf-8")
for a in firstPage.findAll("a", { "class" : "fillet" }):
pageurl = a["href"]
pageurl = absolutize(pageurl, url)
if not loadedURL.has_key(pageurl):
#LOAD URL...
print "PAGE ", pageurl
f2, pageurl = openURL(pageurl, None)
# print f2.read()
parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("BeautifulSoup"))
page = parser.parse(f2)
for img in page.findAll("img"):
# print " ", img
# src = img.getAttribute("src")
src = img["src"]
if src:
src = absolutize(src, pageurl)
#print " ",src
time.sleep(random.random())
try:
downloadURL(src, ""+str(cardnumber)+"/output")
except urllib2.URLError, msg:
print "ncfiles: Urllib2 error (%s)" % msg
except socket.error, (errno, strerror):
print "ncfiles: Socket error (%s) for host %s (%s)" % (errno, host, strerror)
for h3 in page.findAll("h3"):
value = (h3.contents[0])
if value != "Afdeling":
print >> txt, value
import codecs
f = codecs.open("alle.txt", "r", encoding="utf-8")
text = f.read()
f.close()
# open the file again for writing
f = codecs.open("alle.txt", "w", encoding="utf-8")
f.write(value+"\n")
# write the original contents
f.write(text)
f.close()
loadedURL[pageurl] = True
f.close()
f2.close()
system("mkdir "+str(cardnumber)+"/products")
system("mv "+str(cardnumber)+"/output/*_80.jpg "+str(cardnumber)+"/products")
system("montage "+str(cardnumber)+"/products/*.jpg "+str(cardnumber)+"/"+str(cardnumber)+".jpg")
system("convert -delay 100 "+str(cardnumber)+"/products/*.jpg "+str(cardnumber)+"/"+str(cardnumber)+".gif")
system("rm -r "+str(cardnumber)+"/output")
system("rm -r "+str(cardnumber)+"/products")
#system("display "+str(cardnumber)+"/"+str(cardnumber)+".jpg &")
#TRY TO GRAB ALL EXISTING BONUSCARD-DIRECTORIES FOR COMPARISON-LOOP
path = '/home/merglind/Documents/OpenEvents/try04'
html = codecs.open(str(cardnumber)+"/index.html", "w", encoding="utf-8")
friends=[]
for infile in glob.glob( os.path.join(path, '26*') ):
(head, tail) = os.path.split(infile)
#print "current file is: " + tail
compareNumber = tail
if compareNumber != cardnumber:
cmd="./compare.sh "+str(cardnumber)+"/"+str(cardnumber)+".txt "+str(compareNumber)+"/"+str(compareNumber)+".txt"
value=runBash(cmd)
value=round(float(value),2)
# print value, "% Match with bonuscard number", compareNumber
friend = value, compareNumber
#print >> txt02, friend
if value > 10.0:
friends.append(friend)
#print friends
friends.sort(reverse=True)
# import pprint
# pprint.pprint(friends)
##alle.close()
#### TOP10 products
#################HTML STARTS HERE#########################
if line[14:]!="":
nickname=line[14:]
else:
nickname=cardnumber
print >> html, "<html><head><title>"+line[14:]+"'s Profile</title>"
print >> html, """<link rel="stylesheet" href="../style.css"></head>"""
print >> html, """
<body>
<table bgcolor="#00A0E2"width="100%" height="40px "border="0">
<tr>
<td width="25%"><img src="../ah_friends_t.gif"></td>
<td><h1>
"""
print >> html, ""+cardnumber+"'s Profile</></h1></td>"
print >> html, """
</tr>
</table>
<div class="outer"><div class="inner">
"""
print >> html, "<p><h2>This is "+nickname+"</h2>"
print >> html, "<div><a href='../"+cardnumber+"/"+cardnumber+".jpg'><img src='../"+cardnumber+"/"+cardnumber+".gif'></a></div>"
print >> html, "<h3>"+nickname+"'s BEST BONUS-FRIENDS:</h3>"
print >> html, "<ul>"
for friend in friends:
card=friend[1]
score="%0.2f" % friend[0]
if score > 10.:
print >> html, "<li><a href='../"+card+"/index.html'>"+card+"</a> "+score+"% Match</li>"
else:
print >> html, " "
print >> html, "</ul>"
print >> html, "<h3><a href='../"+cardnumber+"/"+cardnumber+".txt'>"+nickname+"'s Shopping List</a></h3>"
#print >> html, "<div><img src='../"+cardnumber+"/"+cardnumber+".jpg'></div>"
print >> html, "</div></div></body></html>"
f2.close()
txt.close()
cardnumber=""
#system("sys.stdout.flush()")
print "*"*50
print "END"
print "*"*50
import codecs
html2 = codecs.open("index.html", "w", encoding="utf-8")
print >> html2, "<html><head><title>Statistics</title>"
print >> html2, """<link rel="stylesheet" href="style.css"></head>"""
print >> html2, """
<body>
<table bgcolor="#00A0E2"width="100%" height="40px "border="0">
<tr>
<td width="25%"><img src="ah_friends_t.gif"></td>
<td><h1>
"""
print >> html2, "Top 20</></h1></td>"
print >> html2, """
</tr>
</table>
<div class="outer"><div class="inner">
"""
print >> html2, "<p><h2>Today's Top 20:</h2>"
topProducts()
compare.sh:
#!/bin/bash
TOTAL=`cat $1 | wc -l`
#echo $TOTAL
#WTWO=wc -l txt02.txt
cat $1 | sort | uniq > /tmp/txt01-uniq.txt
cat $2 | sort | uniq > /tmp/txt02-uniq.txt
cat /tmp/txt01-uniq.txt >> /tmp/txt02-uniq.txt
MATCHES=`cat /tmp/txt02-uniq.txt | sort | uniq -d | wc -l`
BFF=`echo "$MATCHES*100/$TOTAL" | bc -l`
echo "$BFF"