User:Lidia.Pereira/PTSP/SSC

http://headroom.pzwart.wdka.hro.nl/~lpereira/cgi-bin/genderturingtest.cgi

This test is an adaptation of the gender Turing test to distinguish between a man and a woman, which itself derives from the original Turing test to distinguish between a human and machine. However, as Andrew Hodges put it, Turing's experiments are problematic due to their lack of understanding of the impact of sex, society and politics on what people might think, that is, they lack the realization that gender is nothing but a social construction. That said, this adaptation features sets of skills scraped from Linkedin and your role is to drag them to the column where you think they belong. How do you perceive these skills?

1st Step: Scrape it like it's hot

import html5lib
from urlparse import urljoin, urldefrag
import urllib2, urllib
from urllib2 import urlopen
from xml.etree import ElementTree as ET
import time, json, argparse


parser = argparse.ArgumentParser()
parser.add_argument("-url", help="start crawling!")
args = parser.parse_args()

preffix1 = args.url
preffix2 = "http://www.linkedin.com/pub/dir/"
preffix3 = "http://www.linkedin.com/pub/"
start_urls = [preffix1]
middle_ground = []
personal_pages = []
history = []
dbase = open("linkedin_database.txt", "a")
datab = open("tryout_databse.txt", "a")
dictionary = {}
same_name = ''
same_sex = ''
listing = ''
found = False



while start_urls:
    url = start_urls[0]
    print url
    start_urls = start_urls[1:]
    try:
        f = urlopen(url)
        parsed = html5lib.parse(f, namespaceHTMLElements=False)
        ol = parsed.findall(".//ol") + parsed.findall(".//ul")
        for o in ol:
            if o.attrib.get("class") == "primary" or o.attrib.get("class") == "directory":
                main_index = o
                if main_index == None:
                    continue 
                alink = main_index.findall(".//li/a")
                for b in alink:
                    if b.get("href") == None:
                        continue
                    joinity = urljoin(f.geturl(), b.attrib.get("href"))
                    joinity = urldefrag(joinity)[0]
                    if joinity not in history and joinity not in start_urls and joinity.startswith(preffix1):
                        start_urls.append(joinity)
                        history.append(joinity)
                    elif joinity not in history and joinity not in middle_ground and joinity.startswith(preffix2):
                        middle_ground.append(joinity)
                        history.append(joinity)
                    elif joinity not in history and joinity not in personal_pages and joinity.startswith(preffix3) and preffix2 not in joinity:
                        personal_pages.append(joinity)
                        history.append(joinity)
    except urllib2.URLError:
        print "Nee!"

if middle_ground:
    for mg in middle_ground:
        url = middle_ground[0]
        middle_ground[1:]
        try:
            m = urlopen(url)
            parsol = html5lib.parse(m, namespaceHTMLElements=False)
            li = parsol.findall(".//li")
            for i in li:
                if i.attrib.get("class") == "vcard":
                    blink = i.findall(".//a")
                    for bl in blink:
                        href = bl.attrib.get("href")
                        joini = urljoin(mg,href)
                        if joini not in history and joini not in personal_pages and joini.startswith(preffix3):
                            personal_pages.append(joini)
        except urllib2.URLError:
            print "Nee Nee Nee!"


if personal_pages:
    for p in personal_pages:
        url = personal_pages[0]
        print url
        personal_pages = personal_pages[1:]
        try:
            n = urlopen(url)
            parsing = html5lib.parse(n, namespaceHTMLElements=False)
            lo = parsing.findall(".//ol")
            name = parsing.findall(".//span")
            for l in lo:
                if l.attrib.get("id") == "skills-list":
                    found = True
                    datab.write("SKILLS: ")
                    list_item = l.findall(".//li/span")
                    for l_i in list_item:
                        listing += l_i.text.strip() + ", "
                        if l_i != list_item[len(list_item)-1]:
                            datab.write(l_i.text.encode("utf-8").strip() + ", ")
                        elif l_i == list_item[len(list_item)-1]:
                            datab.write(l_i.text.encode("utf-8").strip())
                            datab.write("\n")
            if found:
                for na in name:
                    if na.attrib.get("class") == "given-name":
                        given_name = na.text.encode("utf-8").lower().split()
                        given_name = given_name[0]
                        dictionary["name"] = given_name
                        datab.write("NAME:"+ given_name + '\n')
                        global same_name, same_sex
                        if given_name != same_name:
                            print given_name, "virou"
                            same_name = given_name
                            feed = urllib2.urlopen("https://gender-api.com/get?name="+ given_name +"&key=NhFeoBVUosjBsrAJAb")
                            if feed == None:
                                continue
                            data = json.load(feed)
                            sex = data['gender']
                            same_sex = sex
                            datab.write("SEX:"+ sex + '\n')
                            dictionary["sex"] = sex
                        else:
                            print given_name , "ta na mesma"
                            datab.write("SEX:"+ same_sex + '\n')
                            dictionary["sex"] = same_sex
                datab.write("\n")
                dictionary["skills"] = listing.encode("utf-8").rstrip(", ")
                dbase.write(urllib.urlencode(dictionary) + "\n")
                print dictionary  
            found = False
            listing = ''
        except urllib2.URLError:
            print "Nee!"

2nd Step: Process the Database by adding id's, a m and an f value

import uuid, urlparse, urllib

dbase = open("linkedin_database.txt","r")
datab = open("newLinkedin_database.txt","w")
lines = dbase.readlines()

for line in lines:
    d = urlparse.parse_qs(line.rstrip())
    if "id" not in d and "f" not in d and "m" not in d: 
        d["id"] = [str(uuid.uuid1())]
        d["m"] = [str(0)]
        d["f"] = [str(0)]
    datab.write(urllib.urlencode(d)+"\n")

3rd Step: Create the Interface

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import cgi, urllib
import cgitb; cgitb.enable()
import urlparse, time, random
import urllib2

datab = open("newLinkedin_database.txt", "r")

lines = datab.readlines()
random.shuffle(lines)
lista = []
count = 1


print "Content-Type: text/html"
print 
print """
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset='UTF-8'>
    <title> Gender Turing Test </title>
    <link rel='stylesheet' href='../jquery-ui-1.10.4.custom/css/ui-lightness/jquery-ui-1.10.4.custom.css'>
    <script src='../jquery-ui-1.10.4.custom/js/jquery-1.10.2.js'></script>
    <script src='../jquery-ui-1.10.4.custom/js/jquery-ui-1.10.4.custom.js'></script>
     <style>
    #enquadrado{position:absolute; margin-left:3%; margin-top:1.5%; width:40%} 
    #lesdraggables{ width: 300px; position:absolute; margin-left:47%;}
    #lesdroppables{position:absolute; margin-left:70%; z-index:-999}
    #column{font-family:sans-serif; text-align:center}
    .draggable { width: 103px; height: 20px; padding: 0.5em; margin: 10px 10px 10px 0; float:left}
    #male { width: 150px; height: 550px; padding: 0.5em; margin-right: 10px; float: left }
    #female { width: 150px; height: 550px; padding: 0.5em; margin-right: 10px; float: left }
    p {font-family:georgia; font-size:11px, text-align:center}
    #explanation {font-family:sans-serif; font-size:13px; color: #009999; line-height:17px }
    #title {font-family:sans-serif; font-size: 23px; color: #FF5050}
    select { width: 100px; }
    #visualizer{font-family:sans-serif; font-size:13px; color: #099; text-transform:uppercase; text-decoration: none;}
    #visualizer:hover{color:#FF5050}
    </style>
    <script src= '../genderturingtestskillset.js'>  </script>
    </head>
    <body>
    <div id = 'enquadrado'>
    <h1 id = 'title'> Gender Turing Test </h1>
    <p id = 'explanation'> This test is an adaptation of the gender Turing test to distinguish between a man and a woman, which itself derives from the original
    Turing test to distinguish between a human and machine. However, as Andrew Hodges put it, Turing's experiments are problematic due to their
    lack of understanding of the impact of sex, society and politics on what people might think, that is, they lack the realization that gender 
    is nothing but a social construction. 
    That said, this adaptation features sets of skills scraped from Linkedin and your role is to drag them to
    the column where you think they belong. How do you perceive these skills? (<a href= 'genderturingresults.cgi' id='visualizer'> <strong>Visualizer</strong> </a>)</p>
    <button id='done'>Done!</button> <br>
    
    </div>
    <div id='lesdraggables'>
    
    """

for line in lines:
    if count < 21:
        d = urlparse.parse_qs(line.rstrip())
        skills = d["skills"][0]
        id = str(d['id'][0])
        id = id.strip("['']")
        print "<div id='"+id+"' class='ui-widget-content draggable'>"
        print "<select>"
        print  "<option value='skillset'>Skill Set %s</option>" % count
        for s in skills.split(","):
            if "['" in s:
                s=s.strip("['")
            if "']" in s:
                s=s.strip("']")
            print "<option disabled value ='skill'>"+ s +"</option>"
        print "</select></div>"
    else:
        break
    count += 1

print """</div>
    <div id ='lesdroppables'>
    <div id='male' class='ui-widget-header'>
    <p id = 'column'>Masculine</p>
    </div>
    <div id='female' class='ui-widget-header'>
    <p id = 'column'>Feminine</p>
    </div>
    </div></body>
    </html>
    """

4th Step: Make it interactive and communicate with the back-end

$(function() {
        $('.ui-widget-content').each(function(){
                    $(this).draggable()
                    });
        $("#male").droppable({
            drop: function( event, ui ) {
                console.log("DROP",this, ui)
                $(this)
                    .find('p')
                    .addClass('ui-state-highlight');
                $(ui.draggable)
                    .removeClass('female')
                    .addClass('male')
                }
            });
        $("#female").droppable({
            drop: function( event, ui ) {
                console.log("DROP",this)
                $(this)
                    .find('p')
                    .addClass('ui-state-highlight')
                $(ui.draggable)
                    .removeClass('male')
                    .addClass('female')

            }
        }); 
        $('#done').click(function (){

            var data = {male:[],
                        female:[] };
            $(".draggable.male").each(function(){
                console.log("male",this)
                var id = $(this).attr("id")
                data["male"].push(id)
            })
            $(".draggable.female").each(function(){
                console.log("female",this)
                var id = $(this).attr("id")
                data["female"].push(id)
            })
            $.ajax({
                type: "POST",
                url: "/cgi-bin/genderturingtestvotes.cgi",
                data: data,
                success: function(data){
                    alert("Thank you!")
                    //console.log("python sent", data);
                },
                error: function(obj, e) {
                    console.log(e, "error")
                }
                // contentType: "application/x-www-form-urlencoded; charset=UTF-8",

                //dataType: "json"
            })
            } );
    });

5th Step: The Back-end which processes the votes

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import cgi, urllib, sys
import json, urlparse


print "Content-Type: text/html"
print 

form = cgi.FieldStorage()
malevalue = form.getlist("male[]")
femalevalue = form.getlist("female[]")


dbase = open("newLinkedin_database.txt","r")
lines = dbase.readlines()
dbase.close()
datab = open("newLinkedin_database.txt","w")
dictionary = {}

for line in lines:
    d = urlparse.parse_qs(line.rstrip())
    dictionary["id"]= d["id"][0]
    dictionary["skills"]= d["skills"][0]
    for mv in malevalue:
        if mv in d["id"][0]:
            mvalue = d["m"][0].strip("['']")
            mvalue = int(mvalue) + 1
            dictionary["m"] = [str(mvalue)]
        else:
            dictionary["m"] = d["m"][0]
            
    for fm in femalevalue:
        if fm in d["id"][0]:
            fvalue = d["f"][0].strip("['']")
            fvalue = int(fvalue) + 1
            dictionary["f"] = [str(fvalue)]
        else:
            dictionary["f"] = d["f"][0]
    datab.write(urllib.urlencode(dictionary)+"\n")

6th Step: The Visualizer

#!/usr/bin/env python
#-*- coding:utf-8 -*-

import cgi, urllib, sys
import json, urlparse


voteDb = open("newLinkedin_database.txt","r")
lines = voteDb.readlines()
count = 1

print "Content-Type: text/html"
print 
print """
    <!DOCTYPE html>
    <html>
    <head>
    <meta charset='UTF-8'>
    <script src='/jquery-ui-1.10.4.custom/js/jquery-1.10.2.js'></script>
    <title> Gender Turing Test Visualizer </title>
    <script>

    </script>
    <style>
    
    #mas{
        width:50%;
        float:left;
        height:755px;
        height: 98vh;
        
    }
    #fem {
        width:49%;
        float:left;
        height: 98vh;
        border-right: 2px dashed #FF5050;
    }
    select { width: 100px; }

    h1{
    font-family:sans-serif; font-size: 23px; color: #FF5050; text-align: center;
    }
    #container {width:99%; position:absolute; margin-top:7vh; height:91vh}
    </style>



    </head>
    <body>
    <div id = 'fem'>
    <h1> Feminine </h1>
    </div>
    <div id = 'mas'>
    <h1> Masculine </h1>
    </div>
    <div id='container'>
    """

for line in lines:
    d = urlparse.parse_qs(line.rstrip())
    skills = d["skills"][0]
    if int(d["f"][0].strip("['']")) > 0 or int(d["m"][0].strip("['']")) > 0:
        feminine = int(d["f"][0].strip("['']"))
        masculine = int(d["m"][0].strip("['']"))
        total = int(masculine) + int(feminine)
        percent = str(91*masculine/total)
        print "<div class='mimi'style='margin-left:"+percent+"%;'>"
        print "<select class ='margina'>"
        print "<option value='skillset'>Skill Set %s</option>" % count
        for s in skills.split(","):
            if "['" in s:
                s=s.strip("['")
            if "']" in s:
                s=s.strip("']")
            print "<option disabled value ='skill'>"+ s +"</option>"
        print "</select> </div>"
    count += 1
    
print """</div>
<script>
var coisinhas = document.getElementsByClassName('margina');
for(var i = 0; i < coisinhas.length; i ++) {
    var valor = Math.floor((Math.random() * 44) + 1);
    coisinhas[i].style.marginTop = valor+'px';
}
</script>"""
print "</body>"
print "</html>"