2018 - 06/2019
home/pi/project/praha.eu-scraper/scraper.py
download_datapackage.py
cd /home/michal/dev/wpca/praha_2018-2019/
cp data/votes.csv x_source.csv
wpca.py
, load_datapackage.py
wpca.py
parties.csv
prepare_chart_data.py
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>W-PCA Scatterplot Chart</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="WPCA scatter plot">
<meta name="author" content="Michal Škop">
<script src="//d3js.org/d3.v3.min.js"></script>
<script src="./d3.scatterplot.js"></script>
<script src="./d3.tips.js"></script>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.5/united/bootstrap.min.css">
<style type="text/css">
/* note: we duplicate some of the styles (css, and as attributes of svg elements), so FF displays it correctly, and it is possible to generate png */
.tick {
fill-opacity: 0;
stroke: #000000;
stroke-width: 1;
}
.domain {
fill: none;
fill-opacity: 0;
stroke: black;
stroke-width: 1;
}
.axis line {
fill: none;
fill-opacity: 0;
stroke: black;
stroke-width: 1;
shape-rendering: crispEdges;
}
.axis text {
font-family: sans-serif;
font-size: 11px;
stroke: gray;
}
circle:hover {
fill-opacity: 1;
}
.label {
font-family: sans-serif;
font-size: 15px;
}
.d3-tip {
line-height: 1;
font-weight: bold;
padding: 12px;
background: rgba(0, 0, 0, 0.8);
color: #fff;
border-radius: 2px;
pointer-events: none;
max-width: 400px;
}
/* Creates a small triangle extender for the tooltip */
.d3-tip:after {
box-sizing: border-box;
display: inline;
font-size: 10px;
width: 100%;
line-height: 1;
color: rgba(0, 0, 0, 0.8);
position: absolute;
pointer-events: none;
}
/* Northward tooltips */
.d3-tip:after {
content: "\25BC";
margin: -1px 0 0 0;
top: 100%;
left: 0;
text-align: center;
}
line {
stroke:gray;
stroke-width:0;
opacity: .3;
}
.perfect {
stroke: gray;
stroke-width:0;
opacity: 0.7;
}
#chart {
padding-top:30px;
}
</style>
</head>
<body>
<nav class="navbar navbar-default">
<div class="container">
<div class="navbar-header">
<h2>Rozložení zastupitelů v Praze dle svého hlasování (2018-2019)</h2>
</div>
</div>
</nav>
<div id="chart"></div>
<div class="alert alert-info">
<p>Každý bod reprezentuje jednoho zastupitele.
<p><a href="//bl.ocks.org/michalskop/8514867">W-PCA</a> model.
<script type="text/javascript">
d3.csv("chart_data.csv", function(voters) {
//d3.csv("cutting_lines_ten.csv", function(lines) {
/*linesselected = [];
for (k in lines) {
if ((parseFloat(lines[k]['loss']) < 1.5) && (parseFloat(lines[k]['cl_beta0']) < 50)) {
beta = [lines[k]['normal_x'],lines[k]['normal_y']];
beta0 = lines[k]['cl_beta0'];
if (beta[1] != 0) {
lines[k]['a'] = -beta0/beta[1];
lines[k]['b'] = -beta[0]/beta[1];
} else {
lines[k]['a'] = 0;
lines[k]['b'] = 0;
}
//add class for a perfect cut:
if (lines[k]['loss'] == 0) {
lines[k]['class'] = 'perfect';
} else {
lines[k]['class'] = 'non-perfect';
}
lines[k]['name'] = lines[k]["motion:name"];
linesselected.push(lines[k]);
}
}*/
spdata = [];
voters.forEach(function(d) {
spdata.push({"x":d["wpca:d1"],"y":d["wpca:d2"],"r":d["r"],"color":d["color"],"name":d["name"],"opacity":d["opacity"], "party": d["party"], "country": d["country"]})
});
var scatterplotwithlineplot = [{
"data": spdata,
"margin": {top: 10, right: 10, bottom: 30, left: 30},
"axes": {"labels":{"x":"DIM 1", "y":"DIM 2"}},
"minmax":{"x":{'min':-1.5,'max':1.5},"y":{'min':-1.5,'max':1.5},"r":{'min':0,'max':1},"rrange":{'min':0,'max':100}},
"size":{"width":800,"height":800}//,
//"lines": linesselected
}];
var svg = d3.select("#chart")
.append("svg")
.attr("width",scatterplotwithlineplot[0]['size']['width'])
.attr("height",scatterplotwithlineplot[0]['size']['height']);
/* Initialize tooltip */
tip = d3.tip().attr('class', 'd3-tip').html(function(d) {
return "<span class=\'stronger\'>" + d["name"] + "</span><br>" + d["party"];
});
/* Invoke the tip in the context of your visualization */
svg.call(tip)
var sp = d3.scatterplotwithlineplot()
.data(function(d) {return d.data})
.margin(function(d) {return d.margin})
.axes(function(d) {return d.axes})
.minmax(function(d) {return d.minmax})
.size(function(d) {return d.size})
//.lines(function(d) {return d.lines})
var scatter = svg.selectAll(".scatterplot")
.data(scatterplotwithlineplot)
.enter()
.append("svg:g")
.attr("transform", "translate(" + scatterplotwithlineplot[0].margin.left + "," + scatterplotwithlineplot[0].margin.top + ")")
.call(sp);
//})
})
</script>
</body>
</html>
"id","name","wpca:d1","wpca:d2","wpca:d3","party","color"
6254,"Stanislav Nekolný, MBA",-0.959232016665741,-0.901644078862583,0.734304101228732,"ANO 2011","#261060"
5846,"Ing. Jakob Hurrle",0.66727791802772,0.325407365343984,0.181328402717551,"PRAHA SOBĚ","#e6ac21"
5842,"Martin Arden",0.73763886214504,0.485443818895901,0.322888914628889,"Piráti","#000000"
5818,"PhDr. Pavel Světlík",0.722606857169152,0.044511302241522,0.058185296613972,"PRAHA SOBĚ","#e6ac21"
5816,"Milan Maruštík",-1.16061842733739,-0.019791235245615,0.032175424923091,"ANO 2011","#261060"
5814,"Ing. Lubomír Brož",-1.15297979094009,-0.051749715701771,0.01569084858329,"ANO 2011","#261060"
5812,"Václav Bílek",-1.14249323841448,-0.045247777851564,0.023622984080692,"ANO 2011","#261060"
5810,"Mgr. Marta Gellová",-1.14962805181627,-0.045945197800113,0.020902254760379,"ANO 2011","#261060"
5808,"Ing. Ivan Pilný",-1.14798684031181,-0.018994843404626,0.11000615439912,"ANO 2011","#261060"
5802,"Pavel Zelenka",0.755535381820959,0.001525058422909,-0.017275749355824,"PRAHA SOBĚ","#e6ac21"
5800,"Mgr. Martin Benda",0.753979212730889,0.054574974687427,0.065631439752563,"PRAHA SOBĚ","#e6ac21"
5798,"Ing. arch. PhDr Lenka Burgerová, Ph.D.",0.753214514992205,-0.191506813949039,0.286501032434354,"PRAHA SOBĚ","#e6ac21"
5796,"Mgr. Jiří Knitl",0.732071022177196,0.041451461007905,0.06195773215747,"PRAHA SOBĚ","#e6ac21"
5792,"Pavel Vyhnánek, M.A.",0.716630763864589,0.037711198828574,-0.01860616081433,"PRAHA SOBĚ","#e6ac21"
5790,"Mgr. Milena Johnová",0.760322804958725,0.012689022518075,-0.003707209962841,"PRAHA SOBĚ","#e6ac21"
5788,"Ing. Adam Scheinherr, MSc., Ph.D.",0.766309195617462,0.038062089285244,0.021965768148046,"PRAHA SOBĚ","#e6ac21"
5786,"Ing. Mariana Čapková",0.754797824950398,0.002640481671321,0.028830116766191,"PRAHA SOBĚ","#e6ac21"
5784,"Petr Zeman",0.755608947855756,0.025300407797909,0.016134349781559,"PRAHA SOBĚ","#e6ac21"
5782,"MgA. Hana Třeštíková",0.692902763224279,0.105350807328635,0.002543929299467,"PRAHA SOBĚ","#e6ac21"
5780,"Mgr. Petr Kubíček",0.695654946143861,-0.075488165712629,-0.25049471499378,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5778,"Mgr. Jiří Koubek, DiS.",0.703323858046363,-0.079267811658014,-0.267566804813814,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5776,"Mgr. Jan Chabr",0.491113103432753,-0.304898293952959,-0.268958756967299,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5774,"Mgr. Radek Vondra",0.673334401467465,-0.310411347737056,0.086881356737934,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5772,"Ing. Pavel Richter",0.753881492690599,-0.035604666483593,-0.149642117591318,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5770,"Ing. Petr Hlubuček",0.680367810394071,0.009555373086536,-0.106118501058791,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5768,"doc. Ing. arch. Petr Hlaváček",0.728261037921687,0.046483185006884,-0.096028962593139,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5766,"JUDr. Jiří Pospíšil",0.510655526475468,-0.139072014234621,-0.520131004269068,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5764,"JUDr. Hana Kordová Marvanová",0.596441117966523,-0.053398912822185,-0.281686596551885,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
5762,"Jiří Dohnal",0.730897587408237,0.034096749913347,0.097195199741386,"Piráti","#000000"
5760,"Mgr. Ing. Jaromír Beránek",0.703639342847691,0.105376124212874,0.110911985696931,"Piráti","#000000"
5758,"Ing. Pavel Hájek",0.661819656735838,0.17594841719825,0.115039679171629,"Piráti","#000000"
5754,"Tomáš Murňák",0.742632930214278,0.115389059306092,0.073015184081468,"Piráti","#000000"
5752,"Mgr. Eva Horáková",0.774070688804693,0.121582827987382,0.033121222192994,"Piráti","#000000"
5750,"Bc. Michaela Krausová",0.657736046711737,0.149774310380833,0.08465084297953,"Piráti","#000000"
5748,"PhDr. Mgr. Vít Šimral, Ph.D. et Ph.D.",0.717164554249033,0.056090183127576,0.109238253960404,"Piráti","#000000"
5744,"MUDr. Zdeněk Hřib",0.724427830178935,0.103264751513741,0.137536693899924,"Piráti","#000000"
5742,"Mgr. Karel Hanzlík",-1.07864558780387,0.279728361519317,-0.459825728577147,"ODS","#004494"
5740,"MUDr. Tomáš Kaštovský",-1.08956747379126,-0.006288069151725,-0.020991783262177,"ODS","#004494"
5738,"Ing. Martin Sedeke",-1.11078258175959,0.015544543023342,-0.053718597559604,"ODS","#004494"
5736,"Mgr. Jakub Stárek",-1.10410631572032,0.013463710181798,-0.041562987780895,"ODS","#004494"
5734,"Bc. Tomáš Štampach",-1.12617839281731,0.017829545426368,-0.022135729117743,"ODS","#004494"
5732,"Jiří Zajac",-1.10840280218002,-0.047712786694418,-0.032623655188626,"ODS","#004494"
5730,"Mgr. Zdeněk Zajíček",-1.16031723694442,-0.015601077087229,0.0372666805999,"ODS","#004494"
5728,"Mgr. Tomáš Portlík",-1.09957608874872,-0.032700917841066,-0.061501691281914,"ODS","#004494"
5660,"Ondřej Kallasch",0.730415307801859,-0.176241215901307,0.303054672897712,"Piráti","#000000"
5656,"Viktor Mahrik",0.746742810416319,0.002126801053152,0.138039413183049,"Piráti","#000000"
5407,"Ing. Ladislav Kos",0.737823045616493,0.512386347086298,0.359692619974975,"Piráti","#000000"
5357,"Ing. Ondřej Prokop",-1.15518850976487,-0.050644160253385,0.001180251073552,"ANO 2011","#261060"
4943,"Adam Zábranský",0.729254704560348,0.089726830437933,0.110853466047971,"Piráti","#000000"
4941,"Jan Wolf",0.484534472505008,-0.492316103778312,-0.479335592248462,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
4931,"Ing. Miloš Růžička",0.770836495135921,-0.011834969689436,-0.102862144268656,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
4923,"RNDr. Marcela Plesníková",-1.14422253910118,-0.021923196166826,0.016123805954651,"ANO 2011","#261060"
4921,"RNDr. Jana Plamínková",0.719675831645595,-0.033856983402926,-0.219757683988628,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
4919,"JUDr. Petr Novotný",-1.14174937203561,0.014712227189161,0.110592820025737,"ANO 2011","#261060"
4915,"Radomír Nepil",-1.14073459231867,-0.026427504594059,0.038881152977155,"ANO 2011","#261060"
4913,"Ing. Patrik Nacher",-1.16002933976444,-0.031607826898256,0.019111165943149,"ANO 2011","#261060"
4905,"Ing. Radek Lacko",-1.14509631176118,-0.055491774097251,0.014453985797492,"ANO 2011","#261060"
4895,"JUDr. Jaroslava Janderová",-1.10656283553307,0.069664646546077,-0.011791175734408,"ODS","#004494"
4879,"PharmDr. Petr Fifka",-1.11941806891619,0.053874662836383,0.054151115880558,"ODS","#004494"
4873,"Mgr. Jan Čižinský",0.746232783620718,-0.012479939034381,0.022985130641098,"PRAHA SOBĚ","#e6ac21"
4522,"Ing. Ondřej Martan",-1.1099769683673,-0.037251024538113,-0.023382906077744,"ODS","#004494"
4350,"Ing. Alexandra Udženija",-1.12654637131453,0.016870971252531,-0.032469273027985,"ODS","#004494"
4342,"prof. Ing. Mgr. Martin Dlouhý, Dr., MSc.",0.713205791256103,-0.162448132502987,-0.151337141086709,"TOP 09 a STAN - Spojené síly pro Prahu","#723769"
4316,"doc. MUDr. Bohuslav Svoboda, CSc.",-1.14588554808141,0.102922520389228,0.04672507164396,"ODS","#004494"
3925,"Ing. David Vodrážka",-1.10240445711658,0.090155061216936,0.076892922038552,"ODS","#004494"
id,name,wpca:d1,wpca:d2,wpca:d3
3925,Ing. David Vodrážka,-1.1024044571165756,0.09015506121693621,0.07689292203855198
4316,"doc. MUDr. Bohuslav Svoboda, CSc.",-1.1458855480814065,0.10292252038922828,0.046725071643960225
4342,"prof. Ing. Mgr. Martin Dlouhý, Dr., MSc.",0.7132057912561032,-0.16244813250298734,-0.15133714108670918
4350,Ing. Alexandra Udženija,-1.1265463713145343,0.01687097125253053,-0.032469273027985336
4522,Ing. Ondřej Martan,-1.1099769683672975,-0.03725102453811314,-0.02338290607774428
4873,Mgr. Jan Čižinský,0.7462327836207177,-0.012479939034381094,0.022985130641097893
4879,PharmDr. Petr Fifka,-1.119418068916188,0.05387466283638334,0.054151115880557904
4895,JUDr. Jaroslava Janderová,-1.1065628355330688,0.06966464654607654,-0.011791175734407902
4905,Ing. Radek Lacko,-1.145096311761176,-0.0554917740972512,0.014453985797492188
4913,Ing. Patrik Nacher,-1.1600293397644417,-0.03160782689825628,0.019111165943148798
4915,Radomír Nepil,-1.140734592318673,-0.026427504594059164,0.03888115297715526
4919,JUDr. Petr Novotný,-1.141749372035614,0.014712227189161202,0.11059282002573737
4921,RNDr. Jana Plamínková,0.7196758316455952,-0.03385698340292551,-0.2197576839886283
4923,RNDr. Marcela Plesníková,-1.1442225391011767,-0.021923196166826378,0.016123805954651165
4931,Ing. Miloš Růžička,0.7708364951359213,-0.011834969689435813,-0.10286214426865611
4941,Jan Wolf,0.48453447250500836,-0.49231610377831225,-0.47933559224846206
4943,Adam Zábranský,0.7292547045603481,0.08972683043793246,0.11085346604797061
5357,Ing. Ondřej Prokop,-1.1551885097648713,-0.05064416025338505,0.0011802510735522819
5656,Viktor Mahrik,0.7467428104163185,0.0021268010531522176,0.1380394131830493
5660,Ondřej Kallasch,0.7304153078018594,-0.1762412159013072,0.3030546728977116
5728,Mgr. Tomáš Portlík,-1.099576088748716,-0.03270091784106594,-0.06150169128191426
5730,Mgr. Zdeněk Zajíček,-1.160317236944423,-0.01560107708722922,0.037266680599900306
5732,Jiří Zajac,-1.108402802180016,-0.04771278669441747,-0.032623655188626054
5734,Bc. Tomáš Štampach,-1.1261783928173121,0.017829545426367433,-0.022135729117743138
5736,Mgr. Jakub Stárek,-1.1041063157203246,0.01346371018179795,-0.041562987780894914
5738,Ing. Martin Sedeke,-1.1107825817595882,0.015544543023342197,-0.05371859755960425
5740,MUDr. Tomáš Kaštovský,-1.08956747379126,-0.006288069151724715,-0.02099178326217685
5742,Mgr. Karel Hanzlík,-1.0786455878038683,0.27972836151931707,-0.4598257285771474
5744,MUDr. Zdeněk Hřib,0.7244278301789348,0.10326475151374113,0.13753669389992412
5748,"PhDr. Mgr. Vít Šimral, Ph.D. et Ph.D.",0.7171645542490326,0.05609018312757613,0.10923825396040401
5750,Bc. Michaela Krausová,0.6577360467117368,0.14977431038083264,0.08465084297953
5752,Mgr. Eva Horáková,0.774070688804693,0.12158282798738153,0.033121222192993814
5754,Tomáš Murňák,0.7426329302142779,0.11538905930609246,0.07301518408146782
5758,Ing. Pavel Hájek,0.6618196567358375,0.17594841719825027,0.11503967917162891
5760,Mgr. Ing. Jaromír Beránek,0.7036393428476907,0.10537612421287428,0.11091198569693106
5762,Jiří Dohnal,0.730897587408237,0.034096749913347255,0.09719519974138591
5764,JUDr. Hana Kordová Marvanová,0.5964411179665232,-0.05339891282218507,-0.28168659655188527
5766,JUDr. Jiří Pospíšil,0.510655526475468,-0.13907201423462104,-0.520131004269068
5768,doc. Ing. arch. Petr Hlaváček,0.7282610379216866,0.046483185006883664,-0.09602896259313858
5770,Ing. Petr Hlubuček,0.6803678103940707,0.00955537308653582,-0.10611850105879123
5772,Ing. Pavel Richter,0.7538814926905991,-0.035604666483592486,-0.14964211759131774
5774,Mgr. Radek Vondra,0.6733344014674648,-0.3104113477370562,0.08688135673793436
5776,Mgr. Jan Chabr,0.4911131034327528,-0.3048982939529595,-0.26895875696729854
5778,"Mgr. Jiří Koubek, DiS.",0.7033238580463632,-0.07926781165801342,-0.2675668048138142
5780,Mgr. Petr Kubíček,0.6956549461438613,-0.0754881657126288,-0.2504947149937803
5782,MgA. Hana Třeštíková,0.692902763224279,0.10535080732863529,0.0025439292994667998
5784,Petr Zeman,0.755608947855756,0.02530040779790879,0.016134349781558935
5786,Ing. Mariana Čapková,0.7547978249503979,0.0026404816713209785,0.02883011676619082
5788,"Ing. Adam Scheinherr, MSc., Ph.D.",0.7663091956174617,0.03806208928524352,0.02196576814804595
5790,Mgr. Milena Johnová,0.7603228049587252,0.012689022518074734,-0.0037072099628412767
5792,"Pavel Vyhnánek, M.A.",0.7166307638645892,0.03771119882857392,-0.01860616081433033
5796,Mgr. Jiří Knitl,0.7320710221771964,0.04145146100790522,0.06195773215746976
5798,"Ing. arch. PhDr Lenka Burgerová, Ph.D.",0.7532145149922048,-0.19150681394903918,0.28650103243435404
5800,Mgr. Martin Benda,0.7539792127308892,0.054574974687426926,0.06563143975256322
5802,Pavel Zelenka,0.7555353818209595,0.0015250584229093691,-0.017275749355823468
5808,Ing. Ivan Pilný,-1.1479868403118105,-0.018994843404625983,0.11000615439912044
5810,Mgr. Marta Gellová,-1.1496280518162718,-0.045945197800113156,0.02090225476037922
5812,Václav Bílek,-1.1424932384144793,-0.04524777785156428,0.023622984080691786
5814,Ing. Lubomír Brož,-1.1529797909400927,-0.051749715701770636,0.015690848583290208
5816,Milan Maruštík,-1.16061842733739,-0.01979123524561482,0.03217542492309071
5818,PhDr. Pavel Světlík,0.7226068571691519,0.04451130224152184,0.05818529661397171
5846,Ing. Jakob Hurrle,0.6672779180277197,0.32540736534398357,0.18132840271755085
5407,Ing. Ladislav Kos,0.7378230456164933,0.5123863470862978,0.3596926199749751
5842,Martin Arden,0.7376388621450402,0.4854438188959014,0.32288891462888886
6254,"Stanislav Nekolný, MBA",-0.9592320166657408,-0.9016440788625827,0.7343041012287322
/* requires D3 + https://github.com/Caged/d3-tip */
d3.scatterplotwithlineplot = function() {
unit_circle = true;
function scatterplotwithlineplot(selection) {
selection.each(function(d, i) {
//options
var data = (typeof(data) === "function" ? data(d) : d.data),
//lines = (typeof(lines) === "function" ? lines(d) : d.lines),
margin = (typeof(margin) === "function" ? margin(d) : d.margin),
axes = (typeof(axes) === "function" ? axes(d) : d.axes),
minmax = (typeof(minmax) === "function" ? minmax(d) : d.minmax),
size = (typeof(size) === "function" ? size(d) : d.size),
unit_circle_val = (typeof(unit_circle) === "function" ? unit_circle(d) : unit_circle);
// chart sizes
var width = size['width'] - margin.left - margin.right,
height = size['height'] - margin.top - margin.bottom;
//scales
var xScale = d3.scale.linear()
.domain([minmax['x']['min'], minmax['x']['max']])
.range([0, width])
var yScale = d3.scale.linear()
.domain([minmax['y']['min'], minmax['y']['max']])
.range([height, 0])
var rScale = d3.scale.linear()
.domain([minmax['r']['min'],minmax['r']['max']])
.range([minmax['rrange']['min'],minmax['rrange']['max']]);
//axes
var xAxis = d3.svg.axis()
.scale(xScale)
.orient("bottom");
//.ticks(5);
//.tickSize(16, 0);
var yAxis = d3.svg.axis()
.scale(yScale)
.orient("left");
//.ticks(5);
var element = d3.select(this);
//Create X axis
element.append("g")
.attr("class", "axis x-axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
//Create Y axis
element.append("g")
.attr("class", "axis y-axis")
.call(yAxis);
limits = {"x":[minmax["x"]["min"],minmax["x"]["max"]],"y":[minmax["y"]["min"],minmax["y"]["max"]]}
// for (k in lines) {
// ps = linecross(lines[k],limits);
// if (ps.length == 2) {
// lines[k].x1 = ps[0][0];
// lines[k].y1 = ps[0][1];
// lines[k].x2 = ps[1][0];
// lines[k].y2 = ps[1][1];
// }
// way = get_sign(lines[k].b,lines[k].n1,lines[k].n2);
// lines[k].path =[corners(lines[k],limits,way),corners(lines[k],limits,-1*way)];
// }
//ellipse ~ unit_circle
// if (unit_circle_val) {
// element.selectAll(".ellipse")
// .data([0])
// .enter()
// .append("ellipse")
// .attr("cx", xScale(0))
// .attr("cy", yScale(0))
// .attr("rx", Math.abs(xScale(1)-xScale(0)))
// .attr("ry", Math.abs(yScale(1)-yScale(0)))
// .attr("fill-opacity",0)
// .attr("stroke","red")
// .style("stroke-dasharray", ("10,3"));
// }
//lines
// var line = element.selectAll ('.line')
// .data(lines)
// .enter()
// .append("line")
// .attr("x1",function(d) {return xScale(d.x1)})
// .attr("y1",function(d) {return yScale(d.y1)})
// .attr("x2",function(d) {return xScale(d.x2)})
// .attr("y2",function(d) {return yScale(d.y2)})
// .attr("id", function (d, i) {return "q-" + i;})
// .attr("class", function(d) {
// if (typeof(d['class'] != 'undefined')) return d['class'];
// else return 'line';
// })
// //putting it here and not in css, because it is used for generating png:
// .attr("stroke","gray")
// .attr("stroke-width","1")
// .attr("opacity", 0.15)
//
// .on('mouseover', tip.show)
// .on('mouseout', tip.hide);
//points
element.selectAll(".circle")
.data(data)
.enter()
.append("circle")
.attr("cx", function(d) {
return xScale(d.x);
})
.attr("cy", function(d) {
return yScale(d.y);
})
.attr("r", function(d) {
if ((typeof(d['r']) != 'undefined') && (d['r'] != ''))
return rScale(d.r);
else return rScale(0.15);
})
.attr("class", function(d) {
if ((typeof(d['class']) != 'undefined') && (d['class'] != ''))
return d['class'];
else return 'circle';
})
.attr("fill",function(d) {
if ((typeof(d['color']) != 'undefined') && (d['color'] != ''))
return d.color;
else
return "#eee";
})
.attr("stroke",function(d) {
if ((typeof(d['color']) != 'undefined') && (d['color'] != ''))
return d.color;
else
return "#bbb";
})
.attr("stroke-width",function(d) {
if ((typeof(d['r']) != 'undefined') && (d['r'] != ''))
return Math.max(d.r*5,1);
else return 1;
})
.attr("fill-opacity",function(d) {
if ((typeof(d['opacity']) != 'undefined') && (d['opacity'] != ''))
return d.opacity;
else
return 0.5;
})
.on('mouseover', tip.show)
.on('mouseout', tip.hide);
//axis labels
element.append("text")
.attr("class", "x-label label")
.attr("text-anchor", "end")
.attr("x", width)
.attr("y", height-5)
.text(axes['labels']['x']);
element.append("text")
.attr("class", "y label")
.attr("text-anchor", "end")
.attr("y", 5)
.attr("x", 0)
.attr("dy", ".75em")
.attr("transform", "rotate(-90)")
.text(axes['labels']['y']);
// putting it here and not in css, because it is used for generating png:
element.selectAll(".domain")
.attr("fill","none")
.attr("fill-opacity",0)
.attr("stroke","black")
.attr("stroke-width",1);
element.selectAll(".tick")
.attr("fill-opacity",0)
.attr("stroke","#000")
.attr("stroke-width",1);
element.selectAll("text")
.attr("font-family","sans-serif")
.attr("font-size",11)
element.selectAll(".label")
.attr("font-size",15)
.attr("font-weight","bold")
//convert dy("0.71em") to dy("10"), inkscape feature https://www.ruby-forum.com/topic/5505193 :
element.selectAll("text")
.attr("dy",function(d) {
if (d3.select(this).attr("dy")) {
em = parseFloat(d3.select(this).attr("dy").replace("em",""));
if (d3.select(this).attr("font-size"))
px = parseFloat(d3.select(this).attr("font-size"));
else
px = 11;
return em*px + "px";
} else {
return 0;
}
})
});
}
scatterplotwithlineplot.data = function(value) {
if (!arguments.length) return value;
data = value;
return scatterplotwithlineplot;
};
// scatterplotwithlineplot.lines = function(value) {
// if (!arguments.length) return value;
// lines = value;
// return scatterplotwithlineplot;
// };
scatterplotwithlineplot.margin = function(value) {
if (!arguments.length) return value;
margin = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.axes = function(value) {
if (!arguments.length) return value;
axes = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.minmax = function(value) {
if (!arguments.length) return value;
minmax = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.size = function(value) {
if (!arguments.length) return value;
size = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.unit_circle = function(value) {
if (!arguments.length) return value;
unit_circle = value;
return scatterplotwithlineplot;
};
return scatterplotwithlineplot;
function corners(l,e,o) {
//l = {"a":0,"b":1} //line, a and slope, i.e., y=a+bx
//e = {"x": [-10,10], "y": [-10,10]} //limits
//o = 1 //orientation -1 or 1
//crossing x0, x1
//crossing y0, y1
outp = linecross (l,e);
out = [];
//vertices
for (i=0;i<=1;i++){
for (j=0;j<=1;j++){
if (o*(l.a+l.b*e.x[i]-e.y[j]) > 0)
outp.push([e.x[i],e.y[j]]);
}
}
//sort the outps, anticlockwise
if (outp.length > 0) {
mid = [0,0];
for (i in outp) {
mid[0] += outp[i][0];
mid[1] += outp[i][1];
}
mid[0] = mid[0] / outp.length;
mid[1] = mid[1] / outp.length;
for (i in outp) {
p = outp[i][1] - mid[1];
q = outp[i][0] - mid[0];
if (q != 0)
outp[i][2] = Math.atan(p/q) + (q<0 ? Math.PI : 0);
else
outp[i][2] = Math.PI/2 + Math.PI*sign(p);
}
outp = outp.sort(function(w,z) {
return w[2] > z[2];
});
for (i in outp) {
outp[i].splice(2,1);
out.push({"x":outp[i][0],"y":outp[i][1]});
}
}
return out;
}
function linecross (l,e) {
out = [];
//crossing x0, x1
for (i=0;i<=1;i++){
Y = l.a + l.b*e.x[i];
if ((Y > e.y[0]) && (Y < e.y[1]))
out.push([e.x[i],Y]);
}
//crossing y0, y1
for (j=0;j<=1;j++){
if (l.b != 0) {
X = (e.y[j] - l.a)/l.b;
if ((X > e.x[0]) && (X < e.x[1]))
out.push([X,e.y[j]]);
}
}
return out;
}
function get_sign(b,d1,d2) {
t = b*d1-d2;
if (t > 0) return 1;
if (t < 0) return -1;
return 0;
}
}
/* requires D3 + https://github.com/Caged/d3-tip */
d3.scatterplotwithlineplot = function() {
function scatterplotwithlineplot(selection) {
selection.each(function(d, i) {
//options
var data = (typeof(data) === "function" ? data(d) : d.data),
lines = (typeof(lines) === "function" ? lines(d) : d.lines),
margin = (typeof(margin) === "function" ? margin(d) : d.margin),
axes = (typeof(axes) === "function" ? axes(d) : d.axes),
minmax = (typeof(minmax) === "function" ? minmax(d) : d.minmax),
size = (typeof(size) === "function" ? size(d) : d.size);
// chart sizes
var width = size['width'] - margin.left - margin.right,
height = size['height'] - margin.top - margin.bottom;
//scales
var xScale = d3.scale.linear()
.domain([minmax['x']['min'], minmax['x']['max']])
.range([0, width])
var yScale = d3.scale.linear()
.domain([minmax['y']['min'], minmax['y']['max']])
.range([height, 0])
var rScale = d3.scale.linear()
.domain([minmax['r']['min'],minmax['r']['max']])
.range([minmax['rrange']['min'],minmax['rrange']['max']]);
//axes
var xAxis = d3.svg.axis()
.scale(xScale)
.orient("bottom");
//.ticks(5);
//.tickSize(16, 0);
var yAxis = d3.svg.axis()
.scale(yScale)
.orient("left");
//.ticks(5);
var element = d3.select(this);
//Create X axis
element.append("g")
.attr("class", "axis x-axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
//Create Y axis
element.append("g")
.attr("class", "axis y-axis")
.call(yAxis);
limits = {"x":[minmax["x"]["min"],minmax["x"]["max"]],"y":[minmax["y"]["min"],minmax["y"]["max"]]}
for (k in lines) {
ps = linecross(lines[k],limits);
if (ps.length == 2) {
lines[k].x1 = ps[0][0];
lines[k].y1 = ps[0][1];
lines[k].x2 = ps[1][0];
lines[k].y2 = ps[1][1];
}
way = get_sign(lines[k].b,lines[k].n1,lines[k].n2);
lines[k].path =[corners(lines[k],limits,way),corners(lines[k],limits,-1*way)];
}
var line = element.selectAll ('.line')
.data(lines)
.enter()
.append("line")
.attr("x1",function(d) {return xScale(d.x1)})
.attr("y1",function(d) {return yScale(d.y1)})
.attr("x2",function(d) {return xScale(d.x2)})
.attr("y2",function(d) {return yScale(d.y2)})
.attr("id", function (d, i) {return "q-" + i;})
.attr("class", function(d) {
if (typeof(d['class'] != 'undefined')) return d['class'];
else return 'line';
})
.on('mouseover', tip.show)
.on('mouseout', tip.hide);
element.selectAll(".circle")
.data(data)
.enter()
.append("circle")
.attr("cx", function(d) {
return xScale(d.x);
})
.attr("cy", function(d) {
return yScale(d.y);
})
.attr("r", function(d) {
return rScale(d.r);
})
.attr("class", function(d) {
if (typeof(d['class'] != 'undefined')) return d['class'];
else return 'circle';
})
.attr("fill",function(d) {return d.color})
.attr("stroke",function(d) {return d.color})
.attr("fill-opacity",0.33)
.on('mouseover', tip.show)
.on('mouseout', tip.hide);
//axis labels
element.append("text")
.attr("class", "x-label label")
.attr("text-anchor", "end")
.attr("x", width)
.attr("y", height-5)
.text(axes['labels']['x']);
element.append("text")
.attr("class", "y label")
.attr("text-anchor", "end")
.attr("y", 5)
.attr("x", 0)
.attr("dy", ".75em")
.attr("transform", "rotate(-90)")
.text(axes['labels']['y']);
});
}
scatterplotwithlineplot.data = function(value) {
if (!arguments.length) return value;
data = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.lines = function(value) {
if (!arguments.length) return value;
lines = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.margin = function(value) {
if (!arguments.length) return value;
margin = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.axes = function(value) {
if (!arguments.length) return value;
axes = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.minmax = function(value) {
if (!arguments.length) return value;
minmax = value;
return scatterplotwithlineplot;
};
scatterplotwithlineplot.size = function(value) {
if (!arguments.length) return value;
size = value;
return scatterplotwithlineplot;
};
return scatterplotwithlineplot;
function corners(l,e,o) {
//l = {"a":0,"b":1} //line, a and slope, i.e., y=a+bx
//e = {"x": [-10,10], "y": [-10,10]} //limits
//o = 1 //orientation -1 or 1
//crossing x0, x1
//crossing y0, y1
outp = linecross (l,e);
out = [];
//vertices
for (i=0;i<=1;i++){
for (j=0;j<=1;j++){
if (o*(l.a+l.b*e.x[i]-e.y[j]) > 0)
outp.push([e.x[i],e.y[j]]);
}
}
//sort the outps, anticlockwise
if (outp.length > 0) {
mid = [0,0];
for (i in outp) {
mid[0] += outp[i][0];
mid[1] += outp[i][1];
}
mid[0] = mid[0] / outp.length;
mid[1] = mid[1] / outp.length;
for (i in outp) {
p = outp[i][1] - mid[1];
q = outp[i][0] - mid[0];
if (q != 0)
outp[i][2] = Math.atan(p/q) + (q<0 ? Math.PI : 0);
else
outp[i][2] = Math.PI/2 + Math.PI*sign(p);
}
outp = outp.sort(function(w,z) {
return w[2] > z[2];
});
for (i in outp) {
outp[i].splice(2,1);
out.push({"x":outp[i][0],"y":outp[i][1]});
}
}
return out;
}
function linecross (l,e) {
out = [];
//crossing x0, x1
for (i=0;i<=1;i++){
Y = l.a + l.b*e.x[i];
if ((Y > e.y[0]) && (Y < e.y[1]))
out.push([e.x[i],Y]);
}
//crossing y0, y1
for (j=0;j<=1;j++){
if (l.b != 0) {
X = (e.y[j] - l.a)/l.b;
if ((X > e.x[0]) && (X < e.x[1]))
out.push([X,e.y[j]]);
}
}
return out;
}
function get_sign(b,d1,d2) {
t = b*d1-d2;
if (t > 0) return 1;
if (t < 0) return -1;
return 0;
}
}
// d3.tip
// Copyright (c) 2013 Justin Palmer
//
// Tooltips for d3.js SVG visualizations
(function (root, factory) {
if (typeof define === 'function' && define.amd) {
// AMD. Register as an anonymous module with d3 as a dependency.
define(['d3'], factory)
} else if (typeof module === 'object' && module.exports) {
// CommonJS
module.exports = function(d3) {
d3.tip = factory(d3)
return d3.tip
}
} else {
// Browser global.
root.d3.tip = factory(root.d3)
}
}(this, function (d3) {
// Public - contructs a new tooltip
//
// Returns a tip
return function() {
var direction = d3_tip_direction,
offset = d3_tip_offset,
html = d3_tip_html,
node = initNode(),
svg = null,
point = null,
target = null
function tip(vis) {
svg = getSVGNode(vis)
point = svg.createSVGPoint()
document.body.appendChild(node)
}
// Public - show the tooltip on the screen
//
// Returns a tip
tip.show = function() {
var args = Array.prototype.slice.call(arguments)
if(args[args.length - 1] instanceof SVGElement) target = args.pop()
var content = html.apply(this, args),
poffset = offset.apply(this, args),
dir = direction.apply(this, args),
nodel = d3.select(node),
i = directions.length,
coords,
scrollTop = document.documentElement.scrollTop || document.body.scrollTop,
scrollLeft = document.documentElement.scrollLeft || document.body.scrollLeft
nodel.html(content)
.style({ opacity: 1, 'pointer-events': 'all' })
while(i--) nodel.classed(directions[i], false)
coords = direction_callbacks.get(dir).apply(this)
nodel.classed(dir, true).style({
top: (coords.top + poffset[0]) + scrollTop + 'px',
left: (coords.left + poffset[1]) + scrollLeft + 'px'
})
return tip
}
// Public - hide the tooltip
//
// Returns a tip
tip.hide = function() {
var nodel = d3.select(node)
nodel.style({ opacity: 0, 'pointer-events': 'none' })
return tip
}
// Public: Proxy attr calls to the d3 tip container. Sets or gets attribute value.
//
// n - name of the attribute
// v - value of the attribute
//
// Returns tip or attribute value
tip.attr = function(n, v) {
if (arguments.length < 2 && typeof n === 'string') {
return d3.select(node).attr(n)
} else {
var args = Array.prototype.slice.call(arguments)
d3.selection.prototype.attr.apply(d3.select(node), args)
}
return tip
}
// Public: Proxy style calls to the d3 tip container. Sets or gets a style value.
//
// n - name of the property
// v - value of the property
//
// Returns tip or style property value
tip.style = function(n, v) {
if (arguments.length < 2 && typeof n === 'string') {
return d3.select(node).style(n)
} else {
var args = Array.prototype.slice.call(arguments)
d3.selection.prototype.style.apply(d3.select(node), args)
}
return tip
}
// Public: Set or get the direction of the tooltip
//
// v - One of n(north), s(south), e(east), or w(west), nw(northwest),
// sw(southwest), ne(northeast) or se(southeast)
//
// Returns tip or direction
tip.direction = function(v) {
if (!arguments.length) return direction
direction = v == null ? v : d3.functor(v)
return tip
}
// Public: Sets or gets the offset of the tip
//
// v - Array of [x, y] offset
//
// Returns offset or
tip.offset = function(v) {
if (!arguments.length) return offset
offset = v == null ? v : d3.functor(v)
return tip
}
// Public: sets or gets the html value of the tooltip
//
// v - String value of the tip
//
// Returns html value or tip
tip.html = function(v) {
if (!arguments.length) return html
html = v == null ? v : d3.functor(v)
return tip
}
function d3_tip_direction() { return 'n' }
function d3_tip_offset() { return [0, 0] }
function d3_tip_html() { return ' ' }
var direction_callbacks = d3.map({
n: direction_n,
s: direction_s,
e: direction_e,
w: direction_w,
nw: direction_nw,
ne: direction_ne,
sw: direction_sw,
se: direction_se
}),
directions = direction_callbacks.keys()
function direction_n() {
var bbox = getScreenBBox()
return {
top: bbox.n.y - node.offsetHeight,
left: bbox.n.x - node.offsetWidth / 2
}
}
function direction_s() {
var bbox = getScreenBBox()
return {
top: bbox.s.y,
left: bbox.s.x - node.offsetWidth / 2
}
}
function direction_e() {
var bbox = getScreenBBox()
return {
top: bbox.e.y - node.offsetHeight / 2,
left: bbox.e.x
}
}
function direction_w() {
var bbox = getScreenBBox()
return {
top: bbox.w.y - node.offsetHeight / 2,
left: bbox.w.x - node.offsetWidth
}
}
function direction_nw() {
var bbox = getScreenBBox()
return {
top: bbox.nw.y - node.offsetHeight,
left: bbox.nw.x - node.offsetWidth
}
}
function direction_ne() {
var bbox = getScreenBBox()
return {
top: bbox.ne.y - node.offsetHeight,
left: bbox.ne.x
}
}
function direction_sw() {
var bbox = getScreenBBox()
return {
top: bbox.sw.y,
left: bbox.sw.x - node.offsetWidth
}
}
function direction_se() {
var bbox = getScreenBBox()
return {
top: bbox.se.y,
left: bbox.e.x
}
}
function initNode() {
var node = d3.select(document.createElement('div'))
node.style({
position: 'absolute',
top: 0,
opacity: 0,
'pointer-events': 'none',
'box-sizing': 'border-box'
})
return node.node()
}
function getSVGNode(el) {
el = el.node()
if(el.tagName.toLowerCase() === 'svg')
return el
return el.ownerSVGElement
}
// Private - gets the screen coordinates of a shape
//
// Given a shape on the screen, will return an SVGPoint for the directions
// n(north), s(south), e(east), w(west), ne(northeast), se(southeast), nw(northwest),
// sw(southwest).
//
// +-+-+
// | |
// + +
// | |
// +-+-+
//
// Returns an Object {n, s, e, w, nw, sw, ne, se}
function getScreenBBox() {
var targetel = target || d3.event.target;
while ('undefined' === typeof targetel.getScreenCTM && 'undefined' === targetel.parentNode) {
targetel = targetel.parentNode;
}
var bbox = {},
matrix = targetel.getScreenCTM(),
tbbox = targetel.getBBox(),
width = tbbox.width,
height = tbbox.height,
x = tbbox.x,
y = tbbox.y
point.x = x
point.y = y
bbox.nw = point.matrixTransform(matrix)
point.x += width
bbox.ne = point.matrixTransform(matrix)
point.y += height
bbox.se = point.matrixTransform(matrix)
point.x -= width
bbox.sw = point.matrixTransform(matrix)
point.y -= height / 2
bbox.w = point.matrixTransform(matrix)
point.x += width
bbox.e = point.matrixTransform(matrix)
point.x -= width / 2
point.y -= height / 2
bbox.n = point.matrixTransform(matrix)
point.y += height
bbox.s = point.matrixTransform(matrix)
return bbox
}
return tip
};
}));
{"name": "cz-praha-2018-2022-roll-call-votes", "description": "Roll-call votes from Prague Assembly 2018-2022", "resources": [{"name": "voters", "path": "data/voters.csv", "schema": {"fields": [{"name": "id", "type": "integer"}, {"name": "name", "type": "string"}, {"name": "party", "type": "string"}, {"name": "email", "type": "string"}], "primaryKey": "name"}}, {"name": "vote_events", "path": "data/vote_events.csv", "schema": {"fields": [{"name": "id", "type": "integer"}, {"name": "start_date", "type": "string", "format": "YYYY-mm-dd"}, {"name": "motion:name", "type": "string"}, {"name": "motion:number", "type": "string"}, {"name": "motion:document", "type": "string"}, {"name": "sources:link:url", "type": "string"}, {"name": "legislative_session_id", "type": "string"}, {"name": "result", "type": "string"}, {"name": "counts:option:yes", "type": "integer"}, {"name": "counts:option:no", "type": "integer"}, {"name": "counts:option:abstain", "type": "integer"}, {"name": "number_of_people", "type": "integer"}, {"name": "present", "type": "integer"}, {"name": "identifier", "type": "integer"}], "primaryKey": "id"}}, {"name": "votes", "path": "data/votes.csv", "schema": {"fields": [{"name": "vote_event_id", "type": "integer"}, {"name": "voter_id", "type": "integer"}, {"name": "option", "type": "string"}], "primaryKey": ["vote_event_id", "voter_id"]}}]}
"""Downloads the datapackage from Github."""
import json
import requests
path = "/home/michal/dev/wpca/praha_2018-2019/"
url_path = "https://raw.githubusercontent.com/michalskop/praha.eu-scraper/master/data/2018-2022/"
with open(path + "datapackage.json", "w") as fout:
url = url_path + "datapackage.json"
r = requests.get(url)
dpobj = r.json()
json.dump(dpobj, fout)
for resource in dpobj['resources']:
with open(path + resource['path'], "w") as fout:
url = url_path + resource['path']
r = requests.get(url)
fout.write(r.text)
"""Loads datapackage."""
import csv
import json
import operator
path = "/home/michal/dev/wpca/praha_2018-2019/"
# global data
# print(anal)
datapackage = json.loads(open(path + 'datapackage' + anal['name'] + '.json').read())
# print(path + 'datapackage' + anal['name'] + '.json')
def _row2header(row):
header = {}
j = 0
for h in row:
header[h] = j
j = j + 1
return header
def _row2data(row, header):
out = {}
for h in header:
out[h] = row[header[h]]
return out
# extract info from datapackage
dp = {}
for resource in datapackage['resources']:
# print(resource["name"])
if resource["name"] == "voters":
dp["voters"] = resource
if resource["name"] == ("vote_events"):
dp["vote_events"] = resource
if resource["name"] == ("votes"):
dp["votes"] = resource
# load into variable
headers = {}
data = {"voters": [], "votes": [], "vote_events": []}
for key in data:
i = 0
with open(path + dp[key]["path"], 'r') as csvfile:
csvreader = csv.reader(csvfile)
for row in csvreader:
if i == 0:
headers[key] = _row2header(row)
else:
data[key].append(_row2data(row, headers[key]))
i = i + 1
# reorder for easier access:
people = {}
for person in data["voters"]:
people[person['id']] = person
vote_events = {}
for vote_event in data["vote_events"]:
vote_events[vote_event['id']] = vote_event
# sort votes
data["votes"] = sorted(data["votes"], key=operator.itemgetter("vote_event_id", "voter_id"))
# print(data.keys())
"party","abbreviation","color","rotation:d1","rotation:d2"
"Piráti","Piráti",,1,1
"PRAHA SOBĚ","KDU-ČSL",,,
"TOP 09 a STAN - Spojené síly pro Prahu","TOP 09",,,
"ODS","ODS",,,
"ANO 2011","ANO",,,
"""Prepares data for chart."""
import csv
import io
import math
import requests
path = "/home/michal/dev/wpca/praha_2018-2019/"
fname = "cz-praha-2018-2022-roll-call-votes-wpca.csv"
# data about parties from Github:
url = "https://raw.githubusercontent.com/michalskop/political_parties/master/cz/parties.csv"
# get info about parties from github
r = requests.get(url)
r.encoding = 'utf-8' # useful if encoding is not sent (or not sent properly) by the server
csvio = io.StringIO(r.text, newline="")
political_parties = []
for row in csv.DictReader(csvio):
political_parties.append(row)
def _get_color(abbreviation):
for pp in political_parties:
if pp['abbreviation'] == abbreviation:
return pp['color']
return '#888888'
# get colors forI don't know why it's not a built-in, but I have some thoughts. local parties
local_parties = {}
with open(path + "parties.csv") as fin:
dr = csv.DictReader(fin)
for row in dr:
if row['color'] == '':
row['color'] = _get_color(row['abbreviation'])
local_parties[row['party']] = row
# parties for voters
voters = {}
with open(path + "data/voters.csv") as fin:
dr = csv.DictReader(fin)
for row in dr:
voters[row['id']] = row
# data
data = []
with open(path + fname) as fin:
dr = csv.DictReader(fin)
for row in dr:
row['party'] = voters[row['id']]['party']
row['color'] = local_parties[row['party']]['color']
data.append(row)
header = row.keys()
# rotations
for k in local_parties:
if local_parties[k]['rotation:d1'] != '':
rotation = local_parties[k]
for row in data:
if row['party'] == rotation['party']:
r1 = math.copysign(1, float(row['wpca:d1'])) * int(rotation['rotation:d1'])
r2 = math.copysign(1, float(row['wpca:d2'])) * int(rotation['rotation:d2'])
break
for row in data:
row['wpca:d1'] = r1 * float(row['wpca:d1'])
row['wpca:d2'] = r2 * float(row['wpca:d2'])
with open(path + "chart_data.csv", "w") as fout:
dw = csv.DictWriter(fout, header)
dw.writeheader()
for row in data:
dw.writerow(row)
Xsource = matrix(Xsourcevector,ncol=3,byrow=T)
dimnames(Xsource)[[2]] = c('vote_event_id','voter_id','option')
Xsource = as.data.frame(Xsource)
"""Calculates WPCA."""
import csv
import json
import numpy
import rpy2.robjects as robjects
r = robjects.r
path = "/home/michal/dev/wpca/praha_2018-2019/"
data = {}
people = {}
datapackage = {}
anals = [
# {
# "name": "_vlada1",
# "min": 67018,
# "max": 67896
# },
# {
# "name": "_vlada2",
# "min": 67896,
# "max": 69449 # 16.3.2019
# },
# {
# "name": "_left",
# "min": 69449,
# "max": 70880
# },
{
"name": "",
"min": 0,
"max": 1000000
}
]
for anal in anals:
exec(open(path + "load_datapackage.py").read()), globals()
# load into row vector in R
Xsourcevector = []
'data' in locals()
# print(data.keys())
for row in data["votes"]:
Xsourcevector.append(row["vote_event_id"])
Xsourcevector.append(row["voter_id"])
Xsourcevector.append(row["option"])
robjects.globalenv["Xsourcevector"] = robjects.StrVector(Xsourcevector)
r.source(path + "prepare_matrix.r")
# calculate WPCA
# check wpca_script.r - loading correct library reshape (library(reshape2, ...))
r.source(path + "wpca_script.r")
# save WPCA
with open(path + datapackage['name'] + '-wpca.csv', 'w') as csvfile:
csvwriter = csv.writer(csvfile)
csvwriter.writerow(["id", "name", "wpca:d1", "wpca:d2", "wpca:d3"])
i = 0 # all people
k = 0 # cutted people
rXproju = numpy.array(robjects.globalenv['Xproju'])
rXpeople = robjects.globalenv['Xpeople']
rXvote_events = robjects.globalenv['Xvote_events']
for item in robjects.globalenv['pI']:
if item:
try:
csvwriter.writerow([people[rXpeople[i]]['id'], people[rXpeople[i]]['name'], rXproju[k, 0], rXproju[k, 1], rXproju[k, 2]])
except Exception:
nothing = 0
k = k + 1
i = i + 1
names = r('rownames(Corr0)')
rdata = r('Corr0')
k = 0
dta = {}
for i in range(0, len(names)):
for j in range(0, len(names)):
if names[i] not in dta.keys():
dta[names[i]] = {}
dta[names[i]][names[j]] = rdata[k]
k += 1
with open(path + "correlation" + anal['name'] + ".json", "w") as fout:
json.dump(dta, fout)
# INPUT PARAMETERS
# _X_RAW_DB, _LO_LIMIT_1
# raw data in csv using db structure, i.e., a single row contains:
# code of representative, code of division, encoded vote (i.e. one of -1, 1, 0, NA)
# for example:
# “Joe Europe”,”Division-007”,”1”
#Xsource = read.csv("data/votes.csv")
Xsource$vote_event_id = as.factor(Xsource$vote_event_id)
Xsource$voter_id = as.factor(Xsource$voter_id)
#Xsource = Xsource[c("voter_id","vote_event_id","option")]
Xsource$option_numeric = rep(0,length(Xsource$option))
Xsource$option_numeric[Xsource$option=='yes'] = 1
Xsource$option_numeric[Xsource$option=='no'] = -1
Xsource$option_numeric[Xsource$option=='abstain'] = -1
Xsource$option_numeric[Xsource$option=='not voting'] = NA
Xsource$option_numeric[Xsource$option=='absent'] = NA
Xsource$option_numeric = as.numeric(Xsource$option_numeric)
#Xrawdb = _X_RAW_DB
Xrawdb = Xsource
# lower limit to eliminate from calculations, e.g., .1; number
lo_limit = .1
# reorder data; divisions x persons
# we may need to install and/or load some additional libraries
# install.packages("reshape2")
library("reshape2")
# library("reshape2", lib.loc="/home/michal/R/x86_64-pc-linux-gnu-library/3.6")
# install.packages("sqldf")
# library("sqldf")
#prevent reordering, which is behaviour of acast:
#Xrawdb$V1 = factor(Xrawdb$V1, levels=unique(Xrawdb$V1))
Xrawdb$voter_id = factor(Xrawdb$voter_id, levels=unique(Xrawdb$voter_id))
Xraw = t(acast(Xrawdb,voter_id~vote_event_id,fun.aggregate=mean,value.var='option_numeric'))
Xpeople = dimnames(Xraw)[[2]]
Xvote_events = dimnames(Xraw)[[1]]
# Xraw=apply(Xraw,1,as.numeric)
mode(Xraw) = 'numeric'
# scale data; divisions x persons (mean=0 and sd=1 for each division)
Xstand=t(scale(t(Xraw),scale=TRUE))
# WEIGHTS
# weights 1 for divisions, based on number of persons in division
w1 = apply(abs(Xraw)==1,1,sum,na.rm=TRUE)/max(apply(abs(Xraw)==1,1,sum,na.rm=TRUE))
w1[is.na(w1)] = 0
# weights 2 for divisions, "100:100" vs. "195:5"
w2 = 1 - abs(apply(Xraw==1,1,sum,na.rm=TRUE) - apply(Xraw==-1,1,sum,na.rm=TRUE))/apply(!is.na(Xraw),1,sum)
w2[is.na(w2)] = 0
# analytical charts for weights:
#plot(w1)
#plot(w2)
#plot(w1*w2)
# weighted scaled matrix; divisions x persons
X = Xstand * w1 * w2
# MISSING DATA
# index of missing data; divisions x persons
I = X
I[!is.na(X)] = 1
I[is.na(X)] = 0
# weighted scaled with NA substituted by 0; division x persons
X0 = X
X0[is.na(X)]=0
# EXCLUSION OF REPRESENTATIVES WITH TOO FEW VOTES (WEIGHTED)
# weights for non missing data; division x persons
Iw = I*w1*w2
# sum of weights of divisions for each persons; vector of length “persons”
s = apply(Iw,2,sum)
pw = s/(t(w1)%*%w2)
# index of persons kept in calculation; vector of length “persons”
pI = pw > lo_limit
# weighted scaled with NA->0 and cutted persons with too few weighted votes; division x persons
X0c = X0[,pI]
# index of missing cutted (excluded) persons with too few weighted votes; divisions x persons
Ic = I[,pI]
# indexes of cutted (excluded) persons with too few votes; divisions x persons
Iwc = Iw[,pI]
# “X’X” MATRIX
# weighted X’X matrix with missing values substituted and excluded persons; persons x persons
C=t(X0c)%*%X0c * 1/(t(Iwc)%*%Iwc) * (sum(w1*w1*w2*w2))
# substitution of missing data in "covariance" matrix (the simple way)
C0 = C
C0[is.na(C)] = 0
# DECOMPOSITION
# eigendecomposition
Xe=eigen(C0)
# W (rotation values of persons)
W = Xe$vectors
# projected divisions into dimensions
Xy=X0c%*%W
# analytical charts of projection of divisions and lambdas
#plot(Xy[,1],Xy[,2])
#plot(sqrt(Xe$values[1:10]))
# lambda matrix
sigma = sqrt(Xe$values)
sigma[is.na(sigma)] = 0
lambda = diag(sigma)
# unit scaled lambda matrix
lambdau = sqrt(lambda^2/sum(lambda^2))
# projection of persons into dimensions
Xproj = W%*%lambda
# scaled projection of persons into dimensions
Xproju = W%*%lambdau*sqrt(dim(W)[1])
# analytical chart
# plot(Xproj[,1],Xproj[,2])
# plot(Xproju[,1],Xproju[,2])
# lambda^-1 matrix
lambda_1 = diag(sqrt(1/Xe$values))
lambda_1[is.na(lambda_1)] = 0
# Z (rotation values of divisions)
Z = X0c%*%W%*%lambda_1
# analytical charts
# second projection
Xproj2 = t(X0c) %*% Z
# without missing values, they are equal:
#plot(Xproj[,1],Xproj2[,1])
#plot(Xproj[,2],Xproj2[,2])
Xproj3u = W%*%lambdau * sqrt(lambda[1,1])
Xy3u=X0c%*%W / sqrt(lambda[1,1])
# "correlation matrix" (for charting)
invSC0 = solve(diag(sqrt(diag(C0))))
Corr0 = invSC0 %*% C0 %*% invSC0
rownames(Corr0) = rownames(C0)
colnames(Corr0) = colnames(C0)
# save into JSON file:
# setwd("home/michal/dev/wpca/psp/cs-2017-2019/")
# library("jsonlite", lib.loc="/home/michal/R/x86_64-pc-linux-gnu-library/3.4")
# sink(file="correlation_from_r.json")
# serializeJSON(as.data.frame(Corr0))
# sink()