block by nitaku 633fffc54e45289175d7

Term frequencies (Cassandra)

Full Screen

FIXME undefined data points are not represented correctly

index.js

// Generated by CoffeeScript 1.10.0
(function() {
  var END, MARGIN, MARGIN_BOTTOM, MARGIN_LEFT, START, STEP, chart, chart_height, chart_width, forum, height, keyword, line_generator, redraw, svg, width, x, y;

  MARGIN = 20;

  MARGIN_LEFT = 20;

  MARGIN_BOTTOM = 40;

  keyword = 'cocaine';

  forum = 'drugsforum';

  START = 946598400;

  END = 1450310400;

  STEP = 604800;

  svg = d3.select('svg');

  width = svg.node().getBoundingClientRect().width;

  height = svg.node().getBoundingClientRect().height;

  chart_width = width - 2 * MARGIN - MARGIN_LEFT;

  chart_height = height - 2 * MARGIN - MARGIN_BOTTOM;

  chart = svg.append('g').attr({
    transform: "translate(" + (MARGIN + MARGIN_LEFT) + " " + MARGIN + ")"
  });

  x = d3.time.scale().range([0, chart_width]);

  y = d3.scale.linear().range([chart_height, 0]);

  line_generator = d3_shape.area().x(function(d) {
    return x(d.date);
  }).y1(function(d) {
    return y(d.value);
  }).y0(function(d) {
    return y(0);
  }).curve(d3_shape.stepAfter);

  redraw = function() {
    d3.select('body').classed('wait', true);
    return d3.json("http://wafi.iit.cnr.it/cas-scraper2/api/timeseries/getTermTs.php?db=" + forum + "&term=" + keyword + "&start=" + START + "&end=" + END + "&step=" + STEP, function(result) {
      var data, end, index, start, xAxis, yAxis;
      d3.select('body').classed('wait', false);
      start = new Date(START * 1000);
      end = new Date(END * 1000 + (STEP * 1000));
      index = {};
      result.data.forEach(function(d) {
        d.date = 1000 * d.date;
        return index[d.date] = d;
      });
      data = d3.range(+start, +end, STEP * 1000).map(function(k) {
        if (k in index) {
          return index[k];
        } else {
          return {
            date: new Date(k),
            value: 0
          };
        }
      });
      data.forEach(function(d) {
        return d.date = new Date(d.date);
      });
      x.domain([start, end]);
      y.domain([
        0, d3.max(data, function(d) {
          return d.value;
        })
      ]);
      data.push({
        date: end,
        value: null
      });
      xAxis = d3.svg.axis().orient("bottom").ticks(d3.time.year, 1).scale(x);
      yAxis = d3.svg.axis().orient("left").scale(y);
      chart.selectAll('*').remove();
      chart.append("g").attr("class", "xaxis").attr("transform", "translate(0," + (chart_height + 0.5) + ")").call(xAxis);
      chart.append("g").attr("class", "yaxis").call(yAxis);
      return chart.append('path').datum(data).attr({
        "class": 'area',
        d: line_generator,
        fill: 'orange'
      });
    });
  };

  d3.select("#keyword").node().value = keyword;

  d3.select("#keyword").on('keyup', function() {
    if (d3.event.keyCode === 13) {
      keyword = this.value.replace(/ and /gi, "%2BAND%2B").replace(/ or /gi, "%2BOR%2B").replace(/-/g, "%2D").replace(/ /g, "%2B");
      return redraw();
    }
  });

  d3.select("#forum_ctrl").on("change", function() {
    forum = this.options[this.selectedIndex].value;
    return redraw();
  });

  redraw();

}).call(this);

index.html

<!doctype html>
<html lang="en">
  <head>
    <meta charset="utf-8">
    <title>Term frequencies (Cassandra)</title>
    <link rel="stylesheet" href="index.css">
    <script src="//d3js.org/d3.v3.min.js"></script>
    <script src="d3-path.js"></script>
    <script src="d3-shape.js"></script>
  </head>
  <body>
    <div>
      <label>Select source:</label>
      <select id="forum_ctrl">
        <option value="drugsforum">Drugs-forum</option>
        <option value="bluelight">Bluelight</option>
        <option value="tweets_NPS">Tweets</option>
      </select>
    </div>
    <div id="search">
      <label>Keyword:</label>
      <input type="search" id="keyword">
    </div>
    <svg></svg>
    <script src="index.js"></script>
  </body>
</html>

d3-path.js

(function (global, factory) {
  typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :
  typeof define === 'function' && define.amd ? define('d3-path', ['exports'], factory) :
  factory((global.d3_path = {}));
}(this, function (exports) { 'use strict';

  var pi = Math.PI;
  var tau = 2 * pi;
  var epsilon = 1e-6;
  var tauEpsilon = tau - epsilon;
  function Path() {
    this._x0 = this._y0 = // start of current subpath
    this._x1 = this._y1 = null; // end of current subpath
    this._ = [];
  }

  function path() {
    return new Path;
  }

  Path.prototype = path.prototype = {
    moveTo: function(x, y) {
      this._.push("M", this._x0 = this._x1 = +x, ",", this._y0 = this._y1 = +y);
    },
    closePath: function() {
      if (this._x1 !== null) {
        this._x1 = this._x0, this._y1 = this._y0;
        this._.push("Z");
      }
    },
    lineTo: function(x, y) {
      this._.push("L", this._x1 = +x, ",", this._y1 = +y);
    },
    quadraticCurveTo: function(x1, y1, x, y) {
      this._.push("Q", +x1, ",", +y1, ",", this._x1 = +x, ",", this._y1 = +y);
    },
    bezierCurveTo: function(x1, y1, x2, y2, x, y) {
      this._.push("C", +x1, ",", +y1, ",", +x2, ",", +y2, ",", this._x1 = +x, ",", this._y1 = +y);
    },
    arcTo: function(x1, y1, x2, y2, r) {
      x1 = +x1, y1 = +y1, x2 = +x2, y2 = +y2, r = +r;
      var x0 = this._x1,
          y0 = this._y1,
          x21 = x2 - x1,
          y21 = y2 - y1,
          x01 = x0 - x1,
          y01 = y0 - y1,
          l01_2 = x01 * x01 + y01 * y01;

      // Is the radius negative? Error.
      if (r < 0) throw new Error("negative radius: " + r);

      // Is this path empty? Move to (x1,y1).
      if (this._x1 === null) {
        this._.push(
          "M", this._x1 = x1, ",", this._y1 = y1
        );
      }

      // Or, is (x1,y1) coincident with (x0,y0)? Do nothing.
      else if (!(l01_2 > epsilon));

      // Or, are (x0,y0), (x1,y1) and (x2,y2) collinear?
      // Equivalently, is (x1,y1) coincident with (x2,y2)?
      // Or, is the radius zero? Line to (x1,y1).
      else if (!(Math.abs(y01 * x21 - y21 * x01) > epsilon) || !r) {
        this._.push(
          "L", this._x1 = x1, ",", this._y1 = y1
        );
      }

      // Otherwise, draw an arc!
      else {
        var x20 = x2 - x0,
            y20 = y2 - y0,
            l21_2 = x21 * x21 + y21 * y21,
            l20_2 = x20 * x20 + y20 * y20,
            l21 = Math.sqrt(l21_2),
            l01 = Math.sqrt(l01_2),
            l = r * Math.tan((pi - Math.acos((l21_2 + l01_2 - l20_2) / (2 * l21 * l01))) / 2),
            t01 = l / l01,
            t21 = l / l21;

        // If the start tangent is not coincident with (x0,y0), line to.
        if (Math.abs(t01 - 1) > epsilon) {
          this._.push(
            "L", x1 + t01 * x01, ",", y1 + t01 * y01
          );
        }

        this._.push(
          "A", r, ",", r, ",0,0,", +(y01 * x20 > x01 * y20), ",", this._x1 = x1 + t21 * x21, ",", this._y1 = y1 + t21 * y21
        );
      }
    },
    arc: function(x, y, r, a0, a1, ccw) {
      x = +x, y = +y, r = +r;
      var dx = r * Math.cos(a0),
          dy = r * Math.sin(a0),
          x0 = x + dx,
          y0 = y + dy,
          cw = 1 ^ ccw,
          da = ccw ? a0 - a1 : a1 - a0;

      // Is the radius negative? Error.
      if (r < 0) throw new Error("negative radius: " + r);

      // Is this path empty? Move to (x0,y0).
      if (this._x1 === null) {
        this._.push(
          "M", x0, ",", y0
        );
      }

      // Or, is (x0,y0) not coincident with the previous point? Line to (x0,y0).
      else if (Math.abs(this._x1 - x0) > epsilon || Math.abs(this._y1 - y0) > epsilon) {
        this._.push(
          "L", x0, ",", y0
        );
      }

      // Is this arc empty? We’re done.
      if (!r) return;

      // Is this a complete circle? Draw two arcs to complete the circle.
      if (da > tauEpsilon) {
        this._.push(
          "A", r, ",", r, ",0,1,", cw, ",", x - dx, ",", y - dy,
          "A", r, ",", r, ",0,1,", cw, ",", this._x1 = x0, ",", this._y1 = y0
        );
      }

      // Otherwise, draw an arc!
      else {
        if (da < 0) da = da % tau + tau;
        this._.push(
          "A", r, ",", r, ",0,", +(da >= pi), ",", cw, ",", this._x1 = x + r * Math.cos(a1), ",", this._y1 = y + r * Math.sin(a1)
        );
      }
    },
    rect: function(x, y, w, h) {
      this._.push("M", this._x0 = this._x1 = +x, ",", this._y0 = this._y1 = +y, "h", +w, "v", +h, "h", -w, "Z");
    },
    toString: function() {
      return this._.join("");
    }
  };

  var version = "0.1.2";

  exports.version = version;
  exports.path = path;

}));

index.coffee

MARGIN = 20
MARGIN_LEFT = 20
MARGIN_BOTTOM = 40
keyword = 'cocaine'
forum = 'drugsforum'

# FIXME se la API li cambiano, devono restituire i nuovi valori. Se non lo dicono il valore di end dev'essere una tacca più avanti, altrimenti manca l'ultimo valore.
#START = 915148800 #1999
START = 946598400 #2000
#START = 1041292800 #2003
END = 1450310400

#FIXME lo step non può essere fisso in secondi, ci vuole un'aggregazione per mesi o settimane veri
STEP = 604800 # 1 week
#STEP = 2592000 # 30 days
  
svg = d3.select('svg')
width = svg.node().getBoundingClientRect().width
height = svg.node().getBoundingClientRect().height

chart_width = width-2*MARGIN-MARGIN_LEFT
chart_height = height-2*MARGIN-MARGIN_BOTTOM

chart = svg.append 'g'
  .attr
    transform: "translate(#{MARGIN+MARGIN_LEFT} #{MARGIN})"

x = d3.time.scale()
  .range([0, chart_width])

y = d3.scale.linear()
  .range([chart_height, 0])

  
line_generator = d3_shape.area()
  .x (d) -> x(d.date)
  .y1 (d) -> y(d.value)
  .y0 (d) -> y(0)
  .curve d3_shape.stepAfter # step interpolator is the only one that can be applied after wrapping and wrap without error
  
redraw = () ->
  d3.select('body').classed 'wait', true
  d3.json "http://wafi.iit.cnr.it/cas-scraper2/api/timeseries/getTermTs.php?db=#{forum}&term=#{keyword}&start=#{START}&end=#{END}&step=#{STEP}", (result) ->
    d3.select('body').classed 'wait', false
    
    start = new Date(START*1000)
    end = new Date(END*1000+(STEP*1000))
    
    index = {}
    result.data.forEach (d) ->
      d.date = 1000*d.date
      index[d.date] = d
    
    data = d3.range(+start, +end, STEP*1000).map (k) ->
      if k of index
        return index[k]
      else
        return {date: new Date(k), value: 0}
      
    data.forEach (d) ->
      d.date = new Date(d.date)

    x
      .domain([start, end])
      
    y
      .domain([0, d3.max(data, (d) -> d.value)])
      
    # add a fake data point to extend the stepped curve to include the last value
    data.push {date: end, value: null}
    
    # define the x axis
    xAxis = d3.svg.axis()
      .orient("bottom")
      .ticks(d3.time.year, 1) # FIXME non si sa se d3 posiziona i tick secondo l'ora locale o secondo l'ora UTC. E se scrive considerando l'ora giusta o no.
      .scale(x)
      
    # define the y axis
    yAxis = d3.svg.axis()
      .orient("left")
#       .ticks(10)
      .scale(y)
    
    chart.selectAll('*')
      .remove()
      
      
    # draw x axis with labels and move to the bottom of the chart area
    chart.append("g")
      .attr "class", "xaxis"
      .attr "transform", "translate(0,#{chart_height+0.5})"
      .call(xAxis)
      
    # draw y axis with labels and move to the bottom of the chart area
    chart.append("g")
      .attr "class", "yaxis"
      .call(yAxis)
    
    chart.append('path')
      .datum data
      .attr
        class: 'area'
        d: line_generator
        fill: 'orange'
    
d3.select("#keyword").node().value = keyword

d3.select("#keyword").on 'keyup', () ->
  if(d3.event.keyCode == 13)
    keyword = this.value
      .replace(/ and /gi, "%2BAND%2B") # URL encoding of queries
      .replace(/ or /gi, "%2BOR%2B")
      .replace(/-/g, "%2D")
      .replace(/ /g, "%2B")
    redraw()
    
d3.select "#forum_ctrl"
   .on "change", () ->
      forum = this.options[this.selectedIndex].value
      redraw()
      
# d3.select "#date_start"
#    .on "change", () ->
#       START = +new Date(this.value)/1000
#       console.log START
#       redraw()
      
# d3.select "#date_end"
#    .on "change", () ->
#       END = +new Date(this.value)/1000
#       console.log END
#       redraw()

redraw()

index.css

body, html {
  padding: 0;
  margin: 0;
  width: 100%;
  height: 100%;
  font-family: sans-serif;
  font-size: 12px;
}
body.wait {
  cursor: progress;
}
svg {
  width: 100%;
  height: 100%;
  background: white;
}
.tick line {
  stroke: black;
  stroke-width:  1px;
}
.xaxis path, .axis line, .yaxis path {
  stroke-width:1px;
  fill: none;
  stroke: #000;
  shape-rendering: crispEdges;
  }

input{
  padding : 0 2px;
  margin  : 0;
  width   : 240px;
}

temp.json

{
"data": [
{
"date": 1042156800,
"value": 0
},
{
"date": 1044748800,
"value": 50
},
{
"date": 1047340800,
"value": 100
},
{
"date": 1049932800,
"value": 150
},
{
"date": 1052524800,
"value": 200
},
{
"date": 1055116800,
"value": 250
},
{
"date": 1057708800,
"value": 300
},
{
"date": 1060300800,
"value": 350
},
{
"date": 1062892800,
"value": 375
},
{
"date": 1065484800,
"value": 425
},
{
"date": 1068076800,
"value": 1000
},
{
"date": 1070668800,
"value": 1025
},
{
"date": 1073260800,
"value": 750
},
{
"date": 1075852800,
"value": 0
},
{
"date": 1078444800,
"value": 50
},
{
"date": 1081036800,
"value": 675
},
{
"date": 1083628800,
"value": 50
},
{
"date": 1086220800,
"value": 150
},
{
"date": 1088812800,
"value": 50
}],
"tot": 896311
}