This program fetches data from the Data Canvas - Sense Your City API.
It shows how the API can be accessed using async.js for asynchronous control flow. Data is fetched for each city separately, then combined, then output as CSV.
The API has a limitation of returning at most 1000 rows.
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<!-- Runs the main program found in main.js. -->
<script data-main="main.js" src="//cdnjs.cloudflare.com/ajax/libs/require.js/2.1.14/require.js"></script>
<!-- Configure RequireJS paths for third party libraries. -->
<script>
requirejs.config({
paths: {
d3: "//d3js.org/d3.v3.min",
jquery: "//code.jquery.com/jquery-2.1.1.min",
lodash: "//cdnjs.cloudflare.com/ajax/libs/lodash.js/3.4.0/lodash.min",
async: "//cdnjs.cloudflare.com/ajax/libs/async/0.9.0/async"
}
});
</script>
<!-- Include CSS that styles the visualization. -->
<link rel="stylesheet" href="styles.css">
<title>Fetching Data</title>
</head>
<body>
<!-- The visualization will be injected into this div. -->
<p>
day.csv - past day, every 5 minutes
<span class="loader" id="day-loading"><img src="ajax-loader.gif"></span> <br>
<textarea id="day" rows="15"></textarea>
</p>
<p>
week.csv - past week, every hour
<span class="loader" id="week-loading"><img src="ajax-loader.gif"></span> <br>
<textarea id="week" rows="15"></textarea>
</p>
<p>
month.csv - past month, every hour
<span class="loader" id="month-loading"><img src="ajax-loader.gif"></span> <br>
<textarea id="month" rows="15"></textarea>
</p>
<p>
all.csv - all available data from the API, every 5 minutes
<span class="loader" id="all-loading">
<img src="ajax-loader.gif">
<span id="all-counter"></span>
</span> <br>
<textarea id="all" rows="15"></textarea>
</p>
</body>
</html>
GIF89a � ��� ���BBB bbb������!�Created with ajaxload.info !�
!� NETSCAPE2.0 , 3���0�Ikc:�N�f E�1º���.`��q�-[9ݦ9 Jk�H !�
, 4���N�! � ���DqBQT`1 `LE[�| �u��a� ��C�%$* !�
, 6�2#+�AȐ̔V/�c �N�IBa��p �
̳� ƨ+Y����2�d�� !�
, 3�b%+�2���V_��� �! 1D�a �F��� bR]�= 08,�Ȥr9L !�
, 2�r'+J�d��L� &v�`\bT���� hYB)��@ �< �&,�ȤR� !�
, 3� 9�t�ڞ0� �!.B���W��1 sa��5���0� ���m)J !�
, 2���� ٜU]���qp�`��a��4��AF�0�`��
�@�1���Α !�
, 2���0�I�eBԜ)� � ��q10�ʰ�P�aVڥ ub��[� ; // This module provides an API layer above the
// Data Canvas - Sense Your City API described at
// http://map.datacanvas.org/#!/data
define(["jquery", "lodash", "async"], function ($, _, async){
// See API documentation at http://map.datacanvas.org/#!/data
var API_URL = "http://sensor-api.localdata.com/api/v1/aggregations.csv",
// List of all cities with available data.
cities = ["San Francisco", "Bangalore", "Boston", "Geneva", "Rio de Janeiro", "Shanghai", "Singapore"],
// The default parameters to pass into the API.
defaultParams = {
// Use averaging as the aggregation operator.
op: "mean",
// Include temperature only.
fields: "temperature,light,airquality_raw,sound,humidity,dust"
}
// Fetches the latest data for a given city.
function getDataForCity(options){
return function(city, callback){
// Get data for the last 24 hours.
// 1000 milliseconds/second, 60 seconds/minute, 5 minutes
var params = _.extend({
"over.city": city
}, options, defaultParams);
// Use jQuery to fetch the data.
// jQuery is used here rather than D3 because of its nice parameter syntax.
$.get(API_URL, params, function(csv) {
// Parse the CSV string.
callback(null, d3.csv.parse(csv, function(d){
// Parse ISO date strings into Date objects.
// d.date = new Date(d.timestamp);
// Parse strings into Numbers for numeric fields.
// d.temperature = +d.temperature;
//d.light = +d.light
//d.airquality_raw = +d.airquality_raw
//d.sound = +d.sound
//d.humidity = +d.humidity
//d.dust = +d.dust
return d;
}));
});
};
}
// Fetches the current temperature across all cities.
return function getData(options, callback){
async.map(cities, getDataForCity(options), function(err, results){
callback(err, _.flatten(results));
});
}
});
// This is the main program that fetches Data Canvas - Sense Your City API.
// Curran Kelleher March 2015
require(["d3", "getData", "async"], function (d3, getData, async) {
var now = Date.now(),
pastDay = now - 1000 * 60 * 60 * 24,
pastWeek = now - 1000 * 60 * 60 * 24 * 7,
pastMonth = now - 1000 * 60 * 60 * 24 * 7 * 4;
async.series([
function(callback){
d3.select("#day-loading").style("visibility", "visible");
getData({
resolution: "5m",
from: new Date(pastDay).toISOString(),
before: new Date(now).toISOString(),
},function(err, data){
d3.select("#day").text(toCSV(data));
d3.select("#day-loading").style("visibility", "hidden");
d3.select("#week-loading").style("visibility", "visible");
callback();
});
},
function(callback){
getData({
resolution: "1h",
from: new Date(pastWeek).toISOString(),
before: new Date(now).toISOString(),
},function(err, data){
d3.select("#week").text(toCSV(data));
d3.select("#week-loading").style("visibility", "hidden");
d3.select("#month-loading").style("visibility", "visible");
callback();
});
},
function(callback){
getData({
resolution: "1h",
from: new Date(pastMonth).toISOString(),
before: new Date(now).toISOString(),
},function(err, data){
d3.select("#month").text(toCSV(data));
d3.select("#month-loading").style("visibility", "hidden");
callback();
});
},
function(callback){
// 1 day in milliseconds.
var msInterval = 1000 * 60 * 60 * 24,
// Start fetching data from right now.
msTime = Date.now(),
// Get data from back in time 100 days.
intervalCountMax = 100,
// The running count of days we've fetched data for.
intervalCount = 0,
// The cumulative data as an array of objects.
data = [];
d3.select("#all-loading").style("visibility", "visible");
async.whilst(
function () { return intervalCount < intervalCountMax; },
function (cb) {
getData({
resolution: "5m",
from: new Date(msTime - msInterval * (intervalCount + 1)).toISOString(),
before: new Date(msTime - msInterval * intervalCount).toISOString()
},function(err, newData){
data = _.sortBy(_.union(data, newData), "timestamp");
d3.select("#all-counter").text([
"Fetched " + data.length + " records",
"over " + (intervalCount + 1) + " of " + intervalCountMax + " days"
].join("\n"));
intervalCount++;
cb();
});
}, function(err){
d3.select("#all").text(toCSV(data));
d3.select("#all-loading").style("visibility", "hidden");
console.log("Done fetching data");
}
);
callback();
}
]);
function toCSV(data){
var columns = Object.keys(data[0]);
return [columns.join(",")].concat(data.map(function(d){
return columns.map(function(column){
return d[column];
}).join(",");
})).join("\n");
}
});
/* Make the visualization container fill the page. */
textarea{
width: 100%;
}
.loader {
visibility: hidden;
}