Prepping for seasonality talk
<!DOCTYPE html>
<head>
<meta charset="utf-8">
<link rel="stylesheet" href="stl.css">
<link href='//fonts.googleapis.com/css?family=Raleway:400,700' rel='stylesheet' type='text/css'>
</head>
<body>
<div>Seasonal Decomposition</div>
<div>
<div class="monthly"></div>
<div class="monthlyGrowth"></div>
<div class="quarterly"></div>
<div class="quarterlyGrowth"></div>
<div class="weekly"></div>
<div class="weeklyGrowth"></div>
</div>
<script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js"></script>
<script src="aggregateData.js"></script>
<script src="fakeData.js"></script>
<script src="timeSeriesLayout.js"></script>
<script src="stl.js"></script>
</body>
// everything after here can take daily data and plot it, so would work for "real" data too
function aggregateData() {
var dateFormat = d3.time.format("%Y-%m-%d")
var granularity = '';
var data = [];
// incoming data object
var dailyData = {};
var days = [];
var getQuarterFloor = function(date) {
var quarterConvert = [0, 0, 0, 3, 3, 3, 6, 6, 6, 9, 9, 9]
return new Date(date.getFullYear(), quarterConvert[date.getMonth()], 1)
}
var rollupFunction = {
'daily': function(d) {
return d
},
'weekly': function(d) {
return d3.time.week(d);
},
'monthly': function(d) {
return d3.time.month(d);
},
'quarterly': function(d) {
return getQuarterFloor(d)
}
}
// rollup functions
// TODO: generalize this better
var rollupSums = function(leaves) {
if ((granularity == 'weekly' && leaves.length < 7) || (granularity == 'quarterly' && leaves.length < 90)) {
// if a partial week, return empty object (it will complain)
// same for partial quarter - this isn't a perfect check, as could still be missing a day or two
return {}
}
return {
trendDow: d3.sum(leaves, function(a) {
return dailyData[dateFormat(a)].trendDow
}),
trendDowSeasonal: d3.sum(leaves, function(a) {
return dailyData[dateFormat(a)].trendDowSeasonal
}),
trendDowSeasonalHolidays: d3.sum(leaves, function(a) {
return dailyData[dateFormat(a)].trendDowSeasonalHolidays
}),
trendDowSeasonalHolidaysAdjust: d3.sum(leaves, function(a) {
return dailyData[dateFormat(a)].trendDowSeasonalHolidaysAdjust
}),
}
}
var update = function() {
data = d3.nest().key(rollupFunction[granularity])
.rollup(rollupSums)
.entries(days);
}
var getDateExtent = function() {
return [d3.min(dailyData, function(period) {
return d3.min(period.key)
}), d3.max([0, d3.min(dailyData, function(period) {
return d3.max(period.key)
})])]
}
function aggregate(newDailyData, newDays) {
dailyData = newDailyData;
days = newDays;
update()
return this
}
aggregate.getValueExtent = function() {
return [d3.min([0, d3.min(data, function(period) {
return d3.min([period.values.trendDow, period.values.trendDowSeasonal, period.values.trendDowSeasonalHolidays])
})]), d3.max(data, function(period) {
return d3.max([period.values.trendDow, period.values.trendDowSeasonal, period.values.trendDowSeasonalHolidays])
})]
}
aggregate.granularity = function(newGranularity) {
if (!arguments.length) return granularity;
granularity = newGranularity;
update()
return this;
}
aggregate.getData = function() {
update()
return data
}
return aggregate
}// fakeData helps us create a fake dataset
function fakeData() {
var dateFormat = d3.time.format("%Y-%m-%d")
var today = new Date();
var firstWeekValue = 1,
weeklyGrowthRate = 1,
// first position is Sunday, data given as % of week value
dayOfWeekFactors = Array(7).fill(1 / 7),
//annual seasonality
weekOfYearFactors = Array(53).fill(1),
dateRange = [new Date() - 24 * 60 * 60 * 1000 * 728, new Date()],
holidays = [],
adjustment = [];
var dailyData = {}
function getPeriodArray(days) {
return d3.time.day.utc.range(dateRange[0], dateRange[1], days);
}
function updateData() {
// take in information & update it
var trend = [];
getPeriodArray(7).forEach(function(d, i) {
trend[i] = {
date: d,
value: firstWeekValue * Math.pow((weeklyGrowthRate), i)
}
})
// set daily data, based on trend + weekly data
var dailyDataArray = []
getPeriodArray(1).forEach(function(d, i) {
var daily = trend[Math.floor(i / 7)].value * dayOfWeekFactors[d.getDay()]
dailyData[dateFormat(d)] = {
trendDow: daily,
trendDowSeasonal: daily * weekOfYearFactors[d3.time.weekOfYear(d)],
trendDowSeasonalHolidays: daily * weekOfYearFactors[d3.time.weekOfYear(d)],
trendDowSeasonalHolidaysAdjust: daily * weekOfYearFactors[d3.time.weekOfYear(d)],
}
})
// use holiday list to adjust
holidays.forEach(function(holiday) {
holiday.dates.forEach(function(date) {
if (dailyData[date]) {
dailyData[date].trendDowSeasonalHolidays = dailyData[date].trendDowSeasonalHolidays * holiday.impact
dailyData[date].trendDowSeasonalHolidaysAdjust = dailyData[date].trendDowSeasonalHolidaysAdjust * holiday.impact
}
})
})
adjustment.forEach(function(impact) {
if (impact.dateRange[1] == '') {
impact.dateRange[1] = dateRange[1];
}
if (impact.dateRange[0] == '') {
impact.dateRange[0] = dateRange[0]
}
var impactDays = d3.time.day.utc.range(impact.dateRange[0], impact.dateRange[1], 1);
impactDays.forEach(function(date) {
var adjustDate = dateFormat(date)
if (dailyData[adjustDate]) {
dailyData[adjustDate].trendDowSeasonalHolidaysAdjust = dailyData[adjustDate].trendDowSeasonalHolidaysAdjust * impact.factor;
}
})
})
}
function setUpData() {
updateData();
return dailyData;
}
setUpData.firstWeek = function(newFirstWeekValue) {
if (!arguments.length) return firstWeekValue;
firstWeekValue = newFirstWeekValue;
// update data
return this;
}
// array of
setUpData.dayOfWeekFactors = function(newDayOfWeekFactors) {
if (!arguments.length) return dayOfWeekFactors;
dayOfWeekFactors = newDayOfWeekFactors;
return this;
}
// set holidays
setUpData.holidays = function(newHolidays) {
if (!arguments.length) return holidays;
holidays = newHolidays;
return this;
}
// start/end dates of date range, expects an array of length two with dates
setUpData.dateRange = function(newDateRange) {
if (!arguments.length) return dateRange;
dateRange = newDateRange;
return this;
}
// start/end dates of date range, expects an array of length two with dates
setUpData.weeklyGrowth = function(newWeeklyGrowth) {
if (!arguments.length) return weeklyGrowthRate;
weeklyGrowthRate = newWeeklyGrowth;
return this;
}
// manual adjustment
setUpData.adjustment = function(newAdjustment) {
if (!arguments.length) return adjustment;
adjustment = newAdjustment;
return this;
}
// start/end dates of date range, expects an array of length two with dates
setUpData.weekOfYear = function(newWeekOfYearFactors) {
if (!arguments.length) return weekOfYearFactors;
weekOfYearFactors = newWeekOfYearFactors;
return this;
}
// array of days from start to end of range
setUpData.getDays = function() {
return getPeriodArray(1)
}
return setUpData;
}.hidden {
display: none;
}
.lineGraph {
fill: none;
stroke: #4682b4;
stroke-width: 1.5px;
}
.axis path {
fill: none;
stroke: #808080;
shape-rendering: crispEdges;
stroke-width: 1px;
}
.axis .x {
font-size: 11px;
}
.axis .x .tick {
text-anchor: start;
}
.axis .x .tick text {
fill: #808080;
transform: rotate(-90deg);
}
.axis .x.weekly {
font-size: 8px;
}
.bars {
fill: #d0d0d0;
}
"use strict";
// import from other file
var fakedData = fakeData()
// parameters for fake data
var firstWeekValue = 29000,
weeklyGrowthRate = 1.0003,
// first position is Sunday, data given as % of week value
dayOfWeekFactors = [.2, 0, .12, .08, .1, .2, .3],
//annual seasonality
weekOfYearFactors = [
.9, .9, .9, .9,
.9, .9, .9, .9,
.9, .9, .9, .9,
.9, .9, 1.1, 1.1,
1.1, 1.2, 1.2, 1.2,
1.2, 1.2, 1.2, 1.2,
1.2, 1.2, 1.2, 1.2,
1.2, 1.2, 1.2, 1.2,
1.2, 1.1, 1.1, 1.1,
.9, .9, .9, .9,
.9, .9, .9, .9,
.9, .9, .9, .9,
.9, .9, .9, .9, .9
],
holidays = [],
/*holidays = [
{
name: 'Easter',
dates: ['2010-04-04', '2011-04-24', '2012-04-08', '2013-03-31', '2014-04-20', '2015-04-05', '2016-03-27'],
impact: 1.5
},
{
name: 'Christmas',
dates: ['2010-12-25', '2011-12-25', '2012-12-25', '2013-12-25', '2014-12-25', '2015-12-25', '2016-12-25'],
impact: 0
}, {
name: 'Mothers Day',
dates: ['2010-05-09', '2011-05-08', '2012-05-13', '2013-05-12', '2014-05-11', '2015-05-10', '2016-05-08'],
impact: 1.5
}, {
name: 'Thanksgiving',
dates: ['2010-11-25', '2011-11-24', '2012-11-22', '2013-11-28', '2014-11-27', '2015-11-26', '2016-11-24'],
impact: 1.5
}
],
*/
adjustment = [{
// march
dateRange: [new Date(2016, 2, 1), new Date(2016, 3, 1)],
factor: 1
}, {
//rest of quarter
dateRange: [new Date(2016, 3, 1), new Date(2016, 6, 1)],
factor: 1
}],
dateRange = [new Date(2012, 0, 1), new Date(2016, 6, 1)];
// apply parameters to fake data
fakedData.firstWeek(firstWeekValue)
.dayOfWeekFactors(dayOfWeekFactors)
.dateRange(dateRange)
.holidays(holidays)
.adjustment(adjustment)
.weeklyGrowth(weeklyGrowthRate)
.weekOfYear(weekOfYearFactors);
// get data from fakeData functions
var dailyData = fakedData()
console.log(dailyData, fakedData.getDays())
var dateFormat = d3.time.format("%Y-%m-%d")
/* method for printing out data
var csvContent = "data:text/csv;charset=utf-8,";
fakedData.getDays().forEach(function(date, index) {
var daysData = dailyData[dateFormat(date)];
var dataString = dateFormat(date) + "," + Object.keys(daysData).map(function(k) {
return daysData[k]
}).join(",");
csvContent += index < fakedData.getDays().length ? dataString + "\n" : dataString;
});
console.log(csvContent)
var encodedUri = encodeURI(csvContent);
var link = document.createElement("a");
link.setAttribute("href", encodedUri);
link.setAttribute("download", "my_data.csv");
link.click()
*/
var plotType = 'trendDowSeasonalHolidaysAdjust'
function lineChart(data, dateRange) {
function setChart(elem, granularity, plotValue, type) {
var chart = d3.select("." + elem).append("svg").attr("width", 1700).attr("height", 300).append("g").attr("transform", "translate(100,30)");
var timeSeries = timeSeriesLayout().granularity(granularity).type(type).dataString(plotValue);
var line = d3.svg.line().x(function(d) {
return d.x
}).y(function(d) {
return d.y
}).interpolate('linear')
chart.append("path")
.datum(timeSeries(data, dateRange))
.attr("d", line)
.attr("class", "lineGraph " + granularity + " " + type)
var axis = chart.append("g").attr("class", "axis")
var xAxis = d3.svg.axis().scale(timeSeries.getXScale()).orient("bottom").ticks(getTickInterval(granularity), getTickFrequency(granularity)).tickFormat(getTimeFormat(granularity));
axis.append("g").attr("class", "x " + granularity).call(xAxis).attr("transform", "translate(0," + timeSeries.getYScale()(0) + ")").selectAll("text").attr("dx", "-10px").style("text-anchor", 'end').attr("dy", "-.5em")
var yAxis = d3.svg.axis().scale(timeSeries.getYScale()).orient("left").tickFormat(getYAxisFormat(type))
axis.append("g").attr("class", "y").call(yAxis).attr("transform", "translate(0,0)")
}
return setChart;
}
function getYAxisFormat(type) {
if (type == 'growth') {
return d3.format("%")
} else {
return d3.format("s")
}
}
function getTimeFormat(granularity) {
var granularityToFormat = {
'weekly': d3.time.format("%d %b"),
'quarterly': d3.time.format.multi([
["%Y-Q1", function(d) {
return d.getMonth() < 3;
}],
["Q2", function(d) {
return d.getMonth() < 6;
}],
["Q3", function(d) {
return d.getMonth() < 9;
}],
["Q4", function(d) {
return true;
}],
]),
'monthly': d3.time.format.multi([
["%b", function(d) {
return d.getMonth();
}],
["%Y", function() {
return true;
}]
]),
'daily': d3.time.format("%d %b"),
}
return granularityToFormat[granularity]
}
function getTickFrequency(granularity) {
var granularityToTickFreq = {
'weekly': 3,
'quarterly': 3,
'monthly': 1,
'daily': 14,
}
return granularityToTickFreq[granularity]
}
function getTickInterval(granularity) {
var granularityToTickFreq = {
'weekly': d3.time.week,
'quarterly': d3.time.month,
'monthly': d3.time.month,
'daily': d3.time.day,
}
return granularityToTickFreq[granularity]
}
function barChart(data, dateRange) {
function setBarChart(elem, granularity, plotValue, type) {
var chart = d3.select("." + elem).append("svg").attr("width", 1700).attr("height", 300).append("g").attr("transform", "translate(100,30)");
var timeSeries = timeSeriesLayout().granularity(granularity).type(type).dataString(plotValue);
var yScale = timeSeries.getYScale();
chart.selectAll(".bars")
.data(timeSeries(data, dateRange))
.enter()
.append("rect")
.attr({
'x': function(d) {
return d.x
},
'y': function(d) {
return d.y < yScale(0) ? d.y : yScale(0)
},
'height': function(d) {
return d.height
},
'width': function(d) {
return d.width
},
'class': 'bars'
})
var axis = chart.append("g").attr("class", "axis")
var xAxis = d3.svg.axis().scale(timeSeries.getXScale()).orient("bottom").ticks(getTickInterval(granularity), getTickFrequency(granularity)).tickFormat(getTimeFormat(granularity))
axis.append("g").attr("class", "x " + granularity).call(xAxis).attr("transform", "translate(0," + yScale(0) + ")").selectAll("text").attr("dx", "-70px").style("text-anchor", 'end').attr("dy", "-.2em")
var yAxis = d3.svg.axis().scale(timeSeries.getYScale()).orient("left").tickFormat(getYAxisFormat(type))
axis.append("g").attr("class", "y").call(yAxis).attr("transform", "translate(0,0)")
}
return setBarChart;
}
lineChart(dailyData, fakedData.dateRange())('monthly', 'monthly', plotType, 'value')
barChart(dailyData, fakedData.dateRange())('monthlyGrowth', 'monthly', plotType, 'growth')
lineChart(dailyData, fakedData.dateRange())('weekly', 'weekly', plotType, 'value')
barChart(dailyData, fakedData.dateRange())('weeklyGrowth', 'weekly', plotType, 'growth')
lineChart(dailyData, fakedData.dateRange())('quarterly', 'quarterly', plotType, 'value')
barChart(dailyData, fakedData.dateRange())('quarterlyGrowth', 'quarterly', plotType, 'growth').hidden
display: none
.lineGraph
fill: none
stroke: steelblue
stroke-width: 1.5px
.axis path
fill: none
stroke: gray
shape-rendering: crispEdges
stroke-width: 1px
.axis
.x
font-size: 11px
.tick
text-anchor: start
text
fill: gray
transform: rotate(-90deg)
.axis
.x
&.weekly
font-size: 8px
.bars
fill: #d0d0d0
function timeSeriesLayout() {
// 'growth' or 'value'
var type = 'value'
// daily, weekly, monthly, quarterly, annual
var granularity = 'daily'
var width = 1000;
var height = 200;
var dataString = 'value';
//
var periodLookup = {
'monthly': 12,
'quarterly': 4,
'weekly': 52,
'daily': 364
};
// set up scales
var xScale = d3.time.scale().range([0, width]);
var yScale = d3.scale.linear().range([height, 0]);
var heightScale = d3.scale.linear()
function processTimeSeries(data, dateExtent) {
var layoutData = [];
// read in data object
var aggregate = aggregateData();
aggregate.granularity(granularity)
var days = d3.time.day.utc.range(dateExtent[0], dateExtent[1], 1);
aggregate(data, days)
var extent = aggregate.getValueExtent()
var aggData = aggregate.getData();
var barWidth = Math.floor((width) / aggData.length) - 2;
// set scales
xScale.domain(dateExtent)
if (type == 'growth') {
heightScale.domain([0, .1]).range([0, height / 2]);
yScale.domain([-.1, .1]);
} else {
heightScale.domain([0, extent[1]]).range([0, height]);
yScale.domain([0, extent[1]]);
}
if (type == 'value') {
aggData.forEach(function(day, i) {
layoutData[i] = {};
layoutData[i].x = xScale(Date.parse(day.key));
layoutData[i].y = yScale(day.values[dataString]);
layoutData[i].height = heightScale(day.values[dataString]);
layoutData[i].width = barWidth;
})
} else {
aggData.forEach(function(day, i) {
var period = periodLookup[granularity];
var growth = (i - period) >= 0 ? aggData[i].values[dataString] / aggData[i - period].values[dataString] - 1 : 0;
layoutData[i] = {};
layoutData[i].x = xScale(Date.parse(day.key));
layoutData[i].y = yScale(growth);
layoutData[i].height = heightScale(Math.abs(growth));
layoutData[i].width = barWidth;
})
}
return layoutData;
}
processTimeSeries.type = function(newType) {
if (!arguments.length) return type;
type = newType;
return this;
}
processTimeSeries.dataString = function(newString) {
if (!arguments.length) return dataString;
dataString = newString;
return this;
}
processTimeSeries.granularity = function(newGranularity) {
if (!arguments.length) return newGranularity;
granularity = newGranularity;
return this;
}
processTimeSeries.height = function(newHeight) {
if (!arguments.length) return newHeight;
height = newHeight;
return this;
}
processTimeSeries.width = function(newWidth) {
if (!arguments.length) return newWidth;
width = newWidth;
return this;
}
processTimeSeries.getXScale = function() {
return xScale;
}
processTimeSeries.getYScale = function() {
return yScale;
}
return processTimeSeries;
}