Overview
While we can set up alerts in Google Analytics to monitor the health of our tracking events, we currently have no robust out-of-the-box way to check whether our custom javascript and marketing tags are still working. The following tutorial shows the steps needed to set up a tag monitoring system using Google Tag Manager, Google Sheet, and Google Appscript. There is no BigQuery or or Cloud Functions involved.
End Result
Google Tag Manager Configuration
Set up a template in GTM with the following code
const createQueue = require('createQueue');
const dataLayerPush = createQueue('dataLayer');
const readFromDataLayer = require('copyFromDataLayer');
const addEventCallback = require('addEventCallback');
const event = readFromDataLayer('event');
addEventCallback((ctid, eventData) => {
const tags = eventData.tags.filter(t => t.monitor == 'true');
//const tags = eventData.tags;
tags.forEach(t => {
dataLayerPush({
event: "custom.logger",
eventName: event,
tagObj: t
});
});
});
data.gtmOnSuccess
();
Set the permission for the template created as below
Google App Script Configuration
Method 1
Getting the Tag Name from the Tag Monitor GA View. This method has a disadvantage that it can only fetch the tag names of tags that are being monitored, not the full list of current tags in the GTM Container. To get the full list of tags in the GTM container, please use method 2.
function onOpen() {
var ui = SpreadsheetApp.getUi();
// Or DocumentApp or FormApp.
ui.createMenu('Tag Monitor Menu')
.addItem('Fetch New Data', 'menuFetchNewData')
.addToUi();
}
function menuFetchNewData() {
main();
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert('New Data Loaded');
}
function getTagsList(tableId, metrics, gaOptions, startDate, endDate){ // in the future, upgrade this to access the GTM tag list directly
var tableId = tableId; // from Query Explorer, the number is the profile id
var options = gaOptions;
//Fetch the report
var report = gaGet(tableId, startDate, endDate, metrics, options);
var data = report.rows;
data.forEach(data => data.splice(1,1));
var arr1d = [].concat(...data);
return arr1d;
}
function getDateXDaysAgo(numOfDays, date = new Date()) {
const daysAgo = new Date(date.getTime());
daysAgo.setDate(date.getDate() - numOfDays);
return daysAgo;
}
function writeHeadersToSheet(data){
var sheetName ='ga_tag_monitor_tag_status_data'; // Google sheet sheet
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
//const spreadsheet = SpreadsheetApp.create('Google Analytics Report');
var sheet = spreadsheet.getSheetByName(sheetName);
//get the range to write and write the results
var writeRange = sheet.getRange(1, 1, data.length, data[0].length) //// Read reference for getRange arguments
//getRange(row: any, column: any, numRows: any, numColumns: any): SpreadsheetApp.Range
writeRange.setValues(data);
}
function writeToSheet(headerData, contentData){
var sheetName ='ga_tag_monitor_tag_status_data'; // Google sheet sheet
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
//const spreadsheet = SpreadsheetApp.create('Google Analytics Report');
var sheet = spreadsheet.getSheetByName(sheetName);
sheet.clearContents();
writeHeadersToSheet(headerData);
//get the range to write and write the results
var writeRange = sheet.getRange(2, 1, contentData.length, contentData[0].length) //// Read reference for getRange arguments
//getRange(row: any, column: any, numRows: any, numColumns: any): SpreadsheetApp.Range
writeRange.setValues(contentData);
Logger.log('Report spreadsheet created: %s',spreadsheet.getUrl());
}
function main() {
var tableId = 'ga:270339408'; // from Query Explorer, the number is the profile id
var metrics = ['ga:totalEvents'];
var options = {
'dimensions': 'ga:eventAction',
//'filters':'',
'sort': '-ga:eventAction',
//'segment': '',
'samplingLevel': 'HIGHER_PRECISION'
//'max-results':'5'
}
var anchorStartDate = '30daysAgo'; // 'yyyy-MM-dd'
var anchorEndDate = '1daysAgo';
var monitorTimeFrame=14;
//get tag names
tags = getTagsList(tableId, metrics, options, anchorStartDate, anchorEndDate);
//set up the object
var tagsStatusData={};
for(var i=0;i<tags.length;i++){
//var ob={[tags[i]]:[]};
tagsStatusData[tags[i]]=[];
}
var headerData = [["tagName"]];
//fetch tags' status data
for(var dateCounter=monitorTimeFrame;dateCounter>0;dateCounter--){
var startDate=dateCounter+"daysAgo";
var endDate=dateCounter+"daysAgo";
//console.log(tags,startDate,endDate);
var report = gaGet(tableId, startDate, endDate, metrics, options);
var data = report.rows;
for(var i=0;i<data.length;i++){
if(data[i][0] in tagsStatusData){ // safe-check if key exist
tagsStatusData[data[i][0]].push(data[i][1]);
}
var currentLength = tagsStatusData[data[i][0]].length;
}
//loop through all objects and check if their [] length is less than the current, then add 0 in
for (let key in tagsStatusData) {
if(tagsStatusData[key].length<currentLength){
tagsStatusData[key].push(0);
}
}
//add header
headerData[0].push(getDateXDaysAgo(startDate.match(/\d+/)[0]));
}
//set up monitoring metrics
headerData[0].push("averageBeforeYesterday");
headerData[0].push("% Change");
headerData[0].push("Tag Health");
for (let key in tagsStatusData) {
//get sum
var a = tagsStatusData[key];
var sum=0;
var average;
var num;
for(var i = 0; i<a.length-1; i++){
num=a[i];
num=+num;
sum=sum+num;
}
average=sum/(monitorTimeFrame-1);
tagsStatusData[key].push(average);
//get average
var last = a[a.length-2];
last=1.0*last;
var percentageChange;
if(average!=0){
percentageChange = 1-last/average;
}else{
percentageChange ="inactive";
}
tagsStatusData[key].push(percentageChange);
//get monitor status
if(percentageChange<=-0.2 || percentageChange>=0.2){
tagsStatusData[key].push("bad");
}else if(percentageChange=="inactive"){
tagsStatusData[key].push("inactive");
}else{
tagsStatusData[key].push("good");
}
}
//convert tagsStatusData to 2 dimensional array
twoDTagsData=Object.keys(tagsStatusData).map((key)=>[key,...tagsStatusData[key]]);
//** WRITE VALUES TO SHEET */
writeToSheet(headerData, twoDTagsData);
}
function gaGet(tableId, startDate, endDate, metrics, options) {
// Apply standard options
options = options || {};
options['max-results'] = options['max-results'] || '10000';
// If errors persist up to 5 times then terminate the program.
for (var i = 0; i < 5; i++) {
try {
return Analytics.Data.Ga.get(tableId, startDate, endDate, metrics, options); // 503
} catch (err) {
// https://developers.google.com/analytics/devguides/reporting/core/v3/coreErrors
if (err.message.indexOf('a server error occurred') > -1) {
Logger.log('Backend Error');
// Note: Don't listen to Google's reply and retry request after 2 minutes
Utilities.sleep(2 * 60 * 1000);
} else if (err.message.indexOf('User Rate') > -1) {
Logger.log('Rate Limit Error');
// Exponential Backoff
Utilities.sleep(1000 * Math.pow((i + 1), 2));
} else if (err.message.indexOf('too many concurrent connections') > -1) {
Logger.log('Concurrent Connections Error');
// Exponential Backoff
Utilities.sleep(1000 * Math.pow((i + 1), 2));
} else {
Logger.log(err);
throw err;
}
}
}
throw 'Error. Max retries reached';
}
Method 2
The following gets the list of tag names from the GTM, unlike the above which gets it from GA.
function getTagsListFromGA(tableId, metrics, gaOptions, startDate, endDate){ // in the future, upgrade this to access the GTM tag list directly
var tableId = tableId; // from Query Explorer, the number is the profile id
var options = gaOptions;
//Fetch the report
var report = gaGet(tableId, startDate, endDate, metrics, options);
var data = report.rows;
data.forEach(data => data.splice(1,1));
var arr1d = [].concat(...data);
return arr1d;
}
function getTagsListFromGTM(path){
var tags = TagManager.Accounts.Containers.Workspaces.Tags.list(path).tag;
var tagsList = tags.map(function(item) { return item["name"]; });
//Logger.log(tagsList);
return tagsList;
}
function getDateXDaysAgo(numOfDays, date = new Date()) {
const daysAgo = new Date(date.getTime());
daysAgo.setDate(date.getDate() - numOfDays);
return daysAgo;
}
function ga_tag_monitor_v1_main() {
var tableId = 'ga:270339408'; // from Query Explorer, the number is the profile id
var metrics = ['ga:totalEvents'];
var options = {
'dimensions': 'ga:eventAction',
//'filters':'',
'sort': '-ga:eventAction',
//'segment': '',
'samplingLevel': 'HIGHER_PRECISION'
//'max-results':'5'
}
var monitorTimeFrame=29; // UPDATE THIS TO EXTEND TIMEFRAME WHEN WE HAVE MORE DATA
var monitorThreshold=0.1; // 10% tolorence
//get tag names
//tags = getTagsListFromGA(tableId, metrics, options, anchorStartDate, anchorEndDate);
tags = getTagsListFromGTM("accounts/364874/containers/31314328/workspaces/861");
//set up the object
var tagsStatusData={};
for(var i=0;i<tags.length;i++){
//var ob={[tags[i]]:[]};
tagsStatusData[tags[i]]=[];
}
var headerData = [["tagName"]];
//fetch tags' status data
for(var dateCounter=monitorTimeFrame;dateCounter>0;dateCounter--){
var startDate=dateCounter+"daysAgo";
var endDate=dateCounter+"daysAgo";
//console.log(tags,startDate,endDate);
var report = gaGet(tableId, startDate, endDate, metrics, options);
var data = report.rows;
var currentLength=0;
for(var i=0;i<data.length;i++){
if(data[i][0] in tagsStatusData){ // safe-check if key exist
tagsStatusData[data[i][0]].push(data[i][1]);
}
if(tagsStatusData[data[i][0]]){ // in case the lastone is empty (doesn't exist)
currentLength = tagsStatusData[data[i][0]].length; //this without the conditional statement wouldn't work if the last one is empty
}
}
//loop through all objects and check if their [] length is less than the current, then add 0 in
for (let key in tagsStatusData) {
if(tagsStatusData[key].length<currentLength){
tagsStatusData[key].push(0);
}
}
//add header
headerData[0].push(getDateXDaysAgo(startDate.match(/\d+/)[0]));
}
console.log(tagsStatusData);
//set up monitoring metrics - yesterday vs same day last week (7daysAgo). CAN MULTIPLE COLUMN OF VS 7daysAgo*2, * 3, *4, *5
var maxWeeksAgo=3;
for(var i=1;i<maxWeeksAgo+1;i++){
var header = "ytdy % vs "+i+"wa";
headerData[0].push(header);
}
var tagsMonitoringStatusData = getTagMonitoringStatus("accounts/364874/containers/31314328/workspaces/861");
for (let key in tagsStatusData) {
var a=tagsStatusData[key];
var last = a[a.length-1];
//var oneWeekAgo = a[a.length-1-7];
//var twoWeekAgo = a[a.length-1-7*2];
//var threeWeekAgo = a[a.length-1-7*3];
for(var i=1;i<maxWeeksAgo+1;i++){
var xWeeksAgoValue;
if(i==1){
xWeeksAgoValue = (a[a.length-1-(i*7)]);
}else{
xWeeksAgoValue = (a[a.length-1-(i*7)-(i-1)]);
}
if(xWeeksAgoValue==0 && last!=0){
tagsStatusData[key].push("0 -> !0");
}else if(xWeeksAgoValue==0 && last==0){
tagsStatusData[key].push(0);
}else{
tagsStatusData[key].push(last/xWeeksAgoValue-1);
}
}
//add monitor data
var monitoringStatus=tagsMonitoringStatusData[key];
tagsStatusData[key].push(monitoringStatus); // <yes|no>
var oneWeekAgoPercentageChange = a[a.length-maxWeeksAgo-1];
console.log(key, oneWeekAgoPercentageChange);
var tagHealthStatus = "ok";
if(monitoringStatus== "no"){
tagHealthStatus = "not monitored";
}
if(oneWeekAgoPercentageChange<(monitorThreshold*-1)){
tagHealthStatus = "drop off";
}else if(oneWeekAgoPercentageChange>monitorThreshold){
tagHealthStatus = "spike";
}
tagsStatusData[key].push(tagHealthStatus);
}
//convert tagsStatusData to 2 dimensional array
twoDTagsData=Object.keys(tagsStatusData).map((key)=>[key,...tagsStatusData[key]]);
headerData[0].push("monitoring?");
headerData[0].push("Tag Health");
//** WRITE VALUES TO SHEET */
writeToSheet("ga_tag_monitor", headerData, twoDTagsData);
}
function sendTagMonitoringAlert(){
}
function getTagMonitoringStatus(path){
if(!path){
path = "accounts/364874/containers/31314328/workspaces/861";
}
var tags = TagManager.Accounts.Containers.Workspaces.Tags.list(path).tag;
var tagsStatus={};
tags.forEach(function(tag){
var flag = "no";
if(tag.monitoringMetadata && tag.monitoringMetadata.map){
tag.monitoringMetadata.map.forEach(function(mapItem){
if(mapItem.key && mapItem.key == "monitor" && mapItem.value && mapItem.value=="true"){
flag="yes";
}
});
}
tagsStatus[[tag.name]] = flag;
//console.log(tag.name,flag);
});
return tagsStatus;
}
function gaGet(tableId, startDate, endDate, metrics, options) {
// Apply standard options
options = options || {};
options['max-results'] = options['max-results'] || '10000';
// If errors persist up to 5 times then terminate the program.
for (var i = 0; i < 5; i++) {
try {
return Analytics.Data.Ga.get(tableId, startDate, endDate, metrics, options); // 503
} catch (err) {
// https://developers.google.com/analytics/devguides/reporting/core/v3/coreErrors
if (err.message.indexOf('a server error occurred') > -1) {
Logger.log('Backend Error');
// Note: Don't listen to Google's reply and retry request after 2 minutes
Utilities.sleep(2 * 60 * 1000);
} else if (err.message.indexOf('User Rate') > -1) {
Logger.log('Rate Limit Error');
// Exponential Backoff
Utilities.sleep(1000 * Math.pow((i + 1), 2));
} else if (err.message.indexOf('too many concurrent connections') > -1) {
Logger.log('Concurrent Connections Error');
// Exponential Backoff
Utilities.sleep(1000 * Math.pow((i + 1), 2));
} else {
Logger.log(err);
throw err;
}
}
}
throw 'Error. Max retries reached';
}
Utility Functions
function onOpen() {
var ui = SpreadsheetApp.getUi();
// Or DocumentApp or FormApp.
ui.createMenu('Tag Monitor Menu')
.addItem('Fetch New Tag Status Data', 'menuFetchNewData')
.addToUi();
ui.createMenu('GTM Utilities')
.addItem('Fetch Unused Variables', 'menuGtmFindUnusedVariables')
.addToUi();
}
function menuFetchNewData() {
ga_tag_monitor_v1_main();
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert('New Data Loaded in sheet ga_tag_monitor_tag_status_data');
}
function menuGtmFindUnusedVariables(){
gtmFindUnusedVariables();
SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
.alert('New Data Loaded in sheet gtm_unsed_variables_report');
}
function writeHeadersToSheet(sheetName, data){
//var sheetName ='ga_tag_monitor_tag_status_data'; // Google sheet sheet
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
//const spreadsheet = SpreadsheetApp.create('Google Analytics Report');
var sheet = spreadsheet.getSheetByName(sheetName);
//get the range to write and write the results
var writeRange = sheet.getRange(1, 1, data.length, data[0].length) //// Read reference for getRange arguments
//getRange(row: any, column: any, numRows: any, numColumns: any): SpreadsheetApp.Range
writeRange.setValues(data);
}
function writeToSheet(sheetName, headerData, contentData){
//var sheetName ='ga_tag_monitor_tag_status_data'; // Google sheet sheet
var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();
//const spreadsheet = SpreadsheetApp.create('Google Analytics Report');
var sheet = spreadsheet.getSheetByName(sheetName);
sheet.clearContents();
writeHeadersToSheet(sheetName,headerData);
//get the range to write and write the results
var writeRange = sheet.getRange(2, 1, contentData.length, contentData[0].length) //// Read reference for getRange arguments
//getRange(row: any, column: any, numRows: any, numColumns: any): SpreadsheetApp.Range
writeRange.setValues(contentData);
Logger.log('Report spreadsheet created: %s',spreadsheet.getUrl());
}
Analytics-Mastery Pty Ltd 