[
  {
    "path": "LICENSE",
    "content": "The MIT License (MIT)\n\nCopyright (c) 2014 Andy Davies\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n"
  },
  {
    "path": "README.md",
    "content": "<p align=\"center\"><img src=\"https://docs.webpagetest.org/img/wpt-navy-logo.png\" alt=\"WebPageTest Logo\" /></p>\n<p align=\"center\"><a href=\"https://docs.webpagetest.org/api/integrations/#officially-supported-integrations\">Learn about more WebPageTest API Integrations in our docs</a></p>\n\n# WebPageTest Google Sheets Bulk Tester\n[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](/LICENSE)\n\nUse Google Sheets to test multiple URLs using WebPageTest (either webpagetest.org if you have an API key, or another publicly accessible instance)\n\nEach test uses one of a defined set of parameters (a scenario) so tests can either share the same parameters, or use different sets depending on need.\n\nWhen a test completes successfully, selected values from the results are extracted and added to the Tests tab.\n\nComments, suggestions, improvements etc. welcome.\n\nThere are brief instructions below but for more detailed one see the Performance Advent Calender post - http://calendar.perfplanet.com/2014/driving-webpagetest-from-a-google-docs-spreadsheet/\n\n\n# Using\n\n1. Make a copy of Spreadsheet\n\n\t[WPT Bulk Tester v0.7](https://docs.google.com/spreadsheets/d/10-FAt5eelHXjzQqgx5o-JvUqrKAMIux2kp-sAwHARwk)\n\n\tThe spreadsheet is shared read-only so you'll first need to make a copy\n\n2. Configuring Spreadsheet - Settings Tab\n\n\tAdd your own WPT API key\n\tCustomise the parameters and results maps to include the parameters you want to specify and the results values to be extracted\n\n3. Defining Tests - Scenarios Tab\n\n\tCreate one or more test scenarios (a scenario is a named set of test parameters)\n\tFirst column must always be the name of the scenario, other columns are defined by the Parameters map in the Settings tab\n\n4. Specifying URLs to be Tested - Tests Tab\n\n\tAdd URLs to be tested in the first column, and scenario in the second (a drop down can be created via the Data > Validation menu, or just copy cell from previous row)\n\n5. Running Tests\n\n\tOnce the URLs to be tested and the corresponding scenario have been defined, choose 'Run Tests' from WebPageTest menu (on first run the app will need to be authorised and the test re-submitted)\n\n\tOnce the tests have been submitted the results will be polled until they have all completed. Polling interval is based on number of tests 1 min <= 5 tests, 5 mins <= 10 tests otherwise 30 mins\n\n\tTo re-run a test delete the WPT URL and then choose 'Run Tests' from WebPageTest menu\n\tTo re-retrieve the results delete the status (and corresponding results) and choose 'Get Results' from the WebPageTest menu\n\n\n# Changes\n\n## v0.6 - 26th Oct 2020\n\n- Change submission to use POST as GET requests are limited to 2,000 bytes (Thx @dougsillars)\n- Add `normalizekeys=1` to request for results so fields names containing `.` and `-` can be accessed without array notation (Thx @Nooshu)\n- Add silent error handling around requests for non-existent fields in results\n\n## v0.7 - 26th Mar 2024\n\n- Change API key submission to pass key in 'X-WPT-API-KEY' header instead of url 'k' parameter\n\n\n"
  },
  {
    "path": "webpagetest.gs",
    "content": "/*\n * App Script to submit tests to WebPageTest and retrieve results\n *\n * Will only work in the context of the matching Google Spreadsheet.\n * https://docs.google.com/spreadsheets/d/1Hz_8griZtkDhCVqSCmeyxuHHZGbzm885nt7ws65GvIM\n *\n * This is currently a work-in-progress and the code has too many 'magic numbers' for my liking e.g. row and column offsets\n */\n\n/*\n * License\n *\n * Copyright (c) 2013-2020 Andy Davies, @andydavies, http://andydavies.me\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction,\n * including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,\n * and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so,\n * subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\n * TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL\n * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\n * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n\n/**\n * Globals\n */\n\n// Named tabs\n\nvar TESTS_TAB = \"Tests\";\nvar SCENARIOS_TAB = \"Scenarios\";\n\n// Named ranges on settings tabs\n\nvar SERVER_URL = \"ServerURL\";\nvar API_KEY = \"APIKey\";\nvar NORMALIZE_KEYS = \"NormalizeKeys\";\n\nvar PARAMETERS_MAP = \"ParametersMap\";\nvar RESULTS_MAP = \"ResultsMap\";\n\n/**\n * Adds WebPageTest menu, with actions to submit tests, check their progress and clear results\n */\nfunction onOpen() {\n  var spreadsheet = SpreadsheetApp.getActive();\n\n  var entries = [\n    { name: \"Run Tests\", functionName: \"submitTests\" },\n    { name: \"Get Results\", functionName: \"getResults\" },\n    null,\n    { name: \"Update Scenario Columns\", functionName: \"updateScenarioColumns\" },\n    { name: \"Update Test Columns\", functionName: \"updateTestColumns\" },\n  ];\n\n  spreadsheet.addMenu(\"WebPageTest\", entries);\n}\n\n/**\n * Extracts parameters from spreadsheet and submits tests to WPT\n */\n\nfunction submitTests() {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var sheet = spreadsheet.getSheetByName(TESTS_TAB);\n\n  spreadsheet.toast(\"Submitting tests…\", \"Status\", 5);\n\n  var range = sheet.getRange(2, 1, sheet.getLastRow() - 1, 4);\n\n  var server = getServerURL();\n  var APIKey = getAPIKey();\n\n  var testScenarios = getTestScenarios();\n\n  var submitted = 0; // Track how many tests were submitted\n\n  for (n = 0; n < range.getNumRows(); n++) {\n    var cells = range.offset(n, 0, 1, 4).getValues();\n\n    var pageURL = cells[0][0];\n    var scenario = testScenarios[cells[0][1]];\n    var testURL = cells[0][2];\n    var testStatus = cells[0][3];\n\n    // If there's no URL for test then it's not been submitted (TODO: what about submission failures i.e. statusCode 400)\n    if (testURL == \"\" && scenario != undefined) {\n      var params = [\n        {\n          param: \"url\",\n          value: pageURL,\n        },\n        {\n          param: \"f\",\n          value: \"json\",\n        },\n      ];\n\n      params = params.concat(scenario); // TODO: what happens if scenerio doesn't exist?\n\n      var querystring = buildQueryString(params);\n\n      // Submit tests via POST to allow URLs that exceed 2K\n      var wptAPI = server + \"/runtest.php\";\n\n      var options = {\n        method: \"post\",\n        payload: querystring,\n        headers: {\n          \"X-WPT-API-KEY\": APIKey,\n        },\n      };\n\n      var response = UrlFetchApp.fetch(wptAPI, options);\n      var result = JSON.parse(response.getContentText());\n\n      // get a new offset for result cells\n      var responseCells = range.offset(n, 2, 1, 2); // TODO: Why not just do this earlier and have two ranges?\n\n      if (result.statusCode == 200) {\n        responseCells.setValues([[result.data.userUrl, \"\"]]);\n        responseCells.clearNote();\n        submitted++;\n      } else {\n        responseCells.setValues([[\"\", result.statusCode]]);\n        responseCells.setNote(response);\n      }\n    }\n  }\n\n  // If any tests submitted, get a first pass of results and start trigger to poll for results\n  if (submitted > 0) {\n    getResults(); // get result of test submission\n\n    var pollingInterval = getPollingInterval(submitted);\n\n    spreadsheet.toast(\n      \"Polling for results until all tests complete…\",\n      \"Status\",\n      60\n    );\n\n    startTrigger(pollingInterval);\n  }\n}\n\n/**\n * Checks the status of any uncompleted tests, retrieves the results and inserts into sheet\n */\n\nfunction getResults() {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var sheet = spreadsheet.getSheetByName(TESTS_TAB);\n\n  // Build querystring, allowing the WPT fields to be normalised (remove - and .) or not\n  var normalizeKeys = getNormalizeKeys();\n\n  var params = [\n    {\n      param: \"f\",\n      value: \"json\",\n    },\n    {\n      param: \"normalizekeys\",\n      value: normalizeKeys,\n    },\n  ];\n\n  var querystring = buildQueryString(params);\n\n  var range = sheet.getRange(2, 3, sheet.getLastRow() - 1, 2); // Just get URL for test, and status columns\n\n  var urls_array = range.getValues();\n\n  var resultsMap = getResultsMap();\n\n  var outstandingResults = 0; // track how many tests yet to complete\n\n  for (var i = 0; i < urls_array.length; i++) {\n    var url = urls_array[i][0];\n    var status = urls_array[i][1];\n\n    if (url && status < 200) {\n      // WebPageTest\n      var wptAPI = url + \"?\" + querystring;\n\n      var response = UrlFetchApp.fetch(wptAPI);\n      var result = JSON.parse(response.getContentText());\n\n      e = sheet.setActiveCell(\"D\" + (2 + i));\n      e.setValue(result.statusCode);\n\n      if (result.statusCode < 200) {\n        outstandingResults++;\n      } else if (result.statusCode == 200) {\n        for (var column in resultsMap) {\n          cell = sheet.setActiveCell(column + (2 + i));\n\n          try {\n            var value = eval(\"result.\" + resultsMap[column].value); // TODO: remove eval\n\n            // some results field may not exist in some tests e.g. SpeedIndex relies on video capture\n            if (value != undefined) {\n              cell.setValue(eval(\"result.\" + resultsMap[column].value));\n            }\n          } catch (e) {\n            // do nothing\n          }\n        }\n      }\n    }\n  }\n\n  // If all tests have completed cancel the trigger\n  if (outstandingResults == 0) {\n    cancelTrigger();\n  }\n}\n\n/**\n * Retrieves WPT server URL from Settings tab\n *\n * @return {string} server URL\n */\n\nfunction getServerURL() {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var range = spreadsheet.getRange(SERVER_URL);\n\n  return range.getValue(); // TODO check for trailing / and add if necessary\n}\n\n/**\n * Retrieves WPT API key from Settings tab\n *\n * @return {string} API key\n */\n\nfunction getAPIKey() {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var range = spreadsheet.getRange(API_KEY);\n\n  return range.getValue();\n}\n\n/**\n * Retrieves normalizeKeys parameter from Settings tab\n *\n * @return {boolean} normalizeKey\n */\n\nfunction getNormalizeKeys() {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var range = spreadsheet.getRange(NORMALIZE_KEYS);\n\n  return range.getValue();\n}\n\n/**\n * Builds a querystring\n *\n * @param {Array.<{param: string, value: string}>} key/value pairs of URL parameters\n *\n * @return {string} querystring\n */\n\nfunction buildQueryString(params) {\n  var querystring = params.reduce(function (a, b) {\n    return a.concat(\n      encodeURIComponent(b.param) + \"=\" + encodeURIComponent(b.value)\n    );\n  }, []);\n\n  return querystring.join(\"&\");\n}\n\n/**\n * get the parameters map\n */\n\nfunction getParametersMap() {\n  return getMap(PARAMETERS_MAP);\n}\n\n/**\n * get the results map\n */\n\nfunction getResultsMap() {\n  return getMap(RESULTS_MAP);\n}\n\n/**\n * Retrieves map of column name, title and API param or results value the column is mapped to\n *\n * @param {string} rangeName - named range within Spreadsheet\n *\n * @return {dictionary} Object.<string, {name: string, value: string}>\n *\n * TODO: check range has 3 columns\n */\n\nfunction getMap(rangeName) {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var range = spreadsheet.getRange(rangeName);\n  var values = range.getValues();\n\n  var map = {};\n\n  for (n = 0; n < values.length; n++) {\n    map[values[n][0]] = {\n      name: values[n][1],\n      value: values[n][2],\n    };\n  }\n\n  return map;\n}\n\n/**\n * Sets column headers on Scenarios tab\n */\n\nfunction updateScenarioColumns() {\n  var map = getParametersMap();\n\n  var spreadsheet = SpreadsheetApp.getActive();\n  for (var column in map) {\n    cell = spreadsheet.getRange(SCENARIOS_TAB + \"!\" + column + \"1\");\n    cell.setValue(map[column].name);\n  }\n}\n\n/**\n * Sets column headers on Tests tab\n */\n\nfunction updateTestColumns() {\n  var map = getResultsMap();\n\n  var spreadsheet = SpreadsheetApp.getActive();\n  for (var column in map) {\n    cell = spreadsheet.getRange(TESTS_TAB + \"!\" + column + \"1\");\n    cell.setValue(map[column].name);\n  }\n}\n\n/**\n * Builds dictionary of test parameters from Scenarios tab\n *\n * @return {dictionary} Object.<string, {param: string, value: string}>\n */\n\nfunction getTestScenarios() {\n  var spreadsheet = SpreadsheetApp.getActive();\n  var sheet = spreadsheet.getSheetByName(SCENARIOS_TAB);\n\n  var range = sheet.getRange(\n    2,\n    1,\n    sheet.getLastRow() - 1,\n    sheet.getLastColumn()\n  );\n\n  var map = getParametersMap();\n\n  var scenarios = {};\n\n  for (y = 1; y < sheet.getLastRow(); y++) {\n    var scenario = [];\n\n    var cell = range.getCell(y, 1);\n    var name = cell.getValue();\n\n    for (x = 2; x <= sheet.getLastColumn(); x++) {\n      var cell = range.getCell(y, x);\n\n      if (!cell.isBlank()) {\n        var cellName = cell.getA1Notation();\n        var column = cellName.match(\"[A-Z]*\")[0]; // TODO: Urgh\n\n        scenario.push({\n          param: map[column].value,\n          value: cell.getValue(),\n        });\n      }\n    }\n\n    scenarios[name] = scenario;\n  }\n\n  return scenarios;\n}\n\n/**\n * Starts a trigger to call getResults at a defined interval\n *\n * @param {int} number of minutes between each check (must be 1, 5, 10, 15, 30)\n */\n\nfunction startTrigger(interval) {\n  // Check for existing trigger, if it doesn't exist create a new one\n  var spreadsheet = SpreadsheetApp.getActive();\n  var triggerId = ScriptProperties.getProperty(spreadsheet.getId());\n\n  if (!triggerId) {\n    var trigger = ScriptApp.newTrigger(\"getResults\")\n      .timeBased()\n      .everyMinutes(interval)\n      .create();\n\n    ScriptProperties.setProperty(spreadsheet.getId(), trigger.getUniqueId());\n  }\n}\n\n/**\n * Cancels trigger for onResults\n */\n\nfunction cancelTrigger() {\n  var spreadsheet = SpreadsheetApp.getActiveSpreadsheet();\n  var triggerId = ScriptProperties.getProperty(spreadsheet.getId());\n\n  ScriptProperties.deleteProperty(spreadsheet.getId());\n\n  // Locate a trigger by unique ID\n  var allTriggers = ScriptApp.getProjectTriggers();\n\n  // Loop over all triggers\n  for (var i = 0; i < allTriggers.length; i++) {\n    if (allTriggers[i].getUniqueId() == triggerId) {\n      // Found the trigger so now delete it\n      ScriptApp.deleteTrigger(allTriggers[i]);\n      break;\n    }\n  }\n}\n\n/**\n * Determine polling interval for checking test results\n *\n * @param {int} number of tests submitted\n *\n * @return {int} interval between check for test status\n *\n * Appscript supports polling intervals of 1, 5, 10, 15, 30 minutes\n *\n * Need to vary polling interval as can exceed urlfetch quota in large test runs\n */\n\nfunction getPollingInterval(tests) {\n  var pollingInterval;\n\n  if (tests <= 5) {\n    pollingInterval = 1;\n  } else if (tests <= 10) {\n    pollingInterval = 5;\n  } else {\n    pollingInterval = 30;\n  }\n\n  return pollingInterval;\n}\n"
  }
]