Implementing prebid-cache to support Video ads (prebid#1277)

* Added code to cache video content, and made the cache key accessible through the bid.

* Renamed the method we add to the bid.

* Removed an extraneous function.

* Removed some accidental console.log messages

* Added unit tests for the video cache, and fixed a bug or two.

* Refactored the bidmanager a bit to help wrap my head around it, and changed video support so that only bids which have been cached are valid.

* Implemented video end-to-end. Waiting on the modules PR so that it can be transformed into a module.

* Better fix to the utils bug. Modernized the imports.

* Updated documentation.

* Added a few unit tests, and added fixtures for shared data structures.

* Added more thorough tests, and refactored a bit for better code reuse.

* Renamed a function.

* Added tests for adding video bids when the cache fails.

* Removed an unused import.

* Added tests for the adServerManager, and renamed some files for consistency with the normal conventions.

* Renamed the dfpVideo module, and added unit tests for it.

* Moved dfpAdServerVideo into the modules directory.

* Fixed a bug, and added a regression test to catch it in the future.

* Added a bare-bones example page.

* Made some cosmetic changes to names and documentation.

* Removed the shifty API. Updated unit tests. Renamed a property because Ive messed it up multiple times myself already during testing, expecting it to have a different name.

* Most code review comments. Still need to look into the details of VAST.

* Deleted some unused code, and upped the vast version to use 3.0 everywhere.

* Made the cache use the new endpoints

* Fixed style.

* Moved the test page into the tests directory.

* include the bid that ended the auction

modules/dfpAdServerVideo.js

@@ -0,0 +1,88 @@
+/**
+ * This module adds [DFP support]{@link https://www.doubleclickbygoogle.com/} for Video to Prebid.
+ */
+
+import { registerVideoSupport } from '../src/adServerManager';
+import { getWinningBids } from '../src/targeting';
+import { formatQS, format as buildUrl } from '../src/url';
+import { parseSizesInput } from '../src/utils';
+
+/**
+ * @typedef {Object} DfpVideoParams
+ *
+ * This object contains the params needed to form a URL which hits the
+ * [DFP API]{@link https://support.google.com/dfp_premium/answer/1068325?hl=en}.
+ *
+ * All params (except iu, mentioned below) should be considered optional. This module will choose reasonable
+ * defaults for all of the other required params.
+ *
+ * The cust_params property, if present, must be an object. It will be merged with the rest of the
+ * standard Prebid targeting params (hb_adid, hb_bidder, etc).
+ *
+ * @param {string} iu This param *must* be included, in order for us to create a valid request.
+ * @param [string] description_url This field is required if you want Ad Exchange to bid on our ad unit...
+ *   but otherwise optional
+ */
+
+/**
+ * @typedef {Object} DfpVideoOptions
+ *
+ * @param {Object} adUnit The adUnit which this bid is supposed to help fill.
+ * @param [Object] bid The bid which should be considered alongside the rest of the adserver's demand.
+ *   If this isn't defined, then we'll use the winning bid for the adUnit.
+ *
+ * @param {DfpVideoParams} params Query params which should be set on the DFP request.
+ *   These will override this module's defaults whenever they conflict.
+ */
+
+/** Safe defaults which work on pretty much all video calls. */
+const defaultParamConstants = {
+  env: 'vp',
+  gdfp_req: 1,
+  output: 'xml_vast3',
+  unviewed_position_start: 1,
+};
+
+/**
+ * Merge all the bid data and publisher-supplied options into a single URL, and then return it.
+ *
+ * @see [The DFP API]{@link https://support.google.com/dfp_premium/answer/1068325?hl=en#env} for details.
+ *
+ * @param {DfpVideoOptions} options Options which should be used to construct the URL.
+ *
+ * @return {string} A URL which calls DFP, letting options.bid
+ *   (or the auction's winning bid for this adUnit, if undefined) compete alongside the rest of the
+ *   demand in DFP.
+ */
+export default function buildDfpVideoUrl(options) {
+  const adUnit = options.adUnit;
+  const bid = options.bid || getWinningBids(adUnit.code)[0];
+
+  const derivedParams = {
+    correlator: Date.now(),
+    sz: parseSizesInput(adUnit.sizes).join('|'),
+    url: location.href,
+  };
+
+  const customParams = Object.assign({},
+    bid.adserverTargeting,
+    { hb_uuid: bid.videoCacheKey },
+    options.params.cust_params);
+
+  const queryParams = Object.assign({},
+    defaultParamConstants,
+    derivedParams,
+    options.params,
+    { cust_params: encodeURIComponent(formatQS(customParams))});
+
+  return buildUrl({
+    protocol: 'https',
+    host: 'pubads.g.doubleclick.net',
+    pathname: '/gampad/ads',
+    search: queryParams
+  });
+}
+
+registerVideoSupport('dfp', {
+  buildVideoUrl: buildDfpVideoUrl
+});

src/adServerManager.js

@@ -0,0 +1,54 @@
+import { getGlobal } from 'src/prebidGlobal';
+import { logWarn } from 'src/utils';
+
+const prebid = getGlobal();
+
+/**
+ * This file defines the plugin points in prebid-core for AdServer-specific functionality.
+ *
+ * Its main job is to expose functions for AdServer modules to append functionality to the Prebid public API.
+ * For a given Ad Server with name "adServerName", these functions will only change the API in the
+ * $$PREBID_GLOBAL$$.adServers[adServerName] namespace.
+ */
+
+/**
+ * @typedef {Object} CachedVideoBid
+ *
+ * @property {string} videoCacheId The ID which can be used to retrieve this video from prebid-server.
+ *   This is the same ID given to the callback in the videoCache's store function.
+ */
+
+/**
+ * @function VideoAdUrlBuilder
+ *
+ * @param {CachedVideoBid} bid The winning Bid which the ad server should show, assuming it beats out
+ *   the competition.
+ *
+ * @param {Object} options Options required by the Ad Server to make a valid AdServer URL.
+ *   This object will have different properties depending on the specific ad server supported.
+ *   For more information, see the docs inside the ad server module you're supporting.
+ *
+ * @return {string} A URL which can be passed into the Video player to play an ad.
+ */
+
+/**
+ * @typedef {Object} VideoSupport
+ *
+ * @function {VideoAdUrlBuilder} buildVideoAdUrl
+ */
+
+/**
+ * Enable video support for the Ad Server.
+ *
+ * @property {string} name The identifying name for this adserver.
+ * @property {VideoSupport} videoSupport An object with the functions needed to support video in Prebid.
+ */
+export function registerVideoSupport(name, videoSupport) {
+  prebid.adServers = prebid.adServers || { };
+  prebid.adServers[name] = prebid.adServers[name] || { };
+  if (prebid.adServers[name].buildVideoUrl) {
+    logWarn(`Multiple calls to registerVideoSupport for AdServer ${name}. Expect surprising behavior.`);
+    return;
+  }
+  prebid.adServers[name].buildVideoUrl = videoSupport.buildVideoUrl;
+}

src/bidmanager.js

@@ -1,6 +1,7 @@
 import { uniques, flatten, adUnitsFilter, getBidderRequest } from './utils';
 import {getPriceBucketString} from './cpmBucketManager';
 import {NATIVE_KEYS, nativeBidIsValid} from './native';
+import { store } from './videoCache';
 
 var CONSTANTS = require('./constants.json');
 var AUCTION_END = CONSTANTS.EVENTS.AUCTION_END;
@@ -87,17 +88,43 @@ exports.bidsBackAll = function () {
  *   This function should be called to by the bidder adapter to register a bid response
  */
 exports.addBidResponse = function (adUnitCode, bid) {
-  if (!adUnitCode) {
-    utils.logWarn('No adUnitCode supplied to addBidResponse, response discarded');
-    return;
+  if (isValid()) {
+    prepareBidForAuction();
+
+    if (bid.mediaType === 'video') {
+      tryAddVideoBid(bid);
+    } else {
+      doCallbacksIfNeeded();
+      addBidToAuction(bid);
+    }
   }
 
-  if (bid) {
+  // Actual method logic is above. Everything below is helper functions.
+
+  // Validate the arguments sent to us by the adapter. If this returns false, the bid should be totally ignored.
+  function isValid() {
+    function errorMessage(msg) {
+      return `Invalid bid from ${bid.bidderCode}. Ignoring bid: ${msg}`;
+    }
+
+    if (!adUnitCode) {
+      utils.logWarn(errorMessage('No adUnitCode was supplied to addBidResponse.'));
+      return false;
+    }
     if (bid.mediaType === 'native' && !nativeBidIsValid(bid)) {
-      utils.logError(`Native bid response does not contain all required assets. This bid won't be addeed to the auction`);
-      return;
+      utils.logError(errorMessage('Native bid missing some required properties.'));
+      return false;
     }
+    if (bid.mediaType === 'video' && !bid.vastUrl) {
+      utils.logError(errorMessage(`Video bid does not have required vastUrl property.`));
+      return false;
+    }
+    return true;
+  }
 
+  // Postprocess the bids so that all the universal properties exist, no matter which bidder they came from.
+  // This should be called before addBidToAuction().
+  function prepareBidForAuction() {
     const { requestId, start } = getBidderRequest(bid.bidderCode, adUnitCode);
     Object.assign(bid, {
       requestId: requestId,
@@ -110,18 +137,6 @@ exports.addBidResponse = function (adUnitCode, bid) {
 
     bid.timeToRespond = bid.responseTimestamp - bid.requestTimestamp;
 
-    if (bid.timeToRespond > $$PREBID_GLOBAL$$.cbTimeout + $$PREBID_GLOBAL$$.timeoutBuffer) {
-      const timedOut = true;
-
-      exports.executeCallback(timedOut);
-    }
-
-    // emit the bidAdjustment event before bidResponse, so bid response has the adjusted bid value
-    events.emit(CONSTANTS.EVENTS.BID_ADJUSTMENT, bid);
-
-    // emit the bidResponse event
-    events.emit(CONSTANTS.EVENTS.BID_RESPONSE, bid);
-
     // append price strings
     const priceStringsObj = getPriceBucketString(bid.cpm, _customPriceBucket);
     bid.pbLg = priceStringsObj.low;
@@ -138,15 +153,44 @@ exports.addBidResponse = function (adUnitCode, bid) {
     }
 
     bid.adserverTargeting = keyValues;
-    $$PREBID_GLOBAL$$._bidsReceived.push(bid);
   }
 
-  if (bid && bid.adUnitCode && bidsBackAdUnit(bid.adUnitCode)) {
-    triggerAdUnitCallbacks(bid.adUnitCode);
+  function doCallbacksIfNeeded() {
+    if (bid.timeToRespond > $$PREBID_GLOBAL$$.cbTimeout + $$PREBID_GLOBAL$$.timeoutBuffer) {
+      const timedOut = true;
+      exports.executeCallback(timedOut);
+    }
+  }
+
+  // Add a bid to the auction.
+  function addBidToAuction() {
+    // Make sure that the bidAdjustment event fires before bidResponse, so that the bid response
+    // has the adjusted bid value
+    events.emit(CONSTANTS.EVENTS.BID_ADJUSTMENT, bid);
+    events.emit(CONSTANTS.EVENTS.BID_RESPONSE, bid);
+
+    $$PREBID_GLOBAL$$._bidsReceived.push(bid);
+
+    if (bid.adUnitCode && bidsBackAdUnit(bid.adUnitCode)) {
+      triggerAdUnitCallbacks(bid.adUnitCode);
+    }
+
+    if (bidsBackAll()) {
+      exports.executeCallback();
+    }
   }
 
-  if (bidsBackAll()) {
-    exports.executeCallback();
+  // Video bids may fail if the cache is down, or there's trouble on the network.
+  function tryAddVideoBid(bid) {
+    store([bid], function(error, cacheIds) {
+      if (error) {
+        utils.logWarn(`Failed to save to the video cache: ${error}. Video bid must be discarded.`);
+      } else {
+        bid.videoCacheKey = cacheIds[0].uuid;
+        addBidToAuction(bid);
+      }
+      doCallbacksIfNeeded();
+    });
   }
 };
 

src/prebid.js

@@ -657,6 +657,11 @@ $$PREBID_GLOBAL$$.getAllWinningBids = function () {
  * Build master video tag from publishers adserver tag
  * @param {string} adserverTag default url
  * @param {object} options options for video tag
+ *
+ * @deprecated Include the dfpVideoSupport module in your build, and use the
+ *   $$PREBID_GLOBAL$$.adserver.buildVideoAdUrl (if DFP is your only Ad Server) or
+ *   $$PREBID_GLOBAL$$.adservers.dfp.buildVideoAdUrl (if you use other Ad Servers too)
+ *   function instead. This function will be removed in Prebid 1.0.
  */
 $$PREBID_GLOBAL$$.buildMasterVideoTagFromAdserverTag = function (adserverTag, options) {
   utils.logInfo('Invoking $$PREBID_GLOBAL$$.buildMasterVideoTagFromAdserverTag', arguments);

src/utils.js

@@ -112,7 +112,7 @@ exports.transformAdServerTargetingObj = function (targeting) {
  * @param  {array[array|number]} sizeObj Input array or double array [300,250] or [[300,250], [728,90]]
  * @return {array[string]}  Array of strings like `["300x250"]` or `["300x250", "728x90"]`
  */
-exports.parseSizesInput = function (sizeObj) {
+export function parseSizesInput(sizeObj) {
   var parsedSizes = [];
 
   // if a string for now we can assume it is a single size, like "300x250"
@@ -137,11 +137,11 @@ exports.parseSizesInput = function (sizeObj) {
     if (sizeArrayLength > 0) {
       // if we are a 2 item array of 2 numbers, we must be a SingleSize array
       if (sizeArrayLength === 2 && typeof sizeObj[0] === objectType_number && typeof sizeObj[1] === objectType_number) {
-        parsedSizes.push(this.parseGPTSingleSizeArray(sizeObj));
+        parsedSizes.push(parseGPTSingleSizeArray(sizeObj));
       } else {
         // otherwise, we must be a MultiSize array
         for (var i = 0; i < sizeArrayLength; i++) {
-          parsedSizes.push(this.parseGPTSingleSizeArray(sizeObj[i]));
+          parsedSizes.push(parseGPTSingleSizeArray(sizeObj[i]));
         }
       }
     }
@@ -152,9 +152,9 @@ exports.parseSizesInput = function (sizeObj) {
 
 // parse a GPT style sigle size array, (i.e [300,250])
 // into an AppNexus style string, (i.e. 300x250)
-exports.parseGPTSingleSizeArray = function (singleSize) {
+export function parseGPTSingleSizeArray(singleSize) {
   // if we aren't exactly 2 items in this array, it is invalid
-  if (this.isArray(singleSize) && singleSize.length === 2 && (!isNaN(singleSize[0]) && !isNaN(singleSize[1]))) {
+  if (exports.isArray(singleSize) && singleSize.length === 2 && (!isNaN(singleSize[0]) && !isNaN(singleSize[1]))) {
     return singleSize[0] + 'x' + singleSize[1];
   }
 };