Reporting active A/B tests to Slack using Split, Sidekiq, Sidetiq

report_open_experiments_to_yappie_job.rb

class ReportOpenExperimentsToYappieJob
  include Sidekiq::Worker
  sidekiq_options :queue => :yappie

  include Sidetiq::Schedulable
  recurrence { daily.hour_of_day(9) }

  def perform
    active_experiments = Split::Experiment.all.select{ |exp| exp.winner.nil? }
    active_experiments.each do |experiment|
      attachments = []
      experiment.alternatives.each do |alternative|
        attachments.push({
          fallback: "Experiment data not shown in this client.",
          pretext: "Experiment status:",
          color: alternative.to_s == experiment.winner.to_s ? "good" : "#222222",
          fields: [
            {
              title: "Alternative",
              value: alternative.name,
              short: true
            },
            {
              title: "Participants",
              value: alternative.participant_count,
              short: true
            },
            {
              title: "Completed",
              value: alternative.completed_count,
              short: true
            },
            {
              title: "Conversion Rate",
              value: conversion_rate(alternative, experiment),
              short: true
            },
            {
              title: "Confidence",
              value: confidence_level(alternative.z_score),
              short: true
            }
          ]
        })
      end
      message = "Active experiment: #{experiment.name}"
      puts message
      puts attachments
      Slack.new.post_message(message, attachments)
    end
  end

  def conversion_rate(alternative, experiment)
    [number_to_percentage(round(alternative.conversion_rate, 3)), diff_percent(experiment, alternative)].compact.join(' ')
  end

  def round(number, precision = 2)
    BigDecimal.new(number.to_s).round(precision).to_f
  end

  def number_to_percentage(number, precision = 2)
    "#{round(number * 100)}%"
  end

  def confidence_level(z_score)
    return z_score if z_score.is_a? String

    z = round(z_score.to_s.to_f, 3).abs

    if z == 0.0
      'No Change'
    elsif z < 1.645
      'no confidence'
    elsif z < 1.96
      '95% confidence'
    elsif z < 2.57
      '99% confidence'
    else
      '99.9% confidence'
    end
  end

  def diff_percent(experiment, alternative)
    if experiment.control.conversion_rate > 0 && !alternative.control?
      if alternative.conversion_rate > experiment.control.conversion_rate
        "(+#{number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1)})"
      elsif alternative.conversion_rate < experiment.control.conversion_rate
        "(#{number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1)})"
      end
    end
  end
end

slack.rb

class Slack
  def initialize(organization = "yapp", token = ENV['SLACK_TOKEN'])
    @token = token
    @url = "https://#{organization}.slack.com/services/hooks/incoming-webhook?token=#{token}"
  end

  def post_message(message, attachments = nil)
    payload = {
      text: message,
      channel: "#metrics",
      username: "yappie",
      icon_url: "https://www.yapp.us/images/yappie_48.gif"
    }
    if attachments
      payload[:attachments] = attachments
    end

    conn = Faraday.new(:url => @url) do |faraday|
      faraday.request  :url_encoded
      faraday.response :logger
      faraday.adapter  Faraday.default_adapter
    end

    if !@token
      Rails.logger.warn("No token was provided for slack posting, so we will skip it. Payload was: " + payload.to_json)
    else
      conn.post do |req|
        req.headers['Content-Type'] = 'application/json'
        req.body = payload.to_json
      end
    end
  end
end