class OurEelHacks::Autoscaler

Constants

API_CALLS_PER_SCALE
MILLIS_PER_DAY

Attributes

app_name[RW]
dynos[R]
entered_soft[R]
heroku_api_key[RW]
heroku_rate_limit[RW]
heroku_rate_limit_margin[RW]
last_reading[R]
last_scaled[R]
logger[RW]
lower_limits[RW]
max_dynos[RW]
millis_til_next_scale[R]
min_dynos[RW]
ps_type[RW]
scaling_frequency[RW]
soft_duration[RW]
soft_side[R]
upper_limits[RW]

Public Class Methods

configure(flavor = :web, &block) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 19
def configure(flavor = :web, &block)
  get_instance(flavor).configure(flavor, &block)
end
get_instance(flavor) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 13
def get_instance(flavor)
  flavor = flavor.to_sym
  @instances ||= Hash.new{ |h,k| h[k] = self.new }
  return @instances[flavor]
end
instance_for(flavor = :web) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 23
def instance_for(flavor = :web)
  instance = get_instance(flavor)
  instance.check_settings
  return instance
end
new() click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 67
def initialize()
  @dynos = nil
  @soft_side = nil

  @memoed_dyno_info = nil

  @last_scaled = Time.at(0)
  @entered_soft = Time.at(0)

  @app_name = nil
  @ps_type = nil
  @heroku_api_key = nil

  @min_dynos = 1
  @max_dynos = 10
  @lower_limits = LowerLimit.new(5, 1)
  @upper_limits = UpperLimit.new(30, 50)

  @soft_duration = 10000
  @scaling_frequency = 5000
  @heroku_rate_limit = 80_000
  @heroku_rate_limit_margin = 0.1

  @millis_til_next_scale = nil

  @logger = NullLogger.new
end

Public Instance Methods

check_settings() click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 105
def check_settings
  errors = []
  errors << "No heroku api key set" if @heroku_api_key.nil?
  errors << "No app name set" if @app_name.nil?
  errors << "No process type set" if @ps_type.nil?
  if (MILLIS_PER_DAY / @heroku_rate_limit) *
    (1.0 - @heroku_rate_limit_margin) *
    API_CALLS_PER_SCALE > @scaling_frequency
    errors << "Scaling frequency will lock up Heroku"
  end
  unless errors.empty?
    logger.warn{ "Problems configuring Autoscaler: #{errors.inspect}" }
    raise "OurEelHacks::Autoscaler, configuration problem: " + errors.join(", ")
  end
end
clear_dyno_info() click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 215
def clear_dyno_info
  @memoed_dyno_info = nil
end
configure(flavor = nil) { |self| ... } click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 95
def configure(flavor = nil)
  yield self
  check_settings
  logger.info{ "Autoscaler configured for #{flavor || "{{unknown flavor}}"}"}

  update_dynos(dyno_info.count, Time.now)
  update_scaling_delay(0)
end
dyno_info() click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 219
def dyno_info
  return @memoed_dyno_info ||=
    begin
      regexp = /^#{ps_type}[.].*/
      heroku.ps(app_name).find_all do |dyno|
        dyno["process"] =~ regexp
      end
    end
end
dynos_stable?() click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 229
def dynos_stable?
  return dyno_info.all? do |dyno|
    dyno["state"] == "up"
  end
end
elapsed(start, finish) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 125
def elapsed(start, finish)
  seconds = finish.to_i - start.to_i
  micros = finish.usec - start.usec
  diff = seconds * 1000 + micros / 1000
  logger.debug{ "Elapsed: #{start.to_s}:#{finish.to_s} : #{diff}ms" }
  return diff
end
heroku() click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 235
def heroku
  @heroku ||= HerokuClient.new(logger, heroku_api_key)
end
scale(metric_hash) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 134
def scale(metric_hash)
  logger.debug{ "Scaling request for #{@ps_type}: metrics are: #{metric_hash.inspect}" }

  #TODO: multi-metric scaling logic
  metric = metric_hash.to_a.last.last #Yeah, this is awful
  moment = Time.now
  if elapsed(last_scaled, moment) < millis_til_next_scale
    logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than computed #{millis_til_next_scale}" }
    return
  end

  clear_dyno_info

  starting_wait = millis_til_next_scale

  update_dynos(dyno_info.count, moment)

  target_dynos = target_scale(metric, moment)

  target_dynos = [[target_dynos, max_dynos].min, min_dynos].max
  logger.debug{ "Target dynos at: #{min_dynos}/#{target_dynos}/#{max_dynos} (vs. current: #{@dynos})" }

  set_dynos(target_dynos, moment)

  update_scaling_delay(starting_wait)
rescue => ex
  logger.warn{ "Problem scaling: #{ex.inspect} \t#{ex.backtrace.join("\t\n")}" }
end
set_dynos(count,moment) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 239
def set_dynos(count,moment)
  if count == dynos
    logger.debug{ "Not scaling: #{count} ?= #{dynos}" }
    return
  end

  if not (stable = dynos_stable?)
    logger.debug{ "Not scaling: dynos not stable (iow: not all #{ps_type} dynos are up)" }
    return
  end
  logger.info{ "Scaling from #{dynos} to #{count} dynos for #{ps_type}" }
  heroku.ps_scale(app_name, ps_type, count)
  update_dynos(count, moment)
end
soft_limit(metric, moment) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 174
def soft_limit(metric, moment)
  hit_limit = [lower_limits, upper_limits].find{|lim| lim.includes? metric}

  if soft_side == hit_limit
    if elapsed(entered_soft, moment) > soft_duration
      entered_soft = moment
      case hit_limit
      when upper_limits
        return +1
      when lower_limits
        return -1
      else
        return 0
      end
    else
      return 0
    end
  else
    @entered_soft = moment
  end

  @soft_side = hit_limit
  return 0
end
target_scale(metric, moment) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 163
def target_scale(metric, moment)
  if lower_limits > metric
    return dynos - 1
  elsif upper_limits < metric
    return dynos + 1
  elsif
    result = (dynos + soft_limit(metric, moment))
    return result
  end
end
update_dynos(new_value, moment) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 207
def update_dynos(new_value, moment)
  if new_value != dynos
    @entered_soft = moment
  end
  @dynos = new_value
  @last_scaled = moment
end
update_scaling_delay(starting_wait) click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 199
def update_scaling_delay(starting_wait)
  @millis_til_next_scale = scaling_frequency * @dynos
  if starting_wait > millis_til_next_scale
    logger.debug{ "Adjusting scaling delay for cadence between #{@millis_til_next_scale.inspect} and #{starting_wait.inspect}" }
    @millis_til_next_scale += rand(starting_wait - @millis_til_next_scale)
  end
end