class OurEelHacks::Autoscaler
Constants
- API_CALLS_PER_SCALE
- MILLIS_PER_DAY
Attributes
app_name[RW]
dynos[R]
entered_soft[R]
heroku_api_key[RW]
heroku_rate_limit[RW]
heroku_rate_limit_margin[RW]
last_reading[R]
last_scaled[R]
logger[RW]
lower_limits[RW]
max_dynos[RW]
millis_til_next_scale[R]
min_dynos[RW]
ps_type[RW]
scaling_frequency[RW]
soft_duration[RW]
soft_side[R]
upper_limits[RW]
Public Class Methods
configure(flavor = :web, &block)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 19 def configure(flavor = :web, &block) get_instance(flavor).configure(flavor, &block) end
get_instance(flavor)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 13 def get_instance(flavor) flavor = flavor.to_sym @instances ||= Hash.new{ |h,k| h[k] = self.new } return @instances[flavor] end
instance_for(flavor = :web)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 23 def instance_for(flavor = :web) instance = get_instance(flavor) instance.check_settings return instance end
new()
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 67 def initialize() @dynos = nil @soft_side = nil @memoed_dyno_info = nil @last_scaled = Time.at(0) @entered_soft = Time.at(0) @app_name = nil @ps_type = nil @heroku_api_key = nil @min_dynos = 1 @max_dynos = 10 @lower_limits = LowerLimit.new(5, 1) @upper_limits = UpperLimit.new(30, 50) @soft_duration = 10000 @scaling_frequency = 5000 @heroku_rate_limit = 80_000 @heroku_rate_limit_margin = 0.1 @millis_til_next_scale = nil @logger = NullLogger.new end
Public Instance Methods
check_settings()
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 105 def check_settings errors = [] errors << "No heroku api key set" if @heroku_api_key.nil? errors << "No app name set" if @app_name.nil? errors << "No process type set" if @ps_type.nil? if (MILLIS_PER_DAY / @heroku_rate_limit) * (1.0 - @heroku_rate_limit_margin) * API_CALLS_PER_SCALE > @scaling_frequency errors << "Scaling frequency will lock up Heroku" end unless errors.empty? logger.warn{ "Problems configuring Autoscaler: #{errors.inspect}" } raise "OurEelHacks::Autoscaler, configuration problem: " + errors.join(", ") end end
clear_dyno_info()
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 215 def clear_dyno_info @memoed_dyno_info = nil end
configure(flavor = nil) { |self| ... }
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 95 def configure(flavor = nil) yield self check_settings logger.info{ "Autoscaler configured for #{flavor || "{{unknown flavor}}"}"} update_dynos(dyno_info.count, Time.now) update_scaling_delay(0) end
dyno_info()
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 219 def dyno_info return @memoed_dyno_info ||= begin regexp = /^#{ps_type}[.].*/ heroku.ps(app_name).find_all do |dyno| dyno["process"] =~ regexp end end end
dynos_stable?()
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 229 def dynos_stable? return dyno_info.all? do |dyno| dyno["state"] == "up" end end
elapsed(start, finish)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 125 def elapsed(start, finish) seconds = finish.to_i - start.to_i micros = finish.usec - start.usec diff = seconds * 1000 + micros / 1000 logger.debug{ "Elapsed: #{start.to_s}:#{finish.to_s} : #{diff}ms" } return diff end
heroku()
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 235 def heroku @heroku ||= HerokuClient.new(logger, heroku_api_key) end
scale(metric_hash)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 134 def scale(metric_hash) logger.debug{ "Scaling request for #{@ps_type}: metrics are: #{metric_hash.inspect}" } #TODO: multi-metric scaling logic metric = metric_hash.to_a.last.last #Yeah, this is awful moment = Time.now if elapsed(last_scaled, moment) < millis_til_next_scale logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than computed #{millis_til_next_scale}" } return end clear_dyno_info starting_wait = millis_til_next_scale update_dynos(dyno_info.count, moment) target_dynos = target_scale(metric, moment) target_dynos = [[target_dynos, max_dynos].min, min_dynos].max logger.debug{ "Target dynos at: #{min_dynos}/#{target_dynos}/#{max_dynos} (vs. current: #{@dynos})" } set_dynos(target_dynos, moment) update_scaling_delay(starting_wait) rescue => ex logger.warn{ "Problem scaling: #{ex.inspect} \t#{ex.backtrace.join("\t\n")}" } end
set_dynos(count,moment)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 239 def set_dynos(count,moment) if count == dynos logger.debug{ "Not scaling: #{count} ?= #{dynos}" } return end if not (stable = dynos_stable?) logger.debug{ "Not scaling: dynos not stable (iow: not all #{ps_type} dynos are up)" } return end logger.info{ "Scaling from #{dynos} to #{count} dynos for #{ps_type}" } heroku.ps_scale(app_name, ps_type, count) update_dynos(count, moment) end
soft_limit(metric, moment)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 174 def soft_limit(metric, moment) hit_limit = [lower_limits, upper_limits].find{|lim| lim.includes? metric} if soft_side == hit_limit if elapsed(entered_soft, moment) > soft_duration entered_soft = moment case hit_limit when upper_limits return +1 when lower_limits return -1 else return 0 end else return 0 end else @entered_soft = moment end @soft_side = hit_limit return 0 end
target_scale(metric, moment)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 163 def target_scale(metric, moment) if lower_limits > metric return dynos - 1 elsif upper_limits < metric return dynos + 1 elsif result = (dynos + soft_limit(metric, moment)) return result end end
update_dynos(new_value, moment)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 207 def update_dynos(new_value, moment) if new_value != dynos @entered_soft = moment end @dynos = new_value @last_scaled = moment end
update_scaling_delay(starting_wait)
click to toggle source
# File lib/our-eel-hacks/autoscaler.rb, line 199 def update_scaling_delay(starting_wait) @millis_til_next_scale = scaling_frequency * @dynos if starting_wait > millis_til_next_scale logger.debug{ "Adjusting scaling delay for cadence between #{@millis_til_next_scale.inspect} and #{starting_wait.inspect}" } @millis_til_next_scale += rand(starting_wait - @millis_til_next_scale) end end