class FeedUsGrabber
Public Class Methods
new()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 14 def initialize @mintCacheIntervalUnit = CI_FOREVER @mstrCacheFolder = File.join(Rails.root.to_s,'tmp','CachedWebContent') @mstrCacheFileExt = ".cache" @mstrCacheGroup = "" @mstrStateFile = File.join(Rails.root.to_s,'tmp','FeedUsGrabberState') @mstrClientWhiteList = "" @mstrDebugOutput = "" end
Public Instance Methods
addToDebugOutput(debugOutput, info)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 363 def addToDebugOutput(debugOutput, info) debugOutput << info + "<br />" puts " Debug Trace -> " + info end
autoCacheToFile()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 133 def autoCacheToFile appendDebugOutput("Start autoCacheToFile. bIsPostBack = #{@bIsPostBack}") if @bIsPostBack return end appendDebugOutput("Cache command received = #{@mstrCacheCommand}") if @mstrCacheCommand.nil? || @mstrCacheCommand == '' if (!self.cachedFileExists) or self.cacheFileIsExpired self.createCacheFile end else if @mstrCacheCommand == "clear" if @mstrCacheGroup != '' self.clearCacheGroupFiles(@mstrCacheGroup) elsif @mstrCacheFolder self.clearCacheFolder(@mstrCacheFolder) end end if @mstrCacheCommand == 'clearall' || @mstrCacheCommand == CACHE_COMMAND_FORCE self.clearAllCachedFiles end end end
cacheFileIsExpired()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 162 def cacheFileIsExpired() # Ruby 1.9.x Time.parse expects d/m/y strDate = File.mtime(@mstrCachedFileName).strftime('%d/%m/%Y %I:%M:%S %p') case @mintCacheIntervalUnit when CI_MINUTES then strUnit = 'n' when CI_HOURS then strUnit = 'h' when CI_DAYS then strUnit = 'd' when CI_FOREVER then return false end if self.datediff(strUnit,strDate,Time.now().strftime('%d/%m/%Y %I:%M:%S %p')) > @mintCacheIntervalLength return true else return false end end
cachedFileExists()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 158 def cachedFileExists() File.readable?(@mstrCachedFileName) end
canConnectToFeedUs()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 334 def canConnectToFeedUs() # Used for phone home check canConnect = false if @mstrCacheCommand == CACHE_COMMAND_FORCE canConnect = true else begin r = Net::HTTP.get_response(URI.parse(@mstrDynURL)) if r.body.nil? == false && r.body != ERROR_RESPONSE canConnect = true end rescue logError("Unable to connect to Feed.Us. Cache will not be cleared. URL #{@mstrDynURL}") end end return canConnect; end
clearAllCachedFiles()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 313 def clearAllCachedFiles logMessage = "Clearing all caches at path = #{@mstrCacheFolder}" appendDebugOutput(logMessage) logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a'); grabber_logger = FeedUsGrabberLogger.new(logfile) grabber_logger.info(logMessage) logfile.close self.clearCacheFolder(@mstrCacheFolder); end
clearCacheFolder(folder)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 323 def clearCacheFolder(folder) canConnect = canConnectToFeedUs() if canConnect == true appendDebugOutput("Clear cache folder can connect, removing cache from #{folder}") FileUtils.rm_r Dir.glob("#{folder}/*") else logError("Unable to connect to Feed.Us. Cache will not be cleared. URL that was checked: #{@mstrDynURL}") end end
clearCacheGroupFiles(group)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 296 def clearCacheGroupFiles(group) if group == '.' || group == '..' logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a'); grabber_logger = FeedUsGrabberLogger.new(logfile) grabber_logger.warn("someone requested to delete . OR .. ") logfile.close return end logMessage = "Clearing cache at group #{group} at path #{File.join(@mstrCacheFolder,group)}" appendDebugOutput(logMessage) logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a'); grabber_logger = FeedUsGrabberLogger.new(logfile) grabber_logger.info(logMessage) logfile.close self.clearCacheFolder(File.join(@mstrCacheFolder,group)) end
createCacheFile()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 215 def createCacheFile() sFile = @mstrCachedFileName # Fetch the url begin r = Net::HTTP.get_response(URI.parse(@mstrDynURL)) rescue logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a'); grabber_logger = FeedUsGrabberLogger.new(logfile) grabber_logger.error("Unable to fetch URL #{@mstrDynURL}") logfile.close return; end if self.makeDirectory(File.join(@mstrCacheFolder,@mstrCacheGroup)) File.open(sFile,'w') do |file| file.write(r.body) end return true end return false end
createStateFile()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 191 def createStateFile() File.new(@mstrStateFile,'w') end
datediff(interval, datefrom, dateto, using_timestamps = false)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 245 def datediff(interval, datefrom, dateto, using_timestamps = false) #$interval can be: #yyyy - Number of full years #q - Number of full quarters #m - Number of full months #y - Difference between day numbers #(eg 1st Jan 2004 is "1", the first day. 2nd Feb 2003 is "33". The datediff is "-32".) #d - Number of full days #w - Number of full weekdays #ww - Number of full weeks #h - Number of full hours #n - Number of full minutes #s - Number of full seconds (default) unless using_timestamps datefrom = Time.parse(datefrom).to_i dateto = Time.parse(dateto).to_i end difference = dateto - datefrom # Difference in seconds if interval == 'd' datediff = (difference / 86400).floor elsif interval == 'h' datediff = (difference / 3600).floor elsif interval == 'n' datediff = (difference / 60).floor else datediff = difference end datediff end
getCacheCommand()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 48 def getCacheCommand @mstrCacheCommand end
getCacheFileQualifiers(strURLParams)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 127 def getCacheFileQualifiers(strURLParams) strResults = nil strResults = '_' + strURLParams.gsub(/[=&%\.\/:]+/,'_') strResults end
getCacheFolder()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 52 def getCacheFolder @mstrCacheFolder end
getCacheGroup()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 64 def getCacheGroup @mstrCacheGroup end
getCacheIntervalLength()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 75 def getCacheIntervalLength @mintCacheIntervalLength end
getCacheIntervalUnit()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 36 def getCacheIntervalUnit @mintCacheIntervalUnit end
getCacheNames(sURL)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 107 def getCacheNames(sURL) if (sURL =~ /\?/) > 0 base = sURL.split('?')[0] params = sURL.split('?')[1] else base = sURL params = '' end #strip off the file extension base = base.chomp(File.extname(base)) base.sub!('http://','') base.gsub!(/[\?:\/\.]/,'_') @mstrScriptBaseName = base @mstrCacheFileQualifiers = self.getCacheFileQualifiers(params) @mstrCachedFileName =File.join( @mstrCacheFolder, @mstrCacheGroup, @mstrScriptBaseName + "_" + @mstrCacheFileQualifiers + @mstrCacheFileExt) @mstrCachedFileNameShort = @mstrScriptBaseName + "_" + @mstrCacheFileQualifiers + @mstrCacheFileExt end
getCachedFileName()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 67 def getCachedFileName @getCachedFileName end
getClientWhiteList()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 90 def getClientWhiteList self.readState @mstrClientWhiteList end
getDebugOutput()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 99 def getDebugOutput @mstrDebugOutput end
getDynURL()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 103 def getDynURL @mstrDynURL end
getIncludeFlag()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 60 def getIncludeFlag @mblnInclude end
logError(contents)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 354 def logError(contents) logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a'); grabber_logger = FeedUsGrabberLogger.new(logfile) grabber_logger.error(contents) logfile.close appendDebugOutput(contents) end
makeDirectory(dir, mode = 0755)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 238 def makeDirectory(dir, mode = 0755) if File.directory?(dir) || FileUtils.mkdir_p(dir) return true end return false end
readState()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 187 def readState() readStateFile end
readStateFile()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 195 def readStateFile() if (self.stateFileExists) File.open(@mstrStateFile, "r").each_line do |line| @mstrClientWhiteList = line end end appendDebugOutput("Read from state file #{@mstrStateFile} value #{@mstrClientWhiteList}") end
renderCacheFromFile()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 277 def renderCacheFromFile sFile = @mstrCachedFileName data = '' begin File.open(sFile,'r') do |file| while temp_data = file.gets data = data + temp_data end end data rescue logfile = File.open(File.join(Rails.root.to_s,'log','FeedUsGrabber.log'),'a'); grabber_logger = FeedUsGrabberLogger.new(logfile) grabber_logger.error("Unable to render/open #{@mstrCachedFileName}") logfile.close end end
saveState()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 179 def saveState appendDebugOutput("Does state file exist " + self.stateFileExists.to_s) if (!self.stateFileExists) self.createStateFile end self.writeStateFile end
setCacheCommand(strNewValue)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 44 def setCacheCommand(strNewValue) @mstrCacheCommand = strNewValue end
setCacheFileExt(strNewValue)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 40 def setCacheFileExt(strNewValue) @mstrCacheFileExt = strNewValue end
setCacheFolder(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 28 def setCacheFolder(param) @mstrCacheFolder = param end
setCacheGroup(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 24 def setCacheGroup(param) @mstrCacheGroup = param end
setCacheIntervalLength(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 71 def setCacheIntervalLength(param) @mintCacheIntervalLength = param end
setCacheIntervalUnit(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 32 def setCacheIntervalUnit(param) @mintCacheIntervalUnit = param end
setClientWhiteList(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 84 def setClientWhiteList(param) # Use file to store state across requests @mstrClientWhiteList = param self.saveState end
setDebugOutput(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 95 def setDebugOutput(param) @mstrDebugOutput = param end
setDynURL(param)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 79 def setDynURL(param) @mstrDynURL = param self.getCacheNames(param) end
setIncludeFlag(strNewValue)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 56 def setIncludeFlag(strNewValue) @mblnInclude = $strNewValue end
stateFileExists()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 211 def stateFileExists() File.readable?(@mstrStateFile) end
writeStateFile()
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 204 def writeStateFile appendDebugOutput("Writing to state file #{@mstrStateFile} value #{@mstrClientWhiteList}") File.open(@mstrStateFile,'w') do |file| file.write(@mstrClientWhiteList) end end
Private Instance Methods
appendDebugOutput(info)
click to toggle source
# File lib/app/models/feed_us_grabber.rb, line 370 def appendDebugOutput(info) self.addToDebugOutput(@mstrDebugOutput, info) end