module Mobilize::Gfile

Public Class Methods

add_admin_acl_by_path(path) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 54
def Gfile.add_admin_acl_by_path(path)
  file = Gfile.find_by_path(path)
  file.add_admin_acl
  return true
end
add_worker_acl_by_path(path) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 60
def Gfile.add_worker_acl_by_path(path)
  file = Gfile.find_by_path(path)
  file.add_worker_acl
  return true
end
config() click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 4
def Gfile.config
  Base.config('gfile')
end
find_by_path(path) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 72
def Gfile.find_by_path(path)
  #file must be owned by owner
  gdrive_slot = Gdrive.owner_email
  files = Gdrive.files(gdrive_slot,{"title"=>path,"title-exact"=>"true"})
  dst = Dataset.find_or_create_by_handler_and_path('gfile',path)
  #there should only be one file with each path, otherwise we have fail
  file = nil
  if files.length>1
    #keep most recent file, delete the rest
    files.sort_by do |f| 
      (f.entry_hash[:published] || Time.now).to_time
      end.reverse.each_with_index do |f,f_i|
      if f_i == 0
        file = f
      else
        #delete the old file
        f.delete
        ("Deleted duplicate file #{path}").oputs
      end
    end
  else
    file = files.first
  end
  #always make sure dataset http URL is up to date
  #and that it has admin acl
  if file
    api_url = file.human_url.split("&").first
    dst.update_attributes(:http_url=>api_url)
    file.add_admin_acl
  end
  return file
end
max_length() click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 8
def Gfile.max_length
  Gfile.config['max_length']
end
path_to_dst(path,stage_path,gdrive_slot) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 12
def Gfile.path_to_dst(path,stage_path,gdrive_slot)
  s = Stage.where(:path=>stage_path).first
  params = s.params
  target_path = params['target']
  #if this is the target, it doesn't have to exist already
  is_target = true if path == target_path
  #don't need the ://
  path = path.split("://").last if path.index("://")
  if is_target or Gfile.find_by_path(path)
    handler = "gfile"
    Dataset.find_or_create_by_url("#{handler}://#{path}")
  else
    raise "unable to find #{path}"
  end
end
read_by_dataset_path(dst_path,user_name,*args) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 28
def Gfile.read_by_dataset_path(dst_path,user_name,*args)
  #expects gdrive slot as first arg, otherwise chooses random
  gdrive_slot = args
  worker_emails = Gdrive.worker_emails.sort_by{rand}
  gdrive_slot = worker_emails.first unless worker_emails.include?(gdrive_slot)
  file = Gfile.find_by_path(dst_path)
  file.read(user_name) if file
end
update_acl_by_path(path,gdrive_slot,role="writer",target_email=nil) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 66
def Gfile.update_acl_by_path(path,gdrive_slot,role="writer",target_email=nil)
  file = Gfile.find_by_path(path,target_email)
  raise "File #{path} not found" unless file
  file.update_acl(gdrive_slot,"user",role)
end
write_by_dataset_path(dst_path,string,user_name,*args) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 37
def Gfile.write_by_dataset_path(dst_path,string,user_name,*args)
  #ignores *args as all files must be created and owned by owner
  file = Gfile.find_by_path(dst_path)
  file.delete if file
  owner_root = Gdrive.root(Gdrive.owner_email)
  file = owner_root.upload_from_string(string,
                                dst_path,
                                :content_type=>"test/plain",
                                :convert=>false)
  file.add_admin_acl
  #update http url for file
  dst = Dataset.find_by_handler_and_path("gfile",dst_path)
  api_url = file.human_url.split("&").first
  dst.update_attributes(:http_url=>api_url)
  true
end
write_by_stage_path(stage_path) click to toggle source
# File lib/mobilize-base/handlers/google/gfile.rb, line 105
def Gfile.write_by_stage_path(stage_path)
  gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
  #return blank response if there are no slots available
  return nil unless gdrive_slot
  s = Stage.where(:path=>stage_path).first
  u = s.job.runner.user
  retries = 0
  stdout,stderr = []
  while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
    begin
      #get tsv to write from stage
      source = s.sources(gdrive_slot).first
      raise "Need source for gfile write" unless source
      tsv = source.read(u.name,gdrive_slot)
      raise "No data source found for #{source.url}" unless tsv.to_s.length>0
      if tsv.length > Gfile.max_length
        raise "Too much data; you have #{tsv.length.to_s}, max is #{Gfile.max_length.to_s}"
      end
      stdout = if tsv.length == 0
           #soft error; no data to write. Stage will complete.
           "Write skipped for #{s.target.url}"
         else
           Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot)
           #update status
           "Write successful for #{s.target.url}"
         end
      Gdrive.unslot_worker_by_path(stage_path)
      stderr = nil
      s.update_status(stdout)
      signal = 0
    rescue => exc
      if retries < Gdrive.max_file_write_retries
        retries +=1
        sleep Gdrive.file_write_retry_delay
      else
        stdout = nil
        stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
        signal = 500
      end
    end
  end
  return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
end