class PatriotAWS::Command::S3Command

Constants

COMMAND_COPY
MODE_UPLOAD_FROM_LOCAL_TO_S3
S3_COMMANDS
S3_PROTOCOLS

Public Instance Methods

configure() click to toggle source

@see Patriot::Command::Base#configure

# File lib/patriot_aws/command/s3.rb, line 21
def configure
  @name_suffix ||= _date_
  self
end
execute() click to toggle source
# File lib/patriot_aws/command/s3.rb, line 26
def execute
  @logger.info "start s3 #{@command}"

  @options ||= {}
  @options = @options.symbolize_keys
  @options = _set_options(@inifile, @options)

  _check_attrs(@command, @options, @src, @dest)

  config_aws(@options)
  s3_cli = Aws::S3::Client.new

  case @command
  when COMMAND_COPY.to_s
    _copy(s3_cli, @src, @dest, @options)
  else
    # should not reach here because this check is already done
    # in configure
    raise Exception,
          'command is invalid. '\
          "supported commands are #{S3_COMMANDS.map(&:to_s)}"
  end
end
job_id() click to toggle source
# File lib/patriot_aws/command/s3.rb, line 15
def job_id
  job_id = "#{command_name}_#{@command}_#{@name}_#{@name_suffix}"
  job_id
end

Private Instance Methods

_check_attrs(command, options, src, dest) click to toggle source

@private check command_attr @param String command @param Hash options @param String src @param String dest

# File lib/patriot_aws/command/s3.rb, line 81
def _check_attrs(command, options, src, dest)
  # check region
  raise Exception, 'region is not set.' if options[:region].nil?

  # check credentials
  if options[:access_key_id].nil?
    raise Exception, 'access_key_id is not set.'
  end
  if options[:secret_access_key].nil?
    raise Exception, 'secret_access_key is not set.'
  end

  # check command
  raise Exception, 's3 comamnd is not set.' if command.nil?

  # check command and resources
  case command
  when COMMAND_COPY.to_s
    raise Exception, 'src or dest are not set.' if src.nil? || dest.nil?

    # check target if file exists
    raise Exception, 'src file does not exist.' unless File.exist?(src)

    # check if file is not empty
    raise Exception, 'The target file is empty.' unless File.size?(src)
  else
    raise Exception,
          'command is invalid. '\
          "supported commands are #{S3_COMMANDS.map(&:to_s)}"
  end
end
_copy(s3_cli, src, dest, options) click to toggle source

@private copy file(s) between file and s3 @param Aws::S3::Client s3_cli @param String src @param String dest @param Hash options

# File lib/patriot_aws/command/s3.rb, line 120
def _copy(s3_cli, src, dest, options)
  @logger.info "source is #{@src}"
  @logger.info "destination is #{@dest}"
  @logger.info "region is #{options[:region]}"

  path_info = {}
  # path_info is going to be like:
  # path_info = {
  #   "src"=>{
  #     "protocol"=>"file",
  #     "path"=>"/path/to/file"
  #   }
  #   "dest"=>{
  #     "protocol"=>"s3",
  #     "bucket"=>"bucket",
  #     "key"=>"test_key"
  #   }
  # }
  path_info['src'] = _path_info(src)
  path_info['dest'] = _path_info(dest)

  mode = _mode_of_copy(path_info)
  @logger.info "copy mode is #{mode}"

  if mode == MODE_UPLOAD_FROM_LOCAL_TO_S3
    _upload_from_local_to_s3(s3_cli, path_info, options)
  end
end
_divide_bucket_and_key(path) click to toggle source

@private divide s3 bucket + key into bucket and key @param String bucket + key @return Array [bucket, key]

# File lib/patriot_aws/command/s3.rb, line 242
def _divide_bucket_and_key(path)
  path.split('/', 2)
end
_divide_protocol_and_bucket_key(path) click to toggle source

@private divide path into protocol and s3 bucket + key @param String path @return Array [protocol, bucket + key]

# File lib/patriot_aws/command/s3.rb, line 233
def _divide_protocol_and_bucket_key(path)
  path.split('://')
end
_mode_of_copy(path_info) click to toggle source

@private get the mode of copy only MODE_UPLOAD_FROM_LOCAL_TO_S3 is supported now @param Hash path_info @return Symbol

# File lib/patriot_aws/command/s3.rb, line 182
def _mode_of_copy(path_info)
  # TODO: need to implement
  # MODE_UPLOAD_FROM_LOCAL_TO_S3
  # MODE_UPLOAD_FROM_HDFS_TO_S3
  # MODE_GET_FROM_S3_TO_LOCAL
  # MODE_GET_FROM_S3_TO_HDFS
  # MODE_COPY and
  # MODE_RECURSIVE_UPLOAD_FROM_LOCAL_TO_S3
  # MODE_RECURSIVE_UPLOAD_FROM_HDFS_TO_S3
  # puts path_info['src']['protocol']
  # puts path_info['dest']['protocol']
  # puts S3_PROTOCOLS
  # puts S3_PROTOCOLS.class
  # puts S3_PROTOCOLS.include?(path_info['dest']['protocol'])

  if path_info['src']['protocol'] == 'file' &&
     S3_PROTOCOLS.include?(path_info['dest']['protocol'])
    return MODE_UPLOAD_FROM_LOCAL_TO_S3
  else
    raise Exception,
          'only the mode to "copy" from local to s3 is supported now'
  end
end
_path_info(path) click to toggle source

@private get path information @param String path @return Hash {protocol: 'file', path: '/path/to/file'}

or {protocol: 's3' or sth,, bucket: 'bucket', key: 'key'}
# File lib/patriot_aws/command/s3.rb, line 155
def _path_info(path)
  path_info = {}
  path_info['protocol'], bucket_key =
    _divide_protocol_and_bucket_key(path)

  # path has no protocol
  if path_info['protocol'] == 'file' || bucket_key.nil?
    path_info['path'] = path_info['protocol']
    path_info['protocol'] = 'file'
  # path has a protocol
  elsif bucket_key
    path_info['bucket'], path_info['key'] =
      _divide_bucket_and_key(bucket_key)
    raise Exception, 's3 object key is not set' if path_info['key'].nil?
  else
    raise Exception, 's3 bucket and object key are not set'
  end

  path_info
end
_set_options(inifile, options) click to toggle source

@private get inifile info and set options parameters @param String inifile @param Hash options @return Hash options

# File lib/patriot_aws/command/s3.rb, line 55
def _set_options(inifile, options)
  if inifile
    ini = IniFile.load(inifile)
    raise Exception, 'inifile not found.' if ini.nil?

    options[:access_key_id] =
      ini['common']['access_key_id'] || ini['s3']['access_key_id']
    options[:secret_access_key] =
      ini['s3']['secret_access_key'] || ini['common']['secret_access_key']
    options[:region] ||=
      ini['s3']['region'] || ini['common']['region']
  end

  options[:cmd_opts] = {} if options[:cmd_opts].nil?
  options[:cmd_opts][:multipart_threshold] ||= 15_728_640

  options
end
_upload_from_local_to_s3(s3_cli, path_info, options) click to toggle source

@private upload file from local to s3 @param Aws::S3::Client s3_cli @param Hash path_info @param Hash options

# File lib/patriot_aws/command/s3.rb, line 212
def _upload_from_local_to_s3(s3_cli, path_info, options)
  @logger.info 'start uploading...'
  s3_resource = Aws::S3::Resource.new(client: s3_cli)
  obj = s3_resource
        .bucket(path_info['dest']['bucket'])
        .object(path_info['dest']['key'])

  obj.upload_file(
    path_info['src']['path'],
    options[:cmd_opts].symbolize_keys
  )
  @logger.info 'end upload'
rescue Aws::S3::MultipartUploadError => errors
  puts "failed upload file to S3: #{errors.message}"
end