class BigShift::Factory

Public Class Methods

new(config) click to toggle source
# File lib/bigshift/cli.rb, line 189
def initialize(config)
  @config = config
end

Public Instance Methods

big_query_dataset() click to toggle source
# File lib/bigshift/cli.rb, line 205
def big_query_dataset
  @big_query_dataset ||= BigQuery::Dataset.new(bq_service, gcp_project, @config[:bq_dataset_id], logger: logger)
end
cleaner() click to toggle source
# File lib/bigshift/cli.rb, line 209
def cleaner
  @cleaner ||= Cleaner.new(s3_resource, cs_service, logger: logger)
end
cloud_storage_transfer() click to toggle source
# File lib/bigshift/cli.rb, line 197
def cloud_storage_transfer
  @cloud_storage_transfer ||= CloudStorageTransfer.new(cs_transfer_service, gcp_project, aws_credentials, logger: logger)
end
create_unload_manifest(s3_bucket_name, s3_table_prefix) click to toggle source
# File lib/bigshift/cli.rb, line 224
def create_unload_manifest(s3_bucket_name, s3_table_prefix)
  UnloadManifest.new(s3_resource, cs_service, @config[:s3_bucket_name], s3_table_prefix)
end
logger() click to toggle source
# File lib/bigshift/cli.rb, line 220
def logger
  @logger ||= Logger.new($stderr)
end
redshift_table_schema() click to toggle source
# File lib/bigshift/cli.rb, line 201
def redshift_table_schema
  @redshift_table_schema ||= RedshiftTableSchema.new(@config[:rs_schema_name], @config[:rs_table_name], create_rs_connection)
end
redshift_unloader() click to toggle source
# File lib/bigshift/cli.rb, line 193
def redshift_unloader
  @redshift_unloader ||= RedshiftUnloader.new(create_rs_connection, aws_credentials, logger: logger)
end
s3_resource() click to toggle source
# File lib/bigshift/cli.rb, line 213
def s3_resource
  @s3_resource ||= Aws::S3::Resource.new(
    region: aws_region,
    credentials: aws_credentials
  )
end

Private Instance Methods

aws_credentials() click to toggle source
# File lib/bigshift/cli.rb, line 272
def aws_credentials
  @aws_credentials ||= begin
    if @config[:aws_credentials]
      credentials = Aws::Credentials.new(*@config[:aws_credentials].values_at('access_key_id', 'secret_access_key'))
    else
      credentials = Aws::CredentialProviderChain.new.resolve
    end
  end
end
aws_region() click to toggle source
# File lib/bigshift/cli.rb, line 282
def aws_region
  @aws_region ||= begin
    if @config[:aws_credentials]
      region = @config[:aws_credentials]['region']
    else
      region = ENV['AWS_REGION'] || ENV['AWS_DEFAULT_REGION']
    end

    if !region
      raise BigShiftError.new('AWS Region not specified')
    end
  end
end
bq_service() click to toggle source
# File lib/bigshift/cli.rb, line 264
def bq_service
  @bq_service ||= begin
    s = Google::Apis::BigqueryV2::BigqueryService.new
    s.authorization = gcp_credentials
    s
  end
end
create_rs_connection() click to toggle source
# File lib/bigshift/cli.rb, line 230
def create_rs_connection
  rs_connection = PG.connect(
    host: @config[:rs_credentials]['host'],
    port: @config[:rs_credentials]['port'],
    dbname: @config[:rs_database_name],
    user: @config[:rs_credentials]['username'],
    password: @config[:rs_credentials]['password'],
    sslmode: 'require'
  )
  socket = Socket.for_fd(rs_connection.socket)
  socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1)
  socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_KEEPCNT, 5)
  socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_KEEPINTVL, 2)
  socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_KEEPIDLE, 2) if defined?(Socket::TCP_KEEPIDLE)
  rs_connection.exec("SET search_path = \"#{@config[:rs_schema_name]}\"")
  rs_connection
end
cs_service() click to toggle source
# File lib/bigshift/cli.rb, line 256
def cs_service
  @cs_service ||= begin
    s = Google::Apis::StorageV1::StorageService.new
    s.authorization = gcp_credentials
    s
  end
end
cs_transfer_service() click to toggle source
# File lib/bigshift/cli.rb, line 248
def cs_transfer_service
  @cs_transfer_service ||= begin
    s = Google::Apis::StoragetransferV1::StoragetransferService.new
    s.authorization = gcp_credentials
    s
  end
end
gcp_credentials() click to toggle source
# File lib/bigshift/cli.rb, line 304
def gcp_credentials
  @gcp_credentials ||= begin
    if @config[:gcp_credentials]
      credentials = Google::Auth::ServiceAccountCredentials.make_creds(
        json_key_io: StringIO.new(JSON.dump(@config[:gcp_credentials])),
        scope: Google::Apis::StoragetransferV1::AUTH_CLOUD_PLATFORM
      )
    else
      credentials = Google::Auth::GCECredentials.new
    end
  end
end
gcp_project() click to toggle source
# File lib/bigshift/cli.rb, line 296
def gcp_project
  if @config[:gcp_credentials]
    @config[:gcp_credentials]['project_id']
  else
    Google::Cloud.env.project_id
  end
end