class BigShift::Factory
Public Class Methods
new(config)
click to toggle source
# File lib/bigshift/cli.rb, line 189 def initialize(config) @config = config end
Public Instance Methods
big_query_dataset()
click to toggle source
# File lib/bigshift/cli.rb, line 205 def big_query_dataset @big_query_dataset ||= BigQuery::Dataset.new(bq_service, gcp_project, @config[:bq_dataset_id], logger: logger) end
cleaner()
click to toggle source
# File lib/bigshift/cli.rb, line 209 def cleaner @cleaner ||= Cleaner.new(s3_resource, cs_service, logger: logger) end
cloud_storage_transfer()
click to toggle source
# File lib/bigshift/cli.rb, line 197 def cloud_storage_transfer @cloud_storage_transfer ||= CloudStorageTransfer.new(cs_transfer_service, gcp_project, aws_credentials, logger: logger) end
create_unload_manifest(s3_bucket_name, s3_table_prefix)
click to toggle source
# File lib/bigshift/cli.rb, line 224 def create_unload_manifest(s3_bucket_name, s3_table_prefix) UnloadManifest.new(s3_resource, cs_service, @config[:s3_bucket_name], s3_table_prefix) end
logger()
click to toggle source
# File lib/bigshift/cli.rb, line 220 def logger @logger ||= Logger.new($stderr) end
redshift_table_schema()
click to toggle source
# File lib/bigshift/cli.rb, line 201 def redshift_table_schema @redshift_table_schema ||= RedshiftTableSchema.new(@config[:rs_schema_name], @config[:rs_table_name], create_rs_connection) end
redshift_unloader()
click to toggle source
# File lib/bigshift/cli.rb, line 193 def redshift_unloader @redshift_unloader ||= RedshiftUnloader.new(create_rs_connection, aws_credentials, logger: logger) end
s3_resource()
click to toggle source
# File lib/bigshift/cli.rb, line 213 def s3_resource @s3_resource ||= Aws::S3::Resource.new( region: aws_region, credentials: aws_credentials ) end
Private Instance Methods
aws_credentials()
click to toggle source
# File lib/bigshift/cli.rb, line 272 def aws_credentials @aws_credentials ||= begin if @config[:aws_credentials] credentials = Aws::Credentials.new(*@config[:aws_credentials].values_at('access_key_id', 'secret_access_key')) else credentials = Aws::CredentialProviderChain.new.resolve end end end
aws_region()
click to toggle source
# File lib/bigshift/cli.rb, line 282 def aws_region @aws_region ||= begin if @config[:aws_credentials] region = @config[:aws_credentials]['region'] else region = ENV['AWS_REGION'] || ENV['AWS_DEFAULT_REGION'] end if !region raise BigShiftError.new('AWS Region not specified') end end end
bq_service()
click to toggle source
# File lib/bigshift/cli.rb, line 264 def bq_service @bq_service ||= begin s = Google::Apis::BigqueryV2::BigqueryService.new s.authorization = gcp_credentials s end end
create_rs_connection()
click to toggle source
# File lib/bigshift/cli.rb, line 230 def create_rs_connection rs_connection = PG.connect( host: @config[:rs_credentials]['host'], port: @config[:rs_credentials]['port'], dbname: @config[:rs_database_name], user: @config[:rs_credentials]['username'], password: @config[:rs_credentials]['password'], sslmode: 'require' ) socket = Socket.for_fd(rs_connection.socket) socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1) socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_KEEPCNT, 5) socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_KEEPINTVL, 2) socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_KEEPIDLE, 2) if defined?(Socket::TCP_KEEPIDLE) rs_connection.exec("SET search_path = \"#{@config[:rs_schema_name]}\"") rs_connection end
cs_service()
click to toggle source
# File lib/bigshift/cli.rb, line 256 def cs_service @cs_service ||= begin s = Google::Apis::StorageV1::StorageService.new s.authorization = gcp_credentials s end end
cs_transfer_service()
click to toggle source
# File lib/bigshift/cli.rb, line 248 def cs_transfer_service @cs_transfer_service ||= begin s = Google::Apis::StoragetransferV1::StoragetransferService.new s.authorization = gcp_credentials s end end
gcp_credentials()
click to toggle source
# File lib/bigshift/cli.rb, line 304 def gcp_credentials @gcp_credentials ||= begin if @config[:gcp_credentials] credentials = Google::Auth::ServiceAccountCredentials.make_creds( json_key_io: StringIO.new(JSON.dump(@config[:gcp_credentials])), scope: Google::Apis::StoragetransferV1::AUTH_CLOUD_PLATFORM ) else credentials = Google::Auth::GCECredentials.new end end end
gcp_project()
click to toggle source
# File lib/bigshift/cli.rb, line 296 def gcp_project if @config[:gcp_credentials] @config[:gcp_credentials]['project_id'] else Google::Cloud.env.project_id end end