class GoodData::Model::ProjectCreator
Public Class Methods
Source
# File lib/gooddata/models/project_creator.rb, line 75 def apply_maqls(ca_chunks, project, replaced_maqls, opts) errors = [] replaced_maqls.each do |replaced_maql_chunks| begin fallback_hard_sync = replaced_maql_chunks['updateScript']['fallbackHardSync'].nil? ? false : replaced_maql_chunks['updateScript']['fallbackHardSync'] replaced_maql_chunks['updateScript']['maqlDdlChunks'].each do |chunk| GoodData.logger.debug(chunk) execute_maql_result = project.execute_maql(chunk) process_fallback_hard_sync_result(execute_maql_result, project) if fallback_hard_sync end rescue => e GoodData.logger.error("Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{replaced_maql_chunks.inspect}") errors << e next end end if ca_chunks && opts[:execute_ca_scripts] begin ca_chunks.each { |chunk| project.execute_maql(chunk) } rescue => e GoodData.logger.error("Error occured when executing MAQL, project: \"#{project.title}\" reason: \"#{e.message}\", chunks: #{ca_chunks.inspect}") errors << e end end if (!errors.empty?) && (errors.length == replaced_maqls.length) messages = errors.map { |err| GoodData::Helpers.interpolate_error_messages(err.data['wTaskStatus']['messages']) } fail MaqlExecutionError.new("Unable to migrate LDM, reason(s): \n #{messages.join("\n")}", errors) end end
Source
# File lib/gooddata/models/project_creator.rb, line 136 def execute_tests(_project, spec) spec.each do |assert| result = GoodData::ReportDefinition.execute(assert[:report]) fail "Test did not pass. Got #{result.table.inspect}, expected #{assert[:result].inspect}" if result.table != assert[:result] end end
Source
# File lib/gooddata/models/project_creator.rb, line 127 def load(project, spec) if spec.key?(:uploads) # rubocop:disable Style/GuardClause spec[:uploads].each do |load| schema = GoodData::Model::Schema.new(spec[:datasets].find { |d| d[:name] == load[:dataset] }) project.upload(load[:source], schema, load[:mode]) end end end
Source
# File lib/gooddata/models/project_creator.rb, line 17 def migrate(opts = {}) opts = { client: GoodData.connection, execute_ca_scripts: true }.merge(opts) client = opts[:client] fail ArgumentError, 'No :client specified' if client.nil? spec = opts[:spec] || fail('You need to provide spec for migration') bp = ProjectBlueprint.new(spec) fail GoodData::ValidationError, "Blueprint is invalid #{bp.validate.inspect}" unless bp.valid? spec = bp.to_hash project = opts[:project] || client.create_project(opts.merge(:title => opts[:title] || spec[:title], :client => client, :environment => opts[:environment])) _maqls, ca_maqls = migrate_datasets(spec, opts.merge(project: project, client: client)) load(p, spec) migrate_metrics(p, spec[:metrics] || []) migrate_reports(p, spec[:reports] || []) migrate_dashboards(p, spec[:dashboards] || []) execute_tests(p, spec[:assert_tests] || []) opts[:execute_ca_scripts] ? project : ca_maqls end
Source
# File lib/gooddata/models/project_creator.rb, line 113 def migrate_dashboards(project, spec) spec.each do |dash| project.add_dashboard(dash) end end
Source
# File lib/gooddata/models/project_creator.rb, line 38 def migrate_datasets(spec, opts = {}) opts = { client: GoodData.connection }.merge(opts) dry_run = opts[:dry_run] replacements = opts['maql_replacements'] || opts[:maql_replacements] || {} update_preference = opts[:update_preference] exist_fallback_to_hard_sync_config = !update_preference.nil? && !update_preference[:fallback_to_hard_sync].nil? include_maql_fallback_hard_sync = exist_fallback_to_hard_sync_config && GoodData::Helpers.to_boolean(update_preference[:fallback_to_hard_sync]) _, project = GoodData.get_client_and_project(opts) bp = ProjectBlueprint.new(spec) response = opts[:maql_diff] unless response maql_diff_params = [:includeGrain] maql_diff_params << :excludeFactRule if opts[:exclude_fact_rule] maql_diff_params << :includeDeprecated if opts[:include_deprecated] maql_diff_params << :includeMaqlFallbackHardSync if include_maql_fallback_hard_sync maql_diff_time = Benchmark.realtime do response = project.maql_diff(blueprint: bp, params: maql_diff_params) end GoodData.logger.debug("MAQL diff took #{maql_diff_time.round} seconds") end GoodData.logger.debug("projectModelDiff") { response.pretty_inspect } chunks = response['projectModelDiff']['updateScripts'] return [], nil if chunks.empty? ca_maql = response['projectModelDiff']['computedAttributesScript'] if response['projectModelDiff']['computedAttributesScript'] ca_chunks = ca_maql && ca_maql['maqlDdlChunks'] maqls = include_maql_fallback_hard_sync ? pick_correct_chunks_hard_sync(chunks, opts) : pick_correct_chunks(chunks, opts) replaced_maqls = apply_replacements_on_maql(maqls, replacements) apply_maqls(ca_chunks, project, replaced_maqls, opts) unless dry_run [replaced_maqls, ca_maql] end
Source
# File lib/gooddata/models/project_creator.rb, line 119 def migrate_metrics(project, spec) spec.each do |metric| project.add_metric(metric) end end
Also aliased as: migrate_measures
Source
# File lib/gooddata/models/project_creator.rb, line 107 def migrate_reports(project, spec) spec.each do |report| project.add_report(report) end end
Source
# File lib/gooddata/models/project_creator.rb, line 143 def pick_correct_chunks(chunks, opts = {}) GoodData.logger.debug("update_preference") { opts[:update_preference].pretty_inspect } preference = GoodData::Helpers.symbolize_keys(opts[:update_preference] || {}) preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }] # will use new parameters instead of the old ones if preference.empty? || %i[allow_cascade_drops keep_data].any? { |k| preference.key?(k) } if %i[cascade_drops preserve_data].any? { |k| preference.key?(k) } fail "Please do not mix old parameters (:cascade_drops, :preserve_data) with the new ones (:allow_cascade_drops, :keep_data)." end preference = { allow_cascade_drops: false, keep_data: true }.merge(preference) new_preference = {} new_preference[:cascade_drops] = false unless preference[:allow_cascade_drops] new_preference[:preserve_data] = true if preference[:keep_data] preference = new_preference end # first is cascadeDrops, second is preserveData rules = [ { priority: 1, cascade_drops: false, preserve_data: true }, { priority: 2, cascade_drops: false, preserve_data: false }, { priority: 3, cascade_drops: true, preserve_data: true }, { priority: 4, cascade_drops: true, preserve_data: false } ] stuff = chunks.select do |chunk| chunk['updateScript']['maqlDdlChunks'] end stuff = stuff.map do |chunk| { cascade_drops: chunk['updateScript']['cascadeDrops'], preserve_data: chunk['updateScript']['preserveData'], maql: chunk['updateScript']['maqlDdlChunks'], orig: chunk } end results_from_api = GoodData::Helpers.join( rules, stuff, %i[cascade_drops preserve_data], %i[cascade_drops preserve_data], inner: true ).sort_by { |l| l[:priority] } || [] if preference.empty? [results_from_api.first[:orig]] else results = results_from_api.dup preference.each do |k, v| results = results.select do |result| result[k] == v end end if results.empty? available_chunks = results_from_api .map do |result| { cascade_drops: result[:cascade_drops], preserve_data: result[:preserve_data] } end .map(&:to_s) .join(', ') fail "Synchronize LDM cannot proceed. Adjust your update_preferences and try again. Available chunks with preference: #{available_chunks}" end results.map { |result| result[:orig] } end end
Source
# File lib/gooddata/models/project_creator.rb, line 213 def pick_correct_chunks_hard_sync(chunks, opts = {}) preference = GoodData::Helpers.symbolize_keys(opts[:update_preference] || {}) preference = Hash[preference.map { |k, v| [k, GoodData::Helpers.to_boolean(v)] }] # Old configure using cascade_drops and preserve_data parameters. New configure using allow_cascade_drops and # keep_data parameters. Need translate from new configure to old configure before processing if preference.empty? || %i[allow_cascade_drops keep_data].any? { |k| preference.key?(k) } if %i[cascade_drops preserve_data].any? { |k| preference.key?(k) } fail "Please do not mix old parameters (:cascade_drops, :preserve_data) with the new ones (:allow_cascade_drops, :keep_data)." end # Default allow_cascade_drops=false and keep_data=true preference = { allow_cascade_drops: false, keep_data: true }.merge(preference) new_preference = {} new_preference[:cascade_drops] = preference[:allow_cascade_drops] new_preference[:preserve_data] = preference[:keep_data] preference = new_preference end preference[:fallback_to_hard_sync] = true # Filter chunk with fallbackHardSync = true result = chunks.select do |chunk| chunk['updateScript']['maqlDdlChunks'] && !chunk['updateScript']['fallbackHardSync'].nil? && chunk['updateScript']['fallbackHardSync'] end # The API model/diff only returns one result for MAQL fallback hard synchronize result = pick_chunks_hard_sync(result[0], preference) if !result.nil? && !result.empty? if result.nil? || result.empty? available_chunks = chunks .map do |chunk| { cascade_drops: chunk['updateScript']['cascadeDrops'], preserve_data: chunk['updateScript']['preserveData'], fallback_hard_sync: chunk['updateScript']['fallbackHardSync'].nil? ? false : chunk['updateScript']['fallbackHardSync'] } end .map(&:to_s) .join(', ') fail "Synchronize LDM cannot proceed. Adjust your update_preferences and try again. Available chunks with preference: #{available_chunks}" end result end
Private Class Methods
Source
# File lib/gooddata/models/project_creator.rb, line 262 def apply_replacements_on_maql(maqls, replacements = {}) maqls.map do |maql| GoodData::Helpers.deep_dup(maql).tap do |m| m['updateScript']['maqlDdlChunks'] = m['updateScript']['maqlDdlChunks'].map do |chunk| replacements.reduce(chunk) { |a, (k, v)| a.gsub(Regexp.new(k), v) } end end end end
Source
# File lib/gooddata/models/project_creator.rb, line 286 def pick_chunks_hard_sync(chunk, preference) # Make sure default values for cascade_drops working_preference = { cascade_drops: false }.merge(preference) if working_preference[:cascade_drops] || chunk['updateScript']['cascadeDrops'] == working_preference[:cascade_drops] [chunk] else [] end end
In case fallback hard synchronize, then the API model/diff only returns one result with preserve_data is always false and cascade_drops is true or false. So pick chunk for fallback hard synchronize, we will ignore the preserve_data parameter and only check the cascade_drops parameter in preference.
Source
# File lib/gooddata/models/project_creator.rb, line 274 def process_fallback_hard_sync_result(result, project) messages = result['wTaskStatus']['messages'] if !messages.nil? && messages.size.positive? warning_message = GoodData::Helpers.interpolate_error_messages(messages) log_message = "Project #{project.pid} failed to preserve data, truncated data of some datasets. MAQL diff execution messages: \"#{warning_message}\"" GoodData.logger.warn(log_message) end end
Fallback hard synchronize although execute result success but some cases there are errors during executing. In this cases, then export the errors to execution log as warning