diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index b47a646c0b..e341c4d9cf 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -2,4 +2,4 @@ module Gitlab module ImportExport class Error < StandardError; end end -end \ No newline at end of file +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 36128909df..66e6ae86cf 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,7 +1,8 @@ # Model relationships to be included in the project import/export project_tree: - issues: - - :notes + - notes: + :author - :labels - :milestones - :snippets @@ -10,8 +11,9 @@ project_tree: - project_members: - :user - merge_requests: + - notes: + :author - :merge_request_diff - - :notes - ci_commits: - :statuses @@ -31,6 +33,8 @@ included_attributes: - :id - :email - :username + author: + - :name # Do not include the following attributes for the models specified. excluded_attributes: diff --git a/lib/gitlab/import_export/import_export_reader.rb b/lib/gitlab/import_export/import_export_reader.rb index f637756b35..11a736ecab 100644 --- a/lib/gitlab/import_export/import_export_reader.rb +++ b/lib/gitlab/import_export/import_export_reader.rb @@ -15,7 +15,7 @@ module Gitlab def project_tree @attributes_parser.find_included(:project).merge(include: build_hash(@tree)) rescue => e - @shared.error(e.message) + @shared.error(e) end private @@ -39,10 +39,22 @@ module Gitlab @attributes_parser.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash } handle_model_object(current_key, model_object) + process_sub_model(current_key, model_object) if model_object.is_a?(Hash) end @json_config_hash end + def process_sub_model(current_key, model_object) + sub_model_json = build_json_config_hash(model_object).dup + @json_config_hash.slice!(current_key) + + if @json_config_hash[current_key] && @json_config_hash[current_key][:include] + @json_config_hash[current_key][:include] << sub_model_json + else + @json_config_hash[current_key] = { include: sub_model_json } + end + end + def handle_model_object(current_key, model_object) if @json_config_hash[current_key] add_model_value(current_key, model_object) @@ -55,7 +67,7 @@ module Gitlab parsed_hash = { include: value } @attributes_parser.parse(value) do |hash| - parsed_hash = { include: hash_or_merge(value, hash) } + parsed_hash = { include: hash_or_merge(value, hash) } end @json_config_hash[current_key] = parsed_hash end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 20a93f7f6d..d73ce43b49 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -16,7 +16,7 @@ module Gitlab FileUtils.mkdir_p(@shared.export_path) decompress_archive rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index d6124106f5..da8aa47565 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -2,18 +2,25 @@ module Gitlab module ImportExport class MembersMapper - def self.map(*args) - new(*args).map - end + attr_reader :map, :note_member_list def initialize(exported_members:, user:, project_id:) @exported_members = exported_members @user = user @project_id = project_id + @note_member_list = [] + + @project_member_map = Hash.new do |_, key| + @note_member_list << key + default_project_member + end + + @map = generate_map end - def map - @project_member_map = Hash.new(default_project_member) + private + + def generate_map @exported_members.each do |member| existing_user = User.where(find_project_user_query(member)).first assign_member(existing_user, member) if existing_user @@ -21,8 +28,6 @@ module Gitlab @project_member_map end - private - def assign_member(existing_user, member) old_user_id = member['user']['id'] member['user'] = existing_user diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 06bf86a178..a840c9f947 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -16,7 +16,7 @@ module Gitlab @project_members = @tree_hash.delete('project_members') create_relations rescue => e - @shared.error(e.message) + @shared.error(e) false end @@ -26,9 +26,10 @@ module Gitlab private - def members_map - @members ||= Gitlab::ImportExport::MembersMapper.map( - exported_members: @project_members, user: @user, project_id: project.id) + def members_mapper + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members, + user: @user, + project_id: project.id) end def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash) @@ -61,11 +62,18 @@ module Gitlab end def create_sub_relations(relation, tree_hash) - tree_hash[relation.keys.first.to_s].each do |relation_item| + relation_key = relation.keys.first.to_s + tree_hash[relation_key].each do |relation_item| relation.values.flatten.each do |sub_relation| - relation_hash = relation_item[sub_relation.to_s] - next if relation_hash.blank? - process_sub_relation(relation_hash, relation_item, sub_relation) + + if sub_relation.is_a?(Hash) + relation_hash = relation_item[sub_relation.keys.first.to_s] + sub_relation = sub_relation.keys.first + else + relation_hash = relation_item[sub_relation.to_s] + end + + process_sub_relation(relation_hash, relation_item, sub_relation) unless relation_hash.blank? end end end @@ -87,7 +95,7 @@ module Gitlab def relation_from_factory(relation, relation_hash) Gitlab::ImportExport::RelationFactory.create( - relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map) + relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper) end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 2287524c8a..ed8ca31d93 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -14,7 +14,7 @@ module Gitlab File.write(full_path, project_json_tree) true rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index dd992ef443..65e2a935fa 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -6,11 +6,12 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze - def create(relation_sym:, relation_hash:, members_map:) + def create(relation_sym:, relation_hash:, members_mapper:) relation_sym = parse_relation_sym(relation_sym) klass = parse_relation(relation_hash, relation_sym) - update_user_references(relation_hash, members_map) + update_missing_author(relation_hash, members_mapper) if relation_sym == :notes + update_user_references(relation_hash, members_mapper.map) update_project_references(relation_hash, klass) imported_object(klass, relation_hash) @@ -26,6 +27,21 @@ module Gitlab end end + def update_missing_author(relation_hash, members_map) + old_author_id = relation_hash['author_id'].dup + relation_hash['author_id'] = members_map.map[old_author_id] + return unless members_map.note_member_list.include?(old_author_id) + + relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank? + relation_hash['note'].join(missing_author_note(relation_hash['updated_at'], + relation_hash['author']['name'])) + relation_hash.delete('author') + end + + def missing_author_note(updated_at, author_name) + "\n\n *By #{author_name} on #{updated_at} (imported from GitLab project)*" + end + def update_project_references(relation_hash, klass) project_id = relation_hash.delete('project_id') diff --git a/lib/gitlab/import_export/repo_bundler.rb b/lib/gitlab/import_export/repo_bundler.rb index bcf976fb62..af809f4c38 100644 --- a/lib/gitlab/import_export/repo_bundler.rb +++ b/lib/gitlab/import_export/repo_bundler.rb @@ -22,7 +22,7 @@ module Gitlab FileUtils.mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 3909d44744..36094b95aa 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -17,7 +17,7 @@ module Gitlab git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index a2ff43a3ce..f38229c6c5 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -20,7 +20,7 @@ module Gitlab false end rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 050b0428f4..01ac332981 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -13,9 +13,11 @@ module Gitlab @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path]) end - def error(message) - error_out(message, caller[0].dup) - @errors << message + def error(error) + error_out(error.message, caller[0].dup) + @errors << error.message + # Debug: + Rails.logger.error(error.backtrace) end private diff --git a/lib/gitlab/import_export/wiki_repo_bundler.rb b/lib/gitlab/import_export/wiki_repo_bundler.rb index a0000176bb..e1e7f75372 100644 --- a/lib/gitlab/import_export/wiki_repo_bundler.rb +++ b/lib/gitlab/import_export/wiki_repo_bundler.rb @@ -12,7 +12,7 @@ module Gitlab FileUtils.mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e - @shared.error(e.message) + @shared.error(e) false end diff --git a/spec/lib/gitlab/import_export/members_mapper_spec.rb b/spec/lib/gitlab/import_export/members_mapper_spec.rb index a4d3d6d122..78706f64fb 100644 --- a/spec/lib/gitlab/import_export/members_mapper_spec.rb +++ b/spec/lib/gitlab/import_export/members_mapper_spec.rb @@ -42,9 +42,11 @@ describe Gitlab::ImportExport::MembersMapper, services: true do it 'defaults to importer project member if it does not exist' do expect(members_mapper.map[-1]).to eq(user.id) end - end - def project_member_user_id(id) - members_mapper.map[id] + it 'updates missing author IDs on missing project member' do + members_mapper.map[-1] + + expect(members_mapper.note_member_list.first).to eq(-1) + end end end diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index 7c6a14fc8a..6464b0141f 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -7,12 +7,9 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do let(:namespace) { create(:namespace, owner: user) } let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: "../../../spec/lib/gitlab/import_export/", project_path: 'path') } let(:project_tree_restorer) { described_class.new(user: user, shared: shared, namespace_id: namespace.id) } + let!(:restored_project_json) { project_tree_restorer.restore } context 'JSON' do - let(:restored_project_json) do - project_tree_restorer.restore - end - it 'restores models based on JSON' do expect(restored_project_json).to be true end diff --git a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb index 67e6267b74..b9cf88d3f7 100644 --- a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb @@ -65,6 +65,10 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do expect(saved_project_json['issues'].first['notes']).not_to be_empty end + it 'has author on issue comments' do + expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty + end + it 'has project members' do expect(saved_project_json['project_members']).not_to be_empty end @@ -77,6 +81,9 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty end + it 'has author on merge requests comments' do + expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty + end it 'has commit statuses' do expect(saved_project_json['ci_commits'].first['statuses']).not_to be_empty end