Merge branch 'feature/project-import' of gitlab.com:gitlab-org/gitlab-ce into feature/project-export-ui-experimental

lib/gitlab/import_export/error.rb

@@ -2,4 +2,4 @@ module Gitlab
   module ImportExport
     class Error < StandardError; end
   end
-end
+end

lib/gitlab/import_export/import_export.yml

@@ -1,7 +1,8 @@
 # Model relationships to be included in the project import/export
 project_tree:
   - issues:
-    - :notes
+    - notes:
+        :author
   - :labels
   - :milestones
   - :snippets
@@ -10,8 +11,9 @@ project_tree:
   - project_members:
     - :user
   - merge_requests:
+    - notes:
+        :author
     - :merge_request_diff
-    - :notes
   - ci_commits:
     - :statuses
 
@@ -31,6 +33,8 @@ included_attributes:
     - :id
     - :email
     - :username
+  author:
+    - :name
 
 # Do not include the following attributes for the models specified.
 excluded_attributes:

lib/gitlab/import_export/import_export_reader.rb

@@ -15,7 +15,7 @@ module Gitlab
       def project_tree
         @attributes_parser.find_included(:project).merge(include: build_hash(@tree))
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
       end
 
       private
@@ -39,10 +39,22 @@ module Gitlab
           @attributes_parser.parse(current_key) { |hash| @json_config_hash[current_key] ||= hash }
 
           handle_model_object(current_key, model_object)
+          process_sub_model(current_key, model_object) if model_object.is_a?(Hash)
         end
         @json_config_hash
       end
 
+      def process_sub_model(current_key, model_object)
+        sub_model_json = build_json_config_hash(model_object).dup
+        @json_config_hash.slice!(current_key)
+
+        if @json_config_hash[current_key] && @json_config_hash[current_key][:include]
+          @json_config_hash[current_key][:include] << sub_model_json
+        else
+          @json_config_hash[current_key] = { include: sub_model_json }
+        end
+      end
+
       def handle_model_object(current_key, model_object)
         if @json_config_hash[current_key]
           add_model_value(current_key, model_object)
@@ -55,7 +67,7 @@ module Gitlab
         parsed_hash = { include: value }
 
         @attributes_parser.parse(value) do |hash|
-            parsed_hash = { include: hash_or_merge(value, hash) }
+          parsed_hash = { include: hash_or_merge(value, hash) }
         end
         @json_config_hash[current_key] = parsed_hash
       end

lib/gitlab/import_export/importer.rb

@@ -16,7 +16,7 @@ module Gitlab
         FileUtils.mkdir_p(@shared.export_path)
         decompress_archive
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 

lib/gitlab/import_export/members_mapper.rb

@@ -2,18 +2,25 @@ module Gitlab
   module ImportExport
     class MembersMapper
 
-      def self.map(*args)
-        new(*args).map
-      end
+      attr_reader :map, :note_member_list
 
       def initialize(exported_members:, user:, project_id:)
         @exported_members = exported_members
         @user = user
         @project_id = project_id
+        @note_member_list = []
+
+        @project_member_map = Hash.new do |_, key|
+          @note_member_list << key
+          default_project_member
+        end
+
+        @map = generate_map
       end
 
-      def map
-        @project_member_map = Hash.new(default_project_member)
+      private
+
+      def generate_map
         @exported_members.each do |member|
           existing_user = User.where(find_project_user_query(member)).first
           assign_member(existing_user, member) if existing_user
@@ -21,8 +28,6 @@ module Gitlab
         @project_member_map
       end
 
-      private
-
       def assign_member(existing_user, member)
         old_user_id = member['user']['id']
         member['user'] = existing_user

lib/gitlab/import_export/project_tree_restorer.rb

@@ -16,7 +16,7 @@ module Gitlab
         @project_members = @tree_hash.delete('project_members')
         create_relations
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 
@@ -26,9 +26,10 @@ module Gitlab
 
       private
 
-      def members_map
-        @members ||= Gitlab::ImportExport::MembersMapper.map(
-          exported_members: @project_members, user: @user, project_id: project.id)
+      def members_mapper
+        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+                                                                    user: @user,
+                                                                    project_id: project.id)
       end
 
       def create_relations(relation_list = default_relation_list, tree_hash = @tree_hash)
@@ -61,11 +62,18 @@ module Gitlab
       end
 
       def create_sub_relations(relation, tree_hash)
-        tree_hash[relation.keys.first.to_s].each do |relation_item|
+        relation_key = relation.keys.first.to_s
+        tree_hash[relation_key].each do |relation_item|
           relation.values.flatten.each do |sub_relation|
-            relation_hash = relation_item[sub_relation.to_s]
-            next if relation_hash.blank?
-            process_sub_relation(relation_hash, relation_item, sub_relation)
+
+            if sub_relation.is_a?(Hash)
+              relation_hash =  relation_item[sub_relation.keys.first.to_s]
+              sub_relation = sub_relation.keys.first
+            else
+              relation_hash = relation_item[sub_relation.to_s]
+            end
+
+            process_sub_relation(relation_hash, relation_item, sub_relation) unless relation_hash.blank?
           end
         end
       end
@@ -87,7 +95,7 @@ module Gitlab
 
       def relation_from_factory(relation, relation_hash)
         Gitlab::ImportExport::RelationFactory.create(
-          relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_map: members_map)
+          relation_sym: relation.to_sym, relation_hash: relation_hash.merge('project_id' => project.id), members_mapper: members_mapper)
       end
     end
   end

lib/gitlab/import_export/project_tree_saver.rb

@@ -14,7 +14,7 @@ module Gitlab
         File.write(full_path, project_json_tree)
         true
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 

lib/gitlab/import_export/relation_factory.rb

@@ -6,11 +6,12 @@ module Gitlab
       OVERRIDES = { snippets: :project_snippets, ci_commits: 'Ci::Commit', statuses: 'commit_status' }.freeze
       USER_REFERENCES = %w(author_id assignee_id updated_by_id).freeze
 
-      def create(relation_sym:, relation_hash:, members_map:)
+      def create(relation_sym:, relation_hash:, members_mapper:)
         relation_sym = parse_relation_sym(relation_sym)
         klass = parse_relation(relation_hash, relation_sym)
 
-        update_user_references(relation_hash, members_map)
+        update_missing_author(relation_hash, members_mapper) if relation_sym == :notes
+        update_user_references(relation_hash, members_mapper.map)
         update_project_references(relation_hash, klass)
 
         imported_object(klass, relation_hash)
@@ -26,6 +27,21 @@ module Gitlab
         end
       end
 
+      def update_missing_author(relation_hash, members_map)
+        old_author_id = relation_hash['author_id'].dup
+        relation_hash['author_id'] = members_map.map[old_author_id]
+        return unless members_map.note_member_list.include?(old_author_id)
+
+        relation_hash['note'] = ('*Blank note*') if relation_hash['note'].blank?
+        relation_hash['note'].join(missing_author_note(relation_hash['updated_at'],
+                                                       relation_hash['author']['name']))
+        relation_hash.delete('author')
+      end
+
+      def missing_author_note(updated_at, author_name)
+        "\n\n *By #{author_name} on #{updated_at} (imported from GitLab project)*"
+      end
+
       def update_project_references(relation_hash, klass)
         project_id = relation_hash.delete('project_id')
 

lib/gitlab/import_export/repo_bundler.rb

@@ -22,7 +22,7 @@ module Gitlab
         FileUtils.mkdir_p(@shared.export_path)
         git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 

lib/gitlab/import_export/repo_restorer.rb

@@ -17,7 +17,7 @@ module Gitlab
 
         git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle)
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 

lib/gitlab/import_export/saver.rb

@@ -20,7 +20,7 @@ module Gitlab
           false
         end
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 

lib/gitlab/import_export/shared.rb

@@ -13,9 +13,11 @@ module Gitlab
         @export_path ||= Gitlab::ImportExport.export_path(relative_path: @opts[:relative_path])
       end
 
-      def error(message)
-        error_out(message, caller[0].dup)
-        @errors << message
+      def error(error)
+        error_out(error.message, caller[0].dup)
+        @errors << error.message
+        # Debug:
+        Rails.logger.error(error.backtrace)
       end
 
       private

lib/gitlab/import_export/wiki_repo_bundler.rb

@@ -12,7 +12,7 @@ module Gitlab
         FileUtils.mkdir_p(@shared.export_path)
         git_bundle(repo_path: path_to_repo, bundle_path: @full_path)
       rescue => e
-        @shared.error(e.message)
+        @shared.error(e)
         false
       end
 

spec/lib/gitlab/import_export/members_mapper_spec.rb

@@ -42,9 +42,11 @@ describe Gitlab::ImportExport::MembersMapper, services: true do
     it 'defaults to importer project member if it does not exist' do
       expect(members_mapper.map[-1]).to eq(user.id)
     end
-  end
 
-  def project_member_user_id(id)
-    members_mapper.map[id]
+    it 'updates missing author IDs on missing project member' do
+      members_mapper.map[-1]
+
+      expect(members_mapper.note_member_list.first).to eq(-1)
+    end
   end
 end

spec/lib/gitlab/import_export/project_tree_restorer_spec.rb

@@ -7,12 +7,9 @@ describe Gitlab::ImportExport::ProjectTreeRestorer, services: true do
     let(:namespace) { create(:namespace, owner: user) }
     let(:shared) { Gitlab::ImportExport::Shared.new(relative_path: "../../../spec/lib/gitlab/import_export/", project_path: 'path') }
     let(:project_tree_restorer) { described_class.new(user: user, shared: shared, namespace_id: namespace.id) }
+    let!(:restored_project_json) { project_tree_restorer.restore }
 
     context 'JSON' do
-      let(:restored_project_json) do
-        project_tree_restorer.restore
-      end
-
       it 'restores models based on JSON' do
         expect(restored_project_json).to be true
       end

spec/lib/gitlab/import_export/project_tree_saver_spec.rb

@@ -65,6 +65,10 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
         expect(saved_project_json['issues'].first['notes']).not_to be_empty
       end
 
+      it 'has author on issue comments' do
+        expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty
+      end
+
       it 'has project members' do
         expect(saved_project_json['project_members']).not_to be_empty
       end
@@ -77,6 +81,9 @@ describe Gitlab::ImportExport::ProjectTreeSaver, services: true do
         expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty
       end
 
+      it 'has author on merge requests comments' do
+        expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty
+      end
       it 'has commit statuses' do
         expect(saved_project_json['ci_commits'].first['statuses']).not_to be_empty
       end