Merge branch 'ce-to-ee' into 'master'

CE to EE

See merge request !375

CHANGELOG

@@ -1,6 +1,21 @@
 Please view this file on the master branch, on stable branches it's out of date.
 
+v 7.11.0 (unreleased)
+  - Fix clone URL field and X11 Primary selection (Dmitry Medvinsky)
+  - Ignore invalid lines in .gitmodules
+  -
+  -
+  -
+  -
+  -
+  -
+  -
+  - Improve new project command options (Ben Bodenmiller)
+
 v 7.10.0 (unreleased)
+  - Ignore submodules that are defined in .gitmodules but are checked in as directories.
+  - Allow projects to be imported from Google Code.
+  - Remove access control for uploaded images to fix broken images in emails (Hannes Rosenögger)
   - Allow users to be invited by email to join a group or project.
   - Don't crash when project repository doesn't exist.
   - Add config var to block auto-created LDAP users.
@@ -15,6 +30,7 @@ v 7.10.0 (unreleased)
   - Don't leak existence of group or project via search.
   - Fix bug where Wiki pages that included a '/' were no longer accessible (Stan Hu)
   - Fix bug where error messages from Dropzone would not be displayed on the issues page (Stan Hu)
+  - Add a rake task to check repository integrity with `git fsck`
   - Add ability to configure Reply-To address in gitlab.yml (Stan Hu)
   - Move current user to the top of the list in assignee/author filters (Stan Hu)
   - Fix broken side-by-side diff view on merge request page (Stan Hu)
@@ -75,7 +91,6 @@ v 7.9.0 (unreleased)
   - Fix admin user projects lists.
   - Don't leak private group existence by redirecting from namespace controller to group controller.
   - Ability to skip some items from backup (database, respositories or uploads)
-  - Fix "Hello @username." references not working by no longer allowing usernames to end in period.
   - Archive repositories in background worker.
   - Import GitHub, Bitbucket or GitLab.com projects owned by authenticated user into current namespace.
   - Project labels are now available over the API under the "tag_list" field (Cristian Medina) 
@@ -92,6 +107,12 @@ v 7.9.0 (unreleased)
   - Remove truncation from issue titles on milestone page (Jason Blanchard)
   - Fix stuck Merge Request merging events from old installations (Ben Bodenmiller)
   - Fix merge request comments on files with multiple commits
+  - Fix Resource Owner Password Authentication Flow
+  
+v 7.9.4
+  - Security: Fix project import URL regex to prevent arbitary local repos from being imported
+  - Fixed issue where only 25 commits would load in file listings
+  - Fix LDAP identities  after config update
 
 v 7.9.3
   - Contains no changes

Gemfile

@@ -39,7 +39,7 @@ gem "browser"
 
 # Extracting information from a git repository
 # Provide access to Gitlab::Git library
-gem "gitlab_git", '~> 7.1.6'
+gem "gitlab_git", '~> 7.1.10'
 
 # Ruby/Rack Git Smart-HTTP Server Handler
 gem 'gitlab-grack', '~> 2.0.0.rc2', require: 'grack'

Gemfile.lock

@@ -212,7 +212,7 @@ GEM
       mime-types (~> 1.19)
     gitlab_emoji (0.1.0)
       gemojione (~> 2.0)
-    gitlab_git (7.1.6)
+    gitlab_git (7.1.10)
       activesupport (~> 4.0)
       charlock_holmes (~> 0.6)
       gitlab-linguist (~> 3.0)
@@ -703,7 +703,7 @@ DEPENDENCIES
   gitlab-grack (~> 2.0.0.rc2)
   gitlab-linguist (~> 3.0.1)
   gitlab_emoji (~> 0.1)
-  gitlab_git (~> 7.1.6)
+  gitlab_git (~> 7.1.10)
   gitlab_meta (= 7.0)
   gitlab_omniauth-ldap (= 1.2.1)
   gollum-lib (~> 4.0.2)

app/assets/javascripts/application.js.coffee

@@ -115,8 +115,8 @@ if location.hash
 window.addEventListener "hashchange", shiftWindow
 
 $ ->
-  # Click a .one_click_select field, select the contents
-  $(".one_click_select").on 'click', -> $(@).select()
+  # Click a .js-select-on-focus field, select the contents
+  $(".js-select-on-focus").on "focusin", -> $(this).select()
 
   $('.remove-row').bind 'ajax:success', ->
     $(this).closest('li').fadeOut()

app/assets/javascripts/branch-graph.js.coffee

@@ -214,7 +214,7 @@ class @BranchGraph
       stroke: @colors[commit.space]
       "stroke-width": 2
     )
-    r.image(gon.relative_url_root + commit.author.icon, avatar_box_x, avatar_box_y, 20, 20)
+    r.image(commit.author.icon, avatar_box_x, avatar_box_y, 20, 20)
     r.text(@offsetX + @unitSpace * @mspace + 35, y, commit.message.split("\n")[0]).attr(
       "text-anchor": "start"
       font: "14px Monaco, monospace"

app/assets/javascripts/merge_request.js.coffee

@@ -58,14 +58,6 @@ class @MergeRequest
       , 'json'
 
   bindEvents: ->
-    this.$('.merge-request-tabs').on 'click', 'a', (event) =>
-      a = $(event.currentTarget)
-
-      href = a.attr('href')
-      History.replaceState {path: href}, document.title, href
-
-      event.preventDefault()
-
     this.$('.merge-request-tabs').on 'click', 'li', (event) =>
       this.activateTab($(event.currentTarget).data('action'))
 

app/controllers/help_controller.rb

@@ -3,38 +3,52 @@ class HelpController < ApplicationController
   end
 
   def show
-    @filepath = clean_path_info(params[:filepath])
-    @format = params[:format]
+    category = clean_path_info(path_params[:category])
+    file = path_params[:file]
 
     respond_to do |format|
-      format.md { render_doc }
-      format.all { send_file_data }
+      format.any(:markdown, :md, :html) do
+        path = Rails.root.join('doc', category, "#{file}.md")
+
+        if File.exist?(path)
+          @markdown = File.read(path)
+
+          render 'show.html.haml'
+        else
+          # Force template to Haml
+          render 'errors/not_found.html.haml', layout: 'errors', status: 404
+        end
+      end
+
+      # Allow access to images in the doc folder
+      format.any(:png, :gif, :jpeg) do
+        path = Rails.root.join('doc', category, "#{file}.#{params[:format]}")
+
+        if File.exist?(path)
+          send_file(path, disposition: 'inline')
+        else
+          head :not_found
+        end
+      end
+
+      # Any other format we don't recognize, just respond 404
+      format.any { head :not_found }
     end
   end
 
   def shortcuts
   end
 
+  def ui
+  end
+
   private
 
-  def render_doc
-    if File.exists?(Rails.root.join('doc', @filepath + '.md'))
-      render 'show.html.haml'
-    else
-      not_found!
-    end
-  end
+  def path_params
+    params.require(:category)
+    params.require(:file)
 
-  def send_file_data
-    path = Rails.root.join('doc', "#{@filepath}.#{@format}")
-    if File.exists?(path)
-      send_file(path, disposition: 'inline')
-    else
-      head :not_found
-    end
-  end
-
-  def ui
+    params
   end
 
   PATH_SEPS = Regexp.union(*[::File::SEPARATOR, ::File::ALT_SEPARATOR].compact)

app/controllers/import/google_code_controller.rb

@@ -54,6 +54,11 @@ class Import::GoogleCodeController < Import::BaseController
       render "new_user_map" and return
     end
 
+    # This is the default, so let's not save it into the database.
+    user_map.reject! do |key, value|
+      value == Gitlab::GoogleCodeImport::Client.mask_email(key)
+    end
+
     session[:google_code_user_map] = user_map
 
     flash[:notice] = "The user map has been saved. Continue by selecting the projects you want to import."

app/controllers/projects/commit_controller.rb

@@ -10,11 +10,11 @@ class Projects::CommitController < Projects::ApplicationController
   def show
     return git_not_found! unless @commit
 
-    @line_notes = @project.notes.for_commit_id(commit.id).inline
+    @line_notes = commit.notes(@project).inline
     @diffs = @commit.diffs
     @note = @project.build_commit_note(commit)
-    @notes_count = @project.notes.for_commit_id(commit.id).count
-    @notes = @project.notes.for_commit_id(@commit.id).not_inline.fresh
+    @notes_count = commit.notes(@project).count
+    @notes = commit.notes(@project).not_inline.fresh
     @noteable = @commit
     @comments_allowed = @reply_allowed = true
     @comments_target  = {

app/controllers/projects/uploads_controller.rb

@@ -1,7 +1,11 @@
 class Projects::UploadsController < Projects::ApplicationController
   layout 'project'
 
-  before_filter :project
+  # We want to skip these filters for only the `show` action if `image?` is true, 
+  # but `skip_before_filter` doesn't work with both `only` and `if`, so we accomplish the same like this.
+  skipped_filters = [:authenticate_user!, :reject_blocked!, :project, :repository]
+  skip_before_filter  *skipped_filters, only: [:show]
+  before_filter       *skipped_filters, only: [:show], unless: :image?
 
   def create
     link_to_file = ::Projects::UploadService.new(project, params[:file]).
@@ -21,15 +25,32 @@ class Projects::UploadsController < Projects::ApplicationController
   end
 
   def show
-    uploader = FileUploader.new(project, params[:secret])
-
-    return redirect_to uploader.url unless uploader.file_storage?
-
-    uploader.retrieve_from_store!(params[:filename])
-
-    return not_found! unless uploader.file.exists?
+    return not_found! if uploader.nil? || !uploader.file.exists?
 
     disposition = uploader.image? ? 'inline' : 'attachment'
     send_file uploader.file.path, disposition: disposition
   end
+
+  def uploader
+    return @uploader if defined?(@uploader)
+
+    namespace = params[:namespace_id]
+    id = params[:project_id]
+
+    file_project = Project.find_with_namespace("#{namespace}/#{id}")
+
+    if file_project.nil?
+      @uploader = nil 
+      return
+    end
+
+    @uploader = FileUploader.new(file_project, params[:secret])
+    @uploader.retrieve_from_store!(params[:filename])
+
+    @uploader
+  end
+
+  def image?
+    uploader && uploader.file.exists? && uploader.image?
+  end
 end

app/models/commit.rb

@@ -134,6 +134,25 @@ class Commit
     User.find_for_commit(committer_email, committer_name)
   end
 
+  def participants(project, current_user = nil)
+    users = []
+    users << author
+    users << committer
+    
+    users.push *self.mentioned_users(current_user, project)
+
+    notes(project).each do |note|
+      users << note.author
+      users.push *note.mentioned_users(current_user, project)
+    end
+
+    users.uniq
+  end
+
+  def notes(project)
+    project.notes.for_commit_id(self.id)
+  end
+
   def method_missing(m, *args, &block)
     @raw.send(m, *args, &block)
   end

app/models/concerns/issuable.rb

@@ -122,15 +122,15 @@ module Issuable
     users = []
     users << author
     users << assignee if is_assigned?
-    mentions = []
-    mentions << self.mentioned_users(current_user)
+
+    users.push *self.mentioned_users(current_user)
 
     notes.each do |note|
       users << note.author
-      mentions << note.mentioned_users(current_user)
+      users.push *note.mentioned_users(current_user)
     end
 
-    users.concat(mentions.reduce([], :|)).uniq
+    users.uniq
   end
 
   def subscribed?(user)

app/models/concerns/mentionable.rb

@@ -42,10 +42,10 @@ module Mentionable
     Note.cross_reference_exists?(target, local_reference)
   end
 
-  def mentioned_users(current_user = nil)
+  def mentioned_users(current_user = nil, p = project)
     return [] if mentionable_text.blank?
 
-    ext = Gitlab::ReferenceExtractor.new(self.project, current_user)
+    ext = Gitlab::ReferenceExtractor.new(p, current_user)
     ext.analyze(mentionable_text)
     ext.users.uniq
   end

app/models/label.rb

@@ -13,6 +13,8 @@
 class Label < ActiveRecord::Base
   DEFAULT_COLOR = '#428BCA'
 
+  default_value_for :color, DEFAULT_COLOR
+
   belongs_to :project
   has_many :label_links, dependent: :destroy
   has_many :issues, through: :label_links, source: :target, source_type: 'Issue'

app/models/namespace.rb

@@ -59,6 +59,7 @@ class Namespace < ActiveRecord::Base
     end
 
     def clean_path(path)
+      path = path.dup
       path.gsub!(/@.*\z/,             "")
       path.gsub!(/\.git\z/,           "")
       path.gsub!(/\A-+/,              "")

app/models/note.rb

@@ -332,14 +332,6 @@ class Note < ActiveRecord::Base
     current_application_settings.max_attachment_size.megabytes.to_i
   end
 
-  def commit_author
-    @commit_author ||=
-      project.team.users.find_by(email: noteable.author_email) ||
-      project.team.users.find_by(name: noteable.author_name)
-  rescue
-    nil
-  end
-
   def cross_reference?
     note.start_with?(self.class.cross_reference_note_prefix)
   end

app/models/project.rb

@@ -27,7 +27,6 @@
 #  import_type            :string(255)
 #  import_source          :string(255)
 #  avatar                 :string(255)
-#  import_data            :text
 #
 
 require 'carrierwave/orm/activerecord'
@@ -51,8 +50,6 @@ class Project < ActiveRecord::Base
   default_value_for :wall_enabled, false
   default_value_for :snippets_enabled, gitlab_config_features.snippets
 
-  serialize :import_data, JSON
-
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
   def set_last_activity_at
@@ -122,6 +119,7 @@ class Project < ActiveRecord::Base
 
   has_many :project_group_links, dependent: :destroy
   has_many :invited_groups, through: :project_group_links, source: :group
+  has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
 
   delegate :name, to: :owner, allow_nil: true, prefix: true
   delegate :members, to: :team, prefix: true
@@ -273,8 +271,7 @@ class Project < ActiveRecord::Base
   end
 
   def clear_import_data
-    self.import_data = nil
-    self.save
+    self.import_data.destroy if self.import_data
   end
 
   def import?

app/models/project_import_data.rb

@@ -0,0 +1,19 @@
+# == Schema Information
+#
+# Table name: project_import_datas
+#
+#  id           :integer          not null, primary key
+#  project_id   :integer
+#  data         :text
+#
+
+require 'carrierwave/orm/activerecord'
+require 'file_size_validator'
+
+class ProjectImportData < ActiveRecord::Base
+  belongs_to :project
+  
+  serialize :data, JSON
+
+  validates :project, presence: true
+end

app/models/snippet.rb

@@ -87,6 +87,18 @@ class Snippet < ActiveRecord::Base
     visibility_level
   end
 
+  def participants(current_user = self.author)
+    users = []
+    users << author
+    
+    notes.each do |note|
+      users << note.author
+      users.push *note.mentioned_users(current_user)
+    end
+
+    users.uniq
+  end
+
   class << self
     def search(query)
       where('(title LIKE :query OR file_name LIKE :query)', query: "%#{query}%")

app/services/notification_service.rb

@@ -127,17 +127,12 @@ class NotificationService
 
     recipients = []
 
-    if note.commit_id.present?
-      recipients << note.commit_author
-    end
-    
     # Add all users participating in the thread (author, assignee, comment authors)
     participants = 
-      if target.respond_to?(:participants)
-        target.participants
-      elsif target.is_a?(Commit)
-        author_ids = Note.for_commit_id(target.id).pluck(:author_id).uniq
-        User.where(id: author_ids)
+      if target.is_a?(Commit)
+        target.participants(note.project, note.author)
+      elsif target.respond_to?(:participants)
+        target.participants(note.author)
       else
         note.mentioned_users
       end

app/services/projects/participants_service.rb

@@ -13,19 +13,21 @@ module Projects
     end
 
     def participants_in(type, id)
-      users = case type
-              when "Issue"
-                issue = project.issues.find_by_iid(id)
-                issue ? issue.participants(current_user) : []
-              when "MergeRequest"
-                merge_request = project.merge_requests.find_by_iid(id)
-                merge_request ? merge_request.participants(current_user) : []
-              when "Commit"
-                author_ids = Note.for_commit_id(id).pluck(:author_id).uniq
-                User.where(id: author_ids)
-              else
-                []
-              end
+      users = 
+        case type
+        when "Issue"
+          issue = project.issues.find_by_iid(id)
+          issue.participants(current_user) if issue
+        when "MergeRequest"
+          merge_request = project.merge_requests.find_by_iid(id)
+          merge_request.participants(current_user) if merge_request
+        when "Commit"
+          commit = project.repository.commit(id)
+          commit.participants(project, current_user) if commit
+        end
+
+      return [] unless users
+      
       sorted(users)
     end
 

app/views/help/show.html.haml

@@ -1,2 +1,2 @@
 .documentation.wiki
-  = markdown File.read(Rails.root.join('doc', @filepath + '.md')).gsub("$your_email", current_user.email)
+  = markdown @markdown.gsub('$your_email', current_user.email)

app/views/layouts/_bootlint.haml

@@ -0,0 +1,4 @@
+:javascript
+  jQuery(document).ready(function() {
+    javascript:(function(){var s=document.createElement("script");s.onload=function(){bootlint.showLintReportForCurrentDocument([], {hasProblems: false, problemFree: false});};s.src="https://maxcdn.bootstrapcdn.com/bootlint/latest/bootlint.min.js";document.body.appendChild(s)})();
+  });

app/views/layouts/_head.html.haml

@@ -16,6 +16,7 @@
 
   = render 'layouts/google_analytics' if extra_config.has_key?('google_analytics_id')
   = render 'layouts/piwik' if extra_config.has_key?('piwik_url') && extra_config.has_key?('piwik_site_id')
+  = render 'layouts/bootlint' if Rails.env == 'development'
 
   -# Atom feed
   - if current_user

app/views/projects/commits/_commit.html.haml

@@ -12,7 +12,7 @@
       - if @note_counts
         - note_count = @note_counts.fetch(commit.id, 0)
       - else
-        - notes = project.notes.for_commit_id(commit.id)
+        - notes = commit.notes(project)
         - note_count = notes.user.count
 
       - if note_count > 0

app/views/projects/empty.html.haml

@@ -27,20 +27,19 @@
     %legend Create a new repository
     %pre.dark
       :preserve
-        mkdir #{@project.path}
+        git clone #{ content_tag(:span, default_url_to_repo, class: 'clone')}
         cd #{@project.path}
-        git init
         touch README.md
         git add README.md
-        git commit -m "first commit"
-        git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
+        git commit -m "add README"
         git push -u origin master
 
   %fieldset
-    %legend Push an existing Git repository
+    %legend Existing folder or Git repository
     %pre.dark
       :preserve
-        cd existing_git_repo
+        cd existing_folder
+        git init
         git remote add origin #{ content_tag(:span, default_url_to_repo, class: 'clone')}
         git push -u origin master
 

app/views/projects/labels/_form.html.haml

@@ -16,7 +16,7 @@
     .col-sm-10
       .input-group
         .input-group-addon.label-color-preview  
-        = f.color_field :color, value: "#AA33EE", class: "form-control"
+        = f.color_field :color, class: "form-control"
       .help-block
         Choose any color.
         %br

app/views/projects/merge_requests/_show.html.haml

@@ -36,17 +36,17 @@
 
   - if @commits.present?
     %ul.nav.nav-tabs.merge-request-tabs
-      %li.notes-tab{data: {action: 'notes'}}
+      %li.notes-tab{data: {action: 'notes', toggle: 'tab'}}
         = link_to merge_request_path(@merge_request) do
           %i.fa.fa-comments
           Discussion
           %span.badge= @merge_request.mr_and_commit_notes.user.count
-      %li.commits-tab{data: {action: 'commits'}}
+      %li.commits-tab{data: {action: 'commits', toggle: 'tab'}}
         = link_to merge_request_path(@merge_request), title: 'Commits' do
           %i.fa.fa-history
           Commits
           %span.badge= @commits.size
-      %li.diffs-tab{data: {action: 'diffs'}}
+      %li.diffs-tab{data: {action: 'diffs', toggle: 'tab'}}
         = link_to diffs_namespace_project_merge_request_path(@project.namespace, @project, @merge_request) do
           %i.fa.fa-list-alt
           Changes

app/views/shared/_clone_panel.html.haml

@@ -15,7 +15,7 @@
       :"data-html" => "true",
       :"data-container" => "body"}
       = gitlab_config.protocol.upcase
-  = text_field_tag :project_clone, default_url_to_repo(project), class: "one_click_select form-control", readonly: true
+  = text_field_tag :project_clone, default_url_to_repo(project), class: "js-select-on-focus form-control", readonly: true
   - if project.kind_of?(Project)
     .input-group-addon
       .visibility-level-label.has_tooltip{'data-title' => "#{visibility_level_label(project.visibility_level)} project" }

config/initializers/doorkeeper.rb

@@ -11,7 +11,7 @@ Doorkeeper.configure do
   end
 
   resource_owner_from_credentials do |routes|
-    u = User.find_by(email: params[:username])
+    u = User.find_by(email: params[:username]) || User.find_by(username: params[:username])
     u if u && u.valid_password?(params[:password])
   end
 
@@ -83,7 +83,7 @@ Doorkeeper.configure do
   #
   # If not specified, Doorkeeper enables all the four grant flows.
   #
-  # grant_flows %w(authorization_code implicit password client_credentials)
+  grant_flows %w(authorization_code password client_credentials)
 
   # Under some circumstances you might want to have applications auto-approved,
   # so that the user skips the authorization step.

config/initializers/mime_types.rb

@@ -6,4 +6,5 @@
 
 Mime::Type.register_alias "text/plain", :diff
 Mime::Type.register_alias "text/plain", :patch
-Mime::Type.register_alias 'text/html', :md
+Mime::Type.register_alias 'text/html',  :markdown
+Mime::Type.register_alias 'text/html',  :md

config/routes.rb

@@ -39,9 +39,9 @@ Gitlab::Application.routes.draw do
 
   # Help
   get 'help'                  => 'help#index'
+  get 'help/:category/:file'  => 'help#show', as: :help_page, constraints: { category: /.*/, file: /[^\/\.]+/ }
   get 'help/shortcuts'
   get 'help/ui'               => 'help#ui'
-  get 'help/:filepath'        => 'help#show', as: :help_page, constraints: { filepath: /[^\.]+/ }
 
   #
   # Global snippets

db/migrate/20150324133047_remove_periods_at_ends_of_usernames.rb

@@ -1,76 +0,0 @@
-class RemovePeriodsAtEndsOfUsernames < ActiveRecord::Migration
-  include Gitlab::ShellAdapter
-
-  class Namespace < ActiveRecord::Base
-    class << self
-      def by_path(path)
-        where('lower(path) = :value', value: path.downcase).first
-      end
-
-      def clean_path(path)
-        path = path.dup
-        path.gsub!(/@.*\z/,             "")
-        path.gsub!(/\.git\z/,           "")
-        path.gsub!(/\A-/,               "")
-        path.gsub!(/.\z/,               "")
-        path.gsub!(/[^a-zA-Z0-9_\-\.]/, "")
-
-        counter = 0
-        base = path
-        while Namespace.by_path(path).present?
-          counter += 1
-          path = "#{base}#{counter}"
-        end
-
-        path
-      end
-    end
-  end
-
-  def up
-    changed_paths = {}
-
-    select_all("SELECT id, username FROM users WHERE username LIKE '%.'").each do |user|
-      username_was = user["username"]
-      username = Namespace.clean_path(username_was)
-      changed_paths[username_was] = username
-
-      username = quote_string(username)
-      execute "UPDATE users SET username = '#{username}' WHERE id = #{user["id"]}"
-      execute "UPDATE namespaces SET path = '#{username}', name = '#{username}' WHERE type IS NULL AND owner_id = #{user["id"]}"
-    end
-
-    select_all("SELECT id, path FROM namespaces WHERE type = 'Group' AND path LIKE '%.'").each do |group|
-      path_was = group["path"]
-      path = Namespace.clean_path(path_was)
-      changed_paths[path_was] = path
-
-      path = quote_string(path)
-      execute "UPDATE namespaces SET path = '#{path}' WHERE id = #{group["id"]}"
-    end
-
-    changed_paths.each do |path_was, path|
-      if gitlab_shell.mv_namespace(path_was, path)
-        # If repositories moved successfully we need to remove old satellites
-        # and send update instructions to users.
-        # However we cannot allow rollback since we moved namespace dir
-        # So we basically we mute exceptions in next actions
-        begin
-          gitlab_shell.rm_satellites(path_was)
-          # We cannot send update instructions since models and mailers
-          # can't safely be used from migrations as they may be written for 
-          # later versions of the database.
-          # send_update_instructions
-        rescue
-          # Returning false does not rollback after_* transaction but gives
-          # us information about failing some of tasks
-          false
-        end
-      else
-        # if we cannot move namespace directory we should rollback
-        # db changes in order to prevent out of sync between db and fs
-        raise Exception.new('namespace directory cannot be moved')
-      end
-    end
-  end
-end

db/migrate/20150417121913_create_project_import_data.rb

@@ -0,0 +1,8 @@
+class CreateProjectImportData < ActiveRecord::Migration
+  def change
+    create_table :project_import_data do |t|
+      t.references :project
+      t.text :data
+    end
+  end
+end

db/migrate/20150417122318_remove_import_data_from_project.rb

@@ -0,0 +1,5 @@
+class RemoveImportDataFromProject < ActiveRecord::Migration
+  def change
+    remove_column :projects, :import_data
+  end
+end

db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150413192223) do
+ActiveRecord::Schema.define(version: 20150417122318) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -381,6 +381,11 @@ ActiveRecord::Schema.define(version: 20150413192223) do
     t.integer  "group_access", default: 30, null: false
   end
 
+  create_table "project_import_data", force: true do |t|
+    t.integer "project_id"
+    t.text    "data"
+  end
+
   create_table "projects", force: true do |t|
     t.string   "name"
     t.string   "path"
@@ -409,7 +414,6 @@ ActiveRecord::Schema.define(version: 20150413192223) do
     t.string   "import_type"
     t.string   "import_source"
     t.boolean  "merge_requests_rebase_default", default: true
-    t.text     "import_data"
   end
 
   add_index "projects", ["created_at", "id"], name: "index_projects_on_created_at_and_id", using: :btree

doc/ssh/README.md

@@ -71,3 +71,9 @@ keys of all the projects you have access to are available. This project
 access can happen through being a direct member of the project, or through
 a group. See `def accessible_deploy_keys` in `app/models/user.rb` for more
 information.
+
+## Applications
+
+### Eclipse
+
+How to add your ssh key to Eclipse: http://wiki.eclipse.org/EGit/User_Guide#Eclipse_SSH_Configuration

features/steps/dashboard/help.rb

@@ -8,7 +8,7 @@ class Spinach::Features::DashboardHelp < Spinach::FeatureSteps
   end
 
   step 'I visit the "Rake Tasks" help page' do
-    visit help_page_path('raketasks/maintenance', format: 'md')
+    visit help_page_path("raketasks", "maintenance")
   end
 
   step 'I should see "Rake Tasks" page markdown rendered' do

lib/gitlab/google_code_import/importer.rb

@@ -5,7 +5,10 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @repo = GoogleCodeImport::Repository.new(project.import_data["repo"])
+
+        import_data = project.import_data.try(:data)
+        repo_data = import_data["repo"] if import_data
+        @repo = GoogleCodeImport::Repository.new(repo_data)
 
         @closed_statuses = []
         @known_labels = Set.new
@@ -27,9 +30,10 @@ module Gitlab
 
       def user_map
         @user_map ||= begin
-          user_map = Hash.new { |hash, user| hash[user] = Client.mask_email(user) }
+          user_map = Hash.new { |hash, user| Client.mask_email(user) }
 
-          stored_user_map = project.import_data["user_map"]
+          import_data = project.import_data.try(:data)
+          stored_user_map = import_data["user_map"] if import_data
           user_map.update(stored_user_map) if stored_user_map
 
           user_map
@@ -58,24 +62,7 @@ module Gitlab
       def import_issues
         return unless repo.issues
 
-        last_id = 0
-
-        deleted_issues = []
-
-        repo.issues.each do |raw_issue|
-          while raw_issue["id"] > last_id + 1
-            last_id += 1
-
-            issue = project.issues.create!(
-              title:        "Deleted issue",
-              description:  "*This issue has been deleted*",
-              author_id:    project.creator_id,
-              state:        "closed"
-            )
-            deleted_issues << issue
-          end
-          last_id = raw_issue["id"]
-
+        while raw_issue = repo.issues.shift
           author  = user_map[raw_issue["author"]["name"]]
           date    = DateTime.parse(raw_issue["published"]).to_formatted_s(:long)
 
@@ -112,7 +99,8 @@ module Gitlab
             end
           end
 
-          issue = project.issues.create!(
+          issue = Issue.create!(
+            project_id:   project.id,
             title:        raw_issue["title"],
             description:  body,
             author_id:    project.creator_id,
@@ -121,39 +109,46 @@ module Gitlab
           )
           issue.add_labels_by_names(labels)
 
+          if issue.iid != raw_issue["id"]
+            issue.update_attribute(:iid, raw_issue["id"])
+          end
+
           import_issue_comments(issue, comments)
         end
-
-        deleted_issues.each(&:destroy!)
       end
 
       def import_issue_comments(issue, comments)
-        comments.each do |raw_comment|
-          next if raw_comment.has_key?("deletedBy")
+        Note.transaction do
+          while raw_comment = comments.shift
+            next if raw_comment.has_key?("deletedBy")
 
-          content     = format_content(raw_comment["content"])
-          updates     = format_updates(raw_comment["updates"])
-          attachments = format_attachments(issue.iid, raw_comment["id"], raw_comment["attachments"])
+            content     = format_content(raw_comment["content"])
+            updates     = format_updates(raw_comment["updates"])
+            attachments = format_attachments(issue.iid, raw_comment["id"], raw_comment["attachments"])
 
-          next if content.blank? && updates.blank? && attachments.blank?
+            next if content.blank? && updates.blank? && attachments.blank?
 
-          author  = user_map[raw_comment["author"]["name"]]
-          date    = DateTime.parse(raw_comment["published"]).to_formatted_s(:long)
+            author  = user_map[raw_comment["author"]["name"]]
+            date    = DateTime.parse(raw_comment["published"]).to_formatted_s(:long)
 
-          body = format_issue_comment_body(
-            raw_comment["id"],
-            author,
-            date,
-            content,
-            updates,
-            attachments
-          )
+            body = format_issue_comment_body(
+              raw_comment["id"],
+              author,
+              date,
+              content,
+              updates,
+              attachments
+            )
 
-          issue.notes.create!(
-            project_id: project.id,
-            author_id:  project.creator_id,
-            note:       body
-          )
+            # Needs to match order of `comment_columns` below.
+            Note.create!(
+              project_id:     project.id,
+              noteable_type:  "Issue",
+              noteable_id:    issue.id,
+              author_id:      project.creator_id,
+              note:           body
+            )
+          end
         end
       end
 
@@ -232,7 +227,7 @@ module Gitlab
 
       def create_label(name)
         color = nice_label_color(name)
-        project.labels.create!(name: name, color: color)
+        Label.create!(project_id: project.id, name: name, color: color)
       end
 
       def format_content(raw_content)

lib/gitlab/google_code_import/project_creator.rb

@@ -11,12 +11,7 @@ module Gitlab
       end
 
       def execute
-        import_data = {
-          "repo"      => repo.raw_data,
-          "user_map"  => user_map
-        }
-
-        @project = Project.new(
+        project = ::Projects::CreateService.new(current_user,
           name: repo.name,
           path: repo.name,
           description: repo.summary,
@@ -25,21 +20,17 @@ module Gitlab
           visibility_level: Gitlab::VisibilityLevel::PUBLIC,
           import_type: "google_code",
           import_source: repo.name,
-          import_url: repo.import_url,
-          import_data: import_data
+          import_url: repo.import_url
+        ).execute
+
+        import_data = project.create_import_data(
+          data: {
+            "repo"      => repo.raw_data,
+            "user_map"  => user_map
+          }
         )
 
-        if @project.save!
-          @project.reload
-
-          if @project.import_failed?
-            @project.import_retry
-          else
-            @project.import_start
-          end
-        end
-
-        @project
+        project
       end
     end
   end

lib/gitlab/markdown.rb

@@ -68,23 +68,8 @@ module Gitlab
       @options      = options
       @html_options = html_options
 
-      # Extract pre blocks so they are not altered
-      # from http://github.github.com/github-flavored-markdown/
-      text.gsub!(%r{<pre>.*?</pre>|<code>.*?</code>}m) { |match| extract_piece(match) }
-      # Extract links with probably parsable hrefs
-      text.gsub!(%r{<a.*?>.*?</a>}m) { |match| extract_piece(match) }
-      # Extract images with probably parsable src
-      text.gsub!(%r{<img.*?>}m) { |match| extract_piece(match) }
-
       # TODO: add popups with additional information
 
-      text = parse(text, project)
-
-      # Insert pre block extractions
-      text.gsub!(/\{gfm-extraction-(\h{32})\}/) do
-        insert_piece($1)
-      end
-
       # Used markdown pipelines in GitLab:
       # GitlabEmojiFilter - performs emoji replacement.
       # SanitizationFilter - remove unsafe HTML tags and attributes
@@ -129,6 +114,21 @@ module Gitlab
 
       text = result[:output].to_html(save_with: save_options)
 
+      # Extract pre blocks so they are not altered
+      # from http://github.github.com/github-flavored-markdown/
+      text.gsub!(%r{<pre>.*?</pre>|<code>.*?</code>}m) { |match| extract_piece(match) }
+      # Extract links with probably parsable hrefs
+      text.gsub!(%r{<a.*?>.*?</a>}m) { |match| extract_piece(match) }
+      # Extract images with probably parsable src
+      text.gsub!(%r{<img.*?>}m) { |match| extract_piece(match) }
+
+      text = parse(text, project)
+
+      # Insert pre block extractions
+      text.gsub!(/\{gfm-extraction-(\h{32})\}/) do
+        insert_piece($1)
+      end
+
       if options[:parse_tasks]
         text = parse_tasks(text)
       end
@@ -150,7 +150,7 @@ module Gitlab
       @extractions[id]
     end
 
-    # Private: Parses text for references and emoji
+    # Private: Parses text for references
     #
     # text - Text to parse
     #

lib/gitlab/regex.rb

@@ -2,7 +2,7 @@ module Gitlab
   module Regex
     extend self
 
-    NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze
+    NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*)'.freeze
 
     def namespace_regex
       @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze
@@ -10,7 +10,7 @@ module Gitlab
 
     def namespace_regex_message
       "can contain only letters, digits, '_', '-' and '.'. " \
-      "Cannot start with '-' or end in '.'." \
+      "Cannot start with '-'." \
     end
 
 

lib/tasks/gitlab/check.rake

@@ -687,6 +687,23 @@ namespace :gitlab do
     end
   end
 
+  namespace :repo do
+    desc "GITLAB | Check the integrity of the repositories managed by GitLab"
+    task check: :environment do
+      namespace_dirs = Dir.glob(
+        File.join(Gitlab.config.gitlab_shell.repos_path, '*')
+      )
+
+      namespace_dirs.each do |namespace_dir|
+        repo_dirs = Dir.glob(File.join(namespace_dir, '*'))
+        repo_dirs.each do |dir|
+          puts "\nChecking repo at #{dir}"
+          system(*%w(git fsck), chdir: dir)
+        end
+      end
+    end
+  end
+
   # Helper methods
   ##########################
 

public/deploy.html

@@ -1,11 +1,11 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <title>Deploy in progress. Please try again in few minutes</title>
+    <title>Deploy in progress. Please try again in a few minutes</title>
     <link href="/static.css" media="screen" rel="stylesheet" type="text/css" />
   </head>
   <body>
     <h1><center><img src="/gitlab_logo.png"/></center>Deploy in progress</h1>
-    <h3>Please try again in few minutes or contact your administrator.</h3>
+    <h3>Please try again in a few minutes or contact your administrator.</h3>
   </body>
 </html>

spec/controllers/help_controller_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+
+describe HelpController do
+  let(:user) { create(:user) }
+
+  before do
+    sign_in(user)
+  end
+
+  describe 'GET #show' do
+    context 'for Markdown formats' do
+      context 'when requested file exists' do
+        before do
+          get :show, category: 'ssh', file: 'README', format: :md
+        end
+
+        it 'assigns to @markdown' do
+          expect(assigns[:markdown]).not_to be_empty
+        end
+
+        it 'renders HTML' do
+          expect(response).to render_template('show.html.haml')
+          expect(response.content_type).to eq 'text/html'
+        end
+      end
+
+      context 'when requested file is missing' do
+        it 'renders not found' do
+          get :show, category: 'foo', file: 'bar', format: :md
+          expect(response).to be_not_found
+        end
+      end
+    end
+
+    context 'for image formats' do
+      context 'when requested file exists' do
+        it 'renders the raw file' do
+          get :show, category: 'workflow/protected_branches',
+            file: 'protected_branches1', format: :png
+          expect(response).to be_success
+          expect(response.content_type).to eq 'image/png'
+          expect(response.headers['Content-Disposition']).to match(/^inline;/)
+        end
+      end
+
+      context 'when requested file is missing' do
+        it 'renders not found' do
+          get :show, category: 'foo', file: 'bar', format: :png
+          expect(response).to be_not_found
+        end
+      end
+    end
+
+    context 'for other formats' do
+      it 'always renders not found' do
+        get :show, category: 'ssh', file: 'README', format: :foo
+        expect(response).to be_not_found
+      end
+    end
+  end
+end

spec/controllers/projects/uploads_controller_spec.rb

@@ -54,4 +54,227 @@ describe Projects::UploadsController do
       end
     end
   end
+
+  describe "GET #show" do
+    let(:go) do
+      get :show,
+        namespace_id: project.namespace.to_param,
+        project_id:   project.to_param,
+        secret:       "123456",
+        filename:     "image.jpg"
+    end
+
+    context "when the project is public" do
+      before do
+        project.update_attribute(:visibility_level, Project::PUBLIC)
+      end
+
+      context "when not signed in" do
+        context "when the file exists" do
+          before do
+            allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
+            allow(jpg).to receive(:exists?).and_return(true)
+          end
+
+          it "responds with status 200" do
+            go
+
+            expect(response.status).to eq(200)
+          end
+        end
+
+        context "when the file doesn't exist" do
+          it "responds with status 404" do
+            go
+
+            expect(response.status).to eq(404)
+          end
+        end
+      end
+
+      context "when signed in" do
+        before do
+          sign_in(user)
+        end
+
+        context "when the file exists" do
+          before do
+            allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
+            allow(jpg).to receive(:exists?).and_return(true)
+          end
+
+          it "responds with status 200" do
+            go
+
+            expect(response.status).to eq(200)
+          end
+        end
+
+        context "when the file doesn't exist" do
+          it "responds with status 404" do
+            go
+
+            expect(response.status).to eq(404)
+          end
+        end
+      end
+    end
+
+    context "when the project is private" do
+      before do
+        project.update_attribute(:visibility_level, Project::PRIVATE)
+      end
+
+      context "when not signed in" do
+        context "when the file exists" do
+          before do
+            allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
+            allow(jpg).to receive(:exists?).and_return(true)
+          end
+
+          context "when the file is an image" do
+            before do
+              allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
+            end
+
+            it "responds with status 200" do
+              go
+
+              expect(response.status).to eq(200)
+            end
+          end
+
+          context "when the file is not an image" do
+            it "redirects to the sign in page" do
+              go
+
+              expect(response).to redirect_to(new_user_session_path)
+            end
+          end
+        end
+
+        context "when the file doesn't exist" do
+          it "redirects to the sign in page" do
+            go
+
+            expect(response).to redirect_to(new_user_session_path)
+          end
+        end
+      end
+
+      context "when signed in" do
+        before do
+          sign_in(user)
+        end
+
+        context "when the user has access to the project" do
+          before do
+            project.team << [user, :master]
+          end
+
+          context "when the user is blocked" do
+            before do
+              user.block
+              project.team << [user, :master]
+            end
+
+            context "when the file exists" do
+              before do
+                allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
+                allow(jpg).to receive(:exists?).and_return(true)
+              end
+
+              context "when the file is an image" do
+                before do
+                  allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
+                end
+
+                it "responds with status 200" do
+                  go
+
+                  expect(response.status).to eq(200)
+                end
+              end
+
+              context "when the file is not an image" do
+                it "redirects to the sign in page" do
+                  go
+
+                  expect(response).to redirect_to(new_user_session_path)
+                end
+              end
+            end
+
+            context "when the file doesn't exist" do
+              it "redirects to the sign in page" do
+                go
+
+                expect(response).to redirect_to(new_user_session_path)
+              end
+            end
+          end
+
+          context "when the user isn't blocked" do
+            context "when the file exists" do
+              before do
+                allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
+                allow(jpg).to receive(:exists?).and_return(true)
+              end
+
+              it "responds with status 200" do
+                go
+
+                expect(response.status).to eq(200)
+              end
+            end
+
+            context "when the file doesn't exist" do
+              it "responds with status 404" do
+                go
+
+                expect(response.status).to eq(404)
+              end
+            end
+          end
+        end
+
+        context "when the user doesn't have access to the project" do
+          context "when the file exists" do
+            before do
+              allow_any_instance_of(FileUploader).to receive(:file).and_return(jpg)
+              allow(jpg).to receive(:exists?).and_return(true)
+            end
+
+            context "when the file is an image" do
+              before do
+                allow_any_instance_of(FileUploader).to receive(:image?).and_return(true)
+              end
+
+              it "responds with status 200" do
+                go
+
+                expect(response.status).to eq(200)
+              end
+            end
+
+            context "when the file is not an image" do
+              it "responds with status 404" do
+                go
+
+                expect(response.status).to eq(404)
+              end
+            end
+          end
+
+          context "when the file doesn't exist" do
+            it "responds with status 404" do
+              go
+
+              expect(response.status).to eq(404)
+            end
+          end
+        end
+      end
+    end
+  end
 end

spec/features/help_pages_spec.rb

@@ -6,7 +6,7 @@ describe 'Help Pages', feature: true do
       login_as :user
     end
     it 'replace the variable $your_email with the email of the user' do
-      visit help_page_path(filepath: 'ssh/README', format: 'md')
+      visit help_page_path('ssh', 'README')
       expect(page).to have_content("ssh-keygen -t rsa -C \"#{@user.email}\"")
     end
   end

spec/lib/gitlab/google_code_import/importer_spec.rb

@@ -12,9 +12,13 @@ describe Gitlab::GoogleCodeImport::Importer do
       }
     } 
   }
-  let(:project) { create(:project, import_data: import_data) }
+  let(:project) { create(:project) }
   subject       { described_class.new(project) }
 
+  before do
+    project.create_import_data(data: import_data)
+  end
+
   describe "#execute" do
 
     it "imports status labels" do

spec/routing/routing_spec.rb

@@ -64,50 +64,35 @@ describe SnippetsController, "routing" do
   end
 end
 
-#              help GET    /help(.:format)              help#index
-#  help_permissions GET    /help/permissions(.:format)  help#permissions
-#     help_workflow GET    /help/workflow(.:format)     help#workflow
-#          help_api GET    /help/api(.:format)          help#api
-#    help_web_hooks GET    /help/web_hooks(.:format)    help#web_hooks
-# help_system_hooks GET    /help/system_hooks(.:format) help#system_hooks
-#     help_markdown GET    /help/markdown(.:format)     help#markdown
-#          help_ssh GET    /help/ssh(.:format)          help#ssh
-#    help_raketasks GET    /help/raketasks(.:format)    help#raketasks
-describe HelpController, 'routing' do
-  it 'to #index' do
-    expect(get('/help')).to route_to('help#index')
+#            help GET /help(.:format)                 help#index
+#       help_page GET /help/:category/:file(.:format) help#show {:category=>/.*/, :file=>/[^\/\.]+/}
+#  help_shortcuts GET /help/shortcuts(.:format)       help#shortcuts
+#         help_ui GET /help/ui(.:format)              help#ui
+describe HelpController, "routing" do
+  it "to #index" do
+    expect(get("/help")).to route_to('help#index')
   end
 
-  it 'to #permissions' do
-    expect(get('/help/permissions/permissions')).to route_to('help#show', filepath: 'permissions/permissions')
+  it 'to #show' do
+    path = '/help/markdown/markdown.md'
+    expect(get(path)).to route_to('help#show',
+                                  category: 'markdown',
+                                  file: 'markdown',
+                                  format: 'md')
+
+    path = '/help/workflow/protected_branches/protected_branches1.png'
+    expect(get(path)).to route_to('help#show',
+                                  category: 'workflow/protected_branches',
+                                  file: 'protected_branches1',
+                                  format: 'png')
   end
 
-  it 'to #workflow' do
-    expect(get('/help/workflow/README')).to route_to('help#show', filepath: 'workflow/README')
+  it 'to #shortcuts' do
+    expect(get('/help/shortcuts')).to route_to('help#shortcuts')
   end
 
-  it 'to #api' do
-    expect(get('/help/api/README')).to route_to('help#show', filepath: 'api/README')
-  end
-
-  it 'to #web_hooks' do
-    expect(get('/help/web_hooks/web_hooks')).to route_to('help#show', filepath: 'web_hooks/web_hooks')
-  end
-
-  it 'to #system_hooks' do
-    expect(get('/help/system_hooks/system_hooks')).to route_to('help#show', filepath: 'system_hooks/system_hooks')
-  end
-
-  it 'to #markdown' do
-    expect(get('/help/markdown/markdown')).to route_to('help#show',filepath: 'markdown/markdown')
-  end
-
-  it 'to #ssh' do
-    expect(get('/help/ssh/README')).to route_to('help#show', filepath: 'ssh/README')
-  end
-
-  it 'to #raketasks' do
-    expect(get('/help/raketasks/README')).to route_to('help#show', filepath: 'raketasks/README')
+  it 'to #ui' do
+    expect(get('/help/ui')).to route_to('help#ui')
   end
 end
 

spec/services/notification_service_spec.rb

@@ -149,7 +149,7 @@ describe NotificationService do
 
       before do
         build_team(note.project)
-        note.stub(:commit_author => @u_committer)
+        allow_any_instance_of(Commit).to receive(:author).and_return(@u_committer)
       end
 
       describe :new_note do