Merge branch 'master' of https://gitlab.com/gitlab-org/gitlab-ce into git-http-controller

lib/gitlab/badge/build.rb

@@ -0,0 +1,46 @@
+module Gitlab
+  module Badge
+    ##
+    # Build badge
+    #
+    class Build
+      include Gitlab::Application.routes.url_helpers
+      include ActionView::Helpers::AssetTagHelper
+      include ActionView::Helpers::UrlHelper
+
+      def initialize(project, ref)
+        @project, @ref = project, ref
+        @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref)
+      end
+
+      def type
+        'image/svg+xml'
+      end
+
+      def data
+        File.read(@image[:path])
+      end
+
+      def to_s
+        @image[:name].sub(/\.svg$/, '')
+      end
+
+      def to_html
+        link_to(image_tag(image_url, alt: 'build status'), link_url)
+      end
+
+      def to_markdown
+        "[![build status](#{image_url})](#{link_url})"
+      end
+
+      def image_url
+        build_namespace_project_badges_url(@project.namespace,
+                                           @project, @ref, format: :svg)
+      end
+
+      def link_url
+        namespace_project_commits_url(@project.namespace, @project, id: @ref)
+      end
+    end
+  end
+end

lib/gitlab/current_settings.rb

@@ -21,7 +21,6 @@ module Gitlab
         default_branch_protection: Settings.gitlab['default_branch_protection'],
         signup_enabled: Settings.gitlab['signup_enabled'],
         signin_enabled: Settings.gitlab['signin_enabled'],
-        twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'],
         gravatar_enabled: Settings.gravatar['enabled'],
         sign_in_text: Settings.extra['sign_in_text'],
         restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'],

lib/gitlab/email/message/repository_push.rb

@@ -5,7 +5,7 @@ module Gitlab
         attr_accessor :recipient
         attr_reader :author_id, :ref, :action
 
-        include Gitlab::Application.routes.url_helpers
+        include Gitlab::Routing.url_helpers
 
         delegate :namespace, :name_with_namespace, to: :project, prefix: :project
         delegate :name, to: :author, prefix: :author

lib/gitlab/email/receiver.rb

@@ -45,12 +45,12 @@ module Gitlab
         note = create_note(reply)
 
         unless note.persisted?
-          message = "The comment could not be created for the following reasons:"
+          msg = "The comment could not be created for the following reasons:"
           note.errors.full_messages.each do |error|
-            message << "\n\n- #{error}"
+            msg << "\n\n- #{error}"
           end
 
-          raise InvalidNoteError, message
+          raise InvalidNoteError, msg
         end
       end
 
@@ -63,9 +63,24 @@ module Gitlab
       end
 
       def reply_key
-        reply_key = nil
+        key_from_to_header || key_from_additional_headers
+      end
+
+      def key_from_to_header
+        key = nil
         message.to.each do |address|
-          reply_key = Gitlab::IncomingEmail.key_from_address(address)
+          key = Gitlab::IncomingEmail.key_from_address(address)
+          break if key
+        end
+
+        key
+      end
+
+      def key_from_additional_headers
+        reply_key = nil
+
+        Array(message.references).each do |message_id|
+          reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id)
           break if reply_key
         end
 

lib/gitlab/exclusive_lease.rb

@@ -15,6 +15,25 @@ module Gitlab
   # seconds then two overlapping operations may hold a lease for the same
   # key at the same time.
   #
+  # This class has no 'cancel' method. I originally decided against adding
+  # it because it would add complexity and a false sense of security. The
+  # complexity: instead of setting '1' we would have to set a UUID, and to
+  # delete it we would have to execute Lua on the Redis server to only
+  # delete the key if the value was our own UUID. Otherwise there is a
+  # chance that when you intend to cancel your lease you actually delete
+  # someone else's. The false sense of security: you cannot design your
+  # system to rely too much on the lease being cancelled after use because
+  # the calling (Ruby) process may crash or be killed. You _cannot_ count
+  # on begin/ensure blocks to cancel a lease, because the 'ensure' does
+  # not always run. Think of 'kill -9' from the Unicorn master for
+  # instance.
+  # 
+  # If you find that leases are getting in your way, ask yourself: would
+  # it be enough to lower the lease timeout? Another thing that might be
+  # appropriate is to only use a lease for bulk/automated operations, and
+  # to ignore the lease when you get a single 'manual' user request (a
+  # button click).
+  #
   class ExclusiveLease
     def initialize(key, timeout:)
       @key, @timeout = key, timeout
@@ -24,9 +43,13 @@ module Gitlab
     # false if the lease is already taken.
     def try_obtain
       # Performing a single SET is atomic
-      !!redis.set(redis_key, '1', nx: true, ex: @timeout)
+      Gitlab::Redis.with do |redis|
+        !!redis.set(redis_key, '1', nx: true, ex: @timeout)
+      end
     end
 
+    # No #cancel method. See comments above!
+
     private
 
     def redis

lib/gitlab/fogbugz_import/client.rb

@@ -26,7 +26,7 @@ module Gitlab
       def user_map
         users = {}
         res = @api.command(:listPeople)
-        res['people']['person'].each do |user|
+        [res['people']['person']].flatten.each do |user|
           users[user['ixPerson']] = { name: user['sFullName'], email: user['sEmail'] }
         end
         users

lib/gitlab/gfm/reference_rewriter.rb

@@ -34,16 +34,21 @@ module Gitlab
         @source_project = source_project
         @current_user = current_user
         @original_html = markdown(text)
+        @pattern = Gitlab::ReferenceExtractor.references_pattern
       end
 
       def rewrite(target_project)
-        pattern = Gitlab::ReferenceExtractor.references_pattern
+        return @text unless needs_rewrite?
 
-        @text.gsub(pattern) do |reference|
+        @text.gsub(@pattern) do |reference|
           unfold_reference(reference, Regexp.last_match, target_project)
         end
       end
 
+      def needs_rewrite?
+        @text =~ @pattern
+      end
+
       private
 
       def unfold_reference(reference, match, target_project)

lib/gitlab/gfm/uploads_rewriter.rb

@@ -0,0 +1,51 @@
+module Gitlab
+  module Gfm
+    ##
+    # Class that rewrites markdown links for uploads
+    #
+    # Using a pattern defined in `FileUploader` it copies files to a new
+    # project and rewrites all links to uploads in in a given text.
+    #
+    #
+    class UploadsRewriter
+      def initialize(text, source_project, _current_user)
+        @text = text
+        @source_project = source_project
+        @pattern = FileUploader::MARKDOWN_PATTERN
+      end
+
+      def rewrite(target_project)
+        return @text unless needs_rewrite?
+
+        @text.gsub(@pattern) do |markdown|
+          file = find_file(@source_project, $~[:secret], $~[:file])
+          return markdown unless file.try(:exists?)
+
+          new_uploader = FileUploader.new(target_project)
+          new_uploader.store!(file)
+          new_uploader.to_markdown
+        end
+      end
+
+      def needs_rewrite?
+        files.any?
+      end
+
+      def files
+        referenced_files = @text.scan(@pattern).map do
+          find_file(@source_project, $~[:secret], $~[:file])
+        end
+
+        referenced_files.compact.select(&:exists?)
+      end
+
+      private
+
+      def find_file(project, secret, file)
+        uploader = FileUploader.new(project, secret)
+        uploader.retrieve_from_store!(file)
+        uploader.file
+      end
+    end
+  end
+end

lib/gitlab/incoming_email.rb

@@ -1,13 +1,10 @@
 module Gitlab
   module IncomingEmail
     class << self
-      def enabled?
-        config.enabled && address_formatted_correctly?
-      end
+      FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze
 
-      def address_formatted_correctly?
-        config.address &&
-          config.address.include?("%{key}")
+      def enabled?
+        config.enabled && config.address
       end
 
       def reply_address(key)
@@ -24,6 +21,13 @@ module Gitlab
         match[1]
       end
 
+      def key_from_fallback_reply_message_id(message_id)
+        match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX)
+        return unless match
+
+        match[1]
+      end
+
       def config
         Gitlab.config.incoming_email
       end

lib/gitlab/ldap/access.rb

@@ -33,7 +33,10 @@ module Gitlab
 
       def allowed?
         if ldap_user
-          return true unless ldap_config.active_directory
+          unless ldap_config.active_directory
+            user.activate if user.ldap_blocked?
+            return true
+          end
 
           # Block user in GitLab if he/she was blocked in AD
           if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter)

lib/gitlab/metrics.rb

@@ -70,6 +70,40 @@ module Gitlab
       value.to_s.gsub('=', '\\=')
     end
 
+    # Measures the execution time of a block.
+    #
+    # Example:
+    #
+    #     Gitlab::Metrics.measure(:find_by_username_duration) do
+    #       User.find_by_username(some_username)
+    #     end
+    #
+    # name - The name of the field to store the execution time in.
+    #
+    # Returns the value yielded by the supplied block.
+    def self.measure(name)
+      trans = current_transaction
+
+      return yield unless trans
+
+      real_start = Time.now.to_f
+      cpu_start = System.cpu_time
+
+      retval = yield
+
+      cpu_stop = System.cpu_time
+      real_stop = Time.now.to_f
+
+      real_time = (real_stop - real_start) * 1000.0
+      cpu_time = cpu_stop - cpu_start
+
+      trans.increment("#{name}_real_time", real_time)
+      trans.increment("#{name}_cpu_time", cpu_time)
+      trans.increment("#{name}_call_count", 1)
+
+      retval
+    end
+
     # When enabled this should be set before being used as the usual pattern
     # "@foo ||= bar" is _not_ thread-safe.
     if enabled?
@@ -81,5 +115,11 @@ module Gitlab
           new(udp: { host: host, port: port })
       end
     end
+
+    private
+
+    def self.current_transaction
+      Transaction.current
+    end
   end
 end

lib/gitlab/metrics/metric.rb

@@ -2,6 +2,8 @@ module Gitlab
   module Metrics
     # Class for storing details of a single metric (label, value, etc).
     class Metric
+      JITTER_RANGE = 0.000001..0.001
+
       attr_reader :series, :values, :tags, :created_at
 
       # series - The name of the series (as a String) to store the metric in.
@@ -16,11 +18,29 @@ module Gitlab
 
       # Returns a Hash in a format that can be directly written to InfluxDB.
       def to_hash
+        # InfluxDB overwrites an existing point if a new point has the same
+        # series, tag set, and timestamp. In a highly concurrent environment
+        # this means that using the number of seconds since the Unix epoch is
+        # inevitably going to collide with another timestamp. For example, two
+        # Rails requests processed by different processes may end up generating
+        # metrics using the _exact_ same timestamp (in seconds).
+        #
+        # Due to the way InfluxDB is set up there's no solution to this problem,
+        # all we can do is lower the amount of collisions. We do this by using
+        # Time#to_f which returns the seconds as a Float providing greater
+        # accuracy. We then add a small random value that is large enough to
+        # distinguish most timestamps but small enough to not alter the amount
+        # of seconds.
+        #
+        # See https://gitlab.com/gitlab-com/operations/issues/175 for more
+        # information.
+        time = @created_at.to_f + rand(JITTER_RANGE)
+
         {
           series:    @series,
           tags:      @tags,
           values:    @values,
-          timestamp: @created_at.to_i * 1_000_000_000
+          timestamp: (time * 1_000_000_000).to_i
         }
       end
     end

lib/gitlab/metrics/subscribers/rails_cache.rb

@@ -0,0 +1,39 @@
+module Gitlab
+  module Metrics
+    module Subscribers
+      # Class for tracking the total time spent in Rails cache calls
+      class RailsCache < ActiveSupport::Subscriber
+        attach_to :active_support
+
+        def cache_read(event)
+          increment(:cache_read_duration, event.duration)
+        end
+
+        def cache_write(event)
+          increment(:cache_write_duration, event.duration)
+        end
+
+        def cache_delete(event)
+          increment(:cache_delete_duration, event.duration)
+        end
+
+        def cache_exist?(event)
+          increment(:cache_exists_duration, event.duration)
+        end
+
+        def increment(key, duration)
+          return unless current_transaction
+
+          current_transaction.increment(:cache_duration, duration)
+          current_transaction.increment(key, duration)
+        end
+
+        private
+
+        def current_transaction
+          Transaction.current
+        end
+      end
+    end
+  end
+end

lib/gitlab/metrics/system.rb

@@ -30,6 +30,17 @@ module Gitlab
           0
         end
       end
+
+      # THREAD_CPUTIME is not supported on OS X
+      if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID)
+        def self.cpu_time
+          Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond)
+        end
+      else
+        def self.cpu_time
+          Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond)
+        end
+      end
     end
   end
 end

lib/gitlab/note_data_builder.rb

@@ -41,7 +41,7 @@ module Gitlab
           data[:issue] = note.noteable.hook_attrs
         elsif note.for_merge_request?
           data[:merge_request] = note.noteable.hook_attrs
-        elsif note.for_project_snippet?
+        elsif note.for_snippet?
           data[:snippet] = note.noteable.hook_attrs
         end
 

lib/gitlab/redis.rb

@@ -0,0 +1,48 @@
+module Gitlab
+  class Redis
+    CACHE_NAMESPACE = 'cache:gitlab'
+
+    attr_reader :url
+
+    # To be thread-safe we must be careful when writing the class instance
+    # variables @url and @pool. Because @pool depends on @url we need two
+    # mutexes to prevent deadlock.
+    URL_MUTEX = Mutex.new
+    POOL_MUTEX = Mutex.new
+    private_constant :URL_MUTEX, :POOL_MUTEX
+
+    def self.url
+      @url || URL_MUTEX.synchronize { @url = new.url }
+    end
+
+    def self.with
+      if @pool.nil?
+        POOL_MUTEX.synchronize do
+          @pool = ConnectionPool.new { ::Redis.new(url: url) }
+        end
+      end
+      @pool.with { |redis| yield redis }
+    end
+    
+    def self.redis_store_options
+      url = new.url
+      redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url)
+      # Redis::Store does not handle Unix sockets well, so let's do it for them
+      redis_uri = URI.parse(url)
+      if redis_uri.scheme == 'unix'
+        redis_config_hash[:path] = redis_uri.path
+      end
+      redis_config_hash
+    end
+
+    def initialize(rails_env=nil)
+      rails_env ||= Rails.env
+      config_file = File.expand_path('../../../config/resque.yml', __FILE__)
+  
+      @url = "redis://localhost:6379"
+      if File.exists?(config_file)
+        @url =YAML.load_file(config_file)[rails_env]
+      end
+    end
+  end
+end

lib/gitlab/redis_config.rb

@@ -1,30 +0,0 @@
-module Gitlab
-  class RedisConfig
-    attr_reader :url
-
-    def self.url
-      new.url
-    end
-    
-    def self.redis_store_options
-      url = new.url
-      redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url)
-      # Redis::Store does not handle Unix sockets well, so let's do it for them
-      redis_uri = URI.parse(url)
-      if redis_uri.scheme == 'unix'
-        redis_config_hash[:path] = redis_uri.path
-      end
-      redis_config_hash
-    end
-
-    def initialize(rails_env=nil)
-      rails_env ||= Rails.env
-      config_file = File.expand_path('../../../config/resque.yml', __FILE__)
-  
-      @url = "redis://localhost:6379"
-      if File.exists?(config_file)
-        @url =YAML.load_file(config_file)[rails_env]
-      end
-    end
-  end
-end

lib/gitlab/routing.rb

@@ -0,0 +1,13 @@
+module Gitlab
+  module Routing
+    # Returns the URL helpers Module.
+    #
+    # This method caches the output as Rails' "url_helpers" method creates an
+    # anonymous module every time it's called.
+    #
+    # Returns a Module.
+    def self.url_helpers
+      @url_helpers ||= Gitlab::Application.routes.url_helpers
+    end
+  end
+end

lib/gitlab/saml/auth_hash.rb

@@ -0,0 +1,19 @@
+module Gitlab
+  module Saml
+    class AuthHash < Gitlab::OAuth::AuthHash
+
+      def groups
+        get_raw(Gitlab::Saml::Config.groups)
+      end
+
+      private
+
+      def get_raw(key)
+        # Needs to call `all` because of https://git.io/vVo4u
+        # otherwise just the first value is returned
+        auth_hash.extra[:raw_info].all[key]
+      end
+
+    end
+  end
+end

lib/gitlab/saml/config.rb

@@ -0,0 +1,21 @@
+module Gitlab
+  module Saml
+    class Config
+
+      class << self
+        def options
+          Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' }
+        end
+
+        def groups
+          options[:groups_attribute]
+        end
+
+        def external_groups
+          options[:external_groups]
+        end
+      end
+
+    end
+  end
+end

lib/gitlab/saml/user.rb

@@ -18,7 +18,7 @@ module Gitlab
           @user ||= find_or_create_ldap_user
         end
 
-        if auto_link_saml_enabled?
+        if auto_link_saml_user?
           @user ||= find_by_email
         end
 
@@ -26,6 +26,16 @@ module Gitlab
           @user ||= build_new_user
         end
 
+        if external_users_enabled? && @user
+          # Check if there is overlap between the user's groups and the external groups
+          # setting then set user as external or internal.
+          if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty?
+            @user.external = false
+          else
+            @user.external = true
+          end
+        end
+
         @user
       end
 
@@ -37,11 +47,24 @@ module Gitlab
         end
       end
 
+      def changed?
+        return true unless gl_user
+        gl_user.changed? || gl_user.identities.any?(&:changed?)
+      end
+
       protected
 
-      def auto_link_saml_enabled?
+      def auto_link_saml_user?
         Gitlab.config.omniauth.auto_link_saml_user
       end
+
+      def external_users_enabled?
+        !Gitlab::Saml::Config.external_groups.nil?
+      end
+
+      def auth_hash=(auth_hash)
+        @auth_hash = Gitlab::Saml::AuthHash.new(auth_hash)
+      end
     end
   end
 end

lib/gitlab/url_builder.rb

@@ -1,7 +1,8 @@
 module Gitlab
   class UrlBuilder
-    include Gitlab::Application.routes.url_helpers
+    include Gitlab::Routing.url_helpers
     include GitlabRoutingHelper
+    include ActionView::RecordIdentifier
 
     def initialize(type)
       @type = type
@@ -37,19 +38,16 @@ module Gitlab
         namespace_project_commit_url(namespace_id: note.project.namespace,
                                      id: note.commit_id,
                                      project_id: note.project,
-                                     anchor: "note_#{note.id}")
+                                     anchor: dom_id(note))
       elsif note.for_issue?
         issue = Issue.find(note.noteable_id)
-        issue_url(issue,
-                  anchor: "note_#{note.id}")
+        issue_url(issue, anchor: dom_id(note))
       elsif note.for_merge_request?
         merge_request = MergeRequest.find(note.noteable_id)
-        merge_request_url(merge_request,
-                          anchor: "note_#{note.id}")
-      elsif note.for_project_snippet?
+        merge_request_url(merge_request, anchor: dom_id(note))
+      elsif note.for_snippet?
         snippet = Snippet.find(note.noteable_id)
-        project_snippet_url(snippet,
-                            anchor: "note_#{note.id}")
+        project_snippet_url(snippet, anchor: dom_id(note))
       end
     end
   end