From 640bf8ab7212988fda0189e1534d46dcbf0c126f Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Thu, 12 Nov 2015 10:52:20 +0100
Subject: [PATCH 1/4] Retry failed tests

---
 .gitlab-ci.yml      | 2 ++
 Gemfile             | 1 +
 Gemfile.lock        | 3 +++
 spec/spec_helper.rb | 4 ++++
 4 files changed, 10 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c477721f9d..f77b5bdc95 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,6 +12,8 @@ cache:
 
 variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
+  # retry tests only in CI environment
+  RSPEC_RETRY_RETRY_COUNT: "3"
 
 before_script:
   - source ./scripts/prepare_build.sh
diff --git a/Gemfile b/Gemfile
index c66ef3cffa..a5c244fb7b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -263,6 +263,7 @@ group :development, :test do
   gem 'database_cleaner',   '~> 1.4.0'
   gem 'factory_girl_rails', '~> 4.6.0'
   gem 'rspec-rails',        '~> 3.3.0'
+  gem 'rspec-retry'
   gem 'spinach-rails',      '~> 0.2.1'
 
   # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826)
diff --git a/Gemfile.lock b/Gemfile.lock
index 22c86e4ae8..7f3c5911e5 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -679,6 +679,8 @@ GEM
       rspec-expectations (~> 3.3.0)
       rspec-mocks (~> 3.3.0)
       rspec-support (~> 3.3.0)
+    rspec-retry (0.4.5)
+      rspec-core
     rspec-support (3.3.0)
     rubocop (0.35.1)
       astrolabe (~> 1.3)
@@ -999,6 +1001,7 @@ DEPENDENCIES
   rouge (~> 1.10.1)
   rqrcode-rails3 (~> 0.1.7)
   rspec-rails (~> 3.3.0)
+  rspec-retry
   rubocop (~> 0.35.0)
   ruby-fogbugz (~> 0.2.1)
   sanitize (~> 2.0)
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 8f381f46e5..159fb96417 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -15,6 +15,7 @@ require 'rspec/rails'
 require 'shoulda/matchers'
 require 'sidekiq/testing/inline'
 require 'benchmark/ips'
+require 'rspec/retry'
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
@@ -25,6 +26,9 @@ RSpec.configure do |config|
   config.use_instantiated_fixtures  = false
   config.mock_with :rspec
 
+  config.verbose_retry = true
+  config.display_try_failure_messages = true
+
   config.include Devise::TestHelpers, type: :controller
   config.include LoginHelpers,        type: :feature
   config.include LoginHelpers,        type: :request

From af43401602d637c5b91def323ba3a0bac824b785 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 9 Mar 2016 14:12:08 +0100
Subject: [PATCH 2/4] Retry spinach tests

---
 Gemfile                   |  1 +
 Gemfile.lock              |  3 ++
 features/support/env.rb   |  1 +
 features/support/rerun.rb | 14 +++++++++
 lib/tasks/spinach.rake    | 62 +++++++++++++++++++++------------------
 5 files changed, 53 insertions(+), 28 deletions(-)
 create mode 100644 features/support/rerun.rb

diff --git a/Gemfile b/Gemfile
index a5c244fb7b..9aa76f92aa 100644
--- a/Gemfile
+++ b/Gemfile
@@ -265,6 +265,7 @@ group :development, :test do
   gem 'rspec-rails',        '~> 3.3.0'
   gem 'rspec-retry'
   gem 'spinach-rails',      '~> 0.2.1'
+  gem 'spinach-rerun-reporter', '~> 0.0.2'
 
   # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826)
   gem 'minitest', '~> 5.7.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 7f3c5911e5..dcfef4cd6b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -766,6 +766,8 @@ GEM
       capybara (>= 2.0.0)
       railties (>= 3)
       spinach (>= 0.4)
+    spinach-rerun-reporter (0.0.2)
+      spinach (~> 0.8)
     spring (1.6.4)
     spring-commands-rspec (1.0.4)
       spring (>= 0.9.1)
@@ -1020,6 +1022,7 @@ DEPENDENCIES
   six (~> 0.2.0)
   slack-notifier (~> 1.2.0)
   spinach-rails (~> 0.2.1)
+  spinach-rerun-reporter (~> 0.0.2)
   spring (~> 1.6.4)
   spring-commands-rspec (~> 1.0.4)
   spring-commands-spinach (~> 1.0.0)
diff --git a/features/support/env.rb b/features/support/env.rb
index 62c80b9c94..357d164d87 100644
--- a/features/support/env.rb
+++ b/features/support/env.rb
@@ -14,6 +14,7 @@ require 'sidekiq/testing/inline'
 
 require_relative 'capybara'
 require_relative 'db_cleaner'
+require_relative 'rerun'
 
 %w(select2_helper test_env repo_helpers).each do |f|
   require Rails.root.join('spec', 'support', f)
diff --git a/features/support/rerun.rb b/features/support/rerun.rb
new file mode 100644
index 0000000000..8b176c5be8
--- /dev/null
+++ b/features/support/rerun.rb
@@ -0,0 +1,14 @@
+# The spinach-rerun-reporter doesn't define the on_undefined_step
+# See it here: https://github.com/javierav/spinach-rerun-reporter/blob/master/lib/spinach/reporter/rerun.rb
+module Spinach
+  class Reporter
+    class Rerun
+      def on_undefined_step(step_data, failure, step_definitions = nil)
+        super step_data, failure, step_definitions
+
+        # save feature file and scenario line
+        @rerun << "#{current_feature.filename}:#{current_scenario.line}"
+      end
+    end
+  end
+end
diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake
index 3acfc6e207..01d23b89bb 100644
--- a/lib/tasks/spinach.rake
+++ b/lib/tasks/spinach.rake
@@ -4,53 +4,59 @@ namespace :spinach do
   namespace :project do
     desc "GitLab | Spinach | Run project commits, issues and merge requests spinach features"
     task :half do
-      cmds = [
-        %W(rake gitlab:setup),
-        %W(spinach --tags @project_commits,@project_issues,@project_merge_requests),
-      ]
-      run_commands(cmds)
+      run_spinach_tests('@project_commits,@project_issues,@project_merge_requests')
     end
 
     desc "GitLab | Spinach | Run remaining project spinach features"
     task :rest do
-      cmds = [
-        %W(rake gitlab:setup),
-        %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests),
-      ]
-      run_commands(cmds)
+      run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets,~@project_commits,~@project_issues,~@project_merge_requests')
     end
   end
 
   desc "GitLab | Spinach | Run project spinach features"
   task :project do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(spinach --tags ~@admin,~@dashboard,~@profile,~@public,~@snippets),
-    ]
-    run_commands(cmds)
+    run_spinach_tests('~@admin,~@dashboard,~@profile,~@public,~@snippets')
   end
 
   desc "GitLab | Spinach | Run other spinach features"
   task :other do
-    cmds = [
-      %W(rake gitlab:setup),
-      %W(spinach --tags @admin,@dashboard,@profile,@public,@snippets),
-    ]
-    run_commands(cmds)
+    run_spinach_tests('@admin,@dashboard,@profile,@public,@snippets')
+  end
+
+  desc "GitLab | Spinach | Run other spinach features"
+  task :builds do
+    run_spinach_tests('@builds')
   end
 end
 
 desc "GitLab | Run spinach"
 task :spinach do
-  cmds = [
-    %W(rake gitlab:setup),
-    %W(spinach),
-  ]
-  run_commands(cmds)
+  run_spinach_tests(nil)
 end
 
-def run_commands(cmds)
-  cmds.each do |cmd|
-    system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!")
+def run_command(cmd)
+  system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd)
+end
+
+def run_spinach_command(args)
+  run_command(%w(spinach -r rerun) + args)
+end
+
+def run_spinach_tests(tags)
+  #run_command(%w(rake gitlab:setup)) or raise('gitlab:setup failed!')
+
+  success = run_spinach_command(%W(--tags #{tags}))
+  3.times do |_|
+    break if success
+    break unless File.exists?('tmp/spinach-rerun.txt')
+
+    tests = File.foreach('tmp/spinach-rerun.txt').map(&:chomp)
+    puts ''
+    puts "Spinach tests for #{tags}: Retrying tests... #{tests}".red
+    puts ''
+    sleep(3)
+    success = run_spinach_command(tests)
   end
+
+  raise("spinach tests for #{tags} failed!") unless success
 end

From 978703abe287519784b8258296ca71d783f67794 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 9 Mar 2016 14:12:29 +0100
Subject: [PATCH 3/4] Retry apt-get installation and bundle install

---
 .gitlab-ci.yml           |  2 +-
 scripts/prepare_build.sh | 30 +++++++++++++++++++++---------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f77b5bdc95..ffefeb6dfd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,7 +23,7 @@ before_script:
   - cp config/gitlab.yml.example config/gitlab.yml
   - touch log/application.log
   - touch log/test.log
-  - bundle install --without postgres production --jobs $(nproc)  "${FLAGS[@]}"
+  - bundle install --without postgres production --jobs $(nproc) "${FLAGS[@]}"
   - RAILS_ENV=test bundle exec rake db:drop db:create db:schema:load db:migrate
 
 stages:
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index b6f076a90c..bce9bce99c 100755
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -2,15 +2,27 @@
 
 if [ -f /.dockerinit ]; then
     mkdir -p vendor
-    if [ ! -e vendor/phantomjs_1.9.8-0jessie_amd64.deb ]; then
-        wget -q https://gitlab.com/axil/phantomjs-debian/raw/master/phantomjs_1.9.8-0jessie_amd64.deb
-        mv phantomjs_1.9.8-0jessie_amd64.deb vendor/
-    fi
-    dpkg -i vendor/phantomjs_1.9.8-0jessie_amd64.deb
 
-    apt-get update -qq
-    apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \
-        libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip
+    # Install phantomjs package
+    pushd vendor
+    if [ ! -e phantomjs_1.9.8-0jessie_amd64.deb ]; then
+        wget -q https://gitlab.com/axil/phantomjs-debian/raw/master/phantomjs_1.9.8-0jessie_amd64.deb
+    fi
+    dpkg -i phantomjs_1.9.8-0jessie_amd64.deb
+    popd
+
+    # Try to install packages
+    for i in $(seq 1 3); do
+      apt-get update -yqqq || true
+
+      if apt-get -o dir::cache::archives="vendor/apt" install -y -qq --force-yes \
+          libicu-dev libkrb5-dev cmake nodejs postgresql-client mysql-client unzip; then
+          break
+      fi
+
+      sleep 3s
+      echo "Retrying package installation..."
+    fi
 
     cp config/database.yml.mysql config/database.yml
     sed -i 's/username:.*/username: root/g' config/database.yml
@@ -20,7 +32,7 @@ if [ -f /.dockerinit ]; then
     cp config/resque.yml.example config/resque.yml
     sed -i 's/localhost/redis/g' config/resque.yml
 
-    export FLAGS=(--path vendor)
+    export FLAGS=(--path vendor --retry 3)
 else
     export PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin
     cp config/database.yml.mysql config/database.yml

From b651cd227efae152ef56c112576c5464af4ee725 Mon Sep 17 00:00:00 2001
From: Kamil Trzcinski <ayufan@ayufan.eu>
Date: Wed, 9 Mar 2016 14:26:49 +0100
Subject: [PATCH 4/4] Fix script error

---
 scripts/prepare_build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index bce9bce99c..82de51a9a2 100755
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -22,7 +22,7 @@ if [ -f /.dockerinit ]; then
 
       sleep 3s
       echo "Retrying package installation..."
-    fi
+    done
 
     cp config/database.yml.mysql config/database.yml
     sed -i 's/username:.*/username: root/g' config/database.yml