From 16f8ca566b8637dc8092a6b630c23a82a905b437 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 1 Oct 2015 23:40:29 -0400 Subject: [PATCH 01/13] Add custom protocol whitelisting to SanitizationFilter Addresses internal https://dev.gitlab.org/gitlab/gitlabhq/issues/2613 --- lib/gitlab/markdown/sanitization_filter.rb | 19 ++++ .../markdown/sanitization_filter_spec.rb | 101 ++++++++++++++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/lib/gitlab/markdown/sanitization_filter.rb b/lib/gitlab/markdown/sanitization_filter.rb index e368de7d84..ffb9dc33b6 100644 --- a/lib/gitlab/markdown/sanitization_filter.rb +++ b/lib/gitlab/markdown/sanitization_filter.rb @@ -48,6 +48,12 @@ module Gitlab # Allow span elements whitelist[:elements].push('span') + # Allow any protocol in `a` elements... + whitelist[:protocols].delete('a') + + # ...but then remove links with the `javascript` protocol + whitelist[:transformers].push(remove_javascript_links) + # Remove `rel` attribute from `a` elements whitelist[:transformers].push(remove_rel) @@ -57,6 +63,19 @@ module Gitlab whitelist end + def remove_javascript_links + lambda do |env| + node = env[:node] + + return unless node.name == 'a' + return unless node.has_attribute?('href') + + if node['href'].start_with?('javascript', ':javascript') + node.remove_attribute('href') + end + end + end + def remove_rel lambda do |env| if env[:node_name] == 'a' diff --git a/spec/lib/gitlab/markdown/sanitization_filter_spec.rb b/spec/lib/gitlab/markdown/sanitization_filter_spec.rb index e50c82d0b3..27cd00e805 100644 --- a/spec/lib/gitlab/markdown/sanitization_filter_spec.rb +++ b/spec/lib/gitlab/markdown/sanitization_filter_spec.rb @@ -44,7 +44,7 @@ module Gitlab::Markdown instance = described_class.new('Foo') 3.times { instance.whitelist } - expect(instance.whitelist[:transformers].size).to eq 4 + expect(instance.whitelist[:transformers].size).to eq 5 end it 'allows syntax highlighting' do @@ -77,19 +77,100 @@ module Gitlab::Markdown end it 'removes `rel` attribute from `a` elements' do - doc = filter(%q{Link}) + act = %q{Link} + exp = %q{Link} - expect(doc.css('a').size).to eq 1 - expect(doc.at_css('a')['href']).to eq '#' - expect(doc.at_css('a')['rel']).to be_nil + expect(filter(act).to_html).to eq exp end - it 'removes script-like `href` attribute from `a` elements' do - html = %q{Hi} - doc = filter(html) + # Adapted from the Sanitize test suite: http://git.io/vczrM + protocols = { + 'protocol-based JS injection: simple, no spaces' => { + input: 'foo', + output: 'foo' + }, - expect(doc.css('a').size).to eq 1 - expect(doc.at_css('a')['href']).to be_nil + 'protocol-based JS injection: simple, spaces before' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: simple, spaces after' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: simple, spaces before and after' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: preceding colon' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: UTF-8 encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: long UTF-8 encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: long UTF-8 encoding without semicolons' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: hex encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: long hex encoding' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: hex encoding without semicolons' => { + input: 'foo', + output: 'foo' + }, + + 'protocol-based JS injection: null char' => { + input: "foo", + output: '' + }, + + 'protocol-based JS injection: spaces and entities' => { + input: 'foo', + output: 'foo' + }, + } + + protocols.each do |name, data| + it "handles #{name}" do + doc = filter(data[:input]) + + expect(doc.to_html).to eq data[:output] + end + end + + it 'allows non-standard anchor schemes' do + exp = %q{IRC} + act = filter(exp) + + expect(act.to_html).to eq exp + end + + it 'allows relative links' do + exp = %q{foo/bar.md} + act = filter(exp) + + expect(act.to_html).to eq exp end end From d3734fbd89c069d35856b440f12109af8a7ef9c9 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 14:43:19 +0200 Subject: [PATCH 02/13] Use tar for intermediate backup storage During the backup we create an intermediate copy of two directories: builds and uploads. Instead of creating many small files with 'cp -r', we now use tar (and fast gzip) to create single intermediate files. This saves on disk IO and disk space while creating a backup. --- lib/backup/builds.rb | 31 ++----------------------------- lib/backup/files.rb | 39 +++++++++++++++++++++++++++++++++++++++ lib/backup/manager.rb | 4 ++-- lib/backup/uploads.rb | 30 ++---------------------------- 4 files changed, 45 insertions(+), 59 deletions(-) create mode 100644 lib/backup/files.rb diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 6f56f680bb..d269f8e260 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,34 +1,7 @@ module Backup - class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir - + class Builds < Files def initialize - @app_builds_dir = Settings.gitlab_ci.builds_path - @backup_dir = Gitlab.config.backup.path - @backup_builds_dir = File.join(Gitlab.config.backup.path, 'builds') - end - - # Copy builds from builds directory to backup/builds - def dump - FileUtils.rm_rf(backup_builds_dir) - # Ensure the parent dir of backup_builds_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_builds_dir before us - FileUtils.mkdir(backup_builds_dir, mode: 0700) - FileUtils.cp_r(app_builds_dir, backup_dir) - end - - def restore - backup_existing_builds_dir - - FileUtils.cp_r(backup_builds_dir, app_builds_dir) - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end + super(Settings.gitlab_ci.builds_path) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb new file mode 100644 index 0000000000..d0a6e8f27b --- /dev/null +++ b/lib/backup/files.rb @@ -0,0 +1,39 @@ +require 'open3' + +module Backup + class Files + attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir + + def initialize(app_files_dir) + @app_files_dir = File.realpath(app_files_dir) + @name = File.basename(app_files_dir) + @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_dir = Gitlab.config.backup.path + @backup_tarball = File.join(@backup_dir, name + '.tar.gz') + end + + # Copy files from public/files to backup/files + def dump + FileUtils.mkdir_p(Gitlab.config.backup.path) + run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end + + def restore + backup_existing_files_dir + + run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball) + end + + def backup_existing_files_dir + timestamped_files_path = File.join(files_parent_dir, "#{name}.#{Time.now.to_i}") + if File.exists?(app_files_dir) + FileUtils.mv(app_files_dir, File.expand_path(timestamped_files_path)) + end + end + + def run_pipeline!(cmd_list, options={}) + status_list = Open3.pipeline(*cmd_list, options) + abort 'Backup failed' unless status_list.compact.all?(&:success?) + end + end +end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 5c42f25f4a..f011fd03de 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories db uploads builds} + folders = %w{repositories db} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 1f9626644e..7c0838cc8b 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -1,34 +1,8 @@ module Backup - class Uploads - attr_reader :app_uploads_dir, :backup_uploads_dir, :backup_dir + class Uploads < Files def initialize - @app_uploads_dir = File.realpath(Rails.root.join('public', 'uploads')) - @backup_dir = Gitlab.config.backup.path - @backup_uploads_dir = File.join(Gitlab.config.backup.path, 'uploads') - end - - # Copy uploads from public/uploads to backup/uploads - def dump - FileUtils.rm_rf(backup_uploads_dir) - # Ensure the parent dir of backup_uploads_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_uploads_dir before us - FileUtils.mkdir(backup_uploads_dir, mode: 0700) - FileUtils.cp_r(app_uploads_dir, backup_dir) - end - - def restore - backup_existing_uploads_dir - - FileUtils.cp_r(backup_uploads_dir, app_uploads_dir) - end - - def backup_existing_uploads_dir - timestamped_uploads_path = File.join(app_uploads_dir, '..', "uploads.#{Time.now.to_i}") - if File.exists?(app_uploads_dir) - FileUtils.mv(app_uploads_dir, File.expand_path(timestamped_uploads_path)) - end + super(Rails.root.join('public/uploads')) end end end From 90ddf140b9390647002771572d0375da0bb9dfa4 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:06:15 +0200 Subject: [PATCH 03/13] Reduce disk IO during SQL backup By using light gzip compression we can save a lot of disk IO during the backup. --- lib/backup/database.rb | 55 +++++++++++++++++++----------------------- lib/backup/manager.rb | 4 +-- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 959ac4b786..4bdf6e1c62 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -2,26 +2,27 @@ require 'yaml' module Backup class Database - attr_reader :config, :db_dir + attr_reader :config, :db_file_name def initialize @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_dir = File.join(Gitlab.config.backup.path, 'db') + @db_file_name = File.join(Gitlab.config.backup.path, 'database.sql.gz') end def dump - FileUtils.rm_rf(@db_dir) - # Ensure the parent dir of @db_dir exists - FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create @db_dir before us - FileUtils.mkdir(@db_dir, mode: 0700) + FileUtils.rm_f(db_file_name) + compress_rd, compress_wr = IO.pipe + compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_rd.close - success = case config["adapter"] + FileUtils.mkdir_p(Gitlab.config.backup.path) + + dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysqldump', *mysql_args, config['database'], out: db_file_name) + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env @@ -30,48 +31,42 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end - system('pg_dump', *pgsql_args, config['database'], out: db_file_name) + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end + compress_wr.close + + success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Backup failed' unless success - - $progress.print 'Compressing database ... ' - success = system('gzip', db_file_name) - report_success(success) - abort 'Backup failed: compress error' unless success end def restore - $progress.print 'Decompressing database ... ' - success = system('gzip', '-d', db_file_name_gz) - report_success(success) - abort 'Restore failed: decompress error' unless success + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close - success = case config["adapter"] + restore_pid = case config["adapter"] when /^mysql/ then $progress.print "Restoring MySQL database #{config['database']} ... " # Workaround warnings from MySQL 5.6 about passwords on cmd line ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - system('mysql', *mysql_args, config['database'], in: db_file_name) + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then $progress.print "Restoring PostgreSQL database #{config['database']} ... " pg_env - system('psql', config['database'], '-f', db_file_name) + spawn('psql', config['database'], in: decompress_rd) end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Restore failed' unless success end protected - def db_file_name - File.join(db_dir, 'database.sql') - end - - def db_file_name_gz - File.join(db_dir, 'database.sql.gz') - end - def mysql_args args = { 'host' => '--host', diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index f011fd03de..53e79d4d1f 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["database.sql.gz", "uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories db} + folders = %w{repositories} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } From 7b71727c562b6f6337a180ae136be94bf0f6ed31 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:10:13 +0200 Subject: [PATCH 04/13] Remove old "files" tarball explicitly --- lib/backup/files.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/backup/files.rb b/lib/backup/files.rb index d0a6e8f27b..1b08e3324d 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -15,6 +15,7 @@ module Backup # Copy files from public/files to backup/files def dump FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.rm_f(backup_tarball) run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end From e789644783fae55f1095ffcc38b32f810f549caa Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:21:15 +0200 Subject: [PATCH 05/13] Keep old path: db/database.sql.gz Documentation elsewhere refers to this internal path, let's keep it. --- lib/backup/database.rb | 3 ++- lib/backup/manager.rb | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4bdf6e1c62..fe0434361e 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -6,10 +6,11 @@ module Backup def initialize @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_file_name = File.join(Gitlab.config.backup.path, 'database.sql.gz') + @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end def dump + FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 53e79d4d1f..f011fd03de 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -150,11 +150,11 @@ module Backup private def backup_contents - folders_to_backup + ["database.sql.gz", "uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] + folders_to_backup + ["uploads.tar.gz", "builds.tar.gz", "backup_information.yml"] end def folders_to_backup - folders = %w{repositories} + folders = %w{repositories db} if ENV["SKIP"] return folders.reject{ |folder| ENV["SKIP"].include?(folder) } From 7d58489fd908b2263f02e8919b1bd0b3fae1201d Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:22:08 +0200 Subject: [PATCH 06/13] Remove unused variable --- lib/backup/files.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 1b08e3324d..5a210a0e46 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -2,14 +2,13 @@ require 'open3' module Backup class Files - attr_reader :name, :app_files_dir, :backup_tarball, :backup_dir, :files_parent_dir + attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir def initialize(app_files_dir) @app_files_dir = File.realpath(app_files_dir) @name = File.basename(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) - @backup_dir = Gitlab.config.backup.path - @backup_tarball = File.join(@backup_dir, name + '.tar.gz') + @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end # Copy files from public/files to backup/files From 901f5445785a754227d8b77ca535947ab8cbbfca Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 15:38:21 +0200 Subject: [PATCH 07/13] Remove superfluous mkdir -p --- lib/backup/database.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index fe0434361e..67b2a64bd1 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -16,8 +16,6 @@ module Backup compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) compress_rd.close - FileUtils.mkdir_p(Gitlab.config.backup.path) - dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " From 852526e07fdac8cc11bdd4dd47cab71b39ae192c Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 16:10:14 +0200 Subject: [PATCH 08/13] Spec fixes for new backup contents --- spec/tasks/gitlab/backup_rake_spec.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 3be7dd4e52..9a881455ea 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -112,14 +112,14 @@ describe 'gitlab:app namespace rake task' do it 'should set correct permissions on the tar contents' do tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads repositories builds} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz} ) expect(exit_status).to eq(0) expect(tar_contents).to match('db/') - expect(tar_contents).to match('uploads/') + expect(tar_contents).to match('uploads.tar.gz') expect(tar_contents).to match('repositories/') - expect(tar_contents).to match('builds/') - expect(tar_contents).not_to match(/^.{4,9}[rwx].* (db|uploads|repositories|builds)\/$/) + expect(tar_contents).to match('builds.tar.gz') + expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz)\/$/) end it 'should delete temp directories' do @@ -160,12 +160,12 @@ describe 'gitlab:app namespace rake task' do it "does not contain skipped item" do tar_contents, _exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads repositories builds} + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz} ) expect(tar_contents).to match('db/') - expect(tar_contents).to match('uploads/') - expect(tar_contents).to match('builds/') + expect(tar_contents).to match('uploads.tar.gz') + expect(tar_contents).to match('builds.tar.gz') expect(tar_contents).not_to match('repositories/') end From b847baf8c405935415fcd4ad4a620d577f6526f8 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 6 Oct 2015 16:11:59 +0200 Subject: [PATCH 09/13] One more backup spec fix Stop the 'uploads' part from actually running. --- spec/tasks/gitlab/backup_rake_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 9a881455ea..386ac9c837 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -55,6 +55,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke) + expect(Rake::Task["gitlab:backup:uploads:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke) expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end From 58260a0327a953499a07e9cad8d9aaad2d25699b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 16 Oct 2015 17:16:17 +0200 Subject: [PATCH 10/13] Do no rely on basename of builds, uploads --- lib/backup/builds.rb | 6 +++++- lib/backup/files.rb | 9 +++++---- lib/backup/uploads.rb | 6 +++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index d269f8e260..800f30c214 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,7 +1,11 @@ module Backup class Builds < Files def initialize - super(Settings.gitlab_ci.builds_path) + super('builds', Settings.gitlab_ci.builds_path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) end end end diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 5a210a0e46..654b4d1c89 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -4,9 +4,9 @@ module Backup class Files attr_reader :name, :app_files_dir, :backup_tarball, :files_parent_dir - def initialize(app_files_dir) + def initialize(name, app_files_dir) + @name = name @app_files_dir = File.realpath(app_files_dir) - @name = File.basename(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end @@ -15,13 +15,14 @@ module Backup def dump FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) - run_pipeline!([%W(tar -C #{files_parent_dir} -cf - #{name}), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end def restore backup_existing_files_dir + create_files_dir - run_pipeline!([%W(gzip -cd), %W(tar -C #{files_parent_dir} -xf -)], in: backup_tarball) + run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 7c0838cc8b..0a0ec564ba 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,7 +2,11 @@ module Backup class Uploads < Files def initialize - super(Rails.root.join('public/uploads')) + super('uploads', Rails.root.join('public/uploads')) + end + + def create_files_dir + Dir.mkdir(app_files_dir) end end end From 3b0039f659eab1c29e735ef2a5443e26b33d5d18 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Sat, 31 Oct 2015 16:06:06 +0100 Subject: [PATCH 11/13] Persist blob editor's value on submit, not on click Prior, the value of the Ace editor was only being persisted if the user physically clicked the submit button, which the "quick submit" behavior doesn't do. Now the value will be properly transferred before any form is submitted. --- app/assets/javascripts/blob/edit_blob.js.coffee | 8 ++++---- app/assets/javascripts/blob/new_blob.js.coffee | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/assets/javascripts/blob/edit_blob.js.coffee b/app/assets/javascripts/blob/edit_blob.js.coffee index 050888f9c1..f6bf836f19 100644 --- a/app/assets/javascripts/blob/edit_blob.js.coffee +++ b/app/assets/javascripts/blob/edit_blob.js.coffee @@ -11,10 +11,10 @@ class @EditBlob if ace_mode editor.getSession().setMode "ace/mode/" + ace_mode - $(".js-commit-button").click -> - $("#file-content").val editor.getValue() - $(".file-editor form").submit() - return false + # Before a form submission, move the content from the Ace editor into the + # submitted textarea + $('form').submit -> + $("#file-content").val(editor.getValue()) editModePanes = $(".js-edit-mode-pane") editModeLinks = $(".js-edit-mode a") diff --git a/app/assets/javascripts/blob/new_blob.js.coffee b/app/assets/javascripts/blob/new_blob.js.coffee index 1f36a53f19..68c5e5195e 100644 --- a/app/assets/javascripts/blob/new_blob.js.coffee +++ b/app/assets/javascripts/blob/new_blob.js.coffee @@ -11,10 +11,10 @@ class @NewBlob if ace_mode editor.getSession().setMode "ace/mode/" + ace_mode - $(".js-commit-button").click -> - $("#file-content").val editor.getValue() - $(".file-editor form").submit() - return false + # Before a form submission, move the content from the Ace editor into the + # submitted textarea + $('form').submit -> + $("#file-content").val(editor.getValue()) editor: -> return @editor From 1045a72eb7fd16ab09b7e68e1a467ff0f5c9215d Mon Sep 17 00:00:00 2001 From: zheng_b Date: Mon, 2 Nov 2015 07:59:04 +0100 Subject: [PATCH 12/13] Go to gitlab installation folder before initialize database --- doc/install/installation.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/install/installation.md b/doc/install/installation.md index b48acb655e..8ed1eba560 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -329,6 +329,10 @@ GitLab Shell is an SSH access and repository management software developed speci sudo -u git -H make ### Initialize Database and Activate Advanced Features + + # Go to Gitlab installation folder + + cd /home/git/gilab sudo -u git -H bundle exec rake gitlab:setup RAILS_ENV=production From 1b8d324762787805fabee72d585ac40e244af4d7 Mon Sep 17 00:00:00 2001 From: Valery Sizov Date: Mon, 2 Nov 2015 14:14:38 +0200 Subject: [PATCH 13/13] Add ability to fetch the commit ID of the last commit that actually touched a file --- CHANGELOG | 1 + doc/api/repository_files.md | 3 ++- lib/api/files.rb | 4 +++- spec/requests/api/files_spec.rb | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6bec4f606e..588909d257 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,7 @@ v 8.2.0 (unreleased) - Fix: 500 error returned if destroy request without HTTP referer (Kazuki Shimizu) - Remove deprecated CI events from project settings page - Use issue editor as cross reference comment author when issue is edited with a new mention. + - [API] Add ability to fetch the commit ID of the last commit that actually touched a file v 8.1.1 - Fix cloning Wiki repositories via HTTP (Stan Hu) diff --git a/doc/api/repository_files.md b/doc/api/repository_files.md index 25311b0710..623063f357 100644 --- a/doc/api/repository_files.md +++ b/doc/api/repository_files.md @@ -23,7 +23,8 @@ Example response: "content": "IyA9PSBTY2hlbWEgSW5mb3...", "ref": "master", "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50" + "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", + "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d" } ``` diff --git a/lib/api/files.rb b/lib/api/files.rb index 308c84dd13..a7a768f889 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -43,7 +43,8 @@ module API # "content": "IyA9PSBTY2hlbWEgSW5mb3...", # "ref": "master", # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50" + # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", + # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", # } # get ":id/repository/files" do @@ -71,6 +72,7 @@ module API ref: ref, blob_id: blob.id, commit_id: commit.id, + last_commit_id: user_project.repository.last_commit_for_path(commit.sha, file_path).id } else not_found! 'File' diff --git a/spec/requests/api/files_spec.rb b/spec/requests/api/files_spec.rb index 042e635256..8efa09f75f 100644 --- a/spec/requests/api/files_spec.rb +++ b/spec/requests/api/files_spec.rb @@ -19,6 +19,7 @@ describe API::API, api: true do expect(response.status).to eq(200) expect(json_response['file_path']).to eq(file_path) expect(json_response['file_name']).to eq('popen.rb') + expect(json_response['last_commit_id']).to eq('570e7b2abdd848b95f2f578043fc23bd6f6fd24d') expect(Base64.decode64(json_response['content']).lines.first).to eq("require 'fileutils'\n") end