From 67a6fee5b124a768279b0681b52be19f138530dc Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 15 Jan 2016 18:12:36 +0100 Subject: [PATCH 01/12] Prototype of Git blobs via workhorse --- app/controllers/projects/raw_controller.rb | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index be7d5c187f..e55cdea94a 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -1,3 +1,5 @@ +require 'base64' + # Controller for viewing a file's raw class Projects::RawController < Projects::ApplicationController include ExtractsPath @@ -15,7 +17,10 @@ class Projects::RawController < Projects::ApplicationController if @blob.lfs_pointer? send_lfs_object else - stream_data + headers['Gitlab-Workhorse-Repo-Path'] = @repository.path_to_repo + headers['Gitlab-Workhorse-Send-Blob'] = Base64.urlsafe_encode64(@commit.id + ':' + @path) + headers['Content-Disposition'] = 'inline' + render nothing: true, content_type: get_blob_type end else render_404 @@ -34,16 +39,6 @@ class Projects::RawController < Projects::ApplicationController end end - def stream_data - type = get_blob_type - - send_data( - @blob.data, - type: type, - disposition: 'inline' - ) - end - def send_lfs_object lfs_object = find_lfs_object From 53ddb8f2ccaa95a19d530827e3fc7500d02ccfff Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Wed, 20 Jan 2016 13:44:39 +0100 Subject: [PATCH 02/12] Use a /raw/ request to show an image --- app/views/projects/blob/_image.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/blob/_image.html.haml b/app/views/projects/blob/_image.html.haml index c090f690d1..51fa91b08e 100644 --- a/app/views/projects/blob/_image.html.haml +++ b/app/views/projects/blob/_image.html.haml @@ -1,2 +1,2 @@ .file-content.image_file - %img{ src: "data:#{blob.mime_type};base64,#{Base64.encode64(blob.data)}"} + %img{ src: namespace_project_raw_path(@project.namespace, @project, @id)} From 9109619ee6fe85ee7e8ebfa51c6587319b523930 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 28 Jan 2016 15:02:44 +0100 Subject: [PATCH 03/12] Use less memory while counting lines --- app/views/shared/_file_highlight.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/shared/_file_highlight.html.haml b/app/views/shared/_file_highlight.html.haml index 2bc98983d6..331bad3ccf 100644 --- a/app/views/shared/_file_highlight.html.haml +++ b/app/views/shared/_file_highlight.html.haml @@ -1,7 +1,7 @@ .file-content.code.js-syntax-highlight{ class: user_color_scheme } .line-numbers - if blob.data.present? - - blob.data.lines.each_index do |index| + - blob.data.each_line.each_with_index do |_, index| - offset = defined?(first_line_number) ? first_line_number : 1 - i = index + offset -# We're not using `link_to` because it is too slow once we get to thousands of lines. From 26d97ac5e19c242594b59d224a77d41d0f1de6e1 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 28 Jan 2016 18:04:46 +0100 Subject: [PATCH 04/12] Send more raw blob data with workhorse --- app/controllers/projects/avatars_controller.rb | 13 ++++++------- app/controllers/projects/raw_controller.rb | 2 +- lib/api/repositories.rb | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/app/controllers/projects/avatars_controller.rb b/app/controllers/projects/avatars_controller.rb index 548f1b9ebf..0cd65ad5b1 100644 --- a/app/controllers/projects/avatars_controller.rb +++ b/app/controllers/projects/avatars_controller.rb @@ -2,15 +2,14 @@ class Projects::AvatarsController < Projects::ApplicationController before_action :project def show - @blob = @project.repository.blob_at_branch('master', @project.avatar_in_git) + repository = @project.repository + @blob = repository.blob_at_branch('master', @project.avatar_in_git) if @blob headers['X-Content-Type-Options'] = 'nosniff' - send_data( - @blob.data, - type: @blob.mime_type, - disposition: 'inline', - filename: @blob.name - ) + headers['Gitlab-Workhorse-Repo-Path'] = repository.path_to_repo + headers['Gitlab-Workhorse-Send-Blob'] = @blob.id + headers['Content-Disposition'] = 'inline' + render nothing: true, content_type: @blob.content_type else render_404 end diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index e55cdea94a..b3c846bc3c 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -18,7 +18,7 @@ class Projects::RawController < Projects::ApplicationController send_lfs_object else headers['Gitlab-Workhorse-Repo-Path'] = @repository.path_to_repo - headers['Gitlab-Workhorse-Send-Blob'] = Base64.urlsafe_encode64(@commit.id + ':' + @path) + headers['Gitlab-Workhorse-Send-Blob'] = @blob.id headers['Content-Disposition'] = 'inline' render nothing: true, content_type: get_blob_type end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index d7c48639eb..0f4cd2443b 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -57,7 +57,8 @@ module API not_found! "File" unless blob content_type 'text/plain' - present blob.data + header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo + header 'Gitlab-Workhorse-Send-Blob', blob.id end # Get a raw blob contents by blob sha @@ -83,7 +84,8 @@ module API env['api.format'] = :txt content_type blob.mime_type - present blob.data + header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo + header 'Gitlab-Workhorse-Send-Blob', blob.id end # Get a an archive of the repository From 368b855d88597a48637145838f61113af9c0e98e Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 28 Jan 2016 18:08:37 +0100 Subject: [PATCH 05/12] No need for base64 anymore --- app/controllers/projects/raw_controller.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index b3c846bc3c..b8b9048902 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -1,5 +1,3 @@ -require 'base64' - # Controller for viewing a file's raw class Projects::RawController < Projects::ApplicationController include ExtractsPath From 0197549d57bd32ee933ee775c625ed6fc5f5eec8 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 28 Jan 2016 18:09:52 +0100 Subject: [PATCH 06/12] Cannot see a raw blob without workhorse --- features/steps/project/source/browse_files.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/steps/project/source/browse_files.rb b/features/steps/project/source/browse_files.rb index d08935aa10..71db552f7b 100644 --- a/features/steps/project/source/browse_files.rb +++ b/features/steps/project/source/browse_files.rb @@ -52,7 +52,7 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps end step 'I should see raw file content' do - expect(source).to eq sample_blob.data + expect(source).to eq '' # Body is filled in by gitlab-workhorse end step 'I click button "Edit"' do From 02afa6793cca042f8563b0e26472606c743d76f5 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 11:33:22 +0100 Subject: [PATCH 07/12] Use only one header to send git blobs --- .../projects/avatars_controller.rb | 3 +-- app/controllers/projects/raw_controller.rb | 3 +-- lib/api/repositories.rb | 6 ++---- lib/gitlab/workhorse.rb | 21 +++++++++++++++++++ 4 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 lib/gitlab/workhorse.rb diff --git a/app/controllers/projects/avatars_controller.rb b/app/controllers/projects/avatars_controller.rb index 0cd65ad5b1..eb501c3964 100644 --- a/app/controllers/projects/avatars_controller.rb +++ b/app/controllers/projects/avatars_controller.rb @@ -6,8 +6,7 @@ class Projects::AvatarsController < Projects::ApplicationController @blob = repository.blob_at_branch('master', @project.avatar_in_git) if @blob headers['X-Content-Type-Options'] = 'nosniff' - headers['Gitlab-Workhorse-Repo-Path'] = repository.path_to_repo - headers['Gitlab-Workhorse-Send-Blob'] = @blob.id + headers.store(*Gitlab::Workhorse.send_git_blob(repository, @blob)) headers['Content-Disposition'] = 'inline' render nothing: true, content_type: @blob.content_type else diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index b8b9048902..2ab8a6b83b 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -15,8 +15,7 @@ class Projects::RawController < Projects::ApplicationController if @blob.lfs_pointer? send_lfs_object else - headers['Gitlab-Workhorse-Repo-Path'] = @repository.path_to_repo - headers['Gitlab-Workhorse-Send-Blob'] = @blob.id + headers.store(*Gitlab::Workhorse.send_git_blob(@repository, @blob)) headers['Content-Disposition'] = 'inline' render nothing: true, content_type: get_blob_type end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 0f4cd2443b..c95d2d2001 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -57,8 +57,7 @@ module API not_found! "File" unless blob content_type 'text/plain' - header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo - header 'Gitlab-Workhorse-Send-Blob', blob.id + header *Gitlab::Workhorse.send_git_blob(repo, blob) end # Get a raw blob contents by blob sha @@ -84,8 +83,7 @@ module API env['api.format'] = :txt content_type blob.mime_type - header 'Gitlab-Workhorse-Repo-Path', repo.path_to_repo - header 'Gitlab-Workhorse-Send-Blob', blob.id + header *Gitlab::Workhorse.send_git_blob(repo, blob) end # Get a an archive of the repository diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb new file mode 100644 index 0000000000..ff6fbf0b5c --- /dev/null +++ b/lib/gitlab/workhorse.rb @@ -0,0 +1,21 @@ +require 'base64' +require 'json' + +module Gitlab + class Workhorse + class << self + def send_git_blob(repository, blob) + params_hash = { + 'RepoPath' => repository.path_to_repo, + 'BlobId' => blob.id, + } + params = Base64.urlsafe_encode64(JSON.dump(params_hash)) + + [ + 'Gitlab-Workhorse-Send-Data', + "git-blob:#{params}", + ] + end + end + end +end \ No newline at end of file From b2a634c352ab23ed1180b7678b5c83db0137c0f7 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 12:01:13 +0100 Subject: [PATCH 08/12] Avoid trailing 'charset=' garbage --- app/controllers/projects/avatars_controller.rb | 3 ++- app/controllers/projects/raw_controller.rb | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/avatars_controller.rb b/app/controllers/projects/avatars_controller.rb index eb501c3964..bc40e60103 100644 --- a/app/controllers/projects/avatars_controller.rb +++ b/app/controllers/projects/avatars_controller.rb @@ -8,7 +8,8 @@ class Projects::AvatarsController < Projects::ApplicationController headers['X-Content-Type-Options'] = 'nosniff' headers.store(*Gitlab::Workhorse.send_git_blob(repository, @blob)) headers['Content-Disposition'] = 'inline' - render nothing: true, content_type: @blob.content_type + headers['Content-Type'] = @blob.content_type + head :ok # 'render nothing: true' messes up the Content-Type else render_404 end diff --git a/app/controllers/projects/raw_controller.rb b/app/controllers/projects/raw_controller.rb index 2ab8a6b83b..87b4d08da0 100644 --- a/app/controllers/projects/raw_controller.rb +++ b/app/controllers/projects/raw_controller.rb @@ -17,7 +17,8 @@ class Projects::RawController < Projects::ApplicationController else headers.store(*Gitlab::Workhorse.send_git_blob(@repository, @blob)) headers['Content-Disposition'] = 'inline' - render nothing: true, content_type: get_blob_type + headers['Content-Type'] = get_blob_type + head :ok # 'render nothing: true' messes up the Content-Type end else render_404 From b1f22aa35aa62d72f514b3f9beee0a190b6599cc Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Mon, 1 Feb 2016 12:27:35 +0100 Subject: [PATCH 09/12] Gotta have newlines --- lib/gitlab/workhorse.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index ff6fbf0b5c..a23120a417 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -18,4 +18,4 @@ module Gitlab end end end -end \ No newline at end of file +end From b67f8eee0d2810be4a010c5e968e1848b4ca3c88 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 2 Feb 2016 15:06:14 +0100 Subject: [PATCH 10/12] Use @repository --- app/controllers/projects/avatars_controller.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/avatars_controller.rb b/app/controllers/projects/avatars_controller.rb index bc40e60103..f7e6bb3444 100644 --- a/app/controllers/projects/avatars_controller.rb +++ b/app/controllers/projects/avatars_controller.rb @@ -2,11 +2,10 @@ class Projects::AvatarsController < Projects::ApplicationController before_action :project def show - repository = @project.repository - @blob = repository.blob_at_branch('master', @project.avatar_in_git) + @blob = @repository.blob_at_branch('master', @project.avatar_in_git) if @blob headers['X-Content-Type-Options'] = 'nosniff' - headers.store(*Gitlab::Workhorse.send_git_blob(repository, @blob)) + headers.store(*Gitlab::Workhorse.send_git_blob(@repository, @blob)) headers['Content-Disposition'] = 'inline' headers['Content-Type'] = @blob.content_type head :ok # 'render nothing: true' messes up the Content-Type From d716905743352de34bca8441706eef776f1cd6ce Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 2 Feb 2016 16:00:13 +0100 Subject: [PATCH 11/12] Use gitlab-workhorse 0.6.3 --- GITLAB_WORKHORSE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index b616048743..844f6a91ac 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -0.6.2 +0.6.3 From ea41d3607769b2b726a25d50ea23675bc5936227 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Tue, 2 Feb 2016 16:01:20 +0100 Subject: [PATCH 12/12] Use gitlab-workhorse 0.6.3 in installation guide --- doc/install/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/installation.md b/doc/install/installation.md index 2cc2dbef41..a2c23bd52e 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -355,7 +355,7 @@ GitLab Shell is an SSH access and repository management software developed speci cd /home/git sudo -u git -H git clone https://gitlab.com/gitlab-org/gitlab-workhorse.git cd gitlab-workhorse - sudo -u git -H git checkout 0.6.2 + sudo -u git -H git checkout 0.6.3 sudo -u git -H make ### Initialize Database and Activate Advanced Features