From 31a8ba8a79ffbb7c7b45463b2edf7d51ea8ad7e0 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Tue, 11 Dec 2018 17:42:32 +0200 Subject: [PATCH 1/2] Add group based Maven API endpoint User will be able to download any maven package within the group with URL like /api/v4/groups/GROUP_ID_OR_NAME/packages/maven Signed-off-by: Dmitriy Zaporozhets --- doc/user/project/packages/maven_repository.md | 28 +++++ .../finders/packages/maven_package_finder.rb | 46 ++++++-- ee/app/models/packages/package.rb | 10 ++ .../find_or_create_maven_package_service.rb | 2 +- .../dz-group-level-maven-endpoint.yml | 5 + ee/lib/api/maven_packages.rb | 45 ++++++- .../packages/maven_package_finder_spec.rb | 30 ++++- ee/spec/requests/api/maven_packages_spec.rb | 110 +++++++++++++++++- 8 files changed, 255 insertions(+), 21 deletions(-) create mode 100644 ee/changelogs/unreleased/dz-group-level-maven-endpoint.yml diff --git a/doc/user/project/packages/maven_repository.md b/doc/user/project/packages/maven_repository.md index 5be82cc8ca2..175842d7b54 100644 --- a/doc/user/project/packages/maven_repository.md +++ b/doc/user/project/packages/maven_repository.md @@ -163,6 +163,34 @@ the `distributionManagement` section: If you have a self-hosted GitLab installation, replace `gitlab.com` with your domain name. +## Group level Maven endpoint + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/8798) in GitLab Premium 11.7. + +If you rely on many packages, it might be inefficient to include the `repository` section +with a unique URL for each package. Instead, you can use the group level endpoint for +all your maven packages stored within one GitLab group. Only packages you have access to +will be available for download. Here's how the relevant `repository` section of +your `pom.xml` would look like: + +```xml + + + gitlab-maven + https://gitlab.com/api/v4/groups/my-company/-/packages/maven + + +``` + +If you have a self-hosted GitLab installation, replace `gitlab.com` with your +domain name. + +**Notes**: + +- Group level endpoint works with any package names. That means if you have a flexibility of naming compared to instance level endpoint. However, that means GitLab will not guarantee uniqueness of package names withing the group. You can have two projects with a same package name and a package version. As result, GitLab will serve whichever one is more recent. +- You still need a project specific URL for uploading a package +in the `distributionManagement` section. + ## Uploading packages Once you have set up the [authorization](#authorizing-with-the-gitlab-maven-repository) diff --git a/ee/app/finders/packages/maven_package_finder.rb b/ee/app/finders/packages/maven_package_finder.rb index 6297b5325a4..92f7b0c6314 100644 --- a/ee/app/finders/packages/maven_package_finder.rb +++ b/ee/app/finders/packages/maven_package_finder.rb @@ -1,34 +1,58 @@ # frozen_string_literal: true class Packages::MavenPackageFinder - attr_reader :path, :project + attr_reader :path, :current_user, :project, :group - def initialize(path, project = nil) + def initialize(path, current_user, project: nil, group: nil) @path = path + @current_user = current_user @project = project + @group = group end def execute - packages.last + packages_with_path.last end def execute! - packages.last! + packages_with_path.last! end private - def scope + def base if project - project.packages + packages_for_a_single_project + elsif group + packages_for_multiple_projects else - ::Packages::Package.all + packages end end - # rubocop: disable CodeReuse/ActiveRecord + def packages_with_path + base.only_maven_packages_with_path(path) + end + + # Produces a query that returns all packages. def packages - scope.joins(:maven_metadatum) - .where(packages_maven_metadata: { path: path }) + ::Packages::Package.all + end + + # Produces a query that retrieves packages from a single project. + def packages_for_a_single_project + project.packages + end + + # Produces a query that retrieves packages from multiple projects that + # the current user can view within a group. + def packages_for_multiple_projects + ::Packages::Package.for_projects(projects_visible_to_current_user) + end + + # Returns the projects that the current user can view within a group. + def projects_visible_to_current_user + ::Project + .in_namespace(group.self_and_descendants.select(:id)) + .public_or_visible_to_user(current_user) end - # rubocop: enable CodeReuse/ActiveRecord end diff --git a/ee/app/models/packages/package.rb b/ee/app/models/packages/package.rb index b593f9546eb..e149bbbfc69 100644 --- a/ee/app/models/packages/package.rb +++ b/ee/app/models/packages/package.rb @@ -11,4 +11,14 @@ class Packages::Package < ActiveRecord::Base validates :name, presence: true, format: { with: Gitlab::Regex.package_name_regex } + + def self.for_projects(projects) + return none unless projects.any? + + where(project_id: projects) + end + + def self.only_maven_packages_with_path(path) + joins(:maven_metadatum).where(packages_maven_metadata: { path: path }) + end end diff --git a/ee/app/services/packages/find_or_create_maven_package_service.rb b/ee/app/services/packages/find_or_create_maven_package_service.rb index 1e43bf863b5..4727022448f 100644 --- a/ee/app/services/packages/find_or_create_maven_package_service.rb +++ b/ee/app/services/packages/find_or_create_maven_package_service.rb @@ -5,7 +5,7 @@ module Packages def execute package = ::Packages::MavenPackageFinder - .new(params[:path], project).execute + .new(params[:path], current_user, project: project).execute unless package if params[:file_name] == MAVEN_METADATA_FILE diff --git a/ee/changelogs/unreleased/dz-group-level-maven-endpoint.yml b/ee/changelogs/unreleased/dz-group-level-maven-endpoint.yml new file mode 100644 index 00000000000..4064abbed3a --- /dev/null +++ b/ee/changelogs/unreleased/dz-group-level-maven-endpoint.yml @@ -0,0 +1,5 @@ +--- +title: Add a group-level endpoint for downloading maven packages +merge_request: 8798 +author: +type: added diff --git a/ee/lib/api/maven_packages.rb b/ee/lib/api/maven_packages.rb index 5a182f33869..cc8facf22e3 100644 --- a/ee/lib/api/maven_packages.rb +++ b/ee/lib/api/maven_packages.rb @@ -76,7 +76,8 @@ module API authorize!(:read_package, project) - package = ::Packages::MavenPackageFinder.new(params[:path], project).execute! + package = ::Packages::MavenPackageFinder + .new(params[:path], current_user, project: project).execute! forbidden! unless package.project.feature_available?(:packages) @@ -93,6 +94,46 @@ module API end end + desc 'Download the maven package file at a group level' do + detail 'This feature was introduced in GitLab 11.7' + end + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :path, type: String, desc: 'Package path' + requires :file_name, type: String, desc: 'Package file name' + end + route_setting :authentication, job_token_allowed: true + get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + file_name, format = extract_format(params[:file_name]) + + group = find_group(params[:id]) + + not_found!('Group') unless can?(current_user, :read_group, group) + + package = ::Packages::MavenPackageFinder + .new(params[:path], current_user, group: group).execute! + + forbidden! unless package.project.feature_available?(:packages) + + authorize!(:read_package, package.project) + + package_file = ::Packages::PackageFileFinder + .new(package, file_name).execute! + + case format + when 'md5' + package_file.file_md5 + when 'sha1' + package_file.file_sha1 + when nil + present_carrierwave_file!(package_file.file) + end + end + end + params do requires :id, type: String, desc: 'The ID of a project' end @@ -115,7 +156,7 @@ module API file_name, format = extract_format(params[:file_name]) package = ::Packages::MavenPackageFinder - .new(params[:path], user_project).execute! + .new(params[:path], current_user, project: user_project).execute! package_file = ::Packages::PackageFileFinder .new(package, file_name).execute! diff --git a/ee/spec/finders/packages/maven_package_finder_spec.rb b/ee/spec/finders/packages/maven_package_finder_spec.rb index 0e1792e58ef..a250f064881 100644 --- a/ee/spec/finders/packages/maven_package_finder_spec.rb +++ b/ee/spec/finders/packages/maven_package_finder_spec.rb @@ -2,19 +2,25 @@ require 'spec_helper' describe Packages::MavenPackageFinder do - let(:project) { create(:project) } + let(:user) { create(:user) } + let(:group) { create(:group) } + let(:project) { create(:project, namespace: group) } let(:package) { create(:maven_package, project: project) } + before do + group.add_developer(user) + end + describe '#execute!' do context 'within the project' do it 'returns a package' do - finder = described_class.new(package.maven_metadatum.path, project) + finder = described_class.new(package.maven_metadatum.path, user, project: project) expect(finder.execute!).to eq(package) end it 'raises an error' do - finder = described_class.new('com/example/my-app/1.0-SNAPSHOT', project) + finder = described_class.new('com/example/my-app/1.0-SNAPSHOT', user, project: project) expect { finder.execute! }.to raise_error(ActiveRecord::RecordNotFound) end @@ -22,13 +28,27 @@ describe Packages::MavenPackageFinder do context 'across all projects' do it 'returns a package' do - finder = described_class.new(package.maven_metadatum.path) + finder = described_class.new(package.maven_metadatum.path, user) + + expect(finder.execute!).to eq(package) + end + + it 'raises an error' do + finder = described_class.new('com/example/my-app/1.0-SNAPSHOT', user) + + expect { finder.execute! }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context 'within a group' do + it 'returns a package' do + finder = described_class.new(package.maven_metadatum.path, user, group: group) expect(finder.execute!).to eq(package) end it 'raises an error' do - finder = described_class.new('com/example/my-app/1.0-SNAPSHOT') + finder = described_class.new('com/example/my-app/1.0-SNAPSHOT', user, group: group) expect { finder.execute! }.to raise_error(ActiveRecord::RecordNotFound) end diff --git a/ee/spec/requests/api/maven_packages_spec.rb b/ee/spec/requests/api/maven_packages_spec.rb index 1dd7c83d3c6..477b0e95a95 100644 --- a/ee/spec/requests/api/maven_packages_spec.rb +++ b/ee/spec/requests/api/maven_packages_spec.rb @@ -2,8 +2,9 @@ require 'spec_helper' describe API::MavenPackages do - let(:user) { create(:user) } - let(:project) { create(:project, :public) } + let(:group) { create(:group) } + let(:user) { create(:user) } + let(:project) { create(:project, :public, namespace: group) } let(:personal_access_token) { create(:personal_access_token, user: user) } let(:jwt_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } let(:headers) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => jwt_token } } @@ -125,6 +126,111 @@ describe API::MavenPackages do end end + describe 'GET /api/v4/groups/:id/-/packages/maven/*path/:file_name' do + let(:package) { create(:maven_package, project: project) } + let(:maven_metadatum) { package.maven_metadatum } + let(:package_file_xml) { package.package_files.find_by(file_type: 'xml') } + + before do + project.team.truncate + group.add_developer(user) + end + + context 'a public project' do + it 'returns the file' do + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + + it 'returns sha1 of the file' do + download_file(package_file_xml.file_name + '.sha1') + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('text/plain') + expect(response.body).to eq(package_file_xml.file_sha1) + end + end + + context 'internal project' do + before do + group.group_member(user).destroy + project.update!(visibility_level: Gitlab::VisibilityLevel::INTERNAL) + end + + it 'returns the file' do + download_file_with_token(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + + it 'denies download when no private token' do + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(404) + end + + it 'allows download with job token' do + download_file(package_file_xml.file_name, job_token: job.token) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + end + + context 'private project' do + before do + project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE) + end + + it 'returns the file' do + download_file_with_token(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + + it 'denies download when not enough permissions' do + group.add_guest(user) + + download_file_with_token(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(403) + end + + it 'denies download when no private token' do + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(404) + end + + it 'allows download with job token' do + download_file(package_file_xml.file_name, job_token: job.token) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + end + + it 'rejects request if feature is not in the license' do + stub_licensed_features(packages: false) + + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(403) + end + + def download_file(file_name, params = {}, request_headers = headers) + get api("/groups/#{group.id}/-/packages/maven/#{maven_metadatum.path}/#{file_name}"), params, request_headers + end + + def download_file_with_token(file_name, params = {}, request_headers = headers_with_token) + download_file(file_name, params, request_headers) + end + end + describe 'GET /api/v4/projects/:id/packages/maven/*path/:file_name' do let(:package) { create(:maven_package, project: project) } let(:maven_metadatum) { package.maven_metadatum } -- 2.24.1 From 5277c0c238b02fb6d0e171900b954e016db701dc Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Fri, 14 Dec 2018 13:27:31 +0100 Subject: [PATCH 2/2] Place all 3 different Maven endpoints under the same section --- doc/user/project/packages/maven_repository.md | 106 +++++++++++++----- 1 file changed, 75 insertions(+), 31 deletions(-) diff --git a/doc/user/project/packages/maven_repository.md b/doc/user/project/packages/maven_repository.md index 175842d7b54..5ad2535d893 100644 --- a/doc/user/project/packages/maven_repository.md +++ b/doc/user/project/packages/maven_repository.md @@ -89,7 +89,26 @@ You can read more on ## Configuring your project to use the GitLab Maven repository URL To download and upload packages from GitLab, you need a `repository` and -`distributionManagement` section respectively in your `pom.xml` file: +`distributionManagement` section in your `pom.xml` file. + +Depending on your workflow and the amount of Maven packages you have, there are +3 ways you can configure your project to use the GitLab endpoint for Maven packages: + +- **Project level**: Useful when you have few Maven packages which are not under + the same GitLab group. +- **Group level**: Useful when you have many Maven packages under the same GitLab + group. +- **Instance level**: Useful when you have many Maven packages under different + GitLab groups or on their own namespace. + +NOTE: **Note:** +In all cases, you need a project specific URL for uploading a package in +the `distributionManagement` section. + +### Project level Maven endpoint + +The example below shows how the relevant `repository` section of your `pom.xml` +would look like: ```xml @@ -113,29 +132,26 @@ To download and upload packages from GitLab, you need a `repository` and The `id` must be the same with what you [defined in `settings.xml`](#authorizing-with-the-maven-repository). -In both examples, replace `PROJECT_ID` with your project ID which can be found -on the home page of your project. +Replace `PROJECT_ID` with your project ID which can be found on the home page +of your project. If you have a self-hosted GitLab installation, replace `gitlab.com` with your domain name. -## Instance level Maven endpoint +### Group level Maven endpoint -> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/8274) in GitLab Premium 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/8798) in GitLab Premium 11.7. If you rely on many packages, it might be inefficient to include the `repository` section -with a unique URL for each package. Instead, you can use the instance level endpoint for -all maven packages stored in GitLab and the packages you have access to will be available -for download. +with a unique URL for each package. Instead, you can use the group level endpoint for +all your Maven packages stored within one GitLab group. Only packages you have access to +will be available for download. -Note that only packages that have the same path as the project are exposed via -the instance level endpoint. - -| Project | Package | Instance level endpoint available | -| ------- | ------- | --------------------------------- | -| `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | -| `gitlab-org/gitlab-ce` | `foo/bar/1.0-SNAPSHOT` | No | -| `gitlab-org/gitlab-ce` | `gitlab-org/gitlab-ce/1.0-SNAPSHOT` | Yes | +The group level endpoint works with any package names, which means the you +have the flexibility of naming compared to [instance level endpoint](#instance-level-maven-endpoint). +However, GitLab will not guarantee the uniqueness of the package names within +the group. You can have two projects with the same package name and package +version. As a result, GitLab will serve whichever one is more recent. The example below shows how the relevant `repository` section of your `pom.xml` would look like. You still need a project specific URL for uploading a package in @@ -145,7 +161,7 @@ the `distributionManagement` section: gitlab-maven - https://gitlab.com/api/v4/packages/maven + https://gitlab.com/api/v4/groups/my-group/-/packages/maven @@ -160,36 +176,64 @@ the `distributionManagement` section: ``` +The `id` must be the same with what you +[defined in `settings.xml`](#authorizing-with-the-maven-repository). + +Replace `my-group` with your group name and `PROJECT_ID` with your project ID +which can be found on the home page of your project. + If you have a self-hosted GitLab installation, replace `gitlab.com` with your domain name. -## Group level Maven endpoint +### Instance level Maven endpoint -> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/8798) in GitLab Premium 11.7. +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/8274) in GitLab Premium 11.7. -If you rely on many packages, it might be inefficient to include the `repository` section -with a unique URL for each package. Instead, you can use the group level endpoint for -all your maven packages stored within one GitLab group. Only packages you have access to -will be available for download. Here's how the relevant `repository` section of -your `pom.xml` would look like: +If you rely on many packages, it might be inefficient to include the `repository` section +with a unique URL for each package. Instead, you can use the instance level endpoint for +all maven packages stored in GitLab and the packages you have access to will be available +for download. + +Note that **only packages that have the same path as the project** are exposed via +the instance level endpoint. + +| Project | Package | Instance level endpoint available | +| ------- | ------- | --------------------------------- | +| `foo/bar` | `foo/bar/1.0-SNAPSHOT` | Yes | +| `gitlab-org/gitlab-ce` | `foo/bar/1.0-SNAPSHOT` | No | +| `gitlab-org/gitlab-ce` | `gitlab-org/gitlab-ce/1.0-SNAPSHOT` | Yes | + +The example below shows how the relevant `repository` section of your `pom.xml` +would look like. You still need a project specific URL for uploading a package in +the `distributionManagement` section: ```xml gitlab-maven - https://gitlab.com/api/v4/groups/my-company/-/packages/maven + https://gitlab.com/api/v4/packages/maven + + + gitlab-maven + https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven + + + gitlab-maven + https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven + + ``` -If you have a self-hosted GitLab installation, replace `gitlab.com` with your -domain name. +The `id` must be the same with what you +[defined in `settings.xml`](#authorizing-with-the-maven-repository). -**Notes**: +Replace `PROJECT_ID` with your project ID which can be found on the home page +of your project. -- Group level endpoint works with any package names. That means if you have a flexibility of naming compared to instance level endpoint. However, that means GitLab will not guarantee uniqueness of package names withing the group. You can have two projects with a same package name and a package version. As result, GitLab will serve whichever one is more recent. -- You still need a project specific URL for uploading a package -in the `distributionManagement` section. +If you have a self-hosted GitLab installation, replace `gitlab.com` with your +domain name. ## Uploading packages -- 2.24.1