Commit 89519349 authored by Nick Thomas's avatar Nick Thomas 💃

Merge branch 'ac-subgroups' into 'master'

pages for subgroups

Closes gitlab-ce#30548

See merge request !123
parents 5dc404ef f7fd1fa5
Pipeline #43012924 passed with stage
in 1 minute and 48 seconds
......@@ -10,6 +10,7 @@ import (
"net/http/httptest"
"net/url"
"os"
"path"
"testing"
"time"
......@@ -128,6 +129,11 @@ func TestKnownHostReturns200(t *testing.T) {
host: "CapitalGroup.gitlab-example.com",
path: "CapitalProject/",
},
{
name: "subgroup",
host: "group.gitlab-example.com",
path: "subgroup/project/",
},
}
for _, test := range tests {
......@@ -143,6 +149,51 @@ func TestKnownHostReturns200(t *testing.T) {
}
}
func TestNestedSubgroups(t *testing.T) {
skipUnlessEnabled(t)
maxNestedSubgroup := 21
pagesRoot, err := ioutil.TempDir("", "pages-root")
require.NoError(t, err)
defer os.RemoveAll(pagesRoot)
makeProjectIndex := func(subGroupPath string) {
projectPath := path.Join(pagesRoot, "nested", subGroupPath, "project", "public")
require.NoError(t, os.MkdirAll(projectPath, 0755))
projectIndex := path.Join(projectPath, "index.html")
require.NoError(t, ioutil.WriteFile(projectIndex, []byte("index"), 0644))
}
makeProjectIndex("")
paths := []string{""}
for i := 1; i < maxNestedSubgroup*2; i++ {
subGroupPath := fmt.Sprintf("%ssub%d/", paths[i-1], i)
paths = append(paths, subGroupPath)
makeProjectIndex(subGroupPath)
}
teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-pages-root", pagesRoot)
defer teardown()
for nestingLevel, path := range paths {
t.Run(fmt.Sprintf("nested level %d", nestingLevel), func(t *testing.T) {
for _, spec := range listeners {
rsp, err := GetPageFromListener(t, spec, "nested.gitlab-example.com", path+"project/")
require.NoError(t, err)
rsp.Body.Close()
if nestingLevel <= maxNestedSubgroup {
require.Equal(t, http.StatusOK, rsp.StatusCode)
} else {
require.Equal(t, http.StatusNotFound, rsp.StatusCode)
}
}
})
}
}
func TestCORSWhenDisabled(t *testing.T) {
skipUnlessEnabled(t)
teardown := RunPagesProcess(t, *pagesBinary, listeners, "", "-disable-cross-origin-requests")
......@@ -828,13 +879,13 @@ func TestAccessControl(t *testing.T) {
case "/api/v4/user":
assert.Equal(t, "Bearer abc", r.Header.Get("Authorization"))
w.WriteHeader(http.StatusOK)
case "/api/v4/projects/1000/pages_access":
case "/api/v4/projects/1000/pages_access", "/api/v4/projects/1001/pages_access":
assert.Equal(t, "Bearer abc", r.Header.Get("Authorization"))
w.WriteHeader(http.StatusOK)
case "/api/v4/projects/2000/pages_access":
case "/api/v4/projects/2000/pages_access", "/api/v4/projects/2001/pages_access":
assert.Equal(t, "Bearer abc", r.Header.Get("Authorization"))
w.WriteHeader(http.StatusUnauthorized)
case "/api/v4/projects/3000/pages_access":
case "/api/v4/projects/3000/pages_access", "/api/v4/projects/3001/pages_access":
assert.Equal(t, "Bearer abc", r.Header.Get("Authorization"))
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprint(w, "{\"error\":\"invalid_token\"}")
......@@ -896,6 +947,41 @@ func TestAccessControl(t *testing.T) {
http.StatusNotFound,
false,
"no project should redirect to login and then return 404",
}, // subgroups
{
"group.auth.gitlab-example.com",
"/subgroup/private.project/",
http.StatusOK,
false,
"[subgroup] project with access",
},
{
"group.auth.gitlab-example.com",
"/subgroup/private.project.1/",
http.StatusNotFound, // Do not expose project existed
false,
"[subgroup] project without access",
},
{
"group.auth.gitlab-example.com",
"/subgroup/private.project.2/",
http.StatusFound,
true,
"[subgroup] invalid token test should redirect back",
},
{
"group.auth.gitlab-example.com",
"/subgroup/nonexistent/",
http.StatusNotFound,
false,
"[subgroup] no project should redirect to login and then return 404",
},
{
"nonexistent.gitlab-example.com",
"/subgroup/nonexistent/",
http.StatusNotFound,
false,
"[subgroup] no project should redirect to login and then return 404",
},
}
......
......@@ -21,6 +21,14 @@ import (
"gitlab.com/gitlab-org/gitlab-pages/internal/httputil"
)
const (
subgroupScanLimit int = 21
// maxProjectDepth is set to the maximum nested project depth in gitlab (21) plus 3.
// One for the project, one for the first empty element of the split (URL.Path starts with /),
// and one for the real file path
maxProjectDepth int = subgroupScanLimit + 3
)
type locationDirectoryError struct {
FullPath string
RelativePath string
......@@ -33,11 +41,9 @@ type project struct {
ID uint64
}
type projects map[string]*project
// D is a domain that gitlab-pages can serve.
type D struct {
group string
group
// custom domains:
projectName string
......@@ -46,19 +52,16 @@ type D struct {
certificate *tls.Certificate
certificateError error
certificateOnce sync.Once
// group domains:
projects projects
}
// String implements Stringer.
func (d *D) String() string {
if d.group != "" && d.projectName != "" {
return d.group + "/" + d.projectName
if d.group.name != "" && d.projectName != "" {
return d.group.name + "/" + d.projectName
}
if d.group != "" {
return d.group
if d.group.name != "" {
return d.group.name
}
return d.projectName
......@@ -110,11 +113,11 @@ func getHost(r *http.Request) string {
func (d *D) getProjectWithSubpath(r *http.Request) (*project, string, string) {
// Check for a project specified in the URL: http://group.gitlab.io/projectA
// If present, these projects shadow the group domain.
split := strings.SplitN(r.URL.Path, "/", 3)
split := strings.SplitN(r.URL.Path, "/", maxProjectDepth)
if len(split) >= 2 {
projectName := strings.ToLower(split[1])
if project := d.projects[projectName]; project != nil {
return project, split[1], strings.Join(split[2:], "/")
project, projectPath, urlPath := d.digProjectWithSubpath("", split[1:])
if project != nil {
return project, projectPath, urlPath
}
}
......@@ -314,7 +317,7 @@ func (d *D) serveCustomFile(w http.ResponseWriter, r *http.Request, code int, or
// Resolve the HTTP request to a path on disk, converting requests for
// directories to requests for index.html inside the directory if appropriate.
func (d *D) resolvePath(projectName string, subPath ...string) (string, error) {
publicPath := filepath.Join(d.group, projectName, "public")
publicPath := filepath.Join(d.group.name, projectName, "public")
// Don't use filepath.Join as cleans the path,
// where we want to traverse full path as supplied by user
......
......@@ -38,7 +38,8 @@ func TestDomainConfigValidness(t *testing.T) {
}
func TestDomainConfigRead(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
d := domainsConfig{}
err := d.Read("test-group", "test-project")
......
......@@ -11,7 +11,6 @@ import (
"testing"
"time"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
......@@ -27,16 +26,19 @@ func serveFileOrNotFound(domain *D) http.HandlerFunc {
}
func TestGroupServeHTTP(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
testGroup := &D{
group: "group",
projectName: "",
projects: map[string]*project{
"group.test.io": &project{},
"group.gitlab-example.com": &project{},
"project": &project{},
"project2": &project{},
group: group{
name: "group",
projects: map[string]*project{
"group.test.io": &project{},
"group.gitlab-example.com": &project{},
"project": &project{},
"project2": &project{},
},
},
}
......@@ -64,10 +66,11 @@ func TestGroupServeHTTP(t *testing.T) {
}
func TestDomainServeHTTP(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
testDomain := &D{
group: "group",
group: group{name: "group"},
projectName: "project2",
config: &domainConfig{
Domain: "test.domain.com",
......@@ -95,7 +98,7 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Custom domain with HTTPS-only enabled",
domain: &D{
group: "group",
group: group{name: "group"},
projectName: "project",
config: &domainConfig{HTTPSOnly: true},
},
......@@ -105,7 +108,7 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Custom domain with HTTPS-only disabled",
domain: &D{
group: "group",
group: group{name: "group"},
projectName: "project",
config: &domainConfig{HTTPSOnly: false},
},
......@@ -115,9 +118,11 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Default group domain with HTTPS-only enabled",
domain: &D{
group: "group",
projectName: "project",
projects: projects{"test-domain": &project{HTTPSOnly: true}},
group: group{
name: "group",
projects: projects{"test-domain": &project{HTTPSOnly: true}},
},
},
url: "http://test-domain",
expected: true,
......@@ -125,9 +130,11 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Default group domain with HTTPS-only disabled",
domain: &D{
group: "group",
projectName: "project",
projects: projects{"test-domain": &project{HTTPSOnly: false}},
group: group{
name: "group",
projects: projects{"test-domain": &project{HTTPSOnly: false}},
},
},
url: "http://test-domain",
expected: false,
......@@ -135,9 +142,11 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Case-insensitive default group domain with HTTPS-only enabled",
domain: &D{
group: "group",
projectName: "project",
projects: projects{"test-domain": &project{HTTPSOnly: true}},
group: group{
name: "group",
projects: projects{"test-domain": &project{HTTPSOnly: true}},
},
},
url: "http://Test-domain",
expected: true,
......@@ -145,9 +154,11 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Other group domain with HTTPS-only enabled",
domain: &D{
group: "group",
projectName: "project",
projects: projects{"project": &project{HTTPSOnly: true}},
group: group{
name: "group",
projects: projects{"project": &project{HTTPSOnly: true}},
},
},
url: "http://test-domain/project",
expected: true,
......@@ -155,9 +166,11 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Other group domain with HTTPS-only disabled",
domain: &D{
group: "group",
projectName: "project",
projects: projects{"project": &project{HTTPSOnly: false}},
group: group{
name: "group",
projects: projects{"project": &project{HTTPSOnly: false}},
},
},
url: "http://test-domain/project",
expected: false,
......@@ -165,7 +178,7 @@ func TestIsHTTPSOnly(t *testing.T) {
{
name: "Unknown project",
domain: &D{
group: "group",
group: group{name: "group"},
projectName: "project",
},
url: "http://test-domain/project",
......@@ -209,16 +222,19 @@ func testHTTPGzip(t *testing.T, handler http.HandlerFunc, mode, url string, valu
}
func TestGroupServeHTTPGzip(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
testGroup := &D{
group: "group",
projectName: "",
projects: map[string]*project{
"group.test.io": &project{},
"group.gitlab-example.com": &project{},
"project": &project{},
"project2": &project{},
group: group{
name: "group",
projects: map[string]*project{
"group.test.io": &project{},
"group.gitlab-example.com": &project{},
"project": &project{},
"project2": &project{},
},
},
}
......@@ -285,17 +301,20 @@ func testHTTP404(t *testing.T, handler http.HandlerFunc, mode, url string, value
}
func TestGroup404ServeHTTP(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
testGroup := &D{
group: "group.404",
projectName: "",
projects: map[string]*project{
"domain.404": &project{},
"group.404.test.io": &project{},
"project.404": &project{},
"project.404.symlink": &project{},
"project.no.404": &project{},
group: group{
name: "group.404",
projects: map[string]*project{
"domain.404": &project{},
"group.404.test.io": &project{},
"project.404": &project{},
"project.404.symlink": &project{},
"project.no.404": &project{},
},
},
}
......@@ -311,10 +330,11 @@ func TestGroup404ServeHTTP(t *testing.T) {
}
func TestDomain404ServeHTTP(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
testDomain := &D{
group: "group.404",
group: group{name: "group.404"},
projectName: "domain.404",
config: &domainConfig{
Domain: "domain.404.com",
......@@ -326,10 +346,11 @@ func TestDomain404ServeHTTP(t *testing.T) {
}
func TestPredefined404ServeHTTP(t *testing.T) {
setUpTests()
cleanup := setUpTests(t)
defer cleanup()
testDomain := &D{
group: "group",
group: group{name: "group"},
}
testHTTP404(t, serveFileOrNotFound(testDomain), "GET", "http://group.test.io/not-existing-file", nil, "The page you're looking for could not be found")
......@@ -337,7 +358,7 @@ func TestPredefined404ServeHTTP(t *testing.T) {
func TestGroupCertificate(t *testing.T) {
testGroup := &D{
group: "group",
group: group{name: "group"},
projectName: "",
}
......@@ -348,7 +369,7 @@ func TestGroupCertificate(t *testing.T) {
func TestDomainNoCertificate(t *testing.T) {
testDomain := &D{
group: "group",
group: group{name: "group"},
projectName: "project2",
config: &domainConfig{
Domain: "test.domain.com",
......@@ -366,7 +387,7 @@ func TestDomainNoCertificate(t *testing.T) {
func TestDomainCertificate(t *testing.T) {
testDomain := &D{
group: "group",
group: group{name: "group"},
projectName: "project2",
config: &domainConfig{
Domain: "test.domain.com",
......@@ -381,10 +402,15 @@ func TestDomainCertificate(t *testing.T) {
}
func TestCacheControlHeaders(t *testing.T) {
cleanup := setUpTests(t)
defer cleanup()
testGroup := &D{
group: "group",
projects: map[string]*project{
"group.test.io": &project{},
group: group{
name: "group",
projects: map[string]*project{
"group.test.io": &project{},
},
},
}
w := httptest.NewRecorder()
......@@ -431,15 +457,27 @@ func TestOpenNoFollow(t *testing.T) {
var chdirSet = false
func setUpTests() {
func setUpTests(t require.TestingT) func() {
return chdirInPath(t, "../../shared/pages")
}
func chdirInPath(t require.TestingT, path string) func() {
noOp := func() {}
if chdirSet {
return
return noOp
}
err := os.Chdir("../../shared/pages")
if err != nil {
log.WithError(err).Print("chdir")
} else {
chdirSet = true
cwd, err := os.Getwd()
require.NoError(t, err, "Cannot Getwd")
err = os.Chdir(path)
require.NoError(t, err, "Cannot Chdir")
chdirSet = true
return func() {
err := os.Chdir(cwd)
require.NoError(t, err, "Cannot Chdir in cleanup")
chdirSet = false
}
}
package domain
import (
"path"
"strings"
)
type projects map[string]*project
type subgroups map[string]*group
type group struct {
name string
// nested groups
subgroups subgroups
// group domains:
projects projects
}
func (g *group) digProjectWithSubpath(parentPath string, keys []string) (*project, string, string) {
if len(keys) >= 1 {
head := keys[0]
tail := keys[1:]
currentPath := path.Join(parentPath, head)
search := strings.ToLower(head)
if project := g.projects[search]; project != nil {
return project, currentPath, path.Join(tail...)
}
if subgroup := g.subgroups[search]; subgroup != nil {
return subgroup.digProjectWithSubpath(currentPath, tail)
}
}
return nil, "", ""
}
package domain
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGroupDig(t *testing.T) {
matchingProject := &project{ID: 1}
tests := []struct {
name string
g group
path string
expectedProject *project
expectedProjectPath string
expectedPath string
}{
{
name: "empty group",
path: "projectb/demo/features.html",
g: group{},
},
{
name: "group with project",
path: "projectb/demo/features.html",
g: group{
projects: projects{"projectb": matchingProject},
},
expectedProject: matchingProject,
expectedProjectPath: "projectb",
expectedPath: "demo/features.html",
},
{
name: "group with project and no path in URL",
path: "projectb",
g: group{
projects: projects{"projectb": matchingProject},
},
expectedProject: matchingProject,
expectedProjectPath: "projectb",
},
{
name: "group with subgroup and project",
path: "projectb/demo/features.html",
g: group{
projects: projects{"projectb": matchingProject},
subgroups: subgroups{
"sub1": &group{
projects: projects{"another": &project{}},
},
},
},
expectedProject: matchingProject,
expectedProjectPath: "projectb",
expectedPath: "demo/features.html",
},
{
name: "group with project inside a subgroup",
path: "sub1/projectb/demo/features.html",
g: group{
subgroups: subgroups{
"sub1": &group{
projects: projects{"projectb": matchingProject},
},
},
projects: projects{"another": &project{}},
},
expectedProject: matchingProject,
expectedProjectPath: "sub1/projectb",
expectedPath: "demo/features.html",
},
{
name: "group with matching subgroup but no project",
path: "sub1/projectb/demo/features.html",
g: group{
subgroups: subgroups{
"sub1": &group{
projects: projects{"another": &project{}},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
project, projectPath, urlPath := test.g.digProjectWithSubpath("", strings.Split(test.path, "/"))
assert.Equal(t, test.expectedProject, project)
assert.Equal(t, test.expectedProjectPath, projectPath)
assert.Equal(t, test.expectedPath, urlPath)
})
}
}
......@@ -34,9 +34,9 @@ func (dm Map) updateDomainMap(domainName string, domain *D) {
dm[domainName] = domain
}
func (dm Map) addDomain(rootDomain, group, projectName string, config *domainConfig) {
func (dm Map) addDomain(rootDomain, groupName, projectName string, config *domainConfig) {
newDomain := &D{
group: group,
group: group{name: groupName},
projectName: projectName,
config: config,
}
......@@ -46,19 +46,41 @@ func (dm Map) addDomain(rootDomain, group, projectName string, config *domainCon
dm.updateDomainMap(domainName, newDomain)
}
func (dm Map) updateGroupDomain(rootDomain, group, projectName string, httpsOnly bool, accessControl bool, id uint64) {
domainName := strings.ToLower(group + "." + rootDomain)
func (dm Map) updateGroupDomain(rootDomain, groupName, projectPath string, httpsOnly bool, accessControl bool, id uint64) {
domainName := strings.ToLower(groupName + "." + rootDomain)
groupDomain := dm[domainName]
if groupDomain == nil {
groupDomain = &D{
group: group,
projects: make(projects),
group: group{
name: groupName,
projects: make(projects),
subgroups: make(subgroups),
},
}
}
groupDomain.projects[strings.ToLower(projectName)] = &project{
NamespaceProject: domainName == strings.ToLower(projectName),
split := strings.SplitN(strings.ToLower(projectPath), "/", maxProjectDepth)
projectName := split[len(split)-1]
g := &groupDomain.group
for i := 0; i < len(split)-1; i++ {
subgroupName := split[i]
subgroup := g.subgroups[subgroupName]
if subgroup == nil {
subgroup = &group{
name: subgroupName,
projects: make(projects),
subgroups: make(subgroups),
}
g.subgroups[subgroupName] = subgroup
}
g = subgroup
}
g.projects[projectName] = &project{
NamespaceProject: domainName == projectName,
HTTPSOnly: httpsOnly,
AccessControl: accessControl,
ID: id,
......@@ -86,7 +108,7 @@ func (dm Map) readProjectConfig(rootDomain string, group, projectName string, co
}
}
func readProject(group, projectName string, fanIn chan<- jobResult) {
func readProject(group, parent, projectName string, level int, fanIn chan<- jobResult) {
if strings.HasPrefix(projectName, ".") {
return
}
......@@ -96,25 +118,34 @@ func readProject(group, projectName string, fanIn chan<- jobResult) {
return
}
if _, err := os.Lstat(filepath.Join(group, projectName, "public")); err != nil {
projectPath := filepath.Join(parent, projectName)
if _, err := os.Lstat(filepath.Join(group, projectPath, "public")); err != nil {