From 254fe284ffbd6f1978820097f2bd1f6ff6664982 Mon Sep 17 00:00:00 2001 From: Serge Zaitsev Date: Mon, 22 Oct 2018 14:38:22 +0200 Subject: [PATCH] Feature/nested modules (#6) * add support for nested go modules * update dependencies --- go.mod | 12 +------- go.sum | 18 ++++++++++-- pkg/vcs/git.go | 68 +++++++++++++++++++++++++++++++++++--------- pkg/vcs/git_test.go | 14 +++++++++ pkg/vcs/meta.go | 23 ++++++++------- pkg/vcs/meta_test.go | 48 ++++++++++++++++++------------- 6 files changed, 126 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index ce977fa..b97c740 100644 --- a/go.mod +++ b/go.mod @@ -2,17 +2,7 @@ module github.com/sixt/gomodproxy require ( github.com/emirpasic/gods v1.11.0 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect - github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e // indirect - github.com/mitchellh/go-homedir v1.0.0 // indirect - github.com/pelletier/go-buffruneio v0.2.0 // indirect - github.com/sergi/go-diff v1.0.0 // indirect - github.com/src-d/gcfg v1.3.0 // indirect - github.com/xanzy/ssh-agent v0.2.0 // indirect golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b // indirect golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 // indirect - golang.org/x/text v0.3.0 // indirect - gopkg.in/src-d/go-billy.v4 v4.2.1 // indirect - gopkg.in/src-d/go-git.v4 v4.7.0 - gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/src-d/go-git.v4 v4.7.1 ) diff --git a/go.sum b/go.sum index 28598d8..c80e604 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ -github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.11.0 h1:8KhjokrJy1+REZkLeSlnJvLKI4tRR8g65a4C07oQQ50= github.com/emirpasic/gods v1.11.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -13,14 +19,19 @@ github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnG github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg= github.com/src-d/gcfg v1.3.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b h1:2b9XGzhjiYsYPnKXoEfL7klWZQIt8IfyRCz62gCqqlQ= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -29,7 +40,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo= gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= -gopkg.in/src-d/go-git.v4 v4.7.0 h1:WXB+2gCoRhQiAr//IMHpIpoDsTrDgvjDORxt57e8XTA= -gopkg.in/src-d/go-git.v4 v4.7.0/go.mod h1:CzbUWqMn4pvmvndg3gnh5iZFmSsbhyhUWdI0IQ60AQo= +gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.7.1 h1:phAV/kNULxfYEvyInGdPuq3U2MtPpJdgmtOUF3cghkQ= +gopkg.in/src-d/go-git.v4 v4.7.1/go.mod h1:xrJH/YX8uSWewT6evfocf8qsivF18JgCN7/IMitOptY= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go index 9018808..777d8ae 100644 --- a/pkg/vcs/git.go +++ b/pkg/vcs/git.go @@ -9,6 +9,7 @@ import ( "io" "io/ioutil" "os" + "path" "path/filepath" "strings" "time" @@ -29,6 +30,7 @@ type gitVCS struct { log logger dir string module string + prefix string auth Auth } @@ -62,12 +64,16 @@ func (g *gitVCS) List(ctx context.Context) ([]Version, error) { list := []Version{} masterHash := "" + tagPrefix := "" + if g.prefix != "" { + tagPrefix = g.prefix + "/" + } for _, ref := range refs { name := ref.Name() if name == plumbing.Master { masterHash = ref.Hash().String() - } else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/v") { - list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/"))) + } else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/"+tagPrefix+"v") { + list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/"+tagPrefix))) } } @@ -122,31 +128,70 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error b := &bytes.Buffer{} zw := zip.NewWriter(b) + modules := map[string]bool{} + files := []*object.File{} tree.Files().ForEach(func(f *object.File) error { + dir, file := path.Split(f.Name) + if file == "go.mod" { + modules[dir] = true + } + files = append(files, f) + return nil + }) + prefix := g.prefix + if prefix != "" { + prefix = prefix + "/" + } + submodule := func(name string) bool { + for { + dir, _ := path.Split(name) + if len(dir) <= len(prefix) { + return false + } + if modules[dir] { + return true + } + name = dir[:len(dir)-1] + } + } + for _, f := range files { // go mod strips vendored directories from the zip, and we do the same // to match the checksums in the go.sum if isVendoredPackage(f.Name) { - return nil + continue } - w, err := zw.Create(filepath.Join(g.module+"@"+string(version), f.Name)) + if submodule(f.Name) { + continue + } + name := f.Name + if strings.HasPrefix(name, prefix) { + name = strings.TrimPrefix(name, prefix) + } else { + continue + } + w, err := zw.Create(filepath.Join(g.module+"@"+string(version), name)) if err != nil { - return err + return nil, err } r, err := f.Reader() if err != nil { - return err + return nil, err } defer r.Close() io.Copy(w, r) - return nil - }) + } zw.Close() return ioutil.NopCloser(bytes.NewBuffer(b.Bytes())), nil } func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) { + repoRoot, path, err := RepoRoot(ctx, g.module) + if err != nil { + return nil, err + } + g.prefix = path if g.dir != "" { - dir := filepath.Join(g.dir, g.module) + dir := filepath.Join(g.dir, repoRoot) if _, err := os.Stat(dir); os.IsNotExist(err) { os.MkdirAll(dir, 0755) repo, err = git.PlainInit(dir, true) @@ -163,10 +208,7 @@ func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) { if g.auth.Key != "" { schema = "ssh://" } - repoRoot := g.module - if meta, err := MetaImports(ctx, g.module); err == nil { - repoRoot = meta - } + g.log("repo", "url", schema+repoRoot+".git", "prefix", g.prefix) _, err = repo.CreateRemote(&config.RemoteConfig{ Name: remoteName, URLs: []string{schema + repoRoot + ".git"}, diff --git a/pkg/vcs/git_test.go b/pkg/vcs/git_test.go index fe9fe80..34e61ab 100644 --- a/pkg/vcs/git_test.go +++ b/pkg/vcs/git_test.go @@ -52,6 +52,20 @@ func TestGit(t *testing.T) { Timestamp: "2018-09-21", Checksum: "YIusKhLwlEKiuwowBFdEElpP0hIGDOCaSnQaMufLB00=", }, + { + // Repository that contains go.mod + Module: "bitbucket.org/gomodproxytest/nested", + Tag: "v1.0.0", + Timestamp: "2018-10-18", + Checksum: "rFPd/yPMmhjy4FdmLTySAHVcjHmZh/XP3xNrHKLiFss=", + }, + { + // Submodule that contains its own go.mod and tag + Module: "bitbucket.org/gomodproxytest/nested/child", + Tag: "v1.0.0", + Timestamp: "2018-10-18", + Checksum: "RN9U68h7BnmrsLP24VM37/Zeyb+21R8KmGZtxTUBd74=", + }, { // Just a frequently used module from github Module: "github.com/pkg/errors", diff --git a/pkg/vcs/meta.go b/pkg/vcs/meta.go index bb6fe71..2187e0e 100644 --- a/pkg/vcs/meta.go +++ b/pkg/vcs/meta.go @@ -13,15 +13,20 @@ var ( errMetaNotFound = errors.New("go-import meta tag not found") ) -// MetaImports resolved module import path for certain hosts using the special tag. -func MetaImports(ctx context.Context, module string) (string, error) { +func RepoRoot(ctx context.Context, module string) (root string, path string, err error) { + // For common VCS hosters we can figure out repo root by the URL if strings.HasPrefix(module, "github.com/") || strings.HasPrefix(module, "bitbucket.org/") { - return module, nil + parts := strings.Split(module, "/") + if len(parts) < 3 { + return "", "", errors.New("bad module name") + } + return strings.Join(parts[0:3], "/"), strings.Join(parts[3:], "/"), nil } + // Otherwise we shall make a `?go-get=1` HTTP request // TODO: use context res, err := http.Get("https://" + module + "?go-get=1") if err != nil { - return "", err + return "", "", err } defer res.Body.Close() html := struct { @@ -38,21 +43,19 @@ func MetaImports(ctx context.Context, module string) (string, error) { dec.AutoClose = xml.HTMLAutoClose dec.Entity = xml.HTMLEntity if err := dec.Decode(&html); err != nil { - return "", err + return "", "", err } for _, meta := range html.Head.Meta { if meta.Name == "go-import" { if f := strings.Fields(meta.Content); len(f) == 3 { - if f[0] != module { - return "", errPrefixDoesNotMatch - } url := f[2] if i := strings.Index(url, "://"); i >= 0 { url = url[i+3:] } - return url, nil + path = strings.TrimPrefix(strings.TrimPrefix(module, f[0]), "/") + return url, path, nil } } } - return "", errMetaNotFound + return "", "", errMetaNotFound } diff --git a/pkg/vcs/meta_test.go b/pkg/vcs/meta_test.go index 4e80b7d..3a57cba 100644 --- a/pkg/vcs/meta_test.go +++ b/pkg/vcs/meta_test.go @@ -10,7 +10,7 @@ import ( "testing" ) -func TestMetaImports(t *testing.T) { +func TestRepoRoot(t *testing.T) { var hostname string http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -31,39 +31,47 @@ func TestMetaImports(t *testing.T) { defer ts.Close() hostname = strings.TrimPrefix(ts.URL, "https://") - if url, err := MetaImports(context.Background(), hostname+"/foo/bar"); err != nil { + if root, path, err := RepoRoot(context.Background(), hostname+"/foo/bar"); err != nil { t.Fatal(err) - } else if url != "example.com/foo/bar" { - t.Fatal(url) + } else if root != "example.com/foo/bar" { + t.Fatal(root) + } else if path != "" { + t.Fatal(path) } } -func TestMetaImportsExternal(t *testing.T) { +func TestRepoRootExternal(t *testing.T) { if testing.Short() { t.Skip("testing with external VCS might be slow") } for _, test := range []struct { - Pkg string - URL string + Pkg string + Root string + Path string }{ // Common VCS should be resolved immediately without any checks - {Pkg: "github.com/user/repo", URL: "github.com/user/repo"}, - {Pkg: "bitbucket.org/user/repo", URL: "bitbucket.org/user/repo"}, + {Pkg: "github.com/user/repo", Root: "github.com/user/repo", Path: ""}, + {Pkg: "bitbucket.org/user/repo", Root: "bitbucket.org/user/repo", Path: ""}, + {Pkg: "github.com/user/repo/sub/dir", Root: "github.com/user/repo", Path: "sub/dir"}, + {Pkg: "bitbucket.org/user/repo/sub/dir", Root: "bitbucket.org/user/repo", Path: "sub/dir"}, // Otherwise, HTML meta tag should be checked - {Pkg: "golang.org/x/sys", URL: "go.googlesource.com/sys"}, - {Pkg: "gopkg.in/warnings.v0", URL: "gopkg.in/warnings.v0"}, - {Pkg: "gopkg.in/src-d/go-git.v4", URL: "gopkg.in/src-d/go-git.v4"}, + {Pkg: "golang.org/x/sys", Root: "go.googlesource.com/sys", Path: ""}, + {Pkg: "golang.org/x/sys/unix", Root: "go.googlesource.com/sys", Path: "unix"}, + {Pkg: "golang.org/x/net/websocket", Root: "go.googlesource.com/net", Path: "websocket"}, + {Pkg: "gopkg.in/warnings.v0", Root: "gopkg.in/warnings.v0", Path: ""}, + {Pkg: "gopkg.in/src-d/go-git.v4", Root: "gopkg.in/src-d/go-git.v4", Path: ""}, // On errors URL should be empty and error should be not nil - {Pkg: "google.com/foo", URL: ""}, - {Pkg: "golang.org/x/sys/unix", URL: ""}, - {Pkg: "example.com/foo", URL: ""}, - {Pkg: "foo/bar", URL: ""}, + {Pkg: "google.com/foo", Root: "", Path: ""}, + {Pkg: "example.com/foo", Root: "", Path: ""}, + {Pkg: "foo/bar", Root: "", Path: ""}, } { - url, err := MetaImports(context.Background(), test.Pkg) - if url != test.URL { - t.Fatal(test, url, err) + root, path, err := RepoRoot(context.Background(), test.Pkg) + if root != test.Root { + t.Fatal(test, root, err) + } else if path != test.Path { + t.Fatal(test, path, err) } - if url == "" && err == nil { + if root == "" && path == "" && err == nil { t.Fatal(test, "error should be set if module import can not be resolved") } }