Feature/nested modules (#6)
* add support for nested go modules * update dependencies
This commit is contained in:
Serge Zaitsev
GitHub
parent
e892000edb
commit
254fe284ff
12
go.mod
12
go.mod
@ -2,17 +2,7 @@ module github.com/sixt/gomodproxy
|
||||
|
||||
require (
|
||||
github.com/emirpasic/gods v1.11.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e // indirect
|
||||
github.com/mitchellh/go-homedir v1.0.0 // indirect
|
||||
github.com/pelletier/go-buffruneio v0.2.0 // indirect
|
||||
github.com/sergi/go-diff v1.0.0 // indirect
|
||||
github.com/src-d/gcfg v1.3.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.2.0 // indirect
|
||||
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b // indirect
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 // indirect
|
||||
golang.org/x/text v0.3.0 // indirect
|
||||
gopkg.in/src-d/go-billy.v4 v4.2.1 // indirect
|
||||
gopkg.in/src-d/go-git.v4 v4.7.0
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/src-d/go-git.v4 v4.7.1
|
||||
)
|
||||
|
||||
18
go.sum
18
go.sum
@ -1,9 +1,15 @@
|
||||
github.com/emirpasic/gods v1.9.0 h1:rUF4PuzEjMChMiNsVjdI+SyLu7rEqpQ5reNFnhC7oFo=
|
||||
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/emirpasic/gods v1.11.0 h1:8KhjokrJy1+REZkLeSlnJvLKI4tRR8g65a4C07oQQ50=
|
||||
github.com/emirpasic/gods v1.11.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
|
||||
github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@ -13,14 +19,19 @@ github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnG
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
|
||||
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg=
|
||||
github.com/src-d/gcfg v1.3.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
|
||||
github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b h1:2b9XGzhjiYsYPnKXoEfL7klWZQIt8IfyRCz62gCqqlQ=
|
||||
golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
|
||||
golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -29,7 +40,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo=
|
||||
gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
|
||||
gopkg.in/src-d/go-git.v4 v4.7.0 h1:WXB+2gCoRhQiAr//IMHpIpoDsTrDgvjDORxt57e8XTA=
|
||||
gopkg.in/src-d/go-git.v4 v4.7.0/go.mod h1:CzbUWqMn4pvmvndg3gnh5iZFmSsbhyhUWdI0IQ60AQo=
|
||||
gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
|
||||
gopkg.in/src-d/go-git.v4 v4.7.1 h1:phAV/kNULxfYEvyInGdPuq3U2MtPpJdgmtOUF3cghkQ=
|
||||
gopkg.in/src-d/go-git.v4 v4.7.1/go.mod h1:xrJH/YX8uSWewT6evfocf8qsivF18JgCN7/IMitOptY=
|
||||
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
|
||||
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
@ -29,6 +30,7 @@ type gitVCS struct {
|
||||
log logger
|
||||
dir string
|
||||
module string
|
||||
prefix string
|
||||
auth Auth
|
||||
}
|
||||
|
||||
@ -62,12 +64,16 @@ func (g *gitVCS) List(ctx context.Context) ([]Version, error) {
|
||||
|
||||
list := []Version{}
|
||||
masterHash := ""
|
||||
tagPrefix := ""
|
||||
if g.prefix != "" {
|
||||
tagPrefix = g.prefix + "/"
|
||||
}
|
||||
for _, ref := range refs {
|
||||
name := ref.Name()
|
||||
if name == plumbing.Master {
|
||||
masterHash = ref.Hash().String()
|
||||
} else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/v") {
|
||||
list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/")))
|
||||
} else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/"+tagPrefix+"v") {
|
||||
list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/"+tagPrefix)))
|
||||
}
|
||||
}
|
||||
|
||||
@ -122,31 +128,70 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error
|
||||
|
||||
b := &bytes.Buffer{}
|
||||
zw := zip.NewWriter(b)
|
||||
modules := map[string]bool{}
|
||||
files := []*object.File{}
|
||||
tree.Files().ForEach(func(f *object.File) error {
|
||||
dir, file := path.Split(f.Name)
|
||||
if file == "go.mod" {
|
||||
modules[dir] = true
|
||||
}
|
||||
files = append(files, f)
|
||||
return nil
|
||||
})
|
||||
prefix := g.prefix
|
||||
if prefix != "" {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
submodule := func(name string) bool {
|
||||
for {
|
||||
dir, _ := path.Split(name)
|
||||
if len(dir) <= len(prefix) {
|
||||
return false
|
||||
}
|
||||
if modules[dir] {
|
||||
return true
|
||||
}
|
||||
name = dir[:len(dir)-1]
|
||||
}
|
||||
}
|
||||
for _, f := range files {
|
||||
// go mod strips vendored directories from the zip, and we do the same
|
||||
// to match the checksums in the go.sum
|
||||
if isVendoredPackage(f.Name) {
|
||||
return nil
|
||||
continue
|
||||
}
|
||||
w, err := zw.Create(filepath.Join(g.module+"@"+string(version), f.Name))
|
||||
if submodule(f.Name) {
|
||||
continue
|
||||
}
|
||||
name := f.Name
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
name = strings.TrimPrefix(name, prefix)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
w, err := zw.Create(filepath.Join(g.module+"@"+string(version), name))
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
r, err := f.Reader()
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
io.Copy(w, r)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
zw.Close()
|
||||
return ioutil.NopCloser(bytes.NewBuffer(b.Bytes())), nil
|
||||
}
|
||||
|
||||
func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) {
|
||||
repoRoot, path, err := RepoRoot(ctx, g.module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g.prefix = path
|
||||
if g.dir != "" {
|
||||
dir := filepath.Join(g.dir, g.module)
|
||||
dir := filepath.Join(g.dir, repoRoot)
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
os.MkdirAll(dir, 0755)
|
||||
repo, err = git.PlainInit(dir, true)
|
||||
@ -163,10 +208,7 @@ func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) {
|
||||
if g.auth.Key != "" {
|
||||
schema = "ssh://"
|
||||
}
|
||||
repoRoot := g.module
|
||||
if meta, err := MetaImports(ctx, g.module); err == nil {
|
||||
repoRoot = meta
|
||||
}
|
||||
g.log("repo", "url", schema+repoRoot+".git", "prefix", g.prefix)
|
||||
_, err = repo.CreateRemote(&config.RemoteConfig{
|
||||
Name: remoteName,
|
||||
URLs: []string{schema + repoRoot + ".git"},
|
||||
|
||||
@ -52,6 +52,20 @@ func TestGit(t *testing.T) {
|
||||
Timestamp: "2018-09-21",
|
||||
Checksum: "YIusKhLwlEKiuwowBFdEElpP0hIGDOCaSnQaMufLB00=",
|
||||
},
|
||||
{
|
||||
// Repository that contains go.mod
|
||||
Module: "bitbucket.org/gomodproxytest/nested",
|
||||
Tag: "v1.0.0",
|
||||
Timestamp: "2018-10-18",
|
||||
Checksum: "rFPd/yPMmhjy4FdmLTySAHVcjHmZh/XP3xNrHKLiFss=",
|
||||
},
|
||||
{
|
||||
// Submodule that contains its own go.mod and tag
|
||||
Module: "bitbucket.org/gomodproxytest/nested/child",
|
||||
Tag: "v1.0.0",
|
||||
Timestamp: "2018-10-18",
|
||||
Checksum: "RN9U68h7BnmrsLP24VM37/Zeyb+21R8KmGZtxTUBd74=",
|
||||
},
|
||||
{
|
||||
// Just a frequently used module from github
|
||||
Module: "github.com/pkg/errors",
|
||||
|
||||
@ -13,15 +13,20 @@ var (
|
||||
errMetaNotFound = errors.New("go-import meta tag not found")
|
||||
)
|
||||
|
||||
// MetaImports resolved module import path for certain hosts using the special <meta> tag.
|
||||
func MetaImports(ctx context.Context, module string) (string, error) {
|
||||
func RepoRoot(ctx context.Context, module string) (root string, path string, err error) {
|
||||
// For common VCS hosters we can figure out repo root by the URL
|
||||
if strings.HasPrefix(module, "github.com/") || strings.HasPrefix(module, "bitbucket.org/") {
|
||||
return module, nil
|
||||
parts := strings.Split(module, "/")
|
||||
if len(parts) < 3 {
|
||||
return "", "", errors.New("bad module name")
|
||||
}
|
||||
return strings.Join(parts[0:3], "/"), strings.Join(parts[3:], "/"), nil
|
||||
}
|
||||
// Otherwise we shall make a `?go-get=1` HTTP request
|
||||
// TODO: use context
|
||||
res, err := http.Get("https://" + module + "?go-get=1")
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", "", err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
html := struct {
|
||||
@ -38,21 +43,19 @@ func MetaImports(ctx context.Context, module string) (string, error) {
|
||||
dec.AutoClose = xml.HTMLAutoClose
|
||||
dec.Entity = xml.HTMLEntity
|
||||
if err := dec.Decode(&html); err != nil {
|
||||
return "", err
|
||||
return "", "", err
|
||||
}
|
||||
for _, meta := range html.Head.Meta {
|
||||
if meta.Name == "go-import" {
|
||||
if f := strings.Fields(meta.Content); len(f) == 3 {
|
||||
if f[0] != module {
|
||||
return "", errPrefixDoesNotMatch
|
||||
}
|
||||
url := f[2]
|
||||
if i := strings.Index(url, "://"); i >= 0 {
|
||||
url = url[i+3:]
|
||||
}
|
||||
return url, nil
|
||||
path = strings.TrimPrefix(strings.TrimPrefix(module, f[0]), "/")
|
||||
return url, path, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", errMetaNotFound
|
||||
return "", "", errMetaNotFound
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMetaImports(t *testing.T) {
|
||||
func TestRepoRoot(t *testing.T) {
|
||||
var hostname string
|
||||
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
||||
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
@ -31,39 +31,47 @@ func TestMetaImports(t *testing.T) {
|
||||
defer ts.Close()
|
||||
hostname = strings.TrimPrefix(ts.URL, "https://")
|
||||
|
||||
if url, err := MetaImports(context.Background(), hostname+"/foo/bar"); err != nil {
|
||||
if root, path, err := RepoRoot(context.Background(), hostname+"/foo/bar"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if url != "example.com/foo/bar" {
|
||||
t.Fatal(url)
|
||||
} else if root != "example.com/foo/bar" {
|
||||
t.Fatal(root)
|
||||
} else if path != "" {
|
||||
t.Fatal(path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetaImportsExternal(t *testing.T) {
|
||||
func TestRepoRootExternal(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("testing with external VCS might be slow")
|
||||
}
|
||||
for _, test := range []struct {
|
||||
Pkg string
|
||||
URL string
|
||||
Pkg string
|
||||
Root string
|
||||
Path string
|
||||
}{
|
||||
// Common VCS should be resolved immediately without any checks
|
||||
{Pkg: "github.com/user/repo", URL: "github.com/user/repo"},
|
||||
{Pkg: "bitbucket.org/user/repo", URL: "bitbucket.org/user/repo"},
|
||||
{Pkg: "github.com/user/repo", Root: "github.com/user/repo", Path: ""},
|
||||
{Pkg: "bitbucket.org/user/repo", Root: "bitbucket.org/user/repo", Path: ""},
|
||||
{Pkg: "github.com/user/repo/sub/dir", Root: "github.com/user/repo", Path: "sub/dir"},
|
||||
{Pkg: "bitbucket.org/user/repo/sub/dir", Root: "bitbucket.org/user/repo", Path: "sub/dir"},
|
||||
// Otherwise, HTML meta tag should be checked
|
||||
{Pkg: "golang.org/x/sys", URL: "go.googlesource.com/sys"},
|
||||
{Pkg: "gopkg.in/warnings.v0", URL: "gopkg.in/warnings.v0"},
|
||||
{Pkg: "gopkg.in/src-d/go-git.v4", URL: "gopkg.in/src-d/go-git.v4"},
|
||||
{Pkg: "golang.org/x/sys", Root: "go.googlesource.com/sys", Path: ""},
|
||||
{Pkg: "golang.org/x/sys/unix", Root: "go.googlesource.com/sys", Path: "unix"},
|
||||
{Pkg: "golang.org/x/net/websocket", Root: "go.googlesource.com/net", Path: "websocket"},
|
||||
{Pkg: "gopkg.in/warnings.v0", Root: "gopkg.in/warnings.v0", Path: ""},
|
||||
{Pkg: "gopkg.in/src-d/go-git.v4", Root: "gopkg.in/src-d/go-git.v4", Path: ""},
|
||||
// On errors URL should be empty and error should be not nil
|
||||
{Pkg: "google.com/foo", URL: ""},
|
||||
{Pkg: "golang.org/x/sys/unix", URL: ""},
|
||||
{Pkg: "example.com/foo", URL: ""},
|
||||
{Pkg: "foo/bar", URL: ""},
|
||||
{Pkg: "google.com/foo", Root: "", Path: ""},
|
||||
{Pkg: "example.com/foo", Root: "", Path: ""},
|
||||
{Pkg: "foo/bar", Root: "", Path: ""},
|
||||
} {
|
||||
url, err := MetaImports(context.Background(), test.Pkg)
|
||||
if url != test.URL {
|
||||
t.Fatal(test, url, err)
|
||||
root, path, err := RepoRoot(context.Background(), test.Pkg)
|
||||
if root != test.Root {
|
||||
t.Fatal(test, root, err)
|
||||
} else if path != test.Path {
|
||||
t.Fatal(test, path, err)
|
||||
}
|
||||
if url == "" && err == nil {
|
||||
if root == "" && path == "" && err == nil {
|
||||
t.Fatal(test, "error should be set if module import can not be resolved")
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user