stale/gomodproxy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature/nested modules (#6)

Browse Source 
* add support for nested go modules

* update dependencies
This commit is contained in:
Serge Zaitsev
2018-10-22 14:38:22 +02:00
committed by GitHub
parent e892000edb
commit 254fe284ff
6 changed files with 126 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
pkg/vcs/git.go
View File

@@ -9,6 +9,7 @@ import (
"io"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"time"
@@ -29,6 +30,7 @@ type gitVCS struct {
log logger
dir string
module string
prefix string
auth Auth
}
@@ -62,12 +64,16 @@ func (g *gitVCS) List(ctx context.Context) ([]Version, error) {
list := []Version{}
masterHash := ""
tagPrefix := ""
if g.prefix != "" {
tagPrefix = g.prefix + "/"
}
for _, ref := range refs {
name := ref.Name()
if name == plumbing.Master {
masterHash = ref.Hash().String()
} else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/v") {
list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/")))
} else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/"+tagPrefix+"v") {
list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/"+tagPrefix)))
}
}
@@ -122,31 +128,70 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error
b := &bytes.Buffer{}
zw := zip.NewWriter(b)
modules := map[string]bool{}
files := []*object.File{}
tree.Files().ForEach(func(f *object.File) error {
dir, file := path.Split(f.Name)
if file == "go.mod" {
modules[dir] = true
}
files = append(files, f)
return nil
})
prefix := g.prefix
if prefix != "" {
prefix = prefix + "/"
}
submodule := func(name string) bool {
for {
dir, _ := path.Split(name)
if len(dir) <= len(prefix) {
return false
}
if modules[dir] {
return true
}
name = dir[:len(dir)-1]
}
}
for _, f := range files {
// go mod strips vendored directories from the zip, and we do the same
// to match the checksums in the go.sum
if isVendoredPackage(f.Name) {
return nil
continue
}
w, err := zw.Create(filepath.Join(g.module+"@"+string(version), f.Name))
if submodule(f.Name) {
continue
}
name := f.Name
if strings.HasPrefix(name, prefix) {
name = strings.TrimPrefix(name, prefix)
} else {
continue
}
w, err := zw.Create(filepath.Join(g.module+"@"+string(version), name))
if err != nil {
return err
return nil, err
}
r, err := f.Reader()
if err != nil {
return err
return nil, err
}
defer r.Close()
io.Copy(w, r)
return nil
})
}
zw.Close()
return ioutil.NopCloser(bytes.NewBuffer(b.Bytes())), nil
}
func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) {
repoRoot, path, err := RepoRoot(ctx, g.module)
if err != nil {
return nil, err
}
g.prefix = path
if g.dir != "" {
dir := filepath.Join(g.dir, g.module)
dir := filepath.Join(g.dir, repoRoot)
if _, err := os.Stat(dir); os.IsNotExist(err) {
os.MkdirAll(dir, 0755)
repo, err = git.PlainInit(dir, true)
@@ -163,10 +208,7 @@ func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) {
if g.auth.Key != "" {
schema = "ssh://"
}
repoRoot := g.module
if meta, err := MetaImports(ctx, g.module); err == nil {
repoRoot = meta
}
g.log("repo", "url", schema+repoRoot+".git", "prefix", g.prefix)
_, err = repo.CreateRemote(&config.RemoteConfig{
Name: remoteName,
URLs: []string{schema + repoRoot + ".git"},

14
pkg/vcs/git_test.go
View File

@@ -52,6 +52,20 @@ func TestGit(t *testing.T) {
Timestamp: "2018-09-21",
Checksum: "YIusKhLwlEKiuwowBFdEElpP0hIGDOCaSnQaMufLB00=",
},
{
// Repository that contains go.mod
Module: "bitbucket.org/gomodproxytest/nested",
Tag: "v1.0.0",
Timestamp: "2018-10-18",
Checksum: "rFPd/yPMmhjy4FdmLTySAHVcjHmZh/XP3xNrHKLiFss=",
},
{
// Submodule that contains its own go.mod and tag
Module: "bitbucket.org/gomodproxytest/nested/child",
Tag: "v1.0.0",
Timestamp: "2018-10-18",
Checksum: "RN9U68h7BnmrsLP24VM37/Zeyb+21R8KmGZtxTUBd74=",
},
{
// Just a frequently used module from github
Module: "github.com/pkg/errors",

23
pkg/vcs/meta.go
View File

@@ -13,15 +13,20 @@ var (
errMetaNotFound = errors.New("go-import meta tag not found")
)
// MetaImports resolved module import path for certain hosts using the special <meta> tag.
func MetaImports(ctx context.Context, module string) (string, error) {
func RepoRoot(ctx context.Context, module string) (root string, path string, err error) {
// For common VCS hosters we can figure out repo root by the URL
if strings.HasPrefix(module, "github.com/") || strings.HasPrefix(module, "bitbucket.org/") {
return module, nil
parts := strings.Split(module, "/")
if len(parts) < 3 {
return "", "", errors.New("bad module name")
}
return strings.Join(parts[0:3], "/"), strings.Join(parts[3:], "/"), nil
}
// Otherwise we shall make a `?go-get=1` HTTP request
// TODO: use context
res, err := http.Get("https://" + module + "?go-get=1")
if err != nil {
return "", err
return "", "", err
}
defer res.Body.Close()
html := struct {
@@ -38,21 +43,19 @@ func MetaImports(ctx context.Context, module string) (string, error) {
dec.AutoClose = xml.HTMLAutoClose
dec.Entity = xml.HTMLEntity
if err := dec.Decode(&html); err != nil {
return "", err
return "", "", err
}
for _, meta := range html.Head.Meta {
if meta.Name == "go-import" {
if f := strings.Fields(meta.Content); len(f) == 3 {
if f[0] != module {
return "", errPrefixDoesNotMatch
}
url := f[2]
if i := strings.Index(url, "://"); i >= 0 {
url = url[i+3:]
}
return url, nil
path = strings.TrimPrefix(strings.TrimPrefix(module, f[0]), "/")
return url, path, nil
}
}
}
return "", errMetaNotFound
return "", "", errMetaNotFound
}

48
pkg/vcs/meta_test.go
View File

@@ -10,7 +10,7 @@ import (
"testing"
)
func TestMetaImports(t *testing.T) {
func TestRepoRoot(t *testing.T) {
var hostname string
http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -31,39 +31,47 @@ func TestMetaImports(t *testing.T) {
defer ts.Close()
hostname = strings.TrimPrefix(ts.URL, "https://")
if url, err := MetaImports(context.Background(), hostname+"/foo/bar"); err != nil {
if root, path, err := RepoRoot(context.Background(), hostname+"/foo/bar"); err != nil {
t.Fatal(err)
} else if url != "example.com/foo/bar" {
t.Fatal(url)
} else if root != "example.com/foo/bar" {
t.Fatal(root)
} else if path != "" {
t.Fatal(path)
}
}
func TestMetaImportsExternal(t *testing.T) {
func TestRepoRootExternal(t *testing.T) {
if testing.Short() {
t.Skip("testing with external VCS might be slow")
}
for _, test := range []struct {
Pkg string
URL string
Pkg string
Root string
Path string
}{
// Common VCS should be resolved immediately without any checks
{Pkg: "github.com/user/repo", URL: "github.com/user/repo"},
{Pkg: "bitbucket.org/user/repo", URL: "bitbucket.org/user/repo"},
{Pkg: "github.com/user/repo", Root: "github.com/user/repo", Path: ""},
{Pkg: "bitbucket.org/user/repo", Root: "bitbucket.org/user/repo", Path: ""},
{Pkg: "github.com/user/repo/sub/dir", Root: "github.com/user/repo", Path: "sub/dir"},
{Pkg: "bitbucket.org/user/repo/sub/dir", Root: "bitbucket.org/user/repo", Path: "sub/dir"},
// Otherwise, HTML meta tag should be checked
{Pkg: "golang.org/x/sys", URL: "go.googlesource.com/sys"},
{Pkg: "gopkg.in/warnings.v0", URL: "gopkg.in/warnings.v0"},
{Pkg: "gopkg.in/src-d/go-git.v4", URL: "gopkg.in/src-d/go-git.v4"},
{Pkg: "golang.org/x/sys", Root: "go.googlesource.com/sys", Path: ""},
{Pkg: "golang.org/x/sys/unix", Root: "go.googlesource.com/sys", Path: "unix"},
{Pkg: "golang.org/x/net/websocket", Root: "go.googlesource.com/net", Path: "websocket"},
{Pkg: "gopkg.in/warnings.v0", Root: "gopkg.in/warnings.v0", Path: ""},
{Pkg: "gopkg.in/src-d/go-git.v4", Root: "gopkg.in/src-d/go-git.v4", Path: ""},
// On errors URL should be empty and error should be not nil
{Pkg: "google.com/foo", URL: ""},
{Pkg: "golang.org/x/sys/unix", URL: ""},
{Pkg: "example.com/foo", URL: ""},
{Pkg: "foo/bar", URL: ""},
{Pkg: "google.com/foo", Root: "", Path: ""},
{Pkg: "example.com/foo", Root: "", Path: ""},
{Pkg: "foo/bar", Root: "", Path: ""},
} {
url, err := MetaImports(context.Background(), test.Pkg)
if url != test.URL {
t.Fatal(test, url, err)
root, path, err := RepoRoot(context.Background(), test.Pkg)
if root != test.Root {
t.Fatal(test, root, err)
} else if path != test.Path {
t.Fatal(test, path, err)
}
if url == "" && err == nil {
if root == "" && path == "" && err == nil {
t.Fatal(test, "error should be set if module import can not be resolved")
}
}