ignore symlinks to match go1.11.4 checksum algorithm changes

cmd/gomodproxy/main.go

@@ -97,6 +97,7 @@ func (f *listFlag) Set(s string) error { *f = append(*f, s); return nil }
 
 func main() {
 	gitPaths := listFlag{}
+	vcsPaths := listFlag{}
 
 	addr := flag.String("addr", ":0", "http server address")
 	verbose := flag.Bool("v", false, "verbose logging")
@@ -106,7 +107,9 @@ func main() {
 	dir := flag.String("dir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/cache"), "modules cache directory")
 	gitdir := flag.String("gitdir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/git"), "git cache directory")
 	memLimit := flag.Int64("mem", 256, "in-memory cache size in MB")
+	workers := flag.Int("workers", 1, "number of parallel VCS workers")
 	flag.Var(&gitPaths, "git", "list of git settings")
+	flag.Var(&vcsPaths, "vcs", "list of custom VCS handlers")
 
 	flag.Parse()
 
@@ -129,6 +132,14 @@ func main() {
 	}
 	options = append(options, api.Log(logger))
 
+	for _, path := range vcsPaths {
+		kv := strings.SplitN(path, ":", 2)
+		if len(kv) != 2 {
+			log.Fatal("bad VCS syntax:", path)
+		}
+		options = append(options, api.CustomVCS(kv[0], kv[1]))
+	}
+
 	for _, path := range gitPaths {
 		kv := strings.SplitN(path, ":", 2)
 		if len(kv) != 2 {
@@ -138,6 +149,7 @@ func main() {
 	}
 
 	options = append(options,
+		api.VCSWorkers(*workers),
 		api.GitDir(*gitdir),
 		api.Memory(logger, *memLimit*1024*1024),
 		api.CacheDir(*dir),

go.mod

@@ -1,8 +1,13 @@
 module github.com/sixt/gomodproxy
 
 require (
-	github.com/emirpasic/gods v1.11.0 // indirect
-	golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b // indirect
-	golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 // indirect
-	gopkg.in/src-d/go-git.v4 v4.7.1
+	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/stretchr/testify v1.3.0 // indirect
+	golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc // indirect
+	golang.org/x/net v0.0.0-20190110200230-915654e7eabc // indirect
+	golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb // indirect
+	gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect
+	gopkg.in/src-d/go-git-fixtures.v3 v3.3.0 // indirect
+	gopkg.in/src-d/go-git.v4 v4.8.1
 )

go.sum

@@ -1,47 +1,68 @@
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/emirpasic/gods v1.11.0 h1:8KhjokrJy1+REZkLeSlnJvLKI4tRR8g65a4C07oQQ50=
-github.com/emirpasic/gods v1.11.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
 github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg=
-github.com/src-d/gcfg v1.3.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
+github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
 github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b h1:2b9XGzhjiYsYPnKXoEfL7klWZQIt8IfyRCz62gCqqlQ=
-golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M=
+golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc h1:Yx9JGxI1SBhVLFjpAkWMaO1TF+xyqtHLjZpvQboJGiM=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb h1:1w588/yEchbPNpa9sEvOcMZYbWHedwJjg4VOAdDHWHk=
+golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo=
 gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
+gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek=
+gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
 gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
-gopkg.in/src-d/go-git.v4 v4.7.1 h1:phAV/kNULxfYEvyInGdPuq3U2MtPpJdgmtOUF3cghkQ=
-gopkg.in/src-d/go-git.v4 v4.7.1/go.mod h1:xrJH/YX8uSWewT6evfocf8qsivF18JgCN7/IMitOptY=
+gopkg.in/src-d/go-git-fixtures.v3 v3.3.0 h1:AxUOwLW3at53ysFqs0Lg+H+8KSQXl7AEHBvWj8wEsT8=
+gopkg.in/src-d/go-git-fixtures.v3 v3.3.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
+gopkg.in/src-d/go-git.v4 v4.8.1 h1:aAyBmkdE1QUUEHcP4YFCGKmsMQRAuRmUcPEQR7lOAa0=
+gopkg.in/src-d/go-git.v4 v4.8.1/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=

pkg/api/api.go

@@ -26,6 +26,7 @@ type api struct {
 	gitdir   string
 	vcsPaths []vcsPath
 	stores   []store.Store
+	semc     chan struct{}
 }
 
 type vcsPath struct {
@@ -53,7 +54,7 @@ var (
 
 // New returns a configured http.Handler which implements GOPROXY API.
 func New(options ...Option) http.Handler {
-	api := &api{log: func(...interface{}) {}}
+	api := &api{log: func(...interface{}) {}, semc: make(chan struct{}, 1)}
 	for _, opt := range options {
 		opt(api)
 	}
@@ -85,6 +86,17 @@ func Git(prefix string, auth string) Option {
 	}
 }
 
+func CustomVCS(prefix string, cmd string) Option {
+	return func(api *api) {
+		api.vcsPaths = append(api.vcsPaths, vcsPath{
+			prefix: prefix,
+			vcs: func(module string) vcs.VCS {
+				return vcs.NewCommand(api.log, cmd, module)
+			},
+		})
+	}
+}
+
 // Memory configures API to use in-memory cache for downloaded modules.
 func Memory(log logger, limit int64) Option {
 	return func(api *api) {
@@ -99,6 +111,15 @@ func CacheDir(dir string) Option {
 	}
 }
 
+// VCSWorkers configures API to use at most n parallel workers when fetching
+// from the VCS. The reason to restrict number of workers is to limit their
+// memory usage.
+func VCSWorkers(n int) Option {
+	return func(api *api) {
+		api.semc = make(chan struct{}, n)
+	}
+}
+
 func decodeBangs(s string) string {
 	buf := []rune{}
 	bang := false
@@ -170,6 +191,10 @@ func (api *api) module(ctx context.Context, module string, version vcs.Version)
 	}
 	cacheMisses.Add(module, 1)
 
+	// wait for semaphore
+	api.semc <- struct{}{}
+	defer func() { <-api.semc }()
+
 	timestamp, err := api.vcs(ctx, module).Timestamp(ctx, version)
 	if err != nil {
 		return nil, time.Time{}, err

pkg/vcs/cmd.go

@@ -0,0 +1,75 @@
+package vcs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type cmdVCS struct {
+	log    logger
+	module string
+	cmd    string
+}
+
+func NewCommand(l logger, cmd string, module string) VCS {
+	return &cmdVCS{log: l, cmd: cmd, module: module}
+}
+
+func (c *cmdVCS) List(ctx context.Context) ([]Version, error) {
+	b, err := c.exec(ctx, "MODULE="+c.module, "ACTION=list", "VERSION=latest", "FILEPATH="+c.module+"/@v/list")
+	if err != nil {
+		return nil, err
+	}
+	versions := []Version{}
+	for _, line := range strings.Split(string(b), "\n") {
+		versions = append(versions, Version(line))
+	}
+	return versions, nil
+}
+
+func (c *cmdVCS) Timestamp(ctx context.Context, version Version) (time.Time, error) {
+	b, err := c.exec(ctx, "MODULE="+c.module, "ACTION=timestamp", "VERSION="+version.String(),
+		"FILEPATH="+c.module+"/@v/"+version.String()+".info")
+	if err != nil {
+		return time.Time{}, err
+	}
+	info := struct {
+		Version string
+		Time    time.Time
+	}{}
+	if json.Unmarshal(b, &info) == nil {
+		return info.Time, nil
+	}
+	if t, err := time.Parse(time.RFC3339, string(b)); err == nil {
+		return t, nil
+	}
+	if sec, err := strconv.ParseInt(string(b), 10, 64); err == nil {
+		return time.Unix(sec, 0), nil
+	}
+	return time.Time{}, errors.New("unknown time format")
+}
+
+func (c *cmdVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error) {
+	b, err := c.exec(ctx, "MODULE="+c.module, "ACTION=zip", "VERSION="+version.String(),
+		"FILEPATH="+c.module+"/@v/"+version.String()+".zip")
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.NopCloser(bytes.NewReader(b)), nil
+}
+
+func (c *cmdVCS) exec(ctx context.Context, env ...string) ([]byte, error) {
+	cmd := exec.Command("sh", "-c", c.cmd)
+	cmd.Env = append(os.Environ(), env...)
+	cmd.Stderr = os.Stderr
+	return cmd.Output()
+}

pkg/vcs/git.go

@@ -163,6 +163,9 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error
 		if submodule(f.Name) {
 			continue
 		}
+		if !f.Mode.IsRegular() {
+			continue
+		}
 		name := f.Name
 		if strings.HasPrefix(name, prefix) {
 			name = strings.TrimPrefix(name, prefix)

pkg/vcs/git_test.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"io/ioutil"
 	"sort"
+	"strings"
 	"testing"
 )
 
@@ -73,12 +74,19 @@ func TestGit(t *testing.T) {
 			Timestamp: "2016-09-29",
 			Checksum:  "WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=",
 		},
+		{
+			// A module with symlinks, should match Go 1.11.4 algorithm fix
+			Module:    "github.com/hashicorp/go-rootcerts",
+			Tag:       "v0.0.0-20160503143440-6bb64b370b90",
+			Timestamp: "2016-05-03",
+			Checksum:  "VBj0QYQ0u2MCJzBfeYXGexnAl17GsH1yidnoxCqqD9E=",
+		},
 	} {
 		if test.Module == "" {
 			continue
 		}
 		auth := NoAuth()
-		if test.Tag != "" {
+		if test.Tag != "" && !strings.HasPrefix(test.Tag, "v0.0.0-") {
 			t.Run(test.Module+"/List", func(t *testing.T) {
 				git := NewGit(t.Log, "", test.Module, auth)
 				list, err := git.List(context.Background())

pkg/vcs/vcs.go

@@ -27,6 +27,11 @@ func (v Version) Hash() string {
 	return fields[2]
 }
 
+// String returns a string representation of a version
+func (v Version) String() string {
+	return string(v)
+}
+
 // Module is a source code snapshot for which one can get the commit timestamp
 // or the actual ZIP with the source code in it.
 type Module interface {