encode module name with gomod vcs provider to handle uppercase module paths

* add module invalidation via DELETE request
* add go mod download vcs provider

.travis.yml

@@ -12,7 +12,17 @@ script:
   - go test -v ./...
 
 after_success:
-  - printf 'FROM scratch\nADD gomodproxy /\nCMD ["/gomodproxy"]' > Dockerfile
+  - mkdir gomods
+  - printf 'FROM scratch\nADD gomodproxy /\nADD gomods /\nCMD ["/gomodproxy"]' > Dockerfile
+  - docker build -t "sixtlabs/gomodproxy-slim:latest" .
+  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+  - docker push "sixtlabs/gomodproxy-slim:latest"
+  - if [ ! -z $TRAVIS_TAG ] ; then
+      TAG=$(echo $TRAVIS_TAG | sed 's/^v//');
+      docker tag sixtlabs/gomodproxy-slim:latest sixtlabs/gomodproxy-slim:$TAG;
+      docker push sixtlabs/gomodproxy-slim:$TAG;
+    fi
+  - printf 'FROM golang\nADD gomodproxy /\nADD gomods /\nCMD ["/gomodproxy"]' > Dockerfile
   - docker build -t "sixtlabs/gomodproxy:latest" .
   - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
   - docker push "sixtlabs/gomodproxy:latest"

cmd/gomodproxy/main.go

@@ -97,6 +97,7 @@ func (f *listFlag) Set(s string) error { *f = append(*f, s); return nil }
 
 func main() {
 	gitPaths := listFlag{}
+	vcsPaths := listFlag{}
 
 	addr := flag.String("addr", ":0", "http server address")
 	verbose := flag.Bool("v", false, "verbose logging")
@@ -106,7 +107,9 @@ func main() {
 	dir := flag.String("dir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/cache"), "modules cache directory")
 	gitdir := flag.String("gitdir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/git"), "git cache directory")
 	memLimit := flag.Int64("mem", 256, "in-memory cache size in MB")
+	workers := flag.Int("workers", 1, "number of parallel VCS workers")
 	flag.Var(&gitPaths, "git", "list of git settings")
+	flag.Var(&vcsPaths, "vcs", "list of custom VCS handlers")
 
 	flag.Parse()
 
@@ -137,7 +140,16 @@ func main() {
 		options = append(options, api.Git(kv[0], kv[1]))
 	}
 
+	for _, path := range vcsPaths {
+		kv := strings.SplitN(path, ":", 2)
+		if len(kv) != 2 {
+			log.Fatal("bad VCS syntax:", path)
+		}
+		options = append(options, api.CustomVCS(kv[0], kv[1]))
+	}
+
 	options = append(options,
+		api.VCSWorkers(*workers),
 		api.GitDir(*gitdir),
 		api.Memory(logger, *memLimit*1024*1024),
 		api.CacheDir(*dir),

go.mod

@@ -1,8 +1,5 @@
 module github.com/sixt/gomodproxy
 
-require (
+go 1.13
-	github.com/emirpasic/gods v1.11.0 // indirect
+
-	golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b // indirect
+require gopkg.in/src-d/go-git.v4 v4.13.1
-	golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 // indirect
-	gopkg.in/src-d/go-git.v4 v4.7.1
-)

go.sum

@@ -1,47 +1,60 @@
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
-github.com/emirpasic/gods v1.11.0 h1:8KhjokrJy1+REZkLeSlnJvLKI4tRR8g65a4C07oQQ50=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/emirpasic/gods v1.11.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
-github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/src-d/gcfg v1.3.0 h1:2BEDr8r0I0b8h/fOqwtxCEiq2HJu8n2JGZJQFGXWLjg=
+github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
-github.com/src-d/gcfg v1.3.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
-golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b h1:2b9XGzhjiYsYPnKXoEfL7klWZQIt8IfyRCz62gCqqlQ=
+github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
-golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/src-d/go-billy.v4 v4.2.1 h1:omN5CrMrMcQ+4I8bJ0wEhOBPanIRWzFC953IiXKdYzo=
+gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
-gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
+gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
-gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
+gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
-gopkg.in/src-d/go-git.v4 v4.7.1 h1:phAV/kNULxfYEvyInGdPuq3U2MtPpJdgmtOUF3cghkQ=
+gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
-gopkg.in/src-d/go-git.v4 v4.7.1/go.mod h1:xrJH/YX8uSWewT6evfocf8qsivF18JgCN7/IMitOptY=
+gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=

pkg/api/api.go

@@ -26,6 +26,7 @@ type api struct {
 	gitdir   string
 	vcsPaths []vcsPath
 	stores   []store.Store
+	semc     chan struct{}
 }
 
 type vcsPath struct {
@@ -53,7 +54,7 @@ var (
 
 // New returns a configured http.Handler which implements GOPROXY API.
 func New(options ...Option) http.Handler {
-	api := &api{log: func(...interface{}) {}}
+	api := &api{log: func(...interface{}) {}, semc: make(chan struct{}, 1)}
 	for _, opt := range options {
 		opt(api)
 	}
@@ -85,6 +86,17 @@ func Git(prefix string, auth string) Option {
 	}
 }
 
+func CustomVCS(prefix string, cmd string) Option {
+	return func(api *api) {
+		api.vcsPaths = append(api.vcsPaths, vcsPath{
+			prefix: prefix,
+			vcs: func(module string) vcs.VCS {
+				return vcs.NewCommand(api.log, cmd, module)
+			},
+		})
+	}
+}
+
 // Memory configures API to use in-memory cache for downloaded modules.
 func Memory(log logger, limit int64) Option {
 	return func(api *api) {
@@ -99,6 +111,15 @@ func CacheDir(dir string) Option {
 	}
 }
 
+// VCSWorkers configures API to use at most n parallel workers when fetching
+// from the VCS. The reason to restrict number of workers is to limit their
+// memory usage.
+func VCSWorkers(n int) Option {
+	return func(api *api) {
+		api.semc = make(chan struct{}, n)
+	}
+}
+
 func decodeBangs(s string) string {
 	buf := []rune{}
 	bang := false
@@ -137,6 +158,10 @@ func (api *api) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				version = m[2]
 			}
 			module = decodeBangs(module)
+			if r.Method == http.MethodDelete && version != "" {
+				api.delete(w, r, module, version)
+				return
+			}
 			httpRequests.Add(route.id, 1)
 			defer func() {
 				v := &expvar.Float{}
@@ -158,7 +183,7 @@ func (api *api) vcs(ctx context.Context, module string) vcs.VCS {
 			return path.vcs(module)
 		}
 	}
-	return vcs.NewGit(api.log, api.gitdir, module, vcs.NoAuth())
+	return vcs.NewGoMod(api.log, module)
 }
 
 func (api *api) module(ctx context.Context, module string, version vcs.Version) ([]byte, time.Time, error) {
@@ -170,6 +195,10 @@ func (api *api) module(ctx context.Context, module string, version vcs.Version)
 	}
 	cacheMisses.Add(module, 1)
 
+	// wait for semaphore
+	api.semc <- struct{}{}
+	defer func() { <-api.semc }()
+
 	timestamp, err := api.vcs(ctx, module).Timestamp(ctx, version)
 	if err != nil {
 		return nil, time.Time{}, err
@@ -207,7 +236,7 @@ func (api *api) list(w http.ResponseWriter, r *http.Request, module, version str
 	if err != nil {
 		api.log("api.list", "module", module, "error", err)
 		httpErrors.Add(module, 1)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+		http.Error(w, err.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -223,7 +252,7 @@ func (api *api) info(w http.ResponseWriter, r *http.Request, module, version str
 	if err != nil {
 		api.log("api.info", "module", module, "version", version, "error", err)
 		httpErrors.Add(module, 1)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+		http.Error(w, err.Error(), http.StatusNotFound)
 		return
 	}
 
@@ -258,8 +287,17 @@ func (api *api) zip(w http.ResponseWriter, r *http.Request, module, version stri
 	if err != nil {
 		api.log("api.zip", "module", module, "version", version, "error", err)
 		httpErrors.Add(module, 1)
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+		http.Error(w, err.Error(), http.StatusNotFound)
 		return
 	}
 	io.Copy(w, bytes.NewReader(b))
 }
+
+func (api *api) delete(w http.ResponseWriter, r *http.Request, module, version string) {
+	for _, store := range api.stores {
+		if err := store.Del(r.Context(), module, vcs.Version(version)); err != nil {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+	}
+}

pkg/store/disk.go

@@ -47,4 +47,15 @@ func (d disk) Get(ctx context.Context, module string, version vcs.Version) (Snap
 	return s, err
 }
 
+func (d disk) Del(ctx context.Context, module string, version vcs.Version) error {
+	dir := string(d)
+	s := Snapshot{Module: module, Version: version}
+	err := os.Remove(filepath.Join(dir, s.Key()+".time"))
+	if err != nil {
+		return err
+	}
+	err = os.Remove(filepath.Join(dir, s.Key()+".zip"))
+	return err
+}
+
 func (d disk) Close() error { return nil }

pkg/store/mem.go

@@ -45,17 +45,41 @@ func (m *memory) Put(ctx context.Context, snapshot Snapshot) error {
 func (m *memory) Get(ctx context.Context, module string, version vcs.Version) (Snapshot, error) {
 	m.Lock()
 	defer m.Unlock()
-	return m.lookup(module, version)
+	item, err := m.lookup(module, version)
+	if err != nil {
+		return Snapshot{}, err
+	}
+	return item.Snapshot, nil
 }
 
-func (m *memory) lookup(module string, version vcs.Version) (Snapshot, error) {
+func (m *memory) Del(ctx context.Context, module string, version vcs.Version) error {
+	m.Lock()
+	defer m.Unlock()
+	item, err := m.lookup(module, version)
+	if err != nil {
+		return err
+	}
+	if item.prev == nil {
+		m.head = item.next
+	} else {
+		item.prev.next = item.next
+	}
+	if item.next == nil {
+		m.tail = item.prev
+	} else {
+		item.next.prev = item.prev
+	}
+	return nil
+}
+
+func (m *memory) lookup(module string, version vcs.Version) (*lruItem, error) {
 	for item := m.head; item != nil; item = item.next {
 		if item.Module == module && item.Version == version {
 			m.update(item)
-			return item.Snapshot, nil
+			return item, nil
 		}
 	}
-	return Snapshot{}, errors.New("not found")
+	return nil, errors.New("not found")
 }
 
 func (m *memory) insert(item *lruItem) {

pkg/store/store.go

@@ -14,6 +14,7 @@ type logger = func(...interface{})
 type Store interface {
 	Put(ctx context.Context, snapshot Snapshot) error
 	Get(ctx context.Context, module string, version vcs.Version) (Snapshot, error)
+	Del(ctx context.Context, module string, version vcs.Version) error
 	Close() error
 }
 

pkg/vcs/cmd.go

@@ -0,0 +1,107 @@
+package vcs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type cmdVCS struct {
+	log           logger
+	module        string
+	moduleEncoded string
+	cmd           string
+}
+
+func encodeBangs(s string) string {
+	buf := []byte{}
+	for _, r := range s {
+		if 'A' <= r && r <= 'Z' {
+			buf = append(buf, '!', byte(r+'a'-'A'))
+		} else {
+			buf = append(buf, byte(r))
+		}
+	}
+	return string(buf)
+}
+
+func NewCommand(l logger, cmd string, module string) VCS {
+	return &cmdVCS{log: l, cmd: cmd, module: module, moduleEncoded: encodeBangs(module)}
+}
+
+func (c *cmdVCS) List(ctx context.Context) ([]Version, error) {
+	b, err := c.exec(ctx,
+		"MODULE="+c.module,
+		"MODULE_ENCODED="+c.moduleEncoded,
+		"ACTION=list",
+		"VERSION=latest",
+		"FILEPATH="+c.module+"/@v/list",
+		"FILEPATH_ENCODED="+c.moduleEncoded+"/@v/list",
+	)
+	if err != nil {
+		return nil, err
+	}
+	versions := []Version{}
+	for _, line := range strings.Split(string(b), "\n") {
+		versions = append(versions, Version(line))
+	}
+	return versions, nil
+}
+
+func (c *cmdVCS) Timestamp(ctx context.Context, version Version) (time.Time, error) {
+	b, err := c.exec(ctx,
+		"MODULE="+c.module,
+		"MODULE_ENCODED="+c.moduleEncoded,
+		"ACTION=timestamp",
+		"VERSION="+version.String(),
+		"FILEPATH="+c.module+"/@v/"+version.String()+".info",
+		"FILEPATH_ENCODED="+c.moduleEncoded+"/@v/"+version.String()+".info",
+	)
+	if err != nil {
+		return time.Time{}, err
+	}
+	info := struct {
+		Version string
+		Time    time.Time
+	}{}
+	if json.Unmarshal(b, &info) == nil {
+		return info.Time, nil
+	}
+	if t, err := time.Parse(time.RFC3339, string(b)); err == nil {
+		return t, nil
+	}
+	if sec, err := strconv.ParseInt(string(b), 10, 64); err == nil {
+		return time.Unix(sec, 0), nil
+	}
+	return time.Time{}, errors.New("unknown time format")
+}
+
+func (c *cmdVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error) {
+	b, err := c.exec(ctx,
+		"MODULE="+c.module,
+		"MODULE_ENCODED="+c.moduleEncoded,
+		"ACTION=zip",
+		"VERSION="+version.String(),
+		"FILEPATH="+c.module+"/@v/"+version.String()+".zip",
+		"FILEPATH_ENCODED="+c.moduleEncoded+"/@v/"+version.String()+".zip",
+	)
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.NopCloser(bytes.NewReader(b)), nil
+}
+
+func (c *cmdVCS) exec(ctx context.Context, env ...string) ([]byte, error) {
+	cmd := exec.Command("sh", "-c", c.cmd)
+	cmd.Env = append(os.Environ(), env...)
+	cmd.Stderr = os.Stderr
+	return cmd.Output()
+}

pkg/vcs/git.go

@@ -163,6 +163,13 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error
 		if submodule(f.Name) {
 			continue
 		}
+		mode, err := f.Mode.ToOSFileMode()
+		if err != nil {
+			return nil, err
+		}
+		if !mode.IsRegular() {
+			continue
+		}
 		name := f.Name
 		if strings.HasPrefix(name, prefix) {
 			name = strings.TrimPrefix(name, prefix)
@@ -228,6 +235,7 @@ func (g *gitVCS) commit(ctx context.Context, version Version) (*object.Commit, e
 	err = repo.FetchContext(ctx, &git.FetchOptions{
 		RemoteName: remoteName,
 		Auth:       auth,
+		Tags:       git.AllTags,
 	})
 	if err != nil && err != git.NoErrAlreadyUpToDate {
 		return nil, err

pkg/vcs/git_test.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"io/ioutil"
 	"sort"
+	"strings"
 	"testing"
 )
 
@@ -73,12 +74,19 @@ func TestGit(t *testing.T) {
 			Timestamp: "2016-09-29",
 			Checksum:  "WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=",
 		},
+		{
+			// A module with symlinks, should match Go 1.11.4 algorithm fix
+			Module:    "github.com/hashicorp/go-rootcerts",
+			Tag:       "v0.0.0-20160503143440-6bb64b370b90",
+			Timestamp: "2016-05-03",
+			Checksum:  "VBj0QYQ0u2MCJzBfeYXGexnAl17GsH1yidnoxCqqD9E=",
+		},
 	} {
 		if test.Module == "" {
 			continue
 		}
 		auth := NoAuth()
-		if test.Tag != "" {
+		if test.Tag != "" && !strings.HasPrefix(test.Tag, "v0.0.0-") {
 			t.Run(test.Module+"/List", func(t *testing.T) {
 				git := NewGit(t.Log, "", test.Module, auth)
 				list, err := git.List(context.Background())

pkg/vcs/gomod.go

@@ -0,0 +1,87 @@
+package vcs
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type goVCS struct {
+	dir    string
+	log    logger
+	module string
+}
+
+func NewGoMod(l logger, module string) VCS {
+	return &goVCS{log: l, module: module, dir: "/tmp/_go"}
+}
+
+func (g *goVCS) List(ctx context.Context) ([]Version, error) {
+	if err := g.download(ctx, "latest"); err != nil {
+		return nil, err
+	}
+	b, err := g.file("list")
+	if err != nil {
+		return nil, err
+	}
+	versions := []Version{}
+	for _, line := range strings.Split(string(b), "\n") {
+		versions = append(versions, Version(line))
+	}
+	return versions, nil
+}
+
+func (g *goVCS) Timestamp(ctx context.Context, version Version) (time.Time, error) {
+	if err := g.download(ctx, version.String()); err != nil {
+		return time.Time{}, err
+	}
+	b, err := g.file(version.String() + ".info")
+	if err != nil {
+		return time.Time{}, err
+	}
+	info := struct {
+		Version string
+		Time    time.Time
+	}{}
+	if json.Unmarshal(b, &info) == nil {
+		return info.Time, nil
+	}
+	if t, err := time.Parse(time.RFC3339, string(b)); err == nil {
+		return t, nil
+	}
+	if sec, err := strconv.ParseInt(string(b), 10, 64); err == nil {
+		return time.Unix(sec, 0), nil
+	}
+	return time.Time{}, nil
+}
+
+func (g *goVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error) {
+	if err := g.download(ctx, version.String()); err != nil {
+		return nil, err
+	}
+	b, err := g.file(version.String() + ".zip")
+	if err != nil {
+		return nil, err
+	}
+	return ioutil.NopCloser(bytes.NewReader(b)), nil
+}
+
+func (g *goVCS) download(ctx context.Context, version string) error {
+	cmd := exec.Command("go", "mod", "download", g.module+"@"+version)
+	cmd.Env = append(os.Environ(), "GOPATH="+g.dir)
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func (g *goVCS) file(name string) ([]byte, error) {
+	path := filepath.Join(g.dir, "pkg", "mod", "cache", "download", encodeBangs(g.module), "@v", name)
+	return ioutil.ReadFile(path)
+}

pkg/vcs/vcs.go

@@ -27,6 +27,11 @@ func (v Version) Hash() string {
 	return fields[2]
 }
 
+// String returns a string representation of a version
+func (v Version) String() string {
+	return string(v)
+}
+
 // Module is a source code snapshot for which one can get the commit timestamp
 // or the actual ZIP with the source code in it.
 type Module interface {