Merge pull request #4 from sixt/feature/git-disk-storage

use disk storage for fetching git repos as an alternative to in-memory store

cmd/gomodproxy/main.go

@@ -58,7 +58,8 @@ func main() {
 	verbose := flag.Bool("v", false, "verbose logging")
 	debug := flag.Bool("debug", false, "enable debug HTTP API (pprof/expvar)")
 	json := flag.Bool("json", false, "json structured logging")
-	dir := flag.String("dir", filepath.Join(os.Getenv("HOME"), ".gomodproxy"), "cache directory")
+	dir := flag.String("dir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/cache"), "modules cache directory")
+	gitdir := flag.String("gitdir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/git"), "git cache directory")
 	memLimit := flag.Int64("mem", 256, "in-memory cache size in MB")
 	flag.Var(&gitPaths, "git", "list of git settings")
 
@@ -92,6 +93,7 @@ func main() {
 	}
 
 	options = append(options,
+		api.GitDir(*gitdir),
 		api.Memory(logger, *memLimit*1024*1024),
 		api.CacheDir(*dir),
 	)

pkg/api/api.go

@@ -22,6 +22,7 @@ type logger = func(v ...interface{})
 
 type api struct {
 	log      logger
+	gitdir   string
 	vcsPaths []vcsPath
 	stores   []store.Store
 }
@@ -54,6 +55,9 @@ func New(options ...Option) http.Handler {
 // testing.T.Log or any other custom logger.
 func Log(log logger) Option { return func(api *api) { api.log = log } }
 
+// GitDir configures API to use a specific directory for bare git repos.
+func GitDir(dir string) Option { return func(api *api) { api.gitdir = dir } }
+
 // Git configures API to use a specific git client when trying to download a
 // repository with the given prefix. Auth string can be a path to the SSK key,
 // or a colon-separated username:password string.
@@ -66,7 +70,7 @@ func Git(prefix string, auth string) Option {
 		api.vcsPaths = append(api.vcsPaths, vcsPath{
 			prefix: prefix,
 			vcs: func(module string) vcs.VCS {
-				return vcs.NewGit(api.log, module, a)
+				return vcs.NewGit(api.log, api.gitdir, module, a)
 			},
 		})
 	}
@@ -137,7 +141,7 @@ func (api *api) vcs(ctx context.Context, module string) vcs.VCS {
 			return path.vcs(module)
 		}
 	}
-	return vcs.NewGit(api.log, module, vcs.NoAuth())
+	return vcs.NewGit(api.log, api.gitdir, module, vcs.NoAuth())
 }
 
 func (api *api) module(ctx context.Context, module string, version vcs.Version) ([]byte, time.Time, error) {

pkg/vcs/git.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -26,14 +27,15 @@ const remoteName = "origin"
 
 type gitVCS struct {
 	log    logger
+	dir    string
 	module string
 	auth   Auth
 }
 
 // NewGit return a go-git VCS client implementation that provides information
 // about the specific module using the pgiven authentication mechanism.
-func NewGit(l logger, module string, auth Auth) VCS {
+func NewGit(l logger, dir string, module string, auth Auth) VCS {
-	return &gitVCS{log: l, module: module, auth: auth}
+	return &gitVCS{log: l, dir: dir, module: module, auth: auth}
 }
 
 func (g *gitVCS) List(ctx context.Context) ([]Version, error) {
@@ -142,8 +144,18 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error
 	return ioutil.NopCloser(bytes.NewBuffer(b.Bytes())), nil
 }
 
-func (g *gitVCS) repo(ctx context.Context) (*git.Repository, error) {
+func (g *gitVCS) repo(ctx context.Context) (repo *git.Repository, err error) {
-	repo, err := git.Init(memory.NewStorage(), nil)
+	if g.dir != "" {
+		dir := filepath.Join(g.dir, g.module)
+		if _, err := os.Stat(dir); os.IsNotExist(err) {
+			os.MkdirAll(dir, 0755)
+			repo, err = git.PlainInit(dir, true)
+		} else {
+			return git.PlainOpen(dir)
+		}
+	} else {
+		repo, err = git.Init(memory.NewStorage(), nil)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -175,7 +187,7 @@ func (g *gitVCS) commit(ctx context.Context, version Version) (*object.Commit, e
 		RemoteName: remoteName,
 		Auth:       auth,
 	})
-	if err != nil {
+	if err != nil && err != git.NoErrAlreadyUpToDate {
 		return nil, err
 	}
 

pkg/vcs/git_test.go

@@ -66,7 +66,7 @@ func TestGit(t *testing.T) {
 		auth := NoAuth()
 		if test.Tag != "" {
 			t.Run(test.Module+"/List", func(t *testing.T) {
-				git := NewGit(t.Log, test.Module, auth)
+				git := NewGit(t.Log, "", test.Module, auth)
 				list, err := git.List(context.Background())
 				if err != nil {
 					t.Fatal(err)
@@ -81,7 +81,7 @@ func TestGit(t *testing.T) {
 		}
 		if test.Timestamp != "" {
 			t.Run(test.Module+"/Timestamp", func(t *testing.T) {
-				git := NewGit(t.Log, test.Module, auth)
+				git := NewGit(t.Log, "", test.Module, auth)
 				timestamp, err := git.Timestamp(context.Background(), Version(test.Tag))
 				if err != nil {
 					t.Fatal(err)
@@ -93,7 +93,7 @@ func TestGit(t *testing.T) {
 		}
 		if test.Checksum != "" {
 			t.Run(test.Module+"/ZIP", func(t *testing.T) {
-				git := NewGit(t.Log, test.Module, auth)
+				git := NewGit(t.Log, "", test.Module, auth)
 				r, err := git.Zip(context.Background(), Version(test.Tag))
 				if err != nil {
 					t.Fatal(err)