diff --git a/cmd/gomodproxy/main.go b/cmd/gomodproxy/main.go index 46cf1fc..f3cd2a7 100644 --- a/cmd/gomodproxy/main.go +++ b/cmd/gomodproxy/main.go @@ -106,6 +106,7 @@ func main() { dir := flag.String("dir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/cache"), "modules cache directory") gitdir := flag.String("gitdir", filepath.Join(os.Getenv("HOME"), ".gomodproxy/git"), "git cache directory") memLimit := flag.Int64("mem", 256, "in-memory cache size in MB") + workers := flag.Int("workers", 1, "number of parallel VCS workers") flag.Var(&gitPaths, "git", "list of git settings") flag.Parse() @@ -138,6 +139,7 @@ func main() { } options = append(options, + api.VCSWorkers(*workers), api.GitDir(*gitdir), api.Memory(logger, *memLimit*1024*1024), api.CacheDir(*dir), diff --git a/pkg/api/api.go b/pkg/api/api.go index 1bd7153..b533cd8 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -26,6 +26,7 @@ type api struct { gitdir string vcsPaths []vcsPath stores []store.Store + semc chan struct{} } type vcsPath struct { @@ -53,7 +54,7 @@ var ( // New returns a configured http.Handler which implements GOPROXY API. func New(options ...Option) http.Handler { - api := &api{log: func(...interface{}) {}} + api := &api{log: func(...interface{}) {}, semc: make(chan struct{}, 1)} for _, opt := range options { opt(api) } @@ -99,6 +100,15 @@ func CacheDir(dir string) Option { } } +// VCSWorkers configures API to use at most n parallel workers when fetching +// from the VCS. The reason to restrict number of workers is to limit their +// memory usage. +func VCSWorkers(n int) Option { + return func(api *api) { + api.semc = make(chan struct{}, n) + } +} + func decodeBangs(s string) string { buf := []rune{} bang := false @@ -170,6 +180,10 @@ func (api *api) module(ctx context.Context, module string, version vcs.Version) } cacheMisses.Add(module, 1) + // wait for semaphore + api.semc <- struct{}{} + defer func() { <-api.semc }() + timestamp, err := api.vcs(ctx, module).Timestamp(ctx, version) if err != nil { return nil, time.Time{}, err