initial commit

2018-09-19 11:20:09 +02:00
parent 95a937ef2c
commit ce767e10ee
11 changed files with 918 additions and 1 deletions
--- a/pkg/api/api.go
+++ b/pkg/api/api.go
@@ -0,0 +1,230 @@
+package api
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/sixt/gomodproxy/pkg/store"
+	"github.com/sixt/gomodproxy/pkg/vcs"
+)
+
+type logger = func(v ...interface{})
+
+type api struct {
+	log      logger
+	vcsPaths []vcsPath
+	stores   []store.Store
+}
+
+type vcsPath struct {
+	prefix string
+	vcs    func(module string) vcs.VCS
+}
+
+type Option func(*api)
+
+var (
+	apiList = regexp.MustCompile(`^/(?P<module>.*)/@v/list$`)
+	apiInfo = regexp.MustCompile(`^/(?P<module>.*)/@v/(?P<version>.*).info$`)
+	apiMod  = regexp.MustCompile(`^/(?P<module>.*)/@v/(?P<version>.*).mod$`)
+	apiZip  = regexp.MustCompile(`^/(?P<module>.*)/@v/(?P<version>.*).zip$`)
+)
+
+func New(options ...Option) http.Handler {
+	api := &api{log: func(...interface{}) {}}
+	for _, opt := range options {
+		opt(api)
+	}
+	return api
+}
+
+func Log(log logger) Option { return func(api *api) { api.log = log } }
+
+func Git(prefix string, auth string) Option {
+	a := vcs.Key(auth)
+	if creds := strings.SplitN(auth, ":", 2); len(creds) == 2 {
+		a = vcs.Password(creds[0], creds[1])
+	}
+	return func(api *api) {
+		api.vcsPaths = append(api.vcsPaths, vcsPath{
+			prefix: prefix,
+			vcs: func(module string) vcs.VCS {
+				return vcs.NewGit(api.log, module, a)
+			},
+		})
+	}
+}
+
+func Memory() Option {
+	return func(api *api) {
+		api.stores = append(api.stores, store.Memory())
+	}
+}
+
+func CacheDir(dir string) Option {
+	return func(api *api) {
+		api.stores = append(api.stores, store.Disk(dir))
+	}
+}
+
+func decodeBangs(s string) string {
+	buf := []rune{}
+	bang := false
+	for _, r := range []rune(s) {
+		if bang {
+			bang = false
+			buf = append(buf, unicode.ToUpper(r))
+			continue
+		}
+		if r == '!' {
+			bang = true
+			continue
+		}
+		buf = append(buf, r)
+	}
+	return string(buf)
+}
+
+func (api *api) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	now := time.Now()
+	defer func() { api.log("api.ServeHTTP", "method", r.Method, "url", r.URL, "time", time.Since(now)) }()
+
+	for _, route := range []struct {
+		regexp  *regexp.Regexp
+		handler func(w http.ResponseWriter, r *http.Request, module, version string)
+	}{
+		{apiList, api.list},
+		{apiInfo, api.info},
+		{apiMod, api.mod},
+		{apiZip, api.zip},
+	} {
+		if m := route.regexp.FindStringSubmatch(r.URL.Path); m != nil {
+			module, version := m[1], ""
+			if len(m) > 2 {
+				version = m[2]
+			}
+			module = decodeBangs(module)
+			route.handler(w, r, module, version)
+			return
+		}
+	}
+
+	http.NotFound(w, r)
+}
+
+func (api *api) vcs(ctx context.Context, module string) vcs.VCS {
+	for _, path := range api.vcsPaths {
+		if strings.HasPrefix(module, path.prefix) {
+			return path.vcs(module)
+		}
+	}
+	return vcs.NewGit(api.log, module, vcs.NoAuth())
+}
+
+func (api *api) module(ctx context.Context, module string, version vcs.Version) ([]byte, time.Time, error) {
+	for _, store := range api.stores {
+		if snapshot, err := store.Get(ctx, module, version); err == nil {
+			return snapshot.Data, snapshot.Timestamp, nil
+		}
+	}
+
+	timestamp, err := api.vcs(ctx, module).Timestamp(ctx, version)
+	if err != nil {
+		return nil, time.Time{}, err
+	}
+
+	b := &bytes.Buffer{}
+	zr, err := api.vcs(ctx, module).Zip(ctx, version)
+	if err != nil {
+		return nil, time.Time{}, err
+	}
+	defer zr.Close()
+
+	if _, err := io.Copy(b, zr); err != nil {
+		return nil, time.Time{}, err
+	}
+
+	for i := len(api.stores) - 1; i >= 0; i-- {
+		err := api.stores[i].Put(ctx, store.Snapshot{
+			Module:    module,
+			Version:   version,
+			Timestamp: timestamp,
+			Data:      b.Bytes(),
+		})
+		if err != nil {
+			api.log("api.module.Put", "module", module, "version", version, "error", err)
+		}
+	}
+
+	return b.Bytes(), timestamp, nil
+}
+
+func (api *api) list(w http.ResponseWriter, r *http.Request, module, version string) {
+	api.log("api.list", "module", module)
+	list, err := api.vcs(r.Context(), module).List(r.Context())
+	if err != nil {
+		api.log("api.list", "module", module, "error", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	for _, v := range list {
+		fmt.Fprintln(w, string(v))
+	}
+}
+
+func (api *api) info(w http.ResponseWriter, r *http.Request, module, version string) {
+	api.log("api.info", "module", module, "version", version)
+	_, t, err := api.module(r.Context(), module, vcs.Version(version))
+
+	if err != nil {
+		api.log("api.info", "module", module, "version", version, "error", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	json.NewEncoder(w).Encode(struct {
+		Version string
+		Time    time.Time
+	}{version, t})
+}
+
+func (api *api) mod(w http.ResponseWriter, r *http.Request, module, version string) {
+	api.log("api.mod", "module", module, "version", version)
+	b, _, err := api.module(r.Context(), module, vcs.Version(version))
+	if err == nil {
+		if zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b))); err == nil {
+			for _, f := range zr.File {
+				if f.Name == filepath.Join(module+"@"+string(version), "go.mod") {
+					if r, err := f.Open(); err == nil {
+						defer r.Close()
+						io.Copy(w, r)
+						return
+					}
+				}
+			}
+		}
+	}
+	w.Write([]byte(fmt.Sprintf("module %s\n", module)))
+}
+
+func (api *api) zip(w http.ResponseWriter, r *http.Request, module, version string) {
+	api.log("api.zip", "module", module, "version", version)
+	b, _, err := api.module(r.Context(), module, vcs.Version(version))
+	if err != nil {
+		api.log("api.zip", "module", module, "version", version, "error", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	io.Copy(w, bytes.NewReader(b))
+}
--- a/pkg/store/disk.go
+++ b/pkg/store/disk.go
@@ -0,0 +1,49 @@
+package store
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/sixt/gomodproxy/pkg/vcs"
+)
+
+type disk string
+
+func Disk(dir string) Store { return disk(dir) }
+
+func (d disk) Put(ctx context.Context, snapshot Snapshot) error {
+	dir := string(d)
+	timeFile := filepath.Join(dir, snapshot.Key()+".time")
+	zipFile := filepath.Join(dir, snapshot.Key()+".zip")
+
+	if err := os.MkdirAll(filepath.Dir(timeFile), 0755); err != nil {
+		return err
+	}
+
+	t, err := snapshot.Timestamp.MarshalText()
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(timeFile, t, 0644); err != nil {
+		return err
+	}
+	return ioutil.WriteFile(zipFile, snapshot.Data, 0644)
+}
+
+func (d disk) Get(ctx context.Context, module string, version vcs.Version) (Snapshot, error) {
+	dir := string(d)
+	s := Snapshot{Module: module, Version: version}
+	t, err := ioutil.ReadFile(filepath.Join(dir, s.Key()+".time"))
+	if err != nil {
+		return Snapshot{}, err
+	}
+	if err := s.Timestamp.UnmarshalText(t); err != nil {
+		return Snapshot{}, err
+	}
+	s.Data, err = ioutil.ReadFile(filepath.Join(dir, s.Key()+".zip"))
+	return s, err
+}
+
+func (d disk) Close() error { return nil }
--- a/pkg/store/mem.go
+++ b/pkg/store/mem.go
@@ -0,0 +1,41 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	"github.com/sixt/gomodproxy/pkg/vcs"
+)
+
+type memory struct {
+	sync.Mutex
+	cache []Snapshot
+}
+
+func Memory() Store { return &memory{} }
+
+func (m *memory) Put(ctx context.Context, snapshot Snapshot) error {
+	m.Lock()
+	defer m.Unlock()
+	for _, item := range m.cache {
+		if item.Module == snapshot.Module && item.Version == snapshot.Version {
+			return nil
+		}
+	}
+	m.cache = append(m.cache, snapshot)
+	return nil
+}
+
+func (m *memory) Get(ctx context.Context, module string, version vcs.Version) (Snapshot, error) {
+	m.Lock()
+	defer m.Unlock()
+	for _, snapshot := range m.cache {
+		if snapshot.Module == module && snapshot.Version == version {
+			return snapshot, nil
+		}
+	}
+	return Snapshot{}, errors.New("not found")
+}
+
+func (m *memory) Close() error { return nil }
--- a/pkg/store/store.go
+++ b/pkg/store/store.go
@@ -0,0 +1,25 @@
+package store
+
+import (
+	"context"
+	"time"
+
+	"github.com/sixt/gomodproxy/pkg/vcs"
+)
+
+type Store interface {
+	Put(ctx context.Context, snapshot Snapshot) error
+	Get(ctx context.Context, module string, version vcs.Version) (Snapshot, error)
+	Close() error
+}
+
+type Snapshot struct {
+	Module    string
+	Version   vcs.Version
+	Timestamp time.Time
+	Data      []byte
+}
+
+func (s Snapshot) Key() string {
+	return s.Module + "@" + string(s.Version)
+}
--- a/pkg/vcs/git.go
+++ b/pkg/vcs/git.go
@@ -0,0 +1,221 @@
+package vcs
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"gopkg.in/src-d/go-git.v4"
+	"gopkg.in/src-d/go-git.v4/config"
+	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/object"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
+)
+
+const remoteName = "origin"
+
+type gitVCS struct {
+	log    logger
+	module string
+	auth   Auth
+}
+
+func NewGit(l logger, module string, auth Auth) VCS {
+	return &gitVCS{log: l, module: module, auth: auth}
+}
+
+func (g *gitVCS) List(ctx context.Context) ([]Version, error) {
+	g.log("gitVCS.List", "module", g.module)
+	repo, err := g.repo(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	remote, err := repo.Remote(remoteName)
+	if err != nil {
+		return nil, err
+	}
+
+	auth, err := g.authMethod()
+	if err != nil {
+		return nil, err
+	}
+
+	refs, err := remote.List(&git.ListOptions{Auth: auth})
+	if err != nil {
+		return nil, err
+	}
+
+	list := []Version{}
+	masterHash := ""
+	for _, ref := range refs {
+		name := ref.Name()
+		if name == plumbing.Master {
+			masterHash = ref.Hash().String()
+		} else if name.IsTag() && strings.HasPrefix(name.String(), "refs/tags/v") {
+			list = append(list, Version(strings.TrimPrefix(name.String(), "refs/tags/")))
+		}
+	}
+
+	if len(list) == 0 {
+		if masterHash == "" {
+			return nil, errors.New("no tags and no master branch found")
+		}
+		short := masterHash[:12]
+		t, err := g.Timestamp(ctx, Version("v0.0.0-20060102150405-"+short))
+		if err != nil {
+			return nil, err
+		}
+		list = []Version{Version(fmt.Sprintf("v0.0.0-%s-%s", t.Format("20060102150405"), short))}
+	}
+
+	g.log("gitVCS.List", "module", g.module, "list", list)
+	return list, nil
+}
+
+func (g *gitVCS) Timestamp(ctx context.Context, version Version) (time.Time, error) {
+	g.log("gitVCS.Timestamp", "module", g.module, "version", version)
+	ci, err := g.commit(ctx, version)
+	if err != nil {
+		return time.Time{}, err
+	}
+	g.log("gitVCS.Timestamp", "module", g.module, "version", version, "timestamp", ci.Committer.When)
+	return ci.Committer.When, nil
+}
+
+func isVendoredPackage(name string) bool {
+	var i int
+	if strings.HasPrefix(name, "vendor/") {
+		i += len("vendor/")
+	} else if j := strings.Index(name, "/vendor/"); j >= 0 {
+		i += len("/vendor/")
+	} else {
+		return false
+	}
+	return strings.Contains(name[i:], "/")
+}
+
+func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error) {
+	g.log("gitVCS.Zip", "module", g.module, "version", version)
+	ci, err := g.commit(ctx, version)
+	if err != nil {
+		return nil, err
+	}
+	tree, err := ci.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	b := &bytes.Buffer{}
+	zw := zip.NewWriter(b)
+	tree.Files().ForEach(func(f *object.File) error {
+		// go mod strips vendored directories from the zip, and we do the same
+		// to match the checksums in the go.sum
+		if isVendoredPackage(f.Name) {
+			return nil
+		}
+		w, err := zw.Create(filepath.Join(g.module+"@"+string(version), f.Name))
+		if err != nil {
+			return err
+		}
+		r, err := f.Reader()
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		io.Copy(w, r)
+		return nil
+	})
+	zw.Close()
+	return ioutil.NopCloser(bytes.NewBuffer(b.Bytes())), nil
+}
+
+func (g *gitVCS) repo(ctx context.Context) (*git.Repository, error) {
+	repo, err := git.Init(memory.NewStorage(), nil)
+	if err != nil {
+		return nil, err
+	}
+	schema := "https://"
+	if g.auth.Key != "" {
+		schema = "ssh://"
+	}
+	repoRoot := g.module
+	if meta, err := MetaImports(ctx, g.module); err == nil {
+		repoRoot = meta
+	}
+	_, err = repo.CreateRemote(&config.RemoteConfig{
+		Name: remoteName,
+		URLs: []string{schema + repoRoot + ".git"},
+	})
+	return repo, err
+}
+
+func (g *gitVCS) commit(ctx context.Context, version Version) (*object.Commit, error) {
+	repo, err := g.repo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	auth, err := g.authMethod()
+	if err != nil {
+		return nil, err
+	}
+	err = repo.FetchContext(ctx, &git.FetchOptions{
+		RemoteName: remoteName,
+		Auth:       auth,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	version = Version(strings.TrimSuffix(string(version), "+incompatible"))
+	hash := version.Hash()
+	if version.IsSemVer() {
+		tags, err := repo.Tags()
+		if err != nil {
+			return nil, err
+		}
+		tags.ForEach(func(t *plumbing.Reference) error {
+			if t.Name().String() == "refs/tags/"+string(version) {
+				hash = t.Hash().String()
+				annotated, err := repo.TagObject(t.Hash())
+				if err == nil {
+					hash = annotated.Target.String()
+				}
+			}
+			return nil
+		})
+	} else {
+		commits, err := repo.CommitObjects()
+		if err != nil {
+			return nil, err
+		}
+		commits.ForEach(func(ci *object.Commit) error {
+			if strings.HasPrefix(ci.Hash.String(), version.Hash()) {
+				hash = ci.Hash.String()
+			}
+			return nil
+		})
+	}
+
+	g.log("gitVCS.commit", "module", g.module, "version", version, "hash", hash)
+	return repo.CommitObject(plumbing.NewHash(hash))
+}
+
+func (g *gitVCS) authMethod() (transport.AuthMethod, error) {
+	if g.auth.Key != "" {
+		return ssh.NewPublicKeysFromFile("git", g.auth.Key, "")
+	} else if g.auth.Username != "" {
+		return &http.BasicAuth{Username: g.auth.Username, Password: g.auth.Password}, nil
+	}
+	return nil, nil
+}
--- a/pkg/vcs/vcs.go
+++ b/pkg/vcs/vcs.go
@@ -0,0 +1,90 @@
+package vcs
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+)
+
+type logger = func(v ...interface{})
+
+type Version string
+
+var reSemVer = regexp.MustCompile(`^v\d+\.\d+\.\d+$`)
+
+func (v Version) IsSemVer() bool { return reSemVer.MatchString(string(v)) }
+func (v Version) Hash() string {
+	fields := strings.Split(string(v), "-")
+	if len(fields) != 3 {
+		return ""
+	}
+	return fields[2]
+}
+
+type Module interface {
+	Timestamp(ctx context.Context, version Version) (time.Time, error)
+	Zip(ctx context.Context, version Version) (io.ReadCloser, error)
+}
+
+type VCS interface {
+	List(ctx context.Context) ([]Version, error)
+	Module
+}
+
+type Auth struct {
+	Username string
+	Password string
+	Key      string
+}
+
+func NoAuth() Auth                            { return Auth{} }
+func Password(username, password string) Auth { return Auth{Username: username, Password: password} }
+func Key(key string) Auth                     { return Auth{Key: key} }
+
+func MetaImports(ctx context.Context, module string) (string, error) {
+	if strings.HasPrefix(module, "github.com/") || strings.HasPrefix(module, "bitbucket.org/") {
+		return module, nil
+	}
+	// TODO: use context
+	res, err := http.Get("https://" + module + "?go-get=1")
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	html := struct {
+		HTML string `xml:"html"`
+		Head struct {
+			Meta []struct {
+				Name    string `xml:"name,attr"`
+				Content string `xml:"content,attr"`
+			} `xml:"meta"`
+		} `xml:"head"`
+	}{}
+	dec := xml.NewDecoder(res.Body)
+	dec.Strict = false
+	dec.AutoClose = xml.HTMLAutoClose
+	dec.Entity = xml.HTMLEntity
+	if err := dec.Decode(&html); err != nil {
+		return "", err
+	}
+	for _, meta := range html.Head.Meta {
+		if meta.Name == "go-import" {
+			if f := strings.Fields(meta.Content); len(f) == 3 {
+				if f[0] != module {
+					return "", errors.New("prefix does not match the module")
+				}
+				url := f[2]
+				if i := strings.Index(url, "://"); i >= 0 {
+					url = url[i+3:]
+				}
+				return url, nil
+			}
+		}
+	}
+	return "", errors.New("go-import meta tag not found")
+}