From e08d2382436dbd16c144277059767f6fed5859da Mon Sep 17 00:00:00 2001 From: Marcin Bilski Date: Wed, 22 Dec 2021 14:04:57 +0100 Subject: [PATCH] Add support for ephemeral tags. --- cmd/gomodproxy/main.go | 2 +- go.sum | 15 +++++ pkg/api/api.go | 55 +++++++++++++++++ pkg/vcs/git.go | 7 ++- pkg/vcs/tags.go | 130 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 pkg/vcs/tags.go diff --git a/cmd/gomodproxy/main.go b/cmd/gomodproxy/main.go index cab1b0f..6444f09 100644 --- a/cmd/gomodproxy/main.go +++ b/cmd/gomodproxy/main.go @@ -137,7 +137,7 @@ func main() { if len(kv) != 2 { log.Fatal("bad git path:", path) } - options = append(options, api.Git(kv[0], kv[1])) + options = append(options, api.GitWithEphemeralTags(kv[0], kv[1])) } for _, path := range vcsPaths { diff --git a/go.sum b/go.sum index 3cfdd61..1b95e08 100644 --- a/go.sum +++ b/go.sum @@ -1,27 +1,38 @@ +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -29,6 +40,7 @@ github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= @@ -47,12 +59,15 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e h1:D5TXcfTk7xF7hvieo4QErS3qqCB4teTffacDWr7CI+0= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= diff --git a/pkg/api/api.go b/pkg/api/api.go index 75c68b1..09c97e2 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -27,6 +27,8 @@ type api struct { vcsPaths []vcsPath stores []store.Store semc chan struct{} + + ephemeralTagStorage *vcs.EphemeralTagStorage } type vcsPath struct { @@ -42,6 +44,7 @@ var ( apiInfo = regexp.MustCompile(`^/(?P.*)/@v/(?P.*).info$`) apiMod = regexp.MustCompile(`^/(?P.*)/@v/(?P.*).mod$`) apiZip = regexp.MustCompile(`^/(?P.*)/@v/(?P.*).zip$`) + apiTag = regexp.MustCompile(`^/tags/(?P.*)/@v/(?P.*)$`) ) var ( @@ -86,6 +89,28 @@ func Git(prefix string, auth string) Option { } } +// GitWithEphemeralTags configures API to use a specific git client when trying +// to download a repository with the given prefix. Auth string can be a path to +// the SSK key, or a colon-separated username:password string. +func GitWithEphemeralTags(prefix string, auth string) Option { + + storage := vcs.NewEphemeralTagStorage() + + a := vcs.Key(auth) + if creds := strings.SplitN(auth, ":", 2); len(creds) == 2 { + a = vcs.Password(creds[0], creds[1]) + } + return func(api *api) { + api.ephemeralTagStorage = storage + api.vcsPaths = append(api.vcsPaths, vcsPath{ + prefix: prefix, + vcs: func(module string) vcs.VCS { + return vcs.NewGitWithEphemeralTags(api.log, api.gitdir, module, a, storage) + }, + }) + } +} + func CustomVCS(prefix string, cmd string) Option { return func(api *api) { api.vcsPaths = append(api.vcsPaths, vcsPath{ @@ -151,6 +176,7 @@ func (api *api) ServeHTTP(w http.ResponseWriter, r *http.Request) { {"info", apiInfo, api.info}, {"api", apiMod, api.mod}, {"zip", apiZip, api.zip}, + {"tag", apiTag, api.tag}, } { if m := route.regexp.FindStringSubmatch(r.URL.Path); m != nil { module, version := m[1], "" @@ -301,3 +327,32 @@ func (api *api) delete(w http.ResponseWriter, r *http.Request, module, version s } } } + +func (api *api) tag(w http.ResponseWriter, r *http.Request, module, version string) { + api.log("api.tag", "module", module, "version", version) + + taggable, ok := api.vcs(r.Context(), module).(vcs.Taggable) + if !ok { + err := fmt.Errorf("repository for module %v is not taggable", module) + api.log("api.tag", "module", module, "version", version, "error", err) + httpErrors.Add(module, 1) + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + defer r.Body.Close() + + req := struct { + Short string `json:"short"` + }{} + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + api.log("api.tag", "module", module, "version", version, "error", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + taggable.Tag(vcs.Version(version), req.Short) + + // TODO(bilus): Response +} diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go index 75644ae..42941d3 100644 --- a/pkg/vcs/git.go +++ b/pkg/vcs/git.go @@ -143,6 +143,11 @@ func isVendoredPackage(name string) bool { } func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error) { + dirName := g.module + "@" + string(version) + return g.zipUnder(ctx, version, dirName) +} + +func (g *gitVCS) zipUnder(ctx context.Context, version Version, dirName string) (io.ReadCloser, error) { g.log("gitVCS.Zip", "module", g.module, "version", version) ci, err := g.commit(ctx, version) if err != nil { @@ -203,7 +208,7 @@ func (g *gitVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error } else { continue } - w, err := zw.Create(filepath.Join(g.module+"@"+string(version), name)) + w, err := zw.Create(filepath.Join(dirName, name)) if err != nil { return nil, err } diff --git a/pkg/vcs/tags.go b/pkg/vcs/tags.go new file mode 100644 index 0000000..f198686 --- /dev/null +++ b/pkg/vcs/tags.go @@ -0,0 +1,130 @@ +package vcs + +import ( + "context" + "fmt" + "io" + "time" +) + +type ephemeralTag struct { + semVer Version + short string +} + +type EphemeralTagStorage struct { + tagsByModule map[moduleName][]ephemeralTag +} + +func NewEphemeralTagStorage() *EphemeralTagStorage { + return &EphemeralTagStorage{ + tagsByModule: make(map[moduleName][]ephemeralTag), + } +} + +func (s *EphemeralTagStorage) Tag(module string, semVer Version, short string) { + tags := s.tagsByModule[module] + tmp := tags[:0] + for _, t := range tags { + if t.semVer != semVer { + tmp = append(tmp, t) + } + } + s.tagsByModule[module] = append(tmp, ephemeralTag{semVer, short}) +} + +func (s *EphemeralTagStorage) tags(module string) []ephemeralTag { + return s.tagsByModule[module] +} + +type moduleName = string + +type taggableVCS struct { + wrapped *gitVCS + module string + storage *EphemeralTagStorage +} + +type Taggable interface { + Tag(semVer Version, short string) +} + +// NewGitWithEphemeralTags return a go-git VCS client implementation that +// provides information about the specific module using the given +// authentication mechanism while adding support to ephemeral tags. +func NewGitWithEphemeralTags(l logger, dir string, module string, auth Auth, storage *EphemeralTagStorage) VCS { + git := NewGit(l, dir, module, auth).(*gitVCS) + return &taggableVCS{ + wrapped: git, + module: module, + storage: storage, + } +} + +func (v *taggableVCS) Tag(semVer Version, short string) { + v.storage.Tag(v.module, semVer, short) +} + +func (v *taggableVCS) List(ctx context.Context) ([]Version, error) { + remoteVersions, err := v.wrapped.List(ctx) + if err != nil { + return nil, err + } + + tags := v.storage.tags(v.module) + // Remote versions win. + return appendEphemeralVersion(remoteVersions, tags...), nil +} + +func appendEphemeralVersion(versions []Version, tags ...ephemeralTag) []Version { + ephemeral := make([]Version, 0) + for _, tag := range tags { + if !versionExists(versions, tag.semVer) { + ephemeral = append(ephemeral, tag.semVer) + } + } + return append(versions, ephemeral...) +} + +func versionExists(versions []Version, v Version) bool { + for _, v2 := range versions { + if v == v2 { + return true + } + } + return false +} + +func (v *taggableVCS) Timestamp(ctx context.Context, version Version) (time.Time, error) { + version2, err := v.resolveVersion(ctx, version) + if err != nil { + return time.Time{}, err + } + return v.wrapped.Timestamp(ctx, version2) +} + +func (v *taggableVCS) Zip(ctx context.Context, version Version) (io.ReadCloser, error) { + version2, err := v.resolveVersion(ctx, version) + if err != nil { + return nil, err + } + // Zip must contain the ephemeral version. + dirName := v.module + "@" + string(version) + return v.wrapped.zipUnder(ctx, version2, dirName) +} + +func (v *taggableVCS) resolveVersion(ctx context.Context, version Version) (Version, error) { + for _, tag := range v.storage.tags(v.module) { + if tag.semVer == version { + // TODO(bilus): Duplicated in git.go. + t, err := v.wrapped.Timestamp(ctx, Version("v0.0.0-20060102150405-"+tag.short)) + if err != nil { + return Version(""), err + } + version2 := Version(fmt.Sprintf("v0.0.0-%s-%s", t.Format("20060102150405"), tag.short)) + + return version2, nil + } + } + return version, nil +}