Merge pull request #270 from berty/dev/moul/artifact-caching

.gitignore

@@ -3,6 +3,8 @@ coverage.txt
 packrd/
 *-packr.go
 *.tmp
+cache/
+buildkite_*
 
 # Junk files
 *~

go/Makefile

@@ -1,4 +1,4 @@
-DEV_RUN_OPTS ?= --dev-mode
+DEV_RUN_OPTS ?= --dev-mode --artifacts-cache-path=./cache/artifacts --http-cache-path=./cache/http
 
 GO_TEST_OPTS ?= -test.timeout=60s
 

go/cmd/yolo/main.go

@@ -55,6 +55,7 @@ func yolo(args []string) error {
 		githubToken        string
 		bintrayUsername    string
 		bintrayToken       string
+		artifactsCachePath string
 		circleciToken      string
 		dbStorePath        string
 		withPreloading     bool
@@ -85,8 +86,7 @@ func yolo(args []string) error {
 	serverFlagSet.StringVar(&circleciToken, "circleci-token", "", "CircleCI API Token")
 	serverFlagSet.StringVar(&githubToken, "github-token", "", "GitHub API Token")
 	serverFlagSet.StringVar(&dbStorePath, "db-path", ":memory:", "DB Store path")
-	storeFlagSet.StringVar(&dbStorePath, "db-path", ":memory:", "DB Store path")
-	storeFlagSet.BoolVar(&withPreloading, "with-preloading", false, "with auto DB preloading")
+	serverFlagSet.StringVar(&artifactsCachePath, "artifacts-cache-path", "", "Artifacts caching path")
 	serverFlagSet.IntVar(&maxBuilds, "max-builds", 100, "maximum builds to fetch from external services (pagination)")
 	serverFlagSet.StringVar(&httpBind, "http-bind", ":8000", "HTTP bind address")
 	serverFlagSet.StringVar(&grpcBind, "grpc-bind", ":9000", "gRPC bind address")
@@ -98,6 +98,8 @@ func yolo(args []string) error {
 	serverFlagSet.StringVar(&bearerSecretKey, "bearer-secretkey", "", "optional Bearer.sh Secret Key")
 	serverFlagSet.StringVar(&authSalt, "auth-salt", "", "salt used to generate authentication tokens at the end of the URLs")
 	serverFlagSet.StringVar(&httpCachePath, "http-cache-path", "", "if set, will cache http client requests")
+	storeFlagSet.StringVar(&dbStorePath, "db-path", ":memory:", "DB Store path")
+	storeFlagSet.BoolVar(&withPreloading, "with-preloading", false, "with auto DB preloading")
 
 	server := &ffcli.Command{
 		Name:      `server`,
@@ -157,15 +159,22 @@ func yolo(args []string) error {
 				logger.Warn("--dev-mode: insecure helpers are enabled")
 			}
 
+			if artifactsCachePath != "" {
+				if err := os.MkdirAll(artifactsCachePath, 0755); err != nil {
+					return err
+				}
+			}
+
 			// service
 			svc, err := yolosvc.NewService(db, yolosvc.ServiceOpts{
-				Logger:          logger,
-				BuildkiteClient: bkc,
-				CircleciClient:  ccc,
-				BintrayClient:   btc,
-				GithubClient:    ghc,
-				AuthSalt:        authSalt,
-				DevMode:         devMode,
+				Logger:             logger,
+				BuildkiteClient:    bkc,
+				CircleciClient:     ccc,
+				BintrayClient:      btc,
+				GithubClient:       ghc,
+				AuthSalt:           authSalt,
+				DevMode:            devMode,
+				ArtifactsCachePath: artifactsCachePath,
 			})
 			if err != nil {
 				return err

go/pkg/bintray/bintray.go

@@ -81,7 +81,7 @@ func (c Client) GetVersion(subject, repo, pkg, version string) (GetVersionRespon
 	return result, err
 }
 
-func DownloadContent(url string, w http.ResponseWriter) error {
+func DownloadContent(url string, w io.Writer) error {
 	resp, err := http.Get(url)
 	if err != nil {
 		return err

go/pkg/yolosvc/api_download.go

@@ -2,25 +2,43 @@ package yolosvc
 
 import (
 	"fmt"
+	"io"
 	"net/http"
+	"os"
 	"path"
+	"path/filepath"
 
 	"berty.tech/yolo/v2/go/pkg/bintray"
 	"berty.tech/yolo/v2/go/pkg/yolopb"
 	"github.com/go-chi/chi"
 	"google.golang.org/grpc/codes"
 )
 
-func (svc service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) {
-	id := chi.URLParam(r, "artifactID")
+func (svc *service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) {
 
+	id := chi.URLParam(r, "artifactID")
 	var artifact yolopb.Artifact
 	err := svc.db.First(&artifact, "ID = ?", id).Error
 	if err != nil {
 		httpError(w, err, codes.InvalidArgument)
 		return
 	}
 
+	cache := filepath.Join(svc.artifactsCachePath, artifact.ID)
+	// download missing cache
+	if svc.artifactsCachePath != "" {
+		svc.artifactsCacheMutex.Lock()
+		if !fileExists(cache) {
+			err := svc.artifactDownloadToFile(&artifact, cache)
+			if err != nil {
+				svc.artifactsCacheMutex.Unlock()
+				httpError(w, err, codes.Internal)
+				return
+			}
+		}
+		svc.artifactsCacheMutex.Unlock()
+	}
+
 	base := path.Base(artifact.LocalPath)
 	w.Header().Add("Content-Disposition", fmt.Sprintf("inline; filename=%s", base))
 	if artifact.FileSize > 0 {
@@ -30,20 +48,62 @@ func (svc service) ArtifactDownloader(w http.ResponseWriter, r *http.Request) {
 		w.Header().Add("Content-Type", artifact.MimeType)
 	}
 
+	if svc.artifactsCachePath != "" {
+		// send cache
+		f, err := os.Open(cache)
+		if err != nil {
+			httpError(w, err, codes.Internal)
+			return
+		}
+		defer f.Close()
+		_, err = io.Copy(w, f)
+		if err != nil {
+			httpError(w, err, codes.Internal)
+		}
+	} else {
+		// proxy
+		err = svc.artifactDownloadFromProvider(&artifact, w)
+		if err != nil {
+			httpError(w, err, codes.Internal)
+		}
+	}
+}
+
+func (svc *service) artifactDownloadToFile(artifact *yolopb.Artifact, dest string) error {
+	out, err := os.Create(dest + ".tmp")
+	if err != nil {
+		return err
+	}
+
+	err = svc.artifactDownloadFromProvider(artifact, out)
+	if err != nil {
+		out.Close()
+		return err
+	}
+
+	out.Close()
+
+	err = os.Rename(dest+".tmp", dest)
+	if err != nil {
+		return err
+	}
+
+	// FIXME: parse file and update the db with new metadata
+
+	return nil
+}
+
+func (svc *service) artifactDownloadFromProvider(artifact *yolopb.Artifact, w io.Writer) error {
 	switch artifact.Driver {
 	case yolopb.Driver_Buildkite:
 		if svc.bkc == nil {
-			err = fmt.Errorf("buildkite token required")
-		} else {
-			_, err = svc.bkc.Artifacts.DownloadArtifactByURL(artifact.DownloadURL, w)
+			return fmt.Errorf("buildkite token required")
 		}
+		_, err := svc.bkc.Artifacts.DownloadArtifactByURL(artifact.DownloadURL, w)
+		return err
 	case yolopb.Driver_Bintray:
-		err = bintray.DownloadContent(artifact.DownloadURL, w)
-	// case Driver_CircleCI:
-	default:
-		err = fmt.Errorf("download not supported for this driver")
-	}
-	if err != nil {
-		httpError(w, err, codes.Internal)
+		return bintray.DownloadContent(artifact.DownloadURL, w)
+		// case Driver_CircleCI:
 	}
+	return fmt.Errorf("download not supported for this driver")
 }

go/pkg/yolosvc/service.go

@@ -3,6 +3,7 @@ package yolosvc
 import (
 	"context"
 	"net/http"
+	"sync"
 	"time"
 
 	"berty.tech/yolo/v2/go/pkg/bintray"
@@ -27,27 +28,30 @@ type Service interface {
 }
 
 type service struct {
-	startTime  time.Time
-	db         *gorm.DB
-	logger     *zap.Logger
-	bkc        *buildkite.Client
-	btc        *bintray.Client
-	ccc        *circleci.Client
-	ghc        *github.Client
-	authSalt   string
-	devMode    bool
-	clearCache *abool.AtomicBool
+	startTime           time.Time
+	db                  *gorm.DB
+	logger              *zap.Logger
+	bkc                 *buildkite.Client
+	btc                 *bintray.Client
+	ccc                 *circleci.Client
+	ghc                 *github.Client
+	authSalt            string
+	devMode             bool
+	clearCache          *abool.AtomicBool
+	artifactsCachePath  string
+	artifactsCacheMutex sync.Mutex
 }
 
 type ServiceOpts struct {
-	BuildkiteClient *buildkite.Client
-	CircleciClient  *circleci.Client
-	BintrayClient   *bintray.Client
-	GithubClient    *github.Client
-	Logger          *zap.Logger
-	AuthSalt        string
-	DevMode         bool
-	ClearCache      *abool.AtomicBool
+	BuildkiteClient    *buildkite.Client
+	CircleciClient     *circleci.Client
+	BintrayClient      *bintray.Client
+	GithubClient       *github.Client
+	Logger             *zap.Logger
+	AuthSalt           string
+	DevMode            bool
+	ClearCache         *abool.AtomicBool
+	ArtifactsCachePath string
 }
 
 func NewService(db *gorm.DB, opts ServiceOpts) (Service, error) {
@@ -59,16 +63,17 @@ func NewService(db *gorm.DB, opts ServiceOpts) (Service, error) {
 	}
 
 	return &service{
-		startTime:  time.Now(),
-		db:         db,
-		logger:     opts.Logger,
-		bkc:        opts.BuildkiteClient,
-		btc:        opts.BintrayClient,
-		ccc:        opts.CircleciClient,
-		ghc:        opts.GithubClient,
-		authSalt:   opts.AuthSalt,
-		devMode:    opts.DevMode,
-		clearCache: opts.ClearCache,
+		startTime:          time.Now(),
+		db:                 db,
+		logger:             opts.Logger,
+		bkc:                opts.BuildkiteClient,
+		btc:                opts.BintrayClient,
+		ccc:                opts.CircleciClient,
+		ghc:                opts.GithubClient,
+		authSalt:           opts.AuthSalt,
+		devMode:            opts.DevMode,
+		clearCache:         opts.ClearCache,
+		artifactsCachePath: opts.ArtifactsCachePath,
 	}, nil
 }
 

go/pkg/yolosvc/util.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
 
@@ -67,3 +68,8 @@ func guessMissingBuildInfo(build *yolopb.Build) {
 		build.VCSTagURL = fmt.Sprintf("%s/tree/%s", build.HasProjectID, build.VCSTag)
 	}
 }
+
+func fileExists(path string) bool {
+	_, err := os.Stat(path)
+	return !os.IsNotExist(err)
+}