Merge pull request #5 from berty/dev/moul/build-artifacts

feat: list builds' artifacts

api/yolo.proto

@@ -15,7 +15,6 @@ service YoloService {
   rpc Ping(Ping.Request)                 returns (Ping.Response)         { option (google.api.http) = {get: "/ping"}; };
   rpc Status(Status.Request)             returns (Status.Response)       { option (google.api.http) = {get: "/status"}; };
   rpc BuildList(BuildList.Request)       returns (BuildList.Response)    { option (google.api.http) = {get: "/build-list"}; }
-  rpc ArtifactList(ArtifactList.Request) returns (ArtifactList.Response) { option (google.api.http) = {get: "/artifact-list"}; }
 }
 
 //
@@ -38,18 +37,11 @@ message Status {
 }
 
 message BuildList {
-  message Request {}
-  message Response {
-    repeated Build builds = 1;
-  }
-}
-
-message ArtifactList {
   message Request {
-    Artifact.Kind kind = 1;
+    Artifact.Kind artifact_kind = 1;
   }
   message Response {
-    repeated Artifact artifacts = 1;
+    repeated Build builds = 1;
   }
 }
 
@@ -69,6 +61,8 @@ message Build {
   string commit = 9 [(gogoproto.moretags) = "quad:\"schema:commit,optional\""];
   string branch = 10 [(gogoproto.moretags) = "quad:\"schema:branch,optional\""];
 
+  repeated Artifact has_artifacts = 101 [(gogoproto.moretags) = "quad:\"schema:hasBuild < *,optional\""];
+
   enum State {
     UnknownState = 0;
     Running = 1;
@@ -92,7 +86,7 @@ message Artifact {
   State state = 8 [(gogoproto.moretags) = "quad:\"schema:state,optional\""];
   Kind kind = 9 [(gogoproto.moretags) = "quad:\"schema:kind,optional\""];
 
-  Build has_build = 101 [(gogoproto.moretags) = "quad:\"hasBuild,optional\""];
+  Build has_build = 101 [(gogoproto.moretags) = "quad:\"schema:hasBuild,optional\""];
 
   enum State {
     UnknownState = 0;

buildkite.go

@@ -9,7 +9,8 @@ import (
 
 	"github.com/buildkite/go-buildkite/buildkite"
 	"github.com/cayleygraph/cayley"
-	"github.com/cayleygraph/cayley/graph/path"
+	"github.com/cayleygraph/cayley/query/path"
+	"github.com/cayleygraph/cayley/schema"
 	"github.com/cayleygraph/quad"
 	"go.uber.org/zap"
 )
@@ -20,7 +21,7 @@ type BuildkiteWorkerOpts struct {
 }
 
 // BuildkiteWorker goals is to manage the buildkite update routine, it should try to support as much errors as possible by itself
-func BuildkiteWorker(ctx context.Context, db *cayley.Handle, bkc *buildkite.Client, opts BuildkiteWorkerOpts) error {
+func BuildkiteWorker(ctx context.Context, db *cayley.Handle, bkc *buildkite.Client, schema *schema.Config, opts BuildkiteWorkerOpts) error {
 	if opts.Logger == nil {
 		opts.Logger = zap.NewNop()
 	}
@@ -40,7 +41,7 @@ func BuildkiteWorker(ctx context.Context, db *cayley.Handle, bkc *buildkite.Clie
 		if err != nil {
 			logger.Warn("fetch buildkite", zap.Error(err))
 		} else {
-			if err := saveBatches(ctx, db, batches); err != nil {
+			if err := saveBatches(ctx, db, batches, schema); err != nil {
 				logger.Warn("save batches", zap.Error(err))
 			}
 		}
@@ -186,6 +187,8 @@ func buildkiteArtifactsToBatch(artifacts []buildkite.Artifact, build buildkite.B
 			newArtifact.Kind = Artifact_IPA
 		case ".apk":
 			newArtifact.Kind = Artifact_APK
+		default:
+			newArtifact.Kind = Artifact_UnknownKind
 		}
 		switch *artifact.State {
 		case "finished":
@@ -197,6 +200,7 @@ func buildkiteArtifactsToBatch(artifacts []buildkite.Artifact, build buildkite.B
 		case "deleted":
 			newArtifact.State = Artifact_Deleted
 		default:
+			newArtifact.State = Artifact_UnknownState
 			fmt.Println("unknown state: ", *artifact.State)
 		}
 		batch.Artifacts = append(batch.Artifacts, &newArtifact)

cmd/yolo/main.go

@@ -79,7 +79,7 @@ func main() {
 				return err
 			}
 			opts := yolo.BuildkiteWorkerOpts{Logger: logger, MaxPages: maxPages}
-			gr.Add(func() error { return yolo.BuildkiteWorker(ctx, db, bkc, opts) }, func(_ error) { cancel() })
+			gr.Add(func() error { return yolo.BuildkiteWorker(ctx, db, bkc, dbSchema, opts) }, func(_ error) { cancel() })
 
 			// server
 			svc := yolo.NewService(db, dbSchema, yolo.ServiceOpts{

db.go

@@ -17,7 +17,7 @@ func SchemaConfig() *schema.Config {
 	return config
 }
 
-func saveBatches(ctx context.Context, db *cayley.Handle, batches []Batch) error {
+func saveBatches(ctx context.Context, db *cayley.Handle, batches []Batch, schema *schema.Config) error {
 	tx := cayley.NewTransaction()
 	dw := graph.NewTxWriter(tx, graph.Delete)
 	iw := graph.NewTxWriter(tx, graph.Add)

server.go

@@ -21,7 +21,7 @@ import (
 	chilogger "github.com/treastech/logger"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
-	"moul.io/depviz/pkg/chiutil"
+	"moul.io/depviz/v3/pkg/chiutil"
 )
 
 type Server struct {
@@ -101,8 +101,8 @@ func NewServer(ctx context.Context, svc Service, opts ServerOpts) (*Server, erro
 		r.Use(cors.Handler)
 	}
 	r.Use(chilogger.Logger(srv.logger))
-	r.Use(middleware.Recoverer)
 	r.Use(middleware.Timeout(opts.RequestTimeout))
+	r.Use(middleware.Recoverer)
 	gwmux := runtime.NewServeMux(
 		runtime.WithMarshalerOption(runtime.MIMEWildcard, &gateway.JSONPb{EmitDefaults: false, Indent: "  ", OrigName: true}),
 		runtime.WithProtoErrorHandler(runtime.DefaultHTTPProtoErrorHandler),

service.go

@@ -5,13 +5,14 @@ import (
 	"fmt"
 	"net/http"
 	"path"
+	"reflect"
 	"sort"
 	"time"
 
 	plistgen "berty.tech/yolo/v2/pkg/plistgen"
 	"github.com/buildkite/go-buildkite/buildkite"
 	"github.com/cayleygraph/cayley"
-	cayleypath "github.com/cayleygraph/cayley/graph/path"
+	cayleypath "github.com/cayleygraph/cayley/query/path"
 	"github.com/cayleygraph/cayley/schema"
 	"github.com/cayleygraph/quad"
 	"github.com/go-chi/chi"
@@ -62,8 +63,8 @@ func (svc service) Status(ctx context.Context, req *Status_Request) (*Status_Res
 	// db
 	stats, err := svc.db.Stats(ctx, false)
 	if err == nil {
-		resp.DbNodes = stats.Nodes.Size
-		resp.DbQuads = stats.Quads.Size
+		resp.DbNodes = stats.Nodes.Value
+		resp.DbQuads = stats.Quads.Value
 	} else {
 		resp.DbErr = err.Error()
 	}
@@ -75,14 +76,41 @@ func (svc service) BuildList(ctx context.Context, req *BuildList_Request) (*Buil
 	resp := BuildList_Response{}
 
 	p := cayleypath.StartPath(svc.db).
-		Both().
-		Has(quad.IRI("rdf:type"), quad.IRI("yolo:Build")).
-		Limit(300)
+		Has(quad.IRI("rdf:type"), quad.IRI("yolo:Build"))
+	if req.ArtifactKind > 0 {
+		// this will filter builds with at least one artifact of the good kind
+		// but I don't know how to filter them during loading, so I will cleanup
+		// the result later, feel free to help me make things in a better way
+		p = p.HasPath(
+			cayleypath.StartMorphism().
+				In(quad.IRI("schema:hasBuild")).
+				Has(quad.IRI("schema:kind"), quad.Int(req.ArtifactKind)),
+		)
+	}
+	p = p.Limit(300)
 
 	builds := []Build{}
-	if err := svc.schema.LoadPathTo(ctx, svc.db, &builds, p); err != nil {
+	if err := svc.schema.LoadIteratorToDepth(ctx, svc.db, reflect.ValueOf(&builds), 1, p.BuildIterator(ctx)); err != nil {
 		return nil, fmt.Errorf("load builds: %w", err)
 	}
+	// clean up the result
+	for idx, build := range builds {
+		// avoid infinite loop by removing already existing pointers
+		for _, artifact := range build.HasArtifacts {
+			artifact.HasBuild = nil
+		}
+		// cleanup artifact with invalid requested type (see comment above)
+		if req.ArtifactKind > 0 {
+			n := 0
+			for _, artifact := range build.HasArtifacts {
+				if artifact.Kind == req.ArtifactKind {
+					build.HasArtifacts[n] = artifact
+					n++
+				}
+			}
+			builds[idx].HasArtifacts = build.HasArtifacts[:n]
+		}
+	}
 
 	resp.Builds = make([]*Build, len(builds))
 	for i := range builds {
@@ -96,33 +124,6 @@ func (svc service) BuildList(ctx context.Context, req *BuildList_Request) (*Buil
 	return &resp, nil
 }
 
-func (svc service) ArtifactList(ctx context.Context, req *ArtifactList_Request) (*ArtifactList_Response, error) {
-	resp := ArtifactList_Response{}
-
-	p := cayleypath.StartPath(svc.db).
-		Has(quad.IRI("rdf:type"), quad.IRI("yolo:Artifact"))
-	if req.Kind != 0 {
-		p = p.Has(quad.IRI("schema:kind"), quad.Int(req.Kind))
-	}
-	p = p.Limit(300)
-
-	artifacts := []Artifact{}
-	if err := svc.schema.LoadPathTo(ctx, svc.db, &artifacts, p); err != nil {
-		return nil, fmt.Errorf("load artifacts: %w", err)
-	}
-
-	resp.Artifacts = make([]*Artifact, len(artifacts))
-	for i := range artifacts {
-		resp.Artifacts[i] = &artifacts[i]
-	}
-
-	sort.Slice(resp.Artifacts[:], func(i, j int) bool {
-		return resp.Artifacts[i].CreatedAt.After(*resp.Artifacts[j].CreatedAt)
-	})
-
-	return &resp, nil
-}
-
 func (svc service) PlistGenerator(w http.ResponseWriter, r *http.Request) {
 	id := chi.URLParam(r, "artifactID")
 

web/index.html

@@ -56,36 +56,48 @@ <h1 class="jumbotron-heading">Yolo World!</h1>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/mustache.js/3.1.0/mustache.min.js" integrity="sha256-MPgtcamIpCPKRRm1ppJHkvtNBAuE71xcOM+MmQytXi8=" crossorigin="anonymous"></script>
 
     <script id="releasesTpl" type="text/template">
-      {{#artifacts}}
-      <div class="col-md-4">
-        <div class="card mb-4 box-shadow">
+      {{#builds}}
+      <div class="col-md-12 my-2">
+        <div class="card mb-12 box-shadow">
           <div class="card-body">
-            <p class="card-text">{{has_build.id}}</a>
-            <p class="card-text"><small class="text-muted">branch: {{has_build.branch}}</small></p>
-            <p class="card-text"><small class="text-muted">message: {{has_build.message}}</small></p>
-            <p class="card-text"><small class="text-muted">path: {{local_path}}</small></p>
+            <p class="card-text">{{id}}</a>
+            <p class="card-text"><small class="text-muted">branch: {{branch}}</small></p>
             <p class="card-text"><small class="text-muted">date: {{created_at}}</small></p>
+            <p class="card-text"><small class="text-muted">message: {{message}}</small></p>
+            <p class="card-text"><small class="text-muted">commit: {{commit}}</small></p>
             <p class="card-text"><small class="text-muted">state: {{state}}</small></p>
-            <div class="d-flex justify-content-between align-items-center">
-              <div class="btn-group">
-                <a type="button" class="btn btn-sm btn-outline-secondary" href="/api/artifact-dl?id={{id}}">DL</a>
-                <a type="button" class="btn btn-sm btn-outline-secondary" href="itms-services://?action=download-manifest&url={{baseURL}}/api/plist-gen/{{id}}.plist">Plist</a>
-                <a type="button" class="btn btn-sm btn-outline-secondary" href="{{has_build}}">Logs</a>
+            <a type="button" class="btn btn-sm btn-outline-secondary" href="{{has_build}}">Logs</a>
+            <div class="container">
+              <div class="row">
+                {{#has_artifacts}}
+                  <div class="col-md-4 my-3">
+                    <div class="card bg-warning mb-4 box-shadow p-2">
+                      <p class="card-text"><small class="text-muted">path: {{local_path}}</small></p>
+                      <p class="card-text"><small class="text-muted">state: {{state}}</small></p>
+                      <div class="d-flex justify-content-between align-items-center">
+                        <div class="btn-group">
+                          <a type="button" class="btn btn-sm btn-outline-secondary" href="/api/artifact-dl?id={{id}}">DL</a>
+                          <a type="button" class="btn btn-sm btn-outline-secondary" href="itms-services://?action=download-manifest&url={{baseURL}}/api/plist-gen/{{id}}.plist">Plist</a>
+                        </div>
+                        <small class="text-muted">{{file_size}}b</small>
+                      </div>
+                    </div>
+                  </div>
+                {{/has_artifacts}}
               </div>
-              <small class="text-muted">{{file_size}}b</small>
             </div>
           </div>
         </div>
       </div>
-      {{/artifacts}}
+      {{/builds}}
     </script>
 
     <script type="text/javascript">
       function fetch() {
-        var url = "/api/artifact-list?";
+        var url = "/api/build-list?";
         var kind = $('#kind option:selected').attr('id');
         if (kind !== undefined && kind != "") {
-          url += "kind=" + kind + "&";
+          url += "artifact_kind=" + kind + "&";
         }
         $.ajax({
           url: url,