diff --git a/api/yolo.proto b/api/yolo.proto index b1dbcaa2..a67af136 100644 --- a/api/yolo.proto +++ b/api/yolo.proto @@ -15,7 +15,6 @@ service YoloService { rpc Ping(Ping.Request) returns (Ping.Response) { option (google.api.http) = {get: "/ping"}; }; rpc Status(Status.Request) returns (Status.Response) { option (google.api.http) = {get: "/status"}; }; rpc BuildList(BuildList.Request) returns (BuildList.Response) { option (google.api.http) = {get: "/build-list"}; } - rpc ArtifactList(ArtifactList.Request) returns (ArtifactList.Response) { option (google.api.http) = {get: "/artifact-list"}; } } // @@ -38,18 +37,11 @@ message Status { } message BuildList { - message Request {} - message Response { - repeated Build builds = 1; - } -} - -message ArtifactList { message Request { - Artifact.Kind kind = 1; + Artifact.Kind artifact_kind = 1; } message Response { - repeated Artifact artifacts = 1; + repeated Build builds = 1; } } @@ -69,6 +61,8 @@ message Build { string commit = 9 [(gogoproto.moretags) = "quad:\"schema:commit,optional\""]; string branch = 10 [(gogoproto.moretags) = "quad:\"schema:branch,optional\""]; + repeated Artifact has_artifacts = 101 [(gogoproto.moretags) = "quad:\"schema:hasBuild < *,optional\""]; + enum State { UnknownState = 0; Running = 1; @@ -92,7 +86,7 @@ message Artifact { State state = 8 [(gogoproto.moretags) = "quad:\"schema:state,optional\""]; Kind kind = 9 [(gogoproto.moretags) = "quad:\"schema:kind,optional\""]; - Build has_build = 101 [(gogoproto.moretags) = "quad:\"hasBuild,optional\""]; + Build has_build = 101 [(gogoproto.moretags) = "quad:\"schema:hasBuild,optional\""]; enum State { UnknownState = 0; diff --git a/buildkite.go b/buildkite.go index f1637517..355477de 100644 --- a/buildkite.go +++ b/buildkite.go @@ -9,7 +9,8 @@ import ( "github.com/buildkite/go-buildkite/buildkite" "github.com/cayleygraph/cayley" - "github.com/cayleygraph/cayley/graph/path" + "github.com/cayleygraph/cayley/query/path" + "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "go.uber.org/zap" ) @@ -20,7 +21,7 @@ type BuildkiteWorkerOpts struct { } // BuildkiteWorker goals is to manage the buildkite update routine, it should try to support as much errors as possible by itself -func BuildkiteWorker(ctx context.Context, db *cayley.Handle, bkc *buildkite.Client, opts BuildkiteWorkerOpts) error { +func BuildkiteWorker(ctx context.Context, db *cayley.Handle, bkc *buildkite.Client, schema *schema.Config, opts BuildkiteWorkerOpts) error { if opts.Logger == nil { opts.Logger = zap.NewNop() } @@ -40,7 +41,7 @@ func BuildkiteWorker(ctx context.Context, db *cayley.Handle, bkc *buildkite.Clie if err != nil { logger.Warn("fetch buildkite", zap.Error(err)) } else { - if err := saveBatches(ctx, db, batches); err != nil { + if err := saveBatches(ctx, db, batches, schema); err != nil { logger.Warn("save batches", zap.Error(err)) } } @@ -186,6 +187,8 @@ func buildkiteArtifactsToBatch(artifacts []buildkite.Artifact, build buildkite.B newArtifact.Kind = Artifact_IPA case ".apk": newArtifact.Kind = Artifact_APK + default: + newArtifact.Kind = Artifact_UnknownKind } switch *artifact.State { case "finished": @@ -197,6 +200,7 @@ func buildkiteArtifactsToBatch(artifacts []buildkite.Artifact, build buildkite.B case "deleted": newArtifact.State = Artifact_Deleted default: + newArtifact.State = Artifact_UnknownState fmt.Println("unknown state: ", *artifact.State) } batch.Artifacts = append(batch.Artifacts, &newArtifact) diff --git a/cmd/yolo/main.go b/cmd/yolo/main.go index fa793957..dcbf20ba 100644 --- a/cmd/yolo/main.go +++ b/cmd/yolo/main.go @@ -79,7 +79,7 @@ func main() { return err } opts := yolo.BuildkiteWorkerOpts{Logger: logger, MaxPages: maxPages} - gr.Add(func() error { return yolo.BuildkiteWorker(ctx, db, bkc, opts) }, func(_ error) { cancel() }) + gr.Add(func() error { return yolo.BuildkiteWorker(ctx, db, bkc, dbSchema, opts) }, func(_ error) { cancel() }) // server svc := yolo.NewService(db, dbSchema, yolo.ServiceOpts{ diff --git a/db.go b/db.go index d34b7173..27acd60b 100644 --- a/db.go +++ b/db.go @@ -17,7 +17,7 @@ func SchemaConfig() *schema.Config { return config } -func saveBatches(ctx context.Context, db *cayley.Handle, batches []Batch) error { +func saveBatches(ctx context.Context, db *cayley.Handle, batches []Batch, schema *schema.Config) error { tx := cayley.NewTransaction() dw := graph.NewTxWriter(tx, graph.Delete) iw := graph.NewTxWriter(tx, graph.Add) diff --git a/gen.sum b/gen.sum index 7b501302..213bbd24 100644 --- a/gen.sum +++ b/gen.sum @@ -1,2 +1,2 @@ +785a44458568a93de1427046e5b54c4d955cd8a0 ./api/yolo.proto b39593fbd759edfacb39b5cd10b55ce71c49e7b4 Makefile -de2fd8bc96c38b48ccbcae9b498794da8b588dc0 ./api/yolo.proto diff --git a/go.mod b/go.mod index 30552771..238e1893 100644 --- a/go.mod +++ b/go.mod @@ -23,5 +23,7 @@ require ( google.golang.org/genproto v0.0.0-20200204235621-fb4a7afc5178 google.golang.org/grpc v1.27.0 howett.net/plist v0.0.0-20181124034731-591f970eefbb - moul.io/depviz v1.1.1-0.20200205124305-ba4612b0396d + moul.io/depviz/v3 v3.5.0 ) + +replace github.com/cayleygraph/cayley v0.7.7 => github.com/cayleygraph/cayley v0.7.7-0.20200130230943-9fb4d58e0e07 diff --git a/go.sum b/go.sum index 40403da4..7770ca95 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/buildkite/go-buildkite v2.2.0+incompatible h1:yEjSu1axFC88x4dbufhgMDsEnJztPWlLiZzEvzJggXc= github.com/buildkite/go-buildkite v2.2.0+incompatible/go.mod h1:WTV0aX5KnQ9ofsKMg2CLUBLJNsQ0RwOEKPhrXXZWPcE= +github.com/cayleygraph/cayley v0.7.7-0.20200130230943-9fb4d58e0e07 h1:NWyjQzz6t7Rh3mwv0gMOkIp5GZy1v+NkPypSmgN0J6s= +github.com/cayleygraph/cayley v0.7.7-0.20200130230943-9fb4d58e0e07/go.mod h1:NcUBKkA9+qbBhtOEAXtn+rucziFVTBNw+c9ehZ6QlPQ= github.com/cayleygraph/cayley v0.7.7 h1:z+7xkAbg6bKiXJOtOkEG3zCm2K084sr/aGwFV7xcQNs= github.com/cayleygraph/cayley v0.7.7/go.mod h1:VUd+PInYf94/VY41ePeFtFyP99BAs953kFT4N+6F7Ko= github.com/cayleygraph/quad v1.1.0 h1:w1nXAmn+nz07+qlw89dke9LwWkYpeX+OcvfTvGQRBpM= @@ -532,8 +534,9 @@ honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXe honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= howett.net/plist v0.0.0-20181124034731-591f970eefbb h1:jhnBjNi9UFpfpl8YZhA9CrOqpnJdvzuiHsl/dnxl11M= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= -moul.io/depviz v1.1.1-0.20200205124305-ba4612b0396d h1:2SzpFoKz7ZCRUxQ/p1rTaikeUdWdl1S8gtfYXHVbx5w= -moul.io/depviz v1.1.1-0.20200205124305-ba4612b0396d/go.mod h1:/DwpEFFmgFS59cBGo5bSzGQbjh/CDZMZeVX1Nu6mzRM= +moul.io/depviz v1.1.0 h1:gx+H9uRFroNarvFUVhTz167ryr8I1VyufHkeylj/y48= +moul.io/depviz/v3 v3.5.0 h1:Lt3VZuYy7ygQ3ggez+4k9W8cq6V/Z3SQ1LTmxAZatxM= +moul.io/depviz/v3 v3.5.0/go.mod h1:8FqL05r5TpCwgdJ07DvuwyP7TUe8ziyY0CD2kdOSNOo= moul.io/godev v1.4.0 h1:hEUzBIrRtBRWqZHmp12bFUlLSyKxpvfcMYa3Pipfng8= moul.io/godev v1.4.0/go.mod h1:xsV3hXF1E9vy2T/1b1CLjSImJ1/nQeTfqTDhJHhJCWg= moul.io/graphman v1.6.0/go.mod h1:WfW75G37UTvAu09o4xdmcflGDn+VbZiwStaKmJKxkE0= diff --git a/server.go b/server.go index bfe5f662..a99cd69f 100644 --- a/server.go +++ b/server.go @@ -21,7 +21,7 @@ import ( chilogger "github.com/treastech/logger" "go.uber.org/zap" "google.golang.org/grpc" - "moul.io/depviz/pkg/chiutil" + "moul.io/depviz/v3/pkg/chiutil" ) type Server struct { @@ -101,8 +101,8 @@ func NewServer(ctx context.Context, svc Service, opts ServerOpts) (*Server, erro r.Use(cors.Handler) } r.Use(chilogger.Logger(srv.logger)) - r.Use(middleware.Recoverer) r.Use(middleware.Timeout(opts.RequestTimeout)) + r.Use(middleware.Recoverer) gwmux := runtime.NewServeMux( runtime.WithMarshalerOption(runtime.MIMEWildcard, &gateway.JSONPb{EmitDefaults: false, Indent: " ", OrigName: true}), runtime.WithProtoErrorHandler(runtime.DefaultHTTPProtoErrorHandler), diff --git a/service.go b/service.go index 73a025c3..31213eca 100644 --- a/service.go +++ b/service.go @@ -5,13 +5,14 @@ import ( "fmt" "net/http" "path" + "reflect" "sort" "time" plistgen "berty.tech/yolo/v2/pkg/plistgen" "github.com/buildkite/go-buildkite/buildkite" "github.com/cayleygraph/cayley" - cayleypath "github.com/cayleygraph/cayley/graph/path" + cayleypath "github.com/cayleygraph/cayley/query/path" "github.com/cayleygraph/cayley/schema" "github.com/cayleygraph/quad" "github.com/go-chi/chi" @@ -62,8 +63,8 @@ func (svc service) Status(ctx context.Context, req *Status_Request) (*Status_Res // db stats, err := svc.db.Stats(ctx, false) if err == nil { - resp.DbNodes = stats.Nodes.Size - resp.DbQuads = stats.Quads.Size + resp.DbNodes = stats.Nodes.Value + resp.DbQuads = stats.Quads.Value } else { resp.DbErr = err.Error() } @@ -75,14 +76,41 @@ func (svc service) BuildList(ctx context.Context, req *BuildList_Request) (*Buil resp := BuildList_Response{} p := cayleypath.StartPath(svc.db). - Both(). - Has(quad.IRI("rdf:type"), quad.IRI("yolo:Build")). - Limit(300) + Has(quad.IRI("rdf:type"), quad.IRI("yolo:Build")) + if req.ArtifactKind > 0 { + // this will filter builds with at least one artifact of the good kind + // but I don't know how to filter them during loading, so I will cleanup + // the result later, feel free to help me make things in a better way + p = p.HasPath( + cayleypath.StartMorphism(). + In(quad.IRI("schema:hasBuild")). + Has(quad.IRI("schema:kind"), quad.Int(req.ArtifactKind)), + ) + } + p = p.Limit(300) builds := []Build{} - if err := svc.schema.LoadPathTo(ctx, svc.db, &builds, p); err != nil { + if err := svc.schema.LoadIteratorToDepth(ctx, svc.db, reflect.ValueOf(&builds), 1, p.BuildIterator(ctx)); err != nil { return nil, fmt.Errorf("load builds: %w", err) } + // clean up the result + for idx, build := range builds { + // avoid infinite loop by removing already existing pointers + for _, artifact := range build.HasArtifacts { + artifact.HasBuild = nil + } + // cleanup artifact with invalid requested type (see comment above) + if req.ArtifactKind > 0 { + n := 0 + for _, artifact := range build.HasArtifacts { + if artifact.Kind == req.ArtifactKind { + build.HasArtifacts[n] = artifact + n++ + } + } + builds[idx].HasArtifacts = build.HasArtifacts[:n] + } + } resp.Builds = make([]*Build, len(builds)) for i := range builds { @@ -96,33 +124,6 @@ func (svc service) BuildList(ctx context.Context, req *BuildList_Request) (*Buil return &resp, nil } -func (svc service) ArtifactList(ctx context.Context, req *ArtifactList_Request) (*ArtifactList_Response, error) { - resp := ArtifactList_Response{} - - p := cayleypath.StartPath(svc.db). - Has(quad.IRI("rdf:type"), quad.IRI("yolo:Artifact")) - if req.Kind != 0 { - p = p.Has(quad.IRI("schema:kind"), quad.Int(req.Kind)) - } - p = p.Limit(300) - - artifacts := []Artifact{} - if err := svc.schema.LoadPathTo(ctx, svc.db, &artifacts, p); err != nil { - return nil, fmt.Errorf("load artifacts: %w", err) - } - - resp.Artifacts = make([]*Artifact, len(artifacts)) - for i := range artifacts { - resp.Artifacts[i] = &artifacts[i] - } - - sort.Slice(resp.Artifacts[:], func(i, j int) bool { - return resp.Artifacts[i].CreatedAt.After(*resp.Artifacts[j].CreatedAt) - }) - - return &resp, nil -} - func (svc service) PlistGenerator(w http.ResponseWriter, r *http.Request) { id := chi.URLParam(r, "artifactID") diff --git a/web/index.html b/web/index.html index de300a9e..db88693f 100644 --- a/web/index.html +++ b/web/index.html @@ -56,36 +56,48 @@

Yolo World!