Skip to content

Commit

Permalink
feat(frontend/backend): Allow the ability to sort experiments by last…
Browse files Browse the repository at this point in the history 
… run creation. Fixes #10884 (#11163)

* UPSTREAM: <carry>: add last_run_creation

Signed-off-by: Humair Khan <[email protected]>

* Allow-the-ability-to-sort-Experiments-by-last-run-creation-#10884

Signed-off-by: Elay Aharoni (EXT-Nokia) <[email protected]>

* UPSTREAM: <carry>: chore(backend): Rename UpdateLastRun -> SetLastRunTimestamp

follup up to bf77909. Rename UpdateLastRun -> SetLastRunTimestamp

also tweak a related log message

Signed-off-by: Greg Sheremeta <[email protected]>

* UPSTREAM: <carry>: chore(backend): Rename UpdateLastRun -> SetLastRunTimestamp

follup up to bf77909. Rename UpdateLastRun -> SetLastRunTimestamp

also tweak a related log message

Signed-off-by: Greg Sheremeta <[email protected]>

---------

Signed-off-by: Humair Khan <[email protected]>
Signed-off-by: Elay Aharoni (EXT-Nokia) <[email protected]>
Signed-off-by: Greg Sheremeta <[email protected]>
Co-authored-by: Humair Khan <[email protected]>
Co-authored-by: Elay Aharoni (EXT-Nokia) <[email protected]>
Co-authored-by: Greg Sheremeta <[email protected]>
  • Loading branch information
@ElayAharoni @HumairAK @gregsheremeta
4 people authored Sep 25, 2024
1 parent 3f49522 commit db8669c
Show file tree
Hide file tree
Showing 15 changed files with 496 additions and 297 deletions.
3 changes: 3 additions & 0 deletions backend/api/v2beta1/experiment.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ message Experiment {

// Output. Specifies whether this experiment is in archived or available state.
StorageState storage_state = 6;

// Output. The creation time of the last run in this experiment.
google.protobuf.Timestamp last_run_created_at = 7;
}

message CreateExperimentRequest {
Expand Down
323 changes: 167 additions & 156 deletions backend/api/v2beta1/go_client/experiment.pb.go
View file

Large diffs are not rendered by default.

21 changes: 21 additions & 0 deletions backend/api/v2beta1/go_http_client/experiment_model/v2beta1_experiment.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions backend/api/v2beta1/swagger/experiment.swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,11 @@
"storage_state": {
"$ref": "#/definitions/v2beta1ExperimentStorageState",
"description": "Output. Specifies whether this experiment is in archived or available state."
},
"last_run_created_at": {
"type": "string",
"format": "date-time",
"description": "Output. The time the created time of the last run in this experiment."
}
}
},
Expand Down
5 changes: 5 additions & 0 deletions backend/api/v2beta1/swagger/kfp_api_single_file.swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1613,6 +1613,11 @@
"storage_state": {
"$ref": "#/definitions/v2beta1ExperimentStorageState",
"description": "Output. Specifies whether this experiment is in archived or available state."
},
"last_run_created_at": {
"type": "string",
"format": "date-time",
"description": "Output. The time the created time of the last run in this experiment."
}
}
},
Expand Down
32 changes: 18 additions & 14 deletions backend/src/apiserver/model/experiment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@
package model

type Experiment struct {
UUID string `gorm:"column:UUID; not null; primary_key;"`
Name string `gorm:"column:Name; not null; unique_index:idx_name_namespace;"`
Description string `gorm:"column:Description; not null;"`
CreatedAtInSec int64 `gorm:"column:CreatedAtInSec; not null;"`
Namespace string `gorm:"column:Namespace; not null; unique_index:idx_name_namespace;"`
StorageState StorageState `gorm:"column:StorageState; not null;"`
UUID string `gorm:"column:UUID; not null; primary_key;"`
Name string `gorm:"column:Name; not null; unique_index:idx_name_namespace;"`
Description string `gorm:"column:Description; not null;"`
CreatedAtInSec int64 `gorm:"column:CreatedAtInSec; not null;"`
LastRunCreatedAtInSec int64 `gorm:"column:LastRunCreatedAtInSec; not null;"`
Namespace string `gorm:"column:Namespace; not null; unique_index:idx_name_namespace;"`
StorageState StorageState `gorm:"column:StorageState; not null;"`
}

// Note: Experiment.StorageState can have values: "STORAGE_STATE_UNSPECIFIED", "AVAILABLE" or "ARCHIVED"
Expand All @@ -44,14 +45,15 @@ func (e *Experiment) DefaultSortField() string {
}

var experimentAPIToModelFieldMap = map[string]string{
"id": "UUID", // v1beta1 API
"experiment_id": "UUID", // v2beta1 API
"name": "Name", // v1beta1 API
"display_name": "Name", // v2beta1 API
"created_at": "CreatedAtInSec",
"description": "Description",
"namespace": "Namespace", // v2beta1 API
"storage_state": "StorageState",
"id": "UUID", // v1beta1 API
"experiment_id": "UUID", // v2beta1 API
"name": "Name", // v1beta1 API
"display_name": "Name", // v2beta1 API
"created_at": "CreatedAtInSec",
"last_run_created_at": "LastRunCreatedAtInSec", // v2beta1 API
"description": "Description",
"namespace": "Namespace", // v2beta1 API
"storage_state": "StorageState",
}

// APIToModelFieldMap returns a map from API names to field names for model
Expand Down Expand Up @@ -80,6 +82,8 @@ func (e *Experiment) GetFieldValue(name string) interface{} {
return e.Name
case "CreatedAtInSec":
return e.CreatedAtInSec
case "LastRunCreatedAtInSec":
return e.LastRunCreatedAtInSec
case "Description":
return e.Description
case "Namespace":
Expand Down
11 changes: 11 additions & 0 deletions backend/src/apiserver/resource/resource_manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,13 @@ func (r *ResourceManager) CreateRun(ctx context.Context, run *model.Run) (*model
if err != nil {
return nil, util.Wrap(err, "Failed to create a run")
}

// Upon run creation, update owning experiment
err = r.experimentStore.SetLastRunTimestamp(newRun)
if err != nil {
return nil, util.Wrap(err, fmt.Sprintf("Failed to set last run timestamp on experiment %s for run %s", newRun.ExperimentId, newRun.UUID))
}

return newRun, nil
}

Expand Down Expand Up @@ -1245,6 +1252,10 @@ func (r *ResourceManager) ReportWorkflowResource(ctx context.Context, execSpec u
} else {
runId = run.UUID
}
// Upon run creation, update owning experiment
if updateError = r.experimentStore.SetLastRunTimestamp(run); updateError != nil {
return nil, util.Wrapf(updateError, "Failed to report a workflow for existing run %s during updating the owning experiment.", runId)
}
}
if execStatus.IsInFinalState() {
err := addWorkflowLabel(ctx, r.getWorkflowClient(execSpec.ExecutionNamespace()), execSpec.ExecutionName(), util.LabelKeyWorkflowPersistedFinalState, "true")
Expand Down
13 changes: 7 additions & 6 deletions backend/src/apiserver/server/api_converter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,13 @@ func toApiExperiment(experiment *model.Experiment) *apiv2beta1.Experiment {
storageState = apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["STORAGE_STATE_UNSPECIFIED"])
}
return &apiv2beta1.Experiment{
ExperimentId: experiment.UUID,
DisplayName: experiment.Name,
Description: experiment.Description,
CreatedAt: &timestamp.Timestamp{Seconds: experiment.CreatedAtInSec},
Namespace: experiment.Namespace,
StorageState: storageState,
ExperimentId: experiment.UUID,
DisplayName: experiment.Name,
Description: experiment.Description,
CreatedAt: &timestamp.Timestamp{Seconds: experiment.CreatedAtInSec},
LastRunCreatedAt: &timestamp.Timestamp{Seconds: experiment.LastRunCreatedAtInSec},
Namespace: experiment.Namespace,
StorageState: storageState,
}
}

Expand Down
110 changes: 60 additions & 50 deletions backend/src/apiserver/server/api_converter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1974,77 +1974,87 @@ func TestToApiExperimentsV1(t *testing.T) {

func TestToApiExperiments(t *testing.T) {
exp1 := &model.Experiment{
UUID: "exp1",
CreatedAtInSec: 1,
Name: "experiment1",
Description: "My name is experiment1",
StorageState: "AVAILABLE",
UUID: "exp1",
CreatedAtInSec: 1,
LastRunCreatedAtInSec: 1,
Name: "experiment1",
Description: "My name is experiment1",
StorageState: "AVAILABLE",
}
exp2 := &model.Experiment{
UUID: "exp2",
CreatedAtInSec: 2,
Name: "experiment2",
Description: "My name is experiment2",
StorageState: "ARCHIVED",
UUID: "exp2",
CreatedAtInSec: 2,
LastRunCreatedAtInSec: 2,
Name: "experiment2",
Description: "My name is experiment2",
StorageState: "ARCHIVED",
}
exp3 := &model.Experiment{
UUID: "exp3",
CreatedAtInSec: 1,
Name: "experiment3",
Description: "experiment3 was created using V1 APIV1BETA1",
StorageState: "STORAGESTATE_AVAILABLE",
UUID: "exp3",
CreatedAtInSec: 1,
LastRunCreatedAtInSec: 1,
Name: "experiment3",
Description: "experiment3 was created using V1 APIV1BETA1",
StorageState: "STORAGESTATE_AVAILABLE",
}
exp4 := &model.Experiment{
UUID: "exp4",
CreatedAtInSec: 2,
Name: "experiment4",
Description: "experiment4 was created using V1 APIV1BETA1",
StorageState: "STORAGESTATE_ARCHIVED",
UUID: "exp4",
CreatedAtInSec: 2,
LastRunCreatedAtInSec: 2,
Name: "experiment4",
Description: "experiment4 was created using V1 APIV1BETA1",
StorageState: "STORAGESTATE_ARCHIVED",
}
exp5 := &model.Experiment{
UUID: "exp5",
CreatedAtInSec: 1,
Name: "experiment5",
Description: "My name is experiment5",
StorageState: "this is invalid storage state",
UUID: "exp5",
CreatedAtInSec: 1,
LastRunCreatedAtInSec: 1,
Name: "experiment5",
Description: "My name is experiment5",
StorageState: "this is invalid storage state",
}
apiExps := toApiExperiments([]*model.Experiment{exp1, exp2, exp3, exp4, nil, exp5})
expectedApiExps := []*apiv2beta1.Experiment{
{
ExperimentId: "exp1",
DisplayName: "experiment1",
Description: "My name is experiment1",
CreatedAt: &timestamp.Timestamp{Seconds: 1},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["AVAILABLE"]),
ExperimentId: "exp1",
DisplayName: "experiment1",
Description: "My name is experiment1",
CreatedAt: &timestamp.Timestamp{Seconds: 1},
LastRunCreatedAt: &timestamp.Timestamp{Seconds: 1},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["AVAILABLE"]),
},
{
ExperimentId: "exp2",
DisplayName: "experiment2",
Description: "My name is experiment2",
CreatedAt: &timestamp.Timestamp{Seconds: 2},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["ARCHIVED"]),
ExperimentId: "exp2",
DisplayName: "experiment2",
Description: "My name is experiment2",
CreatedAt: &timestamp.Timestamp{Seconds: 2},
LastRunCreatedAt: &timestamp.Timestamp{Seconds: 2},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["ARCHIVED"]),
},
{
ExperimentId: "exp3",
DisplayName: "experiment3",
Description: "experiment3 was created using V1 APIV1BETA1",
CreatedAt: &timestamp.Timestamp{Seconds: 1},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["AVAILABLE"]),
ExperimentId: "exp3",
DisplayName: "experiment3",
Description: "experiment3 was created using V1 APIV1BETA1",
CreatedAt: &timestamp.Timestamp{Seconds: 1},
LastRunCreatedAt: &timestamp.Timestamp{Seconds: 1},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["AVAILABLE"]),
},
{
ExperimentId: "exp4",
DisplayName: "experiment4",
Description: "experiment4 was created using V1 APIV1BETA1",
CreatedAt: &timestamp.Timestamp{Seconds: 2},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["ARCHIVED"]),
ExperimentId: "exp4",
DisplayName: "experiment4",
Description: "experiment4 was created using V1 APIV1BETA1",
CreatedAt: &timestamp.Timestamp{Seconds: 2},
LastRunCreatedAt: &timestamp.Timestamp{Seconds: 2},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["ARCHIVED"]),
},
{},
{
ExperimentId: "exp5",
DisplayName: "experiment5",
Description: "My name is experiment5",
CreatedAt: &timestamp.Timestamp{Seconds: 1},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["STORAGE_STATE_UNSPECIFIED"]),
ExperimentId: "exp5",
DisplayName: "experiment5",
Description: "My name is experiment5",
CreatedAt: &timestamp.Timestamp{Seconds: 1},
LastRunCreatedAt: &timestamp.Timestamp{Seconds: 1},
StorageState: apiv2beta1.Experiment_StorageState(apiv2beta1.Experiment_StorageState_value["STORAGE_STATE_UNSPECIFIED"]),
},
}
assert.Equal(t, expectedApiExps, apiExps)
Expand Down
Loading

0 comments on commit db8669c

Please sign in to comment.