Merge pull request #792 from pingcap-inc/sprint9

Sprint9

.gitignore

@@ -2,6 +2,7 @@
 .DS_Store
 golangci-lint-report.xml
 results.xml
+.github/ISSUE_TEMPLATE
 .idea/
 data.etcd1/
 *.db
@@ -19,7 +20,7 @@ test/
 /template
 /nohup.out
 .vscode/
-log/
+/log
 logs/
 out.txt
 run.sh

.golangci.yml

@@ -1,6 +1,10 @@
 # options for analysis running
 run:
   timeout: 10m
+  skip-dirs:
+    - test
+  skip-files:
+    - ".*_test\\.go$"
 
 linters:
   enable:

Makefile

@@ -247,9 +247,9 @@ failpoint-disable: build_failpoint_ctl
 # Restoring gofail failpoints...
 	@$(FAILPOINT_DISABLE)
 
-lint:
+lint: mock
 	# refer https://golangci-lint.run/usage/install/#local-installation to install golangci-lint firstly
-	-golangci-lint run --out-format=junit-xml  --timeout=10m -v ./... > golangci-lint-report.xml
+	golangci-lint run --out-format=junit-xml  --timeout=10m -v ./... > golangci-lint-report.xml
 
 gosec:
 	go install github.com/securego/gosec/v2/cmd/gosec@latest
@@ -284,7 +284,7 @@ mock:
 	mockgen -destination ./test/mockutil/mocksshclientexecutor/mock_ssh_client_interface.go -package mocksshclient -source ./util/ssh/ssh_client.go
 	mockgen -destination ./test/mockresource/mock_allocator_recycler.go -package mock_allocator_recycler -source ./micro-cluster/resourcemanager/management/structs/allocator_recycler.go
 	mockgen -destination ./test/mockcdcmanager/mock_cdc_manager.go -package mockcdcmanager -source ./micro-cluster/cluster/switchover/cdc_manager_api.go
-	mockgen -destination ./test/mockmodels/mock_product.go -package mock_product -source ./models/platform/product/product_read_writer.go
+	mockgen -destination ./test/mockmodels/mock_product.go -package mock_product -source ./models/platform/product/readerwriter.go
 	mockgen -destination ./test/mockaccount/mock_account.go -package mockaccount -source ./models/user/account/readerwriter.go
 	mockgen -destination ./test/mockidentification/mock_identification.go -package mockidentification -source ./models/user/identification/readerwriter.go
 	mockgen -destination ./test/mockchangefeed/mock_changefeed.go -package mockchangefeed -source ./micro-cluster/cluster/changefeed/service.go
@@ -294,6 +294,9 @@ mock:
 	mockgen -destination ./test/mockutiltidbhttp/mock_utiltidbhttp.go -package mockutiltidbhttp -source ./util/api/tidb/http/clusterconfig.go
 	mockgen -destination ./test/mockutiltidbsql_config/mock_utiltidbsql_config.go -package mockutiltidbsqlconfig -source ./util/api/tidb/sql/clusterconfig.go
 	mockgen -destination ./test/mockutilcdc/mock_utilcdc_change_feed.go -package mockutilcdc -source ./util/api/cdc/changefeed.go
+	mockgen -destination ./test/mockcheck/mock_check.go -package mock_check -source ./models/platform/check/report_read_writer.go
+	mockgen -destination ./test/mockreport/mock_report.go -package mock_report -source ./micro-cluster/platform/check/handler.go
+	mockgen -destination ./test/mockhostsinspect/mock_hosts_inspect.go -package mock_hosts_inspect -source ./micro-cluster/resourcemanager/inspect/hostinspector.go
 
 swag:
 	$(GO) install github.com/swaggo/swag/cmd/[email protected]

Makefile.common

@@ -70,6 +70,8 @@ MAC       := "Darwin"
 
 PACKAGE_LIST  := go list ./...| grep -vE "docs|proto|common|message|micro-cluster/service|util|micro-cluster/registry|micro-cluster/platform/telemetry|library/framework|micro-api|file-server|test/mock|deprecated|util/scp|^github.com/pingcap-inc/tiem/micro-cluster$$|library/secondparty|micro-cluster/resourcemanager/management|models/resource/management|micro-cluster/cluster/switchover"
 PACKAGES  ?= $$($(PACKAGE_LIST))
+LINT_PACKAGE_LIST  := go list ./...| grep -vE "docs|proto|test"
+LINT_PACKAGES  ?= $$($(LINT_PACKAGE_LIST))
 #PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/$(PROJECT)||'
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST)
 FILES     := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")

common/constants/changefeed.go

@@ -74,3 +74,8 @@ const (
 	DownstreamTypeKafka DownstreamType = "kafka"
 	DownstreamTypeMysql DownstreamType = "mysql"
 )
+
+var DefaultFilterRules = []string{
+	"*.*",
+	"!__TiDB_BR_Temporary*.*",
+}

common/constants/check.go

@@ -0,0 +1,36 @@
+/******************************************************************************
+ * Copyright (c)  2022 PingCAP, Inc.                                          *
+ * Licensed under the Apache License, Version 2.0 (the "License");            *
+ * you may not use this file except in compliance with the License.           *
+ * You may obtain a copy of the License at                                    *
+ *                                                                            *
+ * http://www.apache.org/licenses/LICENSE-2.0                                 *
+ *                                                                            *
+ *  Unless required by applicable law or agreed to in writing, software       *
+ *  distributed under the License is distributed on an "AS IS" BASIS,         *
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
+ *  See the License for the specific language governing permissions and       *
+ *  limitations under the License.                                            *
+ ******************************************************************************/
+
+package constants
+
+const (
+	FlowCheckPlatform = "CheckPlatform"
+	FlowCheckCluster  = "CheckCluster"
+)
+
+type CheckStatus string
+
+const (
+	CheckRunning   CheckStatus = "Running"
+	CheckFailure   CheckStatus = "Failure"
+	CheckCompleted CheckStatus = "Completed"
+)
+
+type ReportType string
+
+const (
+	PlatformReport ReportType = "Platform"
+	ClusterReport  ReportType = "Cluster"
+)

common/constants/cluster.go

@@ -40,6 +40,7 @@ type ClusterMaintenanceStatus string
 const (
 	ClusterMaintenanceCreating                     ClusterMaintenanceStatus = "Creating"
 	ClusterMaintenanceCloning                      ClusterMaintenanceStatus = "Cloning"
+	ClusterMaintenanceBeingCloned                  ClusterMaintenanceStatus = "BeingCloned"
 	ClusterMaintenanceDeleting                     ClusterMaintenanceStatus = "Deleting"
 	ClusterMaintenanceStopping                     ClusterMaintenanceStatus = "Stopping"
 	ClusterMaintenanceRestarting                   ClusterMaintenanceStatus = "Restarting"
@@ -173,6 +174,7 @@ const (
 	DBUserBackupRestore       DBUserRoleType = "EM_Backup_Restore"       // user for backup and restore
 	DBUserParameterManagement DBUserRoleType = "EM_Parameter_Management" // user for managing parameters
 	DBUserCDCDataSync         DBUserRoleType = "CDC_Data_Sync"           // user for CDC data synchronization
+	DBUserGrafana             DBUserRoleType = "Grafana"                 // user for Grafana
 )
 
 var DBUserName = map[DBUserRoleType]string{
@@ -191,3 +193,15 @@ var DBUserPermission = map[DBUserRoleType][]string{
 
 // DefaultRetainedPortRange default retained port range for tiem
 var DefaultRetainedPortRange = "[11000,12000]"
+
+// RequestResourceMode specifying resource mode, enums : SpecificZone,SpecificHost
+type RequestResourceMode string
+
+const (
+	ResourceModeSpecificZone = "SpecificZone"
+	ResourceModeSpecificHost = "SpecificHost"
+)
+
+func DefaultResourceMode() string {
+	return ResourceModeSpecificZone
+}

common/constants/common.go

@@ -105,6 +105,13 @@ var ParasiteComponentIDs = []EMProductComponentIDType{
 	ComponentIDAlertManger,
 }
 
+var KernelComponentIDs = []EMProductComponentIDType{
+	ComponentIDTiDB,
+	ComponentIDTiKV,
+	ComponentIDPD,
+	ComponentIDTiFlash,
+}
+
 //Definition of product component ID provided by Enterprise manager
 const (
 	ComponentIDTiDB    EMProductComponentIDType = "TiDB"

common/constants/metrics.go

@@ -54,8 +54,20 @@ const (
 	MetricsClusterUpgradePath           MetricsType = "cluster/upgrade_path"
 	MetricsClusterUpgradeDiff           MetricsType = "cluster/upgrade_diff"
 
+	MetricsMetadataDeletePhysically     MetricsType = "metadata/delete"
+
+	MetricsPlatformCheck     MetricsType = "platform/check"
+	MetricsClusterCheck      MetricsType = "platform/check/cluster"
+	MetricsQueryCheckReports MetricsType = "platform/reports"
+	MetricsGetCheckReport    MetricsType = "platform/report"
+	MetricsQueryPlatformLog  MetricsType = "platform/log"
+
+	MetricsSystemConfigUpdate MetricsType = "config/update"
+	MetricsSystemConfigGet    MetricsType = "config/get"
+
 	// MetricsBackupCreate define backup metrics
 	MetricsBackupCreate         MetricsType = "backup/create"
+	MetricsBackupCancel         MetricsType = "backup/cancel"
 	MetricsBackupDelete         MetricsType = "backup/delete"
 	MetricsBackupQuery          MetricsType = "backup/query"
 	MetricsBackupQueryStrategy  MetricsType = "backup/query_strategy"
@@ -67,9 +79,6 @@ const (
 	MetricsDataExportImportQuery  MetricsType = "data/query_export_import_record"
 	MetricsDataExportImportDelete MetricsType = "data/delete_export_import_record"
 
-	// MetricsPlatformQueryKnowledge define knowledge metrics
-	MetricsPlatformQueryKnowledge MetricsType = "platform/query_knowledge"
-
 	// MetricsCDCTaskCreate define cdc metrics
 	MetricsCDCTaskCreate MetricsType = "cdc/create"
 	MetricsCDCTaskDelete MetricsType = "cdc/delete"
@@ -129,9 +138,23 @@ const (
 	MetricsResourceDownloadHostTemplateFile MetricsType = "resource/download_host_template_file"
 	MetricsResourceReservedHost             MetricsType = "resource/reserved_host"
 	MetricsResourceModifyHostStatus         MetricsType = "resource/modify_host_status"
-	MetricsResourceImportHosts              MetricsType = "resource/import_host"
-	MetricsResourceDeleteHost               MetricsType = "resource/delete_host"
-	MetricsResourceQueryHosts               MetricsType = "resource/query"
+	MetricsResourceImportHosts              MetricsType = "resource/import_hosts"
+	MetricsResourceDeleteHosts              MetricsType = "resource/delete_hosts"
+	MetricsResourceQueryHosts               MetricsType = "resource/query_hosts"
+	MetricsResourceUpdateHost               MetricsType = "resource/update_host"
+	MetricsResourceCreateDisks              MetricsType = "resource/create_disks"
+	MetricsResourceDeleteDisks              MetricsType = "resource/delete_disks"
+	MetricsResourceUpdateDisk               MetricsType = "resource/update_disk"
+
+	// MetricsProductUpdate define product metrics
+	MetricsProductUpdate         MetricsType = "product/update_products"
+	MetricsProductQuery          MetricsType = "product/query_products"
+	MetricsProductQueryAvailable MetricsType = "product/query_available_products"
+	MetricsProductQueryDetail    MetricsType = "product/query_product_detail"
+
+	MetricsVendorUpdate         MetricsType = "vendor/update_vendors"
+	MetricsVendorQuery          MetricsType = "vendor/query_vendors"
+	MetricsVendorQueryAvailable MetricsType = "vendor/query_available_vendors"
 )
 
 var EMMetrics = []MetricsType{
@@ -154,7 +177,7 @@ var EMMetrics = []MetricsType{
 	MetricsClusterModifyParameter,
 	MetricsClusterInspectParameter,
 	MetricsClusterQueryLogParameter,
-
+	MetricsMetadataDeletePhysically,
 	// MetricsBackupCreate define backup metrics
 	MetricsBackupCreate,
 	MetricsBackupDelete,
@@ -168,9 +191,6 @@ var EMMetrics = []MetricsType{
 	MetricsDataExportImportQuery,
 	MetricsDataExportImportDelete,
 
-	// MetricsPlatformQueryKnowledge define knowledge metrics
-	MetricsPlatformQueryKnowledge,
-
 	// MetricsCDCTaskCreate define cdc metrics
 	MetricsCDCTaskCreate,
 	MetricsCDCTaskDelete,
@@ -205,7 +225,22 @@ var EMMetrics = []MetricsType{
 	MetricsResourceDownloadHostTemplateFile,
 	MetricsResourceModifyHostStatus,
 	MetricsResourceImportHosts,
-	MetricsResourceDeleteHost,
+	MetricsResourceDeleteHosts,
 	MetricsResourceQueryHosts,
 	MetricsResourceReservedHost,
+	MetricsResourceUpdateHost,
+	MetricsResourceCreateDisks,
+	MetricsResourceDeleteDisks,
+	MetricsResourceUpdateDisk,
+
+	// define product metrics
+	MetricsProductUpdate,
+	MetricsProductQuery,
+	MetricsProductQueryAvailable,
+	MetricsProductQueryDetail,
+
+	// define vendor metrics
+	MetricsVendorUpdate,
+	MetricsVendorQuery,
+	MetricsVendorQueryAvailable,
 }

common/constants/platform.go

@@ -43,6 +43,9 @@ const (
 	ConfigPrometheusAddress string = "config_prometheus_address"
 
 	ConfigKeyRetainedPortRange string = "config_retained_port_range"
+
+	ConfigKeyDefaultSSHPort  string = "config_default_ssh_port"
+	ConfigKeyExtraVMFacturer string = "config_extra_vm_facturer"
 )
 
 type SystemState string

common/constants/switchover.go

@@ -27,12 +27,13 @@ const SwitchoverCheckMasterSlaveMaxLagTimeRetriesCount = 3
 const SwitchoverCheckMasterSlaveMaxLagTimeRetryWait = 3000 * time.Millisecond
 
 const SwitchoverCheckSyncChangeFeedTaskHealthTimeInterval = 5000 * time.Millisecond
-const SwitchoverCheckSyncChangeFeedTaskHealthRetriesCount = 3
+const SwitchoverCheckSyncChangeFeedTaskHealthRetriesCount = 11
 const SwitchoverCheckSyncChangeFeedTaskHealthRetryWait = 500 * time.Millisecond
 
-const SwitchoverCheckSyncChangeFeedTaskCaughtUpRetriesCount = 3
-const SwitchoverCheckSyncChangeFeedTaskCaughtUpRetryWait = 1000 * time.Millisecond
-const SwitchoverCheckSyncChangeFeedTaskCaughtUpMaxLagTime = 2000 * time.Millisecond
+const SwitchoverCheckChangeFeedTaskCaughtUpRetriesCount = 60
+const SwitchoverCheckChangeFeedTaskCaughtUpRetryWait = 1000 * time.Millisecond
+const SwitchoverCheckChangeFeedTaskCaughtUpMaxLagTime = 3000 * time.Millisecond
+const SwitchoverCheckChangeFeedTaskCaughtUpMakeSureRetriesCount = 0
 
 const SwitchoverReadWriteHealthTestDBName = "test"
 const SwitchoverReadWriteHealthTestTableNamePrefix = "em_rwtest"

common/constants/user.go

@@ -23,6 +23,8 @@
 
 package constants
 
+import "time"
+
 type TenantStatus string
 
 //Definition tenant status information
@@ -32,6 +34,7 @@ const (
 )
 
 type TenantOnBoardingStatus string
+
 const (
 	TenantOnBoarding  TenantOnBoardingStatus = "On"
 	TenantOFFBoarding TenantOnBoardingStatus = "Off"
@@ -64,4 +67,7 @@ const (
 
 func (s CommonStatus) IsValid() bool {
 	return s == Valid
-}
+}
+
+// 1 year
+const ExpirationTime = 365 * 24 * time.Hour

common/constants/workflow.go

@@ -29,7 +29,9 @@ type WorkFlowStatus string
 const (
 	WorkFlowStatusInitializing = "Initializing"
 	WorkFlowStatusProcessing   = "Processing"
+	WorkFlowStatusCanceling    = "Canceling"
 	WorkFlowStatusFinished     = "Finished"
 	WorkFlowStatusError        = "Error"
 	WorkFlowStatusCanceled     = "Canceled"
+	WorkFlowStatusStopped      = "Stopped"
 )