feat: complete daemon operations and recovery tooling
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
package backendstatus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"yms-daemon/internal/containerengine"
|
||||
"yms-daemon/internal/daemonapi"
|
||||
"yms-daemon/internal/deploymentconfig"
|
||||
"yms-daemon/internal/transaction"
|
||||
)
|
||||
|
||||
// 诊断项稳定标识，供测试与 reconcile 识别，不参与展示。
|
||||
const (
|
||||
codeUndeployed = "container_backend_undeployed"
|
||||
codeUndeployedWithHistory = "container_backend_undeployed_with_history"
|
||||
codeContainerWithoutRecord = "container_present_without_deployment_record"
|
||||
codeCommittedPortOK = "nginx_active_port"
|
||||
codeNginxWrongPort = "nginx_wrong_port"
|
||||
codeActiveContainerOK = "active_container_healthy"
|
||||
codeActiveContainerMissing = "active_container_missing"
|
||||
codeActiveContainerIdentity = "active_container_identity_mismatch"
|
||||
codeActiveContainerNotRunning = "active_container_not_running"
|
||||
codeInactiveContainerOK = "inactive_container_absent"
|
||||
codeInactiveContainerResidual = "inactive_container_residual"
|
||||
codeInactiveContainerRunning = "inactive_container_running"
|
||||
codeDeploymentConfigMismatch = "deployment_record_config_mismatch"
|
||||
)
|
||||
|
||||
// diagnoseContainer 对 container 后端做只读诊断。
|
||||
// 事实来源是 backend_container_deployment 单例记录；现场状态由容器引擎与宿主 Nginx 提供。
|
||||
func (d *Diagnoser) diagnoseContainer(ctx context.Context) (daemonapi.Diagnosis, error) {
|
||||
findings, err := d.containerGather(ctx)
|
||||
if err != nil {
|
||||
return daemonapi.Diagnosis{}, err
|
||||
}
|
||||
return toDiagnosis("backend", deploymentconfig.BackendTypeContainer, findings), nil
|
||||
}
|
||||
|
||||
// containerGather 读取 container 后端的部署记录、历史事务与宿主 Nginx 状态，生成全部诊断项。
|
||||
func (d *Diagnoser) containerGather(ctx context.Context) ([]finding, error) {
|
||||
deployment, depErr := d.store.BackendContainerDeployment(ctx)
|
||||
hasDeployment := depErr == nil
|
||||
if depErr != nil && !errors.Is(depErr, transaction.ErrNotFound) {
|
||||
return nil, fmt.Errorf("read backend container deployment: %w", depErr)
|
||||
}
|
||||
hasHistory, err := d.store.HasCommittedBackendContainerTransactionHistory(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
before, err := d.gateway.Read()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read host Nginx configuration: %w", err)
|
||||
}
|
||||
return d.containerFindings(ctx, deployment, hasDeployment, hasHistory, before.ActivePort)
|
||||
}
|
||||
|
||||
// containerFindings 生成 container 后端的全部诊断项。
|
||||
// deployment 部署记录，hasDeployment 表示是否存在该记录，hasHistory 表示是否存在历史
|
||||
// 已提交事务，activePort 是宿主 Nginx 当前指向的后端端口。
|
||||
func (d *Diagnoser) containerFindings(ctx context.Context, deployment transaction.BackendContainerDeployment, hasDeployment bool, hasHistory bool, activePort int) ([]finding, error) {
|
||||
if !hasDeployment {
|
||||
return d.containerUndeployedFindings(ctx, hasHistory)
|
||||
}
|
||||
|
||||
findings := make([]finding, 0, 6)
|
||||
|
||||
committedSlot, err := d.config.Backend.SlotForPort(deployment.ActivePort)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if deployment.ContainerName != committedSlot.ContainerName {
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeDeploymentConfigMismatch,
|
||||
message: fmt.Sprintf("部署记录容器 %s 与活动端口 %d 的槽位容器 %s 不一致", deployment.ContainerName, deployment.ActivePort, committedSlot.ContainerName),
|
||||
})
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
committedContainer, committedErr := d.engine.InspectContainer(ctx, deployment.ContainerName)
|
||||
committedHealthy := false
|
||||
switch {
|
||||
case errors.Is(committedErr, containerengine.ErrNotFound):
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeActiveContainerMissing,
|
||||
message: "部署记录的活动容器 " + deployment.ContainerName + " 不存在",
|
||||
})
|
||||
case committedErr != nil:
|
||||
return nil, fmt.Errorf("inspect committed backend container %s: %w", deployment.ContainerName, committedErr)
|
||||
case committedContainer.ID != deployment.ContainerID:
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeActiveContainerIdentity,
|
||||
message: "活动容器 " + deployment.ContainerName + " 身份与部署记录不一致",
|
||||
})
|
||||
case !committedContainer.Running || committedContainer.Dead:
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeActiveContainerNotRunning,
|
||||
message: "活动容器 " + deployment.ContainerName + " 未运行",
|
||||
})
|
||||
default:
|
||||
committedHealthy = true
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelOK,
|
||||
code: codeActiveContainerOK,
|
||||
message: "活动容器 " + deployment.ContainerName + " 健康",
|
||||
})
|
||||
}
|
||||
|
||||
if activePort == deployment.ActivePort {
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelOK,
|
||||
code: codeCommittedPortOK,
|
||||
message: fmt.Sprintf("宿主 Nginx 指向部署记录的活动端口 %d", deployment.ActivePort),
|
||||
})
|
||||
} else if committedHealthy {
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelFixable,
|
||||
code: codeNginxWrongPort,
|
||||
message: fmt.Sprintf("宿主 Nginx 指向 %d，但部署记录的活动端口是 %d", activePort, deployment.ActivePort),
|
||||
action: fmt.Sprintf("将宿主 Nginx 切流到端口 %d", deployment.ActivePort),
|
||||
fix: &fixAction{kind: fixSwitchPort, port: deployment.ActivePort},
|
||||
})
|
||||
} else {
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeNginxWrongPort,
|
||||
message: fmt.Sprintf("宿主 Nginx 指向 %d，与部署记录活动端口 %d 不一致，且活动容器不健康", activePort, deployment.ActivePort),
|
||||
})
|
||||
}
|
||||
|
||||
inactiveSlot, err := d.config.Backend.SlotForPort(otherPort(deployment.ActivePort))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inactiveContainer, inactiveErr := d.engine.InspectContainer(ctx, inactiveSlot.ContainerName)
|
||||
switch {
|
||||
case errors.Is(inactiveErr, containerengine.ErrNotFound):
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelOK,
|
||||
code: codeInactiveContainerOK,
|
||||
message: "非活动槽位容器 " + inactiveSlot.ContainerName + " 不存在",
|
||||
})
|
||||
case inactiveErr != nil:
|
||||
return nil, fmt.Errorf("inspect inactive backend container %s: %w", inactiveSlot.ContainerName, inactiveErr)
|
||||
case inactiveContainer.Running && !inactiveContainer.Dead:
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeInactiveContainerRunning,
|
||||
message: "非活动槽位容器 " + inactiveSlot.ContainerName + " 意外处于运行状态",
|
||||
})
|
||||
default:
|
||||
findings = append(findings, finding{
|
||||
level: daemonapi.DiagnosisLevelFixable,
|
||||
code: codeInactiveContainerResidual,
|
||||
message: "非活动槽位容器 " + inactiveSlot.ContainerName + " 已停止，属于残留",
|
||||
action: "移除残留的已停止容器 " + inactiveSlot.ContainerName,
|
||||
fix: &fixAction{kind: fixRemoveContainer, container: inactiveSlot.ContainerName},
|
||||
})
|
||||
}
|
||||
|
||||
return findings, nil
|
||||
}
|
||||
|
||||
// containerUndeployedFindings 处理不存在部署记录的情况，依据容器现场与历史记录判断是否为全新机器。
|
||||
func (d *Diagnoser) containerUndeployedFindings(ctx context.Context, hasHistory bool) ([]finding, error) {
|
||||
var present []string
|
||||
for _, port := range []int{deploymentconfig.BackendPort8080, deploymentconfig.BackendPort8081} {
|
||||
slot, err := d.config.Backend.SlotForPort(port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := d.engine.InspectContainer(ctx, slot.ContainerName); err == nil {
|
||||
present = append(present, slot.ContainerName)
|
||||
} else if !errors.Is(err, containerengine.ErrNotFound) {
|
||||
return nil, fmt.Errorf("inspect backend container %s: %w", slot.ContainerName, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(present) > 0 {
|
||||
return []finding{{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeContainerWithoutRecord,
|
||||
message: fmt.Sprintf("SQLite 没有部署记录，但现场存在容器 %v", present),
|
||||
}}, nil
|
||||
}
|
||||
if hasHistory {
|
||||
return []finding{{
|
||||
level: daemonapi.DiagnosisLevelDrift,
|
||||
code: codeUndeployedWithHistory,
|
||||
message: "存在历史已提交事务，但缺少部署记录且两个槽位容器均不存在",
|
||||
}}, nil
|
||||
}
|
||||
return []finding{{
|
||||
level: daemonapi.DiagnosisLevelOK,
|
||||
code: codeUndeployed,
|
||||
message: "全新机器，尚未部署 backend 容器",
|
||||
}}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user