草稿
[WIP]perf(vmm): redirect stratovirt output to log and reduce unnecessary ResumeVM call #172
ytluo创建于 8月19日
[WIP]perf(vmm): redirect stratovirt output to log and reduce unnecessary ResumeVM call #172
草稿
ytluo创建于 8月19日
共 7 个文件变更+209-12
@@ -22,6 +22,13 @@ const (
22 defaultStateDir = "/var/lib/conch"22 defaultStateDir = "/var/lib/conch"
23)23)
24 24 
25+const (
26+ // DefaultLogDir is the fixed root for logs owned by Conch components.
27+ DefaultLogDir = "/var/log/conchd"
28+ // StratovirtLogDir stores per-sandbox StratoVirt logs under the Conch log root.
29+ StratovirtLogDir = DefaultLogDir + "/stratovirt"
30+)
31+ 
25// Config holds the application configuration32// Config holds the application configuration
26type Config struct {33type Config struct {
27 App AppConfig `yaml:"app"`34 App AppConfig `yaml:"app"`
@@ -336,10 +343,10 @@ func (c *Config) GetLogConfig() (ulog.Config, error) {
336 outputPath = "" // No file output343 outputPath = "" // No file output
337 case "file":344 case "file":
338 stdout = false345 stdout = false
339- outputPath = "/var/log/conchd/"346+ outputPath = DefaultLogDir
340 case "both":347 case "both":
341 stdout = true348 stdout = true
342- outputPath = "/var/log/conchd/"349+ outputPath = DefaultLogDir
343 default:350 default:
344 return ulog.Config{}, fmt.Errorf("invalid log output mode: %s (must be stdout, file, or both)", c.Log.Output)351 return ulog.Config{}, fmt.Errorf("invalid log output mode: %s (must be stdout, file, or both)", c.Log.Output)
345 }352 }
@@ -35,7 +35,7 @@ func TestGetLogConfig(t *testing.T) {
35 Log: LogConfig{Level: "debug", Output: "file"},35 Log: LogConfig{Level: "debug", Output: "file"},
36 },36 },
37 wantStdout: false,37 wantStdout: false,
38- wantPath: "/var/log/conchd/",38+ wantPath: DefaultLogDir,
39 },39 },
40 {40 {
41 name: "both mode",41 name: "both mode",
@@ -43,7 +43,7 @@ func TestGetLogConfig(t *testing.T) {
43 Log: LogConfig{Level: "warn", Output: "both"},43 Log: LogConfig{Level: "warn", Output: "both"},
44 },44 },
45 wantStdout: true,45 wantStdout: true,
46- wantPath: "/var/log/conchd/",46+ wantPath: DefaultLogDir,
47 },47 },
48 {48 {
49 name: "invalid mode",49 name: "invalid mode",
@@ -464,6 +464,9 @@ func TestDefaultConfigRuntimePaths(t *testing.T) {
464 if cfg.Sandbox.CloudHypervisor != nil || cfg.Sandbox.Stratovirt == nil || cfg.Sandbox.Stratovirt.Binary != "/usr/bin/stratovirt" {464 if cfg.Sandbox.CloudHypervisor != nil || cfg.Sandbox.Stratovirt == nil || cfg.Sandbox.Stratovirt.Binary != "/usr/bin/stratovirt" {
465 t.Errorf("DefaultConfig().Sandbox.Stratovirt = %#v, want /usr/bin/stratovirt", cfg.Sandbox.Stratovirt)465 t.Errorf("DefaultConfig().Sandbox.Stratovirt = %#v, want /usr/bin/stratovirt", cfg.Sandbox.Stratovirt)
466 }466 }
467+ if StratovirtLogDir != "/var/log/conchd/stratovirt" {
468+ t.Errorf("StratovirtLogDir = %q, want /var/log/conchd/stratovirt", StratovirtLogDir)
469+ }
467 if cfg.Sandbox.Backend != DefaultSandboxBackend {470 if cfg.Sandbox.Backend != DefaultSandboxBackend {
468 t.Errorf("DefaultConfig().Sandbox.Backend = %q, want %q", cfg.Sandbox.Backend, DefaultSandboxBackend)471 t.Errorf("DefaultConfig().Sandbox.Backend = %q, want %q", cfg.Sandbox.Backend, DefaultSandboxBackend)
469 }472 }
@@ -117,6 +117,7 @@ func RestoreSandbox(
117 VirtioFS: append([]driver.VirtioFSDevice(nil), vmStartSpec.VirtioFS...),117 VirtioFS: append([]driver.VirtioFSDevice(nil), vmStartSpec.VirtioFS...),
118 VsockCID: vsockCID,118 VsockCID: vsockCID,
119 VsockSocketPath: vsockSocketPath,119 VsockSocketPath: vsockSocketPath,
120+ SandboxId: sandboxId,
120 }121 }
121 122 
122 vmmHandle, vmmErr := vmm.NewProcess(123 vmmHandle, vmmErr := vmm.NewProcess(
@@ -50,6 +50,7 @@ func SandboxSocketPath(subDir, sandboxId string) (string, error) {
50 50 
51type Process struct {51type Process struct {
52 cmd *exec.Cmd52 cmd *exec.Cmd
53+ vmmName string
53 SandboxId string54 SandboxId string
54 VmmSocketPath string55 VmmSocketPath string
55 VsockSocketPath string56 VsockSocketPath string
@@ -101,6 +102,7 @@ func NewProcess(
101 }102 }
102 103 
103 p := Process{104 p := Process{
105+ vmmName: vmmName,
104 SandboxId: sandboxId,106 SandboxId: sandboxId,
105 VsockSocketPath: vmmResourceArgs.VsockSocketPath,107 VsockSocketPath: vmmResourceArgs.VsockSocketPath,
106 VmmSocketPath: vmmSocketPath,108 VmmSocketPath: vmmSocketPath,
@@ -143,10 +145,12 @@ func (p *Process) startCmd(
143 ctx context.Context,145 ctx context.Context,
144) error {146) error {
145 logger := ulog.GetLogger()147 logger := ulog.GetLogger()
146- 148+ if p.cmd.Stdout == nil {
147- // TODO: redirect stderr/stdout149+ p.cmd.Stdout = os.Stdout
148- p.cmd.Stdout = os.Stdout150+ }
149- p.cmd.Stderr = os.Stderr151+ if p.cmd.Stderr == nil {
152+ p.cmd.Stderr = os.Stderr
153+ }
150 154 
151 logger.Debug("Starting VMM process")155 logger.Debug("Starting VMM process")
152 err := p.cmd.Start()156 err := p.cmd.Start()
@@ -161,7 +165,6 @@ func (p *Process) startCmd(
161 p.adapter.AfterProcessStart()165 p.adapter.AfterProcessStart()
162 166 
163 go func() {167 go func() {
164- // TODO: close fd after redirecting stderr/stdout
165 waitErr := p.cmd.Wait()168 waitErr := p.cmd.Wait()
166 if waitErr != nil {169 if waitErr != nil {
167 var exitErr *exec.ExitError170 var exitErr *exec.ExitError
@@ -247,7 +250,9 @@ func (p *Process) Restore(ctx context.Context, snapshotPath string) error {
247 }250 }
248 p.markAPIReady()251 p.markAPIReady()
249 252 
250- err = p.adapter.ResumeVM()253+ if p.vmmName != StratovirtName {
254+ err = p.adapter.ResumeVM()
255+ }
251 if err != nil {256 if err != nil {
252 vmmStopErr := p.Stop()257 vmmStopErr := p.Stop()
253 return errors.Join(fmt.Errorf("error resuming vm: %w", err), vmmStopErr)258 return errors.Join(fmt.Errorf("error resuming vm: %w", err), vmmStopErr)
@@ -140,6 +140,75 @@ func (c *blockingDaemonClient) WaitForRestoreReady(context.Context, <-chan error
140}140}
141func (c *blockingDaemonClient) Cleanup() { c.cleanupCalls.Add(1) }141func (c *blockingDaemonClient) Cleanup() { c.cleanupCalls.Add(1) }
142 142 
143+type restoreTrackingAdapter struct {
144+ restoreReadyCalls atomic.Int32
145+ loadCalls atomic.Int32
146+ resumeCalls atomic.Int32
147+ checkCalls atomic.Int32
148+}
149+ 
150+func (a *restoreTrackingAdapter) BuildStartCmd(*ResourceArgs, bool) (string, error) { return "", nil }
151+func (a *restoreTrackingAdapter) PrepareLaunch(*ResourceArgs, bool) error { return nil }
152+func (a *restoreTrackingAdapter) AfterProcessStart() {}
153+func (a *restoreTrackingAdapter) WaitForCreateReady(context.Context, <-chan error) error {
154+ return nil
155+}
156+func (a *restoreTrackingAdapter) WaitForRestoreReady(context.Context, <-chan error) error {
157+ a.restoreReadyCalls.Add(1)
158+ return nil
159+}
160+func (a *restoreTrackingAdapter) CheckAgentAlive(context.Context, <-chan error) error {
161+ a.checkCalls.Add(1)
162+ return nil
163+}
164+func (a *restoreTrackingAdapter) PauseVM() error { return nil }
165+func (a *restoreTrackingAdapter) ResumeVM() error { a.resumeCalls.Add(1); return nil }
166+func (a *restoreTrackingAdapter) DeleteVM() error { return nil }
167+func (a *restoreTrackingAdapter) CreateSnapshot(string) error { return nil }
168+func (a *restoreTrackingAdapter) LoadSnapshot(string, bool) error {
169+ a.loadCalls.Add(1)
170+ return nil
171+}
172+func (a *restoreTrackingAdapter) Cleanup() {}
173+ 
174+func TestRestoreUsesBackendSpecificResumePolicy(t *testing.T) {
175+ for _, tt := range []struct {
176+ name string
177+ vmmName string
178+ wantResumeCalls int32
179+ }{
180+ {name: "stratovirt relies on status check", vmmName: StratovirtName, wantResumeCalls: 0},
181+ {name: "cloud hypervisor resumes explicitly", vmmName: CloudHypervisorName, wantResumeCalls: 1},
182+ } {
183+ t.Run(tt.name, func(t *testing.T) {
184+ adapter := &restoreTrackingAdapter{}
185+ process := &Process{
186+ cmd: exec.Command("true"),
187+ vmmName: tt.vmmName,
188+ SandboxId: "sandbox-test",
189+ adapter: adapter,
190+ exitSignal: make(chan error, 1),
191+ }
192+ 
193+ if err := process.Restore(context.Background(), "/tmp/snapshot"); err != nil {
194+ t.Fatalf("Restore() error = %v", err)
195+ }
196+ if got := adapter.resumeCalls.Load(); got != tt.wantResumeCalls {
197+ t.Fatalf("ResumeVM() calls = %d, want %d", got, tt.wantResumeCalls)
198+ }
199+ if got := adapter.restoreReadyCalls.Load(); got != 1 {
200+ t.Fatalf("WaitForRestoreReady() calls = %d, want 1", got)
201+ }
202+ if got := adapter.loadCalls.Load(); got != 1 {
203+ t.Fatalf("LoadSnapshot() calls = %d, want 1", got)
204+ }
205+ if got := adapter.checkCalls.Load(); got != 1 {
206+ t.Fatalf("CheckAgentAlive() calls = %d, want 1", got)
207+ }
208+ })
209+ }
210+}
211+ 
143func TestStopIgnoresProcessDoneWhenProcessAlreadyFinished(t *testing.T) {212func TestStopIgnoresProcessDoneWhenProcessAlreadyFinished(t *testing.T) {
144 cmd := exec.Command("true")213 cmd := exec.Command("true")
145 if err := cmd.Start(); err != nil {214 if err := cmd.Start(); err != nil {
@@ -9,12 +9,14 @@ import (
9 "net"9 "net"
10 "os"10 "os"
11 "os/exec"11 "os/exec"
12+ "path/filepath"
12 "runtime"13 "runtime"
13 "strconv"14 "strconv"
14 "strings"15 "strings"
15 "text/template"16 "text/template"
16 "time"17 "time"
17 18 
19+ "github.com/openeuler/Conch/internal/config"
18 "github.com/openeuler/Conch/internal/vmm/driver"20 "github.com/openeuler/Conch/internal/vmm/driver"
19 "github.com/openeuler/Conch/pkg/ulog"21 "github.com/openeuler/Conch/pkg/ulog"
20)22)
@@ -56,7 +58,7 @@ const startScriptStratovirt = `{{ .NSenterPath }} --net={{ .NetNSPath }} -- \
56{{ .VirtioFSDevices }} \58{{ .VirtioFSDevices }} \
57{{ .PmemDevices }} \59{{ .PmemDevices }} \
58-device vhost-vsock-pci,id=vsock0,guest-cid={{ .VsockCID }},bus=pcie.0,addr=0x11 \60-device vhost-vsock-pci,id=vsock0,guest-cid={{ .VsockCID }},bus=pcie.0,addr=0x11 \
59--disable-seccomp`61+-disable-seccomp > "{{ .LogPath }}" 2>&1`
J
Jjing-rui8月20日

stratovirt有专门的日志配置参数,应当使用官方用法: -D [log path] output log to logfile (default stderr)

likedislike
60 62 
61const restoreScriptStratovirt = `{{ .NSenterPath }} --net={{ .NetNSPath }} -- \63const restoreScriptStratovirt = `{{ .NSenterPath }} --net={{ .NetNSPath }} -- \
62{{ .VmmBinaryPath }} \64{{ .VmmBinaryPath }} \
@@ -73,7 +75,7 @@ const restoreScriptStratovirt = `{{ .NSenterPath }} --net={{ .NetNSPath }} -- \
73{{ .PmemDevices }} \75{{ .PmemDevices }} \
74-device vhost-vsock-pci,id=vsock0,guest-cid={{ .VsockCID }},bus=pcie.0,addr=0x11 \76-device vhost-vsock-pci,id=vsock0,guest-cid={{ .VsockCID }},bus=pcie.0,addr=0x11 \
75-disable-seccomp \77-disable-seccomp \
76--incoming file:{{ .SnapfilePath }},mapped=true`78+-incoming file:{{ .SnapfilePath }},mapped=true > "{{ .LogPath }}" 2>&1`
77 79 
78type StartScriptStratovirtArgs struct {80type StartScriptStratovirtArgs struct {
79 NSenterPath string81 NSenterPath string
@@ -91,6 +93,7 @@ type StartScriptStratovirtArgs struct {
91 SerialSocket string93 SerialSocket string
92 SnapfilePath string94 SnapfilePath string
93 SandboxId string95 SandboxId string
96+ LogPath string
94 VsockCID uint3297 VsockCID uint32
95 PmemDevices string98 PmemDevices string
96 VirtioFSDevices string99 VirtioFSDevices string
@@ -105,6 +108,7 @@ type StratovirtClient struct {
105 vmmType int108 vmmType int
106 socketPath string109 socketPath string
107 vmmBinary string110 vmmBinary string
111+ logDir string
108}112}
109 113 
110func NewStratovirtClient(vmmType int, socketPath, vmmBinary string) *StratovirtClient {114func NewStratovirtClient(vmmType int, socketPath, vmmBinary string) *StratovirtClient {
@@ -112,6 +116,7 @@ func NewStratovirtClient(vmmType int, socketPath, vmmBinary string) *StratovirtC
112 vmmType: vmmType,116 vmmType: vmmType,
113 socketPath: socketPath,117 socketPath: socketPath,
114 vmmBinary: vmmBinary,118 vmmBinary: vmmBinary,
119+ logDir: config.StratovirtLogDir,
115 }120 }
116}121}
117 122 
@@ -167,6 +172,10 @@ func waitForVmmSocket(ctx context.Context, socketPath string, processExited <-ch
167 172 
168func (s *StratovirtClient) BuildStartCmd(args *driver.ResourceArgs, restore bool) (string, error) {173func (s *StratovirtClient) BuildStartCmd(args *driver.ResourceArgs, restore bool) (string, error) {
169 logger := ulog.GetLogger()174 logger := ulog.GetLogger()
175+ sandboxLogDir := filepath.Join(s.logDir, args.SandboxId)
176+ if err := os.MkdirAll(sandboxLogDir, 0o755); err != nil {
177+ return "", fmt.Errorf("create Stratovirt log directory: %w", err)
178+ }
170 nsenterPath, err := exec.LookPath("nsenter")179 nsenterPath, err := exec.LookPath("nsenter")
171 if err != nil {180 if err != nil {
172 return "", fmt.Errorf("resolve nsenter binary: %w", err)181 return "", fmt.Errorf("resolve nsenter binary: %w", err)
@@ -199,6 +208,7 @@ func (s *StratovirtClient) BuildStartCmd(args *driver.ResourceArgs, restore bool
199 SerialSocket: s.socketPath + ".serial",208 SerialSocket: s.socketPath + ".serial",
200 SnapfilePath: args.SnapfilePath,209 SnapfilePath: args.SnapfilePath,
201 SandboxId: args.SandboxId,210 SandboxId: args.SandboxId,
211+ LogPath: filepath.Join(sandboxLogDir, "vmm.log"),
202 VsockCID: args.VsockCID,212 VsockCID: args.VsockCID,
203 PmemDevices: buildStratovirtPmemDevices(args.PmemPaths),213 PmemDevices: buildStratovirtPmemDevices(args.PmemPaths),
204 VirtioFSDevices: buildStratovirtVirtioFSDevices(args.VirtioFS, len(args.PmemPaths)),214 VirtioFSDevices: buildStratovirtVirtioFSDevices(args.VirtioFS, len(args.PmemPaths)),
@@ -1,8 +1,12 @@
1package stratovirt1package stratovirt
2 2 
3import (3import (
4+ "bufio"
4 "context"5 "context"
6+ "encoding/json"
5 "errors"7 "errors"
8+ "fmt"
9+ "net"
6 "os"10 "os"
7 "path/filepath"11 "path/filepath"
8 "strings"12 "strings"
@@ -12,6 +16,101 @@ import (
12 "github.com/openeuler/Conch/internal/vmm/driver"16 "github.com/openeuler/Conch/internal/vmm/driver"
13)17)
14 18 
19+func startQMPStatusServer(t *testing.T, socketPath string, statuses []string) <-chan string {
20+ t.Helper()
21+ if len(statuses) == 0 {
22+ t.Fatal("startQMPStatusServer requires at least one status")
23+ }
24+ listener, err := net.Listen("unix", socketPath)
25+ if err != nil {
26+ t.Fatalf("Listen() error = %v", err)
27+ }
28+ t.Cleanup(func() { _ = listener.Close() })
29+ 
30+ commands := make(chan string, 8)
31+ go func() {
32+ statusIndex := 0
33+ for {
34+ conn, err := listener.Accept()
35+ if err != nil {
36+ return
37+ }
38+ reader := bufio.NewReader(conn)
39+ _, _ = fmt.Fprintln(conn, `{"QMP":{"version":{}}}`)
40+ _, _ = reader.ReadString('\n')
41+ _, _ = fmt.Fprintln(conn, `{"return":{}}`)
42+ 
43+ line, readErr := reader.ReadString('\n')
44+ if readErr != nil {
45+ _ = conn.Close()
46+ continue
47+ }
48+ var request struct {
49+ Execute string `json:"execute"`
50+ }
51+ if err := json.Unmarshal([]byte(line), &request); err != nil {
52+ _ = conn.Close()
53+ continue
54+ }
55+ commands <- request.Execute
56+ if request.Execute == "query-status" {
57+ status := statuses[len(statuses)-1]
58+ if statusIndex < len(statuses) {
59+ status = statuses[statusIndex]
60+ statusIndex++
61+ }
62+ _, _ = fmt.Fprintf(conn, "{\"return\":{\"status\":%q,\"running\":%t}}\n", status, status == "running")
63+ } else {
64+ _, _ = fmt.Fprintln(conn, `{"return":{}}`)
65+ }
66+ _ = conn.Close()
67+ }
68+ }()
69+ return commands
70+}
71+ 
72+func TestCheckAgentAliveDoesNotResumeRunningVM(t *testing.T) {
73+ socketPath := filepath.Join(t.TempDir(), "qmp.sock")
74+ commands := startQMPStatusServer(t, socketPath, []string{"running"})
75+ client := NewStratovirtClient(1, socketPath, "/opt/vmm/stratovirt")
76+ 
77+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
78+ defer cancel()
79+ if err := client.CheckAgentAlive(ctx, make(chan error)); err != nil {
80+ t.Fatalf("CheckAgentAlive() error = %v", err)
81+ }
82+ if got := <-commands; got != "query-status" {
83+ t.Fatalf("first QMP command = %q, want query-status", got)
84+ }
85+ select {
86+ case got := <-commands:
87+ t.Fatalf("unexpected QMP command %q", got)
88+ default:
89+ }
90+}
91+ 
92+func TestCheckAgentAliveResumesPausedVMOnce(t *testing.T) {
93+ socketPath := filepath.Join(t.TempDir(), "qmp.sock")
94+ commands := startQMPStatusServer(t, socketPath, []string{"paused", "running"})
95+ client := NewStratovirtClient(1, socketPath, "/opt/vmm/stratovirt")
96+ 
97+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
98+ defer cancel()
99+ if err := client.CheckAgentAlive(ctx, make(chan error)); err != nil {
100+ t.Fatalf("CheckAgentAlive() error = %v", err)
101+ }
102+ for i, want := range []string{"query-status", "cont", "query-status"} {
103+ if got := <-commands; got != want {
104+ t.Fatalf("QMP command %d = %q, want %q", i, got, want)
105+ }
106+ }
107+ select {
108+ case got := <-commands:
109+ t.Fatalf("unexpected QMP command %q", got)
110+ default:
111+ }
112+}
113+ 
15func TestStratovirtBuildStartCmd(t *testing.T) {114func TestStratovirtBuildStartCmd(t *testing.T) {
16 configuredDir := t.TempDir()115 configuredDir := t.TempDir()
17 binPath := filepath.Join(configuredDir, "stratovirt")116 binPath := filepath.Join(configuredDir, "stratovirt")
@@ -26,6 +125,7 @@ func TestStratovirtBuildStartCmd(t *testing.T) {
26 t.Setenv("PATH", pathDir+string(os.PathListSeparator)+os.Getenv("PATH"))125 t.Setenv("PATH", pathDir+string(os.PathListSeparator)+os.Getenv("PATH"))
27 126 
28 client := NewStratovirtClient(1, "/tmp/conch-qmp.sock", binPath)127 client := NewStratovirtClient(1, "/tmp/conch-qmp.sock", binPath)
128+ client.logDir = t.TempDir()
29 script, err := client.BuildStartCmd(&driver.ResourceArgs{129 script, err := client.BuildStartCmd(&driver.ResourceArgs{
30 CPUBoot: 2,130 CPUBoot: 2,
31 CPUMax: 4,131 CPUMax: 4,
@@ -49,6 +149,7 @@ func TestStratovirtBuildStartCmd(t *testing.T) {
49 "-device vhost-vsock-pci,id=vsock0,guest-cid=42",149 "-device vhost-vsock-pci,id=vsock0,guest-cid=42",
50 "conch.sandbox_id=sandbox-test",150 "conch.sandbox_id=sandbox-test",
51 "-m 1024M",151 "-m 1024M",
152+ filepath.Join(client.logDir, "sandbox-test", "vmm.log"),
52 } {153 } {
53 if !strings.Contains(script, want) {154 if !strings.Contains(script, want) {
54 t.Fatalf("script missing %q:\n%s", want, script)155 t.Fatalf("script missing %q:\n%s", want, script)
@@ -73,6 +174,7 @@ func TestStratovirtBuildRestoreCmdUsesMappedCheckpoint(t *testing.T) {
73 t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))174 t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
74 175 
75 client := NewStratovirtClient(1, "/tmp/conch-qmp.sock", binPath)176 client := NewStratovirtClient(1, "/tmp/conch-qmp.sock", binPath)
177+ client.logDir = t.TempDir()
76 script, err := client.BuildStartCmd(&driver.ResourceArgs{178 script, err := client.BuildStartCmd(&driver.ResourceArgs{
77 CPUBoot: 1,179 CPUBoot: 1,
78 CPUMax: 1,180 CPUMax: 1,