From d1b1b673a0669b17eb48d2f947dc1a5298ec2275 Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Thu, 2 Jul 2026 10:35:48 +0530 Subject: [PATCH 1/2] Add LCOW live migration source and destination pathways Introduce an end-to-end live-migration workflow for LCOW sandboxes, letting a running sandbox be handed off from a source shim to a destination shim. - Add a migration Controller that sequences a single session through a source or destination lifecycle: the source prepares and exports an opaque sandbox snapshot (VM + per-pod state), while the destination imports it, patches each container onto its new IDs, builds the destination VM, transfers memory over a shared socket, and finalizes with a resume or stop. - Expose the migration gRPC/ttrpc surface on the LCOW shim service (PrepareAndExportSandbox, ImportSandbox, PrepareSandbox, TransferSandbox, FinalizeSandbox, Cancel, Cleanup, CreateDuplicateSocket, Notifications) and register it alongside the task and sandbox services. - Wire migration into the task lifecycle: route destination-side CreateTask for rehydrated containers into a patch path, and reject task mutations (state, delete, kill) while a session is in progress. - Add duplicated transport-socket adoption, a subscriber-based progress notification stream, and proto/HCS conversion helpers for migration options and events. Signed-off-by: Harsh Rawat --- .../service/mocks/mock_service.go | 144 +++++ .../service/service.go | 16 + .../service/service_migration.go | 196 +++++++ .../service/service_migration_internal.go | 238 ++++++++ .../service/service_sandbox_internal_test.go | 2 + .../service/service_task_internal.go | 31 +- .../service/service_task_internal_test.go | 26 +- cmd/containerd-shim-lcow-v2/service/types.go | 36 ++ internal/controller/migration/controller.go | 417 ++++++++++++++ .../migration/controller_destination.go | 263 +++++++++ .../migration/controller_destination_test.go | 414 ++++++++++++++ .../controller/migration/controller_source.go | 126 +++++ .../migration/controller_source_test.go | 279 +++++++++ .../controller/migration/controller_test.go | 534 ++++++++++++++++++ internal/controller/migration/doc.go | 100 ++++ .../migration/mocks/mock_lcow_migration.go | 337 +++++++++++ .../controller/migration/notifications.go | 214 +++++++ .../migration/notifications_test.go | 329 +++++++++++ internal/controller/migration/socket.go | 114 ++++ internal/controller/migration/socket_test.go | 68 +++ internal/controller/migration/state.go | 116 ++++ internal/controller/migration/types.go | 90 +++ pkg/migration/parse.go | 102 ++++ pkg/migration/parse_test.go | 310 ++++++++++ 24 files changed, 4475 insertions(+), 27 deletions(-) create mode 100644 cmd/containerd-shim-lcow-v2/service/service_migration.go create mode 100644 cmd/containerd-shim-lcow-v2/service/service_migration_internal.go create mode 100644 internal/controller/migration/controller.go create mode 100644 internal/controller/migration/controller_destination.go create mode 100644 internal/controller/migration/controller_destination_test.go create mode 100644 internal/controller/migration/controller_source.go create mode 100644 internal/controller/migration/controller_source_test.go create mode 100644 internal/controller/migration/controller_test.go create mode 100644 internal/controller/migration/doc.go create mode 100644 internal/controller/migration/mocks/mock_lcow_migration.go create mode 100644 internal/controller/migration/notifications.go create mode 100644 internal/controller/migration/notifications_test.go create mode 100644 internal/controller/migration/socket.go create mode 100644 internal/controller/migration/socket_test.go create mode 100644 internal/controller/migration/state.go create mode 100644 internal/controller/migration/types.go create mode 100644 pkg/migration/parse_test.go diff --git a/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go b/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go index 7ce70ad64c..f3b20c371f 100644 --- a/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go +++ b/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go @@ -26,6 +26,7 @@ import ( process "github.com/Microsoft/hcsshim/internal/controller/process" vm "github.com/Microsoft/hcsshim/internal/controller/vm" schema2 "github.com/Microsoft/hcsshim/internal/hcs/schema2" + v2 "github.com/Microsoft/hcsshim/internal/hcs/v2" guestresource "github.com/Microsoft/hcsshim/internal/protocol/guestresource" shimdiag "github.com/Microsoft/hcsshim/internal/shimdiag" guestmanager "github.com/Microsoft/hcsshim/internal/vm/guestmanager" @@ -34,6 +35,7 @@ import ( task "github.com/containerd/containerd/api/types/task" specs_go "github.com/opencontainers/runtime-spec/specs-go" gomock "go.uber.org/mock/gomock" + anypb "google.golang.org/protobuf/types/known/anypb" ) // MockvmController is a mock of vmController interface. @@ -119,6 +121,20 @@ func (mr *MockvmControllerMockRecorder) ExitStatus() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExitStatus", reflect.TypeOf((*MockvmController)(nil).ExitStatus)) } +// FinalizeLiveMigration mocks base method. +func (m *MockvmController) FinalizeLiveMigration(ctx context.Context, options *schema2.MigrationFinalizedOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinalizeLiveMigration", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// FinalizeLiveMigration indicates an expected call of FinalizeLiveMigration. +func (mr *MockvmControllerMockRecorder) FinalizeLiveMigration(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinalizeLiveMigration", reflect.TypeOf((*MockvmController)(nil).FinalizeLiveMigration), ctx, options) +} + // Guest mocks base method. func (m *MockvmController) Guest() *guestmanager.Guest { m.ctrl.T.Helper() @@ -133,6 +149,49 @@ func (mr *MockvmControllerMockRecorder) Guest() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Guest", reflect.TypeOf((*MockvmController)(nil).Guest)) } +// Import mocks base method. +func (m *MockvmController) Import(ctx context.Context, env *anypb.Any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Import", ctx, env) + ret0, _ := ret[0].(error) + return ret0 +} + +// Import indicates an expected call of Import. +func (mr *MockvmControllerMockRecorder) Import(ctx, env any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Import", reflect.TypeOf((*MockvmController)(nil).Import), ctx, env) +} + +// InitializeLiveMigrationOnSource mocks base method. +func (m *MockvmController) InitializeLiveMigrationOnSource(ctx context.Context, options *schema2.MigrationInitializeOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitializeLiveMigrationOnSource", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitializeLiveMigrationOnSource indicates an expected call of InitializeLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) InitializeLiveMigrationOnSource(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitializeLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).InitializeLiveMigrationOnSource), ctx, options) +} + +// MigrationNotifications mocks base method. +func (m *MockvmController) MigrationNotifications() (<-chan schema2.OperationSystemMigrationNotificationInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MigrationNotifications") + ret0, _ := ret[0].(<-chan schema2.OperationSystemMigrationNotificationInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MigrationNotifications indicates an expected call of MigrationNotifications. +func (mr *MockvmControllerMockRecorder) MigrationNotifications() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MigrationNotifications", reflect.TypeOf((*MockvmController)(nil).MigrationNotifications)) +} + // NetworkController mocks base method. func (m *MockvmController) NetworkController(networkNamespaceID string) *network.Controller { m.ctrl.T.Helper() @@ -147,6 +206,20 @@ func (mr *MockvmControllerMockRecorder) NetworkController(networkNamespaceID any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkController", reflect.TypeOf((*MockvmController)(nil).NetworkController), networkNamespaceID) } +// Patch mocks base method. +func (m *MockvmController) Patch(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Patch", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockvmControllerMockRecorder) Patch(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockvmController)(nil).Patch), ctx) +} + // Plan9Controller mocks base method. func (m *MockvmController) Plan9Controller() *plan9.Controller { m.ctrl.T.Helper() @@ -161,6 +234,20 @@ func (mr *MockvmControllerMockRecorder) Plan9Controller() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Plan9Controller", reflect.TypeOf((*MockvmController)(nil).Plan9Controller)) } +// Resume mocks base method. +func (m *MockvmController) Resume(ctx context.Context, rebuildBridge bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resume", ctx, rebuildBridge) + ret0, _ := ret[0].(error) + return ret0 +} + +// Resume indicates an expected call of Resume. +func (mr *MockvmControllerMockRecorder) Resume(ctx, rebuildBridge any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resume", reflect.TypeOf((*MockvmController)(nil).Resume), ctx, rebuildBridge) +} + // RuntimeID mocks base method. func (m *MockvmController) RuntimeID() string { m.ctrl.T.Helper() @@ -204,6 +291,49 @@ func (mr *MockvmControllerMockRecorder) SandboxOptions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SandboxOptions", reflect.TypeOf((*MockvmController)(nil).SandboxOptions)) } +// Save mocks base method. +func (m *MockvmController) Save(ctx context.Context) (*anypb.Any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", ctx) + ret0, _ := ret[0].(*anypb.Any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Save indicates an expected call of Save. +func (mr *MockvmControllerMockRecorder) Save(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockvmController)(nil).Save), ctx) +} + +// StartLiveMigrationOnSource mocks base method. +func (m *MockvmController) StartLiveMigrationOnSource(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationOnSource", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationOnSource indicates an expected call of StartLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationOnSource(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationOnSource), ctx, config) +} + +// StartLiveMigrationTransfer mocks base method. +func (m *MockvmController) StartLiveMigrationTransfer(ctx context.Context, options *schema2.MigrationTransferOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationTransfer", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationTransfer indicates an expected call of StartLiveMigrationTransfer. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationTransfer(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationTransfer", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationTransfer), ctx, options) +} + // StartTime mocks base method. func (m *MockvmController) StartTime() time.Time { m.ctrl.T.Helper() @@ -232,6 +362,20 @@ func (mr *MockvmControllerMockRecorder) StartVM(ctx, opts any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartVM", reflect.TypeOf((*MockvmController)(nil).StartVM), ctx, opts) } +// StartWithMigrationOptions mocks base method. +func (m *MockvmController) StartWithMigrationOptions(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartWithMigrationOptions", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartWithMigrationOptions indicates an expected call of StartWithMigrationOptions. +func (mr *MockvmControllerMockRecorder) StartWithMigrationOptions(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartWithMigrationOptions", reflect.TypeOf((*MockvmController)(nil).StartWithMigrationOptions), ctx, config) +} + // State mocks base method. func (m *MockvmController) State() vm.State { m.ctrl.T.Helper() diff --git a/cmd/containerd-shim-lcow-v2/service/service.go b/cmd/containerd-shim-lcow-v2/service/service.go index 536295d04b..8135dc1e2d 100644 --- a/cmd/containerd-shim-lcow-v2/service/service.go +++ b/cmd/containerd-shim-lcow-v2/service/service.go @@ -7,11 +7,13 @@ import ( "fmt" "sync" + "github.com/Microsoft/hcsshim/internal/controller/migration" "github.com/Microsoft/hcsshim/internal/controller/pod" "github.com/Microsoft/hcsshim/internal/controller/vm" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/shim" "github.com/Microsoft/hcsshim/internal/shimdiag" + migrationsvc "github.com/Microsoft/hcsshim/pkg/migration" sandboxsvc "github.com/containerd/containerd/api/runtime/sandbox/v1" tasksvc "github.com/containerd/containerd/api/runtime/task/v3" @@ -54,6 +56,10 @@ type Service struct { // from podControllers. containerPodMapping map[string]string + // migrationController orchestrates the live-migration session for the + // sandbox. There is at most one active session per shim. + migrationController *migration.Controller + // shutdown manages graceful shutdown operations and allows registration of cleanup callbacks. shutdown shutdown.Service } @@ -68,6 +74,7 @@ func NewService(ctx context.Context, eventsPublisher shim.Publisher, sd shutdown vmController: vm.New(), podControllers: make(map[string]*pod.Controller), containerPodMapping: make(map[string]string), + migrationController: migration.New(), shutdown: sd, } @@ -95,6 +102,7 @@ func (s *Service) RegisterTTRPC(server *ttrpc.Server) error { tasksvc.RegisterTTRPCTaskService(server, s) sandboxsvc.RegisterTTRPCSandboxService(server, s) shimdiag.RegisterShimDiagService(server, s) + migrationsvc.RegisterMigrationService(server, s) return nil } @@ -106,6 +114,14 @@ func (s *Service) ensureVMRunning() error { return nil } +// ensureMigrationIdle returns an error if a migration session is in progress. +func (s *Service) ensureMigrationIdle() error { + if state := s.migrationController.State(); state != migration.StateIdle { + return fmt.Errorf("migration session is in progress (state: %s): %w", state, errdefs.ErrFailedPrecondition) + } + return nil +} + // SandboxID returns the unique identifier for the sandbox managed by this Service. func (s *Service) SandboxID() string { return s.sandboxID diff --git a/cmd/containerd-shim-lcow-v2/service/service_migration.go b/cmd/containerd-shim-lcow-v2/service/service_migration.go new file mode 100644 index 0000000000..dfc06e04e5 --- /dev/null +++ b/cmd/containerd-shim-lcow-v2/service/service_migration.go @@ -0,0 +1,196 @@ +//go:build windows && lcow + +package service + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/migration" + + "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" +) + +// Ensure Service implements the MigrationService interface at compile time. +var _ migration.MigrationService = &Service{} + +// PrepareAndExportSandbox prepares the source sandbox for live migration and +// exports an opaque config that the destination shim can use to import it. +// This method is part of the instrumentation layer and business logic is included in prepareAndExportSandboxInternal. +func (s *Service) PrepareAndExportSandbox(ctx context.Context, request *migration.PrepareAndExportSandboxRequest) (resp *migration.PrepareAndExportSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "PrepareAndExportSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.prepareAndExportSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// ImportSandbox imports a sandbox on the destination shim from the opaque +// config produced by PrepareAndExportSandbox on the source. +// This method is part of the instrumentation layer and business logic is included in importSandboxInternal. +func (s *Service) ImportSandbox(ctx context.Context, request *migration.ImportSandboxRequest) (resp *migration.ImportSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "ImportSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.importSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// PrepareSandbox prepares the destination-side compute system to receive the +// migrated sandbox state. +// This method is part of the instrumentation layer and business logic is included in prepareSandboxInternal. +func (s *Service) PrepareSandbox(ctx context.Context, request *migration.PrepareSandboxRequest) (resp *migration.PrepareSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "PrepareSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.prepareSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// TransferSandbox transfers sandbox state between source and destination +// over the previously established migration transport. +// This method is part of the instrumentation layer and business logic is included in transferSandboxInternal. +func (s *Service) TransferSandbox(ctx context.Context, request *migration.TransferSandboxRequest) (resp *migration.TransferSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "TransferSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + if request.Timeout != nil { + span.AddAttributes(trace.Int64Attribute(logfields.Timeout, int64(request.Timeout.AsDuration()))) + } + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.transferSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// FinalizeSandbox finalizes the migration on either side: stop on the +// source, resume on the destination (per the requested action). +// This method is part of the instrumentation layer and business logic is included in finalizeSandboxInternal. +func (s *Service) FinalizeSandbox(ctx context.Context, request *migration.FinalizeSandboxRequest) (resp *migration.FinalizeSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "FinalizeSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID), + trace.StringAttribute(logfields.Action, request.Action.String())) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.finalizeSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// Notifications streams migration progress notifications to the caller for +// the lifetime of the migration session. +// This method is part of the instrumentation layer and business logic is included in notificationsInternal. +func (s *Service) Notifications(ctx context.Context, request *migration.NotificationsRequest, server migration.Migration_NotificationsServer) (err error) { + ctx, span := oc.StartSpan(ctx, "Notifications") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + return errgrpc.ToGRPC(s.notificationsInternal(ctx, request, server)) +} + +// CreateDuplicateSocket duplicates a socket handle from the caller into the +// shim process for use as the migration transport. +// This method is part of the instrumentation layer and business logic is included in createDuplicateSocketInternal. +func (s *Service) CreateDuplicateSocket(ctx context.Context, request *migration.CreateDuplicateSocketRequest) (resp *migration.CreateDuplicateSocketResponse, err error) { + ctx, span := oc.StartSpan(ctx, "CreateDuplicateSocket") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.createDuplicateSocketInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// Cancel aborts an in-flight migration. It is mandatory on both source and +// destination to abort a migration that is already underway; it stops the +// in-flight transfer and closes the migration socket but leaves the shim in +// the migrating state until Cleanup is called. +// This method is part of the instrumentation layer and business logic is included in cancelInternal. +func (s *Service) Cancel(ctx context.Context, request *migration.CancelRequest) (resp *migration.CancelResponse, err error) { + ctx, span := oc.StartSpan(ctx, "Cancel") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.cancelInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// Cleanup is the terminal call of a migration on both sides, issued regardless +// of whether the migration succeeded, failed, or was cancelled. It reverts the +// migration controller's state machine back to the normal state. +// This method is part of the instrumentation layer and business logic is included in cleanupInternal. +func (s *Service) Cleanup(ctx context.Context, request *migration.CleanupRequest) (resp *migration.CleanupResponse, err error) { + ctx, span := oc.StartSpan(ctx, "Cleanup") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.cleanupInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} diff --git a/cmd/containerd-shim-lcow-v2/service/service_migration_internal.go b/cmd/containerd-shim-lcow-v2/service/service_migration_internal.go new file mode 100644 index 0000000000..f27ff69da7 --- /dev/null +++ b/cmd/containerd-shim-lcow-v2/service/service_migration_internal.go @@ -0,0 +1,238 @@ +//go:build windows && lcow + +package service + +import ( + "context" + "fmt" + "time" + + migrationcontroller "github.com/Microsoft/hcsshim/internal/controller/migration" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/pkg/migration" + + "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// prepareAndExportSandboxInternal is the implementation for PrepareAndExportSandbox. +// +// It initializes the source sandbox for migration and exports the +// opaque snapshot consumed by the destination shim's ImportSandbox API. +func (s *Service) prepareAndExportSandboxInternal(ctx context.Context, request *migration.PrepareAndExportSandboxRequest) (*migration.PrepareAndExportSandboxResponse, error) { + // Convert the protobuf migration options to the HCS representation. + migrationOpts, err := migration.InitializeOptionsFromProto(request.InitOptions) + if err != nil { + return nil, fmt.Errorf("convert migration initialize options: %w", err) + } + + // Arm the source for migration. + if err = s.migrationController.PrepareSource(ctx, + &migrationcontroller.PrepareSourceOptions{ + InitOptions: migrationcontroller.InitOptions{ + SessionID: request.SessionID, + Origin: hcsschema.MigrationOriginSource, + VMController: s.vmController, + PodControllers: s.podControllers, + }, + MigrationOpts: migrationOpts, + }); err != nil { + return nil, fmt.Errorf("source: prepare migration: %w", err) + } + + // Produce the opaque sandbox snapshot (VM plus per-pod state) that the + // destination shim consumes through ImportSandbox. + cfg, err := s.migrationController.ExportState(ctx, request.SessionID) + if err != nil { + return nil, fmt.Errorf("source: export migration state: %w", err) + } + + return &migration.PrepareAndExportSandboxResponse{Config: cfg}, nil +} + +// importSandboxInternal is the implementation for ImportSandbox. +// +// It rehydrates the destination shim from the source's opaque snapshot, +// mutating the Service-owned vm controller and pod controllers in place. +func (s *Service) importSandboxInternal(ctx context.Context, request *migration.ImportSandboxRequest) (*migration.ImportSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Rehydrate the destination from the source snapshot, recreating the VM + // and pod state and indexing each container to its owning pod. + if err := s.migrationController.ImportState(ctx, + &migrationcontroller.ImportStateOptions{ + InitOptions: migrationcontroller.InitOptions{ + SessionID: request.SessionID, + Origin: hcsschema.MigrationOriginDestination, + VMController: s.vmController, + PodControllers: s.podControllers, + }, + SandboxID: request.SandboxID, + SavedState: request.Config, + ContainerPodMapping: s.containerPodMapping, + }); err != nil { + return nil, fmt.Errorf("destination: import migration state: %w", err) + } + + s.sandboxID = request.SandboxID + return &migration.ImportSandboxResponse{}, nil +} + +// patchMigratedContainerInternal completes CreateTask for a container already +// rehydrated by migration: it rebinds the container's host-side resources to +// this shim and returns its existing init process PID without starting anything. +func (s *Service) patchMigratedContainerInternal( + ctx context.Context, + request *task.CreateTaskRequest, + spec specs.Spec, +) (*task.CreateTaskResponse, error) { + // Rebind the migrated container's host-side resources to this side's IDs. + err := s.migrationController.PatchResourcePaths(ctx, request, spec) + if err != nil { + return nil, fmt.Errorf("patch migrated container %q: %w", request.ID, err) + } + + ctrCtrl, err := getContainerController(s, request.ID) + if err != nil { + return nil, fmt.Errorf("lookup migrated container %q: %w", request.ID, err) + } + + initProc, err := ctrCtrl.GetProcess("") + if err != nil { + return nil, fmt.Errorf("get init process for migrated container %q: %w", request.ID, err) + } + + return &task.CreateTaskResponse{Pid: uint32(initProc.Pid())}, nil +} + +// prepareSandboxInternal is the implementation for PrepareSandbox. +// +// It creates the destination VM's HCS compute system and re-ACLs its disks so +// it is ready to start; all migrated containers must already be patched. +func (s *Service) prepareSandboxInternal(ctx context.Context, request *migration.PrepareSandboxRequest) (*migration.PrepareSandboxResponse, error) { + migrationOpts, err := migration.InitializeOptionsFromProto(request.InitOptions) + if err != nil { + return nil, fmt.Errorf("convert migration initialize options: %w", err) + } + + // Build the destination VM and prep its disks; every container must be patched first. + if err := s.migrationController.PrepareDestination( + ctx, + request.SessionID, + migrationOpts, + ); err != nil { + return nil, fmt.Errorf("destination: prepare migration: %w", err) + } + + return &migration.PrepareSandboxResponse{}, nil +} + +// transferSandboxInternal is the implementation for TransferSandbox. +// +// It drives the memory transfer between source and destination over the +// duplicated socket. +func (s *Service) transferSandboxInternal(ctx context.Context, request *migration.TransferSandboxRequest) (*migration.TransferSandboxResponse, error) { + var timeout time.Duration + if request.Timeout != nil { + timeout = request.Timeout.AsDuration() + } + + // Wait for the transport socket, then drive the memory transfer within the timeout. + if err := s.migrationController.Transfer( + ctx, + request.SessionID, + timeout, + ); err != nil { + return nil, fmt.Errorf("transfer migration state: %w", err) + } + return &migration.TransferSandboxResponse{}, nil +} + +// finalizeSandboxInternal is the implementation for FinalizeSandbox. +// +// It commits the migration's final outcome on each side per the requested +// action — resuming the sandbox back to running or stopping it — and ends the session. +func (s *Service) finalizeSandboxInternal(ctx context.Context, request *migration.FinalizeSandboxRequest) (*migration.FinalizeSandboxResponse, error) { + // Apply the requested resume/stop outcome and end the session. + if err := s.migrationController.Finalize(ctx, request.SessionID, request.Action, s.events); err != nil { + return nil, fmt.Errorf("finalize migration session: %w", err) + } + + return &migration.FinalizeSandboxResponse{}, nil +} + +// notificationsInternal is the implementation for Notifications. +// +// It forwards migration notifications to the calling stream until the session +// terminates or the client disconnects. +func (s *Service) notificationsInternal(ctx context.Context, request *migration.NotificationsRequest, server migration.Migration_NotificationsServer) error { + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Attach to this session's migration progress notifications. + ch, err := s.migrationController.Subscribe(subCtx, request.SessionID) + if err != nil { + return fmt.Errorf("subscribe to migration notifications: %w", err) + } + + logger := log.G(ctx).WithField(logfields.SessionID, request.SessionID) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case resp, ok := <-ch: + if !ok { + // Session terminated; close the stream cleanly. + return nil + } + + // Forward each notification to the client stream. + if err := server.Send(resp); err != nil { + logger.WithError(err).Warn("send migration notification failed") + return fmt.Errorf("send migration notification: %w", err) + } + } + } +} + +// createDuplicateSocketInternal is the implementation for CreateDuplicateSocket. +// +// It reconstructs the connected migration transport socket from the caller's +// duplicated protocol info and adopts it for the session, unblocking the transfer. +func (s *Service) createDuplicateSocketInternal(ctx context.Context, request *migration.CreateDuplicateSocketRequest) (*migration.CreateDuplicateSocketResponse, error) { + // Adopt the caller's duplicated socket as the transport and unblock the transfer. + if err := s.migrationController.RegisterDuplicateSocket(ctx, request.SessionID, request.ProtocolInfo); err != nil { + return nil, fmt.Errorf("register duplicate migration socket: %w", err) + } + + return &migration.CreateDuplicateSocketResponse{}, nil +} + +// cancelInternal is the implementation for Cancel. +// +// It aborts the in-flight transfer and closes the migration socket without +// reverting the controller state machine; Cleanup performs the final revert. +func (s *Service) cancelInternal(ctx context.Context, request *migration.CancelRequest) (*migration.CancelResponse, error) { + // Abort the in-flight transfer now; the session stays until cleanup. + if err := s.migrationController.Cancel(ctx, request.SessionID); err != nil { + return nil, fmt.Errorf("cancel migration session: %w", err) + } + + return &migration.CancelResponse{}, nil +} + +// cleanupInternal is the implementation for Cleanup. +// +// It reverts the migration controller state machine back to idle, resuming or +// aborting the underlying controllers based on the current migration state. +func (s *Service) cleanupInternal(ctx context.Context, request *migration.CleanupRequest) (*migration.CleanupResponse, error) { + // Tear down the session and revert this side back to idle. + if err := s.migrationController.Cleanup(ctx, request.SessionID, s.events); err != nil { + return nil, fmt.Errorf("cleanup migration session: %w", err) + } + + return &migration.CleanupResponse{}, nil +} diff --git a/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go b/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go index 0036831d12..29aecbf06b 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go +++ b/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go @@ -21,6 +21,7 @@ import ( "github.com/Microsoft/hcsshim/cmd/containerd-shim-lcow-v2/service/mocks" "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + "github.com/Microsoft/hcsshim/internal/controller/migration" "github.com/Microsoft/hcsshim/internal/controller/pod" "github.com/Microsoft/hcsshim/internal/controller/vm" ) @@ -48,6 +49,7 @@ func newTestService(t *testing.T) (*Service, *mocks.MockvmController) { events: make(chan interface{}, 128), podControllers: make(map[string]*pod.Controller), containerPodMapping: make(map[string]string), + migrationController: migration.New(), shutdown: sd, }, mockCtrl } diff --git a/cmd/containerd-shim-lcow-v2/service/service_task_internal.go b/cmd/containerd-shim-lcow-v2/service/service_task_internal.go index 402909898f..36a319b939 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_task_internal.go +++ b/cmd/containerd-shim-lcow-v2/service/service_task_internal.go @@ -75,6 +75,11 @@ func (s *Service) getPodController(podID string) (*pod.Controller, bool) { // stateInternal returns the current status of a process within a container. func (s *Service) stateInternal(_ context.Context, request *task.StateRequest) (*task.StateResponse, error) { + // Task queries are only valid while no migration session is in progress. + if err := s.ensureMigrationIdle(); err != nil { + return nil, err + } + // Look up the container controller for the requested container. ctrCtrl, err := getContainerController(s, request.ID) if err != nil { @@ -93,8 +98,8 @@ func (s *Service) stateInternal(_ context.Context, request *task.StateRequest) ( // createInternal creates a new pod sandbox or workload container based on the OCI spec. func (s *Service) createInternal(ctx context.Context, request *task.CreateTaskRequest) (*task.CreateTaskResponse, error) { - if err := s.ensureVMRunning(); err != nil { - return nil, err + if state := s.vmController.State(); state != vm.StateRunning && state != vm.StateMigratingImported { + return nil, fmt.Errorf("vm is not running or migrating (state: %s): %w", state, errdefs.ErrFailedPrecondition) } // Parse the OCI spec from the bundle. @@ -116,6 +121,14 @@ func (s *Service) createInternal(ctx context.Context, request *task.CreateTaskRe return nil, err } + // Live-migration destination path: the container is already rehydrated + // from the imported snapshot. Delegate to the migration helper, which + // patches host-side resource paths, fixes bookkeeping, and publishes the + // TaskCreate event without driving the creation/start lifecycle. + if _, ok := spec.Annotations[annotations.LiveMigrationSourceContainerID]; ok { + return s.patchMigratedContainerInternal(ctx, request, spec) + } + s.mu.Lock() defer s.mu.Unlock() @@ -279,6 +292,11 @@ func (s *Service) startInternal(ctx context.Context, request *task.StartRequest) // deleteInternal deletes a process, container, or pod sandbox depending on the request. func (s *Service) deleteInternal(ctx context.Context, request *task.DeleteRequest) (*task.DeleteResponse, error) { + // Deletion is only valid while no migration session is in progress. + if err := s.ensureMigrationIdle(); err != nil { + return nil, err + } + // Look up the container controller for the target ID. ctrCtrl, err := getContainerController(s, request.ID) if err != nil { @@ -401,7 +419,8 @@ func (s *Service) checkpointInternal(_ context.Context, _ *task.CheckpointTaskRe // killInternal sends a signal to a process or, when All is set, to every process in the pod. func (s *Service) killInternal(ctx context.Context, request *task.KillRequest) (*emptypb.Empty, error) { - if err := s.ensureVMRunning(); err != nil { + // Kill is only valid while no migration session is in progress. + if err := s.ensureMigrationIdle(); err != nil { return nil, err } @@ -608,10 +627,8 @@ func (s *Service) updateVMResources(ctx context.Context, resources interface{}, // waitInternal blocks until the specified process exits and returns its exit status. func (s *Service) waitInternal(ctx context.Context, request *task.WaitRequest) (*task.WaitResponse, error) { - if err := s.ensureVMRunning(); err != nil { - return nil, err - } - + // No ensureVMRunning: Wait/Status only read controller-cached state so + // they're safe even after the VM has been terminated. ctrCtrl, err := getContainerController(s, request.ID) if err != nil { return nil, fmt.Errorf("failed to find container for wait request: %w", err) diff --git a/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go b/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go index 92fbc132f7..a68f3985b1 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go +++ b/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go @@ -41,10 +41,11 @@ var ( // one representative not-running state (NotCreated); a regression in the guard // would let containerd issue task RPCs against a VM that has not booted. // -// state and delete are intentionally excluded: containerd issues them during -// task teardown, so they operate on container bookkeeping and surface NotFound -// when the container is absent (see TestTaskMethods_RejectUnknownContainer) -// rather than a precondition error. +// state, delete, kill, and wait are intentionally excluded: they omit the +// VM-running guard so they can run during task teardown / migration aborts, so +// they operate on container bookkeeping and surface NotFound when the container +// is absent (see TestTaskMethods_RejectUnknownContainer) rather than a +// precondition error. func TestTaskMethods_RejectVMNotRunning(t *testing.T) { tests := []struct { name string @@ -71,13 +72,6 @@ func TestTaskMethods_RejectVMNotRunning(t *testing.T) { return err }, }, - { - name: "killInternal", - call: func(svc *Service) error { - _, err := svc.killInternal(context.Background(), &task.KillRequest{ID: "ctr"}) - return err - }, - }, { name: "execInternal", call: func(svc *Service) error { @@ -106,13 +100,6 @@ func TestTaskMethods_RejectVMNotRunning(t *testing.T) { return err }, }, - { - name: "waitInternal", - call: func(svc *Service) error { - _, err := svc.waitInternal(context.Background(), &task.WaitRequest{ID: "ctr"}) - return err - }, - }, { name: "statsInternal", call: func(svc *Service) error { @@ -655,8 +642,7 @@ func TestDeleteProcess_Success(t *testing.T) { // TestKill_Success verifies the single-container kill path: killInternal // forwards the exec ID, signal, and all=false to the container controller. func TestKill_Success(t *testing.T) { - svc, mockVM := newTestService(t) - mockVM.EXPECT().State().Return(vm.StateRunning) + svc, _ := newTestService(t) mockCtr := mocks.NewMockcontainerController(gomock.NewController(t)) swapGetContainerController(t, mockCtr) diff --git a/cmd/containerd-shim-lcow-v2/service/types.go b/cmd/containerd-shim-lcow-v2/service/types.go index b393340e1c..785e1d96cf 100644 --- a/cmd/containerd-shim-lcow-v2/service/types.go +++ b/cmd/containerd-shim-lcow-v2/service/types.go @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim/internal/controller/process" "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/internal/vm/guestmanager" @@ -23,6 +24,7 @@ import ( "github.com/containerd/containerd/api/runtime/task/v3" containerdtypes "github.com/containerd/containerd/api/types/task" "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/protobuf/types/known/anypb" ) // vmController is the subset of the VM controller that [Service] depends on. @@ -98,6 +100,40 @@ type vmController interface { // NetworkController returns the network controller for the VM. NetworkController(networkNamespaceID string) *network.Controller + + // The methods below are required by the migration controller, which the + // Service hands this vmController to. Declaring them here lets the same + // field satisfy the migration controller's vmController interface. + + // InitializeLiveMigrationOnSource arms the running source VM for migration. + InitializeLiveMigrationOnSource(ctx context.Context, options *hcsschema.MigrationInitializeOptions) error + + // Save captures the migrating VM's state into a portable snapshot. + Save(ctx context.Context) (*anypb.Any, error) + + // Import rehydrates a destination VM controller from a saved snapshot. + Import(ctx context.Context, env *anypb.Any) error + + // Patch re-ACLs the destination VM's disks after creation. + Patch(ctx context.Context) error + + // StartLiveMigrationOnSource starts the outgoing migration on the source VM. + StartLiveMigrationOnSource(ctx context.Context, config *hcs.MigrationConfig) error + + // StartWithMigrationOptions starts the destination VM against the transport. + StartWithMigrationOptions(ctx context.Context, config *hcs.MigrationConfig) error + + // StartLiveMigrationTransfer begins the memory transfer. + StartLiveMigrationTransfer(ctx context.Context, options *hcsschema.MigrationTransferOptions) error + + // FinalizeLiveMigration applies the migration's final operation to the VM. + FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error + + // Resume returns a migrating VM to the running state. + Resume(ctx context.Context, rebuildBridge bool) error + + // MigrationNotifications returns the VM's migration event stream. + MigrationNotifications() (<-chan hcsschema.OperationSystemMigrationNotificationInfo, error) } // containerController is the subset of the container controller that [Service] diff --git a/internal/controller/migration/controller.go b/internal/controller/migration/controller.go new file mode 100644 index 0000000000..fdabe730de --- /dev/null +++ b/internal/controller/migration/controller.go @@ -0,0 +1,417 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "sync" + "syscall" + "time" + + "github.com/Microsoft/hcsshim/internal/controller/pod" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/pkg/migration" + "github.com/containerd/errdefs" + "golang.org/x/sys/windows" +) + +// defaultSocketReadyTimeout caps the Transfer wait for the duplicate socket. +const defaultSocketReadyTimeout = 10 * time.Minute + +// Controller sequences a single live-migration session for an LCOW sandbox, +// driving the source or destination side through its lifecycle states. +type Controller struct { + // mu guards all mutable fields below. + mu sync.RWMutex + + // state is the session's current lifecycle state. + state State + + // sessionID identifies the active migration session. + sessionID string + + // sandboxID is the sandbox this session migrates; set on the destination. + sandboxID string + + // origin is this host's role in the session: source or destination. + origin hcsschema.MigrationOrigin + + // vmController drives the VM being migrated; borrowed from the service. + vmController vmController + + // podControllers are the sandbox's pods keyed by pod ID; borrowed from the service. + podControllers map[string]*pod.Controller + + // containerPodMapping aliases the service-owned containerID -> podID + // index so ImportState and PatchResourcePaths can keep it in sync with + // pod/container renames. Guarded by mu. + containerPodMapping map[string]string + + // pendingPatches is the set of source container IDs imported by + // ImportState that still need a PatchResourcePaths call; a container + // drops out once it has been patched. Guarded by mu. + pendingPatches map[string]struct{} + + // dupSocket is the duplicated transport socket used for the memory transfer. + dupSocket windows.Handle + + // socketReady is closed by RegisterDuplicateSocket on transition to StateSocketReady. + socketReady chan struct{} + + // notifier is created lazily on the first Subscribe. Guarded by mu. + notifier *notifications +} + +// New returns an idle controller ready to host a migration session. +func New() *Controller { + return &Controller{ + state: StateIdle, + socketReady: make(chan struct{}), + } +} + +// State returns the current session state. +func (c *Controller) State() State { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.state +} + +// Transfer starts the session's memory transfer and returns immediately. The +// wait for the duplicate socket and the transfer itself run in the background; +// failures reach notification subscribers rather than the caller. Duplicate +// calls are no-ops. +func (c *Controller) Transfer(ctx context.Context, sessionID string, timeout time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Only one background goroutine should drive the transfer, so duplicate + // Transfer calls must be no-ops. If the duplicate socket has not arrived + // yet, the first call claims StateSocketWaiting and later calls return + // below. If the socket has already arrived, the state is left unchanged here + // and the goroutines dedup instead: only one goroutine moves + // StateSocketReady to StateTransferring and the rest bail. + switch c.state { + case StateSocketReady: + // Socket already arrived: fall through and start the transfer goroutine. + case StateSourceExported, StateDestinationPrepared: + // Socket not here yet: claim the wait so later calls no-op below. + c.state = StateSocketWaiting + case StateSocketWaiting, StateTransferring, StateTransferCompleted: + // A transfer is already waiting, running, or done: nothing to do. + return nil + default: + // No other state can begin a transfer. + return fmt.Errorf("transfer not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // Fall back to the default wait when the caller gives no timeout. + if timeout <= 0 { + timeout = defaultSocketReadyTimeout + } + socketReady := c.socketReady + + // Drive the transfer in the background so the caller returns immediately; + // its outcome is surfaced to notification subscribers, not returned here. + go func() { + // Detached ctx so the transfer outlives the gRPC call. + ctx = context.WithoutCancel(ctx) + + // Wait for the duplicate socket to arrive, or give up after the timeout. + var socketTimeoutErr error + select { + case <-socketReady: + case <-time.After(timeout): + socketTimeoutErr = fmt.Errorf("timed out waiting for socket ready after %s: %w", timeout, errdefs.ErrFailedPrecondition) + } + + c.mu.Lock() + defer c.mu.Unlock() + + // If the socket connection was not ready within timeout, return an error. + if socketTimeoutErr != nil { + c.failTransfer(ctx, socketTimeoutErr) + return + } + + // Gate: only the first goroutine finds StateSocketReady and claims the + // transfer by moving to StateTransferring; any later goroutine (or a + // torn-down session) bails. + if c.state != StateSocketReady { + return + } + + // Move the state to transferring so that other concurrent callers + // will return early from above and there is a single driver for transfer. + c.state = StateTransferring + + // Run the transfer; a failure marks the session failed for subscribers. + if err := c.runTransfer(ctx); err != nil { + c.failTransfer(ctx, err) + return + } + + c.state = StateTransferCompleted + log.G(ctx).Info("migration transfer completed") + }() + + return nil +} + +// failTransfer marks the session failed and surfaces err to notification +// subscribers as a PHASE_FAILED event. +func (c *Controller) failTransfer(ctx context.Context, err error) { + c.state = StateFailed + + log.G(ctx).WithError(err).Error("migration transfer failed") + + // If there are no subscribers, then we should return early. + if c.notifier == nil { + return + } + + result := hcsschema.MigrationResultDestinationMigrationFailed + if c.origin == hcsschema.MigrationOriginSource { + result = hcsschema.MigrationResultSourceMigrationFailed + } + + // Broadcast the failure to subscribers as a migration-failed event. + c.notifier.broadcast(hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: c.origin, + Event: hcsschema.MigrationEventMigrationFailed, + Result: result, + }) +} + +// sessionIDToUint32 derives a stable uint32 from a session ID. SHA-256 is +// deterministic and uniformly distributed, so the same session ID always maps +// to the same value on both hosts. +func sessionIDToUint32(sessionID string) uint32 { + // Session IDs that parse as a GUID are normalized first so + // formatting/case differences map to the same value. + if g, err := guid.FromString(sessionID); err == nil { + sessionID = g.String() + } + + sum := sha256.Sum256([]byte(sessionID)) + return binary.BigEndian.Uint32(sum[:4]) +} + +// runTransfer issues the per-origin HCS calls for the memory transfer. +func (c *Controller) runTransfer(ctx context.Context) error { + config := &hcs.MigrationConfig{ + Socket: syscall.Handle(c.dupSocket), + SessionID: sessionIDToUint32(c.sessionID), + } + + // Start the live migration on the VM according to this host's role. + switch c.origin { + case hcsschema.MigrationOriginSource: + if err := c.vmController.StartLiveMigrationOnSource(ctx, config); err != nil { + return fmt.Errorf("start live migration on source: %w", err) + } + case hcsschema.MigrationOriginDestination: + if err := c.vmController.StartWithMigrationOptions(ctx, config); err != nil { + return fmt.Errorf("start with migration options: %w", err) + } + default: + return fmt.Errorf("unsupported migration origin %q: %w", c.origin, errdefs.ErrInvalidArgument) + } + + // Begin streaming the memory transfer over the socket. + transferOpts := &hcsschema.MigrationTransferOptions{Origin: c.origin} + if err := c.vmController.StartLiveMigrationTransfer(ctx, transferOpts); err != nil { + return fmt.Errorf("start live migration transfer: %w", err) + } + + return nil +} + +// Finalize applies the session's final outcome (resume or stop) on this host +// and, on success, leaves the session finalized so Cleanup can run. A repeat +// call for an already-finalized session is a no-op. +func (c *Controller) Finalize(ctx context.Context, sessionID string, action migration.FinalizeAction, events chan interface{}) error { + if action == migration.FinalizeAction_FINALIZE_ACTION_UNSPECIFIED { + return fmt.Errorf("finalize action must be specified: %w", errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Idempotent: an already-finalized session succeeds without redoing work. + if c.state == StateFinalized { + return nil + } + + // Finalize is valid only after a completed transfer or a cancellation. + if c.state != StateTransferCompleted && c.state != StateCancelled { + return fmt.Errorf("finalize not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // Map the gRPC finalize action to the HCS final operation. + var finalOp hcsschema.MigrationFinalOperation + switch action { + case migration.FinalizeAction_FINALIZE_ACTION_RESUME: + finalOp = hcsschema.MigrationFinalOperationResume + case migration.FinalizeAction_FINALIZE_ACTION_STOP: + finalOp = hcsschema.MigrationFinalOperationStop + default: + return fmt.Errorf("unsupported finalize action %s: %w", action, errdefs.ErrInvalidArgument) + } + + // Apply the final operation (resume or stop) to the underlying VM. + if err := c.vmController.FinalizeLiveMigration(ctx, + &hcsschema.MigrationFinalizedOptions{ + Origin: c.origin, + FinalizedOperation: finalOp, + }); err != nil { + return fmt.Errorf("finalize live migration (origin=%s, action=%s): %w", c.origin, action, err) + } + + // On RESUME, rebuild the GCS bridge. Destination also walks pods to + // reattach gcs.Container/gcs.Process; source has nothing else to restore. + if finalOp == hcsschema.MigrationFinalOperationResume { + switch c.origin { + case hcsschema.MigrationOriginDestination: + // Resume the destination VM. + if err := c.vmController.Resume(ctx, false /* rebuildBridge */); err != nil { + return fmt.Errorf("resume destination vm from migration: %w", err) + } + + // Reattach each migrated pod's containers and processes on this host. + for podID, podCtrl := range c.podControllers { + if err := podCtrl.Resume(ctx, c.vmController, events, true /* isDestination */); err != nil { + return fmt.Errorf("resume pod %q from migration: %w", podID, err) + } + } + case hcsschema.MigrationOriginSource: + // Resume the source VM, rebuilding its GCS bridge. + if err := c.vmController.Resume(ctx, true /* rebuildBridge */); err != nil { + return fmt.Errorf("resume source vm from migration: %w", err) + } + + // Resume pods to lift the source freeze, skipping destination-only + // steps such as network reset. + for podID, podCtrl := range c.podControllers { + if err := podCtrl.Resume(ctx, c.vmController, events, false /* isDestination */); err != nil { + return fmt.Errorf("resume pod %q from migration: %w", podID, err) + } + } + } + } + + if finalOp == hcsschema.MigrationFinalOperationStop { + // For both source and destination, there is nothing to do for VM. + // vm.FinalizeLiveMigration + STOP will stop the VM itself. + // We only need to take care of tasks. + switch c.origin { + case hcsschema.MigrationOriginDestination: + // On the destination side, we would not create the exit handlers until resume + // which happens in Finalize + Resume. Therefore, containers would not have + // any way to report the exit event. Hence, we explicitly abort the + // in-migration tasks. + for _, podCtrl := range c.podControllers { + podCtrl.AbortMigrated(ctx, events) + } + case hcsschema.MigrationOriginSource: + // On the source side, all the exit handlers for containers and processes + // are already running. Therefore, during vm.FinalizeLiveMigration, if the + // operation was stop, we would collapse the bridge which would lead to all + // the exit handlers observing the VM exiting and report the exit event. + // Therefore, nothing to do here. + } + } + + // Mark the session finalized so Cleanup can run and repeat calls no-op. + c.state = StateFinalized + + log.G(ctx).WithField(logfields.Action, action.String()).Info("migration session finalized") + return nil +} + +// Cancel aborts an in-flight migration session, marking it cancelled so a +// subsequent Finalize and Cleanup can wind it down. A repeat call is a no-op. +func (c *Controller) Cancel(ctx context.Context, sessionID string) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrFailedPrecondition) + } + + // Idempotent: already canceled. + if c.state == StateCancelled { + return nil + } + + // TODO: call into HCS to cancel the in-flight migration once the cancel API is available. + + // Mark the session canceled; Cleanup later returns it to idle. + c.state = StateCancelled + + log.G(ctx).Info("migration session cancelled") + return nil +} + +// Cleanup is the terminal call of a migration session on either side. It +// reverts the controller back to [StateIdle], releasing the transport socket +// and tearing down notification subscribers, regardless of whether the +// migration completed, failed, or was cancelled. A call against an already-idle +// controller is a no-op. +func (c *Controller) Cleanup(ctx context.Context, sessionID string, events chan interface{}) error { + c.mu.Lock() + defer c.mu.Unlock() + + // Already cleaned up / no active session. + if c.state == StateIdle { + return nil + } + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrFailedPrecondition) + } + + // Cleanup is only valid once the session has been finalized. + if c.state != StateFinalized { + return fmt.Errorf("cleanup not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // The transport socket is unused past this point and the notifier is + // scoped to the session, so release both before resetting state. + if c.dupSocket != 0 { + if err := windows.Closesocket(c.dupSocket); err != nil { + log.G(ctx).WithError(err).Warn("close duplicate migration socket") + } + c.dupSocket = 0 + } + + if c.notifier != nil { + c.notifier.close() + c.notifier = nil + } + + // Reset all session-scoped state so the controller can host a new session. + c.sessionID, c.sandboxID, c.origin = "", "", "" + c.vmController = nil + c.podControllers = nil + c.containerPodMapping = nil + c.pendingPatches = nil + c.socketReady = make(chan struct{}) + c.state = StateIdle + + log.G(ctx).Info("migration session cleaned up") + return nil +} diff --git a/internal/controller/migration/controller_destination.go b/internal/controller/migration/controller_destination.go new file mode 100644 index 0000000000..1309eb6cd0 --- /dev/null +++ b/internal/controller/migration/controller_destination.go @@ -0,0 +1,263 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/internal/oci" + hcsannotations "github.com/Microsoft/hcsshim/pkg/annotations" + + "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/errdefs" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +// ImportState rehydrates a source-side migration snapshot onto this controller. +// A repeat call for the same session is a no-op. +func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) error { + // Reject malformed input up front so the controller is never mutated + // on the basis of a half-specified request. + switch { + case opts == nil: + return fmt.Errorf("options are required: %w", errdefs.ErrInvalidArgument) + case opts.SessionID == "": + return fmt.Errorf("session id is required: %w", errdefs.ErrInvalidArgument) + case opts.VMController == nil: + return fmt.Errorf("vm controller is required: %w", errdefs.ErrInvalidArgument) + case opts.SandboxID == "": + return fmt.Errorf("sandbox id is required: %w", errdefs.ErrInvalidArgument) + case opts.PodControllers == nil: + return fmt.Errorf("pod controllers map is required: %w", errdefs.ErrInvalidArgument) + case opts.ContainerPodMapping == nil: + return fmt.Errorf("container-pod mapping is required: %w", errdefs.ErrInvalidArgument) + case opts.SavedState == nil: + return fmt.Errorf("sandbox saved state is required: %w", errdefs.ErrInvalidArgument) + case opts.SavedState.TypeUrl != save.TypeURL: + return fmt.Errorf("unsupported sandbox saved-state type %q: %w", opts.SavedState.TypeUrl, errdefs.ErrInvalidArgument) + case opts.VMController.State() != vm.StateNotCreated: + return fmt.Errorf("vm controller is in invalid state %s: %w", opts.VMController.State(), errdefs.ErrFailedPrecondition) + default: + } + + decoded := &save.Payload{} + if err := proto.Unmarshal(opts.SavedState.Value, decoded); err != nil { + return fmt.Errorf("unmarshal sandbox saved state: %w", err) + } + if decoded.GetSchemaVersion() != save.SchemaVersion { + return fmt.Errorf("sandbox saved-state schema version %d not supported (want %d): %w", + decoded.GetSchemaVersion(), save.SchemaVersion, errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Idempotent retry for the same session; any other non-idle state is a conflict. + if c.state == StateDestinationImported && c.sessionID == opts.SessionID { + return nil + } + if c.state != StateIdle { + return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrAlreadyExists) + } + + // Rehydrate the VM controller from the saved VM payload. + if err := opts.VMController.Import(ctx, decoded.GetVm()); err != nil { + return fmt.Errorf("import vm controller: %w", err) + } + + // Rehydrate each pod and index its containers so PatchResourcePaths + // can look up the owning pod. + pending := make(map[string]struct{}) + for _, podAny := range decoded.GetPods() { + // Rebuild the pod controller from its saved payload. + importedPod, err := pod.Import(ctx, podAny) + if err != nil { + return fmt.Errorf("import pod: %w", err) + } + + opts.PodControllers[importedPod.PodID()] = importedPod + + // Source container IDs must be unique across pods so the later patch + // lookup is unambiguous. + for containerID := range importedPod.ListContainers() { + if _, dup := pending[containerID]; dup { + return fmt.Errorf("duplicate source container id %q across imported pods: %w", containerID, errdefs.ErrInvalidArgument) + } + + // Track the container as awaiting a patch and map it to its pod. + pending[containerID] = struct{}{} + opts.ContainerPodMapping[containerID] = importedPod.PodID() + } + } + + c.sessionID = opts.SessionID + c.sandboxID = opts.SandboxID + c.origin = opts.Origin + c.vmController = opts.VMController + c.podControllers = opts.PodControllers + c.containerPodMapping = opts.ContainerPodMapping + c.pendingPatches = pending + c.state = StateDestinationImported + + log.G(ctx).Info("migration destination state imported") + return nil +} + +// PatchResourcePaths rewrites the imported source container's identifiers to +// the destination IDs carried by request and spec. A container may be patched +// only once; a repeat call for an already-patched container fails. +func (c *Controller) PatchResourcePaths( + ctx context.Context, + request *task.CreateTaskRequest, + spec specs.Spec, +) error { + switch { + case request == nil: + return fmt.Errorf("request is required: %w", errdefs.ErrInvalidArgument) + case request.ID == "": + return fmt.Errorf("destination container id is required: %w", errdefs.ErrInvalidArgument) + case spec.Annotations == nil: + return fmt.Errorf("annotations are required: %w", errdefs.ErrInvalidArgument) + } + + // The source container being rebound is identified by the annotation. + sourceContainerID := spec.Annotations[hcsannotations.LiveMigrationSourceContainerID] + if sourceContainerID == "" { + return fmt.Errorf("annotation %q is required: %w", hcsannotations.LiveMigrationSourceContainerID, errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.state != StateDestinationImported { + return fmt.Errorf("patch not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // A container may be patched only once; once patched, it drops out of + // pendingPatches. Hence, a non-pending ID has already been patched. + if _, pending := c.pendingPatches[sourceContainerID]; !pending { + return fmt.Errorf("source container %q is not pending a patch: %w", sourceContainerID, errdefs.ErrAlreadyExists) + } + + // containerPodMapping is the source-of-truth index built in ImportState + // and rewritten in place here, so a direct lookup beats scanning pods. + sourcePodID, ok := c.containerPodMapping[sourceContainerID] + if !ok { + return fmt.Errorf("source container %q not found: %w", sourceContainerID, errdefs.ErrNotFound) + } + podCtrl := c.podControllers[sourcePodID] + + // Sandbox is detected structurally as container ID == pod ID. + isSandbox := sourceContainerID == sourcePodID + + // The K8s container-type annotation, if present, must agree with the + // structural detection; a mismatch would desync the in-pod and outer-map renames. + if v := spec.Annotations[hcsannotations.KubernetesContainerType]; v != "" { + annotationSaysSandbox := v == string(oci.KubernetesContainerTypeSandbox) + if annotationSaysSandbox != isSandbox { + return fmt.Errorf("annotation %q=%q disagrees with structural sandbox detection (sourceContainerID=%q, sourcePodID=%q): %w", + hcsannotations.KubernetesContainerType, v, sourceContainerID, sourcePodID, errdefs.ErrInvalidArgument) + } + } + + // Fetch the SCSI controller so that we can patch the VHD paths on destination. + scsiCtrl, err := c.vmController.SCSIController(ctx) + if err != nil { + return fmt.Errorf("get scsi controller for patch: %w", err) + } + + // Rebind the container's resources within its pod to the destination IDs. + if err := podCtrl.Patch(ctx, sourceContainerID, isSandbox, scsiCtrl, request, spec); err != nil { + return fmt.Errorf("patch source container %q in pod %q: %w", sourceContainerID, sourcePodID, err) + } + + // Patching the sandbox renames the pod: re-key its podControllers entry and + // repoint every container still mapped to the old pod ID at the new one. + if isSandbox { + // Move the pod controller from the old pod ID to the destination ID. + delete(c.podControllers, sourcePodID) + c.podControllers[request.ID] = podCtrl + + // Repoint any container (this sandbox plus worker containers patched earlier) + // that still references the old pod ID at the new pod ID. + for cid, pid := range c.containerPodMapping { + if pid == sourcePodID { + c.containerPodMapping[cid] = request.ID + } + } + } + + // Re-key this container's mapping entry from its source to destination ID. + delete(c.containerPodMapping, sourceContainerID) + c.containerPodMapping[request.ID] = podCtrl.PodID() + + // The container is now patched, so drop it from the pending set. + delete(c.pendingPatches, sourceContainerID) + + log.G(ctx).WithFields(logrus.Fields{ + logfields.SourceContainerID: sourceContainerID, + logfields.DestinationContainerID: request.ID, + }).Info("migration container resource paths patched") + + return nil +} + +// PrepareDestination materialises the destination HCS compute system once every +// imported container has been patched to its destination identifiers. +func (c *Controller) PrepareDestination(ctx context.Context, sessionID string, migrationOpts *hcsschema.MigrationInitializeOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + if c.state != StateDestinationImported { + return fmt.Errorf("prepare destination not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // Every imported container must be patched before the destination VM is built. + if len(c.pendingPatches) > 0 { + pending := slices.Sorted(maps.Keys(c.pendingPatches)) + return fmt.Errorf("%d container patches still pending [%s]: %w", + len(pending), strings.Join(pending, ", "), errdefs.ErrFailedPrecondition) + } + + // Default options and stamp the destination origin so HCS sees a + // complete config regardless of caller input. + if migrationOpts == nil { + migrationOpts = &hcsschema.MigrationInitializeOptions{} + } + migrationOpts.Origin = c.origin + + // Build the destination VM's HCS compute system from the imported config. + if err := c.vmController.CreateVM(ctx, + &vm.CreateOptions{ + ID: fmt.Sprintf("%s@vm", c.sandboxID), + MigrationOptions: migrationOpts, + }); err != nil { + return fmt.Errorf("create destination vm: %w", err) + } + + // Re-ACL the patched (destination-host) VHDs against the freshly + // created VM's SID so it can open them once it starts. + if err := c.vmController.Patch(ctx); err != nil { + return fmt.Errorf("patch destination vm: %w", err) + } + + c.state = StateDestinationPrepared + log.G(ctx).Info("migration destination prepared") + return nil +} diff --git a/internal/controller/migration/controller_destination_test.go b/internal/controller/migration/controller_destination_test.go new file mode 100644 index 0000000000..53f79eb7e1 --- /dev/null +++ b/internal/controller/migration/controller_destination_test.go @@ -0,0 +1,414 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/errdefs" + "github.com/opencontainers/runtime-spec/specs-go" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/oci" + hcsannotations "github.com/Microsoft/hcsshim/pkg/annotations" +) + +// validImportPayload returns a decodable, current-schema sandbox payload with a +// VM blob and no pods, so import never has to construct a real pod controller. +func validImportPayload() *save.Payload { + return &save.Payload{ + SchemaVersion: save.SchemaVersion, + Vm: &anypb.Any{TypeUrl: "type.example/vm", Value: []byte("vm")}, + } +} + +// importEnvelope marshals a payload and wraps it in an envelope with the +// well-known type URL, matching what the source's ExportState emits. +func importEnvelope(t *testing.T, p *save.Payload) *anypb.Any { + t.Helper() + b, err := proto.Marshal(p) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + return &anypb.Any{TypeUrl: save.TypeURL, Value: b} +} + +// importOptions returns a valid ImportStateOptions bound to the given VM mock +// and saved-state envelope. +func importOptions(vmc vmController, saved *anypb.Any) *ImportStateOptions { + return &ImportStateOptions{ + InitOptions: InitOptions{ + SessionID: "sess-1", + Origin: hcsschema.MigrationOriginDestination, + VMController: vmc, + PodControllers: map[string]*pod.Controller{}, + }, + SandboxID: "sandbox-1", + SavedState: saved, + ContainerPodMapping: map[string]string{}, + } +} + +// importVM returns a VM mock reporting the not-created state that ImportState +// requires before rehydrating a snapshot. +func importVM(ctrl *gomock.Controller) *mocks.MockvmController { + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().State().Return(vm.StateNotCreated).AnyTimes() + return vmc +} + +// patchSpec builds a spec carrying the source-container annotation plus any extras. +func patchSpec(sourceContainerID string, extra map[string]string) specs.Spec { + ann := map[string]string{hcsannotations.LiveMigrationSourceContainerID: sourceContainerID} + for k, v := range extra { + ann[k] = v + } + return specs.Spec{Annotations: ann} +} + +// ───────────────────────────────────────────────────────────────────────────── +// ImportState +// ───────────────────────────────────────────────────────────────────────────── + +// TestImportState_RejectsInvalidArgs verifies a half-specified request is +// refused before the controller is mutated. +func TestImportState_RejectsInvalidArgs(t *testing.T) { + valid := func() *ImportStateOptions { + return importOptions(&mocks.MockvmController{}, importEnvelope(t, validImportPayload())) + } + cases := map[string]*ImportStateOptions{ + "NilOptions": nil, + "EmptySessionID": func() *ImportStateOptions { o := valid(); o.SessionID = ""; return o }(), + "NilVMController": func() *ImportStateOptions { o := valid(); o.VMController = nil; return o }(), + "EmptySandboxID": func() *ImportStateOptions { o := valid(); o.SandboxID = ""; return o }(), + "NilPodControllers": func() *ImportStateOptions { o := valid(); o.PodControllers = nil; return o }(), + "NilContainerPodMapping": func() *ImportStateOptions { o := valid(); o.ContainerPodMapping = nil; return o }(), + "NilSavedState": func() *ImportStateOptions { o := valid(); o.SavedState = nil; return o }(), + "WrongTypeURL": func() *ImportStateOptions { o := valid(); o.SavedState = &anypb.Any{TypeUrl: "type.bogus"}; return o }(), + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + c := New() + + err := c.ImportState(t.Context(), opts) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle after rejected args, got %s", c.state) + } + }) + } +} + +// TestImportState_RejectsUndecodableState verifies a payload this build cannot +// decode is rejected before any controller state is rehydrated. +func TestImportState_RejectsUndecodableState(t *testing.T) { + cases := map[string]*anypb.Any{ + "CorruptPayload": {TypeUrl: save.TypeURL, Value: []byte{0x08}}, + "SchemaMismatch": importEnvelope(t, &save.Payload{SchemaVersion: save.SchemaVersion + 1}), + } + for name, saved := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + + if err := c.ImportState(t.Context(), importOptions(importVM(ctrl), saved)); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateIdle { + t.Errorf("expected state Idle, got %s", c.state) + } + }) + } +} + +// TestImportState_IdempotentSameSession verifies a repeat import for the active +// session is a no-op that does not re-import the VM. +func TestImportState_IdempotentSameSession(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateDestinationImported + c.sessionID = "sess-1" + + if err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestImportState_RejectsConflictingState verifies a snapshot cannot be imported +// while the controller is busy with another (non-idle) session. +func TestImportState_RejectsConflictingState(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateDestinationPrepared + c.sessionID = "other" + + err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))) + if !errors.Is(err, errdefs.ErrAlreadyExists) { + t.Fatalf("expected ErrAlreadyExists, got %v", err) + } +} + +// TestImportState_RejectsWrongVMState verifies a snapshot cannot be imported +// unless the VM controller has not yet been created. +func TestImportState_RejectsWrongVMState(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().State().Return(vm.StateRunning).AnyTimes() + + c := New() + err := c.ImportState(t.Context(), importOptions(vmc, importEnvelope(t, validImportPayload()))) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestImportState_VMImportError verifies a failure rehydrating the VM aborts the +// import without advancing the state. +func TestImportState_VMImportError(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := importVM(ctrl) + vmc.EXPECT().Import(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + if err := c.ImportState(t.Context(), importOptions(vmc, importEnvelope(t, validImportPayload()))); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateIdle { + t.Errorf("expected state Idle after failure, got %s", c.state) + } +} + +// TestImportState_Success verifies a valid snapshot with no pods rehydrates the +// VM, binds the session, and advances the state. +func TestImportState_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := importVM(ctrl) + vmc.EXPECT().Import(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + opts := importOptions(vmc, importEnvelope(t, validImportPayload())) + if err := c.ImportState(t.Context(), opts); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if c.state != StateDestinationImported { + t.Errorf("expected state DestinationImported, got %s", c.state) + } + if c.sessionID != "sess-1" || c.sandboxID != "sandbox-1" || c.vmController != vmc { + t.Errorf("session state not bound: %+v", c) + } + if c.pendingPatches == nil || len(c.pendingPatches) != 0 { + t.Errorf("expected empty pending set, got %+v", c.pendingPatches) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// PatchResourcePaths +// ───────────────────────────────────────────────────────────────────────────── + +// TestPatchResourcePaths_RejectsInvalidArgs verifies a malformed request or spec +// is refused before the controller is touched. +func TestPatchResourcePaths_RejectsInvalidArgs(t *testing.T) { + cases := map[string]struct { + request *task.CreateTaskRequest + spec specs.Spec + }{ + "NilRequest": {nil, patchSpec("ctr-1", nil)}, + "EmptyRequestID": {&task.CreateTaskRequest{ID: ""}, patchSpec("ctr-1", nil)}, + "NilAnnotations": {&task.CreateTaskRequest{ID: "dst-1"}, specs.Spec{}}, + "MissingSourceAnnotation": {&task.CreateTaskRequest{ID: "dst-1"}, specs.Spec{Annotations: map[string]string{}}}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := New() + + err := c.PatchResourcePaths(t.Context(), tc.request, tc.spec) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + }) + } +} + +// TestPatchResourcePaths_RejectsWrongState verifies patching is only valid once +// the destination has imported a snapshot. +func TestPatchResourcePaths_RejectsWrongState(t *testing.T) { + c := New() // StateIdle + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestPatchResourcePaths_RejectsAlreadyPatched verifies a container that is no +// longer pending cannot be patched again. +func TestPatchResourcePaths_RejectsAlreadyPatched(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{} + c.containerPodMapping = map[string]string{} + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) + if !errors.Is(err, errdefs.ErrAlreadyExists) { + t.Fatalf("expected ErrAlreadyExists, got %v", err) + } +} + +// TestPatchResourcePaths_RejectsUnknownContainer verifies a pending container +// missing from the index is reported as not found. +func TestPatchResourcePaths_RejectsUnknownContainer(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{"ctr-1": {}} + c.containerPodMapping = map[string]string{} + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) + if !errors.Is(err, errdefs.ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } +} + +// TestPatchResourcePaths_RejectsAnnotationMismatch verifies a container-type +// annotation that contradicts the structural sandbox detection is rejected. +func TestPatchResourcePaths_RejectsAnnotationMismatch(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{"ctr-1": {}} + c.containerPodMapping = map[string]string{"ctr-1": "pod-1"} // ctr-1 != pod-1, so not a sandbox + c.podControllers = map[string]*pod.Controller{} + + // Annotation claims sandbox, contradicting the structural detection. + spec := patchSpec("ctr-1", map[string]string{ + hcsannotations.KubernetesContainerType: string(oci.KubernetesContainerTypeSandbox), + }) + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, spec) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestPatchResourcePaths_SCSIControllerError verifies a failure obtaining the +// SCSI controller surfaces before the pod is patched. +func TestPatchResourcePaths_SCSIControllerError(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().SCSIController(gomock.Any()).Return(nil, errors.New("boom")) + + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{"ctr-1": {}} + c.containerPodMapping = map[string]string{"ctr-1": "pod-1"} + c.podControllers = map[string]*pod.Controller{} + c.vmController = vmc + + if err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)); err == nil { + t.Fatal("expected error, got nil") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// PrepareDestination +// ───────────────────────────────────────────────────────────────────────────── + +// TestPrepareDestination_RejectsWrongState verifies the destination VM is only +// built from an imported controller. +func TestPrepareDestination_RejectsWrongState(t *testing.T) { + c := New() // StateIdle + c.sessionID = "sess-1" + + if err := c.PrepareDestination(t.Context(), "sess-1", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestPrepareDestination_RejectsPendingPatches verifies the VM is not built while +// any imported container is still awaiting a patch. +func TestPrepareDestination_RejectsPendingPatches(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.sessionID = "sess-1" + c.pendingPatches = map[string]struct{}{"ctr-1": {}, "ctr-2": {}} + + err := c.PrepareDestination(t.Context(), "sess-1", nil) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if !strings.Contains(err.Error(), "ctr-1") || !strings.Contains(err.Error(), "ctr-2") { + t.Errorf("expected error to list pending containers, got: %v", err) + } +} + +// TestPrepareDestination_CreateVMError verifies a failure creating the VM aborts +// the preparation without advancing the state. +func TestPrepareDestination_CreateVMError(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().CreateVM(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.state = StateDestinationImported + c.sandboxID = "sandbox-1" + c.sessionID = "sess-1" + c.pendingPatches = map[string]struct{}{} + c.vmController = vmc + + if err := c.PrepareDestination(t.Context(), "sess-1", nil); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateDestinationImported { + t.Errorf("expected state DestinationImported after failure, got %s", c.state) + } +} + +// TestPrepareDestination_Success verifies a successful preparation stamps the +// origin onto the (defaulted) options, builds the VM, and advances the state. +func TestPrepareDestination_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + + var gotOpts *vm.CreateOptions + vmc.EXPECT().CreateVM(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, o *vm.CreateOptions) error { + gotOpts = o + return nil + }) + vmc.EXPECT().Patch(gomock.Any()).Return(nil) + + c := New() + c.state = StateDestinationImported + c.sandboxID = "sandbox-1" + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginDestination + c.pendingPatches = map[string]struct{}{} + c.vmController = vmc + + if err := c.PrepareDestination(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if c.state != StateDestinationPrepared { + t.Errorf("expected state DestinationPrepared, got %s", c.state) + } + if gotOpts == nil || gotOpts.ID != "sandbox-1@vm" { + t.Fatalf("unexpected create options: %+v", gotOpts) + } + // MigrationOptions is defaulted when nil and stamped with the origin. + if gotOpts.MigrationOptions == nil || gotOpts.MigrationOptions.Origin != hcsschema.MigrationOriginDestination { + t.Errorf("expected migration options stamped with destination origin, got %+v", gotOpts.MigrationOptions) + } +} diff --git a/internal/controller/migration/controller_source.go b/internal/controller/migration/controller_source.go new file mode 100644 index 0000000000..60f4a25ece --- /dev/null +++ b/internal/controller/migration/controller_source.go @@ -0,0 +1,126 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "fmt" + + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + + "github.com/containerd/errdefs" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// PrepareSource readies the source side of a migration session so a subsequent +// [Controller.ExportState] can capture its state. A repeat call for the same +// session is a no-op. +func (c *Controller) PrepareSource(ctx context.Context, opts *PrepareSourceOptions) error { + switch { + case opts == nil: + return fmt.Errorf("options are required: %w", errdefs.ErrInvalidArgument) + case opts.SessionID == "": + return fmt.Errorf("session id is required: %w", errdefs.ErrInvalidArgument) + case opts.VMController == nil: + return fmt.Errorf("vm controller is required: %w", errdefs.ErrInvalidArgument) + case opts.PodControllers == nil: + return fmt.Errorf("pod controllers map is required: %w", errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.state != StateIdle { + // If we already called this API, then this is a no-op. + if c.state == StateSourcePrepared && c.sessionID == opts.SessionID { + return nil + } + return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrFailedPrecondition) + } + + // VM must be running in order to initiate the migration. + if opts.VMController.State() != vm.StateRunning { + return fmt.Errorf("vm controller is in invalid state %s: %w", opts.VMController.State(), errdefs.ErrFailedPrecondition) + } + + // The sandbox must have been created with live migration enabled. + // Reject otherwise so callers learn the session cannot proceed before + // any source-side state is mutated. + sandboxOpts := opts.VMController.SandboxOptions() + if sandboxOpts == nil || !sandboxOpts.LiveMigrationSupportEnabled { + return fmt.Errorf("sandbox is not configured to allow live migration: %w", errdefs.ErrFailedPrecondition) + } + + if opts.MigrationOpts == nil { + opts.MigrationOpts = &hcsschema.MigrationInitializeOptions{} + } + opts.MigrationOpts.Origin = opts.Origin + + // Prepare the running source VM; afterward it accepts only live-migration compatible calls. + if err := opts.VMController.InitializeLiveMigrationOnSource(ctx, opts.MigrationOpts); err != nil { + return fmt.Errorf("initialize live migration on source vm: %w", err) + } + + c.sessionID = opts.SessionID + c.origin = opts.Origin + c.vmController = opts.VMController + c.podControllers = opts.PodControllers + c.state = StateSourcePrepared + + log.G(ctx).Info("migration source prepared") + return nil +} + +// ExportState captures the prepared source sandbox into an opaque, versioned +// saved state that the destination consumes via [Controller.ImportState]. The VM +// and per-pod payloads it carries are themselves opaque, owned by their +// respective controllers. A repeat call returns a fresh snapshot. +func (c *Controller) ExportState(ctx context.Context, sessionID string) (*anypb.Any, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return nil, fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Allow re-export after a prior success so duplicate calls are idempotent. + if c.state != StateSourcePrepared && c.state != StateSourceExported { + return nil, fmt.Errorf("export requires state %s or %s (current: %s): %w", StateSourcePrepared, StateSourceExported, c.state, errdefs.ErrFailedPrecondition) + } + + // Save the VM state. + vmAny, err := c.vmController.Save(ctx) + if err != nil { + return nil, fmt.Errorf("save vm controller: %w", err) + } + + // Save all the pod controller states as opaque envelopes. + pods := make([]*anypb.Any, 0, len(c.podControllers)) + for podID, podCtrl := range c.podControllers { + ps, err := podCtrl.Save(ctx) + if err != nil { + return nil, fmt.Errorf("save pod %s: %w", podID, err) + } + + pods = append(pods, ps) + } + + // Wrap the VM and pod payloads in the versioned sandbox-level envelope. + payload, err := proto.Marshal(&save.Payload{ + SchemaVersion: save.SchemaVersion, + Vm: vmAny, + Pods: pods, + }) + if err != nil { + return nil, fmt.Errorf("marshal sandbox saved state: %w", err) + } + + c.state = StateSourceExported + + log.G(ctx).Info("migration source state exported") + return &anypb.Any{TypeUrl: save.TypeURL, Value: payload}, nil +} diff --git a/internal/controller/migration/controller_source_test.go b/internal/controller/migration/controller_source_test.go new file mode 100644 index 0000000000..5ae7d24514 --- /dev/null +++ b/internal/controller/migration/controller_source_test.go @@ -0,0 +1,279 @@ +//go:build windows && lcow + +package migration + +import ( + "errors" + "testing" + + "github.com/containerd/errdefs" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/pod" + vmpkg "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" +) + +// migrationEnabledOptions returns sandbox options that permit live migration. +func migrationEnabledOptions() *lcow.SandboxOptions { + return &lcow.SandboxOptions{LiveMigrationSupportEnabled: true} +} + +// sourceOptions returns a valid PrepareSourceOptions bound to the given VM mock. +func sourceOptions(vm vmController) *PrepareSourceOptions { + return &PrepareSourceOptions{ + InitOptions: InitOptions{ + SessionID: "sess-1", + Origin: hcsschema.MigrationOriginSource, + VMController: vm, + PodControllers: map[string]*pod.Controller{}, + }, + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// PrepareSource +// ───────────────────────────────────────────────────────────────────────────── + +// TestPrepareSource_RejectsInvalidArgs verifies a half-specified request is +// refused before any source-side state is touched. +func TestPrepareSource_RejectsInvalidArgs(t *testing.T) { + cases := map[string]*PrepareSourceOptions{ + "NilOptions": nil, + "EmptySessionID": {InitOptions: InitOptions{VMController: &mocks.MockvmController{}, PodControllers: map[string]*pod.Controller{}}}, + "NilVMController": {InitOptions: InitOptions{SessionID: "sess-1", PodControllers: map[string]*pod.Controller{}}}, + "NilPodControllers": {InitOptions: InitOptions{SessionID: "sess-1", VMController: &mocks.MockvmController{}}}, + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + c := New() + + err := c.PrepareSource(t.Context(), opts) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle after rejected args, got %s", c.state) + } + }) + } +} + +// TestPrepareSource_RejectsWrongState verifies a session can only be armed from +// idle; a controller busy with another session is rejected. +func TestPrepareSource_RejectsWrongState(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateSourceExported + c.sessionID = "other" + + err := c.PrepareSource(t.Context(), sourceOptions(mocks.NewMockvmController(ctrl))) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestPrepareSource_IdempotentSameSession verifies re-arming the same session is +// a no-op that does not touch the VM. +func TestPrepareSource_IdempotentSameSession(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + + // No VM calls are expected on a duplicate arm. + if err := c.PrepareSource(t.Context(), sourceOptions(mocks.NewMockvmController(ctrl))); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.state != StateSourcePrepared { + t.Errorf("expected state SourcePrepared, got %s", c.state) + } +} + +// TestPrepareSource_RejectsMigrationDisabled verifies a sandbox that was not +// created with live migration enabled cannot be armed. +func TestPrepareSource_RejectsMigrationDisabled(t *testing.T) { + cases := map[string]*lcow.SandboxOptions{ + "NilOptions": nil, + "FeatureDisabled": {LiveMigrationSupportEnabled: false}, + } + for name, sandboxOpts := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateRunning) + vm.EXPECT().SandboxOptions().Return(sandboxOpts) + + c := New() + err := c.PrepareSource(t.Context(), sourceOptions(vm)) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle, got %s", c.state) + } + }) + } +} + +// TestPrepareSource_RejectsWrongVMState verifies the source cannot be armed +// unless the VM is running. +func TestPrepareSource_RejectsWrongVMState(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateCreated).AnyTimes() + + c := New() + if err := c.PrepareSource(t.Context(), sourceOptions(vm)); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle, got %s", c.state) + } +} + +// TestPrepareSource_InitializeError verifies a failure arming the VM leaves the +// controller idle so the session can be retried. +func TestPrepareSource_InitializeError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateRunning) + vm.EXPECT().SandboxOptions().Return(migrationEnabledOptions()) + vm.EXPECT().InitializeLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + if err := c.PrepareSource(t.Context(), sourceOptions(vm)); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateIdle { + t.Errorf("expected state Idle after failure, got %s", c.state) + } +} + +// TestPrepareSource_Success verifies a successful arm records the session, stamps +// the origin onto the (defaulted) migration options, and advances the state. +func TestPrepareSource_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateRunning) + vm.EXPECT().SandboxOptions().Return(migrationEnabledOptions()) + vm.EXPECT().InitializeLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + opts := sourceOptions(vm) + if err := c.PrepareSource(t.Context(), opts); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if c.state != StateSourcePrepared { + t.Errorf("expected state SourcePrepared, got %s", c.state) + } + if c.sessionID != "sess-1" || c.origin != hcsschema.MigrationOriginSource || c.vmController != vm { + t.Errorf("session state not bound: %+v", c) + } + // MigrationOpts is defaulted when nil and stamped with the origin. + if opts.MigrationOpts == nil || opts.MigrationOpts.Origin != hcsschema.MigrationOriginSource { + t.Errorf("expected migration options stamped with source origin, got %+v", opts.MigrationOpts) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// ExportState +// ───────────────────────────────────────────────────────────────────────────── + +// TestExportState_RejectsSessionMismatch verifies a snapshot is only produced +// for the active session. +func TestExportState_RejectsSessionMismatch(t *testing.T) { + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + + env, err := c.ExportState(t.Context(), "other") + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + if env != nil { + t.Errorf("expected nil envelope on failure, got %+v", env) + } +} + +// TestExportState_RejectsWrongState verifies a snapshot is only produced from a +// prepared (or already-exported) source. +func TestExportState_RejectsWrongState(t *testing.T) { + c := New() + c.sessionID = "sess-1" + + env, err := c.ExportState(t.Context(), "sess-1") + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if env != nil { + t.Errorf("expected nil envelope on failure, got %+v", env) + } +} + +// TestExportState_VMSaveError verifies a failure saving the VM aborts the export +// without advancing the state. +func TestExportState_VMSaveError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().Save(gomock.Any()).Return(nil, errors.New("boom")) + + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + c.vmController = vm + c.podControllers = map[string]*pod.Controller{} + + if _, err := c.ExportState(t.Context(), "sess-1"); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateSourcePrepared { + t.Errorf("expected state SourcePrepared after failure, got %s", c.state) + } +} + +// TestExportState_Success verifies the produced envelope is self-describing and +// carries the VM payload, and that the source advances to exported. +func TestExportState_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vmAny := &anypb.Any{TypeUrl: "type.example/vm", Value: []byte("vm-state")} + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().Save(gomock.Any()).Return(vmAny, nil) + + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + c.vmController = vm + c.podControllers = map[string]*pod.Controller{} + + env, err := c.ExportState(t.Context(), "sess-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env.GetTypeUrl() != save.TypeURL { + t.Errorf("expected type URL %q, got %q", save.TypeURL, env.GetTypeUrl()) + } + + got := &save.Payload{} + if err := proto.Unmarshal(env.GetValue(), got); err != nil { + t.Fatalf("unmarshal saved payload: %v", err) + } + if got.GetSchemaVersion() != save.SchemaVersion { + t.Errorf("expected schema version %d, got %d", save.SchemaVersion, got.GetSchemaVersion()) + } + if got.GetVm().GetTypeUrl() != vmAny.TypeUrl || string(got.GetVm().GetValue()) != string(vmAny.Value) { + t.Errorf("vm payload not preserved: %+v", got.GetVm()) + } + if len(got.GetPods()) != 0 { + t.Errorf("expected no pod payloads, got %d", len(got.GetPods())) + } + if c.state != StateSourceExported { + t.Errorf("expected state SourceExported after export, got %s", c.state) + } +} diff --git a/internal/controller/migration/controller_test.go b/internal/controller/migration/controller_test.go new file mode 100644 index 0000000000..94448acb34 --- /dev/null +++ b/internal/controller/migration/controller_test.go @@ -0,0 +1,534 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/containerd/errdefs" + "go.uber.org/mock/gomock" + "golang.org/x/sys/windows" + + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/pkg/migration" +) + +// waitForState polls until the controller reaches want or the deadline elapses, +// so background transfer goroutines can be observed deterministically. +func waitForState(t *testing.T, c *Controller, want State) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if c.State() == want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("state = %s; want %s", c.State(), want) +} + +// ───────────────────────────────────────────────────────────────────────────── +// New / State +// ───────────────────────────────────────────────────────────────────────────── + +// TestNew verifies a fresh controller starts idle and ready to host a session. +func TestNew(t *testing.T) { + c := New() + if c.State() != StateIdle { + t.Errorf("expected state Idle, got %s", c.State()) + } + if c.socketReady == nil { + t.Error("expected socketReady channel to be initialized") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// sessionIDToUint32 +// ───────────────────────────────────────────────────────────────────────────── + +// TestSessionIDToUint32 verifies the mapping is deterministic and that a GUID +// maps to the same value regardless of letter case. +func TestSessionIDToUint32(t *testing.T) { + const lower = "5f0b1190-63be-4e0c-b974-bd0f55675a42" + const upper = "5F0B1190-63BE-4E0C-B974-BD0F55675A42" + + if got, want := sessionIDToUint32(lower), sessionIDToUint32(lower); got != want { + t.Errorf("not deterministic: %d != %d", got, want) + } + if got, want := sessionIDToUint32(upper), sessionIDToUint32(lower); got != want { + t.Errorf("GUID case affected result: %d != %d", got, want) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Transfer +// ───────────────────────────────────────────────────────────────────────────── + +// TestTransfer_RejectsSessionMismatch verifies a call for a different session is +// rejected. +func TestTransfer_RejectsSessionMismatch(t *testing.T) { + c := New() + c.sessionID = "sess-1" + + if err := c.Transfer(t.Context(), "other", time.Minute); !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestTransfer_NoopWhenInProgress verifies a transfer that is already waiting, +// running, or done is a no-op that leaves the state unchanged. +func TestTransfer_NoopWhenInProgress(t *testing.T) { + for _, st := range []State{StateSocketWaiting, StateTransferring, StateTransferCompleted} { + t.Run(st.String(), func(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = st + + if err := c.Transfer(t.Context(), "sess-1", time.Minute); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != st { + t.Errorf("state = %s; want unchanged %s", c.State(), st) + } + }) + } +} + +// TestTransfer_RejectsInvalidState verifies a transfer cannot start from a state +// that has no path to it. +func TestTransfer_RejectsInvalidState(t *testing.T) { + c := New() // StateIdle + + if err := c.Transfer(t.Context(), "", time.Minute); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestTransfer_ClaimsSocketWaiting verifies that starting a transfer before the +// socket arrives claims the wait so later calls no-op. +func TestTransfer_ClaimsSocketWaiting(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateSourceExported + + // Long timeout so the background goroutine simply parks on the socket. + if err := c.Transfer(t.Context(), "sess-1", time.Hour); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateSocketWaiting { + t.Errorf("state = %s; want SocketWaiting", c.State()) + } + + // Unpark the goroutine so it exits without driving a transfer. + close(c.socketReady) +} + +// TestTransfer_Success verifies that with the socket ready the background +// transfer runs to completion. +func TestTransfer_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + c.state = StateSocketReady + close(c.socketReady) // socket already arrived + + if err := c.Transfer(t.Context(), "sess-1", time.Minute); err != nil { + t.Fatalf("unexpected error: %v", err) + } + waitForState(t, c, StateTransferCompleted) +} + +// TestTransfer_SocketTimeout verifies the session fails if the socket never +// arrives within the timeout. +func TestTransfer_SocketTimeout(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateSocketReady // socketReady stays open so the wait times out + + if err := c.Transfer(t.Context(), "sess-1", time.Millisecond); err != nil { + t.Fatalf("unexpected error: %v", err) + } + waitForState(t, c, StateFailed) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Finalize +// ───────────────────────────────────────────────────────────────────────────── + +// TestFinalize_RejectsUnspecifiedAction verifies an unspecified action is refused. +func TestFinalize_RejectsUnspecifiedAction(t *testing.T) { + c := New() + + err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_UNSPECIFIED, nil) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestFinalize_IdempotentFinalized verifies finalizing an already-finalized +// session is a no-op. +func TestFinalize_IdempotentFinalized(t *testing.T) { + c := New() + c.state = StateFinalized + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestFinalize_RejectsWrongState verifies finalize is rejected before a transfer +// has completed or been cancelled. +func TestFinalize_RejectsWrongState(t *testing.T) { + c := New() // StateIdle + + err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestFinalize_RejectsUnsupportedAction verifies an unknown action is rejected. +func TestFinalize_RejectsUnsupportedAction(t *testing.T) { + c := New() + c.state = StateTransferCompleted + + err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction(99), nil) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestFinalize_FinalizeError verifies a failure finalizing the VM aborts without +// advancing the state. +func TestFinalize_FinalizeError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err == nil { + t.Fatal("expected error, got nil") + } + if c.State() != StateTransferCompleted { + t.Errorf("state = %s; want unchanged TransferCompleted", c.State()) + } +} + +// TestFinalize_ResumeSucceeds verifies a resume finalizes the VM, resumes it, +// and advances to finalized on both origins. +func TestFinalize_ResumeSucceeds(t *testing.T) { + cases := map[string]struct { + origin hcsschema.MigrationOrigin + rebuildBridge bool + }{ + "destination": {hcsschema.MigrationOriginDestination, false}, + "source": {hcsschema.MigrationOriginSource, true}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().Resume(gomock.Any(), tc.rebuildBridge).Return(nil) + + c := New() + c.state = StateTransferCompleted + c.origin = tc.origin + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } + }) + } +} + +// TestFinalize_StopSucceeds verifies a stop finalizes the VM and advances to +// finalized without resuming it. +func TestFinalize_StopSucceeds(t *testing.T) { + for _, origin := range []hcsschema.MigrationOrigin{hcsschema.MigrationOriginSource, hcsschema.MigrationOriginDestination} { + t.Run(string(origin), func(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + c.state = StateTransferCompleted + c.origin = origin + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } + }) + } +} + +// TestFinalize_ResumeError verifies a failure resuming the VM aborts without +// advancing the state. +func TestFinalize_ResumeError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().Resume(gomock.Any(), false).Return(errors.New("boom")) + + c := New() + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginDestination + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err == nil { + t.Fatal("expected error, got nil") + } + if c.State() != StateTransferCompleted { + t.Errorf("state = %s; want unchanged TransferCompleted", c.State()) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Cancel +// ───────────────────────────────────────────────────────────────────────────── + +// TestCancel_RejectsSessionMismatch verifies a cancel for a different session is +// rejected. +func TestCancel_RejectsSessionMismatch(t *testing.T) { + c := New() + c.sessionID = "sess-1" + + if err := c.Cancel(t.Context(), "other"); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestCancel_Idempotent verifies cancelling an already-cancelled session is a +// no-op. +func TestCancel_Idempotent(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateCancelled + + if err := c.Cancel(t.Context(), "sess-1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateCancelled { + t.Errorf("state = %s; want Cancelled", c.State()) + } +} + +// TestCancel_Success verifies cancelling an in-flight session moves it to +// cancelled. +func TestCancel_Success(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateTransferCompleted + + if err := c.Cancel(t.Context(), "sess-1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateCancelled { + t.Errorf("state = %s; want Cancelled", c.State()) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Cleanup +// ───────────────────────────────────────────────────────────────────────────── + +// TestCleanup_NoopWhenIdle verifies cleaning up an idle controller is a no-op. +func TestCleanup_NoopWhenIdle(t *testing.T) { + c := New() + + if err := c.Cleanup(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestCleanup_RejectsSessionMismatch verifies cleanup for a different session is +// rejected. +func TestCleanup_RejectsSessionMismatch(t *testing.T) { + c := New() + c.state = StateFinalized + c.sessionID = "sess-1" + + if err := c.Cleanup(t.Context(), "other", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestCleanup_RejectsNotFinalized verifies cleanup is only valid once the +// session has been finalized. +func TestCleanup_RejectsNotFinalized(t *testing.T) { + c := New() + c.state = StateTransferCompleted + c.sessionID = "sess-1" + + if err := c.Cleanup(t.Context(), "sess-1", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestCleanup_Success verifies cleanup returns a finalized session to idle and +// clears its session-scoped state. +func TestCleanup_Success(t *testing.T) { + c := New() + c.state = StateFinalized + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginSource + + if err := c.Cleanup(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateIdle { + t.Errorf("state = %s; want Idle", c.State()) + } + if c.sessionID != "" || c.vmController != nil || c.podControllers != nil { + t.Errorf("session state not cleared: %+v", c) + } +} + +// TestCleanup_ClosesSocketAndNotifier verifies cleanup releases the transport +// socket and tears down the notifier before returning to idle. +func TestCleanup_ClosesSocketAndNotifier(t *testing.T) { + if err := ensureWinsock(); err != nil { + t.Skipf("winsock unavailable: %v", err) + } + sock, err := windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP) + if err != nil { + t.Fatalf("create socket: %v", err) + } + + c := New() + c.state = StateFinalized + c.sessionID = "s" + c.dupSocket = sock + c.notifier = newTestNotifications(hcsschema.MigrationOriginSource) + + if err := c.Cleanup(context.Background(), "s", nil); err != nil { + t.Fatalf("cleanup: %v", err) + } + if c.State() != StateIdle { + t.Errorf("state = %s; want Idle", c.State()) + } + if c.dupSocket != 0 { + t.Error("expected dupSocket to be cleared") + } + if c.notifier != nil { + t.Error("expected notifier to be cleared") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// runTransfer +// ───────────────────────────────────────────────────────────────────────────── + +// TestRunTransfer_Destination verifies the destination issues the destination +// start call followed by the transfer. +func TestRunTransfer_Destination(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartWithMigrationOptions(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + c.sessionID = "s" + c.origin = hcsschema.MigrationOriginDestination + c.vmController = vm + + if err := c.runTransfer(t.Context()); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestRunTransfer_UnsupportedOrigin verifies an unknown origin is rejected. +func TestRunTransfer_UnsupportedOrigin(t *testing.T) { + c := New() + c.origin = hcsschema.MigrationOrigin("bogus") + + if err := c.runTransfer(t.Context()); !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestRunTransfer_StartError verifies a failure starting the migration surfaces. +func TestRunTransfer_StartError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.runTransfer(t.Context()); err == nil { + t.Fatal("expected error, got nil") + } +} + +// TestRunTransfer_TransferError verifies a failure starting the transfer surfaces. +func TestRunTransfer_TransferError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.runTransfer(t.Context()); err == nil { + t.Fatal("expected error, got nil") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// failTransfer +// ───────────────────────────────────────────────────────────────────────────── + +// TestFailTransfer_BroadcastsToSubscribers verifies a failed transfer marks the +// session failed and delivers a failure event to subscribers on either origin. +func TestFailTransfer_BroadcastsToSubscribers(t *testing.T) { + for _, origin := range []hcsschema.MigrationOrigin{hcsschema.MigrationOriginSource, hcsschema.MigrationOriginDestination} { + t.Run(string(origin), func(t *testing.T) { + n := newTestNotifications(origin) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + c := New() + c.origin = origin + c.notifier = n + + c.failTransfer(context.Background(), errors.New("boom")) + + if c.State() != StateFailed { + t.Errorf("state = %s; want Failed", c.State()) + } + if got := recvWithin(t, sub, time.Second); got == nil { + t.Fatal("expected a failure notification") + } + }) + } +} diff --git a/internal/controller/migration/doc.go b/internal/controller/migration/doc.go new file mode 100644 index 0000000000..75db1c582a --- /dev/null +++ b/internal/controller/migration/doc.go @@ -0,0 +1,100 @@ +//go:build windows && lcow + +// Package migration provides a controller for sequencing a single live-migration +// session of an LCOW sandbox between two shims: a source that hands off a running +// sandbox and a destination that receives it. +// +// The [Controller] drives one session at a time. The source captures its sandbox +// as an opaque snapshot ([Controller.PrepareSource], [Controller.ExportState]); +// the destination rehydrates that snapshot ([Controller.ImportState]), rebinds +// each migrated container onto its own IDs ([Controller.PatchResourcePaths]), and +// materializes the VM ([Controller.PrepareDestination]). Once both sides share the +// duplicated transport socket ([Controller.RegisterDuplicateSocket]), the memory +// transfer runs ([Controller.Transfer]); the session is then committed with a +// resume or stop ([Controller.Finalize]) and torn down ([Controller.Cleanup]). +// The VM and pod controllers it drives are owned by the service and only borrowed +// for the session. +// +// # Lifecycle +// +// A session created via [New] starts at [StateIdle] and follows a source or a +// destination path; the two converge once the transport socket is ready: +// +// source destination +// ┌─────────────────────┐ ┌───────────────────────────┐ +// │ StateIdle │ │ StateIdle │ +// └──────────┬──────────┘ └─────────────┬─────────────┘ +// │ PrepareSource │ ImportState +// ▼ ▼ +// ┌─────────────────────┐ ┌───────────────────────────┐ +// │ StateSourcePrepared │ │ StateDestinationImported │ +// └──────────┬──────────┘ └─────────────┬─────────────┘ +// │ ExportState │ PatchResourcePaths (per container) +// ▼ │ then PrepareDestination +// ┌─────────────────────┐ ▼ +// │ StateSourceExported │ ┌───────────────────────────┐ +// └──────────┬──────────┘ │ StateDestinationPrepared │ +// │ └─────────────┬─────────────┘ +// │ RegisterDuplicateSocket │ RegisterDuplicateSocket +// └───────────────┬───────────────────────┘ +// ▼ +// ┌────────────────────┐ +// │ StateSocketReady │ +// └─────────┬──────────┘ +// │ Transfer +// ▼ +// ┌────────────────────┐ +// │ StateTransferring │ +// └─────────┬──────────┘ +// │ transfer ok +// ▼ +// ┌────────────────────────┐ +// │ StateTransferCompleted │ +// └───────────┬────────────┘ +// │ Finalize +// ▼ +// ┌──────────────────┐ +// │ StateFinalized │ +// └────────┬─────────┘ +// │ Cleanup +// ▼ +// ┌──────────────┐ +// │ StateIdle │ +// └──────────────┘ +// +// State descriptions: +// +// - [StateIdle]: no session is active; the initial state and where +// [Controller.Cleanup] returns the controller. +// - [StateSourcePrepared]: [Controller.PrepareSource] has armed the source; the +// next call is [Controller.ExportState]. +// - [StateSourceExported]: [Controller.ExportState] has produced the opaque +// sandbox snapshot; the next call is [Controller.RegisterDuplicateSocket]. +// - [StateDestinationImported]: [Controller.ImportState] has rehydrated the +// snapshot; the next calls are [Controller.PatchResourcePaths] (one per +// container) followed by [Controller.PrepareDestination]. +// - [StateDestinationPrepared]: [Controller.PrepareDestination] has materialized +// the destination HCS compute system; the next call is +// [Controller.RegisterDuplicateSocket]. +// - [StateSocketReady]: the duplicated transport socket has been adopted via +// [Controller.RegisterDuplicateSocket]; the next call is [Controller.Transfer]. +// - [StateSocketWaiting]: [Controller.Transfer] ran before the socket arrived and +// is waiting for it in the background; it advances to [StateTransferring] once +// the socket is registered. +// - [StateTransferring]: [Controller.Transfer] has started the memory transfer; +// it auto-advances to [StateTransferCompleted] or [StateFailed]. +// - [StateTransferCompleted]: the memory transfer finished; the next call is +// [Controller.Finalize]. +// - [StateFinalized]: [Controller.Finalize] has applied the resume or stop; the +// next call is [Controller.Cleanup]. +// - [StateCancelled]: [Controller.Cancel] aborted the session; the next call is +// [Controller.Finalize] followed by [Controller.Cleanup]. +// - [StateFailed]: the memory transfer failed; the next call is +// [Controller.Cancel]. +// +// # Notifications +// +// Callers observe session progress by streaming events with +// [Controller.Subscribe]; a failed transfer is reported to subscribers rather +// than returned from [Controller.Transfer]. +package migration diff --git a/internal/controller/migration/mocks/mock_lcow_migration.go b/internal/controller/migration/mocks/mock_lcow_migration.go new file mode 100644 index 0000000000..d97956308b --- /dev/null +++ b/internal/controller/migration/mocks/mock_lcow_migration.go @@ -0,0 +1,337 @@ +//go:build windows && lcow + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/Microsoft/hcsshim/internal/controller/migration (interfaces: vmController) +// +// Generated by this command: +// +// mockgen -build_flags=-tags=windows,lcow -build_constraint=windows && lcow -package mocks -destination internal/controller/migration/mocks/mock_lcow_migration.go github.com/Microsoft/hcsshim/internal/controller/migration vmController +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + lcow "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + plan9 "github.com/Microsoft/hcsshim/internal/controller/device/plan9" + scsi "github.com/Microsoft/hcsshim/internal/controller/device/scsi" + vpci "github.com/Microsoft/hcsshim/internal/controller/device/vpci" + network "github.com/Microsoft/hcsshim/internal/controller/network" + vm "github.com/Microsoft/hcsshim/internal/controller/vm" + schema2 "github.com/Microsoft/hcsshim/internal/hcs/schema2" + v2 "github.com/Microsoft/hcsshim/internal/hcs/v2" + guestmanager "github.com/Microsoft/hcsshim/internal/vm/guestmanager" + vmmanager "github.com/Microsoft/hcsshim/internal/vm/vmmanager" + gomock "go.uber.org/mock/gomock" + anypb "google.golang.org/protobuf/types/known/anypb" +) + +// MockvmController is a mock of vmController interface. +type MockvmController struct { + ctrl *gomock.Controller + recorder *MockvmControllerMockRecorder + isgomock struct{} +} + +// MockvmControllerMockRecorder is the mock recorder for MockvmController. +type MockvmControllerMockRecorder struct { + mock *MockvmController +} + +// NewMockvmController creates a new mock instance. +func NewMockvmController(ctrl *gomock.Controller) *MockvmController { + mock := &MockvmController{ctrl: ctrl} + mock.recorder = &MockvmControllerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockvmController) EXPECT() *MockvmControllerMockRecorder { + return m.recorder +} + +// CreateVM mocks base method. +func (m *MockvmController) CreateVM(ctx context.Context, opts *vm.CreateOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVM", ctx, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateVM indicates an expected call of CreateVM. +func (mr *MockvmControllerMockRecorder) CreateVM(ctx, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVM", reflect.TypeOf((*MockvmController)(nil).CreateVM), ctx, opts) +} + +// FinalizeLiveMigration mocks base method. +func (m *MockvmController) FinalizeLiveMigration(ctx context.Context, options *schema2.MigrationFinalizedOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinalizeLiveMigration", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// FinalizeLiveMigration indicates an expected call of FinalizeLiveMigration. +func (mr *MockvmControllerMockRecorder) FinalizeLiveMigration(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinalizeLiveMigration", reflect.TypeOf((*MockvmController)(nil).FinalizeLiveMigration), ctx, options) +} + +// Guest mocks base method. +func (m *MockvmController) Guest() *guestmanager.Guest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Guest") + ret0, _ := ret[0].(*guestmanager.Guest) + return ret0 +} + +// Guest indicates an expected call of Guest. +func (mr *MockvmControllerMockRecorder) Guest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Guest", reflect.TypeOf((*MockvmController)(nil).Guest)) +} + +// Import mocks base method. +func (m *MockvmController) Import(ctx context.Context, env *anypb.Any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Import", ctx, env) + ret0, _ := ret[0].(error) + return ret0 +} + +// Import indicates an expected call of Import. +func (mr *MockvmControllerMockRecorder) Import(ctx, env any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Import", reflect.TypeOf((*MockvmController)(nil).Import), ctx, env) +} + +// InitializeLiveMigrationOnSource mocks base method. +func (m *MockvmController) InitializeLiveMigrationOnSource(ctx context.Context, options *schema2.MigrationInitializeOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitializeLiveMigrationOnSource", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitializeLiveMigrationOnSource indicates an expected call of InitializeLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) InitializeLiveMigrationOnSource(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitializeLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).InitializeLiveMigrationOnSource), ctx, options) +} + +// MigrationNotifications mocks base method. +func (m *MockvmController) MigrationNotifications() (<-chan schema2.OperationSystemMigrationNotificationInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MigrationNotifications") + ret0, _ := ret[0].(<-chan schema2.OperationSystemMigrationNotificationInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MigrationNotifications indicates an expected call of MigrationNotifications. +func (mr *MockvmControllerMockRecorder) MigrationNotifications() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MigrationNotifications", reflect.TypeOf((*MockvmController)(nil).MigrationNotifications)) +} + +// NetworkController mocks base method. +func (m *MockvmController) NetworkController(networkNamespaceID string) *network.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NetworkController", networkNamespaceID) + ret0, _ := ret[0].(*network.Controller) + return ret0 +} + +// NetworkController indicates an expected call of NetworkController. +func (mr *MockvmControllerMockRecorder) NetworkController(networkNamespaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkController", reflect.TypeOf((*MockvmController)(nil).NetworkController), networkNamespaceID) +} + +// Patch mocks base method. +func (m *MockvmController) Patch(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Patch", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockvmControllerMockRecorder) Patch(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockvmController)(nil).Patch), ctx) +} + +// Plan9Controller mocks base method. +func (m *MockvmController) Plan9Controller() *plan9.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Plan9Controller") + ret0, _ := ret[0].(*plan9.Controller) + return ret0 +} + +// Plan9Controller indicates an expected call of Plan9Controller. +func (mr *MockvmControllerMockRecorder) Plan9Controller() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Plan9Controller", reflect.TypeOf((*MockvmController)(nil).Plan9Controller)) +} + +// Resume mocks base method. +func (m *MockvmController) Resume(ctx context.Context, rebuildBridge bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resume", ctx, rebuildBridge) + ret0, _ := ret[0].(error) + return ret0 +} + +// Resume indicates an expected call of Resume. +func (mr *MockvmControllerMockRecorder) Resume(ctx, rebuildBridge any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resume", reflect.TypeOf((*MockvmController)(nil).Resume), ctx, rebuildBridge) +} + +// RuntimeID mocks base method. +func (m *MockvmController) RuntimeID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RuntimeID") + ret0, _ := ret[0].(string) + return ret0 +} + +// RuntimeID indicates an expected call of RuntimeID. +func (mr *MockvmControllerMockRecorder) RuntimeID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RuntimeID", reflect.TypeOf((*MockvmController)(nil).RuntimeID)) +} + +// SCSIController mocks base method. +func (m *MockvmController) SCSIController(ctx context.Context) (*scsi.Controller, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SCSIController", ctx) + ret0, _ := ret[0].(*scsi.Controller) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SCSIController indicates an expected call of SCSIController. +func (mr *MockvmControllerMockRecorder) SCSIController(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SCSIController", reflect.TypeOf((*MockvmController)(nil).SCSIController), ctx) +} + +// SandboxOptions mocks base method. +func (m *MockvmController) SandboxOptions() *lcow.SandboxOptions { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SandboxOptions") + ret0, _ := ret[0].(*lcow.SandboxOptions) + return ret0 +} + +// SandboxOptions indicates an expected call of SandboxOptions. +func (mr *MockvmControllerMockRecorder) SandboxOptions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SandboxOptions", reflect.TypeOf((*MockvmController)(nil).SandboxOptions)) +} + +// Save mocks base method. +func (m *MockvmController) Save(ctx context.Context) (*anypb.Any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", ctx) + ret0, _ := ret[0].(*anypb.Any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Save indicates an expected call of Save. +func (mr *MockvmControllerMockRecorder) Save(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockvmController)(nil).Save), ctx) +} + +// StartLiveMigrationOnSource mocks base method. +func (m *MockvmController) StartLiveMigrationOnSource(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationOnSource", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationOnSource indicates an expected call of StartLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationOnSource(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationOnSource), ctx, config) +} + +// StartLiveMigrationTransfer mocks base method. +func (m *MockvmController) StartLiveMigrationTransfer(ctx context.Context, options *schema2.MigrationTransferOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationTransfer", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationTransfer indicates an expected call of StartLiveMigrationTransfer. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationTransfer(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationTransfer", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationTransfer), ctx, options) +} + +// StartWithMigrationOptions mocks base method. +func (m *MockvmController) StartWithMigrationOptions(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartWithMigrationOptions", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartWithMigrationOptions indicates an expected call of StartWithMigrationOptions. +func (mr *MockvmControllerMockRecorder) StartWithMigrationOptions(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartWithMigrationOptions", reflect.TypeOf((*MockvmController)(nil).StartWithMigrationOptions), ctx, config) +} + +// State mocks base method. +func (m *MockvmController) State() vm.State { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "State") + ret0, _ := ret[0].(vm.State) + return ret0 +} + +// State indicates an expected call of State. +func (mr *MockvmControllerMockRecorder) State() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockvmController)(nil).State)) +} + +// VM mocks base method. +func (m *MockvmController) VM() *vmmanager.UtilityVM { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VM") + ret0, _ := ret[0].(*vmmanager.UtilityVM) + return ret0 +} + +// VM indicates an expected call of VM. +func (mr *MockvmControllerMockRecorder) VM() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VM", reflect.TypeOf((*MockvmController)(nil).VM)) +} + +// VPCIController mocks base method. +func (m *MockvmController) VPCIController() *vpci.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VPCIController") + ret0, _ := ret[0].(*vpci.Controller) + return ret0 +} + +// VPCIController indicates an expected call of VPCIController. +func (mr *MockvmControllerMockRecorder) VPCIController() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VPCIController", reflect.TypeOf((*MockvmController)(nil).VPCIController)) +} diff --git a/internal/controller/migration/notifications.go b/internal/controller/migration/notifications.go new file mode 100644 index 0000000000..c69847f8a1 --- /dev/null +++ b/internal/controller/migration/notifications.go @@ -0,0 +1,214 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "fmt" + "sync" + "time" + + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/migration" + + "github.com/containerd/errdefs" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// subscriberBuffer caps how many notifications a subscriber may queue before +// slow readers start dropping updates. +const subscriberBuffer = 64 + +// notifications fans migration events out to every subscriber and replays +// the latest event to late subscribers, all sharing one messageID sequence. +type notifications struct { + // mu guards the mutable fields below. + mu sync.RWMutex + + // subscribers is the set of active streams each event is delivered to. + subscribers map[chan *migration.NotificationsResponse]struct{} + + // lastResponse is the most recent event, replayed to new subscribers. + lastResponse *migration.NotificationsResponse + + // messageID is the monotonically increasing sequence number on each event. + messageID uint32 + + // startTime is when this notifier was created, reported as StartTime on every event. + startTime time.Time + + // done is closed by close to end the session and all subscriber streams. + done chan struct{} + + // origin identifies this host's role (source or destination) in the migration. + origin hcsschema.MigrationOrigin +} + +// newNotifications begins forwarding the VM's migration events to subscribers. +func newNotifications(vmController vmController, origin hcsschema.MigrationOrigin) (*notifications, error) { + // Subscribe to the VM's migration events before any subscriber attaches. + src, err := vmController.MigrationNotifications() + if err != nil { + return nil, fmt.Errorf("get migration notifications channel: %w", err) + } + + notif := ¬ifications{ + subscribers: map[chan *migration.NotificationsResponse]struct{}{}, + startTime: time.Now(), + done: make(chan struct{}), + origin: origin, + } + + // Forward each VM migration event to subscribers until the session is torn down or + // the source stops producing. + go func() { + for { + select { + // Session torn down: stop forwarding. + case <-notif.done: + return + + // Next VM migration event, or the source channel was closed. + case info, ok := <-src: + // Source closed: there is nothing left to forward. + if !ok { + return + } + + // broadcast returns false once the notifier is closed. + if !notif.broadcast(info) { + return + } + } + } + }() + + return notif, nil +} + +// Subscribe returns a stream of migration notifications for the session, +// beginning with the most recent one. The stream ends when ctx is canceled or +// the session is torn down. +func (c *Controller) Subscribe(ctx context.Context, sessionID string) (<-chan *migration.NotificationsResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Reject callers without an active session or whose sessionID does not + // match the active one. + if c.sessionID != sessionID { + return nil, fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Create the notifier on first use; it begins forwarding VM events as + // soon as it exists, independent of whether any subscriber attaches. + if c.notifier == nil { + notifier, err := newNotifications(c.vmController, c.origin) + if err != nil { + return nil, err + } + + c.notifier = notifier + } + + log.G(ctx).Debug("migration notification subscriber attached") + return c.notifier.subscribe(ctx) +} + +// subscribe returns a channel that first replays the latest notification, then +// delivers every later one until ctx is canceled or the notifier closes. +func (n *notifications) subscribe(ctx context.Context) (<-chan *migration.NotificationsResponse, error) { + subscriber := make(chan *migration.NotificationsResponse, subscriberBuffer) + + n.mu.Lock() + defer n.mu.Unlock() + + // Session already terminated: reject the subscription. + select { + case <-n.done: + return nil, fmt.Errorf("migration session already terminated: %w", errdefs.ErrFailedPrecondition) + default: + } + + // Replay the latest event so a late subscriber has immediate context; + // the buffered channel keeps this send non-blocking. + if n.lastResponse != nil { + subscriber <- n.lastResponse + } + n.subscribers[subscriber] = struct{}{} + + // Drop the subscriber once its context ends or the notifier closes. + go func() { + select { + case <-ctx.Done(): + case <-n.done: + return + } + + n.mu.Lock() + defer n.mu.Unlock() + + // Skip if broadcast or close already removed this subscriber, to + // avoid a double close. + if _, ok := n.subscribers[subscriber]; ok { + delete(n.subscribers, subscriber) + close(subscriber) + } + }() + + return subscriber, nil +} + +// broadcast delivers info to every subscriber and caches it for replay, +// returning false once the notifier has been closed. +func (n *notifications) broadcast(info hcsschema.OperationSystemMigrationNotificationInfo) bool { + n.mu.Lock() + defer n.mu.Unlock() + + // Notifier closed: drop the event and signal the forwarder to stop. + select { + case <-n.done: + return false + default: + } + + // Stamp the next sequence number and cache the event for replay. + n.messageID++ + n.lastResponse = &migration.NotificationsResponse{ + MessageID: n.messageID, + Notification: migration.ToNotification(info, n.origin), + StartTime: timestamppb.New(n.startTime), + UpdateTime: timestamppb.Now(), + } + + // Non-blocking send so one slow subscriber cannot stall the others. + for subscriber := range n.subscribers { + select { + case subscriber <- n.lastResponse: + default: + } + } + + return true +} + +// close stops forwarding events and closes every subscriber channel. Safe to +// call more than once. +func (n *notifications) close() { + n.mu.Lock() + defer n.mu.Unlock() + + // Already closed: nothing to do. + select { + case <-n.done: + return + default: + } + + // Signal the forwarder to stop, then close every subscriber stream. + close(n.done) + for subscriber := range n.subscribers { + close(subscriber) + delete(n.subscribers, subscriber) + } +} diff --git a/internal/controller/migration/notifications_test.go b/internal/controller/migration/notifications_test.go new file mode 100644 index 0000000000..dcb6da717b --- /dev/null +++ b/internal/controller/migration/notifications_test.go @@ -0,0 +1,329 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "testing" + "time" + + "go.uber.org/mock/gomock" + + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/pkg/migration" +) + +// newTestNotifications builds a notifier without the source-forwarding +// goroutine so the fan-out logic can be driven directly via broadcast. +func newTestNotifications(origin hcsschema.MigrationOrigin) *notifications { + return ¬ifications{ + subscribers: map[chan *migration.NotificationsResponse]struct{}{}, + startTime: time.Now(), + done: make(chan struct{}), + origin: origin, + } +} + +func setupDoneInfo() hcsschema.OperationSystemMigrationNotificationInfo { + return hcsschema.OperationSystemMigrationNotificationInfo{Event: hcsschema.MigrationEventSetupDone} +} + +// recvWithin returns the next notification or fails if none arrives in time. +func recvWithin(t *testing.T, ch <-chan *migration.NotificationsResponse, d time.Duration) *migration.NotificationsResponse { + t.Helper() + select { + case r := <-ch: + return r + case <-time.After(d): + t.Fatal("timed out waiting for notification") + return nil + } +} + +// waitChannelClosed drains ch until it is closed or fails on timeout. +func waitChannelClosed(t *testing.T, ch <-chan *migration.NotificationsResponse, d time.Duration) { + t.Helper() + deadline := time.After(d) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("channel not closed within timeout") + } + } +} + +// TestNotificationsBroadcastDeliversToSubscriber verifies a broadcast reaches +// an attached subscriber with a populated response. +func TestNotificationsBroadcastDeliversToSubscriber(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + if ok := n.broadcast(setupDoneInfo()); !ok { + t.Fatal("broadcast returned false on an open notifier") + } + + got := recvWithin(t, sub, time.Second) + if got.MessageID != 1 { + t.Fatalf("messageID: got %d want 1", got.MessageID) + } + if got.Notification == nil || got.Notification.Phase != migration.Phase_PHASE_SETUP_DONE { + t.Fatalf("unexpected notification: %+v", got.Notification) + } + if got.Notification.Origin != migration.Origin_ORIGIN_SOURCE { + t.Fatalf("origin: got %s want %s", got.Notification.Origin, migration.Origin_ORIGIN_SOURCE) + } + if got.StartTime == nil || got.UpdateTime == nil { + t.Fatal("expected StartTime and UpdateTime to be set") + } +} + +// TestNotificationsBroadcastFansOutToAllSubscribers verifies every subscriber +// receives the same event. +func TestNotificationsBroadcastFansOutToAllSubscribers(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub1, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe sub1: %v", err) + } + sub2, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe sub2: %v", err) + } + + n.broadcast(setupDoneInfo()) + + if r := recvWithin(t, sub1, time.Second); r.MessageID != 1 { + t.Fatalf("sub1 messageID: got %d want 1", r.MessageID) + } + if r := recvWithin(t, sub2, time.Second); r.MessageID != 1 { + t.Fatalf("sub2 messageID: got %d want 1", r.MessageID) + } +} + +// TestNotificationsBroadcastIncrementsMessageID verifies the per-stream +// counter increases monotonically across events. +func TestNotificationsBroadcastIncrementsMessageID(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + n.broadcast(setupDoneInfo()) + n.broadcast(setupDoneInfo()) + + if r := recvWithin(t, sub, time.Second); r.MessageID != 1 { + t.Fatalf("first messageID: got %d want 1", r.MessageID) + } + if r := recvWithin(t, sub, time.Second); r.MessageID != 2 { + t.Fatalf("second messageID: got %d want 2", r.MessageID) + } +} + +// TestNotificationsSubscribeReplaysLatest verifies a late subscriber +// immediately receives the most recent event. +func TestNotificationsSubscribeReplaysLatest(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Broadcast before anyone subscribes. + n.broadcast(setupDoneInfo()) + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + if r := recvWithin(t, sub, time.Second); r.MessageID != 1 { + t.Fatalf("replayed messageID: got %d want 1", r.MessageID) + } +} + +// TestNotificationsSubscribeAfterCloseFails verifies subscribing to a +// terminated notifier is rejected. +func TestNotificationsSubscribeAfterCloseFails(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + n.close() + + if _, err := n.subscribe(context.Background()); err == nil { + t.Fatal("expected error subscribing to a closed notifier") + } +} + +// TestNotificationsBroadcastAfterCloseReturnsFalse verifies broadcast signals +// the forwarder to stop once the notifier is closed. +func TestNotificationsBroadcastAfterCloseReturnsFalse(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + n.close() + + if n.broadcast(setupDoneInfo()) { + t.Fatal("broadcast should return false after close") + } +} + +// TestNotificationsCloseClosesSubscribers verifies close ends every active +// subscriber stream. +func TestNotificationsCloseClosesSubscribers(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + n.close() + waitChannelClosed(t, sub, time.Second) +} + +// TestNotificationsCloseIsIdempotent verifies close can be called repeatedly. +func TestNotificationsCloseIsIdempotent(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + n.close() + n.close() +} + +// TestNotificationsSubscribeContextCancelDropsSubscriber verifies a canceled +// context removes and closes the subscriber, leaving other delivery intact. +func TestNotificationsSubscribeContextCancelDropsSubscriber(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + cancel() + waitChannelClosed(t, sub, time.Second) + + // The notifier is still open and broadcasting must not panic on the + // dropped subscriber. + if ok := n.broadcast(setupDoneInfo()); !ok { + t.Fatal("broadcast on an open notifier returned false") + } +} + +// TestNotificationsBroadcastDoesNotBlockOnSlowSubscriber verifies a subscriber +// that never reads cannot stall the broadcaster. +func TestNotificationsBroadcastDoesNotBlockOnSlowSubscriber(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Subscribe but never drain the channel. + if _, err := n.subscribe(ctx); err != nil { + t.Fatalf("subscribe: %v", err) + } + + done := make(chan struct{}) + go func() { + // Broadcasting well past the buffer must not block; extras are dropped. + for i := 0; i < subscriberBuffer*2; i++ { + n.broadcast(setupDoneInfo()) + } + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("broadcast blocked on a slow subscriber") + } +} + +// TestControllerSubscribeNoActiveSession verifies Subscribe is rejected when +// no session is active. +func TestControllerSubscribeNoActiveSession(t *testing.T) { + c := &Controller{} + if _, err := c.Subscribe(context.Background(), "any"); err == nil { + t.Fatal("expected error when no session is active") + } +} + +// TestControllerSubscribeSessionMismatch verifies Subscribe rejects a +// sessionID that does not match the active one. +func TestControllerSubscribeSessionMismatch(t *testing.T) { + c := &Controller{sessionID: "active"} + if _, err := c.Subscribe(context.Background(), "other"); err == nil { + t.Fatal("expected error on session mismatch") + } +} + +// TestControllerSubscribeReusesExistingNotifier verifies Subscribe attaches to +// an already-created notifier and delivers its events. +func TestControllerSubscribeReusesExistingNotifier(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + c := &Controller{sessionID: "s", notifier: n} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := c.Subscribe(ctx, "s") + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + n.broadcast(setupDoneInfo()) + if r := recvWithin(t, sub, time.Second); r.MessageID != 1 { + t.Fatalf("messageID: got %d want 1", r.MessageID) + } +} + +// TestControllerSubscribeCreatesNotifier verifies the first Subscribe builds the +// notifier from the VM's event stream and forwards its events to the subscriber. +func TestControllerSubscribeCreatesNotifier(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + + src := make(chan hcsschema.OperationSystemMigrationNotificationInfo, 1) + var recv <-chan hcsschema.OperationSystemMigrationNotificationInfo = src + vm.EXPECT().MigrationNotifications().Return(recv, nil) + + c := &Controller{sessionID: "s", origin: hcsschema.MigrationOriginSource, vmController: vm} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := c.Subscribe(ctx, "s") + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer c.notifier.close() + + // An event on the VM stream is forwarded to the subscriber. + src <- setupDoneInfo() + if got := recvWithin(t, sub, time.Second); got == nil { + t.Fatal("expected forwarded notification") + } +} + +// TestControllerSubscribeNotifierError verifies Subscribe surfaces a failure to +// obtain the VM's notification stream. +func TestControllerSubscribeNotifierError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().MigrationNotifications().Return(nil, errors.New("boom")) + + c := &Controller{sessionID: "s", vmController: vm} + if _, err := c.Subscribe(context.Background(), "s"); err == nil { + t.Fatal("expected error, got nil") + } +} diff --git a/internal/controller/migration/socket.go b/internal/controller/migration/socket.go new file mode 100644 index 0000000000..a97645f14e --- /dev/null +++ b/internal/controller/migration/socket.go @@ -0,0 +1,114 @@ +//go:build windows && lcow + +package migration + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "sync" + "unsafe" + + "github.com/Microsoft/hcsshim/internal/log" + "github.com/containerd/errdefs" + "golang.org/x/sys/windows" +) + +// wsaVersion is the Winsock version requested by [ensureWinsock]. +const wsaVersion uint32 = 0x0202 + +// ensureWinsock initializes Winsock once for the process so a socket can be +// recreated without depending on other code having initialized it first. +// WSACleanup is deliberately never called, so Winsock stays up for the +// process lifetime. +var ensureWinsock = sync.OnceValue(func() error { + var data windows.WSAData + if err := windows.WSAStartup(wsaVersion, &data); err != nil { + return fmt.Errorf("WSAStartup: %w", err) + } + + return nil +}) + +// soConnectTime is the SO_CONNECT_TIME socket option. Querying it with +// getsockopt yields how many seconds the socket has been connected, or +// 0xFFFFFFFF if it has never connected. It is not exported by +// [golang.org/x/sys/windows]. +const soConnectTime int32 = 0x700C + +// connectTimeNotConnected is the SO_CONNECT_TIME sentinel returned for an +// unconnected socket. +const connectTimeNotConnected uint32 = 0xFFFFFFFF + +// RegisterDuplicateSocket adopts a duplicated migration transport socket, +// described by protocolInfo, into this process and makes it available to the +// pending transfer for the given session. A repeat call for an already-adopted +// session is a no-op. +func (c *Controller) RegisterDuplicateSocket(ctx context.Context, sessionID string, protocolInfo []byte) error { + // Reject input too small to hold a serialized socket descriptor before + // attempting to decode it. + wantSize := int(unsafe.Sizeof(windows.WSAProtocolInfo{})) + if len(protocolInfo) < wantSize { + return fmt.Errorf("protocol info is %d bytes, want at least %d: %w", len(protocolInfo), wantSize, errdefs.ErrInvalidArgument) + } + + // Decode the opaque caller-supplied bytes into the socket descriptor + // used to recreate the duplicated socket. + var info windows.WSAProtocolInfo + if err := binary.Read(bytes.NewReader(protocolInfo), binary.LittleEndian, &info); err != nil { + return fmt.Errorf("decode WSAProtocolInfo: %w", err) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Idempotent: a repeat call for the same session is a no-op once the + // socket has been adopted. + if c.dupSocket != 0 { + return nil + } + + // Transfer may have already claimed the session (StateSocketWaiting) and + // be waiting on the socket; allow registration in that case too. + if c.state != StateSourceExported && c.state != StateDestinationPrepared && c.state != StateSocketWaiting { + return fmt.Errorf("register duplicate socket requires state %s or %s (current: %s): %w", StateSourceExported, StateDestinationPrepared, c.state, errdefs.ErrFailedPrecondition) + } + + // Make sure Winsock is up for this process before recreating the socket. + if err := ensureWinsock(); err != nil { + return err + } + + // Recreate the duplicated socket in this process from the descriptor so + // the transfer can use it as its transport. + sock, err := windows.WSASocket(info.AddressFamily, info.SocketType, info.Protocol, &info, 0, 0) + if err != nil { + return fmt.Errorf("WSASocket: %w", err) + } + + // Verify the duplicated handle actually represents a connected socket; + // HCS will fail the migration if we hand it an unconnected endpoint. + var connectTime uint32 + optLen := int32(unsafe.Sizeof(connectTime)) + if err := windows.Getsockopt(sock, windows.SOL_SOCKET, soConnectTime, (*byte)(unsafe.Pointer(&connectTime)), &optLen); err != nil { + _ = windows.Closesocket(sock) + return fmt.Errorf("getsockopt SO_CONNECT_TIME: %w", err) + } + + if connectTime == connectTimeNotConnected { + _ = windows.Closesocket(sock) + return fmt.Errorf("duplicated socket is not connected: %w", errdefs.ErrFailedPrecondition) + } + + c.dupSocket = sock + c.state = StateSocketReady + close(c.socketReady) + + log.G(ctx).Info("duplicate migration socket registered") + return nil +} diff --git a/internal/controller/migration/socket_test.go b/internal/controller/migration/socket_test.go new file mode 100644 index 0000000000..71d2be6759 --- /dev/null +++ b/internal/controller/migration/socket_test.go @@ -0,0 +1,68 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "testing" + "unsafe" + + "github.com/containerd/errdefs" + "golang.org/x/sys/windows" +) + +// validProtocolInfo returns a correctly-sized (all-zero) serialized descriptor +// that passes the size check and decode, so tests can exercise the later guards. +func validProtocolInfo() []byte { + return make([]byte, int(unsafe.Sizeof(windows.WSAProtocolInfo{}))) +} + +// TestRegisterDuplicateSocket_TooSmall verifies a buffer too short to hold a +// socket descriptor is rejected as an invalid argument before any decode. +func TestRegisterDuplicateSocket_TooSmall(t *testing.T) { + c := New() + + err := c.RegisterDuplicateSocket(context.Background(), "", []byte{0x00}) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("got %v, want ErrInvalidArgument", err) + } +} + +// TestRegisterDuplicateSocket_SessionMismatch verifies a call for a session +// other than the active one is rejected as an invalid argument. +func TestRegisterDuplicateSocket_SessionMismatch(t *testing.T) { + c := New() + c.sessionID = "active" + + err := c.RegisterDuplicateSocket(context.Background(), "other", validProtocolInfo()) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("got %v, want ErrInvalidArgument", err) + } +} + +// TestRegisterDuplicateSocket_Idempotent verifies a repeat call once a socket +// has been adopted is a no-op, regardless of state. +func TestRegisterDuplicateSocket_Idempotent(t *testing.T) { + c := New() + c.sessionID = "s" + c.dupSocket = windows.Handle(1) + c.state = StateSourceExported + + if err := c.RegisterDuplicateSocket(context.Background(), "s", validProtocolInfo()); err != nil { + t.Fatalf("got %v, want nil", err) + } +} + +// TestRegisterDuplicateSocket_InvalidState verifies registration is rejected as +// a failed precondition when the session is not awaiting a socket. +func TestRegisterDuplicateSocket_InvalidState(t *testing.T) { + c := New() + c.sessionID = "s" + c.state = StateIdle + + err := c.RegisterDuplicateSocket(context.Background(), "s", validProtocolInfo()) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("got %v, want ErrFailedPrecondition", err) + } +} diff --git a/internal/controller/migration/state.go b/internal/controller/migration/state.go new file mode 100644 index 0000000000..dbd2532e71 --- /dev/null +++ b/internal/controller/migration/state.go @@ -0,0 +1,116 @@ +//go:build windows && lcow + +package migration + +// State is the current state of a live migration session, advanced by the +// [Controller] APIs. The source and destination follow distinct paths that +// converge once the transport socket is ready, and both return to [StateIdle]. +// +// Source progression: +// +// StateIdle → StateSourcePrepared → StateSourceExported → StateSocketReady +// → StateTransferring → StateTransferCompleted → StateFinalized → StateIdle +// +// Destination progression: +// +// StateIdle → StateDestinationImported → StateDestinationPrepared → StateSocketReady +// → StateTransferring → StateTransferCompleted → StateFinalized → StateIdle +// +// [StateSocketWaiting] is entered instead of [StateSocketReady] when +// [Controller.Transfer] runs before the socket arrives. A failed transfer +// enters [StateFailed], from which [Controller.Cancel] moves to +// [StateCancelled]; [Controller.Finalize] then [Controller.Cleanup] wind a +// canceled or completed session down to [StateIdle]. +type State int32 + +const ( + // StateIdle indicates no migration session is active. This is the + // initial state and the state the controller returns to after + // [Controller.Cleanup] completes. + StateIdle State = iota + + // StateSourcePrepared indicates [Controller.PrepareSource] has armed + // the source-side migration. The next valid call is + // [Controller.ExportState]. + StateSourcePrepared + + // StateSourceExported indicates the source has produced its opaque saved-state + // envelope via [Controller.ExportState]. The next valid call is + // [Controller.RegisterDuplicateSocket]. + StateSourceExported + + // StateDestinationImported indicates the destination has rehydrated the source + // snapshot via [Controller.ImportState]. The next valid calls are + // [Controller.PatchResourcePaths] (one per container) followed by + // [Controller.PrepareDestination]. + StateDestinationImported + + // StateDestinationPrepared indicates the destination has materialized the + // HCS compute system via [Controller.PrepareDestination]. The next + // valid call is [Controller.RegisterDuplicateSocket]. + StateDestinationPrepared + + // StateSocketWaiting indicates [Controller.Transfer] has claimed the + // session and a single goroutine is driving it, waiting for the duplicate + // socket if it is not yet ready. + StateSocketWaiting + + // StateSocketReady indicates the duplicated migration transport socket + // has been adopted via [Controller.RegisterDuplicateSocket]. The next + // valid call is [Controller.Transfer]. + StateSocketReady + + // StateTransferring indicates [Controller.Transfer] has kicked off the + // memory transfer. The controller automatically transitions to + // [StateTransferCompleted] or [StateFailed] when the transfer ends. + StateTransferring + + // StateTransferCompleted indicates the memory transfer finished + // successfully. The next valid call is [Controller.Finalize]. + StateTransferCompleted + + // StateFinalized indicates [Controller.Finalize] has applied the final + // operation. The next valid call is [Controller.Cleanup]. + StateFinalized + + // StateCancelled indicates the session was aborted via [Controller.Cancel]. + // The next valid call is [Controller.Finalize], followed by + // [Controller.Cleanup] to return the controller to [StateIdle]. + StateCancelled + + // StateFailed indicates the memory transfer failed. The next valid call is + // [Controller.Cancel] to abort the session. + StateFailed +) + +// String returns a human-readable representation of the migration State. +func (s State) String() string { + switch s { + case StateIdle: + return "Idle" + case StateSourcePrepared: + return "SourcePrepared" + case StateSourceExported: + return "SourceExported" + case StateDestinationImported: + return "DestinationImported" + case StateDestinationPrepared: + return "DestinationPrepared" + case StateSocketWaiting: + return "SocketWaiting" + case StateSocketReady: + return "SocketReady" + case StateTransferring: + return "Transferring" + case StateTransferCompleted: + return "TransferCompleted" + case StateFinalized: + return "Finalized" + case StateCancelled: + return "Cancelled" + case StateFailed: + return "Failed" + default: + return "Unknown" + } +} diff --git a/internal/controller/migration/types.go b/internal/controller/migration/types.go new file mode 100644 index 0000000000..800be0ecf4 --- /dev/null +++ b/internal/controller/migration/types.go @@ -0,0 +1,90 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + "github.com/Microsoft/hcsshim/internal/controller/device/plan9" + "github.com/Microsoft/hcsshim/internal/controller/device/scsi" + "github.com/Microsoft/hcsshim/internal/controller/device/vpci" + "github.com/Microsoft/hcsshim/internal/controller/network" + "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" + "github.com/Microsoft/hcsshim/internal/vm/guestmanager" + "github.com/Microsoft/hcsshim/internal/vm/vmmanager" + + "google.golang.org/protobuf/types/known/anypb" +) + +// vmController is the subset of the VM controller that the migration controller +// drives. It is declared as an interface so tests can substitute a mock; it is +// implemented by [vm.Controller]. +type vmController interface { + State() vm.State + SandboxOptions() *lcow.SandboxOptions + InitializeLiveMigrationOnSource(ctx context.Context, options *hcsschema.MigrationInitializeOptions) error + Save(ctx context.Context) (*anypb.Any, error) + Import(ctx context.Context, env *anypb.Any) error + CreateVM(ctx context.Context, opts *vm.CreateOptions) error + Patch(ctx context.Context) error + StartLiveMigrationOnSource(ctx context.Context, config *hcs.MigrationConfig) error + StartWithMigrationOptions(ctx context.Context, config *hcs.MigrationConfig) error + StartLiveMigrationTransfer(ctx context.Context, options *hcsschema.MigrationTransferOptions) error + FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error + Resume(ctx context.Context, rebuildBridge bool) error + MigrationNotifications() (<-chan hcsschema.OperationSystemMigrationNotificationInfo, error) + + RuntimeID() string + VM() *vmmanager.UtilityVM + Guest() *guestmanager.Guest + SCSIController(ctx context.Context) (*scsi.Controller, error) + VPCIController() *vpci.Controller + Plan9Controller() *plan9.Controller + NetworkController(networkNamespaceID string) *network.Controller +} + +// InitOptions carries the fields common to every migration entry point that +// binds a controller to a session. +type InitOptions struct { + // SessionID correlates all calls belonging to a single migration session. + SessionID string + + // Origin selects whether this side acts as the migration source or destination. + Origin hcsschema.MigrationOrigin + + // VMController is the VM driven by the session: saved on the source, rehydrated + // on the destination. Owned by the service; the controller only borrows it. + VMController vmController + + // PodControllers are the sandbox's pods, keyed by pod ID, migrated alongside the + // VM. Owned by the service; the controller only borrows the map. + PodControllers map[string]*pod.Controller +} + +// PrepareSourceOptions configures the source side of a migration session. +type PrepareSourceOptions struct { + InitOptions + + // MigrationOpts tunes the HCS migration workflow; optional, defaulted when nil. + MigrationOpts *hcsschema.MigrationInitializeOptions +} + +// ImportStateOptions configures rehydrating a source snapshot on the destination side. +type ImportStateOptions struct { + InitOptions + + // SandboxID is the destination sandbox ID the snapshot is imported into. + SandboxID string + + // SavedState is the opaque snapshot produced by the source's ExportState. + SavedState *anypb.Any + + // ContainerPodMapping is the service-owned containerID -> podID index. + // ImportState populates it and PatchResourcePaths renames its entries + // in place; the service continues to own its lifetime. + ContainerPodMapping map[string]string +} diff --git a/pkg/migration/parse.go b/pkg/migration/parse.go index e6c66c4e31..fe2bb15505 100644 --- a/pkg/migration/parse.go +++ b/pkg/migration/parse.go @@ -3,9 +3,11 @@ package migration import ( + "encoding/json" "fmt" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "google.golang.org/protobuf/types/known/timestamppb" ) // InitializeOptionsFromProto converts a protobuf [InitializeOptions] to the @@ -71,3 +73,103 @@ func compressionSettingsFromProto(p *CompressionSettings) *hcsschema.MigrationCo ThrottleWorkerCount: p.ThrottleWorkerCount, } } + +// ToOrigin maps an HCS migration origin to its wire form, falling back to the +// controller-known origin when HCS leaves the field empty. +func ToOrigin(origin, fallback hcsschema.MigrationOrigin) Origin { + if origin == "" { + origin = fallback + } + + switch origin { + case hcsschema.MigrationOriginSource: + return Origin_ORIGIN_SOURCE + case hcsschema.MigrationOriginDestination: + return Origin_ORIGIN_DESTINATION + } + + return Origin_ORIGIN_UNSPECIFIED +} + +// ToPhase maps an HCS migration event to its wire-form phase, returning +// PHASE_UNSPECIFIED for an unrecognized event. +func ToPhase(event hcsschema.MigrationEvent) Phase { + switch event { + case hcsschema.MigrationEventSetupDone: + return Phase_PHASE_SETUP_DONE + case hcsschema.MigrationEventTransferInProgress: + return Phase_PHASE_TRANSFER_IN_PROGRESS + case hcsschema.MigrationEventBlackoutStarted: + return Phase_PHASE_BLACKOUT_STARTED + case hcsschema.MigrationEventOfflineDone: + return Phase_PHASE_OFFLINE_DONE + case hcsschema.MigrationEventBlackoutExited: + return Phase_PHASE_BLACKOUT_EXITED + case hcsschema.MigrationEventMigrationDone: + return Phase_PHASE_DONE + case hcsschema.MigrationEventMigrationRecoveryDone: + return Phase_PHASE_RECOVERY_DONE + case hcsschema.MigrationEventMigrationFailed: + return Phase_PHASE_FAILED + } + + return Phase_PHASE_UNSPECIFIED +} + +// ToPhaseState maps an HCS migration result to its wire-form state. +func ToPhaseState(result hcsschema.MigrationResult, phase Phase) PhaseState { + switch result { + case hcsschema.MigrationResultSuccess: + return PhaseState_PHASE_STATE_SUCCESS + case hcsschema.MigrationResultMigrationCancelled: + return PhaseState_PHASE_STATE_CANCELLED + case hcsschema.MigrationResultGuestInitiatedCancellation: + return PhaseState_PHASE_STATE_GUEST_INITIATED_CANCELLATION + case hcsschema.MigrationResultSourceMigrationFailed: + return PhaseState_PHASE_STATE_SOURCE_FAILED + case hcsschema.MigrationResultDestinationMigrationFailed: + return PhaseState_PHASE_STATE_DESTINATION_FAILED + case hcsschema.MigrationResultMigrationRecoveryFailed: + return PhaseState_PHASE_STATE_RECOVERY_FAILED + } + + // No HCS result: progress phases imply forward progress (failures arrive + // as PHASE_FAILED), so default to SUCCESS; terminal phases stay UNSPECIFIED + // so callers can tell "HCS did not say" from a real outcome. + switch phase { + case Phase_PHASE_SETUP_DONE, + Phase_PHASE_TRANSFER_IN_PROGRESS, + Phase_PHASE_BLACKOUT_STARTED, + Phase_PHASE_OFFLINE_DONE, + Phase_PHASE_BLACKOUT_EXITED: + return PhaseState_PHASE_STATE_SUCCESS + } + + return PhaseState_PHASE_STATE_UNSPECIFIED +} + +// ToNotification converts an HCS migration event into its wire-form notification. +func ToNotification(info hcsschema.OperationSystemMigrationNotificationInfo, fallbackOrigin hcsschema.MigrationOrigin) *Notification { + phase := ToPhase(info.Event) + notification := &Notification{ + Origin: ToOrigin(info.Origin, fallbackOrigin), + Phase: phase, + State: ToPhaseState(info.Result, phase), + } + + if info.Event == hcsschema.MigrationEventBlackoutExited && len(info.AdditionalDetails) > 0 { + // On unmarshal failure we drop PhaseDetails rather than the whole + // notification; the core phase/state info is still useful. + var details hcsschema.BlackoutExitedEventDetails + if err := json.Unmarshal(info.AdditionalDetails, &details); err == nil { + notification.PhaseDetails = &Notification_BlackoutExited{ + BlackoutExited: &BlackoutExitedEventDetails{ + BlackoutDurationMilliseconds: details.BlackoutDurationMilliseconds, + BlackoutStopTimestamp: timestamppb.New(details.BlackoutStopTimestamp), + }, + } + } + } + + return notification +} diff --git a/pkg/migration/parse_test.go b/pkg/migration/parse_test.go new file mode 100644 index 0000000000..2ac7eff2ea --- /dev/null +++ b/pkg/migration/parse_test.go @@ -0,0 +1,310 @@ +//go:build windows + +package migration + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func ptr[T any](v T) *T { return &v } + +// TestInitializeOptionsFromProto verifies the wire init options are converted to +// the HCS options: nil and unsupported transports are handled, nested params are +// copied (or left nil), and an out-of-range throttle percentage wraps to uint8. +func TestInitializeOptionsFromProto(t *testing.T) { + tests := []struct { + name string + in *InitializeOptions + want *hcsschema.MigrationInitializeOptions + wantErr bool + }{ + { + name: "nil input returns nil", + }, + { + name: "unsupported transport returns error", + in: &InitializeOptions{MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_UNSPECIFIED}, + wantErr: true, + }, + { + name: "nil nested params are preserved as nil", + in: &InitializeOptions{ + MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_TCP, + ChecksumVerification: true, + }, + want: &hcsschema.MigrationInitializeOptions{ + MemoryTransport: hcsschema.MigrationMemoryTransportTCP, + ChecksumVerification: true, + }, + }, + { + name: "full conversion with nested params", + in: &InitializeOptions{ + MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_TCP, + MemoryTransferThrottleParams: &MemoryTransferThrottleParams{ + SkipThrottling: ptr(true), + ThrottlingScale: ptr(42.5), + MinimumThrottlePercentage: ptr(uint32(50)), + TargetNumberOfBrownoutTransferPasses: ptr(uint32(3)), + StartingBrownoutPassNumberForThrottling: ptr(uint32(1)), + MaximumNumberOfBrownoutTransferPasses: ptr(uint32(7)), + TargetBlackoutTransferTime: ptr(uint32(100)), + BlackoutTimeThresholdForCancellingMigration: ptr(uint32(200)), + }, + CompressionSettings: &CompressionSettings{ThrottleWorkerCount: ptr(uint32(4))}, + ChecksumVerification: true, + PerfTracingEnabled: true, + CancelIfBlackoutThresholdExceeds: true, + PrepareMemoryTransferMode: true, + }, + want: &hcsschema.MigrationInitializeOptions{ + MemoryTransport: hcsschema.MigrationMemoryTransportTCP, + MemoryTransferThrottleParams: &hcsschema.MemoryMigrationTransferThrottleParams{ + SkipThrottling: ptr(true), + ThrottlingScale: ptr(42.5), + MinimumThrottlePercentage: ptr(uint8(50)), + TargetNumberOfBrownoutTransferPasses: ptr(uint32(3)), + StartingBrownoutPassNumberForThrottling: ptr(uint32(1)), + MaximumNumberOfBrownoutTransferPasses: ptr(uint32(7)), + TargetBlackoutTransferTime: ptr(uint32(100)), + BlackoutTimeThresholdForCancellingMigration: ptr(uint32(200)), + }, + CompressionSettings: &hcsschema.MigrationCompressionSettings{ThrottleWorkerCount: ptr(uint32(4))}, + ChecksumVerification: true, + PerfTracingEnabled: true, + CancelIfBlackoutThresholdExceeds: true, + PrepareMemoryTransferMode: true, + }, + }, + { + // A throttle percentage above 255 wraps when narrowed to uint8. + name: "minimum throttle percentage truncates to uint8", + in: &InitializeOptions{ + MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_TCP, + MemoryTransferThrottleParams: &MemoryTransferThrottleParams{MinimumThrottlePercentage: ptr(uint32(300))}, + }, + want: &hcsschema.MigrationInitializeOptions{ + MemoryTransport: hcsschema.MigrationMemoryTransportTCP, + MemoryTransferThrottleParams: &hcsschema.MemoryMigrationTransferThrottleParams{MinimumThrottlePercentage: ptr(uint8(44))}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := InitializeOptionsFromProto(tt.in) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("got %+v, want %+v", got, tt.want) + } + }) + } +} + +// TestToOrigin verifies a caller sees the migration side mapped to the wire +// origin, with the controller-known fallback used only when HCS omits the origin +// and unknown origins reported as unspecified. +func TestToOrigin(t *testing.T) { + tests := []struct { + name string + origin hcsschema.MigrationOrigin + fallback hcsschema.MigrationOrigin + want Origin + }{ + {name: "empty falls back to source", fallback: hcsschema.MigrationOriginSource, want: Origin_ORIGIN_SOURCE}, + {name: "empty falls back to destination", fallback: hcsschema.MigrationOriginDestination, want: Origin_ORIGIN_DESTINATION}, + {name: "empty with empty fallback is unspecified", want: Origin_ORIGIN_UNSPECIFIED}, + {name: "source ignores fallback", origin: hcsschema.MigrationOriginSource, fallback: hcsschema.MigrationOriginDestination, want: Origin_ORIGIN_SOURCE}, + {name: "destination", origin: hcsschema.MigrationOriginDestination, want: Origin_ORIGIN_DESTINATION}, + {name: "unknown non-empty origin ignores fallback", origin: hcsschema.MigrationOrigin("Bogus"), fallback: hcsschema.MigrationOriginSource, want: Origin_ORIGIN_UNSPECIFIED}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToOrigin(tt.origin, tt.fallback); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +// TestToPhase verifies every HCS migration event maps to its wire phase and that +// unknown or empty events fall back to the unspecified phase. +func TestToPhase(t *testing.T) { + tests := []struct { + event hcsschema.MigrationEvent + want Phase + }{ + {hcsschema.MigrationEventSetupDone, Phase_PHASE_SETUP_DONE}, + {hcsschema.MigrationEventTransferInProgress, Phase_PHASE_TRANSFER_IN_PROGRESS}, + {hcsschema.MigrationEventBlackoutStarted, Phase_PHASE_BLACKOUT_STARTED}, + {hcsschema.MigrationEventOfflineDone, Phase_PHASE_OFFLINE_DONE}, + {hcsschema.MigrationEventBlackoutExited, Phase_PHASE_BLACKOUT_EXITED}, + {hcsschema.MigrationEventMigrationDone, Phase_PHASE_DONE}, + {hcsschema.MigrationEventMigrationRecoveryDone, Phase_PHASE_RECOVERY_DONE}, + {hcsschema.MigrationEventMigrationFailed, Phase_PHASE_FAILED}, + {hcsschema.MigrationEventUnknown, Phase_PHASE_UNSPECIFIED}, + {hcsschema.MigrationEvent(""), Phase_PHASE_UNSPECIFIED}, + } + + for _, tt := range tests { + t.Run(string(tt.event), func(t *testing.T) { + if got := ToPhase(tt.event); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +// TestToPhaseState verifies an HCS result takes precedence when mapping to the +// wire state, and that with no result the phase decides the outcome: progress +// phases default to success while terminal/unspecified phases stay unspecified. +func TestToPhaseState(t *testing.T) { + tests := []struct { + name string + result hcsschema.MigrationResult + phase Phase + want PhaseState + }{ + {name: "success", result: hcsschema.MigrationResultSuccess, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "cancelled", result: hcsschema.MigrationResultMigrationCancelled, want: PhaseState_PHASE_STATE_CANCELLED}, + {name: "guest cancellation", result: hcsschema.MigrationResultGuestInitiatedCancellation, want: PhaseState_PHASE_STATE_GUEST_INITIATED_CANCELLATION}, + {name: "source failed", result: hcsschema.MigrationResultSourceMigrationFailed, want: PhaseState_PHASE_STATE_SOURCE_FAILED}, + {name: "destination failed", result: hcsschema.MigrationResultDestinationMigrationFailed, want: PhaseState_PHASE_STATE_DESTINATION_FAILED}, + {name: "recovery failed", result: hcsschema.MigrationResultMigrationRecoveryFailed, want: PhaseState_PHASE_STATE_RECOVERY_FAILED}, + {name: "result wins over phase", result: hcsschema.MigrationResultSuccess, phase: Phase_PHASE_FAILED, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "no result progress phase defaults to success", phase: Phase_PHASE_TRANSFER_IN_PROGRESS, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "no result terminal phase is unspecified", phase: Phase_PHASE_DONE, want: PhaseState_PHASE_STATE_UNSPECIFIED}, + {name: "no result unspecified phase is unspecified", phase: Phase_PHASE_UNSPECIFIED, want: PhaseState_PHASE_STATE_UNSPECIFIED}, + {name: "invalid result falls through to progress phase", result: hcsschema.MigrationResultInvalid, phase: Phase_PHASE_SETUP_DONE, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "invalid result falls through to terminal phase", result: hcsschema.MigrationResultInvalid, phase: Phase_PHASE_FAILED, want: PhaseState_PHASE_STATE_UNSPECIFIED}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToPhaseState(tt.result, tt.phase); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +// TestToNotification verifies an HCS notification is converted to its wire form: +// origin/phase/state are mapped (with fallback origin), and blackout-exited +// details are attached only on a valid payload and dropped otherwise. +func TestToNotification(t *testing.T) { + stopTime := time.Unix(1700000000, 0).UTC() + blackoutDetails, err := json.Marshal(hcsschema.BlackoutExitedEventDetails{ + BlackoutDurationMilliseconds: 1234, + BlackoutStopTimestamp: stopTime, + }) + if err != nil { + t.Fatalf("marshal details: %v", err) + } + + tests := []struct { + name string + info hcsschema.OperationSystemMigrationNotificationInfo + fallback hcsschema.MigrationOrigin + want *Notification + }{ + { + name: "maps origin phase and state", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: hcsschema.MigrationOriginSource, + Event: hcsschema.MigrationEventMigrationFailed, + Result: hcsschema.MigrationResultSourceMigrationFailed, + }, + want: &Notification{ + Origin: Origin_ORIGIN_SOURCE, + Phase: Phase_PHASE_FAILED, + State: PhaseState_PHASE_STATE_SOURCE_FAILED, + }, + }, + { + name: "empty origin uses fallback", + info: hcsschema.OperationSystemMigrationNotificationInfo{Event: hcsschema.MigrationEventSetupDone}, + fallback: hcsschema.MigrationOriginDestination, + want: &Notification{ + Origin: Origin_ORIGIN_DESTINATION, + Phase: Phase_PHASE_SETUP_DONE, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + { + name: "blackout exited with valid details", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: hcsschema.MigrationOriginSource, + Event: hcsschema.MigrationEventBlackoutExited, + AdditionalDetails: blackoutDetails, + }, + want: &Notification{ + Origin: Origin_ORIGIN_SOURCE, + Phase: Phase_PHASE_BLACKOUT_EXITED, + State: PhaseState_PHASE_STATE_SUCCESS, + PhaseDetails: &Notification_BlackoutExited{ + BlackoutExited: &BlackoutExitedEventDetails{ + BlackoutDurationMilliseconds: 1234, + BlackoutStopTimestamp: timestamppb.New(stopTime), + }, + }, + }, + }, + { + name: "blackout exited with invalid details drops phase details", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: hcsschema.MigrationOriginSource, + Event: hcsschema.MigrationEventBlackoutExited, + AdditionalDetails: json.RawMessage("{invalid"), + }, + want: &Notification{ + Origin: Origin_ORIGIN_SOURCE, + Phase: Phase_PHASE_BLACKOUT_EXITED, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + { + name: "blackout exited without details has no phase details", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Event: hcsschema.MigrationEventBlackoutExited, + }, + want: &Notification{ + Phase: Phase_PHASE_BLACKOUT_EXITED, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + { + name: "details ignored for non-blackout event", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Event: hcsschema.MigrationEventSetupDone, + AdditionalDetails: blackoutDetails, + }, + want: &Notification{ + Phase: Phase_PHASE_SETUP_DONE, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ToNotification(tt.info, tt.fallback) + if !proto.Equal(got, tt.want) { + t.Fatalf("got %+v, want %+v", got, tt.want) + } + }) + } +} From 166ff74ff718aad1895f3230a1b521c8f8643c90 Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Fri, 24 Jul 2026 08:35:00 +0530 Subject: [PATCH 2/2] Harden LCOW sandbox live-migration lifecycle Give the live-migration session predictable, caller-facing behavior: - Cancellation & wind-down: a session can be aborted from any active state and always finalizes then cleans up back to idle, even after a failure or a partly-applied finalize. - Predictable retries: resource snapshots can be retaken and resumes are idempotent, while duplicate or out-of-order session calls are rejected instead of silently reapplied. - Typed errors: calls report invalid-argument, failed-precondition, or already-exists so callers can branch on them. Backed by unit tests and a documented state machine. Signed-off-by: Harsh Rawat --- .../service/mocks/mock_service.go | 14 + .../service/service_migration.go | 98 +++---- .../service/service_task_internal.go | 2 +- cmd/containerd-shim-lcow-v2/service/types.go | 3 + internal/controller/device/plan9/save.go | 8 +- internal/controller/device/plan9/save_test.go | 10 +- internal/controller/device/scsi/disk/save.go | 3 +- .../controller/device/scsi/disk/save_test.go | 9 +- .../controller/device/scsi/mount/save_lcow.go | 3 +- .../device/scsi/mount/save_lcow_test.go | 9 +- internal/controller/device/scsi/save.go | 17 +- internal/controller/device/scsi/save_test.go | 32 ++- internal/controller/device/vpci/save.go | 8 +- .../controller/device/vpci/save_lcow_test.go | 10 +- internal/controller/linuxcontainer/save.go | 68 ++--- .../controller/linuxcontainer/save_test.go | 163 +++-------- internal/controller/migration/controller.go | 192 ++++++++----- .../migration/controller_destination.go | 60 +++-- .../migration/controller_destination_test.go | 85 +++++- .../controller/migration/controller_source.go | 40 ++- .../migration/controller_source_test.go | 75 +----- .../controller/migration/controller_test.go | 252 +++++++++++++++++- internal/controller/migration/doc.go | 34 ++- .../migration/mocks/mock_lcow_migration.go | 28 ++ internal/controller/migration/socket.go | 52 ++-- internal/controller/migration/socket_test.go | 97 ++++++- internal/controller/migration/state.go | 34 ++- internal/controller/migration/types.go | 2 + internal/controller/network/save.go | 34 ++- internal/controller/network/save_test.go | 82 ++++-- internal/controller/pod/save_lcow.go | 37 +-- internal/controller/pod/save_lcow_test.go | 37 ++- internal/controller/process/save.go | 34 +-- internal/controller/process/save_test.go | 88 +++--- internal/controller/vm/save_lcow.go | 63 +++-- internal/controller/vm/vm.go | 4 + internal/controller/vm/vm_migration.go | 31 ++- internal/hcs/v2/migration.go | 24 ++ internal/vm/vmmanager/migration.go | 8 + 39 files changed, 1207 insertions(+), 643 deletions(-) diff --git a/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go b/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go index f3b20c371f..fbb1af0d8e 100644 --- a/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go +++ b/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go @@ -62,6 +62,20 @@ func (m *MockvmController) EXPECT() *MockvmControllerMockRecorder { return m.recorder } +// CancelLiveMigration mocks base method. +func (m *MockvmController) CancelLiveMigration(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelLiveMigration", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// CancelLiveMigration indicates an expected call of CancelLiveMigration. +func (mr *MockvmControllerMockRecorder) CancelLiveMigration(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelLiveMigration", reflect.TypeOf((*MockvmController)(nil).CancelLiveMigration), ctx) +} + // CreateVM mocks base method. func (m *MockvmController) CreateVM(ctx context.Context, opts *vm.CreateOptions) error { m.ctrl.T.Helper() diff --git a/cmd/containerd-shim-lcow-v2/service/service_migration.go b/cmd/containerd-shim-lcow-v2/service/service_migration.go index dfc06e04e5..4a5fba5a78 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_migration.go +++ b/cmd/containerd-shim-lcow-v2/service/service_migration.go @@ -7,12 +7,12 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" - "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/internal/ot" "github.com/Microsoft/hcsshim/pkg/migration" "github.com/containerd/errdefs/pkg/errgrpc" "github.com/sirupsen/logrus" - "go.opencensus.io/trace" + "go.opentelemetry.io/otel/attribute" ) // Ensure Service implements the MigrationService interface at compile time. @@ -22,13 +22,13 @@ var _ migration.MigrationService = &Service{} // exports an opaque config that the destination shim can use to import it. // This method is part of the instrumentation layer and business logic is included in prepareAndExportSandboxInternal. func (s *Service) PrepareAndExportSandbox(ctx context.Context, request *migration.PrepareAndExportSandboxRequest) (resp *migration.PrepareAndExportSandboxResponse, err error) { - ctx, span := oc.StartSpan(ctx, "PrepareAndExportSandbox") + ctx, span := ot.StartSpan(ctx, "PrepareAndExportSandbox") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -41,13 +41,13 @@ func (s *Service) PrepareAndExportSandbox(ctx context.Context, request *migratio // config produced by PrepareAndExportSandbox on the source. // This method is part of the instrumentation layer and business logic is included in importSandboxInternal. func (s *Service) ImportSandbox(ctx context.Context, request *migration.ImportSandboxRequest) (resp *migration.ImportSandboxResponse, err error) { - ctx, span := oc.StartSpan(ctx, "ImportSandbox") + ctx, span := ot.StartSpan(ctx, "ImportSandbox") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -60,13 +60,13 @@ func (s *Service) ImportSandbox(ctx context.Context, request *migration.ImportSa // migrated sandbox state. // This method is part of the instrumentation layer and business logic is included in prepareSandboxInternal. func (s *Service) PrepareSandbox(ctx context.Context, request *migration.PrepareSandboxRequest) (resp *migration.PrepareSandboxResponse, err error) { - ctx, span := oc.StartSpan(ctx, "PrepareSandbox") + ctx, span := ot.StartSpan(ctx, "PrepareSandbox") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -79,15 +79,15 @@ func (s *Service) PrepareSandbox(ctx context.Context, request *migration.Prepare // over the previously established migration transport. // This method is part of the instrumentation layer and business logic is included in transferSandboxInternal. func (s *Service) TransferSandbox(ctx context.Context, request *migration.TransferSandboxRequest) (resp *migration.TransferSandboxResponse, err error) { - ctx, span := oc.StartSpan(ctx, "TransferSandbox") + ctx, span := ot.StartSpan(ctx, "TransferSandbox") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) if request.Timeout != nil { - span.AddAttributes(trace.Int64Attribute(logfields.Timeout, int64(request.Timeout.AsDuration()))) + span.SetAttributes(attribute.Int64(logfields.Timeout, int64(request.Timeout.AsDuration()))) } // Set the session id in the logger context for all subsequent logs in this request. @@ -101,14 +101,14 @@ func (s *Service) TransferSandbox(ctx context.Context, request *migration.Transf // source, resume on the destination (per the requested action). // This method is part of the instrumentation layer and business logic is included in finalizeSandboxInternal. func (s *Service) FinalizeSandbox(ctx context.Context, request *migration.FinalizeSandboxRequest) (resp *migration.FinalizeSandboxResponse, err error) { - ctx, span := oc.StartSpan(ctx, "FinalizeSandbox") + ctx, span := ot.StartSpan(ctx, "FinalizeSandbox") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID), - trace.StringAttribute(logfields.Action, request.Action.String())) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID), + attribute.String(logfields.Action, request.Action.String())) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -121,13 +121,13 @@ func (s *Service) FinalizeSandbox(ctx context.Context, request *migration.Finali // the lifetime of the migration session. // This method is part of the instrumentation layer and business logic is included in notificationsInternal. func (s *Service) Notifications(ctx context.Context, request *migration.NotificationsRequest, server migration.Migration_NotificationsServer) (err error) { - ctx, span := oc.StartSpan(ctx, "Notifications") + ctx, span := ot.StartSpan(ctx, "Notifications") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -139,13 +139,13 @@ func (s *Service) Notifications(ctx context.Context, request *migration.Notifica // shim process for use as the migration transport. // This method is part of the instrumentation layer and business logic is included in createDuplicateSocketInternal. func (s *Service) CreateDuplicateSocket(ctx context.Context, request *migration.CreateDuplicateSocketRequest) (resp *migration.CreateDuplicateSocketResponse, err error) { - ctx, span := oc.StartSpan(ctx, "CreateDuplicateSocket") + ctx, span := ot.StartSpan(ctx, "CreateDuplicateSocket") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -160,13 +160,13 @@ func (s *Service) CreateDuplicateSocket(ctx context.Context, request *migration. // the migrating state until Cleanup is called. // This method is part of the instrumentation layer and business logic is included in cancelInternal. func (s *Service) Cancel(ctx context.Context, request *migration.CancelRequest) (resp *migration.CancelResponse, err error) { - ctx, span := oc.StartSpan(ctx, "Cancel") + ctx, span := ot.StartSpan(ctx, "Cancel") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) @@ -180,13 +180,13 @@ func (s *Service) Cancel(ctx context.Context, request *migration.CancelRequest) // migration controller's state machine back to the normal state. // This method is part of the instrumentation layer and business logic is included in cleanupInternal. func (s *Service) Cleanup(ctx context.Context, request *migration.CleanupRequest) (resp *migration.CleanupResponse, err error) { - ctx, span := oc.StartSpan(ctx, "Cleanup") + ctx, span := ot.StartSpan(ctx, "Cleanup") defer span.End() - defer func() { oc.SetSpanStatus(span, err) }() + defer func() { ot.SetSpanStatus(span, err) }() - span.AddAttributes( - trace.StringAttribute(logfields.SandboxID, s.sandboxID), - trace.StringAttribute(logfields.SessionID, request.SessionID)) + span.SetAttributes( + attribute.String(logfields.SandboxID, s.sandboxID), + attribute.String(logfields.SessionID, request.SessionID)) // Set the session id in the logger context for all subsequent logs in this request. ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) diff --git a/cmd/containerd-shim-lcow-v2/service/service_task_internal.go b/cmd/containerd-shim-lcow-v2/service/service_task_internal.go index 36a319b939..1d0bd5e495 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_task_internal.go +++ b/cmd/containerd-shim-lcow-v2/service/service_task_internal.go @@ -98,7 +98,7 @@ func (s *Service) stateInternal(_ context.Context, request *task.StateRequest) ( // createInternal creates a new pod sandbox or workload container based on the OCI spec. func (s *Service) createInternal(ctx context.Context, request *task.CreateTaskRequest) (*task.CreateTaskResponse, error) { - if state := s.vmController.State(); state != vm.StateRunning && state != vm.StateMigratingImported { + if state := s.vmController.State(); state != vm.StateRunning && state != vm.StateDestinationMigrationImported { return nil, fmt.Errorf("vm is not running or migrating (state: %s): %w", state, errdefs.ErrFailedPrecondition) } diff --git a/cmd/containerd-shim-lcow-v2/service/types.go b/cmd/containerd-shim-lcow-v2/service/types.go index 785e1d96cf..bf9d9007fc 100644 --- a/cmd/containerd-shim-lcow-v2/service/types.go +++ b/cmd/containerd-shim-lcow-v2/service/types.go @@ -126,6 +126,9 @@ type vmController interface { // StartLiveMigrationTransfer begins the memory transfer. StartLiveMigrationTransfer(ctx context.Context, options *hcsschema.MigrationTransferOptions) error + // CancelLiveMigration aborts an in-flight migration on the VM. + CancelLiveMigration(ctx context.Context) error + // FinalizeLiveMigration applies the migration's final operation to the VM. FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error diff --git a/internal/controller/device/plan9/save.go b/internal/controller/device/plan9/save.go index 1610be454c..d57bd216da 100644 --- a/internal/controller/device/plan9/save.go +++ b/internal/controller/device/plan9/save.go @@ -4,16 +4,18 @@ package plan9 import ( "fmt" + + "github.com/containerd/errdefs" ) -// Save is not yet supported for the Plan9 sub-controller; any tracked state -// indicates a live-migration scenario the controller cannot represent. +// Save is a no-op for an empty Plan9 sub-controller; it fails if any shares or +// reservations exist, since that state cannot yet be migrated. func (c *Controller) Save() error { c.mu.Lock() defer c.mu.Unlock() if len(c.sharesByHostPath) > 0 || len(c.reservations) > 0 { - return fmt.Errorf("plan9 controller save not supported: %d shares, %d reservations", len(c.sharesByHostPath), len(c.reservations)) + return fmt.Errorf("plan9 controller save not supported: %d shares, %d reservations: %w", len(c.sharesByHostPath), len(c.reservations), errdefs.ErrFailedPrecondition) } return nil diff --git a/internal/controller/device/plan9/save_test.go b/internal/controller/device/plan9/save_test.go index 2738d8d975..b72153fd04 100644 --- a/internal/controller/device/plan9/save_test.go +++ b/internal/controller/device/plan9/save_test.go @@ -3,9 +3,11 @@ package plan9 import ( + "errors" "testing" "github.com/Microsoft/go-winio/pkg/guid" + "github.com/containerd/errdefs" "github.com/Microsoft/hcsshim/internal/controller/device/plan9/share" ) @@ -27,7 +29,13 @@ func TestSave_NonEmptyErrors(t *testing.T) { sharesByHostPath: map[string]*share.Share{}, } - if err := c.Save(); err == nil { + // Save is unsupported here, so it must report a failed precondition + // that callers can detect with errors.Is. + err := c.Save() + if err == nil { t.Fatal("expected Save to error when reservations are present") } + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } } diff --git a/internal/controller/device/scsi/disk/save.go b/internal/controller/device/scsi/disk/save.go index 55b6f80a28..03abea8506 100644 --- a/internal/controller/device/scsi/disk/save.go +++ b/internal/controller/device/scsi/disk/save.go @@ -7,13 +7,14 @@ import ( "github.com/Microsoft/hcsshim/internal/controller/device/scsi/mount" scsisave "github.com/Microsoft/hcsshim/internal/controller/device/scsi/save" + "github.com/containerd/errdefs" ) // Save returns a migration snapshot of the disk and its mounts. It fails unless // the disk is attached or reserved and every mount can be saved. func (d *Disk) Save() (*scsisave.DiskState, error) { if d.state != StateAttached && d.state != StateReserved { - return nil, fmt.Errorf("scsi disk controller=%d lun=%d in state %s; want %s", d.controller, d.lun, d.state, StateAttached) + return nil, fmt.Errorf("scsi disk controller=%d lun=%d in state %s; want %s: %w", d.controller, d.lun, d.state, StateAttached, errdefs.ErrFailedPrecondition) } out := &scsisave.DiskState{ diff --git a/internal/controller/device/scsi/disk/save_test.go b/internal/controller/device/scsi/disk/save_test.go index 92ebb7e807..1d522ee7d6 100644 --- a/internal/controller/device/scsi/disk/save_test.go +++ b/internal/controller/device/scsi/disk/save_test.go @@ -4,10 +4,12 @@ package disk import ( "context" + "errors" "testing" "github.com/Microsoft/hcsshim/internal/controller/device/scsi/mount" scsisave "github.com/Microsoft/hcsshim/internal/controller/device/scsi/save" + "github.com/containerd/errdefs" ) func TestSave_ErrorWhenDetached(t *testing.T) { @@ -15,9 +17,14 @@ func TestSave_ErrorWhenDetached(t *testing.T) { if err := d.DetachFromVM(context.Background(), &mockVMSCSIRemover{}, newDefaultEjector()); err != nil { t.Fatalf("setup DetachFromVM: %v", err) } - if _, err := d.Save(); err == nil { + // A detached disk cannot be saved; the failure is a failed precondition. + _, err := d.Save() + if err == nil { t.Fatal("expected error saving a detached disk") } + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } } func TestSave_ReservedDisk_RoundTripsConfig(t *testing.T) { diff --git a/internal/controller/device/scsi/mount/save_lcow.go b/internal/controller/device/scsi/mount/save_lcow.go index 9385cebbba..9c01b78322 100644 --- a/internal/controller/device/scsi/mount/save_lcow.go +++ b/internal/controller/device/scsi/mount/save_lcow.go @@ -6,13 +6,14 @@ import ( "fmt" scsisave "github.com/Microsoft/hcsshim/internal/controller/device/scsi/save" + "github.com/containerd/errdefs" ) // Save returns a migration snapshot of the mount. It fails unless the mount // is mounted or reserved. func (m *Mount) Save() (*scsisave.MountState, error) { if m.state != StateMounted && m.state != StateReserved { - return nil, fmt.Errorf("scsi mount controller=%d lun=%d partition=%d in state %s; want %s", m.controller, m.lun, m.config.Partition, m.state, StateMounted) + return nil, fmt.Errorf("scsi mount controller=%d lun=%d partition=%d in state %s; want %s: %w", m.controller, m.lun, m.config.Partition, m.state, StateMounted, errdefs.ErrFailedPrecondition) } return &scsisave.MountState{ Config: &scsisave.MountConfig{ diff --git a/internal/controller/device/scsi/mount/save_lcow_test.go b/internal/controller/device/scsi/mount/save_lcow_test.go index f41f684ee7..23bac94b04 100644 --- a/internal/controller/device/scsi/mount/save_lcow_test.go +++ b/internal/controller/device/scsi/mount/save_lcow_test.go @@ -4,9 +4,11 @@ package mount import ( "context" + "errors" "testing" scsisave "github.com/Microsoft/hcsshim/internal/controller/device/scsi/save" + "github.com/containerd/errdefs" ) func TestSave_ErrorWhenUnmounted(t *testing.T) { @@ -14,9 +16,14 @@ func TestSave_ErrorWhenUnmounted(t *testing.T) { if err := m.UnmountFromGuest(context.Background(), newDefaultUnmounter()); err != nil { t.Fatalf("setup UnmountFromGuest: %v", err) } - if _, err := m.Save(); err == nil { + // An unmounted mount cannot be saved; the failure is a failed precondition. + _, err := m.Save() + if err == nil { t.Fatal("expected error saving an unmounted mount") } + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } } func TestSave_Mounted_RoundTripsConfig(t *testing.T) { diff --git a/internal/controller/device/scsi/save.go b/internal/controller/device/scsi/save.go index 3b0615f4d4..beacfd85ee 100644 --- a/internal/controller/device/scsi/save.go +++ b/internal/controller/device/scsi/save.go @@ -12,6 +12,7 @@ import ( hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/containerd/errdefs" "github.com/sirupsen/logrus" "github.com/Microsoft/go-winio/pkg/guid" @@ -76,7 +77,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { // Reject payloads that did not come from a compatible Save. if env.GetTypeUrl() != scsisave.TypeURL { - return nil, fmt.Errorf("unsupported scsi saved-state type %q", env.GetTypeUrl()) + return nil, fmt.Errorf("unsupported scsi saved-state type %q: %w", env.GetTypeUrl(), errdefs.ErrInvalidArgument) } // Unmarshall the payload. @@ -87,7 +88,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { // Reject payloads written by an incompatible shim version. if v := state.GetSchemaVersion(); v != scsisave.SchemaVersion { - return nil, fmt.Errorf("unsupported scsi saved-state schema version %d (want %d)", v, scsisave.SchemaVersion) + return nil, fmt.Errorf("unsupported scsi saved-state schema version %d (want %d): %w", v, scsisave.SchemaVersion, errdefs.ErrInvalidArgument) } // Create a new controller. @@ -103,7 +104,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { for slot, ds := range state.GetDisks() { idx := int(slot) if idx >= len(c.controllerSlots) { - return nil, fmt.Errorf("invalid controller slot: %d", slot) + return nil, fmt.Errorf("invalid controller slot: %d: %w", slot, errdefs.ErrInvalidArgument) } // Derive the controller and LUN from the slot index and rebuild the disk. @@ -122,7 +123,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { // Rehydrate all the reservations. for idStr, r := range state.GetReservations() { - // Skip any reservation whose ID cannot be parsed. + // Fail the import if any reservation ID cannot be parsed. id, err := guid.FromString(idStr) if err != nil { return nil, fmt.Errorf("invalid reservation id %q: %w", idStr, err) @@ -147,7 +148,7 @@ func (c *Controller) Resume(ctx context.Context, vm VMSCSIOps, guest GuestSCSIOp defer c.mu.Unlock() if !c.isMigrating { - return fmt.Errorf("cannot resume scsi controller: not migrating") + return fmt.Errorf("cannot resume scsi controller: not migrating: %w", errdefs.ErrFailedPrecondition) } c.vm = vm @@ -158,8 +159,8 @@ func (c *Controller) Resume(ctx context.Context, vm VMSCSIOps, guest GuestSCSIOp return nil } -// Disks returns the configuration of every disk currently attached to the -// controller. +// Disks returns the configuration of each attached disk that is indexed by a +// host path. func (c *Controller) Disks() []disk.Config { c.mu.Lock() defer c.mu.Unlock() @@ -217,7 +218,7 @@ func (c *Controller) UpdateDiskHostPath(ctx context.Context, reservationID guid. defer c.mu.Unlock() if !c.isMigrating { - return fmt.Errorf("UpdateDiskHostPath is only valid while migrating") + return fmt.Errorf("UpdateDiskHostPath is only valid while migrating: %w", errdefs.ErrFailedPrecondition) } // Find the reservation. diff --git a/internal/controller/device/scsi/save_test.go b/internal/controller/device/scsi/save_test.go index 4b24d20760..7dc422ce1e 100644 --- a/internal/controller/device/scsi/save_test.go +++ b/internal/controller/device/scsi/save_test.go @@ -4,12 +4,14 @@ package scsi import ( "context" + "errors" "strings" "testing" "github.com/Microsoft/hcsshim/internal/controller/device/scsi/disk" scsisave "github.com/Microsoft/hcsshim/internal/controller/device/scsi/save" "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/containerd/errdefs" "github.com/Microsoft/go-winio/pkg/guid" "google.golang.org/protobuf/proto" @@ -152,31 +154,38 @@ func TestImport_Errors(t *testing.T) { return &anypb.Any{TypeUrl: scsisave.TypeURL, Value: b} } + // wantIs, when set, is the sentinel error the failure must wrap so + // callers can classify it with errors.Is. tests := []struct { name string env *anypb.Any wantErr string + wantIs error }{ - {"nil envelope", nil, "nil"}, - {"wrong type url", &anypb.Any{TypeUrl: "bogus"}, "unsupported scsi saved-state type"}, - {"corrupt payload", &anypb.Any{TypeUrl: scsisave.TypeURL, Value: []byte{0xff}}, "unmarshal"}, - {"schema mismatch", wrap(&scsisave.Payload{SchemaVersion: scsisave.SchemaVersion + 1}), "schema version"}, + {"nil envelope", nil, "nil", nil}, + {"wrong type url", &anypb.Any{TypeUrl: "bogus"}, "unsupported scsi saved-state type", errdefs.ErrInvalidArgument}, + {"corrupt payload", &anypb.Any{TypeUrl: scsisave.TypeURL, Value: []byte{0xff}}, "unmarshal", nil}, + {"schema mismatch", wrap(&scsisave.Payload{SchemaVersion: scsisave.SchemaVersion + 1}), "schema version", errdefs.ErrInvalidArgument}, {"slot out of range", wrap(&scsisave.Payload{ SchemaVersion: scsisave.SchemaVersion, NumControllers: 1, Disks: map[uint32]*scsisave.DiskState{numLUNsPerController: {}}, - }), "invalid controller slot"}, + }), "invalid controller slot", errdefs.ErrInvalidArgument}, {"bad reservation id", wrap(&scsisave.Payload{ SchemaVersion: scsisave.SchemaVersion, NumControllers: 1, Reservations: map[string]*scsisave.Reservation{"not-a-guid": {}}, - }), "invalid reservation id"}, + }), "invalid reservation id", nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if _, err := Import(t.Context(), tt.env); err == nil || !strings.Contains(err.Error(), tt.wantErr) { + _, err := Import(t.Context(), tt.env) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) } + if tt.wantIs != nil && !errors.Is(err, tt.wantIs) { + t.Fatalf("expected error to wrap %v, got %v", tt.wantIs, err) + } }) } } @@ -238,9 +247,14 @@ func TestUpdateDiskHostPath(t *testing.T) { if err := c.Resume(t.Context(), &mockVMOps{}, newMockGuestOps()); err != nil { t.Fatalf("unexpected error: %v", err) } - if err := c.UpdateDiskHostPath(t.Context(), id, `C:\x.vhdx`); err == nil || - !strings.Contains(err.Error(), "migrating") { + // After resume the controller is no longer migrating, so the update + // is rejected as a failed precondition. + err := c.UpdateDiskHostPath(t.Context(), id, `C:\x.vhdx`) + if err == nil || !strings.Contains(err.Error(), "migrating") { t.Fatalf("expected migrating error, got %v", err) } + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } }) } diff --git a/internal/controller/device/vpci/save.go b/internal/controller/device/vpci/save.go index 11ad361dad..b8c155dc45 100644 --- a/internal/controller/device/vpci/save.go +++ b/internal/controller/device/vpci/save.go @@ -4,16 +4,18 @@ package vpci import ( "fmt" + + "github.com/containerd/errdefs" ) -// Save is not yet supported for the VPCI sub-controller; any tracked state -// indicates a live-migration scenario the controller cannot represent. +// Save is a no-op for an empty VPCI sub-controller; it fails if any devices are +// attached, since that state cannot yet be migrated. func (c *Controller) Save() error { c.mu.Lock() defer c.mu.Unlock() if len(c.devices) > 0 { - return fmt.Errorf("vpci controller save not supported: %d devices", len(c.devices)) + return fmt.Errorf("vpci controller save not supported: %d devices: %w", len(c.devices), errdefs.ErrFailedPrecondition) } return nil diff --git a/internal/controller/device/vpci/save_lcow_test.go b/internal/controller/device/vpci/save_lcow_test.go index 3b5e28991c..0c711ec894 100644 --- a/internal/controller/device/vpci/save_lcow_test.go +++ b/internal/controller/device/vpci/save_lcow_test.go @@ -3,9 +3,11 @@ package vpci import ( + "errors" "testing" "github.com/Microsoft/go-winio/pkg/guid" + "github.com/containerd/errdefs" ) func TestSave_EmptyOK(t *testing.T) { @@ -28,7 +30,13 @@ func TestSave_NonEmptyErrors(t *testing.T) { deviceToGUID: map[Device]guid.GUID{dev: g}, } - if err := c.Save(); err == nil { + // Save is unsupported here, so it must report a failed precondition + // that callers can detect with errors.Is. + err := c.Save() + if err == nil { t.Fatal("expected Save to error when devices are present") } + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } } diff --git a/internal/controller/linuxcontainer/save.go b/internal/controller/linuxcontainer/save.go index 7da07bcc09..35f367b2cd 100644 --- a/internal/controller/linuxcontainer/save.go +++ b/internal/controller/linuxcontainer/save.go @@ -23,16 +23,16 @@ import ( ) // Save serializes a running container's current state into a portable -// envelope that can be handed to a migration destination. It succeeds only -// when the container is running, the single stable state a live migration -// can be performed from. On success the source is frozen until it is resumed. +// envelope that can be handed to a migration destination. It succeeds when the +// container is running or already frozen by a prior save (retry-safe), and on +// success the source stays frozen until it is resumed. func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { c.mu.Lock() defer c.mu.Unlock() - // Only a running container is in a stable, migratable state. - if c.state != StateRunning { - return nil, fmt.Errorf("container %q in state %s; want %s", c.containerID, c.state, StateRunning) + // A running container—or one already frozen by a prior save—can be saved. + if c.state != StateRunning && c.state != StateSourceMigrating { + return nil, fmt.Errorf("container %q in state %s; want %s: %w", c.containerID, c.state, StateRunning, errdefs.ErrFailedPrecondition) } // Capture the container's scalar bookkeeping into the snapshot. @@ -69,7 +69,7 @@ func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { // Live migration only supports a container whose sole process is the // init process; reject a missing init process or any additional execs. if _, ok := c.processes[""]; !ok || len(c.processes) > 1 { - return nil, fmt.Errorf("container %q must have only the init process for live migration; has %d processes", c.containerID, len(c.processes)) + return nil, fmt.Errorf("container %q must have only the init process for live migration; has %d processes: %w", c.containerID, len(c.processes), errdefs.ErrFailedPrecondition) } // Snapshot the init process as an opaque payload the process controller @@ -121,7 +121,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { } if env.GetTypeUrl() != lcsave.TypeURL { - return nil, fmt.Errorf("unsupported container saved-state type %q", env.GetTypeUrl()) + return nil, fmt.Errorf("unsupported container saved-state type %q: %w", env.GetTypeUrl(), errdefs.ErrInvalidArgument) } // Decode and reject any payload this build cannot interpret. @@ -131,7 +131,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { } if v := state.GetSchemaVersion(); v != lcsave.SchemaVersion { - return nil, fmt.Errorf("unsupported container saved-state schema version %d (want %d)", v, lcsave.SchemaVersion) + return nil, fmt.Errorf("unsupported container saved-state schema version %d (want %d): %w", v, lcsave.SchemaVersion, errdefs.ErrInvalidArgument) } // Rehydrate into the destination-migrating state: state is restored but no @@ -216,6 +216,11 @@ func (c *Controller) Resume( c.mu.Lock() defer c.mu.Unlock() + // Idempotent: a container resumed by a prior attempt is a no-op on retry. + if c.state == StateRunning { + return nil + } + // Resume is only valid mid-migration, on either the source or destination. if c.state != StateSourceMigrating && c.state != StateDestinationMigrating { return fmt.Errorf("container %s is in state %s; cannot resume: %w", c.containerID, c.state, errdefs.ErrFailedPrecondition) @@ -341,7 +346,7 @@ func (c *Controller) Patch(ctx context.Context, scsiCtrl scsiController, request // so future SCSI operations resolve to local disks. if c.layers != nil { if scsiCtrl == nil { - return fmt.Errorf("scsi controller is required to patch container %q layers", c.containerID) + return fmt.Errorf("scsi controller is required to patch container %q layers: %w", c.containerID, errdefs.ErrInvalidArgument) } lcowLayers, err := layers.ParseLCOWLayers(request.Rootfs, nil) @@ -350,7 +355,7 @@ func (c *Controller) Patch(ctx context.Context, scsiCtrl scsiController, request } if got, want := len(lcowLayers.Layers), len(c.layers.roLayers); got != want { - return fmt.Errorf("ro layer count mismatch: got %d, want %d", got, want) + return fmt.Errorf("ro layer count mismatch: got %d, want %d: %w", got, want, errdefs.ErrInvalidArgument) } for i, ro := range c.layers.roLayers { @@ -389,7 +394,7 @@ func (c *Controller) Patch(ctx context.Context, scsiCtrl scsiController, request } if len(c.processes) > 1 { - return fmt.Errorf("container %q has %d processes; live migration only supports the init process", c.containerID, len(c.processes)) + return fmt.Errorf("container %q has %d processes; live migration only supports the init process: %w", c.containerID, len(c.processes), errdefs.ErrFailedPrecondition) } if err := initProc.Patch(ctx, request.ID, &process.CreateOptions{ @@ -413,42 +418,3 @@ func (c *Controller) Patch(ctx context.Context, scsiCtrl scsiController, request return nil } - -// AbortMigrated discards an imported-but-never-resumed container: it drains -// each imported process, marks the container stopped, and publishes a -// synthetic TaskExit so containerd will accept a Delete. It is a no-op once -// the container has left the migrating state. -func (c *Controller) AbortMigrated(ctx context.Context, events chan interface{}) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.state != StateDestinationMigrating { - return - } - - log.G(ctx).WithField(logfields.DestinationContainerID, c.containerID).Debug("aborting migrated container") - - // Tear down each imported process and the container handle before - // reporting the container as exited. - for _, proc := range c.processes { - proc.AbortMigrated(ctx) - } - - c.state = StateStopped - c.closeContainer() - - // Emit a synthetic exit for the init process so containerd unblocks Delete. - initProc := c.processes[""] - if events == nil || initProc == nil { - return - } - - status := initProc.Status(true) - events <- &eventstypes.TaskExit{ - ContainerID: c.containerID, - ID: status.ExecID, - Pid: status.Pid, - ExitStatus: status.ExitStatus, - ExitedAt: status.ExitedAt, - } -} diff --git a/internal/controller/linuxcontainer/save_test.go b/internal/controller/linuxcontainer/save_test.go index d935ec69da..5590378e5b 100644 --- a/internal/controller/linuxcontainer/save_test.go +++ b/internal/controller/linuxcontainer/save_test.go @@ -13,7 +13,6 @@ import ( procsave "github.com/Microsoft/hcsshim/internal/controller/process/save" "github.com/Microsoft/go-winio/pkg/guid" - eventstypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/api/runtime/task/v3" containerdtypes "github.com/containerd/containerd/api/types" "github.com/containerd/errdefs" @@ -51,18 +50,6 @@ func importedInitProcess(t *testing.T) *process.Controller { return p } -// patchedInitProcess returns a migrating init process whose IO has been opened, -// mirroring an imported-and-patched-but-never-resumed process. Empty IO paths -// avoid real named-pipe connections. -func patchedInitProcess(t *testing.T) *process.Controller { - t.Helper() - p := importedInitProcess(t) - if err := p.Patch(t.Context(), testContainerID, &process.CreateOptions{}); err != nil { - t.Fatalf("patch init process = %v", err) - } - return p -} - // baseContainerPayload returns a fully valid container payload that individual // tests mutate to exercise specific decode failures. func baseContainerPayload(t *testing.T) *lcsave.Payload { @@ -98,10 +85,10 @@ func containerEnvelope(t *testing.T, p *lcsave.Payload) *anypb.Any { // --- Save --- -// TestSave_WrongState verifies that only a running container can be saved. +// TestSave_WrongState verifies Save is refused for non-snapshottable container states. func TestSave_WrongState(t *testing.T) { t.Parallel() - invalidStates := []State{StateNotCreated, StateCreated, StateStopped, StateInvalid, StateDestinationMigrating, StateSourceMigrating} + invalidStates := []State{StateNotCreated, StateCreated, StateStopped, StateInvalid, StateDestinationMigrating} for _, state := range invalidStates { t.Run(state.String(), func(t *testing.T) { @@ -109,8 +96,9 @@ func TestSave_WrongState(t *testing.T) { c, _, _, _, _ := newContainerTestController(t) c.state = state - if _, err := c.Save(t.Context()); err == nil { - t.Errorf("Save() = nil; want error for state %s", state) + _, err := c.Save(t.Context()) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Errorf("Save() = %v; want ErrFailedPrecondition for state %s", err, state) } }) } @@ -140,8 +128,9 @@ func TestSave_ProcessConstraints(t *testing.T) { c.state = StateRunning tc.seedProc(c) - if _, err := c.Save(t.Context()); err == nil { - t.Error("Save() = nil; want error for invalid process set") + _, err := c.Save(t.Context()) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Errorf("Save() = %v; want ErrFailedPrecondition for invalid process set", err) } }) } @@ -170,20 +159,25 @@ func TestImport_InvalidEnvelope(t *testing.T) { badVersion := containerEnvelope(t, &lcsave.Payload{SchemaVersion: lcsave.SchemaVersion + 1}) tests := []struct { - name string - env *anypb.Any + name string + env *anypb.Any + wantIs error }{ {name: "nil envelope", env: nil}, - {name: "wrong type url", env: &anypb.Any{TypeUrl: "type.microsoft.com/other"}}, + {name: "wrong type url", env: &anypb.Any{TypeUrl: "type.microsoft.com/other"}, wantIs: errdefs.ErrInvalidArgument}, {name: "undecodable value", env: &anypb.Any{TypeUrl: lcsave.TypeURL, Value: []byte{0x08, 0xff}}}, - {name: "schema version mismatch", env: badVersion}, + {name: "schema version mismatch", env: badVersion, wantIs: errdefs.ErrInvalidArgument}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() - if _, err := Import(t.Context(), tc.env); err == nil { - t.Error("Import() = nil; want error") + _, err := Import(t.Context(), tc.env) + if err == nil { + t.Fatal("Import() = nil; want error") + } + if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { + t.Errorf("Import() = %v; want %v", err, tc.wantIs) } }) } @@ -320,11 +314,13 @@ func TestPatch_LayerErrors(t *testing.T) { layers *scsiLayers rootfs []*containerdtypes.Mount stubPath bool + wantIs error }{ { name: "nil scsi controller", scsiNil: true, layers: &scsiLayers{scratch: scsiReservation{id: scratchGUID}}, + wantIs: errdefs.ErrInvalidArgument, }, { name: "unparsable rootfs", @@ -336,6 +332,7 @@ func TestPatch_LayerErrors(t *testing.T) { layers: &scsiLayers{roLayers: []scsiReservation{{id: roGUID}, {id: roGUID}}, scratch: scsiReservation{id: scratchGUID}}, rootfs: []*containerdtypes.Mount{{Type: "lcow-layer", Source: `C:\scratch`, Options: []string{`parentLayerPaths=["C:\\layers\\base"]`}}}, stubPath: true, + wantIs: errdefs.ErrInvalidArgument, }, } @@ -355,7 +352,10 @@ func TestPatch_LayerErrors(t *testing.T) { err := c.Patch(t.Context(), sc, &task.CreateTaskRequest{ID: "dest", Rootfs: tc.rootfs}) if err == nil { - t.Error("Patch() = nil; want error") + t.Fatal("Patch() = nil; want error") + } + if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { + t.Errorf("Patch() = %v; want %v", err, tc.wantIs) } }) } @@ -416,12 +416,13 @@ func TestPatch_ProcessConstraints(t *testing.T) { tests := []struct { name string seedProc func(c *Controller) + wantIs error }{ {name: "no init process", seedProc: func(*Controller) {}}, {name: "extra exec process", seedProc: func(c *Controller) { c.processes[""] = importedInitProcess(t) c.processes["exec-1"] = process.New(testContainerID, "exec-1", nil, 0) - }}, + }, wantIs: errdefs.ErrFailedPrecondition}, } for _, tc := range tests { @@ -433,7 +434,10 @@ func TestPatch_ProcessConstraints(t *testing.T) { err := c.Patch(t.Context(), scsiCtrl, &task.CreateTaskRequest{ID: "dest"}) if err == nil { - t.Error("Patch() = nil; want error for invalid process set") + t.Fatal("Patch() = nil; want error for invalid process set") + } + if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { + t.Errorf("Patch() = %v; want %v", err, tc.wantIs) } }) } @@ -507,11 +511,11 @@ func TestResume_SourceRollback(t *testing.T) { } } -// TestResume_WrongState verifies that Resume is rejected unless the container is -// migrating on the source or destination. +// TestResume_WrongState verifies Resume is rejected for states that are neither +// mid-migration nor already running. func TestResume_WrongState(t *testing.T) { t.Parallel() - invalidStates := []State{StateNotCreated, StateCreated, StateRunning, StateStopped, StateInvalid} + invalidStates := []State{StateNotCreated, StateCreated, StateStopped, StateInvalid} for _, state := range invalidStates { t.Run(state.String(), func(t *testing.T) { @@ -527,97 +531,18 @@ func TestResume_WrongState(t *testing.T) { } } -// --- AbortMigrated --- - -// TestAbortMigrated_NoOp verifies that AbortMigrated leaves a non-migrating -// container untouched and publishes nothing. -func TestAbortMigrated_NoOp(t *testing.T) { - t.Parallel() - otherStates := []State{StateNotCreated, StateCreated, StateRunning, StateStopped, StateInvalid, StateSourceMigrating} - - for _, state := range otherStates { - t.Run(state.String(), func(t *testing.T) { - t.Parallel() - c, _, _, _, _ := newContainerTestController(t) - c.state = state - - events := make(chan interface{}, 1) - c.AbortMigrated(t.Context(), events) - - if c.state != state { - t.Errorf("state = %s; want unchanged %s", c.state, state) - } - select { - case <-events: - t.Error("AbortMigrated published an event for a non-migrating container") - default: - } - }) - } -} - -// TestAbortMigrated_Succeeds verifies that AbortMigrated drains the init -// process, marks the container stopped, closes waiters, and publishes a -// synthetic TaskExit when an events channel is supplied. -func TestAbortMigrated_Succeeds(t *testing.T) { +// TestResume_IdempotentWhenRunning verifies that resuming an already-resumed +// container is a no-op, so a retry after a completed resume is safe. +func TestResume_IdempotentWhenRunning(t *testing.T) { t.Parallel() - c, _, _, _, _ := newContainerTestController(t) - c.state = StateDestinationMigrating - c.processes[""] = patchedInitProcess(t) - - events := make(chan interface{}, 1) - c.AbortMigrated(t.Context(), events) - - if c.state != StateStopped { - t.Errorf("state = %s; want StateStopped", c.state) - } - select { - case <-c.terminatedCh: - default: - t.Error("terminatedCh should be closed after AbortMigrated") - } - select { - case ev := <-events: - if _, ok := ev.(*eventstypes.TaskExit); !ok { - t.Errorf("event = %T; want *eventstypes.TaskExit", ev) - } - default: - t.Error("AbortMigrated should publish a TaskExit event") - } -} - -// TestAbortMigrated_NoEventChannel verifies that AbortMigrated still tears the -// container down when no events channel is provided. -func TestAbortMigrated_NoEventChannel(t *testing.T) { - t.Parallel() - c, _, _, _, _ := newContainerTestController(t) - c.state = StateDestinationMigrating - c.processes[""] = importedInitProcess(t) - - c.AbortMigrated(t.Context(), nil) - - if c.state != StateStopped { - t.Errorf("state = %s; want StateStopped", c.state) - } -} - -// TestAbortMigrated_NoInitProcess verifies that AbortMigrated tears the -// container down but publishes nothing when there is no init process. -func TestAbortMigrated_NoInitProcess(t *testing.T) { - t.Parallel() - c, _, _, _, _ := newContainerTestController(t) - c.state = StateDestinationMigrating - - events := make(chan interface{}, 1) - c.AbortMigrated(t.Context(), events) + c, scsiCtrl, plan9Ctrl, vpciCtrl, guestCtrl := newContainerTestController(t) + c.state = StateRunning - if c.state != StateStopped { - t.Errorf("state = %s; want StateStopped", c.state) + if err := c.Resume(t.Context(), testVMID, testPodID, guestCtrl, scsiCtrl, plan9Ctrl, vpciCtrl, nil); err != nil { + t.Fatalf("Resume() = %v; want nil", err) } - select { - case <-events: - t.Error("AbortMigrated should not publish an event without an init process") - default: + if c.state != StateRunning { + t.Errorf("state = %s; want unchanged StateRunning", c.state) } } diff --git a/internal/controller/migration/controller.go b/internal/controller/migration/controller.go index fdabe730de..1957b3af72 100644 --- a/internal/controller/migration/controller.go +++ b/internal/controller/migration/controller.go @@ -12,6 +12,7 @@ import ( "time" "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" "github.com/Microsoft/hcsshim/internal/log" @@ -63,7 +64,8 @@ type Controller struct { // dupSocket is the duplicated transport socket used for the memory transfer. dupSocket windows.Handle - // socketReady is closed by RegisterDuplicateSocket on transition to StateSocketReady. + // socketReady is closed once socket registration completes—on success or + // failure—so a Transfer waiting on it unblocks and then proceeds or bails. socketReady chan struct{} // notifier is created lazily on the first Subscribe. Guarded by mu. @@ -130,27 +132,37 @@ func (c *Controller) Transfer(ctx context.Context, sessionID string, timeout tim // Detached ctx so the transfer outlives the gRPC call. ctx = context.WithoutCancel(ctx) + var err error + // In case we encounter error from this point onwards, fail the migration. + defer func() { + if err != nil { + c.failTransfer(ctx, err) + } + }() + // Wait for the duplicate socket to arrive, or give up after the timeout. - var socketTimeoutErr error select { case <-socketReady: case <-time.After(timeout): - socketTimeoutErr = fmt.Errorf("timed out waiting for socket ready after %s: %w", timeout, errdefs.ErrFailedPrecondition) + // If the socket connection was not ready within timeout, return an error. + err = fmt.Errorf("timed out waiting for socket ready after %s: %w", timeout, errdefs.ErrFailedPrecondition) + return } + // Take lock for checking and modifying the state. c.mu.Lock() - defer c.mu.Unlock() - - // If the socket connection was not ready within timeout, return an error. - if socketTimeoutErr != nil { - c.failTransfer(ctx, socketTimeoutErr) - return - } - // Gate: only the first goroutine finds StateSocketReady and claims the - // transfer by moving to StateTransferring; any later goroutine (or a - // torn-down session) bails. + // Gate: only a goroutine that still finds StateSocketReady claims the + // transfer below (by moving to StateTransferring); any other state bails. if c.state != StateSocketReady { + // StateFailed here means the session failed before the transfer could + // start (e.g. destination socket registration failed after this goroutine + // began waiting), so surface it as an error rather than returning silently. + if c.state == StateFailed { + err = fmt.Errorf("migration session failed before transfer could start: %w", errdefs.ErrFailedPrecondition) + } + + c.mu.Unlock() return } @@ -158,13 +170,23 @@ func (c *Controller) Transfer(ctx context.Context, sessionID string, timeout tim // will return early from above and there is a single driver for transfer. c.state = StateTransferring + // Unlock here prior to running the transfer so that other APIs can be called + // while the long-running transfer is in progress. + c.mu.Unlock() + // Run the transfer; a failure marks the session failed for subscribers. - if err := c.runTransfer(ctx); err != nil { - c.failTransfer(ctx, err) + if err = c.runTransfer(ctx); err != nil { return } - c.state = StateTransferCompleted + c.mu.Lock() + // A concurrent Cancel may have moved the session on while the lock was + // released, so only a still-transferring session completes here. + if c.state == StateTransferring { + c.state = StateTransferCompleted + } + c.mu.Unlock() + log.G(ctx).Info("migration transfer completed") }() @@ -174,6 +196,16 @@ func (c *Controller) Transfer(ctx context.Context, sessionID string, timeout tim // failTransfer marks the session failed and surfaces err to notification // subscribers as a PHASE_FAILED event. func (c *Controller) failTransfer(ctx context.Context, err error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Cancel takes precedence: a session canceled while the transfer ran owns + // the outcome, so the expected transfer error is swallowed without failing. + if c.state == StateCancelled { + log.G(ctx).WithError(err).Debug("migration transfer error ignored; session cancelled") + return + } + c.state = StateFailed log.G(ctx).WithError(err).Error("migration transfer failed") @@ -211,6 +243,7 @@ func sessionIDToUint32(sessionID string) uint32 { } // runTransfer issues the per-origin HCS calls for the memory transfer. +// Caller should ensure that it is called at most once. func (c *Controller) runTransfer(ctx context.Context) error { config := &hcs.MigrationConfig{ Socket: syscall.Handle(c.dupSocket), @@ -251,13 +284,20 @@ func (c *Controller) Finalize(ctx context.Context, sessionID string, action migr c.mu.Lock() defer c.mu.Unlock() - // Idempotent: an already-finalized session succeeds without redoing work. - if c.state == StateFinalized { + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Nothing to finalize, so return early: the session is already finalized + // (idempotent repeat), idle (no active session), or a canceled destination, + // which winds down directly through Cleanup rather than Finalize. + if c.state == StateFinalized || c.state == StateIdle || + (c.origin == hcsschema.MigrationOriginDestination && c.state == StateCancelled) { return nil } - // Finalize is valid only after a completed transfer or a cancellation. - if c.state != StateTransferCompleted && c.state != StateCancelled { + // Finalize is valid only after a completed transfer, a cancellation or a failure. + if c.state != StateTransferCompleted && c.state != StateCancelled && c.state != StateFailed { return fmt.Errorf("finalize not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) } @@ -272,17 +312,38 @@ func (c *Controller) Finalize(ctx context.Context, sessionID string, action migr return fmt.Errorf("unsupported finalize action %s: %w", action, errdefs.ErrInvalidArgument) } - // Apply the final operation (resume or stop) to the underlying VM. - if err := c.vmController.FinalizeLiveMigration(ctx, - &hcsschema.MigrationFinalizedOptions{ - Origin: c.origin, - FinalizedOperation: finalOp, - }); err != nil { - return fmt.Errorf("finalize live migration (origin=%s, action=%s): %w", c.origin, action, err) + // Reconcile the requested operation against the VM's migration state. A first + // finalize finds the VM still migrating and drives it; a retry after a + // partially applied finalize finds the VM already settled and reconciles + // against where it landed rather than re-issuing finalize. + switch c.vmController.State() { + case vm.StateTerminated: + // A prior stop finalize already tore the VM down: a repeat stop is a + // no-op, while a resume is no longer possible. + if finalOp == hcsschema.MigrationFinalOperationResume { + return fmt.Errorf("cannot resume finalize: VM already terminated by a stop finalize: %w", errdefs.ErrFailedPrecondition) + } + case vm.StateMigrationFinalized: + // A prior resume finalize already completed on the VM: a stop now tears + // it down, while a repeat resume falls through to re-run the resume steps. + if finalOp == hcsschema.MigrationFinalOperationStop { + if err := c.vmController.TerminateVM(ctx); err != nil { + return fmt.Errorf("terminate resumed vm on stop finalize: %w", err) + } + } + default: + // The VM is still migrating: apply the final operation. + if err := c.vmController.FinalizeLiveMigration(ctx, + &hcsschema.MigrationFinalizedOptions{ + Origin: c.origin, + FinalizedOperation: finalOp, + }); err != nil { + return fmt.Errorf("finalize live migration (origin=%s, action=%s): %w", c.origin, action, err) + } } - // On RESUME, rebuild the GCS bridge. Destination also walks pods to - // reattach gcs.Container/gcs.Process; source has nothing else to restore. + // On RESUME, resume the VM (the source also rebuilds its GCS bridge), then + // resume every pod on both sides to reattach containers and lift the freeze. if finalOp == hcsschema.MigrationFinalOperationResume { switch c.origin { case hcsschema.MigrationOriginDestination: @@ -313,28 +374,6 @@ func (c *Controller) Finalize(ctx context.Context, sessionID string, action migr } } - if finalOp == hcsschema.MigrationFinalOperationStop { - // For both source and destination, there is nothing to do for VM. - // vm.FinalizeLiveMigration + STOP will stop the VM itself. - // We only need to take care of tasks. - switch c.origin { - case hcsschema.MigrationOriginDestination: - // On the destination side, we would not create the exit handlers until resume - // which happens in Finalize + Resume. Therefore, containers would not have - // any way to report the exit event. Hence, we explicitly abort the - // in-migration tasks. - for _, podCtrl := range c.podControllers { - podCtrl.AbortMigrated(ctx, events) - } - case hcsschema.MigrationOriginSource: - // On the source side, all the exit handlers for containers and processes - // are already running. Therefore, during vm.FinalizeLiveMigration, if the - // operation was stop, we would collapse the bridge which would lead to all - // the exit handlers observing the VM exiting and report the exit event. - // Therefore, nothing to do here. - } - } - // Mark the session finalized so Cleanup can run and repeat calls no-op. c.state = StateFinalized @@ -349,23 +388,49 @@ func (c *Controller) Cancel(ctx context.Context, sessionID string) error { defer c.mu.Unlock() if c.sessionID != sessionID { - return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrFailedPrecondition) + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) } - // Idempotent: already canceled. - if c.state == StateCancelled { + // Idempotent: already canceled or not initialized. + if c.state == StateIdle || c.state == StateCancelled || c.state == StateCancelling { + return nil + } + + // The destination imported the snapshot but never materialized the HCS + // compute system (that happens in PrepareDestination), so there is no live + // migration to abort. Mark it canceled directly so Cleanup can wind the + // session down. + if c.state == StateDestinationImported { + c.state = StateCancelled + log.G(ctx).Info("migration session cancelled") return nil } - // TODO: call into HCS to cancel the in-flight migration once the cancel API is available. + // Enter the cancelling state before the abort so a concurrent transfer yields + // instead of completing; a successful abort then marks the session cancelled. + c.state = StateCancelling + + if err := c.cancelLiveMigration(ctx); err != nil { + // Abort did not take hold; mark failed so the caller can retry Cancel. + c.state = StateFailed + return fmt.Errorf("cancel live migration: %w", err) + } - // Mark the session canceled; Cleanup later returns it to idle. c.state = StateCancelled log.G(ctx).Info("migration session cancelled") return nil } +// cancelLiveMigration issues the HCS abort with c.mu released so the possibly +// blocking call does not stall other callers, re-acquiring it before returning. +// Must be called with c.mu held. +func (c *Controller) cancelLiveMigration(ctx context.Context) error { + c.mu.Unlock() + defer c.mu.Lock() + return c.vmController.CancelLiveMigration(ctx) +} + // Cleanup is the terminal call of a migration session on either side. It // reverts the controller back to [StateIdle], releasing the transport socket // and tearing down notification subscribers, regardless of whether the @@ -381,12 +446,18 @@ func (c *Controller) Cleanup(ctx context.Context, sessionID string, events chan } if c.sessionID != sessionID { - return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrFailedPrecondition) + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) } - // Cleanup is only valid once the session has been finalized. - if c.state != StateFinalized { - return fmt.Errorf("cleanup not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + // Cleanup is valid only from a settled state: + // - StateFinalized: either side after a successful Finalize. The normal + // path finalizes both ends; a cancel or error still finalizes the source. + // - StateCancelled: the destination only, whose Finalize is a no-op, so it + // cleans up directly from the canceled state. + canCleanup := c.state == StateFinalized || + (c.origin == hcsschema.MigrationOriginDestination && c.state == StateCancelled) + if !canCleanup { + return fmt.Errorf("cleanup not valid for %s in state %s: %w", c.origin, c.state, errdefs.ErrFailedPrecondition) } // The transport socket is unused past this point and the notifier is @@ -398,6 +469,7 @@ func (c *Controller) Cleanup(ctx context.Context, sessionID string, events chan c.dupSocket = 0 } + // Tear down the notifier: stop the forwarder and close all subscriber streams. if c.notifier != nil { c.notifier.close() c.notifier = nil diff --git a/internal/controller/migration/controller_destination.go b/internal/controller/migration/controller_destination.go index 1309eb6cd0..55d124872f 100644 --- a/internal/controller/migration/controller_destination.go +++ b/internal/controller/migration/controller_destination.go @@ -25,8 +25,8 @@ import ( "google.golang.org/protobuf/proto" ) -// ImportState rehydrates a source-side migration snapshot onto this controller. -// A repeat call for the same session is a no-op. +// ImportState rehydrates a source-side migration snapshot onto an idle controller. +// A repeat call, or any call once a session is already active, is rejected. func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) error { // Reject malformed input up front so the controller is never mutated // on the basis of a half-specified request. @@ -64,21 +64,16 @@ func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) c.mu.Lock() defer c.mu.Unlock() - // Idempotent retry for the same session; any other non-idle state is a conflict. - if c.state == StateDestinationImported && c.sessionID == opts.SessionID { - return nil - } if c.state != StateIdle { - return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrAlreadyExists) - } - - // Rehydrate the VM controller from the saved VM payload. - if err := opts.VMController.Import(ctx, decoded.GetVm()); err != nil { - return fmt.Errorf("import vm controller: %w", err) + return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrFailedPrecondition) } // Rehydrate each pod and index its containers so PatchResourcePaths - // can look up the owning pod. + // can look up the owning pod. Build into local maps and commit them to the + // caller's borrowed maps only on success, so a mid-loop failure leaves the + // borrowed maps untouched. + podControllers := make(map[string]*pod.Controller) + containerPodMapping := make(map[string]string) pending := make(map[string]struct{}) for _, podAny := range decoded.GetPods() { // Rebuild the pod controller from its saved payload. @@ -87,7 +82,7 @@ func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) return fmt.Errorf("import pod: %w", err) } - opts.PodControllers[importedPod.PodID()] = importedPod + podControllers[importedPod.PodID()] = importedPod // Source container IDs must be unique across pods so the later patch // lookup is unambiguous. @@ -98,10 +93,20 @@ func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) // Track the container as awaiting a patch and map it to its pod. pending[containerID] = struct{}{} - opts.ContainerPodMapping[containerID] = importedPod.PodID() + containerPodMapping[containerID] = importedPod.PodID() } } + // Rehydrate the VM controller from the saved VM payload. + if err := opts.VMController.Import(ctx, decoded.GetVm()); err != nil { + return fmt.Errorf("import vm controller: %w", err) + } + + // Import fully succeeded: merge the local index into the borrowed maps, + // preserving the service/controller shared-map instances. + maps.Copy(opts.PodControllers, podControllers) + maps.Copy(opts.ContainerPodMapping, containerPodMapping) + c.sessionID = opts.SessionID c.sandboxID = opts.SandboxID c.origin = opts.Origin @@ -117,7 +122,10 @@ func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) // PatchResourcePaths rewrites the imported source container's identifiers to // the destination IDs carried by request and spec. A container may be patched -// only once; a repeat call for an already-patched container fails. +// only once. +// +// The controller advances its own bookkeeping only after the patch succeeds; +// once a container is patched, a repeat call for it fails. func (c *Controller) PatchResourcePaths( ctx context.Context, request *task.CreateTaskRequest, @@ -145,18 +153,20 @@ func (c *Controller) PatchResourcePaths( return fmt.Errorf("patch not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) } - // A container may be patched only once; once patched, it drops out of - // pendingPatches. Hence, a non-pending ID has already been patched. + // A container may be patched only once and drops out of the pending set when it + // is; an ID absent from that set was already patched or was never imported. if _, pending := c.pendingPatches[sourceContainerID]; !pending { - return fmt.Errorf("source container %q is not pending a patch: %w", sourceContainerID, errdefs.ErrAlreadyExists) + return fmt.Errorf("source container %q is not pending a patch", sourceContainerID) } // containerPodMapping is the source-of-truth index built in ImportState // and rewritten in place here, so a direct lookup beats scanning pods. sourcePodID, ok := c.containerPodMapping[sourceContainerID] if !ok { - return fmt.Errorf("source container %q not found: %w", sourceContainerID, errdefs.ErrNotFound) + return fmt.Errorf("source container %q not found", sourceContainerID) } + + // Get pod controller from pod ID. podCtrl := c.podControllers[sourcePodID] // Sandbox is detected structurally as container ID == pod ID. @@ -179,6 +189,8 @@ func (c *Controller) PatchResourcePaths( } // Rebind the container's resources within its pod to the destination IDs. + // On failure the controller's own maps are left unadvanced; the pod itself + // may already be partly rebound. if err := podCtrl.Patch(ctx, sourceContainerID, isSandbox, scsiCtrl, request, spec); err != nil { return fmt.Errorf("patch source container %q in pod %q: %w", sourceContainerID, sourcePodID, err) } @@ -190,8 +202,7 @@ func (c *Controller) PatchResourcePaths( delete(c.podControllers, sourcePodID) c.podControllers[request.ID] = podCtrl - // Repoint any container (this sandbox plus worker containers patched earlier) - // that still references the old pod ID at the new pod ID. + // Repoint every container still mapped to the old pod ID at the new pod ID. for cid, pid := range c.containerPodMapping { if pid == sourcePodID { c.containerPodMapping[cid] = request.ID @@ -243,6 +254,8 @@ func (c *Controller) PrepareDestination(ctx context.Context, sessionID string, m migrationOpts.Origin = c.origin // Build the destination VM's HCS compute system from the imported config. + // Till the VM is created, all the above errors are benign and the API can + // be retried. if err := c.vmController.CreateVM(ctx, &vm.CreateOptions{ ID: fmt.Sprintf("%s@vm", c.sandboxID), @@ -254,6 +267,9 @@ func (c *Controller) PrepareDestination(ctx context.Context, sessionID string, m // Re-ACL the patched (destination-host) VHDs against the freshly // created VM's SID so it can open them once it starts. if err := c.vmController.Patch(ctx); err != nil { + // On error, fail the session so the caller can cancel or finalize it + // before cleanup. + c.state = StateFailed return fmt.Errorf("patch destination vm: %w", err) } diff --git a/internal/controller/migration/controller_destination_test.go b/internal/controller/migration/controller_destination_test.go index 53f79eb7e1..ff9c71ca04 100644 --- a/internal/controller/migration/controller_destination_test.go +++ b/internal/controller/migration/controller_destination_test.go @@ -15,9 +15,12 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + lcsave "github.com/Microsoft/hcsshim/internal/controller/linuxcontainer/save" "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + netsave "github.com/Microsoft/hcsshim/internal/controller/network/save" "github.com/Microsoft/hcsshim/internal/controller/pod" + podsave "github.com/Microsoft/hcsshim/internal/controller/pod/save" "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/oci" @@ -60,6 +63,33 @@ func importOptions(vmc vmController, saved *anypb.Any) *ImportStateOptions { } } +// marshalAny marshals a proto message and wraps it in a typed Any envelope. +func marshalAny(t *testing.T, typeURL string, m proto.Message) *anypb.Any { + t.Helper() + b, err := proto.Marshal(m) + if err != nil { + t.Fatalf("marshal %s: %v", typeURL, err) + } + return &anypb.Any{TypeUrl: typeURL, Value: b} +} + +// importablePod builds a valid, importable pod payload envelope carrying the +// given pod ID and container IDs. +func importablePod(t *testing.T, podID string, containerIDs ...string) *anypb.Any { + t.Helper() + containers := make([]*anypb.Any, 0, len(containerIDs)) + for _, id := range containerIDs { + containers = append(containers, marshalAny(t, lcsave.TypeURL, &lcsave.Payload{SchemaVersion: lcsave.SchemaVersion, ContainerID: id})) + } + return marshalAny(t, podsave.TypeURL, &podsave.Payload{ + SchemaVersion: podsave.SchemaVersion, + PodID: podID, + GcsPodID: podID, + Network: marshalAny(t, netsave.TypeURL, &netsave.Payload{SchemaVersion: netsave.SchemaVersion}), + Containers: containers, + }) +} + // importVM returns a VM mock reporting the not-created state that ImportState // requires before rehydrating a snapshot. func importVM(ctrl *gomock.Controller) *mocks.MockvmController { @@ -134,16 +164,18 @@ func TestImportState_RejectsUndecodableState(t *testing.T) { } } -// TestImportState_IdempotentSameSession verifies a repeat import for the active -// session is a no-op that does not re-import the VM. -func TestImportState_IdempotentSameSession(t *testing.T) { +// TestImportState_RejectsAlreadyImported verifies a repeat import while the +// controller already holds an imported snapshot is rejected as a failed +// precondition rather than re-importing the VM. +func TestImportState_RejectsAlreadyImported(t *testing.T) { ctrl := gomock.NewController(t) c := New() c.state = StateDestinationImported c.sessionID = "sess-1" - if err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))); err != nil { - t.Fatalf("unexpected error: %v", err) + err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) } } @@ -156,8 +188,8 @@ func TestImportState_RejectsConflictingState(t *testing.T) { c.sessionID = "other" err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))) - if !errors.Is(err, errdefs.ErrAlreadyExists) { - t.Fatalf("expected ErrAlreadyExists, got %v", err) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) } } @@ -215,6 +247,37 @@ func TestImportState_Success(t *testing.T) { } } +// TestImportState_PartialPodFailureIsAtomic verifies that when a later pod +// fails to import, no pod or container entries from earlier pods are committed +// to the caller's borrowed maps and the controller stays idle. +func TestImportState_PartialPodFailureIsAtomic(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := importVM(ctrl) + + // The first pod imports cleanly; the second is rejected by pod.Import. + payload := validImportPayload() + payload.Pods = []*anypb.Any{ + importablePod(t, "pod-1", "ctr-1"), + {TypeUrl: "type.bogus"}, + } + + c := New() + opts := importOptions(vmc, importEnvelope(t, payload)) + if err := c.ImportState(t.Context(), opts); err == nil { + t.Fatal("expected error, got nil") + } + + if c.state != StateIdle { + t.Errorf("expected state Idle after failure, got %s", c.state) + } + if len(opts.PodControllers) != 0 { + t.Errorf("expected borrowed pod controllers untouched, got %d entries", len(opts.PodControllers)) + } + if len(opts.ContainerPodMapping) != 0 { + t.Errorf("expected borrowed container-pod mapping untouched, got %d entries", len(opts.ContainerPodMapping)) + } +} + // ───────────────────────────────────────────────────────────────────────────── // PatchResourcePaths // ───────────────────────────────────────────────────────────────────────────── @@ -263,8 +326,8 @@ func TestPatchResourcePaths_RejectsAlreadyPatched(t *testing.T) { c.containerPodMapping = map[string]string{} err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) - if !errors.Is(err, errdefs.ErrAlreadyExists) { - t.Fatalf("expected ErrAlreadyExists, got %v", err) + if err == nil || !strings.Contains(err.Error(), "not pending a patch") { + t.Fatalf("expected not-pending error, got %v", err) } } @@ -277,8 +340,8 @@ func TestPatchResourcePaths_RejectsUnknownContainer(t *testing.T) { c.containerPodMapping = map[string]string{} err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) - if !errors.Is(err, errdefs.ErrNotFound) { - t.Fatalf("expected ErrNotFound, got %v", err) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected not-found error, got %v", err) } } diff --git a/internal/controller/migration/controller_source.go b/internal/controller/migration/controller_source.go index 60f4a25ece..c63ebe7729 100644 --- a/internal/controller/migration/controller_source.go +++ b/internal/controller/migration/controller_source.go @@ -7,7 +7,6 @@ import ( "fmt" save "github.com/Microsoft/hcsshim/internal/controller/migration/save" - "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/log" @@ -17,8 +16,7 @@ import ( ) // PrepareSource readies the source side of a migration session so a subsequent -// [Controller.ExportState] can capture its state. A repeat call for the same -// session is a no-op. +// [Controller.ExportState] can capture its state. func (c *Controller) PrepareSource(ctx context.Context, opts *PrepareSourceOptions) error { switch { case opts == nil: @@ -35,32 +33,16 @@ func (c *Controller) PrepareSource(ctx context.Context, opts *PrepareSourceOptio defer c.mu.Unlock() if c.state != StateIdle { - // If we already called this API, then this is a no-op. - if c.state == StateSourcePrepared && c.sessionID == opts.SessionID { - return nil - } return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrFailedPrecondition) } - // VM must be running in order to initiate the migration. - if opts.VMController.State() != vm.StateRunning { - return fmt.Errorf("vm controller is in invalid state %s: %w", opts.VMController.State(), errdefs.ErrFailedPrecondition) - } - - // The sandbox must have been created with live migration enabled. - // Reject otherwise so callers learn the session cannot proceed before - // any source-side state is mutated. - sandboxOpts := opts.VMController.SandboxOptions() - if sandboxOpts == nil || !sandboxOpts.LiveMigrationSupportEnabled { - return fmt.Errorf("sandbox is not configured to allow live migration: %w", errdefs.ErrFailedPrecondition) - } - if opts.MigrationOpts == nil { opts.MigrationOpts = &hcsschema.MigrationInitializeOptions{} } opts.MigrationOpts.Origin = opts.Origin // Prepare the running source VM; afterward it accepts only live-migration compatible calls. + // The session is recorded only after this succeeds; on failure the controller is left unchanged. if err := opts.VMController.InitializeLiveMigrationOnSource(ctx, opts.MigrationOpts); err != nil { return fmt.Errorf("initialize live migration on source vm: %w", err) } @@ -78,8 +60,9 @@ func (c *Controller) PrepareSource(ctx context.Context, opts *PrepareSourceOptio // ExportState captures the prepared source sandbox into an opaque, versioned // saved state that the destination consumes via [Controller.ImportState]. The VM // and per-pod payloads it carries are themselves opaque, owned by their -// respective controllers. A repeat call returns a fresh snapshot. -func (c *Controller) ExportState(ctx context.Context, sessionID string) (*anypb.Any, error) { +// respective controllers. A repeat call after a successful export is rejected, +// and a failure partway through leaves the session unusable. +func (c *Controller) ExportState(ctx context.Context, sessionID string) (_ *anypb.Any, err error) { c.mu.Lock() defer c.mu.Unlock() @@ -87,11 +70,20 @@ func (c *Controller) ExportState(ctx context.Context, sessionID string) (*anypb. return nil, fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) } - // Allow re-export after a prior success so duplicate calls are idempotent. - if c.state != StateSourcePrepared && c.state != StateSourceExported { + if c.state != StateSourcePrepared { return nil, fmt.Errorf("export requires state %s or %s (current: %s): %w", StateSourcePrepared, StateSourceExported, c.state, errdefs.ErrFailedPrecondition) } + // Past the state guard the source begins producing its snapshot, so any + // failure leaves the session partially exported and unsafe to retry. + // Poison the controller into StateFailed. + defer func() { + if err != nil { + c.state = StateFailed + log.G(ctx).WithError(err).Error("migration source export failed, moving to invalid state") + } + }() + // Save the VM state. vmAny, err := c.vmController.Save(ctx) if err != nil { diff --git a/internal/controller/migration/controller_source_test.go b/internal/controller/migration/controller_source_test.go index 5ae7d24514..55fbb75b77 100644 --- a/internal/controller/migration/controller_source_test.go +++ b/internal/controller/migration/controller_source_test.go @@ -11,19 +11,12 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" save "github.com/Microsoft/hcsshim/internal/controller/migration/save" "github.com/Microsoft/hcsshim/internal/controller/pod" - vmpkg "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" ) -// migrationEnabledOptions returns sandbox options that permit live migration. -func migrationEnabledOptions() *lcow.SandboxOptions { - return &lcow.SandboxOptions{LiveMigrationSupportEnabled: true} -} - // sourceOptions returns a valid PrepareSourceOptions bound to the given VM mock. func sourceOptions(vm vmController) *PrepareSourceOptions { return &PrepareSourceOptions{ @@ -78,72 +71,28 @@ func TestPrepareSource_RejectsWrongState(t *testing.T) { } } -// TestPrepareSource_IdempotentSameSession verifies re-arming the same session is -// a no-op that does not touch the VM. -func TestPrepareSource_IdempotentSameSession(t *testing.T) { +// TestPrepareSource_RejectsAlreadyPrepared verifies re-arming a controller that +// has already prepared the same session is rejected without touching the VM. +func TestPrepareSource_RejectsAlreadyPrepared(t *testing.T) { ctrl := gomock.NewController(t) c := New() c.state = StateSourcePrepared c.sessionID = "sess-1" - // No VM calls are expected on a duplicate arm. - if err := c.PrepareSource(t.Context(), sourceOptions(mocks.NewMockvmController(ctrl))); err != nil { - t.Fatalf("unexpected error: %v", err) + // No VM calls are expected: the state guard rejects before the VM is armed. + if err := c.PrepareSource(t.Context(), sourceOptions(mocks.NewMockvmController(ctrl))); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) } if c.state != StateSourcePrepared { t.Errorf("expected state SourcePrepared, got %s", c.state) } } -// TestPrepareSource_RejectsMigrationDisabled verifies a sandbox that was not -// created with live migration enabled cannot be armed. -func TestPrepareSource_RejectsMigrationDisabled(t *testing.T) { - cases := map[string]*lcow.SandboxOptions{ - "NilOptions": nil, - "FeatureDisabled": {LiveMigrationSupportEnabled: false}, - } - for name, sandboxOpts := range cases { - t.Run(name, func(t *testing.T) { - ctrl := gomock.NewController(t) - vm := mocks.NewMockvmController(ctrl) - vm.EXPECT().State().Return(vmpkg.StateRunning) - vm.EXPECT().SandboxOptions().Return(sandboxOpts) - - c := New() - err := c.PrepareSource(t.Context(), sourceOptions(vm)) - if !errors.Is(err, errdefs.ErrFailedPrecondition) { - t.Fatalf("expected ErrFailedPrecondition, got %v", err) - } - if c.state != StateIdle { - t.Errorf("expected state Idle, got %s", c.state) - } - }) - } -} - -// TestPrepareSource_RejectsWrongVMState verifies the source cannot be armed -// unless the VM is running. -func TestPrepareSource_RejectsWrongVMState(t *testing.T) { - ctrl := gomock.NewController(t) - vm := mocks.NewMockvmController(ctrl) - vm.EXPECT().State().Return(vmpkg.StateCreated).AnyTimes() - - c := New() - if err := c.PrepareSource(t.Context(), sourceOptions(vm)); !errors.Is(err, errdefs.ErrFailedPrecondition) { - t.Fatalf("expected ErrFailedPrecondition, got %v", err) - } - if c.state != StateIdle { - t.Errorf("expected state Idle, got %s", c.state) - } -} - // TestPrepareSource_InitializeError verifies a failure arming the VM leaves the // controller idle so the session can be retried. func TestPrepareSource_InitializeError(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) - vm.EXPECT().State().Return(vmpkg.StateRunning) - vm.EXPECT().SandboxOptions().Return(migrationEnabledOptions()) vm.EXPECT().InitializeLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(errors.New("boom")) c := New() @@ -160,8 +109,6 @@ func TestPrepareSource_InitializeError(t *testing.T) { func TestPrepareSource_Success(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) - vm.EXPECT().State().Return(vmpkg.StateRunning) - vm.EXPECT().SandboxOptions().Return(migrationEnabledOptions()) vm.EXPECT().InitializeLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) c := New() @@ -202,8 +149,8 @@ func TestExportState_RejectsSessionMismatch(t *testing.T) { } } -// TestExportState_RejectsWrongState verifies a snapshot is only produced from a -// prepared (or already-exported) source. +// TestExportState_RejectsWrongState verifies a snapshot is refused unless the +// source has been prepared. func TestExportState_RejectsWrongState(t *testing.T) { c := New() c.sessionID = "sess-1" @@ -218,7 +165,7 @@ func TestExportState_RejectsWrongState(t *testing.T) { } // TestExportState_VMSaveError verifies a failure saving the VM aborts the export -// without advancing the state. +// and poisons the session into the failed state. func TestExportState_VMSaveError(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) @@ -233,8 +180,8 @@ func TestExportState_VMSaveError(t *testing.T) { if _, err := c.ExportState(t.Context(), "sess-1"); err == nil { t.Fatal("expected error, got nil") } - if c.state != StateSourcePrepared { - t.Errorf("expected state SourcePrepared after failure, got %s", c.state) + if c.state != StateFailed { + t.Errorf("expected state Failed after failure, got %s", c.state) } } diff --git a/internal/controller/migration/controller_test.go b/internal/controller/migration/controller_test.go index 94448acb34..fe6ee37a17 100644 --- a/internal/controller/migration/controller_test.go +++ b/internal/controller/migration/controller_test.go @@ -13,6 +13,7 @@ import ( "golang.org/x/sys/windows" "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + vmpkg "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/pkg/migration" ) @@ -161,6 +162,35 @@ func TestTransfer_SocketTimeout(t *testing.T) { waitForState(t, c, StateFailed) } +// TestTransfer_CompletionDoesNotClobberCancel verifies that a Cancel landing +// while the transfer runs wins: the successful transfer leaves the session +// Cancelled rather than advancing it to TransferCompleted. +func TestTransfer_CompletionDoesNotClobberCancel(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + + c := New() + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + c.state = StateSocketReady + close(c.socketReady) // socket already arrived + + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + // Simulate a concurrent Cancel arriving mid-transfer, once the goroutine has + // released the lock to run the transfer. + vm.EXPECT().CancelLiveMigration(gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *hcsschema.MigrationTransferOptions) error { + return c.Cancel(context.Background(), "sess-1") + }) + + if err := c.Transfer(t.Context(), "sess-1", time.Minute); err != nil { + t.Fatalf("unexpected error: %v", err) + } + waitForState(t, c, StateCancelled) +} + // ───────────────────────────────────────────────────────────────────────────── // Finalize // ───────────────────────────────────────────────────────────────────────────── @@ -180,16 +210,35 @@ func TestFinalize_RejectsUnspecifiedAction(t *testing.T) { func TestFinalize_IdempotentFinalized(t *testing.T) { c := New() c.state = StateFinalized + c.sessionID = "sess-1" if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err != nil { t.Fatalf("unexpected error: %v", err) } } +// TestFinalize_DestinationCancelledNoop verifies a cancelled destination, which +// winds down through Cleanup rather than Finalize, treats Finalize as a no-op. +func TestFinalize_DestinationCancelledNoop(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateCancelled + c.origin = hcsschema.MigrationOriginDestination + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateCancelled { + t.Errorf("state = %s; want unchanged Cancelled", c.State()) + } +} + // TestFinalize_RejectsWrongState verifies finalize is rejected before a transfer // has completed or been cancelled. func TestFinalize_RejectsWrongState(t *testing.T) { - c := New() // StateIdle + c := New() + c.state = StateTransferring + c.sessionID = "sess-1" err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil) if !errors.Is(err, errdefs.ErrFailedPrecondition) { @@ -213,9 +262,11 @@ func TestFinalize_RejectsUnsupportedAction(t *testing.T) { func TestFinalize_FinalizeError(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateMigrationTransferCompleted) vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(errors.New("boom")) c := New() + c.sessionID = "sess-1" c.state = StateTransferCompleted c.origin = hcsschema.MigrationOriginSource c.vmController = vm @@ -242,10 +293,12 @@ func TestFinalize_ResumeSucceeds(t *testing.T) { t.Run(name, func(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateMigrationTransferCompleted) vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) vm.EXPECT().Resume(gomock.Any(), tc.rebuildBridge).Return(nil) c := New() + c.sessionID = "sess-1" c.state = StateTransferCompleted c.origin = tc.origin c.vmController = vm @@ -267,9 +320,11 @@ func TestFinalize_StopSucceeds(t *testing.T) { t.Run(string(origin), func(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateMigrationTransferCompleted) vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) c := New() + c.sessionID = "sess-1" c.state = StateTransferCompleted c.origin = origin c.vmController = vm @@ -289,10 +344,12 @@ func TestFinalize_StopSucceeds(t *testing.T) { func TestFinalize_ResumeError(t *testing.T) { ctrl := gomock.NewController(t) vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateMigrationTransferCompleted) vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) vm.EXPECT().Resume(gomock.Any(), false).Return(errors.New("boom")) c := New() + c.sessionID = "sess-1" c.state = StateTransferCompleted c.origin = hcsschema.MigrationOriginDestination c.vmController = vm @@ -305,6 +362,94 @@ func TestFinalize_ResumeError(t *testing.T) { } } +// TestFinalize_StopAfterResumeTerminates verifies a stop finalize retried after +// the VM was already resume-finalized tears the VM down via TerminateVM instead +// of re-issuing the finalize. +func TestFinalize_StopAfterResumeTerminates(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateMigrationFinalized) + vm.EXPECT().TerminateVM(gomock.Any()).Return(nil) + + c := New() + c.sessionID = "sess-1" + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } +} + +// TestFinalize_ResumeAfterResumeReRunsResume verifies a resume finalize retried +// after the VM was already resume-finalized skips re-finalizing and re-runs the +// resume steps. +func TestFinalize_ResumeAfterResumeReRunsResume(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateMigrationFinalized) + vm.EXPECT().Resume(gomock.Any(), true).Return(nil) + + c := New() + c.sessionID = "sess-1" + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } +} + +// TestFinalize_StopAfterTerminateNoop verifies a stop finalize retried after the +// VM was already terminated is a no-op that neither re-finalizes nor re-terminates. +func TestFinalize_StopAfterTerminateNoop(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateTerminated) + + c := New() + c.sessionID = "sess-1" + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } +} + +// TestFinalize_ResumeAfterTerminateErrors verifies a resume finalize is rejected +// once the VM has already been terminated by a stop finalize. +func TestFinalize_ResumeAfterTerminateErrors(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateTerminated) + + c := New() + c.sessionID = "sess-1" + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if c.State() != StateTransferCompleted { + t.Errorf("state = %s; want unchanged TransferCompleted", c.State()) + } +} + // ───────────────────────────────────────────────────────────────────────────── // Cancel // ───────────────────────────────────────────────────────────────────────────── @@ -315,8 +460,8 @@ func TestCancel_RejectsSessionMismatch(t *testing.T) { c := New() c.sessionID = "sess-1" - if err := c.Cancel(t.Context(), "other"); !errors.Is(err, errdefs.ErrFailedPrecondition) { - t.Fatalf("expected ErrFailedPrecondition, got %v", err) + if err := c.Cancel(t.Context(), "other"); !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) } } @@ -335,11 +480,16 @@ func TestCancel_Idempotent(t *testing.T) { } } -// TestCancel_Success verifies cancelling an in-flight session moves it to -// cancelled. +// TestCancel_Success verifies cancelling an in-flight session aborts the +// migration in HCS and moves the session to cancelled. func TestCancel_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().CancelLiveMigration(gomock.Any()).Return(nil) + c := New() c.sessionID = "sess-1" + c.vmController = vm c.state = StateTransferCompleted if err := c.Cancel(t.Context(), "sess-1"); err != nil { @@ -350,6 +500,49 @@ func TestCancel_Success(t *testing.T) { } } +// TestCancel_VMError verifies a failure aborting the migration in HCS surfaces +// to the caller and leaves the session failed so Cancel can be retried. +func TestCancel_VMError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().CancelLiveMigration(gomock.Any()).Return(errors.New("boom")) + + c := New() + c.sessionID = "sess-1" + c.vmController = vm + c.state = StateFailed + + if err := c.Cancel(t.Context(), "sess-1"); err == nil { + t.Fatal("expected error, got nil") + } + if c.State() != StateFailed { + t.Errorf("state = %s; want Failed", c.State()) + } +} + +// TestCancel_DestinationImported verifies cancelling a destination that has only +// imported the snapshot moves it to cancelled without issuing an HCS abort, +// since the compute system has not been materialized yet. +func TestCancel_DestinationImported(t *testing.T) { + ctrl := gomock.NewController(t) + // No CancelLiveMigration call is expected: gomock fails the test if the + // controller erroneously issues an HCS abort at this state. + vm := mocks.NewMockvmController(ctrl) + + c := New() + c.sessionID = "sess-1" + c.vmController = vm + c.origin = hcsschema.MigrationOriginDestination + c.state = StateDestinationImported + + if err := c.Cancel(t.Context(), "sess-1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateCancelled { + t.Errorf("state = %s; want Cancelled", c.State()) + } +} + // ───────────────────────────────────────────────────────────────────────────── // Cleanup // ───────────────────────────────────────────────────────────────────────────── @@ -370,8 +563,8 @@ func TestCleanup_RejectsSessionMismatch(t *testing.T) { c.state = StateFinalized c.sessionID = "sess-1" - if err := c.Cleanup(t.Context(), "other", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { - t.Fatalf("expected ErrFailedPrecondition, got %v", err) + if err := c.Cleanup(t.Context(), "other", nil); !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) } } @@ -406,6 +599,22 @@ func TestCleanup_Success(t *testing.T) { } } +// TestCleanup_DestinationCancelled verifies a cancelled destination cleans up +// directly from the cancelled state, returning the controller to idle. +func TestCleanup_DestinationCancelled(t *testing.T) { + c := New() + c.state = StateCancelled + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginDestination + + if err := c.Cleanup(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateIdle { + t.Errorf("state = %s; want Idle", c.State()) + } +} + // TestCleanup_ClosesSocketAndNotifier verifies cleanup releases the transport // socket and tears down the notifier before returning to idle. func TestCleanup_ClosesSocketAndNotifier(t *testing.T) { @@ -532,3 +741,32 @@ func TestFailTransfer_BroadcastsToSubscribers(t *testing.T) { }) } } + +// TestFailTransfer_SkipsWhenCancelled verifies a transfer error is swallowed +// once the session is Cancelled: the state is left Cancelled and no failure +// event is broadcast to subscribers. +func TestFailTransfer_SkipsWhenCancelled(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + c := New() + c.origin = hcsschema.MigrationOriginSource + c.notifier = n + c.state = StateCancelled + + c.failTransfer(context.Background(), errors.New("boom")) + + if c.State() != StateCancelled { + t.Errorf("state = %s; want unchanged Cancelled", c.State()) + } + select { + case got := <-sub: + t.Fatalf("expected no notification, got %+v", got) + case <-time.After(100 * time.Millisecond): + } +} diff --git a/internal/controller/migration/doc.go b/internal/controller/migration/doc.go index 75db1c582a..3d936ca1c5 100644 --- a/internal/controller/migration/doc.go +++ b/internal/controller/migration/doc.go @@ -62,6 +62,10 @@ // │ StateIdle │ // └──────────────┘ // +// If [Controller.Transfer] is called before the socket is registered, the session +// waits in [StateSocketWaiting] and advances to [StateTransferring] once +// [Controller.RegisterDuplicateSocket] supplies the socket. +// // State descriptions: // // - [StateIdle]: no session is active; the initial state and where @@ -69,13 +73,14 @@ // - [StateSourcePrepared]: [Controller.PrepareSource] has armed the source; the // next call is [Controller.ExportState]. // - [StateSourceExported]: [Controller.ExportState] has produced the opaque -// sandbox snapshot; the next call is [Controller.RegisterDuplicateSocket]. +// sandbox snapshot; the next calls are [Controller.RegisterDuplicateSocket] +// and [Controller.Transfer], in either order. // - [StateDestinationImported]: [Controller.ImportState] has rehydrated the // snapshot; the next calls are [Controller.PatchResourcePaths] (one per // container) followed by [Controller.PrepareDestination]. // - [StateDestinationPrepared]: [Controller.PrepareDestination] has materialized -// the destination HCS compute system; the next call is -// [Controller.RegisterDuplicateSocket]. +// the destination HCS compute system; the next calls are +// [Controller.RegisterDuplicateSocket] and [Controller.Transfer], in either order. // - [StateSocketReady]: the duplicated transport socket has been adopted via // [Controller.RegisterDuplicateSocket]; the next call is [Controller.Transfer]. // - [StateSocketWaiting]: [Controller.Transfer] ran before the socket arrived and @@ -89,8 +94,27 @@ // next call is [Controller.Cleanup]. // - [StateCancelled]: [Controller.Cancel] aborted the session; the next call is // [Controller.Finalize] followed by [Controller.Cleanup]. -// - [StateFailed]: the memory transfer failed; the next call is -// [Controller.Cancel]. +// - [StateCancelling]: [Controller.Cancel] is aborting the migration; it +// advances to [StateCancelled] on success or [StateFailed] on error. +// - [StateFailed]: the session hit an unrecoverable error; the next call is +// [Controller.Finalize] or [Controller.Cancel], then [Controller.Cleanup]. +// +// # Failure and cancellation +// +// [Controller.Cancel] aborts a session in progress. A destination that has only +// imported ([StateDestinationImported]) has no compute system to abort and moves +// straight to [StateCancelled]; any other active session passes through +// [StateCancelling] while the abort runs, reaching [StateCancelled], or +// [StateFailed] if the abort itself fails. A cancel landing during +// [StateTransferring] wins over the transfer. +// +// [StateFailed] and [StateCancelled] wind down like a completed transfer: +// +// {StateTransferCompleted | StateFailed | StateCancelled} +// ── Finalize ──▶ StateFinalized ── Cleanup ──▶ StateIdle +// +// A cancelled destination is the exception: its [Controller.Finalize] is a no-op, +// so it cleans up directly from [StateCancelled]. // // # Notifications // diff --git a/internal/controller/migration/mocks/mock_lcow_migration.go b/internal/controller/migration/mocks/mock_lcow_migration.go index d97956308b..6262ee077f 100644 --- a/internal/controller/migration/mocks/mock_lcow_migration.go +++ b/internal/controller/migration/mocks/mock_lcow_migration.go @@ -53,6 +53,20 @@ func (m *MockvmController) EXPECT() *MockvmControllerMockRecorder { return m.recorder } +// CancelLiveMigration mocks base method. +func (m *MockvmController) CancelLiveMigration(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelLiveMigration", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// CancelLiveMigration indicates an expected call of CancelLiveMigration. +func (mr *MockvmControllerMockRecorder) CancelLiveMigration(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelLiveMigration", reflect.TypeOf((*MockvmController)(nil).CancelLiveMigration), ctx) +} + // CreateVM mocks base method. func (m *MockvmController) CreateVM(ctx context.Context, opts *vm.CreateOptions) error { m.ctrl.T.Helper() @@ -308,6 +322,20 @@ func (mr *MockvmControllerMockRecorder) State() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockvmController)(nil).State)) } +// TerminateVM mocks base method. +func (m *MockvmController) TerminateVM(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TerminateVM", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// TerminateVM indicates an expected call of TerminateVM. +func (mr *MockvmControllerMockRecorder) TerminateVM(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateVM", reflect.TypeOf((*MockvmController)(nil).TerminateVM), ctx) +} + // VM mocks base method. func (m *MockvmController) VM() *vmmanager.UtilityVM { m.ctrl.T.Helper() diff --git a/internal/controller/migration/socket.go b/internal/controller/migration/socket.go index a97645f14e..c4b77c6906 100644 --- a/internal/controller/migration/socket.go +++ b/internal/controller/migration/socket.go @@ -3,9 +3,7 @@ package migration import ( - "bytes" "context" - "encoding/binary" "fmt" "sync" "unsafe" @@ -43,9 +41,9 @@ const connectTimeNotConnected uint32 = 0xFFFFFFFF // RegisterDuplicateSocket adopts a duplicated migration transport socket, // described by protocolInfo, into this process and makes it available to the -// pending transfer for the given session. A repeat call for an already-adopted -// session is a no-op. -func (c *Controller) RegisterDuplicateSocket(ctx context.Context, sessionID string, protocolInfo []byte) error { +// pending transfer for the given session. A repeat call once a socket is already +// registered is rejected. +func (c *Controller) RegisterDuplicateSocket(ctx context.Context, sessionID string, protocolInfo []byte) (err error) { // Reject input too small to hold a serialized socket descriptor before // attempting to decode it. wantSize := int(unsafe.Sizeof(windows.WSAProtocolInfo{})) @@ -53,12 +51,10 @@ func (c *Controller) RegisterDuplicateSocket(ctx context.Context, sessionID stri return fmt.Errorf("protocol info is %d bytes, want at least %d: %w", len(protocolInfo), wantSize, errdefs.ErrInvalidArgument) } - // Decode the opaque caller-supplied bytes into the socket descriptor - // used to recreate the duplicated socket. + // The bytes are the descriptor's raw native memory as the caller + // serialized it, so copy them back over the same layout to decode. var info windows.WSAProtocolInfo - if err := binary.Read(bytes.NewReader(protocolInfo), binary.LittleEndian, &info); err != nil { - return fmt.Errorf("decode WSAProtocolInfo: %w", err) - } + copy(unsafe.Slice((*byte)(unsafe.Pointer(&info)), wantSize), protocolInfo) c.mu.Lock() defer c.mu.Unlock() @@ -67,41 +63,59 @@ func (c *Controller) RegisterDuplicateSocket(ctx context.Context, sessionID stri return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) } - // Idempotent: a repeat call for the same session is a no-op once the - // socket has been adopted. + // If socket has already been adopted for a session, return an error. if c.dupSocket != 0 { - return nil + return fmt.Errorf("duplicate socket already registered for session %q: %w", sessionID, errdefs.ErrAlreadyExists) } // Transfer may have already claimed the session (StateSocketWaiting) and // be waiting on the socket; allow registration in that case too. if c.state != StateSourceExported && c.state != StateDestinationPrepared && c.state != StateSocketWaiting { - return fmt.Errorf("register duplicate socket requires state %s or %s (current: %s): %w", StateSourceExported, StateDestinationPrepared, c.state, errdefs.ErrFailedPrecondition) + return fmt.Errorf("invalid register duplicate socket state (current: %s): %w", c.state, errdefs.ErrFailedPrecondition) } + // sock holds the recreated socket once WSASocket succeeds; until then it + // stays 0 so the failure cleanup below only closes a socket we own. + var sock windows.Handle + + // Past the state guard, any failure aborts the session: fail it, drop the + // socket we created (if any), and close socketReady so a Transfer goroutine + // blocked on it wakes, observes the failure, and bails instead of + // hanging until its timeout. + defer func() { + if err != nil { + if sock != 0 { + _ = windows.Closesocket(sock) + } + + c.state = StateFailed + close(c.socketReady) + log.G(ctx).WithError(err).Error("duplicate migration socket registration failed, session failed") + } + }() + // Make sure Winsock is up for this process before recreating the socket. - if err := ensureWinsock(); err != nil { + if err = ensureWinsock(); err != nil { return err } // Recreate the duplicated socket in this process from the descriptor so // the transfer can use it as its transport. - sock, err := windows.WSASocket(info.AddressFamily, info.SocketType, info.Protocol, &info, 0, 0) + s, err := windows.WSASocket(info.AddressFamily, info.SocketType, info.Protocol, &info, 0, 0) if err != nil { return fmt.Errorf("WSASocket: %w", err) } + sock = s // Verify the duplicated handle actually represents a connected socket; // HCS will fail the migration if we hand it an unconnected endpoint. var connectTime uint32 optLen := int32(unsafe.Sizeof(connectTime)) - if err := windows.Getsockopt(sock, windows.SOL_SOCKET, soConnectTime, (*byte)(unsafe.Pointer(&connectTime)), &optLen); err != nil { - _ = windows.Closesocket(sock) + if err = windows.Getsockopt(sock, windows.SOL_SOCKET, soConnectTime, (*byte)(unsafe.Pointer(&connectTime)), &optLen); err != nil { return fmt.Errorf("getsockopt SO_CONNECT_TIME: %w", err) } if connectTime == connectTimeNotConnected { - _ = windows.Closesocket(sock) return fmt.Errorf("duplicated socket is not connected: %w", errdefs.ErrFailedPrecondition) } diff --git a/internal/controller/migration/socket_test.go b/internal/controller/migration/socket_test.go index 71d2be6759..8c7de3d17e 100644 --- a/internal/controller/migration/socket_test.go +++ b/internal/controller/migration/socket_test.go @@ -4,7 +4,9 @@ package migration import ( "context" + "encoding/binary" "errors" + "net" "testing" "unsafe" @@ -41,16 +43,16 @@ func TestRegisterDuplicateSocket_SessionMismatch(t *testing.T) { } } -// TestRegisterDuplicateSocket_Idempotent verifies a repeat call once a socket -// has been adopted is a no-op, regardless of state. -func TestRegisterDuplicateSocket_Idempotent(t *testing.T) { +// TestRegisterDuplicateSocket_RejectsAlreadyRegistered verifies a repeat call +// once a socket has been adopted is rejected as already exists. +func TestRegisterDuplicateSocket_RejectsAlreadyRegistered(t *testing.T) { c := New() c.sessionID = "s" c.dupSocket = windows.Handle(1) c.state = StateSourceExported - if err := c.RegisterDuplicateSocket(context.Background(), "s", validProtocolInfo()); err != nil { - t.Fatalf("got %v, want nil", err) + if err := c.RegisterDuplicateSocket(context.Background(), "s", validProtocolInfo()); !errors.Is(err, errdefs.ErrAlreadyExists) { + t.Fatalf("got %v, want ErrAlreadyExists", err) } } @@ -66,3 +68,88 @@ func TestRegisterDuplicateSocket_InvalidState(t *testing.T) { t.Fatalf("got %v, want ErrFailedPrecondition", err) } } + +// TestProtocolInfoNativeLayout guards the assumption behind decoding the +// descriptor as raw native memory: it has no padding, so the native size we +// validate on the wire equals its packed size. A layout change adds padding. +func TestProtocolInfoNativeLayout(t *testing.T) { + native := int(unsafe.Sizeof(windows.WSAProtocolInfo{})) + packed := binary.Size(windows.WSAProtocolInfo{}) + if native != packed { + t.Fatalf("native size %d != packed size %d; revisit the decode in RegisterDuplicateSocket", native, packed) + } +} + +// connectedSocketDescriptor returns a real connected socket's descriptor, +// serialized the way the caller transmits it: the descriptor's raw native +// memory. The source socket is kept alive for the duration of the test. +func connectedSocketDescriptor(t *testing.T) []byte { + t.Helper() + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + t.Cleanup(func() { _ = ln.Close() }) + + conn, err := net.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("dial: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + + server, err := ln.Accept() + if err != nil { + t.Fatalf("accept: %v", err) + } + t.Cleanup(func() { _ = server.Close() }) + + var handle windows.Handle + raw, err := conn.(*net.TCPConn).SyscallConn() + if err != nil { + t.Fatalf("syscall conn: %v", err) + } + if err := raw.Control(func(fd uintptr) { handle = windows.Handle(fd) }); err != nil { + t.Fatalf("control: %v", err) + } + + // WSADuplicateSocket is exactly how the caller produces the descriptor. + var info windows.WSAProtocolInfo + if err := windows.WSADuplicateSocket(handle, windows.GetCurrentProcessId(), &info); err != nil { + t.Fatalf("WSADuplicateSocket: %v", err) + } + + b := make([]byte, unsafe.Sizeof(info)) + copy(b, unsafe.Slice((*byte)(unsafe.Pointer(&info)), unsafe.Sizeof(info))) + return b +} + +// TestRegisterDuplicateSocket_EndToEnd drives a real OS-produced descriptor +// through registration to verify the decode and socket recreation succeed and +// the session advances to StateSocketReady. +func TestRegisterDuplicateSocket_EndToEnd(t *testing.T) { + c := New() + c.sessionID = "s" + c.state = StateSourceExported + + if err := c.RegisterDuplicateSocket(context.Background(), "s", connectedSocketDescriptor(t)); err != nil { + t.Fatalf("got %v, want nil", err) + } + t.Cleanup(func() { + if c.dupSocket != 0 { + _ = windows.Closesocket(c.dupSocket) + } + }) + + if got := c.State(); got != StateSocketReady { + t.Fatalf("state = %s, want %s", got, StateSocketReady) + } + if c.dupSocket == 0 { + t.Fatal("dupSocket not set") + } + select { + case <-c.socketReady: + default: + t.Fatal("socketReady not closed") + } +} diff --git a/internal/controller/migration/state.go b/internal/controller/migration/state.go index dbd2532e71..ee8193bd6f 100644 --- a/internal/controller/migration/state.go +++ b/internal/controller/migration/state.go @@ -17,10 +17,12 @@ package migration // → StateTransferring → StateTransferCompleted → StateFinalized → StateIdle // // [StateSocketWaiting] is entered instead of [StateSocketReady] when -// [Controller.Transfer] runs before the socket arrives. A failed transfer -// enters [StateFailed], from which [Controller.Cancel] moves to -// [StateCancelled]; [Controller.Finalize] then [Controller.Cleanup] wind a -// canceled or completed session down to [StateIdle]. +// [Controller.Transfer] runs before the socket arrives. A step that fails past its +// point of no return enters [StateFailed] (earlier failures stay retriable in the +// prior state). [Controller.Cancel] moves an active session to [StateCancelled]— +// directly from [StateDestinationImported], otherwise via a transient +// [StateCancelling]. A completed, failed, or cancelled session is wound down by +// [Controller.Finalize] then [Controller.Cleanup], back to [StateIdle]. type State int32 const ( @@ -35,8 +37,8 @@ const ( StateSourcePrepared // StateSourceExported indicates the source has produced its opaque saved-state - // envelope via [Controller.ExportState]. The next valid call is - // [Controller.RegisterDuplicateSocket]. + // envelope via [Controller.ExportState]. The next valid calls are + // [Controller.RegisterDuplicateSocket] and [Controller.Transfer], in either order. StateSourceExported // StateDestinationImported indicates the destination has rehydrated the source @@ -46,13 +48,13 @@ const ( StateDestinationImported // StateDestinationPrepared indicates the destination has materialized the - // HCS compute system via [Controller.PrepareDestination]. The next - // valid call is [Controller.RegisterDuplicateSocket]. + // HCS compute system via [Controller.PrepareDestination]. The next valid calls + // are [Controller.RegisterDuplicateSocket] and [Controller.Transfer], in either order. StateDestinationPrepared - // StateSocketWaiting indicates [Controller.Transfer] has claimed the - // session and a single goroutine is driving it, waiting for the duplicate - // socket if it is not yet ready. + // StateSocketWaiting indicates [Controller.Transfer] has claimed the session + // and is waiting in the background for the duplicate socket; it advances to + // [StateTransferring] once [Controller.RegisterDuplicateSocket] supplies it. StateSocketWaiting // StateSocketReady indicates the duplicated migration transport socket @@ -78,8 +80,12 @@ const ( // [Controller.Cleanup] to return the controller to [StateIdle]. StateCancelled - // StateFailed indicates the memory transfer failed. The next valid call is - // [Controller.Cancel] to abort the session. + // StateCancelling indicates [Controller.Cancel] is aborting the in-flight + // migration; it becomes [StateCancelled] on success or [StateFailed] on error. + StateCancelling + + // StateFailed indicates the session hit an unrecoverable error. The next call + // is [Controller.Finalize] or [Controller.Cancel], then [Controller.Cleanup]. StateFailed ) @@ -108,6 +114,8 @@ func (s State) String() string { return "Finalized" case StateCancelled: return "Cancelled" + case StateCancelling: + return "Cancelling" case StateFailed: return "Failed" default: diff --git a/internal/controller/migration/types.go b/internal/controller/migration/types.go index 800be0ecf4..acd29cbf57 100644 --- a/internal/controller/migration/types.go +++ b/internal/controller/migration/types.go @@ -35,6 +35,8 @@ type vmController interface { StartWithMigrationOptions(ctx context.Context, config *hcs.MigrationConfig) error StartLiveMigrationTransfer(ctx context.Context, options *hcsschema.MigrationTransferOptions) error FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error + CancelLiveMigration(ctx context.Context) error + TerminateVM(ctx context.Context) error Resume(ctx context.Context, rebuildBridge bool) error MigrationNotifications() (<-chan hcsschema.OperationSystemMigrationNotificationInfo, error) diff --git a/internal/controller/network/save.go b/internal/controller/network/save.go index c2f22295d1..b20a33c926 100644 --- a/internal/controller/network/save.go +++ b/internal/controller/network/save.go @@ -13,22 +13,23 @@ import ( "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/vm/guestmanager" "github.com/Microsoft/hcsshim/internal/vm/vmmanager" + "github.com/containerd/errdefs" "github.com/sirupsen/logrus" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) // Save serializes the controller's current network state into a portable -// envelope that can be handed to a migration destination. It succeeds only -// when the network is fully configured, and on success freezes the source -// until it is resumed or torn down. +// envelope that can be handed to a migration destination. It succeeds when the +// network is configured or already frozen by a prior save (retry-safe), and on +// success freezes the source until it is resumed or torn down. func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { c.mu.Lock() defer c.mu.Unlock() - // Only a fully configured network is in a stable, migratable state. - if c.netState != StateConfigured { - return nil, fmt.Errorf("network controller in state %s; want %s", c.netState, StateConfigured) + // A configured network—or one already frozen by a prior save—can be saved. + if c.netState != StateConfigured && c.netState != StateSourceMigrating { + return nil, fmt.Errorf("network controller in state %s; want %s: %w", c.netState, StateConfigured, errdefs.ErrFailedPrecondition) } // Capture the scalar configuration into the snapshot. @@ -72,11 +73,11 @@ func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { // Reject an empty or mistyped envelope before touching its bytes. if env == nil { - return nil, fmt.Errorf("network saved-state envelope is nil") + return nil, fmt.Errorf("network saved-state envelope is nil: %w", errdefs.ErrInvalidArgument) } if env.GetTypeUrl() != netsave.TypeURL { - return nil, fmt.Errorf("unsupported network saved-state type %q", env.GetTypeUrl()) + return nil, fmt.Errorf("unsupported network saved-state type %q: %w", env.GetTypeUrl(), errdefs.ErrInvalidArgument) } // Decode and reject any payload this build cannot interpret. @@ -86,7 +87,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { } if v := state.GetSchemaVersion(); v != netsave.SchemaVersion { - return nil, fmt.Errorf("unsupported network saved-state schema version %d (want %d)", v, netsave.SchemaVersion) + return nil, fmt.Errorf("unsupported network saved-state schema version %d (want %d): %w", v, netsave.SchemaVersion, errdefs.ErrInvalidArgument) } // Rehydrate into the destination-migrating state: state is restored but no @@ -127,7 +128,7 @@ func (c *Controller) Patch(ctx context.Context, networkNamespaceID string) error // Patch only applies to a controller imported on the destination. if c.netState != StateDestinationMigrating { - return fmt.Errorf("network controller in state %s; want %s", c.netState, StateDestinationMigrating) + return fmt.Errorf("network controller in state %s; want %s: %w", c.netState, StateDestinationMigrating, errdefs.ErrFailedPrecondition) } c.migratedNamespaceID = networkNamespaceID @@ -147,9 +148,14 @@ func (c *Controller) Resume(ctx context.Context, vm *vmmanager.UtilityVM, guest c.mu.Lock() defer c.mu.Unlock() + // Idempotent: a network resumed by a prior attempt is a no-op on retry. + if c.netState == StateConfigured { + return nil + } + // Resume is only valid mid-migration, on either the source or destination. if c.netState != StateSourceMigrating && c.netState != StateDestinationMigrating { - return fmt.Errorf("network controller in state %s; want %s or %s", c.netState, StateSourceMigrating, StateDestinationMigrating) + return fmt.Errorf("network controller in state %s; want %s or %s: %w", c.netState, StateSourceMigrating, StateDestinationMigrating, errdefs.ErrFailedPrecondition) } c.vmNetwork = vm @@ -170,6 +176,11 @@ func (c *Controller) ResetAfterMigration(ctx context.Context) error { c.mu.Lock() defer c.mu.Unlock() + // Idempotent: a reset already applied (or a controller never patched for one) is a no-op. + if c.migratedNamespaceID == "" { + return nil + } + // Drop the stale source NICs inherited from the saved state. for nicID, ep := range c.vmEndpoints { if err := c.removeEndpointFromGuestNamespace(ctx, nicID, nil); err != nil { @@ -195,6 +206,7 @@ func (c *Controller) ResetAfterMigration(ctx context.Context) error { if err != nil { return fmt.Errorf("generate NIC GUID: %w", err) } + if err := c.addEndpointToGuestNamespace(ctx, c.namespaceID, nicGUID.String(), endpoint, c.policyBasedRouting); err != nil { return fmt.Errorf("add destination endpoint %s to guest: %w", endpoint.Name, err) } diff --git a/internal/controller/network/save_test.go b/internal/controller/network/save_test.go index 438fa56d49..41cf4a968a 100644 --- a/internal/controller/network/save_test.go +++ b/internal/controller/network/save_test.go @@ -3,9 +3,11 @@ package network import ( + "errors" "strings" "testing" + "github.com/containerd/errdefs" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -42,17 +44,17 @@ func configuredController(eps map[string]*hcn.HostComputeEndpoint) *Controller { // Save // ───────────────────────────────────────────────────────────────────────────── -// TestSave_RejectsUnstableState verifies that a snapshot is only produced from -// a fully-configured network; every other state yields an error and no payload. +// TestSave_RejectsUnstableState verifies a snapshot is refused for states other +// than a configured or already-saved network, yielding an error and no payload. func TestSave_RejectsUnstableState(t *testing.T) { - for _, st := range []State{StateNotConfigured, StateInvalid, StateTornDown, StateDestinationMigrating, StateSourceMigrating} { + for _, st := range []State{StateNotConfigured, StateInvalid, StateTornDown, StateDestinationMigrating} { t.Run(st.String(), func(t *testing.T) { c := configuredController(map[string]*hcn.HostComputeEndpoint{}) c.netState = st env, err := c.Save(t.Context()) - if err == nil { - t.Fatalf("expected error saving from state %s, got nil", st) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition saving from state %s, got %v", st, err) } if env != nil { t.Errorf("expected nil envelope on failure, got %+v", env) @@ -119,6 +121,20 @@ func TestSave_Success(t *testing.T) { } } +// TestSave_FromSourceMigrating verifies a re-save of an already-migrating source +// still succeeds, so a retried export is safe. +func TestSave_FromSourceMigrating(t *testing.T) { + c := configuredController(map[string]*hcn.HostComputeEndpoint{}) + c.netState = StateSourceMigrating + + if _, err := c.Save(t.Context()); err != nil { + t.Fatalf("Save() from SourceMigrating = %v; want nil", err) + } + if c.netState != StateSourceMigrating { + t.Errorf("netState = %s; want unchanged StateSourceMigrating", c.netState) + } +} + // ───────────────────────────────────────────────────────────────────────────── // Import // ───────────────────────────────────────────────────────────────────────────── @@ -128,17 +144,18 @@ func TestSave_Success(t *testing.T) { // unknown schema version, or a binding missing its NIC key. func TestImport_Rejects(t *testing.T) { cases := []struct { - name string - env *anypb.Any + name string + env *anypb.Any + wantIs error }{ - {"NilEnvelope", nil}, - {"WrongTypeURL", &anypb.Any{TypeUrl: "type.microsoft.com/bogus", Value: nil}}, - {"CorruptPayload", &anypb.Any{TypeUrl: netsave.TypeURL, Value: []byte{0x08}}}, - {"SchemaMismatch", mustEnvelope(t, &netsave.Payload{SchemaVersion: netsave.SchemaVersion + 1})}, + {"NilEnvelope", nil, errdefs.ErrInvalidArgument}, + {"WrongTypeURL", &anypb.Any{TypeUrl: "type.microsoft.com/bogus", Value: nil}, errdefs.ErrInvalidArgument}, + {"CorruptPayload", &anypb.Any{TypeUrl: netsave.TypeURL, Value: []byte{0x08}}, nil}, + {"SchemaMismatch", mustEnvelope(t, &netsave.Payload{SchemaVersion: netsave.SchemaVersion + 1}), errdefs.ErrInvalidArgument}, {"EmptyNICKey", mustEnvelope(t, &netsave.Payload{ SchemaVersion: netsave.SchemaVersion, VmEndpoints: map[string]*netsave.EndpointBinding{"": {EndpointID: "ep-1"}}, - })}, + }), nil}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -149,6 +166,9 @@ func TestImport_Rejects(t *testing.T) { if c != nil { t.Errorf("expected nil controller on failure, got %+v", c) } + if tc.wantIs != nil && !errors.Is(err, tc.wantIs) { + t.Errorf("expected error to wrap %v, got %v", tc.wantIs, err) + } }) } } @@ -243,8 +263,9 @@ func TestPatch_RejectsNonDestinationMigrating(t *testing.T) { for _, st := range []State{StateNotConfigured, StateConfigured, StateInvalid, StateTornDown, StateSourceMigrating} { t.Run(st.String(), func(t *testing.T) { c := &Controller{netState: st} - if err := c.Patch(t.Context(), "dst-ns"); err == nil { - t.Errorf("Patch() in state %s = nil; want error", st) + err := c.Patch(t.Context(), "dst-ns") + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Errorf("Patch() in state %s = %v; want ErrFailedPrecondition", st, err) } }) } @@ -272,15 +293,38 @@ func TestResume_TransitionsToConfigured(t *testing.T) { } } -// TestResume_RejectsNonMigrating verifies Resume is refused unless the network -// is mid-migration on the source or destination. +// TestResume_RejectsNonMigrating verifies Resume is refused for states that are +// neither mid-migration nor already configured. func TestResume_RejectsNonMigrating(t *testing.T) { - for _, st := range []State{StateNotConfigured, StateConfigured, StateInvalid, StateTornDown} { + for _, st := range []State{StateNotConfigured, StateInvalid, StateTornDown} { t.Run(st.String(), func(t *testing.T) { c := &Controller{netState: st, vmEndpoints: map[string]*hcn.HostComputeEndpoint{}} - if err := c.Resume(t.Context(), (*vmmanager.UtilityVM)(nil), (*guestmanager.Guest)(nil)); err == nil { - t.Errorf("Resume() in state %s = nil; want error", st) + err := c.Resume(t.Context(), (*vmmanager.UtilityVM)(nil), (*guestmanager.Guest)(nil)) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Errorf("Resume() in state %s = %v; want ErrFailedPrecondition", st, err) } }) } } + +// TestResume_IdempotentWhenConfigured verifies that resuming an already-resumed +// network is a no-op, so a retry after a completed resume is safe. +func TestResume_IdempotentWhenConfigured(t *testing.T) { + c := &Controller{netState: StateConfigured, vmEndpoints: map[string]*hcn.HostComputeEndpoint{}} + if err := c.Resume(t.Context(), (*vmmanager.UtilityVM)(nil), (*guestmanager.Guest)(nil)); err != nil { + t.Fatalf("Resume() = %v; want nil", err) + } + if c.netState != StateConfigured { + t.Errorf("netState = %s; want unchanged Configured", c.netState) + } +} + +// TestResetAfterMigration_IdempotentWhenReset verifies that a second reset, once +// migratedNamespaceID has been cleared, is a no-op rather than re-binding or +// failing on the empty namespace ID. +func TestResetAfterMigration_IdempotentWhenReset(t *testing.T) { + c := &Controller{migratedNamespaceID: "", vmEndpoints: map[string]*hcn.HostComputeEndpoint{}} + if err := c.ResetAfterMigration(t.Context()); err != nil { + t.Fatalf("ResetAfterMigration() = %v; want nil", err) + } +} diff --git a/internal/controller/pod/save_lcow.go b/internal/controller/pod/save_lcow.go index c2473317a3..becdba82d8 100644 --- a/internal/controller/pod/save_lcow.go +++ b/internal/controller/pod/save_lcow.go @@ -14,6 +14,7 @@ import ( "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/errdefs" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" @@ -88,7 +89,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { } if env.GetTypeUrl() != podsave.TypeURL { - return nil, fmt.Errorf("unsupported pod saved-state type %q", env.GetTypeUrl()) + return nil, fmt.Errorf("unsupported pod saved-state type %q: %w", env.GetTypeUrl(), errdefs.ErrInvalidArgument) } // Decode and reject any payload this build cannot interpret. @@ -98,7 +99,7 @@ func Import(ctx context.Context, env *anypb.Any) (*Controller, error) { } if v := state.GetSchemaVersion(); v != podsave.SchemaVersion { - return nil, fmt.Errorf("unsupported pod saved-state schema version %d (want %d)", v, podsave.SchemaVersion) + return nil, fmt.Errorf("unsupported pod saved-state schema version %d (want %d): %w", v, podsave.SchemaVersion, errdefs.ErrInvalidArgument) } // Restore the network controller in its own inert state. @@ -142,14 +143,13 @@ func (c *Controller) Resume(ctx context.Context, vm vmController, events chan in c.mu.Lock() defer c.mu.Unlock() - // Resume only applies to a pod that is mid-migration. + // Idempotent: a pod fully resumed by a prior attempt is a no-op on retry. if !c.isMigrating { - return fmt.Errorf("pod %q is not migrating; nothing to resume", c.podID) + return nil } - // Bind the live VM and re-wire the network to it. + // Bind the live VM; the network is re-wired to it below. c.vm = vm - c.isMigrating = false if err := c.network.Resume(ctx, vm.VM(), vm.Guest()); err != nil { return fmt.Errorf("resume network for migration: %w", err) @@ -185,6 +185,10 @@ func (c *Controller) Resume(ctx context.Context, vm vmController, events chan in } } + // Every resource is live again; clear the migrating flag last so any failure + // above leaves the pod retriable. + c.isMigrating = false + log.G(ctx).WithField(logfields.DestinationPodID, c.podID).Debug("resumed pod") return nil @@ -209,12 +213,12 @@ func (c *Controller) Patch( // A destination request with a container ID is required before we mutate // any state. if request == nil || request.ID == "" { - return fmt.Errorf("invalid create task request: %+v", request) + return fmt.Errorf("invalid create task request: %+v: %w", request, errdefs.ErrInvalidArgument) } // Patch only applies to a pod imported for migration. if !c.isMigrating { - return fmt.Errorf("pod %q is not migrating; cannot patch", c.podID) + return fmt.Errorf("pod %q is not migrating; cannot patch: %w", c.podID, errdefs.ErrFailedPrecondition) } log.G(ctx).WithFields(logrus.Fields{ @@ -267,20 +271,3 @@ func (c *Controller) Patch( log.G(ctx).WithField(logfields.DestinationPodID, c.podID).Debug("patched migrated pod") return nil } - -// AbortMigrated marks every container in the pod as stopped and reports their -// exit, so that containerd no longer sees them as UNKNOWN and can delete them. -// This is primarily used on destination during finalize stop. -func (c *Controller) AbortMigrated(ctx context.Context, events chan interface{}) { - c.mu.Lock() - defer c.mu.Unlock() - - // Nothing to abort unless the pod is mid-migration. - if !c.isMigrating { - return - } - - for _, ctr := range c.containers { - ctr.AbortMigrated(ctx, events) - } -} diff --git a/internal/controller/pod/save_lcow_test.go b/internal/controller/pod/save_lcow_test.go index 17c6e8d30c..f0932845de 100644 --- a/internal/controller/pod/save_lcow_test.go +++ b/internal/controller/pod/save_lcow_test.go @@ -3,9 +3,11 @@ package pod import ( + "errors" "testing" "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/errdefs" "github.com/opencontainers/runtime-spec/specs-go" "go.uber.org/mock/gomock" "google.golang.org/protobuf/proto" @@ -19,8 +21,8 @@ import ( procsave "github.com/Microsoft/hcsshim/internal/controller/process/save" ) -// migratingContainer restores a real container into StateMigrating with a -// single init process whose IO paths are empty, so Patch needs no live guest. +// migratingContainer restores a real container as imported on the destination +// with a single init process whose IO paths are empty, so Patch needs no live guest. func migratingContainer(t *testing.T, id string) *linuxcontainer.Controller { t.Helper() proc := mustAny(t, procsave.TypeURL, &procsave.Payload{SchemaVersion: procsave.SchemaVersion}) @@ -158,6 +160,7 @@ func TestImport(t *testing.T) { name string env func(t *testing.T) *anypb.Any wantErr bool + wantIs error check func(t *testing.T, c *Controller) }{ { @@ -169,6 +172,7 @@ func TestImport(t *testing.T) { name: "wrong type url", env: func(t *testing.T) *anypb.Any { t.Helper(); return &anypb.Any{TypeUrl: "type.microsoft.com/bogus"} }, wantErr: true, + wantIs: errdefs.ErrInvalidArgument, }, { name: "corrupt payload", @@ -185,6 +189,7 @@ func TestImport(t *testing.T) { return mustAny(t, podsave.TypeURL, &podsave.Payload{SchemaVersion: podsave.SchemaVersion + 1, PodID: testPodID}) }, wantErr: true, + wantIs: errdefs.ErrInvalidArgument, }, { name: "network import fails", @@ -264,6 +269,9 @@ func TestImport(t *testing.T) { if err == nil { t.Fatal("expected error, got nil") } + if tt.wantIs != nil && !errors.Is(err, tt.wantIs) { + t.Fatalf("error = %v; want wrapped %v", err, tt.wantIs) + } return } if err != nil { @@ -338,20 +346,22 @@ func TestResume(t *testing.T) { if (err != nil) != tt.wantErr { t.Fatalf("Resume() error = %v, wantErr %v", err, tt.wantErr) } - // Resume clears the migrating guard so normal ops are allowed again. - if c.isMigrating { - t.Error("expected isMigrating to be cleared after Resume") + // isMigrating is cleared only on full success, so a failed resume + // stays retriable while a successful one flips the guard off. + if c.isMigrating != tt.wantErr { + t.Errorf("isMigrating = %v; want %v (cleared only on success)", c.isMigrating, tt.wantErr) } }) } } -// TestResume_RejectsWhenNotMigrating verifies Resume fails on a pod that is not -// mid-migration, before it touches the VM or network. -func TestResume_RejectsWhenNotMigrating(t *testing.T) { +// TestResume_NoOpWhenNotMigrating verifies Resume on a pod that is not +// mid-migration is a no-op (rather than an error), so a retry after a completed +// resume is safe. +func TestResume_NoOpWhenNotMigrating(t *testing.T) { c := &Controller{podID: testPodID, containers: map[string]*linuxcontainer.Controller{}} - if err := c.Resume(t.Context(), nil, nil, false); err == nil { - t.Error("Resume() on a non-migrating pod = nil; want error") + if err := c.Resume(t.Context(), nil, nil, false); err != nil { + t.Errorf("Resume() on a non-migrating pod = %v; want nil (no-op)", err) } } @@ -367,6 +377,7 @@ func TestPatch(t *testing.T) { request *task.CreateTaskRequest spec specs.Spec wantErr bool + wantIs error check func(t *testing.T, c *Controller) }{ { @@ -377,6 +388,7 @@ func TestPatch(t *testing.T) { }, request: nil, wantErr: true, + wantIs: errdefs.ErrInvalidArgument, }, { name: "empty request id", @@ -386,6 +398,7 @@ func TestPatch(t *testing.T) { }, request: &task.CreateTaskRequest{ID: ""}, wantErr: true, + wantIs: errdefs.ErrInvalidArgument, }, { name: "source container not found", @@ -487,6 +500,7 @@ func TestPatch(t *testing.T) { sourceID: "src", request: &task.CreateTaskRequest{ID: "dst"}, wantErr: true, + wantIs: errdefs.ErrFailedPrecondition, }, } for _, tt := range tests { @@ -496,6 +510,9 @@ func TestPatch(t *testing.T) { if (err != nil) != tt.wantErr { t.Fatalf("Patch() error = %v, wantErr %v", err, tt.wantErr) } + if tt.wantIs != nil && !errors.Is(err, tt.wantIs) { + t.Fatalf("Patch() error = %v; want wrapped %v", err, tt.wantIs) + } if err == nil && tt.check != nil { tt.check(t, c) } diff --git a/internal/controller/process/save.go b/internal/controller/process/save.go index b9c06f01f0..9b147ccc95 100644 --- a/internal/controller/process/save.go +++ b/internal/controller/process/save.go @@ -21,16 +21,16 @@ import ( "google.golang.org/protobuf/types/known/durationpb" ) -// Save captures a running process as a portable payload that a destination -// shim can later restore. It is only valid while the process is running, and on -// success freezes the source until it is resumed or terminated. +// Save captures the process as a portable payload that a destination shim can +// later restore. It is valid while the process is running or already frozen by a +// prior save, and on success freezes the source until it is resumed or terminated. func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { c.mu.Lock() defer c.mu.Unlock() // Only a live process has the IO ports and wait id needed to restore it. - if c.state != StateRunning { - return nil, fmt.Errorf("process %q in container %q in state %s; want %s", c.execID, c.containerID, c.state, StateRunning) + if c.state != StateRunning && c.state != StateSourceMigrating { + return nil, fmt.Errorf("process %q in container %q in state %s; want %s: %w", c.execID, c.containerID, c.state, StateRunning, errdefs.ErrFailedPrecondition) } // Capture the host-independent identity of the process. @@ -86,7 +86,7 @@ func Import(ctx context.Context, env *anypb.Any, containerID string) (*Controlle // Refuse envelopes that were not produced by this save format. if env.GetTypeUrl() != procsave.TypeURL { - return nil, fmt.Errorf("unsupported process saved-state type %q", env.GetTypeUrl()) + return nil, fmt.Errorf("unsupported process saved-state type %q: %w", env.GetTypeUrl(), errdefs.ErrInvalidArgument) } state := &procsave.Payload{} @@ -96,7 +96,7 @@ func Import(ctx context.Context, env *anypb.Any, containerID string) (*Controlle // Reject payloads written by an incompatible shim version. if v := state.GetSchemaVersion(); v != procsave.SchemaVersion { - return nil, fmt.Errorf("unsupported process saved-state schema version %d (want %d)", v, procsave.SchemaVersion) + return nil, fmt.Errorf("unsupported process saved-state schema version %d (want %d): %w", v, procsave.SchemaVersion, errdefs.ErrInvalidArgument) } // Rebuild the controller in the destination-migrating state, holding the @@ -187,6 +187,11 @@ func (c *Controller) Resume(ctx context.Context, gcsContainer *gcs.Container, ev c.mu.Lock() defer c.mu.Unlock() + // Idempotent: a process resumed by a prior attempt is a no-op on retry. + if c.state == StateRunning { + return nil + } + // Source rollback: the live process and IO are intact, so just lift the // freeze that Save applied. if c.state == StateSourceMigrating { @@ -224,18 +229,3 @@ func (c *Controller) Resume(ctx context.Context, gcsContainer *gcs.Container, ev log.G(ctx).WithField(logfields.ProcessID, c.processID).Debug("resumed migrated process on destination") return nil } - -// AbortMigrated terminates an imported, not-yet-resumed process so it can be -// deleted. It is a no-op once the process has been resumed or otherwise left -// the migrating state. -func (c *Controller) AbortMigrated(ctx context.Context) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.state != StateDestinationMigrating { - return - } - - log.G(ctx).WithField(logfields.ProcessID, c.processID).Debug("aborting migrated process") - c.abortInternal(ctx, 137) -} diff --git a/internal/controller/process/save_test.go b/internal/controller/process/save_test.go index 6eef63c8ad..ecc3f8b962 100644 --- a/internal/controller/process/save_test.go +++ b/internal/controller/process/save_test.go @@ -28,10 +28,10 @@ const ( testWaitCallID = int64(99) ) -// TestSave_WrongState verifies that only a running process can be saved. +// TestSave_WrongState verifies Save is refused for non-snapshottable process states. func TestSave_WrongState(t *testing.T) { t.Parallel() - invalidStates := []State{StateNotCreated, StateCreated, StateTerminated, StateDestinationMigrating, StateSourceMigrating} + invalidStates := []State{StateNotCreated, StateCreated, StateTerminated, StateDestinationMigrating} for _, state := range invalidStates { t.Run(state.String(), func(t *testing.T) { @@ -39,8 +39,9 @@ func TestSave_WrongState(t *testing.T) { _, _, _, controller := newSetup(t) controller.state = state - if _, err := controller.Save(t.Context()); err == nil { - t.Errorf("Save() = nil; want error for state %s", state) + _, err := controller.Save(t.Context()) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Errorf("Save() = %v; want ErrFailedPrecondition for state %s", err, state) } }) } @@ -119,6 +120,21 @@ func TestSave_Succeeds(t *testing.T) { } } +// TestSave_FromSourceMigrating verifies a re-save of an already-migrating source +// process still succeeds, so a retried export is safe. +func TestSave_FromSourceMigrating(t *testing.T) { + t.Parallel() + _, _, _, controller := newSetup(t) + controller.state = StateSourceMigrating + + if _, err := controller.Save(t.Context()); err != nil { + t.Fatalf("Save() from SourceMigrating = %v; want nil", err) + } + if controller.state != StateSourceMigrating { + t.Errorf("state = %s; want unchanged StateSourceMigrating", controller.state) + } +} + // TestImport_InvalidEnvelope verifies that Import rejects malformed or // incompatible envelopes. func TestImport_InvalidEnvelope(t *testing.T) { @@ -133,18 +149,24 @@ func TestImport_InvalidEnvelope(t *testing.T) { tests := []struct { name string env *anypb.Any + // wantIs, when set, is the sentinel the rejection must wrap. + wantIs error }{ {name: "nil envelope", env: nil}, - {name: "wrong type url", env: &anypb.Any{TypeUrl: "type.microsoft.com/other", Value: nil}}, + {name: "wrong type url", env: &anypb.Any{TypeUrl: "type.microsoft.com/other", Value: nil}, wantIs: errdefs.ErrInvalidArgument}, {name: "undecodable value", env: &anypb.Any{TypeUrl: procsave.TypeURL, Value: []byte{0x08, 0xff}}}, - {name: "schema version mismatch", env: &anypb.Any{TypeUrl: procsave.TypeURL, Value: badVersion}}, + {name: "schema version mismatch", env: &anypb.Any{TypeUrl: procsave.TypeURL, Value: badVersion}, wantIs: errdefs.ErrInvalidArgument}, } for _, testCase := range tests { t.Run(testCase.name, func(t *testing.T) { t.Parallel() - if _, err := Import(t.Context(), testCase.env, testContainerID); err == nil { - t.Errorf("Import() = nil; want error") + _, err := Import(t.Context(), testCase.env, testContainerID) + if err == nil { + t.Fatal("Import() = nil; want error") + } + if testCase.wantIs != nil && !errors.Is(err, testCase.wantIs) { + t.Errorf("Import() = %v; want %v", err, testCase.wantIs) } }) } @@ -292,11 +314,11 @@ func TestPatch_Succeeds(t *testing.T) { } } -// TestResume_WrongState verifies that Resume only operates on a migrating -// process and rejects other states before touching the host. +// TestResume_WrongState verifies Resume rejects states that are neither +// mid-migration nor already running, before touching the host. func TestResume_WrongState(t *testing.T) { t.Parallel() - invalidStates := []State{StateNotCreated, StateCreated, StateRunning, StateTerminated} + invalidStates := []State{StateNotCreated, StateCreated, StateTerminated} for _, state := range invalidStates { t.Run(state.String(), func(t *testing.T) { @@ -328,46 +350,18 @@ func TestResume_SourceRollback(t *testing.T) { } } -// TestAbortMigrated_NoOp verifies that AbortMigrated leaves a non-migrating -// process untouched. -func TestAbortMigrated_NoOp(t *testing.T) { - t.Parallel() - otherStates := []State{StateNotCreated, StateCreated, StateRunning, StateTerminated, StateSourceMigrating} - - for _, state := range otherStates { - t.Run(state.String(), func(t *testing.T) { - t.Parallel() - _, _, _, controller := newSetup(t) - controller.state = state - - controller.AbortMigrated(t.Context()) - if controller.state != state { - t.Errorf("state = %s; want unchanged %s", controller.state, state) - } - }) - } -} - -// TestAbortMigrated_Succeeds verifies that AbortMigrated terminates a migrating -// process, recording exit code 137 and unblocking waiters. -func TestAbortMigrated_Succeeds(t *testing.T) { +// TestResume_IdempotentWhenRunning verifies that resuming an already-resumed +// process is a no-op, so a retry after a completed resume is safe. +func TestResume_IdempotentWhenRunning(t *testing.T) { t.Parallel() _, _, _, controller := newSetup(t) - controller.state = StateDestinationMigrating - // upstreamIO intentionally nil — abort must tolerate it. + controller.state = StateRunning - controller.AbortMigrated(t.Context()) - - if controller.state != StateTerminated { - t.Errorf("state = %s; want StateTerminated", controller.state) - } - if controller.exitCode != 137 { - t.Errorf("exitCode = %d; want 137", controller.exitCode) + if err := controller.Resume(t.Context(), nil, nil); err != nil { + t.Fatalf("Resume() = %v; want nil", err) } - select { - case <-controller.exitedCh: - default: - t.Error("exitedCh should be closed after AbortMigrated") + if controller.state != StateRunning { + t.Errorf("state = %s; want unchanged StateRunning", controller.state) } } diff --git a/internal/controller/vm/save_lcow.go b/internal/controller/vm/save_lcow.go index 29f330c74a..ef67245a99 100644 --- a/internal/controller/vm/save_lcow.go +++ b/internal/controller/vm/save_lcow.go @@ -17,6 +17,8 @@ import ( "github.com/Microsoft/hcsshim/internal/logfields" "github.com/Microsoft/hcsshim/internal/wclayer" + "github.com/containerd/errdefs" + "github.com/Microsoft/go-winio" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -37,7 +39,7 @@ func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { // Save is only valid once the source has begun migrating. if c.vmState != StateSourceMigrationInitialized { - return nil, fmt.Errorf("cannot save VM: VM is in state %s", c.vmState) + return nil, fmt.Errorf("cannot save VM: VM is in state %s: %w", c.vmState, errdefs.ErrFailedPrecondition) } // Seed the payload with the VM identity, creation options, and compat blob. @@ -106,24 +108,24 @@ func (c *Controller) Save(ctx context.Context) (*anypb.Any, error) { // Import rebuilds a controller's static state from a snapshot produced by // Save. The controller comes back inert in the migrating state and performs no // live work until Resume supplies the running VM. -func (c *Controller) Import(ctx context.Context, env *anypb.Any) error { +func (c *Controller) Import(ctx context.Context, env *anypb.Any) (err error) { if env == nil { - return fmt.Errorf("vm saved-state envelope is nil") + return fmt.Errorf("vm saved-state envelope is nil: %w", errdefs.ErrInvalidArgument) } // Reject envelopes that did not originate from a compatible Save. if env.GetTypeUrl() != vmsave.TypeURL { - return fmt.Errorf("unsupported vm saved-state type %q", env.GetTypeUrl()) + return fmt.Errorf("unsupported vm saved-state type %q: %w", env.GetTypeUrl(), errdefs.ErrInvalidArgument) } state := &vmsave.Payload{} - if err := proto.Unmarshal(env.GetValue(), state); err != nil { + if err = proto.Unmarshal(env.GetValue(), state); err != nil { return fmt.Errorf("unmarshal vm saved state: %w", err) } // Reject payloads written by an incompatible shim version. if v := state.GetSchemaVersion(); v != vmsave.SchemaVersion { - return fmt.Errorf("unsupported vm saved-state schema version %d (want %d)", v, vmsave.SchemaVersion) + return fmt.Errorf("unsupported vm saved-state schema version %d (want %d): %w", v, vmsave.SchemaVersion, errdefs.ErrInvalidArgument) } c.mu.Lock() @@ -131,41 +133,41 @@ func (c *Controller) Import(ctx context.Context, env *anypb.Any) error { // We can import a new VM only on a freshly created controller. if c.vmState != StateNotCreated { - return fmt.Errorf("unsupported vm state during Import %q", c.vmState) + return fmt.Errorf("unsupported vm state during Import %q: %w", c.vmState, errdefs.ErrFailedPrecondition) } - // Restore the VM identity, allocator floors, and compat blob, then mark - // the controller migrating so only migration APIs are permitted. - c.vmID = state.GetVmID() - c.sandboxOptions = sandboxOptionsFromProto(state.GetSandboxOptions()) - if c.sandboxOptions != nil { - c.isPhysicallyBacked = c.sandboxOptions.FullyPhysicallyBacked - } - c.nextGuestPort = state.GetGcsNextPort() - c.nextBridgeID = state.GetBridgeNextID() - c.compatInfo = state.GetCompatInfo() - c.vmState = StateDestinationMigrationImported - // Decode the HCS document so [Controller.CreateVM] (called next on the // destination with MigrationOptions populated) can reuse it verbatim. + var doc = &hcsschema.ComputeSystem{} if raw := state.GetHcsDocument(); len(raw) > 0 { - doc := &hcsschema.ComputeSystem{} if err := json.Unmarshal(raw, doc); err != nil { return fmt.Errorf("unmarshal hcs document: %w", err) } - - c.hcsDocument = doc } // Import the SCSI sub-controller. - if env := state.GetScsi(); env != nil { - s, err := scsi.Import(ctx, env) + var scsiCtrl *scsi.Controller + if scsiEnv := state.GetScsi(); scsiEnv != nil { + scsiCtrl, err = scsi.Import(ctx, scsiEnv) if err != nil { return fmt.Errorf("import scsi controller: %w", err) } - c.scsiController = s } + // Restore the VM identity, allocator floors, and compat blob, then mark + // the controller migrating so only migration APIs are permitted. + c.vmID = state.GetVmID() + c.sandboxOptions = sandboxOptionsFromProto(state.GetSandboxOptions()) + if c.sandboxOptions != nil { + c.isPhysicallyBacked = c.sandboxOptions.FullyPhysicallyBacked + } + c.hcsDocument = doc + c.scsiController = scsiCtrl + c.nextGuestPort = state.GetGcsNextPort() + c.nextBridgeID = state.GetBridgeNextID() + c.compatInfo = state.GetCompatInfo() + c.vmState = StateDestinationMigrationImported + log.G(ctx).Debug("imported VM migration state") return nil } @@ -178,11 +180,11 @@ func (c *Controller) Patch(ctx context.Context) error { defer c.mu.Unlock() if c.vmState != StateDestinationMigrationCreated { - return fmt.Errorf("cannot patch VM: VM is in state %s", c.vmState) + return fmt.Errorf("cannot patch VM: VM is in state %s: %w", c.vmState, errdefs.ErrFailedPrecondition) } if c.scsiController == nil { - return fmt.Errorf("cannot patch VM: SCSI controller is nil") + return fmt.Errorf("cannot patch VM: SCSI controller is nil: %w", errdefs.ErrInvalidArgument) } // Grant access only for disk types whose host paths the VM must reach. @@ -207,9 +209,14 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error { c.mu.Lock() defer c.mu.Unlock() + // An already running VM will be a no-op on resume. + if c.vmState == StateRunning { + return nil + } + // Resume returns either migration side to the running state. if c.vmState != StateMigrationFinalized { - return fmt.Errorf("cannot resume from migration: VM is in state %s", c.vmState) + return fmt.Errorf("cannot resume from migration: VM is in state %s: %w", c.vmState, errdefs.ErrFailedPrecondition) } switch { diff --git a/internal/controller/vm/vm.go b/internal/controller/vm/vm.go index cd3f921ab4..aff1db6724 100644 --- a/internal/controller/vm/vm.go +++ b/internal/controller/vm/vm.go @@ -605,6 +605,10 @@ func (c *Controller) TerminateVM(ctx context.Context) (err error) { // Best effort attempt to clean up the open vmmem handle. _ = windows.Close(c.vmmemProcess) + // Explicitly set migrating to false so that we release + // the bridge prior to removing the VM. + c.guest.SetMigrating(false) + // Skip HCS Terminate for a never-started VM (cold-created, or a destination // migration VM created/patched but not yet started). The HCS document sets // ShouldTerminateOnLastHandleClosed, so uvm.Close below is sufficient. diff --git a/internal/controller/vm/vm_migration.go b/internal/controller/vm/vm_migration.go index 267669475a..9eda94358a 100644 --- a/internal/controller/vm/vm_migration.go +++ b/internal/controller/vm/vm_migration.go @@ -186,8 +186,14 @@ func (c *Controller) FinalizeLiveMigration(ctx context.Context, options *hcssche c.mu.Lock() defer c.mu.Unlock() - // Finalize is only valid once the memory transfer has completed. - if c.vmState != StateMigrationTransferCompleted { + // Finalize is valid only while the VM is still migrating. The migration + // controller reconciles the already-finalized and terminated states against + // the requested operation, so they are rejected here. + switch c.vmState { + case StateSourceMigrationInitialized, StateSourceMigrationStarted, + StateDestinationMigrationCreated, StateDestinationMigrationPatched, + StateDestinationMigrationStarted, StateMigrationTransferCompleted: + default: return fmt.Errorf("cannot finalize live migration: VM is in state %s", c.vmState) } @@ -202,6 +208,8 @@ func (c *Controller) FinalizeLiveMigration(ctx context.Context, options *hcssche // scratch-layer unmap on delete is no longer rejected. if options.Origin == hcsschema.MigrationOriginSource && c.scsiController != nil { if err := c.scsiController.Resume(ctx, c.uvm, c.guest); err != nil { + // An error here is unlikely: scsi Resume only fails when the controller + // is not in the migrating state, which should not be the case mid-finalize. return fmt.Errorf("resume scsi controller: %w", err) } } @@ -221,3 +229,22 @@ func (c *Controller) FinalizeLiveMigration(ctx context.Context, options *hcssche log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("finalized live migration") return nil } + +// CancelLiveMigration aborts an in-flight live migration. It intentionally takes +// no lock: it must be able to interrupt an in-flight StartLiveMigrationTransfer, +// which holds mu for the entire (synchronous) transfer. Reading uvm without mu is +// safe because uvm is written once in CreateVM and the migration controller +// serializes Cancel behind its own lock, so that write is already published here. +// A nil uvm means the VM was never created. +func (c *Controller) CancelLiveMigration(ctx context.Context) error { + if c.uvm == nil { + return fmt.Errorf("cannot cancel live migration: VM not created") + } + + if err := c.uvm.CancelLiveMigration(ctx); err != nil { + return fmt.Errorf("failed to cancel live migration: %w", err) + } + + log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("cancelled live migration") + return nil +} diff --git a/internal/hcs/v2/migration.go b/internal/hcs/v2/migration.go index df9bb62715..ec4ad417a6 100644 --- a/internal/hcs/v2/migration.go +++ b/internal/hcs/v2/migration.go @@ -241,6 +241,30 @@ func (computeSystem *System) FinalizeLiveMigration(ctx context.Context, opts *hc return nil } +// CancelLiveMigration is currently a stub: it verifies the compute system is +// still open and returns success without contacting HCS or interrupting any +// migration (pending the compute-core cancel API). +// +// It deliberately takes no handleLock so that, once implemented, it can interrupt +// an in-flight transfer that holds it. +func (computeSystem *System) CancelLiveMigration(ctx context.Context) (err error) { + operation := "hcs::System::CancelLiveMigration" + + _, span := ot.StartSpan(ctx, operation) + defer span.End() + defer func() { ot.SetSpanStatus(span, err) }() + span.SetAttributes(attribute.String("cid", computeSystem.id)) + + if computeSystem.handle == 0 { + return makeSystemError(computeSystem, operation, ErrAlreadyClosed) + } + + // TODO: wire this to the HcsCancelLiveMigration compute-core API once it is + // available. Until then this is a stub that reports success without contacting HCS. + + return nil +} + // MigrationNotifications returns a read-only channel of live migration events // for this System. The channel exists for the System's lifetime and is safe // to subscribe to before any migration call, so callers do not miss early diff --git a/internal/vm/vmmanager/migration.go b/internal/vm/vmmanager/migration.go index 375d50b071..e63f926b3d 100644 --- a/internal/vm/vmmanager/migration.go +++ b/internal/vm/vmmanager/migration.go @@ -45,6 +45,14 @@ func (uvm *UtilityVM) StartLiveMigrationTransfer(ctx context.Context, options *h return nil } +// CancelLiveMigration aborts an in-flight live migration on the utility VM. +func (uvm *UtilityVM) CancelLiveMigration(ctx context.Context) error { + if err := uvm.cs.CancelLiveMigration(ctx); err != nil { + return fmt.Errorf("failed to cancel live migration: %w", err) + } + return nil +} + // FinalizeLiveMigration completes the live migration workflow. If resume is true // the utility VM is resumed; otherwise it is stopped. func (uvm *UtilityVM) FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error {