implement resource identity upgrade on refresh

internal/builtin/providers/terraform/provider.go

@@ -155,6 +155,10 @@ func (p *Provider) UpgradeResourceState(req providers.UpgradeResourceStateReques
 	return upgradeDataStoreResourceState(req)
 }
 
+func (p *Provider) UpgradeResourceIdentity(req providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse {
+	return upgradeDataStoreResourceIdentity(req)
+}
+
 // ReadResource refreshes a resource and returns its current state.
 func (p *Provider) ReadResource(req providers.ReadResourceRequest) providers.ReadResourceResponse {
 	return readDataStoreResourceState(req)

internal/builtin/providers/terraform/resource_data.go

@@ -55,6 +55,11 @@ func upgradeDataStoreResourceState(req providers.UpgradeResourceStateRequest) (r
 	return resp
 }
 
+func upgradeDataStoreResourceIdentity(req providers.UpgradeResourceIdentityRequest) (resp providers.UpgradeResourceIdentityResponse) {
+	// TODO: Implement this
+	panic("not implemented")
+}
+
 func readDataStoreResourceState(req providers.ReadResourceRequest) (resp providers.ReadResourceResponse) {
 	resp.NewState = req.PriorState
 	return resp

internal/plugin/grpc_provider.go

@@ -382,6 +382,50 @@ func (p *GRPCProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
 	return resp
 }
 
+func (p *GRPCProvider) UpgradeResourceIdentity(r providers.UpgradeResourceIdentityRequest) (resp providers.UpgradeResourceIdentityResponse) {
+	logger.Trace("GRPCProvider: UpgradeResourceIdentity")
+
+	schema := p.GetResourceIdentitySchemas()
+	if schema.Diagnostics.HasErrors() {
+		resp.Diagnostics = schema.Diagnostics
+		return resp
+	}
+
+	resSchema, ok := schema.IdentityTypes[r.TypeName]
+	if !ok {
+		resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("unknown resource type %q", r.TypeName))
+		return resp
+	}
+
+	protoReq := &proto.UpgradeResourceIdentity_Request{
+		TypeName:    r.TypeName,
+		Version:     int64(r.Version),
+		RawIdentity: r.RawIdentityJSON,
+	}
+
+	protoResp, err := p.client.UpgradeResourceIdentity(p.ctx, protoReq)
+	if err != nil {
+		resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err))
+		return resp
+	}
+	resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics))
+
+	ty := resSchema.Attributes.ImpliedType()
+	resp.UpgradedIdentity = cty.NullVal(ty)
+	if protoResp.UpgradedIdentity == nil {
+		return resp
+	}
+
+	identity, err := decodeDynamicValue(protoResp.UpgradedIdentity.IdentityData, ty)
+	if err != nil {
+		resp.Diagnostics = resp.Diagnostics.Append(err)
+		return resp
+	}
+	resp.UpgradedIdentity = identity
+
+	return resp
+}
+
 func (p *GRPCProvider) ConfigureProvider(r providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
 	logger.Trace("GRPCProvider: ConfigureProvider")
 

internal/plugin6/grpc_provider.go

@@ -375,6 +375,50 @@ func (p *GRPCProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
 	return resp
 }
 
+func (p *GRPCProvider) UpgradeResourceIdentity(r providers.UpgradeResourceIdentityRequest) (resp providers.UpgradeResourceIdentityResponse) {
+	logger.Trace("GRPCProvider.v6: UpgradeResourceIdentity")
+
+	schema := p.GetResourceIdentitySchemas()
+	if schema.Diagnostics.HasErrors() {
+		resp.Diagnostics = schema.Diagnostics
+		return resp
+	}
+
+	resSchema, ok := schema.IdentityTypes[r.TypeName]
+	if !ok {
+		resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("unknown resource type %q", r.TypeName))
+		return resp
+	}
+
+	protoReq := &proto6.UpgradeResourceIdentity_Request{
+		TypeName:    r.TypeName,
+		Version:     int64(r.Version),
+		RawIdentity: r.RawIdentityJSON,
+	}
+
+	protoResp, err := p.client.UpgradeResourceIdentity(p.ctx, protoReq)
+	if err != nil {
+		resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err))
+		return resp
+	}
+	resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics))
+
+	ty := resSchema.Attributes.ImpliedType()
+	resp.UpgradedIdentity = cty.NullVal(ty)
+	if protoResp.UpgradedIdentity == nil {
+		return resp
+	}
+
+	identity, err := decodeDynamicValue(protoResp.UpgradedIdentity.IdentityData, ty)
+	if err != nil {
+		resp.Diagnostics = resp.Diagnostics.Append(err)
+		return resp
+	}
+	resp.UpgradedIdentity = identity
+
+	return resp
+}
+
 func (p *GRPCProvider) ConfigureProvider(r providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
 	logger.Trace("GRPCProvider.v6: ConfigureProvider")
 

internal/provider-simple-v6/provider.go

@@ -105,6 +105,15 @@ func (p simple) UpgradeResourceState(req providers.UpgradeResourceStateRequest)
 	return resp
 }
 
+func (p simple) UpgradeResourceIdentity(req providers.UpgradeResourceIdentityRequest) (resp providers.UpgradeResourceIdentityResponse) {
+	schema := p.GetResourceIdentitySchemas().IdentityTypes[req.TypeName].Attributes
+	ty := schema.ImpliedType()
+	val, err := ctyjson.Unmarshal(req.RawIdentityJSON, ty)
+	resp.Diagnostics = resp.Diagnostics.Append(err)
+	resp.UpgradedIdentity = val
+	return resp
+}
+
 func (s simple) ConfigureProvider(providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
 	return resp
 }

internal/provider-simple/provider.go

@@ -82,6 +82,15 @@ func (p simple) UpgradeResourceState(req providers.UpgradeResourceStateRequest)
 	return resp
 }
 
+func (p simple) UpgradeResourceIdentity(req providers.UpgradeResourceIdentityRequest) (resp providers.UpgradeResourceIdentityResponse) {
+	schema := p.GetResourceIdentitySchemas().IdentityTypes[req.TypeName].Attributes
+	ty := schema.ImpliedType()
+	val, err := ctyjson.Unmarshal(req.RawIdentityJSON, ty)
+	resp.Diagnostics = resp.Diagnostics.Append(err)
+	resp.UpgradedIdentity = val
+	return resp
+}
+
 func (s simple) ConfigureProvider(providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
 	return resp
 }

internal/providers/mock.go

@@ -136,6 +136,40 @@ func (m *Mock) UpgradeResourceState(request UpgradeResourceStateRequest) (respon
 	return response
 }
 
+func (m *Mock) UpgradeResourceIdentity(request UpgradeResourceIdentityRequest) (response UpgradeResourceIdentityResponse) {
+	// We can't do this from a mocked provider, so we just return whatever identity
+	// is in the request back unchanged.
+
+	schema := m.GetResourceIdentitySchemas()
+	response.Diagnostics = response.Diagnostics.Append(schema.Diagnostics)
+	if schema.Diagnostics.HasErrors() {
+		// We couldn't retrieve the schema for some reason, so the mock
+		// provider can't really function.
+		return response
+	}
+
+	resource, exists := schema.IdentityTypes[request.TypeName]
+	if !exists {
+		// This means something has gone wrong much earlier, we should have
+		// failed a validation somewhere if a resource type doesn't exist.
+		panic(fmt.Errorf("failed to retrieve identity schema for resource %s", request.TypeName))
+	}
+
+	schemaType := resource.Attributes.ImpliedType()
+	value, err := ctyjson.Unmarshal(request.RawIdentityJSON, schemaType)
+
+	if err != nil {
+		// Generally, we shouldn't get an error here. The mocked providers are
+		// only used in tests, and we can't use different versions of providers
+		// within/between tests so the types should always match up. As such,
+		// we're not gonna return a super detailed error here.
+		response.Diagnostics = response.Diagnostics.Append(err)
+		return response
+	}
+	response.UpgradedIdentity = value
+	return response
+}
+
 func (m *Mock) ConfigureProvider(request ConfigureProviderRequest) (response ConfigureProviderResponse) {
 	// Do nothing here, we don't have anything to configure within the mocked
 	// providers. We don't want to call the original providers from here as

internal/providers/provider.go

@@ -43,6 +43,12 @@ type Interface interface {
 	// result is used for any further processing.
 	UpgradeResourceState(UpgradeResourceStateRequest) UpgradeResourceStateResponse
 
+	// UpgradeResourceIdentity is called when the state loader encounters an
+	// instance identity whose schema version is less than the one reported by
+	// the currently-used version of the corresponding provider, and the upgraded
+	// result is used for any further processing.
+	UpgradeResourceIdentity(UpgradeResourceIdentityRequest) UpgradeResourceIdentityResponse
+
 	// Configure configures and initialized the provider.
 	ConfigureProvider(ConfigureProviderRequest) ConfigureProviderResponse
 
@@ -275,6 +281,26 @@ type UpgradeResourceStateResponse struct {
 	Diagnostics tfdiags.Diagnostics
 }
 
+type UpgradeResourceIdentityRequest struct {
+	// TypeName is the name of the resource type being upgraded
+	TypeName string
+
+	// Version is version of the schema that created the current identity.
+	Version int64
+
+	// RawIdentityJSON contains the identity that needs to be
+	// upgraded to match the current schema version.
+	RawIdentityJSON []byte
+}
+
+type UpgradeResourceIdentityResponse struct {
+	// UpgradedState is the newly upgraded resource identity.
+	UpgradedIdentity cty.Value
+
+	// Diagnostics contains any warnings or errors from the method call.
+	Diagnostics tfdiags.Diagnostics
+}
+
 type ConfigureProviderRequest struct {
 	// Terraform version is the version string from the running instance of
 	// terraform. Providers can use TerraformVersion to verify compatibility,

internal/providers/testing/provider_mock.go

@@ -58,6 +58,12 @@ type MockProvider struct {
 	UpgradeResourceStateRequest  providers.UpgradeResourceStateRequest
 	UpgradeResourceStateFn       func(providers.UpgradeResourceStateRequest) providers.UpgradeResourceStateResponse
 
+	UpgradeResourceIdentityCalled   bool
+	UpgradeResourceIdentityTypeName string
+	UpgradeResourceIdentityResponse *providers.UpgradeResourceIdentityResponse
+	UpgradeResourceIdentityRequest  providers.UpgradeResourceIdentityRequest
+	UpgradeResourceIdentityFn       func(providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse
+
 	ConfigureProviderCalled   bool
 	ConfigureProviderResponse *providers.ConfigureProviderResponse
 	ConfigureProviderRequest  providers.ConfigureProviderRequest
@@ -150,6 +156,10 @@ func (p *MockProvider) GetResourceIdentitySchemas() providers.GetResourceIdentit
 	defer p.Unlock()
 	p.GetResourceIdentitySchemasCalled = true
 
+	return p.getResourceIdentitySchemas()
+}
+
+func (p *MockProvider) getResourceIdentitySchemas() providers.GetResourceIdentitySchemasResponse {
 	if p.GetResourceIdentitySchemasResponse != nil {
 		return *p.GetResourceIdentitySchemasResponse
 	}
@@ -323,6 +333,44 @@ func (p *MockProvider) UpgradeResourceState(r providers.UpgradeResourceStateRequ
 	return resp
 }
 
+func (p *MockProvider) UpgradeResourceIdentity(r providers.UpgradeResourceIdentityRequest) (resp providers.UpgradeResourceIdentityResponse) {
+	p.Lock()
+	defer p.Unlock()
+
+	if !p.ConfigureProviderCalled {
+		resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("Configure not called before UpgradeResourceIdentity %q", r.TypeName))
+		return resp
+	}
+	p.UpgradeResourceIdentityCalled = true
+	p.UpgradeResourceIdentityRequest = r
+
+	if p.UpgradeResourceIdentityFn != nil {
+		return p.UpgradeResourceIdentityFn(r)
+	}
+
+	if p.UpgradeResourceIdentityResponse != nil {
+		return *p.UpgradeResourceIdentityResponse
+	}
+
+	identitySchema, ok := p.getResourceIdentitySchemas().IdentityTypes[r.TypeName]
+	if !ok {
+		resp.Diagnostics = resp.Diagnostics.Append(fmt.Errorf("no schema found for %q", r.TypeName))
+		return resp
+	}
+
+	identityType := identitySchema.Attributes.ImpliedType()
+
+	v, err := ctyjson.Unmarshal(r.RawIdentityJSON, identityType)
+
+	if err != nil {
+		resp.Diagnostics = resp.Diagnostics.Append(err)
+		return resp
+	}
+	resp.UpgradedIdentity = v
+
+	return resp
+}
+
 func (p *MockProvider) ConfigureProvider(r providers.ConfigureProviderRequest) (resp providers.ConfigureProviderResponse) {
 	p.Lock()
 	defer p.Unlock()

internal/refactoring/mock_provider.go

@@ -48,6 +48,10 @@ func (provider *mockProvider) UpgradeResourceState(providers.UpgradeResourceStat
 	panic("not implemented in mock")
 }
 
+func (provider *mockProvider) UpgradeResourceIdentity(providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse {
+	panic("not implemented in mock")
+}
+
 func (provider *mockProvider) ConfigureProvider(providers.ConfigureProviderRequest) providers.ConfigureProviderResponse {
 	panic("not implemented in mock")
 }

internal/stacks/stackruntime/internal/stackeval/stubs/errored.go

@@ -184,6 +184,22 @@ func (p *erroredProvider) UpgradeResourceState(req providers.UpgradeResourceStat
 	}
 }
 
+func (p *erroredProvider) UpgradeResourceIdentity(req providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse {
+	// Ideally we'd just skip this altogether and echo back what the caller
+	// provided, but the request is in a different serialization format than
+	// the response and so only the real provider can deal with this one.
+	var diags tfdiags.Diagnostics
+	diags = diags.Append(tfdiags.AttributeValue(
+		tfdiags.Error,
+		"Provider configuration is invalid",
+		"Cannot decode the prior state for this resource instance because its provider configuration is invalid.",
+		nil, // nil attribute path means the overall configuration block
+	))
+	return providers.UpgradeResourceIdentityResponse{
+		Diagnostics: diags,
+	}
+}
+
 // ValidateDataResourceConfig implements providers.Interface.
 func (p *erroredProvider) ValidateDataResourceConfig(req providers.ValidateDataResourceConfigRequest) providers.ValidateDataResourceConfigResponse {
 	// We'll just optimistically assume the configuration is valid, so that

internal/stacks/stackruntime/internal/stackeval/stubs/offline.go

@@ -103,6 +103,19 @@ func (o *offlineProvider) UpgradeResourceState(request providers.UpgradeResource
 	}
 }
 
+func (o *offlineProvider) UpgradeResourceIdentity(request providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse {
+	var diags tfdiags.Diagnostics
+	diags = diags.Append(tfdiags.AttributeValue(
+		tfdiags.Error,
+		"Called UpgradeResourceIdentity on an unconfigured provider",
+		"Cannot upgrade the state of this resource because this provider is not configured. This is a bug in Terraform - please report it.",
+		nil, // nil attribute path means the overall configuration block
+	))
+	return providers.UpgradeResourceIdentityResponse{
+		Diagnostics: diags,
+	}
+}
+
 func (o *offlineProvider) ConfigureProvider(request providers.ConfigureProviderRequest) providers.ConfigureProviderResponse {
 	var diags tfdiags.Diagnostics
 	diags = diags.Append(tfdiags.AttributeValue(

internal/stacks/stackruntime/internal/stackeval/stubs/unknown.go

@@ -74,6 +74,12 @@ func (u *unknownProvider) UpgradeResourceState(request providers.UpgradeResource
 	return u.unconfiguredClient.UpgradeResourceState(request)
 }
 
+func (u *unknownProvider) UpgradeResourceIdentity(request providers.UpgradeResourceIdentityRequest) providers.UpgradeResourceIdentityResponse {
+	// This is offline functionality, so we can hand it off to the unconfigured
+	// client.
+	return u.unconfiguredClient.UpgradeResourceIdentity(request)
+}
+
 func (u *unknownProvider) ConfigureProvider(request providers.ConfigureProviderRequest) providers.ConfigureProviderResponse {
 	// This shouldn't be called, we don't configure an unknown provider within
 	// stacks and Terraform Core shouldn't call this method.

internal/states/instance_object_src.go

@@ -75,7 +75,7 @@ type ResourceInstanceObjectSrc struct {
 }
 
 // DecodeWithIdentity unmarshals the raw representation of the object attributes
-// and identity schema.
+// and identity schema. We expect the caller to make sure upgrades of the resource identity happen beforehand.
 func (os *ResourceInstanceObjectSrc) DecodeWithIdentity(ty cty.Type, identityTy cty.Type, identitySchemaVersion uint64) (*ResourceInstanceObject, error) {
 	// TODO: On call-side this should lead to an upgrade call (plus we need the old schema as well)
 	// We might have no identity data at all
@@ -94,7 +94,7 @@ func (os *ResourceInstanceObjectSrc) DecodeWithIdentity(ty cty.Type, identityTy
 
 	rio.Identity, err = ctyjson.Unmarshal(os.IdentitySchemaJSON, identityTy)
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode identity schema: %e", err)
+		return nil, fmt.Errorf("failed to decode identity schema: %s. This is most likely a bug in the Provider, providers must not change the identity schema without updating the identity schema version", err.Error())
 	}
 
 	return rio, nil