Use lookup maps for transformation request/response body and path par…

…ams (#205)

* Upgrade pulschema to the latest

* Update all other packages as well and fix errors

* Handle request body transformation from SDK names to API names

* Handle request/response body transformation using lookup maps

openapi/properties_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
 )
 
 //go:embed testdata/render_openapi.yml
@@ -35,8 +36,10 @@ func TestFilterReadOnlyProperties(t *testing.T) {
 	})
 
 	doc := GetOpenAPISpec([]byte(renderOpenAPIEmbed))
+	pathItem := doc.Paths.Find("/services")
+	contract.Assertf(pathItem != nil, "/services was not found")
 
-	FilterReadOnlyProperties(ctx, *doc.Paths["/services"].Post.RequestBody.Value.Content.Get("application/json").Schema.Value, inputs)
+	FilterReadOnlyProperties(ctx, *pathItem.Post.RequestBody.Value.Content.Get("application/json").Schema.Value, inputs)
 
 	assert.NotNil(t, inputs)
 

rest/generic_schema_test.go

@@ -0,0 +1,111 @@
+package rest
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+
+	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
+
+	openapigen "github.com/cloudy-sky-software/pulschema/pkg"
+)
+
+var packageName = "generic"
+
+func genericPulumiSchema(openapiDoc *openapi3.T) (pschema.PackageSpec, openapigen.ProviderMetadata, openapi3.T) {
+	pkg := pschema.PackageSpec{
+		Name:        packageName,
+		Description: "A Pulumi package for creating and managing Generic Provider resources.",
+		DisplayName: "Generic",
+		License:     "Apache-2.0",
+		Keywords: []string{
+			"pulumi",
+			packageName,
+			"category/cloud",
+			"kind/native",
+		},
+		Homepage:   "https://cloudysky.software",
+		Publisher:  "Cloudy Sky Software",
+		Repository: "https://github.com/cloudy-sky-software/pulumi-fake-native",
+
+		Config: pschema.ConfigSpec{
+			Variables: map[string]pschema.PropertySpec{
+				"apiKey": {
+					Description: "The API key",
+					TypeSpec:    pschema.TypeSpec{Type: "string"},
+					Language: map[string]pschema.RawMessage{
+						"csharp": rawMessage(map[string]interface{}{
+							"name": "ApiKey",
+						}),
+					},
+					Secret: true,
+				},
+			},
+		},
+
+		Provider: pschema.ResourceSpec{
+			ObjectTypeSpec: pschema.ObjectTypeSpec{
+				Description: "The provider type for the Generic package.",
+				Type:        "object",
+			},
+			InputProperties: map[string]pschema.PropertySpec{
+				"apiKey": {
+					DefaultInfo: &pschema.DefaultSpec{
+						Environment: []string{
+							"FAKE_APIKEY",
+						},
+					},
+					Description: "The API key.",
+					TypeSpec:    pschema.TypeSpec{Type: "string"},
+					Language: map[string]pschema.RawMessage{
+						"csharp": rawMessage(map[string]interface{}{
+							"name": "ApiKey",
+						}),
+					},
+					Secret: true,
+				},
+			},
+		},
+
+		PluginDownloadURL: "github://api.github.com/cloudy-sky-software/pulumi-fake-native",
+		Types:             map[string]pschema.ComplexTypeSpec{},
+		Resources:         map[string]pschema.ResourceSpec{},
+		Functions:         map[string]pschema.FunctionSpec{},
+		Language:          map[string]pschema.RawMessage{},
+	}
+
+	csharpNamespaces := map[string]string{
+		"": "Provider",
+	}
+
+	openAPICtx := &openapigen.OpenAPIContext{
+		Doc: *openapiDoc,
+		Pkg: &pkg,
+	}
+
+	providerMetadata, updatedOpenAPIDoc, err := openAPICtx.GatherResourcesFromAPI(csharpNamespaces)
+	if err != nil {
+		contract.Failf("generating resources from OpenAPI spec: %v", err)
+	}
+
+	metadata := openapigen.ProviderMetadata{
+		ResourceCRUDMap:  providerMetadata.ResourceCRUDMap,
+		AutoNameMap:      providerMetadata.AutoNameMap,
+		SDKToAPINameMap:  providerMetadata.SDKToAPINameMap,
+		APIToSDKNameMap:  providerMetadata.APIToSDKNameMap,
+		PathParamNameMap: providerMetadata.PathParamNameMap,
+	}
+
+	return pkg, metadata, updatedOpenAPIDoc
+}
+
+func rawMessage(v interface{}) pschema.RawMessage {
+	var out bytes.Buffer
+	encoder := json.NewEncoder(&out)
+	encoder.SetEscapeHTML(false)
+	err := encoder.Encode(v)
+	contract.Assert(err == nil)
+	return out.Bytes()
+}

rest/provider.go

@@ -124,7 +124,7 @@ func GetResourceTypeToken(u string) string {
 }
 
 func getResourceName(u string) string {
-	return resource.URN(u).Name().String()
+	return resource.URN(u).Name()
 }
 
 // Attach sends the engine address to an already running plugin.
@@ -343,12 +343,12 @@ func (p *Provider) Diff(ctx context.Context, req *pulumirpc.DiffRequest) (*pulum
 	var updateOp *openapi3.Operation
 	switch {
 	case crudMap.U != nil:
-		updateOp = p.openAPIDoc.Paths[*crudMap.U].Patch
+		updateOp = p.openAPIDoc.Paths.Find(*crudMap.U).Patch
 		if updateOp == nil {
 			return nil, errors.Errorf("openapi doc does not have PATCH endpoint definition for the path %s", *crudMap.U)
 		}
 	case crudMap.P != nil:
-		updateOp = p.openAPIDoc.Paths[*crudMap.P].Put
+		updateOp = p.openAPIDoc.Paths.Find(*crudMap.P).Put
 		if updateOp == nil {
 			return nil, errors.Errorf("openapi doc does not have PUT endpoint definition for the path %s", *crudMap.U)
 		}
@@ -504,6 +504,8 @@ func (p *Provider) Create(ctx context.Context, req *pulumirpc.CreateRequest) (*p
 		return nil, postCreateErr
 	}
 
+	p.TransformBody(ctx, outputsMap, p.metadata.APIToSDKNameMap)
+
 	outputProperties, err := plugin.MarshalProperties(state.GetResourceState(outputsMap, inputs), state.DefaultMarshalOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "marshaling the output properties map")
@@ -630,6 +632,8 @@ func (p *Provider) Read(ctx context.Context, req *pulumirpc.ReadRequest) (*pulum
 		return nil, postReadErr
 	}
 
+	p.TransformBody(ctx, outputs, p.metadata.APIToSDKNameMap)
+
 	outputProperties, err := plugin.MarshalProperties(state.GetResourceState(outputs, inputs), state.DefaultMarshalOpts)
 	if err != nil {
 		return nil, errors.Wrap(err, "marshaling the output properties map")
@@ -684,12 +688,8 @@ func (p *Provider) Update(ctx context.Context, req *pulumirpc.UpdateRequest) (*p
 		method = http.MethodPut
 	}
 
-	reqBody, err := json.Marshal(inputs.Mappable())
-	if err != nil {
-		return nil, errors.Wrap(err, "marshaling inputs")
-	}
-
-	logging.V(3).Infof("REQUEST BODY: %s", string(reqBody))
+	bodyMap := inputs.Mappable()
+	logging.V(3).Infof("REQUEST BODY: %v", bodyMap)
 
 	hasPathParams := strings.Contains(httpEndpointPath, "{")
 	var pathParams map[string]string
@@ -703,18 +703,24 @@ func (p *Provider) Update(ctx context.Context, req *pulumirpc.UpdateRequest) (*p
 			return nil, errors.Wrapf(err, "getting path params (type token: %s)", resourceTypeToken)
 		}
 
-		if reqBody != nil {
+		if bodyMap != nil {
 			logging.V(3).Infoln("Removing path params from request body")
-			updatedBody, err := p.removePathParamsFromRequestBody(reqBody, pathParams)
-			if err != nil {
-				return nil, errors.Wrap(err, "removing path params from request body")
-			}
+			p.removePathParamsFromRequestBody(bodyMap, pathParams)
+		}
+	}
 
-			reqBody = updatedBody
+	var buf *bytes.Buffer
+	// Transform properties in the request body from SDK name to API name.
+	if bodyMap != nil {
+		p.TransformBody(ctx, bodyMap, p.metadata.SDKToAPINameMap)
+
+		reqBody, err := json.Marshal(bodyMap)
+		if err != nil {
+			return nil, errors.Wrap(err, "marshaling inputs")
 		}
+		buf = bytes.NewBuffer(reqBody)
 	}
 
-	buf := bytes.NewBuffer(reqBody)
 	httpReq, err := http.NewRequestWithContext(ctx, method, p.baseURL+httpEndpointPath, buf)
 	if err != nil {
 		return nil, errors.Wrap(err, "initializing request")
@@ -771,6 +777,8 @@ func (p *Provider) Update(ctx context.Context, req *pulumirpc.UpdateRequest) (*p
 		return nil, postUpdateErr
 	}
 
+	p.TransformBody(ctx, outputsMap, p.metadata.APIToSDKNameMap)
+
 	// TODO: Could this erase refreshed inputs that were previously saved in outputs state?
 	outputProperties, err := plugin.MarshalProperties(state.GetResourceState(outputsMap, inputs), state.DefaultMarshalOpts)
 	if err != nil {

rest/provider_test.go

@@ -30,7 +30,7 @@ var tailscaleMetadataEmbed string
 //go:embed testdata/tailscale/schema.json
 var tailscalePulSchemaEmbed string
 
-func makeTestProvider(ctx context.Context, t *testing.T, testServer *httptest.Server) pulumirpc.ResourceProviderServer {
+func makeTestTailscaleProvider(ctx context.Context, t *testing.T, testServer *httptest.Server) pulumirpc.ResourceProviderServer {
 	t.Helper()
 
 	openapiBytes := []byte(tailscaleOpenAPIEmbed)
@@ -101,7 +101,7 @@ func TestResourceReadResultsInNoChanges(t *testing.T) {
 
 	defer testServer.Close()
 
-	p := makeTestProvider(ctx, t, testServer)
+	p := makeTestTailscaleProvider(ctx, t, testServer)
 
 	var inputs map[string]interface{}
 	if err := json.Unmarshal([]byte(inputsJSON), &inputs); err != nil {

rest/request.go

@@ -116,29 +116,19 @@ func (p *Provider) CreateGetRequest(
 	return httpReq, nil
 }
 
-func (p *Provider) removePathParamsFromRequestBody(body []byte, pathParams map[string]string) ([]byte, error) {
-	var bodyMap map[string]interface{}
-
-	if err := json.Unmarshal(body, &bodyMap); err != nil {
-		return nil, errors.Wrap(err, "unmarshaling body")
-	}
-
-	for k := range pathParams {
-		// Delete the path param from the request body since it was added
-		// as a way to take path params as inputs to the resource.
-		delete(bodyMap, k)
-	}
-
-	updatedBody, _ := json.Marshal(bodyMap)
-	logging.V(3).Infof("replacePathParams: UPDATED HTTP REQUEST BODY: %s", string(updatedBody))
-	return updatedBody, nil
-}
-
 func (p *Provider) createHTTPRequestWithBody(ctx context.Context, httpEndpointPath string, httpMethod string, reqBody []byte, inputs resource.PropertyMap) (*http.Request, error) {
 	logging.V(3).Infof("REQUEST BODY: %s", string(reqBody))
 
 	hasPathParams := strings.Contains(httpEndpointPath, "{")
 	var pathParams map[string]string
+
+	var bodyMap map[string]interface{}
+	if reqBody != nil {
+		if err := json.Unmarshal(reqBody, &bodyMap); err != nil {
+			return nil, errors.Wrap(err, "unmarshaling body")
+		}
+	}
+
 	// If the endpoint has path params, peek into the OpenAPI doc
 	// for the param names.
 	if hasPathParams {
@@ -150,16 +140,23 @@ func (p *Provider) createHTTPRequestWithBody(ctx context.Context, httpEndpointPa
 
 		if reqBody != nil {
 			logging.V(3).Infoln("Removing path params from request body")
-			updatedBody, err := p.removePathParamsFromRequestBody(reqBody, pathParams)
-			if err != nil {
-				return nil, errors.Wrap(err, "removing path params from request body")
-			}
+			p.removePathParamsFromRequestBody(bodyMap, pathParams)
+		}
+	}
 
-			reqBody = updatedBody
+	var buf *bytes.Buffer
+	// Transform properties in the request body from SDK name to API name.
+	if bodyMap != nil {
+		p.TransformBody(ctx, bodyMap, p.metadata.SDKToAPINameMap)
+
+		updatedBody, err := json.Marshal(bodyMap)
+		if err != nil {
+			return nil, errors.Wrap(err, "marshaling body")
 		}
+
+		buf = bytes.NewBuffer(updatedBody)
 	}
 
-	buf := bytes.NewBuffer(reqBody)
 	httpReq, err := http.NewRequestWithContext(ctx, httpMethod, p.baseURL+httpEndpointPath, buf)
 	if err != nil {
 		return nil, errors.Wrap(err, "initializing request")
@@ -281,15 +278,15 @@ func (p *Provider) getPathParamsMap(apiPath, requestMethod string, properties re
 
 	switch requestMethod {
 	case http.MethodGet:
-		parameters = p.openAPIDoc.Paths[apiPath].Get.Parameters
+		parameters = p.openAPIDoc.Paths.Find(apiPath).Get.Parameters
 	case http.MethodPost:
-		parameters = p.openAPIDoc.Paths[apiPath].Post.Parameters
+		parameters = p.openAPIDoc.Paths.Find(apiPath).Post.Parameters
 	case http.MethodPatch:
-		parameters = p.openAPIDoc.Paths[apiPath].Patch.Parameters
+		parameters = p.openAPIDoc.Paths.Find(apiPath).Patch.Parameters
 	case http.MethodPut:
-		parameters = p.openAPIDoc.Paths[apiPath].Put.Parameters
+		parameters = p.openAPIDoc.Paths.Find(apiPath).Put.Parameters
 	case http.MethodDelete:
-		parameters = p.openAPIDoc.Paths[apiPath].Delete.Parameters
+		parameters = p.openAPIDoc.Paths.Find(apiPath).Delete.Parameters
 	default:
 		return pathParams, nil
 	}
@@ -305,8 +302,14 @@ func (p *Provider) getPathParamsMap(apiPath, requestMethod string, properties re
 
 		count++
 		paramName := param.Value.Name
+		sdkName := paramName
+		if o, ok := p.metadata.PathParamNameMap[sdkName]; ok {
+			logging.V(3).Infof("Path param %q is overridden in the schema as %q", paramName, o)
+			sdkName = o
+		}
+
 		logging.V(3).Infof("Looking for path param %q in resource inputs %v", paramName, properties)
-		property, ok := properties[resource.PropertyKey(paramName)]
+		property, ok := properties[resource.PropertyKey(sdkName)]
 		// If the path param is not in the properties, check if
 		// we have the old inputs, if we are dealing with the state
 		// of an existing resource.
@@ -315,7 +318,7 @@ func (p *Provider) getPathParamsMap(apiPath, requestMethod string, properties re
 				return nil, errors.Errorf("did not find value for path param %s in output props (old inputs was nil)", paramName)
 			}
 
-			property, ok = oldInputs[resource.PropertyKey(paramName)]
+			property, ok = oldInputs[resource.PropertyKey(sdkName)]
 			if !ok {
 				return nil, errors.Errorf("did not find value for path param %s in output props and old inputs", paramName)
 			}
@@ -342,6 +345,21 @@ func (p *Provider) getPathParamsMap(apiPath, requestMethod string, properties re
 	return pathParams, nil
 }
 
+func (p *Provider) removePathParamsFromRequestBody(bodyMap map[string]interface{}, pathParams map[string]string) {
+	for paramName := range pathParams {
+		if sdkName, ok := p.metadata.PathParamNameMap[paramName]; ok {
+			logging.V(3).Infof("Path param %[1]q is overridden in the schema as %[2]q. Will remove %[2]q from body instead.", paramName, sdkName)
+			paramName = sdkName
+		}
+		// Delete the path param from the request body since it was added
+		// as a way to take path params as inputs to the resource.
+		delete(bodyMap, paramName)
+	}
+
+	updatedBody, _ := json.Marshal(bodyMap)
+	logging.V(3).Infof("replacePathParams: UPDATED HTTP REQUEST BODY: %s", string(updatedBody))
+}
+
 func (p *Provider) replacePathParams(httpReq *http.Request, pathParams map[string]string) error {
 	path := httpReq.URL.Path
 

rest/request_test.go

@@ -34,7 +34,7 @@ func TestRemovePathParamsFromRequestBody(t *testing.T) {
 		t.Fatalf("Failed to unmarshal test payload: %v", err)
 	}
 
-	p := makeTestProvider(ctx, t, nil)
+	p := makeTestTailscaleProvider(ctx, t, nil)
 	httpReq, err := p.(Request).CreatePostRequest(ctx, "/tailnet/{tailnet}/keys", []byte(testCreateJSONPayload), resource.NewPropertyMapFromMap(inputs))
 	assert.Nil(t, err)
 	assert.NotNil(t, httpReq)