Skip to content

Commit

Permalink
Detailed diff in cross tests (#2366)
Browse files Browse the repository at this point in the history 
This exposes the detailed diff in the cross-test interface for both TF
and Pulumi allowing us to write tests which assert on that.

Also adds a cross-test for
#1696 for the
existing `TestChangingMaxItems1FilterProperty`. This is one of the cases
where the detailed diff is load-bearing for the diff decision. The cross
test verifies that TF also finds a diff here, so our core diff algorithm
must be making the wrong decision.

Related to #2293
  • Loading branch information
@VenelinMartinov
VenelinMartinov authored Sep 17, 2024
1 parent 16dec78 commit 4d6cd87
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 20 deletions.
29 changes: 21 additions & 8 deletions pkg/tests/cross-tests/diff_check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,17 @@ type diffTestCase struct {
DeleteBeforeReplace bool
}

func runDiffCheck(t T, tc diffTestCase) []string {
type pulumiDiffResp struct {
DetailedDiff map[string]interface{} `json:"detailedDiff"`
DeleteBeforeReplace bool `json:"deleteBeforeReplace"`
}

type diffResult struct {
TFDiff tfChange
PulumiDiff pulumiDiffResp
}

func runDiffCheck(t T, tc diffTestCase) diffResult {
tfwd := t.TempDir()

lifecycleArgs := lifecycleArgs{CreateBeforeDestroy: !tc.DeleteBeforeReplace}
Expand Down Expand Up @@ -80,21 +90,24 @@ func runDiffCheck(t T, tc diffTestCase) []string {
require.NoErrorf(t, err, "writing Pulumi.yaml")
x := pt.Up()

tfAction := tfd.parseChangesFromTFPlan(*tfDiffPlan)
changes := tfd.parseChangesFromTFPlan(*tfDiffPlan)

var diffResponse map[string]interface{}
diffResponse := pulumiDiffResp{}
for _, entry := range pt.GrpcLog().Entries {
if entry.Method == "/pulumirpc.ResourceProvider/Diff" {
err := json.Unmarshal(entry.Response, &diffResponse)
require.NoError(t, err)
}
}
tc.verifyBasicDiffAgreement(t, tfAction, x.Summary, diffResponse)
tc.verifyBasicDiffAgreement(t, changes.Actions, x.Summary, diffResponse)

return tfAction
return diffResult{
TFDiff: changes,
PulumiDiff: diffResponse,
}
}

func (tc *diffTestCase) verifyBasicDiffAgreement(t T, tfActions []string, us auto.UpdateSummary, diffResponse map[string]interface{}) {
func (tc *diffTestCase) verifyBasicDiffAgreement(t T, tfActions []string, us auto.UpdateSummary, diffResponse pulumiDiffResp) {
t.Logf("UpdateSummary.ResourceChanges: %#v", us.ResourceChanges)
// Action list from https://github.com/opentofu/opentofu/blob/main/internal/plans/action.go#L11
if len(tfActions) == 0 {
Expand Down Expand Up @@ -134,14 +147,14 @@ func (tc *diffTestCase) verifyBasicDiffAgreement(t T, tfActions []string, us aut
rc := *us.ResourceChanges
assert.Equalf(t, 1, rc[string(apitype.OpSame)], "expected the stack to stay the same")
assert.Equalf(t, 1, rc[string(apitype.OpReplace)], "expected the test resource to get a replace plan")
assert.Equalf(t, diffResponse["deleteBeforeReplace"], nil, "expected deleteBeforeReplace to be true")
assert.Equalf(t, diffResponse.DeleteBeforeReplace, false, "expected deleteBeforeReplace to be true")
} else if tfActions[0] == "delete" && tfActions[1] == "create" {
require.NotNilf(t, us.ResourceChanges, "UpdateSummary.ResourceChanges should not be nil")
rc := *us.ResourceChanges
t.Logf("UpdateSummary.ResourceChanges: %#v", rc)
assert.Equalf(t, 1, rc[string(apitype.OpSame)], "expected the stack to stay the same")
assert.Equalf(t, 1, rc[string(apitype.OpReplace)], "expected the test resource to get a replace plan")
assert.Equalf(t, diffResponse["deleteBeforeReplace"], true, "expected deleteBeforeReplace to be true")
assert.Equalf(t, diffResponse.DeleteBeforeReplace, true, "expected deleteBeforeReplace to be true")
} else {
panic("TODO: do not understand this TF action yet: " + fmt.Sprint(tfActions))
}
Expand Down
122 changes: 116 additions & 6 deletions pkg/tests/cross-tests/diff_cross_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/hashicorp/terraform-plugin-go/tftypes"
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hexops/autogold/v2"
"github.com/stretchr/testify/require"
)

Expand Down Expand Up @@ -190,7 +191,7 @@ func TestDiffBasicTypes(t *testing.T) {
Config2: tc.config1,
})

require.Equal(t, []string{"no-op"}, tfAction)
require.Equal(t, []string{"no-op"}, tfAction.TFDiff.Actions)
})

t.Run("diff", func(t *testing.T) {
Expand All @@ -200,7 +201,7 @@ func TestDiffBasicTypes(t *testing.T) {
Config2: tc.config2,
})

require.Equal(t, []string{"update"}, tfAction)
require.Equal(t, []string{"update"}, tfAction.TFDiff.Actions)
})

t.Run("create", func(t *testing.T) {
Expand All @@ -210,7 +211,7 @@ func TestDiffBasicTypes(t *testing.T) {
Config2: tc.config1,
})

require.Equal(t, []string{"create"}, tfAction)
require.Equal(t, []string{"create"}, tfAction.TFDiff.Actions)
})

t.Run("delete", func(t *testing.T) {
Expand All @@ -220,7 +221,7 @@ func TestDiffBasicTypes(t *testing.T) {
Config2: nil,
})

require.Equal(t, []string{"delete"}, tfAction)
require.Equal(t, []string{"delete"}, tfAction.TFDiff.Actions)
})

t.Run("replace", func(t *testing.T) {
Expand All @@ -235,7 +236,7 @@ func TestDiffBasicTypes(t *testing.T) {
Config2: tc.config2,
})

require.Equal(t, []string{"create", "delete"}, tfAction)
require.Equal(t, []string{"create", "delete"}, tfAction.TFDiff.Actions)
})

t.Run("replace delete first", func(t *testing.T) {
Expand All @@ -251,7 +252,7 @@ func TestDiffBasicTypes(t *testing.T) {
DeleteBeforeReplace: true,
})

require.Equal(t, []string{"delete", "create"}, tfAction)
require.Equal(t, []string{"delete", "create"}, tfAction.TFDiff.Actions)
})
})
}
Expand Down Expand Up @@ -831,3 +832,112 @@ func TestComputedSetFieldsNoDiff(t *testing.T) {
Config2: t0,
})
}

func TestMaxItemsOneCollectionOnlyDiff(t *testing.T) {
sch := map[string]*schema.Schema{
"rule": {
Type: schema.TypeList,
Required: true,
MaxItems: 1000,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"filter": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"prefix": {
Type: schema.TypeString,
Optional: true,
},
},
},
},
},
},
},
}

t1 := tftypes.Object{
AttributeTypes: map[string]tftypes.Type{
"prefix": tftypes.String,
},
}

t2 := tftypes.Object{
AttributeTypes: map[string]tftypes.Type{
"filter": tftypes.List{ElementType: t1},
},
}

t3 := tftypes.Object{
AttributeTypes: map[string]tftypes.Type{
"rule": tftypes.List{ElementType: t2},
},
}

v1 := tftypes.NewValue(
t3,
map[string]tftypes.Value{
"rule": tftypes.NewValue(
tftypes.List{ElementType: t2},
[]tftypes.Value{
tftypes.NewValue(
t2,
map[string]tftypes.Value{
"filter": tftypes.NewValue(
tftypes.List{ElementType: t1},
[]tftypes.Value{},
),
},
),
},
),
},
)

v2 := tftypes.NewValue(
t3,
map[string]tftypes.Value{
"rule": tftypes.NewValue(
tftypes.List{ElementType: t2},
[]tftypes.Value{
tftypes.NewValue(
t2,
map[string]tftypes.Value{
"filter": tftypes.NewValue(
tftypes.List{ElementType: t1},
[]tftypes.Value{
tftypes.NewValue(
t1,
map[string]tftypes.Value{
"prefix": tftypes.NewValue(tftypes.String, nil),
},
),
},
),
},
),
},
),
},
)

diff := runDiffCheck(
t,
diffTestCase{
Resource: &schema.Resource{Schema: sch},
Config1: v1,
Config2: v2,
},
)

getFilter := func(val map[string]any) any {
return val["rule"].([]any)[0].(map[string]any)["filter"]
}

require.Equal(t, []string{"update"}, diff.TFDiff.Actions)
require.NotEqual(t, getFilter(diff.TFDiff.Before), getFilter(diff.TFDiff.After))
autogold.Expect(map[string]interface{}{"rules[0].filter": map[string]interface{}{"kind": "UPDATE"}}).Equal(t, diff.PulumiDiff.DetailedDiff)
}
15 changes: 9 additions & 6 deletions pkg/tests/cross-tests/tf_driver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,21 @@ func (d *TfResDriver) write(
d.driver.Write(t, buf.String())
}

type tfChange struct {
Actions []string `json:"actions"`
Before map[string]any `json:"before"`
After map[string]any `json:"after"`
}

// Still discovering the structure of JSON-serialized TF plans. The information required from these is, primarily, is
// whether the resource is staying unchanged, being updated or replaced. Secondarily, would be also great to know
// detailed paths of properties causing the change, though that is more difficult to cross-compare with Pulumi.
//
// For now this is code is similar to `jq .resource_changes[0].change.actions[0] plan.json`.
func (*TfResDriver) parseChangesFromTFPlan(plan tfcheck.TfPlan) []string {
func (*TfResDriver) parseChangesFromTFPlan(plan tfcheck.TfPlan) tfChange {
type p struct {
ResourceChanges []struct {
Change struct {
Actions []string `json:"actions"`
} `json:"change"`
Change tfChange `json:"change"`
} `json:"resource_changes"`
}
jb, err := json.Marshal(plan.RawPlan)
Expand All @@ -131,6 +135,5 @@ func (*TfResDriver) parseChangesFromTFPlan(plan tfcheck.TfPlan) []string {
err = json.Unmarshal(jb, &pp)
contract.AssertNoErrorf(err, "failed to unmarshal terraform plan")
contract.Assertf(len(pp.ResourceChanges) == 1, "expected exactly one resource change")
actions := pp.ResourceChanges[0].Change.Actions
return actions
return pp.ResourceChanges[0].Change
}

0 comments on commit 4d6cd87

Please sign in to comment.