From 0c7da962e76f785f19971e11e3d97e0cdba3d3e0 Mon Sep 17 00:00:00 2001 From: Chris Every Date: Fri, 24 Apr 2020 00:22:28 +0100 Subject: [PATCH] Add tf 0 12 support (#12) * Add tf 0.12 support * Ensure currentResource is maintained over scanning * More versioning of regexes * Add comments * PR fix for random_string.some_password.id * Update main.go Co-authored-by: Erik Osterman --- main.go | 222 +++++++++++++++++++++++++++++++++++++-------------- main_test.go | 169 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+), 62 deletions(-) create mode 100644 main_test.go diff --git a/main.go b/main.go index 0c243a5..ab17468 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,28 @@ import ( "unicode/utf8" ) +type match struct { + leadingWhitespace string + property string // something like `stage.0.action.0.configuration.OAuthToken` + trailingWhitespace string + firstQuote string // < or " + oldValue string + secondQuote string // > or " + thirdQuote string // < or " or ( + newValue string + fourthQuote string // > or " or ) + postfix string +} + +type expression struct { + planStatusRegex *regexp.Regexp + reTfPlanLine *regexp.Regexp + reTfPlanCurrentResource *regexp.Regexp + resourceIndex int + assign string + operator string +} + func init() { // make sure we only have one process and that it runs on the main thread // (so that ideally, when we Exec, we keep our user switches and stuff) @@ -25,82 +47,64 @@ func getEnv(key, fallback string) string { return fallback } +var versionedExpressions = map[string]expression{ + "0.11": expression{ + planStatusRegex: regexp.MustCompile( + "^(.*?): (.*?) +\\(ID: (.*?)\\)$", + ), + reTfPlanLine: regexp.MustCompile( + "^( +)([a-zA-Z0-9%._-]+):( +)([\"<])(.*?)([>\"]) +=> +([\"<])(.*)([>\"])(.*)$", + ), + reTfPlanCurrentResource: regexp.MustCompile( + "^([~/+-]+) (.*?) +(.*)$", + ), + resourceIndex: 2, + assign: ":", + operator: "=>", + }, + "0.12": expression{ + planStatusRegex: regexp.MustCompile( + "^(.*?): (.*?) +\\[id=(.*?)\\]$", + ), + reTfPlanLine: regexp.MustCompile( + "^( +)([ ~a-zA-Z0-9%._-]+)=( +)([\"<])(.*?)([>\"]) +-> +(\\()(.*)(\\))(.*)$", + ), + reTfPlanCurrentResource: regexp.MustCompile( + "^([~/+-]+) (.*?) +(.*) (.*) (.*)$", + ), + resourceIndex: 3, + assign: "=", + operator: "->", + }, +} + func main() { log.SetFlags(0) // no timestamps on our logs // Character used to mask sensitive output var tfmaskChar = getEnv("TFMASK_CHAR", "*") - // Pattern representing sensitive output - var tfmaskValuesRegex = getEnv("TFMASK_VALUES_REGEX", "(?i)^.*(oauth|secret|token|password|key|result).*$") - + var tfmaskValuesRegex = getEnv("TFMASK_VALUES_REGEX", + "(?i)^.*(oauth|secret|token|password|key|result|id).*$") // Pattern representing sensitive resource - var tfmaskResourceRegex = getEnv("TFMASK_RESOURCES_REGEX", "(?i)^(random_id).*$") - - // stage.0.action.0.configuration.OAuthToken: "" => "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - reTfPlanLine := regexp.MustCompile("^( +)([a-zA-Z0-9%._-]+):( +)([\"<])(.*?)([>\"]) +=> +([\"<])(.*)([>\"])(.*)$") - - // random_id.some_id: Refreshing state... (ID: itILf4x5lqleQV9ZwT2gH-Zg3yuXM8pdUu6VFTX...P5vqUmggDweOoxFMPY5t9thA0SJE2EZIhcHbsQ) - reTfPlanStatusLine := regexp.MustCompile("^(.*?): (.*?) +\\(ID: (.*?)\\)$") - + var tfmaskResourceRegex = getEnv("TFMASK_RESOURCES_REGEX", + "(?i)^(random_id|random_string).*$") - // -/+ random_string.postgres_admin_password (tainted) (new resource required) - reTfPlanCurrentResource := regexp.MustCompile("^([~/+-]+) (.*?) +(.*)$") - reTfApplyCurrentResource := regexp.MustCompile("^([a-z].*?): (.*?)$") - currentResource := "" + // Default to tf 0.11, but users can override + var tfenv = getEnv("TFENV", "0.12") reTfValues := regexp.MustCompile(tfmaskValuesRegex) reTfResource := regexp.MustCompile(tfmaskResourceRegex) scanner := bufio.NewScanner(os.Stdin) + versionedExpressions := versionedExpressions[tfenv] + // initialise currentResource once before scanning + currentResource := "" for scanner.Scan() { line := scanner.Text() - if reTfPlanCurrentResource.MatchString(line) { - match := reTfPlanCurrentResource.FindStringSubmatch(line) - currentResource = match[2] - } else if reTfApplyCurrentResource.MatchString(line) { - match := reTfApplyCurrentResource.FindStringSubmatch(line) - currentResource = match[1] - } - - if reTfPlanStatusLine.MatchString(line) { - match := reTfPlanStatusLine.FindStringSubmatch(line) - resource := match[1] - id := match[3] - if reTfResource.MatchString(resource) { - line = strings.Replace(line, id, strings.Repeat(tfmaskChar, utf8.RuneCountInString(id)), 1) - } - fmt.Println(line) - } else if reTfPlanLine.MatchString(line) { - match := reTfPlanLine.FindStringSubmatch(line) - leadingWhitespace := match[1] - property := match[2] // something like `stage.0.action.0.configuration.OAuthToken` - trailingWhitespace := match[3] - firstQuote := match[4] // < or " - oldValue := match[5] - secondQuote := match[6] // > or " - thirdQuote := match[7] // < or " - newValue := match[8] - fourthQuote := match[9] // > or " - postfix := match[10] - - if reTfValues.MatchString(property) || reTfResource.MatchString(currentResource) { - // The value inside the "..." or <...> - if oldValue != "sensitive" && oldValue != "computed" && oldValue != " - if newValue != "sensitive" && newValue != "computed" && newValue != " %v%v%v%v\n", - leadingWhitespace, property, trailingWhitespace, firstQuote, oldValue, secondQuote, thirdQuote, newValue, fourthQuote, postfix) - } else { - fmt.Println(line) - } - } else { - // We matched nothing - fmt.Println(line) - } + currentResource = getCurrentResource(versionedExpressions, + currentResource, line) + fmt.Println(processLine(versionedExpressions, reTfResource, reTfValues, + tfmaskChar, currentResource, line)) } if err := scanner.Err(); err != nil { @@ -108,3 +112,97 @@ func main() { os.Exit(1) } } + +func getCurrentResource(expression expression, currentResource, line string) string { + reTfApplyCurrentResource := regexp.MustCompile("^([a-z].*?): (.*?)$") + if expression.reTfPlanCurrentResource.MatchString(line) { + match := expression.reTfPlanCurrentResource.FindStringSubmatch(line) + // for tf 0.12 the resource is wrapped in quotes, so remove them + strippedResource := strings.Replace(match[expression.resourceIndex], + "\"", "", -1) + currentResource = strippedResource + } else if reTfApplyCurrentResource.MatchString(line) { + match := reTfApplyCurrentResource.FindStringSubmatch(line) + currentResource = match[1] + } + return currentResource +} + +func processLine(expression expression, reTfResource, + reTfValues *regexp.Regexp, tfmaskChar, currentResource, + line string) string { + if expression.planStatusRegex.MatchString(line) { + line = planStatus(expression.planStatusRegex, reTfResource, tfmaskChar, + line) + } else if expression.reTfPlanLine.MatchString(line) { + line = planLine(expression.reTfPlanLine, reTfResource, reTfValues, + currentResource, tfmaskChar, expression.assign, + expression.operator, line) + } + return line +} + +func planStatus(planStatusRegex, reTfResource *regexp.Regexp, tfmaskChar, + line string) string { + match := planStatusRegex.FindStringSubmatch(line) + resource := match[1] + id := match[3] + if reTfResource.MatchString(resource) { + line = strings.Replace(line, id, strings.Repeat(tfmaskChar, + utf8.RuneCountInString(id)), 1) + } + return line +} + +func matchFromLine(reTfPlanLine *regexp.Regexp, line string) match { + subMatch := reTfPlanLine.FindStringSubmatch(line) + return match{ + leadingWhitespace: subMatch[1], + property: subMatch[2], // something like `stage.0.action.0.configuration.OAuthToken` + trailingWhitespace: subMatch[3], + firstQuote: subMatch[4], + oldValue: subMatch[5], + secondQuote: subMatch[6], // > or " + thirdQuote: subMatch[7], // < or " or ( + newValue: subMatch[8], + fourthQuote: subMatch[9], // > or " or ) + postfix: subMatch[10], + } +} + +func planLine(reTfPlanLine, reTfResource, reTfValues *regexp.Regexp, + currentResource, tfmaskChar, assign, operator, line string) string { + match := matchFromLine(reTfPlanLine, line) + if reTfValues.MatchString(match.property) || + reTfResource.MatchString(currentResource) { + // The value inside the "...", <...> or (...) + oldValue := maskValue(match.oldValue, tfmaskChar) + // The value inside the "...", <...> or (...) + newValue := maskValue(match.newValue, tfmaskChar) + line = fmt.Sprintf("%v%v%v%v%v%v%v %v %v%v%v%v", + match.leadingWhitespace, match.property, assign, + match.trailingWhitespace, match.firstQuote, oldValue, + match.secondQuote, operator, match.thirdQuote, + newValue, match.fourthQuote, match.postfix) + } + return line +} + +func maskValue(value, tfmaskChar string) string { + exclusions := []string{"sensitive", "computed", " => ", + " stage.0.action.0.configuration.OAuthToken: <*******> => <*******> ", "0.11"}, + {"random_id.some_id", " stage.0.action.0.configuration.OAuthToken: \"tf.0.11\" => \"tf.0.11\" ", + " stage.0.action.0.configuration.OAuthToken: \"*******\" => \"*******\" ", "0.11"}, + {"random_id.some_id", " stage.0.action.0.configuration.DontObfuscate: => ", + " stage.0.action.0.configuration.DontObfuscate: <*******> => <*******> ", "0.11"}, + {"random_id.some_id", "random_id.some_id: Refreshing state... (ID: itILf4x5lqleQV9ZwT2gH-Zg3yuXM8pdUu6VFTX...P5vqUmggDweOoxFMPY5t9thA0SJE2EZIhcHbsQ)", + "random_id.some_id: Refreshing state... (ID: ********************************************************************************)", + "0.11"}, + {"random_string.some_password", "random_string.some_password: Refreshing state... (ID: 2iB@@h22@12kA2qE)", + "random_string.some_password: Refreshing state... (ID: ****************)", + "0.11"}, + {"random_id.some_id", " id: \"VIxvs2TloohI2XtAsHyu68wQvFQQCTOGgsglqC7zKjsnOmUMIMrZ1y5J6ieOIzl-YXiS1_XmVc8J8gb9fIcwIA\" => (forces new resource)", + " id: \"**************************************************************************************\" => (forces new resource)", + "0.11"}, + // tf 0.12 ------------------------------------ + {"random_id.some_id", "not_secret", "not_secret", "0.12"}, + {"random_id.some_id", " ~ result = \"pkwemfpwmfwf\" -> (known after apply) ", + " ~ result = \"************\" -> (known after apply) ", "0.12"}, + {"random_id.some_id", "random_id.some_id: Refreshing state... [id=itILf4x5lqleQV9ZwT2gH-Zg3yuXM8pdUu6VFTX...P5vqUmggDweOoxFMPY5t9thA0SJE2EZIhcHbsQ]", + "random_id.some_id: Refreshing state... [id=********************************************************************************]", + "0.12"}, + {"", "random_id.some_id: Creation complete after 0s [id=YfK9aF]", + "random_id.some_id: Creation complete after 0s [id=******]", + "0.12"}, + {"random_string.some_password", "random_string.some_password: Refreshing state... [id=2iB@@h22@12kA2qE]", + "random_string.some_password: Refreshing state... [id=****************]", + "0.12"}, + {"random_id.some_id", " ~ id = \"VIxvs2TloohI2XtAsHyu68wQvFQQCTOGgsglqC7zKjsnOmUMIMrZ1y5J6ieOIzl-YXiS1_XmVc8J8gb9fIcwIA\" -> (known after apply)", + " ~ id = \"**************************************************************************************\" -> (known after apply)", + "0.12"}, + {"", "random_string.some_password: Creation complete after 0s [id=5s80SMs@JJpA8e/h]", + "random_string.some_password: Creation complete after 0s [id=****************]", + "0.12"}, +} + +func TestProcessLine(t *testing.T) { + for _, lineTest := range lineTests { + line := lineTest.line + // Character used to mask sensitive output + var tfmaskChar = "*" + // Pattern representing sensitive output + var tfmaskValuesRegex = "(?i)^.*(oauth|secret|token|password|key|result|id).*$" + // Pattern representing sensitive resource + var tfmaskResourceRegex = "(?i)^(random_id|random_string).*$" + + versionedExpressions := versionedExpressions[lineTest.minorVersion] + + currentResource := lineTest.currentResource + reTfValues := regexp.MustCompile(tfmaskValuesRegex) + reTfResource := regexp.MustCompile(tfmaskResourceRegex) + result := processLine(versionedExpressions, + reTfResource, reTfValues, tfmaskChar, currentResource, line) + result = strings.TrimSuffix(result, "\n") + expectedResult := lineTest.expectedResult + if result != expectedResult { + t.Errorf("Got %s, want %s", result, expectedResult) + } + } +} + +var currentResourceTests = []struct { + currentResource string + line string + expectedResult string +}{ + {"", "-/+ random_string.postgres_admin_password (tainted) (new resource required)", + "random_string.postgres_admin_password", + }, + // existing currentResource should persist: + { + "random_string.postgres_admin_password", + " id: \"VIxvs2TloohI2XtAsHyu68wQvFQQCTOGgsglqC7zKjsnOmUMIMrZ1y5J6ieOIzl-YXiS1_XmVc8J8gb9fIcwIA\" => (forces new resource)", + "random_string.postgres_admin_password", + }, +} + +func TestGetCurrentResource(t *testing.T) { + for _, currentResourceTest := range currentResourceTests { + result := getCurrentResource(versionedExpressions["0.11"], currentResourceTest.currentResource, + currentResourceTest.line) + expectedResult := currentResourceTest.expectedResult + if result != expectedResult { + t.Errorf("Got %s, want %s", result, expectedResult) + } + } +} + +var planStatusTests = []struct { + line string + expectedResult string + minorVersion string +}{ + // tf 0.11 ------------------------------------ + { + "random_id.some_id: Refreshing state... (ID: itILf4x5lqleQV9ZwT2gH-Zg3yuXM8pdUu6VFTX...P5vqUmggDweOoxFMPY5t9thA0SJE2EZIhcHbsQ)", + "random_id.some_id: Refreshing state... (ID: ********************************************************************************)", + "0.11", + }, + // the id value isn't sensitive with random_string.some_password: + { + "random_string.some_password: Refreshing state... (ID: 2iB@@h22@12kA2qE)", + "random_string.some_password: Refreshing state... (ID: 2iB@@h22@12kA2qE)", + "0.11", + }, + // tf 0.12 ------------------------------------ + { + "random_id.some_id: Refreshing state... [id=itILf4x5lqleQV9ZwT2gH-Zg3yuXM8pdUu6VFTX...P5vqUmggDweOoxFMPY5t9thA0SJE2EZIhcHbsQ]", + "random_id.some_id: Refreshing state... [id=********************************************************************************]", + "0.12", + }, + // the id value isn't sensitive with random_string.some_password: + { + "random_string.some_password: Refreshing state... [id=2iB@@h22@12kA2qE]", + "random_string.some_password: Refreshing state... [id=2iB@@h22@12kA2qE]", + "0.12", + }, +} + +func TestPlanStatus(t *testing.T) { + var tfmaskResourceRegex = regexp.MustCompile("(?i)^(random_id).*$") + for _, planStatusTest := range planStatusTests { + result := planStatus( + versionedExpressions[planStatusTest.minorVersion].planStatusRegex, + tfmaskResourceRegex, "*", + planStatusTest.line) + if result != planStatusTest.expectedResult { + t.Errorf("Got %s, want %s", result, planStatusTest.expectedResult) + } + } +} + +var maskValueTests = []struct { + value string + tfmaskChar string + expectedResult string +}{ + {"password", "*", "********"}, + {"password", "@", "@@@@@@@@"}, + {"sensitive", "*", "sensitive"}, + {"computed", "*", "computed"}, + {"