Add support to parse Terraform plan and state files

config/terraform.spc

@@ -1,8 +1,9 @@
 connection "terraform" {
   plugin = "terraform"
 
-  # Paths is a list of locations to search for Terraform configuration files
-  # Paths can be configured with a local directory, a remote Git repository URL, or an S3 bucket URL
+  # Configuration file paths is a list of locations to search for Terraform configuration files
+  # Similarly, Plan File Paths is a list of locations to search for Terraform plan files
+  # Configuration file paths can be configured with a local directory, a remote Git repository URL, or an S3 bucket URL
   # Wildcard based searches are supported, including recursive searches
   # Local paths are resolved relative to the current working directory (CWD)
 
@@ -18,5 +19,6 @@ connection "terraform" {
   # the CWD will be matched, which may cause errors if incompatible file types exist
 
   # Defaults to CWD
-  paths = [ "*.tf" ]
+  configuration_file_paths = ["*.tf"]
+  plan_file_paths          = ["tfplan.json", "*.tfplan.json"]
 }

docs/index.md

@@ -16,6 +16,44 @@ A Terraform configuration file is used to declare resources, variables, modules,
 
 [Steampipe](https://steampipe.io) is an open source CLI to instantly query data using SQL.
 
+The plugin supports scanning Terraform configuration files from various sources (e.g., [Local files](#configuring-local-file-paths), [Git](#configuring-remote-git-repository-urls), [S3](#configuring-s3-urls) etc.), and [parsing Terraform plans](#scanning-terraform-plan) as well.
+
+## Documentation
+
+- **[Table definitions & examples →](/plugins/turbot/terraform/tables)**
+
+## Get Started
+
+### Install
+
+Download and install the latest Terraform plugin:
+
+```bash
+steampipe plugin install terraform
+```
+
+### Configuration
+
+Installing the latest terraform plugin will create a config file (`~/.steampipe/config/terraform.spc`) with a single connection named `terraform`:
+
+```hcl
+connection "terraform" {
+  plugin = "terraform"
+
+  configuration_file_paths = ["*.tf"]
+}
+```
+
+For a full list of configuration arguments, please see the [default configuration file](https://github.com/turbot/steampipe-plugin-terraform/blob/main/config/terraform.spc).
+
+### Run a Query
+
+Run steampipe:
+
+```shell
+steampipe query
+```
+
 Query all resources in your Terraform files:
 
 ```sql
@@ -27,7 +65,7 @@ from
   terraform_resource;
 ```
 
-```
+```sh
 > select name, type, jsonb_pretty(arguments) as args from terraform_resource;
 +------------+----------------+--------------------------------------------+
 | name       | type           | args                                       |
@@ -57,70 +95,30 @@ from
 +------------+----------------+--------------------------------------------+
 ```
 
-## Documentation
+## Configuring Paths
 
-- **[Table definitions & examples →](/plugins/turbot/terraform/tables)**
+The plugin requires a list of locations to search for the Terraform configuration files. Paths can be configured with [Local files](#configuring-local-file-paths), [Git URLs](#configuring-remote-git-repository-urls), [S3 URLs](#configuring-s3-urls) etc.
 
-## Get started
-
-### Install
-
-Download and install the latest Terraform plugin:
-
-```bash
-steampipe plugin install terraform
-```
-
-### Credentials
-
-No credentials are required.
-
-### Configuration
-
-Installing the latest terraform plugin will create a config file (`~/.steampipe/config/terraform.spc`) with a single connection named `terraform`:
+**Note:** Local file paths are resolved relative to the current working directory (CWD).
 
 ```hcl
 connection "terraform" {
   plugin = "terraform"
 
-  # Paths is a list of locations to search for Terraform configuration files
-  # Paths can be configured with a local directory, a remote Git repository URL, or an S3 bucket URL
-  # Wildcard based searches are supported, including recursive searches
-  # Local paths are resolved relative to the current working directory (CWD)
-
-  # For example:
-  #  - "*.tf" matches all Terraform configuration files in the CWD
-  #  - "**/*.tf" matches all Terraform configuration files in the CWD and all sub-directories
-  #  - "../*.tf" matches all Terraform configuration files in the CWD's parent directory
-  #  - "steampipe*.tf" matches all Terraform configuration files starting with "steampipe" in the CWD
-  #  - "/path/to/dir/*.tf" matches all Terraform configuration files in a specific directory
-  #  - "/path/to/dir/main.tf" matches a specific file
-
-  # If paths includes "*", all files (including non-Terraform configuration files) in
-  # the CWD will be matched, which may cause errors if incompatible file types exist
-
-  # Defaults to CWD
-  paths = [ "*.tf" ]
+  configuration_file_paths = [
+    "terraform_test.tf",
+    "github.com/turbot/steampipe-plugin-aws//aws-test/tests/aws_acm_certificate//variables.tf"
+  ]
 }
 ```
 
-### Supported Path Formats
-
-The `paths` config argument is flexible and can search for Terraform configuration files from several different sources, e.g., local directory paths, Git, S3.
-
-The following sources are supported:
-
-- [Local files](#configuring-local-file-paths)
-- [Remote Git repositories](#configuring-remote-git-repository-urls)
-- [S3](#configuring-s3-urls)
-
 Paths may [include wildcards](https://pkg.go.dev/path/filepath#Match) and support `**` for recursive matching. For example:
 
 ```hcl
 connection "terraform" {
   plugin = "terraform"
 
-  paths = [
+  configuration_file_paths = [
     "*.tf",
     "~/*.tf",
     "github.com/turbot/steampipe-plugin-aws//aws-test/tests/aws_acm_certificate//*.tf",
@@ -134,7 +132,7 @@ connection "terraform" {
 
 **Note**: If any path matches on `*` without `.tf`, all files (including non-Terraform configuration files) in the directory will be matched, which may cause errors if incompatible file types exist.
 
-#### Configuring Local File Paths
+### Configuring Local File Paths
 
 You can define a list of local directory paths to search for terraform files. Paths are resolved relative to the current working directory. For example:
 
@@ -151,11 +149,11 @@ You can define a list of local directory paths to search for terraform files. Pa
 connection "terraform" {
   plugin = "terraform"
 
-  paths = [ "*.tf", "~/*.tf", "/path/to/dir/main.tf" ]
+  configuration_file_paths = [ "*.tf", "~/*.tf", "/path/to/dir/main.tf" ]
 }
 ```
 
-#### Configuring Remote Git Repository URLs
+### Configuring Remote Git Repository URLs
 
 You can also configure `paths` with any Git remote repository URLs, e.g., GitHub, BitBucket, GitLab. The plugin will then attempt to retrieve any Terraform configuration files from the remote repositories.
 
@@ -176,7 +174,7 @@ You can specify a subdirectory after a double-slash (`//`) if you want to downlo
 connection "terraform" {
   plugin = "terraform"
 
-  paths = [ "github.com/turbot/steampipe-plugin-aws//aws-test/tests/aws_acm_certificate//*.tf" ]
+  configuration_file_paths = [ "github.com/turbot/steampipe-plugin-aws//aws-test/tests/aws_acm_certificate//*.tf" ]
 }
 ```
 
@@ -186,7 +184,7 @@ Similarly, you can define a list of GitLab and BitBucket URLs to search for Terr
 connection "terraform" {
   plugin = "terraform"
 
-  paths = [
+  configuration_file_paths = [
     "github.com/turbot/steampipe-plugin-aws//**/*.tf",
     "github.com/hashicorp/terraform-guides//infrastructure-as-code//**/*.tf",
     "bitbucket.org/benturrell/terraform-arcgis-portal//modules/shared//*.tf",
@@ -197,11 +195,11 @@ connection "terraform" {
 }
 ```
 
-#### Configuring S3 URLs
+### Configuring S3 URLs
 
 You can also query all Terraform configuration files stored inside an S3 bucket (public or private) using the bucket URL.
 
-##### Accessing a Private Bucket
+#### Accessing a Private Bucket
 
 In order to access your files in a private S3 bucket, you will need to configure your credentials. You can use your configured AWS profile from local `~/.aws/config`, or pass the credentials using the standard AWS environment variables, e.g., `AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`.
 
@@ -215,7 +213,7 @@ You can also authenticate your request by setting the AWS profile and region in
 connection "terraform" {
   plugin = "terraform"
 
-  paths = [
+  configuration_file_paths = [
     "s3::https://bucket-2.s3.us-east-1.amazonaws.com//*.tf?aws_profile=<AWS_PROFILE>",
     "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//*.tf?aws_profile=<AWS_PROFILE>"
   ]
@@ -242,21 +240,14 @@ If the bucket is in another AWS account, the bucket policy will need to grant ac
       "Principal": {
         "AWS": "arn:aws:iam::123456789012:user/YOUR_USER"
       },
-      "Action": [
-        "s3:ListBucket",
-        "s3:GetObject",
-        "s3:GetObjectVersion"
-      ],
-      "Resource": [
-        "arn:aws:s3:::test-bucket1",
-        "arn:aws:s3:::test-bucket1/*"
-      ]
+      "Action": ["s3:ListBucket", "s3:GetObject", "s3:GetObjectVersion"],
+      "Resource": ["arn:aws:s3:::test-bucket1", "arn:aws:s3:::test-bucket1/*"]
     }
   ]
 }
 ```
 
-##### Accessing a Public Bucket
+#### Accessing a Public Bucket
 
 Public access granted to buckets and objects through ACLs and bucket policies allows any user access to data in the bucket. We do not recommend making S3 buckets public, but if there are specific objects you'd like to make public, please see [How can I grant public read access to some objects in my Amazon S3 bucket?](https://aws.amazon.com/premiumsupport/knowledge-center/read-access-objects-s3-bucket/).
 
@@ -266,14 +257,44 @@ You can query any public S3 bucket directly using the URL without passing creden
 connection "terraform" {
   plugin = "terraform"
 
-  paths = [
+  configuration_file_paths = [
     "s3::https://bucket-1.s3.us-east-1.amazonaws.com/test_folder//*.tf",
     "s3::https://bucket-2.s3.us-east-1.amazonaws.com/test_folder//**/*.tf"
   ]
 }
 ```
 
-## Get involved
+## Scanning Terraform Plan
+
+The plugin supports scanning the Terraform plans given in JSON, and allows the users to query them using Steampipe.
+
+**Note:** The plugin only scans the resource changes from the Terraform plan.
+
+To get the Terraform plan in JSON format simply follow the below steps:
+
+- Run `terraform plan` with `-out` flag to store the generated plan to the given filename. Terraform will allow any filename for the plan file, but a typical convention is to name it `tfplan`.
+
+```shell
+terraform plan -out=tfplan
+```
+
+- Run `terraform show` command with `-json` flag to get the plan in JSON format, and store the output in a file.
+
+```shell
+terraform show -json tfplan > tfplan.json
+```
+
+- And, finally add the path `tfplan.json` to the `plan_file_paths` argument in the config to read the plan using Steampipe.
+
+```hcl
+connection "terraform" {
+  plugin = "terraform"
+
+  plan_file_paths = ["/path/to/tfplan.json"]
+}
+```
+
+## Get Involved
 
 - Open source: https://github.com/turbot/steampipe-plugin-terraform
 - Community: [Join #steampipe on Slack →](https://turbot.com/community/join)

go.mod

@@ -61,6 +61,7 @@ require (
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/terraform-json v0.13.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
 	github.com/iancoleman/strcase v0.2.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect

terraform/connection_config.go

@@ -6,14 +6,24 @@ import (
 )
 
 type terraformConfig struct {
-	Paths []string `cty:"paths" steampipe:"watch"`
+	ConfigurationFilePaths []string `cty:"configuration_file_paths" steampipe:"watch"`
+	Paths                  []string `cty:"paths" steampipe:"watch"`
+	PlanFilePaths          []string `cty:"plan_file_paths" steampipe:"watch"`
 }
 
 var ConfigSchema = map[string]*schema.Attribute{
+	"configuration_file_paths": {
+		Type: schema.TypeList,
+		Elem: &schema.Attribute{Type: schema.TypeString},
+	},
 	"paths": {
 		Type: schema.TypeList,
 		Elem: &schema.Attribute{Type: schema.TypeString},
 	},
+	"plan_file_paths": {
+		Type: schema.TypeList,
+		Elem: &schema.Attribute{Type: schema.TypeString},
+	},
 }
 
 func ConfigInstance() interface{} {

terraform/table_terraform_data_source.go

@@ -107,7 +107,13 @@ type terraformDataSource struct {
 func listDataSources(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
 	// The path comes from a parent hydate, defaulting to the config paths or
 	// available by the optional key column
-	path := h.Item.(filePath).Path
+	data := h.Item.(filePath)
+	path := data.Path
+
+	// Return if the path is a TF plan path
+	if data.IsTFPlanFilePath {
+		return nil, nil
+	}
 
 	combinedParser, err := Parser()
 	if err != nil {

terraform/table_terraform_local.go

@@ -66,7 +66,13 @@ type terraformLocal struct {
 func listLocals(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
 	// The path comes from a parent hydate, defaulting to the config paths or
 	// available by the optional key column
-	path := h.Item.(filePath).Path
+	data := h.Item.(filePath)
+	path := data.Path
+
+	// Return if the path is a TF plan path
+	if data.IsTFPlanFilePath {
+		return nil, nil
+	}
 
 	combinedParser, err := Parser()
 	if err != nil {

terraform/table_terraform_module.go

@@ -112,9 +112,15 @@ type terraformModule struct {
 }
 
 func listModules(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
-	// The path comes from a parent hydrate, defaulting to the config paths
-	// or available by the optional key column
-	path := h.Item.(filePath).Path
+	// The path comes from a parent hydate, defaulting to the config paths or
+	// available by the optional key column
+	data := h.Item.(filePath)
+	path := data.Path
+
+	// Return if the path is a TF plan path
+	if data.IsTFPlanFilePath {
+		return nil, nil
+	}
 
 	combinedParser, err := Parser()
 	if err != nil {

terraform/table_terraform_output.go

@@ -89,7 +89,13 @@ type terraformOutput struct {
 func listOutputs(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
 	// The path comes from a parent hydate, defaulting to the config paths or
 	// available by the optional key column
-	path := h.Item.(filePath).Path
+	data := h.Item.(filePath)
+	path := data.Path
+
+	// Return if the path is a TF plan path
+	if data.IsTFPlanFilePath {
+		return nil, nil
+	}
 
 	combinedParser, err := Parser()
 	if err != nil {