Import scanner violations into Cloud SQL (forseti-security#239)

.gitignore

@@ -28,3 +28,7 @@ deployment-templates/*.yaml
 build/
 dist/
 out/
+
+# Coverage
+.coverage
+htmlcov/

google/cloud/security/common/data_access/dao.py

@@ -41,6 +41,8 @@
     'raw_project_iam_policies':
         create_tables.CREATE_RAW_PROJECT_IAM_POLICIES_TABLE,
     'raw_org_iam_policies': create_tables.CREATE_RAW_ORG_IAM_POLICIES_TABLE,
+
+    'violations': create_tables.CREATE_VIOLATIONS_TABLE,
 }
 
 SNAPSHOT_FILTER_CLAUSE = ' where status in ({})'
@@ -49,9 +51,6 @@
 class Dao(_db_connector.DbConnector):
     """Data access object (DAO)."""
 
-    def __init__(self):
-        super(Dao, self).__init__()
-
     def _create_snapshot_table(self, resource_name, timestamp):
         """Creates a snapshot table.
 

google/cloud/security/common/data_access/group_dao.py

@@ -25,9 +25,6 @@
 class GroupDao(dao.Dao):
     """Data access object (DAO) for Groups."""
 
-    def __init__(self):
-        super(GroupDao, self).__init__()
-
     def get_group_users(self, resource_name, timestamp):
         """Get the group members who are users.
 

google/cloud/security/common/data_access/organization_dao.py

@@ -35,9 +35,6 @@
 class OrganizationDao(dao.Dao):
     """Data access object (DAO) for Organizations."""
 
-    def __init__(self):
-        super(OrganizationDao, self).__init__()
-
     def get_organizations(self, resource_name, timestamp):
         """Get organizations from snapshot table.
 

google/cloud/security/common/data_access/project_dao.py

@@ -36,9 +36,6 @@
 class ProjectDao(dao.Dao):
     """Data access object (DAO)."""
 
-    def __init__(self):
-        super(ProjectDao, self).__init__()
-
     def get_project_numbers(self, resource_name, timestamp):
         """Select the project numbers from a projects snapshot table.
 

google/cloud/security/common/data_access/sql_queries/create_tables.py

@@ -20,8 +20,8 @@
         `project_number` bigint(20) NOT NULL,
         `project_id` varchar(255) NOT NULL,
         `project_name` varchar(255) DEFAULT NULL,
-        `lifecycle_state` enum('ACTIVE','DELETE_REQUESTED',
-            'DELETE_IN_PROGRESS','DELETED') DEFAULT NULL,
+        `lifecycle_state` enum('LIFECYCLE_STATE_UNSPECIFIED','ACTIVE',
+            'DELETE_REQUESTED','DELETED') NOT NULL,
         `parent_type` varchar(255) DEFAULT NULL,
         `parent_id` varchar(255) DEFAULT NULL,
         `raw_project` json DEFAULT NULL,
@@ -58,8 +58,8 @@
         `org_id` bigint(20) unsigned NOT NULL,
         `name` varchar(255) NOT NULL,
         `display_name` varchar(255) DEFAULT NULL,
-        `lifecycle_state` enum('ACTIVE','DELETE_REQUESTED',
-            'DELETED','LIFECYCLE_STATE_UNSPECIFIED') DEFAULT NULL,
+        `lifecycle_state` enum('LIFECYCLE_STATE_UNSPECIFIED','ACTIVE',
+            'DELETE_REQUESTED', 'DELETED') NOT NULL,
         `raw_org` json DEFAULT NULL,
         `creation_time` datetime DEFAULT NULL,
         PRIMARY KEY (`org_id`)
@@ -115,3 +115,17 @@
 """
 
 # TODO: Add a RAW_GROUP_MEMBERS_TABLE.
+
+CREATE_VIOLATIONS_TABLE = """
+    CREATE TABLE `{0}` (
+        `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+        `resource_type` varchar(255) NOT NULL,
+        `resource_id` varchar(255) NOT NULL,
+        `rule_name` varchar(255) DEFAULT NULL,
+        `rule_index` int DEFAULT NULL,
+        `violation_type` enum('UNSPECIFIED','ADDED','REMOVED') NOT NULL,
+        `role` varchar(255) DEFAULT NULL,
+        `member` varchar(255) DEFAULT NULL,
+        PRIMARY KEY (`id`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+"""

google/cloud/security/common/data_access/sql_queries/load_data.py

@@ -19,3 +19,10 @@
     INTO TABLE {1} FIELDS TERMINATED BY ','
     ({2});
 """
+
+INSERT_VIOLATION = """
+    INSERT INTO {0}
+    (resource_type, resource_id, rule_name, rule_index,
+     violation_type, role, member)
+    VALUES (%s, %s, %s, %s, %s, %s, %s)
+"""

google/cloud/security/common/data_access/violation_dao.py

@@ -0,0 +1,123 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides the data access object (DAO) for Organizations."""
+
+import MySQLdb
+
+from google.cloud.security.common.data_access import dao
+from google.cloud.security.common.data_access import errors as db_errors
+from google.cloud.security.common.data_access.sql_queries import load_data
+from google.cloud.security.common.util import log_util
+
+LOGGER = log_util.get_logger(__name__)
+
+
+class ViolationDao(dao.Dao):
+    """Data access object (DAO) for rule violations."""
+
+    RESOURCE_NAME = 'violations'
+
+    def insert_violations(self, violations, snapshot_timestamp=None):
+        """Import violations into database.
+
+        Args:
+            violations: An iterator of RuleViolations.
+            snapshot_timestamp: The snapshot timestamp to associate these
+                violations with.
+
+        Return:
+            A tuple of (int, list) containing the count of inserted rows and
+            a list of violations that encountered an error during insert.
+
+        Raise:
+            MySQLError if snapshot table could not be created.
+        """
+
+        try:
+            # Make sure to have a reasonable timestamp to use.
+            if not snapshot_timestamp:
+                snapshot_timestamp = self.get_latest_snapshot_timestamp(
+                    ('PARTIAL_SUCCESS', 'SUCCESS'))
+
+            # Create the violations snapshot table.
+            snapshot_table = self._create_snapshot_table(
+                self.RESOURCE_NAME, snapshot_timestamp)
+        except MySQLdb.Error, e:
+            raise db_errors.MySQLError(self.RESOURCE_NAME, e)
+
+        inserted_rows = 0
+        violation_errors = []
+        for violation in violations:
+            for formatted_violation in _format_violation(violation):
+                try:
+                    self.execute_sql_with_commit(
+                        self.RESOURCE_NAME,
+                        load_data.INSERT_VIOLATION.format(snapshot_table),
+                        formatted_violation)
+                    inserted_rows += 1
+                except MySQLdb.Error, e:
+                    LOGGER.error('Unable to insert violation %s due to %s',
+                                 formatted_violation, e)
+                    violation_errors.append(formatted_violation)
+
+        return (inserted_rows, violation_errors)
+
+
+def _format_violation(violation):
+    """Format the violation data into a tuple.
+
+    Also flattens the RuleViolation, since it consists of the resource,
+    rule, and members that don't meet the rule criteria.
+
+    Various properties of RuleViolation may also have values that exceed the
+    declared column length, so truncate as necessary to prevent MySQL errors.
+
+    Args:
+        violation: The RuleViolation.
+
+    Yields:
+        A tuple of the rule violation properties.
+    """
+
+    resource_type = violation.resource_type
+    if resource_type:
+        resource_type = resource_type[:255]
+
+    resource_id = violation.resource_id
+    if resource_id:
+        resource_id = str(resource_id)[:255]
+
+    rule_name = violation.rule_name
+    if rule_name:
+        rule_name = rule_name[:255]
+
+    role = violation.role
+    if role:
+        role = role[:255]
+
+    iam_members = violation.members
+    if iam_members:
+        members = [str(iam_member)[:255] for iam_member in iam_members]
+    else:
+        members = []
+
+    for member in members:
+        yield (resource_type,
+               resource_id,
+               rule_name,
+               violation.rule_index,
+               violation.violation_type,
+               role,
+               member)

google/cloud/security/common/email_templates/scanner_summary.jinja

@@ -21,59 +21,10 @@
   line-height: 16px;
 }
 
-body {
-  background-color: #eee;
-}
-
-.content {
-  background-color: #fff;
-  margin-top: 40px;
-  padding: 20px;
-  margin-left: 20%;
-  margin-right: 20%;
-}
-
-.message {
-  margin: 10px 10px 15px 10px;
-}
-
-.summary .header {
-  font-weight: bold;
-  font-size: 16px;
-  margin-bottom: 5px;
-}
-
-.summary .footer {
-  font-style: italic;
-  font-size: 14px;
-  margin-bottom: 5px;
-}
-
-.diff-summary {
-  font-size: 14px;
-}
-
-.diff-summary > * {
-  margin: 5px 0;
-}
-
 a, a:visited {
   color: #1082d9;
 }
 
-.summary {
-  margin: 10px 10px;
-}
-
-.resource {
-  margin: 20px 10px;
-}
-
-.resource-violations {
-  border-collapse: collapse;
-  border-spacing: 0
-}
-
 .resource-violations tr > td {
   border: 1px solid #ddd;
   border: 1px solid #ddd;
@@ -83,45 +34,29 @@ a, a:visited {
   padding: 4px;
 }
 
-.resource-violations tr:first-child > th {
-  border-bottom: 1px solid #ddd;
-}
-
-.resource-violations tr:last-child > td {
-  border-bottom: 1px solid #ddd;
-}
-
-.resource-violations th {
+th {
   font-size: 16px;
   font-weight: bold;
   padding: 4px;
   text-align: left;
 }
 
-.resource-violations th.left {
-  width: 80%;
-}
-
-.resource-violations th.left > a {
-  font-weight: normal;
-}
-
-.resource-violations td.numeric {
-  padding-left: 18px;
+td {
+  padding: 4px;
 }
 
 </style>
 </head>
-<body>
-<div class="content">
-  <div class="message">
+<body style="background-color: #eee;">
+<div style="background-color: #fff; margin-top: 40px; padding: 20px; margin-left: 20%; margin-right: 20%;">
+  <div style="margin: 10px 10px 15px 10px; font-size: 14px;">
     Forseti Security found some issues during the scan on {{ scan_date }}.
   </div>
-  <div class="summary">
-    <div class="header">
+  <div style="margin: 10px 10px;">
+    <div style="font-weight: bold; font-size: 16px; margin-bottom: 5px;">
       Resource Violations:
     </div>
-    <div class="diff-summary">
+    <div style="font-size: 14px;">
     {% for (resource_type, summary) in resource_summaries.iteritems() %}
     <div>{{ summary['pluralized_resource_type'] }}:
          {{ summary['violations']|length }}
@@ -130,23 +65,34 @@ a, a:visited {
     </div>
     {% endfor %}
     </div>
-    <div class="footer">
+    <div style="margin: 10px 0;">
+      <div style="font-size: 14px;">
+          <strong>Errors:</strong> {{ violation_errors|length }}
+      </div>
+    </div>
+    <div style="font-style: italic; font-size: 14px; margin-bottom: 5px;">
       See attached CSV for full violation details.
     </div>
   </div>
 
-  <div class="details">
+  <br/>
+
+  <div>
     {% for (resource_type, summary) in resource_summaries.iteritems() %}
-    <div class="resource">
-      <table class="resource-violations" width="100%">
+    <div style="margin: 20px 10px;">
+      <table style="border-collapse: collapse; border-spacing: 0;" width="100%">
         <tr>
-          <th class="left">{{ resource_type.title() }}</th>
-          <th># Violations</th>
+          <th style="border-bottom: 1px solid #ddd; width: 40%;">
+            {{ resource_type.title() }}
+          </th>
+          <th style="border-bottom: 1px solid #ddd; width: 40%;">
+            # Violations
+          </th>
         </tr>
         {% for resource_id, count in summary['violations'].iteritems() %}
         <tr>
-          <td>{{ resource_id}}</td>
-          <td class="numeric">{{ count }}</td>
+          <td style="padding: 4px;">{{ resource_id}}</td>
+          <td style="padding: 4px;">{{ count }}</td>
         </tr>
         {% endfor %}
       </table>

google/cloud/security/common/gcp_type/iam_policy.py

@@ -199,7 +199,7 @@ def __hash__(self):
 
     def __repr__(self):
         """String representation of IamPolicyMember."""
-        return 'IamMember <type={}, name={}>'.format(self.type, self.name)
+        return '%s:%s' % (self.type, self.name)
 
     def _member_type_exists(self, member_type):
         """Determine if the member type exists in valid member types."""