update scans

.github/workflows/sftp-scanner.yaml

@@ -14,6 +14,8 @@ jobs:
       contents: read
 
     steps:
+      - uses: actions/checkout@v4
+
       - name: Configure AWS credentials
         id: creds
         uses: aws-actions/configure-aws-credentials@v4
@@ -22,9 +24,30 @@ jobs:
           aws-region: us-east-1
           role-session-name: actions-sftp-scan
 
-      - name: Scan SFTP and log to s3
+      - name: Install deps
+        run: |
+          sudo apt update && sudo apt install nmap wget -y
+          wget https://github.com/aquasecurity/trivy/releases/download/v0.18.3/trivy_0.18.3_Linux-64bit.deb
+          sudo dpkg -i trivy_0.18.3_Linux-64bit.deb
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.10
+
+      - name: Install utils
         run: |
-          sudo apt update && sudo apt install nmap -y
-          nmap -sV --script ssh2-enum-algos -Pn -p 22 sftp.prod-useast1.heartbeathealth.com > scan.txt
+          python -m pip install semgrep
+          python -m pip install python_graphql_client
+
+      - name: Run scan
+        env: 
+          HBH_SCAN_SECRET: ${{ secrets.HBH_SCAN_SECRET }}
+        run: |
+          python ./scripts/scan.py
+
           date=$(date '+%Y-%m-%d')
-          aws s3 cp scan.txt s3://prod-us-east-1-sftp/server-scans/${date}-scan.txt
+          aws s3 cp scripts/nmapoutput.txt s3://prod-us-east-1-sftp/server-scans/${date}-sftpscan.txt
+          aws s3 cp scripts/trivyoutput.txt s3://prod-us-east-1-sftp/server-scans/${date}-trivyscan.txt
+          aws s3 cp scripts/sgoutput.txt s3://prod-us-east-1-sftp/server-scans/${date}-semgrepscan.txt
+

scripts/scan.py

@@ -0,0 +1,72 @@
+from python_graphql_client import GraphqlClient
+import json
+import os
+
+ghapi = os.environ.get("HBH_SCAN_SECRET")
+
+client = GraphqlClient(endpoint="https://api.github.com/graphql")
+
+def make_query(after_cursor=None):
+    return """
+{
+  repos: search(
+    query: "org:useheartbeat fork:false archived:false"
+    type: REPOSITORY
+    first: 100
+    after: AFTER
+  ) {
+    pageInfo {
+      hasNextPage
+      endCursor
+    }
+    repositoryCount
+    edges {
+      node {
+        ... on Repository {
+          nameWithOwner
+          name
+          pushedAt
+          sshUrl
+        }
+      }
+    }
+  }
+}
+""".replace(
+        "AFTER", '"{}"'.format(after_cursor) if after_cursor else "null"
+    )
+
+
+def fetch_releases(oauth_token):
+    repos = []
+    releases = []
+    repo_names = set()
+    has_next_page = True
+    after_cursor = None
+
+    while has_next_page:
+        data = client.execute(
+            query=make_query(after_cursor),
+            headers={"Authorization": "Bearer {}".format(oauth_token)},
+        )
+        for r in data["data"]["repos"]["edges"]:
+            repo=r["node"]
+            if repo["name"] not in repo_names:
+                repos.append(repo)
+                repo_names.add(repo["name"])
+        has_next_page = data["data"]["repos"]["pageInfo"]["hasNextPage"]
+        after_cursor = data["data"]["repos"]["pageInfo"]["endCursor"]
+    return repos
+
+# fetch or update all non-archived repos
+repos=fetch_releases(ghapi)
+os.system("mkdir allcode")
+for r in repos:
+  if os.path.exists("allcode/"+r["name"]):
+    os.system("cd allcode/"+r["name"]+" && git pull")
+  else:
+    os.system("cd allcode && git clone "+r["sshUrl"])
+# run various scans. requires these utilities to be installed
+os.system("cd allcode && trivy fs . > ../trivyoutput.txt")
+os.system("cd allcode && semgrep scan . > ../sgoutput.txt")
+os.system("nmap -sV --script ssh2-enum-algos -Pn -p 22 sftp.prod-useast1.heartbeathealth.com > nmapoutput.txt")