Merge branch 'google:master' into master

.github/workflows/publish-to-pypi.yaml

@@ -44,7 +44,7 @@ jobs:
         build
         --sdist --wheel --outdir dist/ .
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@e53eb8b103ffcb59469888563dc324e3c8ba6f06 # v1.8.12
+      uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
         packages_dir: dist/

.github/workflows/scorecards.yml

@@ -50,6 +50,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@928ff8c822d966a999092a6a35e32177899afb7c # v2.24.6
+        uses: github/codeql-action/upload-sarif@e56cfd0877b4826be144d11aa31e6c64a55828e9 # v2.24.7
         with:
           sarif_file: results.sarif

deployment/build-and-stage.yaml

@@ -123,6 +123,20 @@ steps:
   args: ['push', '--all-tags', 'gcr.io/oss-vdb/alpine-cve-convert']
   waitFor: ['build-alpine-cve-convert', 'cloud-build-queue']
 
+- name: 'gcr.io/cloud-builders/docker'
+  entrypoint: 'bash'
+  args: ['-c', 'docker pull gcr.io/oss-vdb/debian-cve-convert:latest || exit 0']
+  id: 'pull-debian-cve-convert'
+  waitFor: ['setup']
+- name: gcr.io/cloud-builders/docker
+  args: ['build', '-t', 'gcr.io/oss-vdb/debian-cve-convert:latest', '-t', 'gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA', '-f', 'cmd/debian/Dockerfile', '--cache-from', 'gcr.io/oss-vdb/debian-cve-convert:latest', '--pull', '.']
+  dir: 'vulnfeeds'
+  id: 'build-debian-cve-convert'
+  waitFor: ['pull-debian-cve-convert']
+- name: gcr.io/cloud-builders/docker
+  args: ['push', '--all-tags', 'gcr.io/oss-vdb/debian-cve-convert']
+  waitFor: ['build-debian-cve-convert', 'cloud-build-queue']
+
 - name: 'gcr.io/cloud-builders/docker'
   entrypoint: 'bash'
   args: ['-c', 'docker pull gcr.io/oss-vdb/combine-to-osv:latest || exit 0']
@@ -263,6 +277,7 @@ steps:
      debian-convert=gcr.io/oss-vdb/debian-convert:$COMMIT_SHA,\
      combine-to-osv=gcr.io/oss-vdb/combine-to-osv:$COMMIT_SHA,\
      alpine-cve-convert=gcr.io/oss-vdb/alpine-cve-convert:$COMMIT_SHA,\
+     debian-cve-convert=gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA,\
      debian-copyright-mirror=gcr.io/oss-vdb/debian-copyright-mirror:$COMMIT_SHA,\
      cpe-repo-gen=gcr.io/oss-vdb/cpe-repo-gen:$COMMIT_SHA,\
      nvd-cve-osv=gcr.io/oss-vdb/nvd-cve-osv:$COMMIT_SHA,\
@@ -312,6 +327,7 @@ images:
 - 'gcr.io/oss-vdb/alias-computation:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/cron:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/alpine-cve-convert:$COMMIT_SHA'
+- 'gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/combine-to-osv:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/indexer:$COMMIT_SHA'
 - 'gcr.io/oss-vdb/debian-convert:$COMMIT_SHA'

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/debian-cve-convert.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: debian-cve-convert
+spec:
+  schedule: "0 */1 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 3600
+      template:
+        spec:
+          containers:
+          - name: debian-cve-convert
+            image: debian-cve-convert
+            imagePullPolicy: Always
+            env:
+            - name: GOOGLE_CLOUD_PROJECT
+              value: oss-vdb-test
+            - name: OUTPUT_GCS_BUCKET
+              value: osv-test-cve-osv-conversion
+            securityContext:
+              privileged: true
+            resources:
+              requests:
+                cpu: 1
+                memory: "1G"
+              limits:
+                cpu: 1
+                memory: "2G"
+          restartPolicy: OnFailure
+          volumes:
+            - name: "ssd"
+              hostPath:
+                path: "/mnt/disks/ssd0"

deployment/clouddeploy/gke-workers/environments/oss-vdb-test/kustomization.yaml

@@ -1,5 +1,6 @@
 resources:
 - ../../base
+- debian-cve-convert.yaml
 patches:
 - path: workers.yaml
 - path: scaler.yaml

deployment/clouddeploy/osv-api/clouddeploy.yaml

@@ -5,12 +5,21 @@ metadata:
 description: OSV API pipeline
 serialPipeline:
   stages:
-  - targetId: staging-api
+  - targetId: staging-api-multi
     profiles: [ staging ]
-  - targetId: production-api
+  - targetId: production-api-multi
     profiles: [ prod ]
 ---
 
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: staging-api-multi
+description: multi-target oss-vdb-test API instances
+multiTarget:
+  targetIds: [staging-api, staging-api-batch]
+---
+
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
@@ -25,6 +34,32 @@ executionConfigs:
   serviceAccount: [email protected]
 ---
 
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: staging-api-batch
+description: oss-vdb-test API batch query instance
+run:
+  location: projects/oss-vdb-test/locations/us-central1
+deployParameters:
+  serviceName: "osv-grpc-backend-batch"
+  containerConcurrency: "1"
+executionConfigs:
+- usages:
+  - RENDER
+  - DEPLOY
+  serviceAccount: [email protected]
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: production-api-multi
+description: multi-target oss-vdb API instances
+multiTarget:
+  targetIds: [production-api, production-api-batch]
+---
+
 apiVersion: deploy.cloud.google.com/v1
 kind: Target
 metadata:
@@ -37,3 +72,20 @@ executionConfigs:
   - RENDER
   - DEPLOY
   serviceAccount: [email protected]
+---
+
+apiVersion: deploy.cloud.google.com/v1
+kind: Target
+metadata:
+  name: production-api-batch
+description: oss-vdb API batch query instance
+run:
+  location: projects/oss-vdb/locations/us-central1
+deployParameters:
+  serviceName: "osv-grpc-backend-batch"
+  containerConcurrency: "1"
+executionConfigs:
+- usages:
+  - RENDER
+  - DEPLOY
+  serviceAccount: [email protected]

deployment/clouddeploy/osv-api/run.yaml

@@ -1,9 +1,12 @@
 apiVersion: serving.knative.dev/v1
 kind: Service
 metadata:
-  name: osv-grpc-backend
+  name: osv-grpc-backend # from-param: ${serviceName}
 spec:
   template:
+    metadata:
+      annotations:
+        autoscaling.knative.dev/maxScale: '300'
     spec:
       containers:
       - image: osv-server
@@ -13,11 +16,13 @@ spec:
         startupProbe:
           grpc:
             service: osv.v1.OSV
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
         livenessProbe:
           grpc:
             service: osv.v1.OSV
           timeoutSeconds: 5
           failureThreshold: 3
           periodSeconds: 10
       timeoutSeconds: 60
-      containerConcurrency: 10
+      containerConcurrency: 5 # from-param: ${containerConcurrency}

deployment/deploy-prod.yaml

@@ -21,7 +21,7 @@ steps:
 - name: gcr.io/cloud-builders/gcloud
   args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=gke-workers', '--to-target=production-workers', '--annotations=tag=$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
-  args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=osv-api', '--to-target=production-api', '--annotations=tag=$TAG_NAME']
+  args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=osv-api', '--to-target=production-api-multi', '--annotations=tag=$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['deploy', 'releases', 'promote', '--quiet', '--release=osv-$SHORT_SHA', '--region=us-central1', '--delivery-pipeline=gke-indexer', '--to-target=production-indexer', '--annotations=tag=$TAG_NAME']
 
@@ -36,6 +36,8 @@ steps:
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/exporter:$COMMIT_SHA', 'gcr.io/oss-vdb/exporter:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/alpine-cve-convert:$COMMIT_SHA', 'gcr.io/oss-vdb/alpine-cve-convert:$TAG_NAME']
+- name: gcr.io/cloud-builders/gcloud
+  args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/debian-cve-convert:$COMMIT_SHA', 'gcr.io/oss-vdb/debian-cve-convert:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud
   args: ['container', 'images', 'add-tag', '--quiet', 'gcr.io/oss-vdb/combine-to-osv:$COMMIT_SHA', 'gcr.io/oss-vdb/combine-to-osv:$TAG_NAME']
 - name: gcr.io/cloud-builders/gcloud

deployment/terraform/environments/oss-vdb-test/api/api_config.tftpl

@@ -31,3 +31,6 @@ backend:
     - selector: "*"
       address: ${backend_url}
       deadline: 60
+    - selector: "osv.v1.OSV.QueryAffectedBatch"
+      address: ${backend_batch_url}
+      deadline: 60

deployment/terraform/environments/oss-vdb/api/api_config.tftpl

@@ -31,6 +31,9 @@ backend:
     - selector: "*"
       address: ${backend_url}
       deadline: 60
+    - selector: "osv.v1.OSV.QueryAffectedBatch"
+      address: ${backend_batch_url}
+      deadline: 60
 documentation:
   summary: >
     OSV is a vulnerability database for open source projects. It exposes an API

deployment/terraform/modules/osv/osv_api.tf

@@ -29,6 +29,29 @@ resource "google_cloud_run_service" "api_backend" {
   }
 }
 
+resource "google_cloud_run_v2_service" "api_backend_batch" {
+  project  = var.project_id
+  name     = "osv-grpc-backend-batch"
+  location = "us-central1"
+
+  template {
+    containers {
+      image = "us-docker.pkg.dev/cloudrun/container/hello:latest" # Placeholder image.
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      # To be managed by Cloud Deploy.
+      template,
+      traffic,
+      labels,
+      client
+    ]
+    prevent_destroy = true
+  }
+}
+
 variable "_api_descriptor_file" {
   # This isn't actually sensitive, but it's outputted as a massive base64 string which really floods the plan output.
   sensitive = true
@@ -42,8 +65,9 @@ resource "google_endpoints_service" "grpc_service" {
   grpc_config = templatefile(
     "api/api_config.tftpl",
     {
-      service_name = var.api_url,
-      backend_url  = replace(google_cloud_run_service.api_backend.status[0].url, "https://", "grpcs://")
+      service_name      = var.api_url,
+      backend_url       = replace(google_cloud_run_service.api_backend.status[0].url, "https://", "grpcs://")
+      backend_batch_url = replace(google_cloud_run_v2_service.api_backend_batch.uri, "https://", "grpcs://")
   })
   protoc_output_base64 = filebase64(var._api_descriptor_file)
 }

docker/alias/alias_computation.py

@@ -87,7 +87,9 @@ def main():
   AliasGroups and creating new AliasGroups for un-computed bugs."""
 
   # Query for all bugs that have aliases.
-  bugs = osv.Bug.query(osv.Bug.aliases != '')
+  # Use (> '' OR < '') instead of (!= '') / (> '') to de-duplicate results
+  # and avoid datastore emulator problems, see issue #2093
+  bugs = osv.Bug.query(ndb.OR(osv.Bug.aliases > '', osv.Bug.aliases < ''))
   all_alias_group = osv.AliasGroup.query()
   allow_list = {
       allow_entry.bug_id for allow_entry in osv.AliasAllowListEntry.query()

docker/exporter/exporter.py

@@ -24,6 +24,7 @@
 
 from google.cloud import ndb
 from google.cloud import storage
+from google.cloud.storage import retry
 
 import osv
 import osv.logs
@@ -59,7 +60,7 @@ def upload_single(self, bucket, source_path, target_path):
     logging.info('Uploading %s', target_path)
     try:
       blob = bucket.blob(target_path)
-      blob.upload_from_filename(source_path)
+      blob.upload_from_filename(source_path, retry=retry.DEFAULT_RETRY)
     except Exception as e:
       logging.exception('Failed to export: %s', e)
 

docker/importer/importer.py

@@ -459,7 +459,8 @@ def _process_updates_rest(self, source_repo: osv.SourceRepository):
     """Process updates from REST API."""
     logging.info('Begin processing REST: %s', source_repo.name)
 
-    ignore_last_import_time = source_repo.ignore_last_import_time
+    ignore_last_import_time = (
+        source_repo.ignore_last_import_time or not source_repo.last_update_date)
     if ignore_last_import_time:
       source_repo.ignore_last_import_time = False
       source_repo.put()
@@ -468,13 +469,16 @@ def _process_updates_rest(self, source_repo: osv.SourceRepository):
     if request.status_code != 200:
       logging.error('Failed to fetch REST API: %s', request.status_code)
       return
-    last_modified = datetime.datetime.strptime(request.headers['Last-Modified'],
-                                               _HTTP_LAST_MODIFIED_FORMAT)
-    # Check whether endpoint has been modified since last update
-    if not ignore_last_import_time and (last_modified
-                                        < source_repo.last_update_date):
-      logging.info('No changes since last update.')
-      return
+
+    if 'Last-Modified' in request.headers:
+      last_modified = datetime.datetime.strptime(
+          request.headers['Last-Modified'], _HTTP_LAST_MODIFIED_FORMAT)
+      # Check whether endpoint has been modified since last update
+      if not ignore_last_import_time and (last_modified
+                                          < source_repo.last_update_date):
+        logging.info('No changes since last update.')
+        return
+
     request = requests.get(source_repo.rest_api_url, timeout=_TIMEOUT_SECONDS)
     # Parse vulns into Vulnerability objects from the REST API request.
     vulns = osv.parse_vulnerabilities_from_data(

docker/indexer/go.mod

@@ -40,11 +40,11 @@ require (
 	github.com/skeema/knownhosts v1.2.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.16.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect

docker/indexer/go.sum

@@ -145,8 +145,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -168,8 +168,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
 golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
@@ -196,15 +196,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=