From 53a2bbb8a71a837bee0a4d55f2038103f07bf208 Mon Sep 17 00:00:00 2001
From: Michael Uti <utimichael9@gmail.com>
Date: Tue, 27 Aug 2024 23:22:12 +0100
Subject: [PATCH 1/3] chore: add bigquery example

---
 .../databases/bigquery/datastream_corp/app.js | 104 ++++++++++++++
 .../databases/bigquery/datastream_corp/run.sh | 127 ++++++++++++++++++
 .../bigquery/datastream_corp/run_ockam.sh     |  46 +++++++
 .../databases/bigquery/metrics_corp/run.sh    |  38 ++++++
 .../bigquery/metrics_corp/run_ockam.sh        |  83 ++++++++++++
 .../command/portals/databases/bigquery/run.sh |  97 +++++++++++++
 6 files changed, 495 insertions(+)
 create mode 100644 examples/command/portals/databases/bigquery/datastream_corp/app.js
 create mode 100755 examples/command/portals/databases/bigquery/datastream_corp/run.sh
 create mode 100755 examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh
 create mode 100755 examples/command/portals/databases/bigquery/metrics_corp/run.sh
 create mode 100755 examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
 create mode 100755 examples/command/portals/databases/bigquery/run.sh

diff --git a/examples/command/portals/databases/bigquery/datastream_corp/app.js b/examples/command/portals/databases/bigquery/datastream_corp/app.js
new file mode 100644
index 00000000000..eed89c77497
--- /dev/null
+++ b/examples/command/portals/databases/bigquery/datastream_corp/app.js
@@ -0,0 +1,104 @@
+const { BigQuery } = require('@google-cloud/bigquery');
+const process = require('process');
+
+const projectId = process.env.GOOGLE_CLOUD_PROJECT;
+if (!projectId) {
+    console.error('GOOGLE_CLOUD_PROJECT environment variable must be set.');
+    process.exit(1);
+}
+
+const credentials_base64 = process.env.GOOGLE_APPLICATION_CREDENTIALS_BASE64;
+const credentials_json = Buffer.from(credentials_base64, 'base64').toString('utf-8');
+
+const credentials = JSON.parse(credentials_json);
+
+// Configure the endpoint to our portal
+const bigqueryOptions = {
+    projectId: projectId,
+    apiEndpoint: 'http://127.0.0.1:8080',
+    maxRetries: 100,
+    credentials: credentials
+};
+
+// Create a BigQuery client
+const bigquery = new BigQuery(bigqueryOptions);
+
+async function createDataset(datasetId) {
+    const [dataset] = await bigquery.createDataset(datasetId);
+    console.log(`Dataset ${dataset.id} created.`);
+}
+
+async function createTable(datasetId, tableId) {
+    const schema = [
+        { name: 'name', type: 'STRING' },
+        { name: 'age', type: 'INTEGER' },
+        { name: 'email', type: 'STRING' }
+    ];
+
+    const options = {
+        schema: schema
+    };
+
+    const [table] = await bigquery
+        .dataset(datasetId)
+        .createTable(tableId, options);
+
+    console.log(`Table ${table.id} created.`);
+}
+
+async function insertData(datasetId, tableId) {
+    const rows = [
+        { name: 'John Doe', age: 30, email: 'john.doe@example.com' },
+        { name: 'Jane Smith', age: 25, email: 'jane.smith@example.com' }
+    ];
+
+    await bigquery
+        .dataset(datasetId)
+        .table(tableId)
+        .insert(rows);
+
+    console.log(`Inserted ${rows.length} rows into ${tableId}`);
+}
+
+async function queryData(datasetId, tableId) {
+    const query = `
+        SELECT name, age, email
+        FROM \`${bigquery.projectId}.${datasetId}.${tableId}\`
+        WHERE age > 20
+    `;
+
+    const [rows] = await bigquery.query({ query });
+
+    console.log('Query Results:');
+    rows.forEach(row => {
+        console.log(`Name: ${row.name}, Age: ${row.age}, Email: ${row.email}`);
+    });
+}
+
+async function deleteDataset(datasetId) {
+    await bigquery.dataset(datasetId).delete({ force: true }); // force: true deletes all tables within the dataset
+    console.log(`Dataset ${datasetId} deleted.`);
+}
+
+// Running all steps
+(async () => {
+    let datasetId = "ockam_" + (Math.random() + 1).toString(36).substring(7);
+    const tableId = 'ockam_table';
+
+    try {
+        await createDataset(datasetId);
+        await createTable(datasetId, tableId);
+        await insertData(datasetId, tableId);
+        await queryData(datasetId, tableId);
+
+        console.log(
+            "\nThe example run was successful 🥳.\n" +
+            "\nThe app connected with bigquery over an encrypted portal." +
+            "\nInserted some data, and querried it back.\n",
+        );
+    } catch (error) {
+        console.error('Error:', error);
+    }
+
+    await deleteDataset(datasetId);
+})();
diff --git a/examples/command/portals/databases/bigquery/datastream_corp/run.sh b/examples/command/portals/databases/bigquery/datastream_corp/run.sh
new file mode 100755
index 00000000000..1e2f4f10a4f
--- /dev/null
+++ b/examples/command/portals/databases/bigquery/datastream_corp/run.sh
@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+set -ex
+
+run() {
+    enrollment_ticket="$1"
+    GOOGLE_APPLICATION_CREDENTIALS_BASE64=$(echo "$GOOGLE_APPLICATION_CREDENTIALS" | base64)
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # CREATE NETWORK
+
+    # Create a new VPC and tag it.
+    vpc_id=$(aws ec2 create-vpc --cidr-block 10.0.0.0/16 --query 'Vpc.VpcId')
+    aws ec2 create-tags --resources "$vpc_id" --tags "Key=Name,Value=${name}-vpc"
+
+    # Create an Internet Gateway and attach it to the VPC.
+    gw_id=$(aws ec2 create-internet-gateway --query 'InternetGateway.InternetGatewayId')
+    aws ec2 attach-internet-gateway --vpc-id "$vpc_id" --internet-gateway-id "$gw_id"
+
+    # Create a route table and a route to the Internet through the Gateway.
+    rtb_id=$(aws ec2 create-route-table --vpc-id "$vpc_id" --query 'RouteTable.RouteTableId')
+    aws ec2 create-route --route-table-id "$rtb_id" --destination-cidr-block 0.0.0.0/0 --gateway-id "$gw_id"
+
+    # Create a subnet and associate the route table
+    az=$(aws ec2 describe-availability-zones --query "AvailabilityZones[0].ZoneName")
+    subnet_id=$(aws ec2 create-subnet --vpc-id "$vpc_id" --cidr-block 10.0.0.0/24 \
+        --availability-zone "$az" --query 'Subnet.SubnetId')
+    aws ec2 modify-subnet-attribute --subnet-id "$subnet_id" --map-public-ip-on-launch
+    aws ec2 associate-route-table --subnet-id "$subnet_id" --route-table-id "$rtb_id"
+
+    # Create a security group to allow TCP egress to the Internet.
+    sg_id=$(aws ec2 create-security-group --group-name "${name}-sg" --vpc-id "$vpc_id" --query 'GroupId' \
+        --description "Allow TCP egress")
+    aws ec2 authorize-security-group-egress --group-id "$sg_id" --cidr 0.0.0.0/0 --protocol tcp --port 0-65535
+
+    # Allow SSH from the machine where this script is running, so we can provision instances.
+    aws ec2 authorize-security-group-ingress --group-id "$sg_id" --cidr "0.0.0.0/0" --protocol tcp --port 22
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # CREATE INSTANCE
+
+    ami_id=$(aws ec2 describe-images --owners 137112412989 --query "Images | sort_by(@, &CreationDate) | [-1].ImageId" \
+        --filters "Name=name,Values=al2023-ami-2023*" "Name=architecture,Values=x86_64" \
+                  "Name=virtualization-type,Values=hvm" "Name=root-device-type,Values=ebs" )
+
+    aws ec2 create-key-pair --key-name "${name}-key" --query 'KeyMaterial' > key.pem
+    chmod 400 key.pem
+
+    sed "s/\$ENROLLMENT_TICKET/${enrollment_ticket}/g" run_ockam.sh > user_data1.sh
+    sed "s/\$OCKAM_VERSION/${OCKAM_VERSION}/g" user_data1.sh > user_data.sh
+    instance_id=$(aws ec2 run-instances --image-id "$ami_id" --instance-type c5n.large \
+        --subnet-id "$subnet_id" --security-group-ids "$sg_id" \
+        --key-name "${name}-key" --user-data file://user_data.sh --query 'Instances[0].InstanceId')
+    aws ec2 create-tags --resources "$instance_id" --tags "Key=Name,Value=${name}-ec2-instance"
+    aws ec2 wait instance-running --instance-ids "$instance_id"
+    ip=$(aws ec2 describe-instances --instance-ids "$instance_id" --query 'Reservations[0].Instances[0].PublicIpAddress')
+    rm -f user_data*.sh
+
+    until scp -o StrictHostKeyChecking=no -i ./key.pem ./app.js "ec2-user@$ip:app.js"; do sleep 10; done
+    ssh -o StrictHostKeyChecking=no -i ./key.pem "ec2-user@$ip" \
+        'bash -s' << EOS
+          export GOOGLE_CLOUD_PROJECT="$GOOGLE_CLOUD_PROJECT_ID"
+          export GOOGLE_APPLICATION_CREDENTIALS_BASE64="$GOOGLE_APPLICATION_CREDENTIALS_BASE64"
+          sudo yum update -y && sudo yum install nodejs -y
+          npm install @google-cloud/bigquery
+          node app.js
+EOS
+}
+
+cleanup() {
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # DELETE INSTANCE
+
+    instance_ids=$(aws ec2 describe-instances --filters "Name=tag:Name,Values=${name}-ec2-instance" \
+        --query "Reservations[*].Instances[*].InstanceId")
+    for i in $instance_ids; do
+        aws ec2 terminate-instances --instance-ids "$i"
+        aws ec2 wait instance-terminated --instance-ids "$i"
+    done
+
+    if aws ec2 describe-key-pairs --key-names "${name}-key" &>/dev/null; then
+        aws ec2 delete-key-pair --key-name "${name}-key"
+    fi
+
+    rm -rf key.pem
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # DELETE NETWORK
+
+    vpc_ids=$(aws ec2 describe-vpcs --query 'Vpcs[*].VpcId' --filters "Name=tag:Name,Values=${name}-vpc")
+
+    for vpc_id in $vpc_ids; do
+        internet_gateways=$(aws ec2 describe-internet-gateways --query "InternetGateways[*].InternetGatewayId" \
+            --filters Name=attachment.vpc-id,Values="$vpc_id")
+        for i in $internet_gateways; do
+            aws ec2 detach-internet-gateway --internet-gateway-id "$i" --vpc-id "$vpc_id"
+            aws ec2 delete-internet-gateway --internet-gateway-id "$i"
+        done
+
+        subnet_ids=$(aws ec2 describe-subnets --query "Subnets[*].SubnetId" --filters Name=vpc-id,Values="$vpc_id")
+        for i in $subnet_ids; do aws ec2 delete-subnet --subnet-id "$i"; done
+
+        route_tables=$(aws ec2 describe-route-tables  --filters Name=vpc-id,Values="$vpc_id" \
+            --query 'RouteTables[?length(Associations[?Main!=`true`]) > `0` || length(Associations) == `0`].RouteTableId')
+        for i in $route_tables; do aws ec2 delete-route-table --route-table-id "$i" || true; done
+
+        security_groups=$(aws ec2 describe-security-groups --filters Name=vpc-id,Values="$vpc_id" \
+            --query "SecurityGroups[?!contains(GroupName, 'default')].[GroupId]")
+        for i in $security_groups; do aws ec2 delete-security-group --group-id "$i"; done
+
+        if aws ec2 describe-vpcs --vpc-ids "$vpc_id" &>/dev/null; then
+            aws ec2 delete-vpc --vpc-id "$vpc_id"
+        fi
+    done
+}
+
+export AWS_PAGER="";
+export AWS_DEFAULT_OUTPUT="text";
+
+user=""
+command -v sha256sum &>/dev/null && user=$(aws sts get-caller-identity | sha256sum | cut -c 1-20)
+command -v shasum &>/dev/null && user=$(aws sts get-caller-identity | shasum -a 256 | cut -c 1-20)
+export name="ockam-ex-ts-ds-$user"
+
+# Check if the first argument is "cleanup"
+# If it is, call the cleanup function. If not, call the run function.
+if [ "$1" = "cleanup" ]; then cleanup; else run "$1"; fi
diff --git a/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh b/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh
new file mode 100755
index 00000000000..c86e0d892f5
--- /dev/null
+++ b/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+set -ex
+
+# Change into ec2-user's home directory and use sudo to run the commands as ec2-user
+cd /home/ec2-user
+sudo -u ec2-user bash << 'EOS'
+set -ex
+
+# Install Ockam Command
+export OCKAM_VERSION="$OCKAM_VERSION"
+curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash
+source "$HOME/.ockam/env"
+
+# Run `ockam project enroll ...`
+#
+# The `project enroll` command creates a new vault and generates a cryptographic identity with
+# private keys stored in that vault.
+#
+# The enrollment ticket includes routes and identitifiers for the project membership authority
+# and the project’s node that offers the relay service.
+#
+# The enrollment ticket also includes an enrollment token. The project enroll command
+# creates a secure channel with the project membership authority and presents this enrollment token.
+# The authority enrolls presented identity and returns a project membership credential.
+#
+# The command, stores this credential for later use and exits.
+ockam project enroll "$ENROLLMENT_TICKET"
+
+# Create an ockam node.
+#
+# Create an access control policy that only allows project members that possesses a credential with
+# attribute bigquery-outlet="true" to connect to TCP Portal Inlets on this node.
+#
+# Create a TCP Portal Inlet to Google's BigQuery API.
+# This makes the remote API available on all localhost IPs at - 0.0.0.0:8080
+cat << EOF > inlet.yaml
+tcp-inlet:
+  from: 0.0.0.0:8080
+  via: bigquery
+  allow: '(= subject.bigquery-outlet "true")'
+EOF
+
+ockam node create inlet.yaml
+rm inlet.yaml
+
+EOS
diff --git a/examples/command/portals/databases/bigquery/metrics_corp/run.sh b/examples/command/portals/databases/bigquery/metrics_corp/run.sh
new file mode 100755
index 00000000000..c6f6d52b059
--- /dev/null
+++ b/examples/command/portals/databases/bigquery/metrics_corp/run.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+run() {
+    enrollment_ticket="$1"
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # CREATE INSTANCE AND START RELAY
+    sed "s/\$ENROLLMENT_TICKET/${enrollment_ticket}/g" run_ockam.sh > user_data1.sh
+    sed "s/\$OCKAM_VERSION/${OCKAM_VERSION}/g" user_data1.sh > user_data.sh
+
+    gcloud compute instances create "${name}-key" \
+        --project="$GOOGLE_CLOUD_PROJECT_ID" \
+        --zone="us-central1-c" \
+        --create-disk=auto-delete=yes,boot=yes,device-name="${name}-key",image=projects/debian-cloud/global/images/debian-12-bookworm-v20240815,mode=rw,size=10,type=pd-balanced \
+        --machine-type=e2-medium \
+        --network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default \
+        --tags="${name}-key" \
+        --metadata-from-file=startup-script=user_data.sh
+
+    rm -rf user_data*.sh
+}
+
+cleanup() {
+    # ----------------------------------------------------------------------------------------------------------------
+    # DELETE INSTANCE
+    gcloud compute instances delete "${name}-key" --zone="us-central1-c" --project="$GOOGLE_CLOUD_PROJECT_ID" --quiet || true
+    rm -rf user_data*.sh
+}
+
+
+user=""
+command -v sha256sum &>/dev/null && user=$(gcloud auth list --format json | jq -r '.[0].account' | sha256sum | cut -c 1-20)
+command -v shasum &>/dev/null && user=$(gcloud auth list --format json | jq -r '.[0].account' | shasum -a 256 | cut -c 1-20)
+export name="ockam-ex-ts-m-$user"
+
+# Check if the first argument is "cleanup"
+# If it is, call the cleanup function. If not, call the run function.
+if [ "$1" = "cleanup" ]; then cleanup; else run "$1"; fi
diff --git a/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh b/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
new file mode 100755
index 00000000000..9e097ef0944
--- /dev/null
+++ b/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+set -ex
+
+# Change into ec2-user's home directory and use sudo to run the commands as user
+cd ~
+sudo bash << 'EOS'
+set -ex
+
+# Create a reverse proxy to the BigQuery API
+cat << 'EOF' > proxy.js
+const http = require('http');
+const httpProxy = require('http-proxy');
+
+// Create a proxy server to forward requests to the BigQuery API
+const proxy = httpProxy.createProxyServer({
+    target: 'https://bigquery.googleapis.com',
+    changeOrigin: true,
+});
+
+const server = http.createServer((req, res) => {
+    proxy.web(req, res, (err) => {
+        if (err) {
+            console.error(`Error proxying request: ${err.message}`);
+            res.writeHead(502, { 'Content-Type': 'text/plain' });
+            res.end('Bad Gateway');
+        }
+    });
+});
+
+const port = 8000;
+server.listen(port, () => {
+    console.log(`Reverse proxy server listening on port ${port}`);
+});
+EOF
+
+# Install Node.js and start the reverse proxy at the background
+sudo apt update -y && sudo apt install nodejs npm -y
+npm install http-proxy
+node proxy.js &
+
+# Install Ockam Command
+export OCKAM_VERSION="$OCKAM_VERSION"
+curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash
+source "$HOME/.ockam/env"
+
+ockam --version
+
+# Run `ockam project enroll ...`
+#
+# The `project enroll` command creates a new vault and generates a cryptographic identity with
+# private keys stored in that vault.
+#
+# The enrollment ticket includes routes and identifiers for the project membership authority
+# and the project’s node that offers the relay service.
+#
+# The enrollment ticket also includes an enrollment token. The project enroll command
+# creates a secure channel with the project membership authority and presents this enrollment token.
+# The authority enrolls presented identity and returns a project membership credential.
+#
+# The command, stores this credential for later use and exits.
+ockam project enroll "$ENROLLMENT_TICKET"
+
+# Create an ockam node.
+#
+# Create an encrypted relay to the reverse proxy, ensuring requests made through the relay is received by the proxy.
+# The relay makes this node reachable by other project members.
+#
+# Create an access control policy that only allows project members that possesses a credential with
+# attribute bigquery-inlet="true" to connect to TCP Portal Outlets on this node.
+#
+# Create a TCP Portal Outlet to our reverse proxy
+cat << EOF > outlet.yaml
+tcp-outlet:
+  to: "127.0.0.1:8000"
+  allow: '(= subject.bigquery-inlet "true")'
+
+relay: bigquery
+EOF
+
+ockam node create outlet.yaml
+rm outlet.yaml
+
+EOS
diff --git a/examples/command/portals/databases/bigquery/run.sh b/examples/command/portals/databases/bigquery/run.sh
new file mode 100755
index 00000000000..ec70cbd9be1
--- /dev/null
+++ b/examples/command/portals/databases/bigquery/run.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+set -e
+
+# This script, `./run.sh ...` is invoked on a developer’s work machine.
+#
+# This hands-on example uses Ockam to create an end-to-end encrypted portal to BigQuery. We host a reverse proxy in GCP cloud
+# and access through an Amazon VPC.
+#
+# The example uses AWS CLI and gcloud CLI to create these VPCs.
+#
+# You can read a detailed walkthrough of this example at:
+# https://docs.ockam.io/portals/databases/influxdb/timestream
+
+run() {
+    # Run `ockam enroll`.
+    #
+    # The enroll command creates a new vault and generates a cryptographic identity with private keys stored in that
+    # vault. It then guides you to sign in to Ockam Orchestrator.
+    #
+    # If this is your first time signing in, the Orchestrator creates a new dedicated project for you. A project offers
+    # two services: a membership authority and a relay service.
+    #
+    # The enroll command then asks this project’s membership authority to sign and issue a credential that attests that
+    # your identifier is a member of this project. Since your account in Orchestrator is the creator and hence first
+    # administrator on this new project, the membership authority issues this credential. The enroll command stores the
+    # credential for later use and exits.
+    ockam enroll
+
+    # Create an enrollment ticket to enroll the identity used by an ockam node that will run adjacent to the bigquery
+    # server in metrics_corp's network.
+    #
+    # The identity that enrolls with the generated ticket will be given a project membership credential in which the
+    # project membership authority will cryptographically attest to the specified attributes - bigquery-outlet=true.
+    #
+    # The identity will also allowed to create a relay in the project at the address `bigquery`.
+    metrics_corp_ticket=$(ockam project ticket --usage-count 1 --expires-in 60m \
+        --attribute "bigquery-outlet=true" --relay bigquery)
+
+    # Create an enrollment ticket to enroll the identity used by an ockam node that will run adjacent to the node
+    # that will access bigquery
+    #  in datastream_corp's network.
+    #
+    # The identity that enrolls with the generated ticket will be given a project membership credential in which the
+    # project membership authority will cryptographically attest to the specified attributes - bigquery-inlet=true.
+    datastream_corp_ticket=$(ockam project ticket --usage-count 1 --expires-in 60m \
+        --attribute "bigquery-inlet=true")
+
+    if [[ -n "$OCKAM_VERSION" ]]; then
+        export OCKAM_VERSION="v${OCKAM_VERSION}";
+    fi
+
+    # Ensure that a project ID is set
+    if [[ -z "$GOOGLE_CLOUD_PROJECT_ID" ]]; then
+        echo "ERROR: Please set the GOOGLE_CLOUD_PROJECT_ID environment variable"
+        exit 1
+    fi
+
+    if [[ -z "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then
+        echo "ERROR: Please set the GOOGLE_APPLICATION_CREDENTIALS environment variable"
+        exit 1
+    fi
+
+    echo "$GOOGLE_APPLICATION_CREDENTIALS"
+
+    # Invoke `metrics_corp/run.sh` in the directory that has metrics_corp's configuration. Pass the above enrollment ticket
+    # as the first argument to run.sh script. Read metrics_corp/run.sh to understand the parts that are provisioned in
+    # metrics_corp's virtual private cloud.
+    echo; pushd metrics_corp; ./run.sh "$metrics_corp_ticket"; popd
+
+    # Invoke `datastream_corp/run.sh` in the directory that has datastream_corp's configuration. Pass the above enrollment
+    # ticket as the first argument to run.sh script. Read datastream_corp/run.sh to understand the parts that are
+    # provisioned in datastream_corp's virtual private cloud.
+    echo; pushd datastream_corp; ./run.sh "$datastream_corp_ticket"; popd
+}
+
+# Cleanup after the example - `./run.sh cleanup`
+# Remove all resources that were created in AWS.
+cleanup() {
+    pushd metrics_corp; ./run.sh cleanup; popd
+    pushd datastream_corp; ./run.sh cleanup; popd
+}
+
+# Check if Ockam Command is already installed and available in path.
+# If it's not, then install it
+if ! type ockam &>/dev/null; then
+    curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash
+    source "$HOME/.ockam/env"
+fi
+
+# Check that tools we need are installed.
+for c in aws curl; do
+    if ! type "$c" &>/dev/null; then echo "ERROR: Please install: $c" && exit 1; fi
+done
+
+# Check if the first argument is "cleanup"
+# If it is, call the cleanup function. If not, call the run function.
+if [ "$1" = "cleanup" ]; then cleanup; else run; fi

From a838cb7fa6fd0f55f578b9aeec9de7ccce58b334 Mon Sep 17 00:00:00 2001
From: Michael Uti <utimichael9@gmail.com>
Date: Wed, 28 Aug 2024 20:27:01 +0100
Subject: [PATCH 2/3] chore: remove reverse proxy feature in bigquery example

---
 .../databases/bigquery/datastream_corp/app.js | 89 +++++++++++++++----
 .../databases/bigquery/datastream_corp/run.sh |  2 +
 .../bigquery/metrics_corp/run_ockam.sh        | 39 +-------
 3 files changed, 78 insertions(+), 52 deletions(-)

diff --git a/examples/command/portals/databases/bigquery/datastream_corp/app.js b/examples/command/portals/databases/bigquery/datastream_corp/app.js
index eed89c77497..2fdaa5c49b7 100644
--- a/examples/command/portals/databases/bigquery/datastream_corp/app.js
+++ b/examples/command/portals/databases/bigquery/datastream_corp/app.js
@@ -1,5 +1,7 @@
+const axios = require('axios');
+const { JWT } = require('google-auth-library');
 const { BigQuery } = require('@google-cloud/bigquery');
-const process = require('process');
+
 
 const projectId = process.env.GOOGLE_CLOUD_PROJECT;
 if (!projectId) {
@@ -8,27 +10,75 @@ if (!projectId) {
 }
 
 const credentials_base64 = process.env.GOOGLE_APPLICATION_CREDENTIALS_BASE64;
-const credentials_json = Buffer.from(credentials_base64, 'base64').toString('utf-8');
+if (!credentials_base64) {
+    console.error('GOOGLE_APPLICATION_CREDENTIALS_BASE64 environment variable must be set.');
+    process.exit(1);
+}
 
+const credentials_json = Buffer.from(credentials_base64, 'base64').toString('utf-8');
 const credentials = JSON.parse(credentials_json);
 
-// Configure the endpoint to our portal
-const bigqueryOptions = {
-    projectId: projectId,
-    apiEndpoint: 'http://127.0.0.1:8080',
-    maxRetries: 100,
-    credentials: credentials
+// Function to get Bearer token
+const getAuthToken = async () => {
+    // Create a JWT client using the credentials
+    const client = new JWT({
+        email: credentials.client_email,
+        key: credentials.private_key,
+        scopes: ['https://www.googleapis.com/auth/bigquery'],
+    });
+
+    // Authorize the client and get the Bearer token
+    const token = await client.authorize();
+    return token.access_token;
 };
 
-// Create a BigQuery client
-const bigquery = new BigQuery(bigqueryOptions);
+// Custom BigQuery Client
+class CustomBigQueryClient extends BigQuery {
+    constructor(projectID) {
+        super();
+        this.projectId = projectID;
+    }
+
+    async request(reqOpts, callback) {
+        try {
+            const token = await getAuthToken();
+            const url = `http://127.0.0.1:8080/bigquery/v2/projects/${this.projectId}/${reqOpts.uri}`;
+            const checkedURl = url.replace(/([^:]\/)\/+/g, "$1");
+
+            // When deleting dataset, body is sent as an object named qs
+            const body = reqOpts.json || reqOpts.qs;
+
+            const config = {
+                method: reqOpts.method,
+                url: checkedURl,
+                headers: {
+                    ...reqOpts.headers,
+                    'Authorization': `Bearer ${token}`,
+                    'Content-Type': 'application/json',
+                    'Host': 'bigquery.googleapis.com',
+                },
+                data: body,
+            };
+
+            const response = await axios(config);
+            callback(null, response.data, response);
+        } catch (error) {
+            callback(error, null, null);
+        }
+    }
+}
+
+const bigQueryClient = new CustomBigQueryClient('effortless-cat-433609-h1');
 
 async function createDataset(datasetId) {
-    const [dataset] = await bigquery.createDataset(datasetId);
+    console.log(`Creating Dataset ${datasetId}`);
+    const [dataset] = await bigQueryClient.createDataset(datasetId);
     console.log(`Dataset ${dataset.id} created.`);
 }
 
 async function createTable(datasetId, tableId) {
+    console.log(`Creating Table ${tableId} for dataset ${datasetId}`);
+
     const schema = [
         { name: 'name', type: 'STRING' },
         { name: 'age', type: 'INTEGER' },
@@ -39,7 +89,7 @@ async function createTable(datasetId, tableId) {
         schema: schema
     };
 
-    const [table] = await bigquery
+    const [table] = await bigQueryClient
         .dataset(datasetId)
         .createTable(tableId, options);
 
@@ -47,12 +97,14 @@ async function createTable(datasetId, tableId) {
 }
 
 async function insertData(datasetId, tableId) {
+    console.log(`Inserting data to Table ${tableId} for dataset ${datasetId}`);
+
     const rows = [
         { name: 'John Doe', age: 30, email: 'john.doe@example.com' },
         { name: 'Jane Smith', age: 25, email: 'jane.smith@example.com' }
     ];
 
-    await bigquery
+    await bigQueryClient
         .dataset(datasetId)
         .table(tableId)
         .insert(rows);
@@ -61,13 +113,15 @@ async function insertData(datasetId, tableId) {
 }
 
 async function queryData(datasetId, tableId) {
+    console.log(`Querying data for Table ${tableId} for dataset ${datasetId}`);
+
     const query = `
         SELECT name, age, email
-        FROM \`${bigquery.projectId}.${datasetId}.${tableId}\`
+        FROM \`${bigQueryClient.projectId}.${datasetId}.${tableId}\`
         WHERE age > 20
     `;
 
-    const [rows] = await bigquery.query({ query });
+    const [rows] = await bigQueryClient.query({ query });
 
     console.log('Query Results:');
     rows.forEach(row => {
@@ -76,11 +130,12 @@ async function queryData(datasetId, tableId) {
 }
 
 async function deleteDataset(datasetId) {
-    await bigquery.dataset(datasetId).delete({ force: true }); // force: true deletes all tables within the dataset
+    console.log(`Deleting dataset ${datasetId}`);
+    await bigQueryClient.dataset(datasetId).delete({ force: true }); // force: true deletes all tables within the dataset
     console.log(`Dataset ${datasetId} deleted.`);
 }
 
-// Running all steps
+// Run the example
 (async () => {
     let datasetId = "ockam_" + (Math.random() + 1).toString(36).substring(7);
     const tableId = 'ockam_table';
diff --git a/examples/command/portals/databases/bigquery/datastream_corp/run.sh b/examples/command/portals/databases/bigquery/datastream_corp/run.sh
index 1e2f4f10a4f..c7a70f03160 100755
--- a/examples/command/portals/databases/bigquery/datastream_corp/run.sh
+++ b/examples/command/portals/databases/bigquery/datastream_corp/run.sh
@@ -62,6 +62,8 @@ run() {
           export GOOGLE_APPLICATION_CREDENTIALS_BASE64="$GOOGLE_APPLICATION_CREDENTIALS_BASE64"
           sudo yum update -y && sudo yum install nodejs -y
           npm install @google-cloud/bigquery
+          npm install google-auth-library
+          npm install axios
           node app.js
 EOS
 }
diff --git a/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh b/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
index 9e097ef0944..02b4fa1fa3d 100755
--- a/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
+++ b/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
@@ -6,38 +6,6 @@ cd ~
 sudo bash << 'EOS'
 set -ex
 
-# Create a reverse proxy to the BigQuery API
-cat << 'EOF' > proxy.js
-const http = require('http');
-const httpProxy = require('http-proxy');
-
-// Create a proxy server to forward requests to the BigQuery API
-const proxy = httpProxy.createProxyServer({
-    target: 'https://bigquery.googleapis.com',
-    changeOrigin: true,
-});
-
-const server = http.createServer((req, res) => {
-    proxy.web(req, res, (err) => {
-        if (err) {
-            console.error(`Error proxying request: ${err.message}`);
-            res.writeHead(502, { 'Content-Type': 'text/plain' });
-            res.end('Bad Gateway');
-        }
-    });
-});
-
-const port = 8000;
-server.listen(port, () => {
-    console.log(`Reverse proxy server listening on port ${port}`);
-});
-EOF
-
-# Install Node.js and start the reverse proxy at the background
-sudo apt update -y && sudo apt install nodejs npm -y
-npm install http-proxy
-node proxy.js &
-
 # Install Ockam Command
 export OCKAM_VERSION="$OCKAM_VERSION"
 curl --proto '=https' --tlsv1.2 -sSfL https://install.command.ockam.io | bash
@@ -62,16 +30,17 @@ ockam project enroll "$ENROLLMENT_TICKET"
 
 # Create an ockam node.
 #
-# Create an encrypted relay to the reverse proxy, ensuring requests made through the relay is received by the proxy.
+# Create an encrypted relay to this node in the project at address: bigquery.googleapis.com.
 # The relay makes this node reachable by other project members.
 #
 # Create an access control policy that only allows project members that possesses a credential with
 # attribute bigquery-inlet="true" to connect to TCP Portal Outlets on this node.
 #
-# Create a TCP Portal Outlet to our reverse proxy
+# Create a TCP Portal Outlet to BigQuery API at at - bigquery.googleapis.com:443.
 cat << EOF > outlet.yaml
 tcp-outlet:
-  to: "127.0.0.1:8000"
+  to: bigquery.googleapis.com:443
+  tls: true
   allow: '(= subject.bigquery-inlet "true")'
 
 relay: bigquery

From d4c936c1bb31069e4b3395de30ca479d720e424d Mon Sep 17 00:00:00 2001
From: Michael Uti <utimichael9@gmail.com>
Date: Mon, 9 Sep 2024 17:57:29 +0100
Subject: [PATCH 3/3] chore: add private endpoint support in bigquery example

---
 .../databases/bigquery/datastream_corp/app.js | 11 +++-
 .../databases/bigquery/datastream_corp/run.sh |  8 +--
 .../bigquery/datastream_corp/run_ockam.sh     |  2 +-
 .../databases/bigquery/metrics_corp/run.sh    | 66 ++++++++++++++++---
 .../bigquery/metrics_corp/run_ockam.sh        | 12 ++--
 .../command/portals/databases/bigquery/run.sh | 12 ++--
 6 files changed, 82 insertions(+), 29 deletions(-)

diff --git a/examples/command/portals/databases/bigquery/datastream_corp/app.js b/examples/command/portals/databases/bigquery/datastream_corp/app.js
index 2fdaa5c49b7..810d4b344a2 100644
--- a/examples/command/portals/databases/bigquery/datastream_corp/app.js
+++ b/examples/command/portals/databases/bigquery/datastream_corp/app.js
@@ -2,7 +2,6 @@ const axios = require('axios');
 const { JWT } = require('google-auth-library');
 const { BigQuery } = require('@google-cloud/bigquery');
 
-
 const projectId = process.env.GOOGLE_CLOUD_PROJECT;
 if (!projectId) {
     console.error('GOOGLE_CLOUD_PROJECT environment variable must be set.');
@@ -15,6 +14,12 @@ if (!credentials_base64) {
     process.exit(1);
 }
 
+const private_endpoint_name = process.env.PRIVATE_ENDPOINT_NAME;
+if (!private_endpoint_name) {
+    console.error('PRIVATE_ENDPOINT_NAME environment variable must be set.');
+    process.exit(1);
+}
+
 const credentials_json = Buffer.from(credentials_base64, 'base64').toString('utf-8');
 const credentials = JSON.parse(credentials_json);
 
@@ -55,7 +60,7 @@ class CustomBigQueryClient extends BigQuery {
                     ...reqOpts.headers,
                     'Authorization': `Bearer ${token}`,
                     'Content-Type': 'application/json',
-                    'Host': 'bigquery.googleapis.com',
+                    'Host': `bigquery-${private_endpoint_name}.p.googleapis.com`,
                 },
                 data: body,
             };
@@ -68,7 +73,7 @@ class CustomBigQueryClient extends BigQuery {
     }
 }
 
-const bigQueryClient = new CustomBigQueryClient('effortless-cat-433609-h1');
+const bigQueryClient = new CustomBigQueryClient(projectId);
 
 async function createDataset(datasetId) {
     console.log(`Creating Dataset ${datasetId}`);
diff --git a/examples/command/portals/databases/bigquery/datastream_corp/run.sh b/examples/command/portals/databases/bigquery/datastream_corp/run.sh
index c7a70f03160..90e8784a654 100755
--- a/examples/command/portals/databases/bigquery/datastream_corp/run.sh
+++ b/examples/command/portals/databases/bigquery/datastream_corp/run.sh
@@ -58,13 +58,13 @@ run() {
     until scp -o StrictHostKeyChecking=no -i ./key.pem ./app.js "ec2-user@$ip:app.js"; do sleep 10; done
     ssh -o StrictHostKeyChecking=no -i ./key.pem "ec2-user@$ip" \
         'bash -s' << EOS
+          # Wait for private endpoint to be up.
+          while ! curl -H "Host: bigquery-${PRIVATE_ENDPOINT_NAME}.p.googleapis.com" http://127.0.0.1:8080/discovery/v1/apis/bigquery/v2/rest --connect-timeout 2 --max-time 5 --silent > /dev/null; do sleep 5 && echo "private endpoint not up yet... retrying"; done
           export GOOGLE_CLOUD_PROJECT="$GOOGLE_CLOUD_PROJECT_ID"
           export GOOGLE_APPLICATION_CREDENTIALS_BASE64="$GOOGLE_APPLICATION_CREDENTIALS_BASE64"
           sudo yum update -y && sudo yum install nodejs -y
-          npm install @google-cloud/bigquery
-          npm install google-auth-library
-          npm install axios
-          node app.js
+          npm install @google-cloud/bigquery google-auth-library axios
+          PRIVATE_ENDPOINT_NAME="$PRIVATE_ENDPOINT_NAME" node app.js
 EOS
 }
 
diff --git a/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh b/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh
index c86e0d892f5..95eecbc1cd8 100755
--- a/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh
+++ b/examples/command/portals/databases/bigquery/datastream_corp/run_ockam.sh
@@ -37,7 +37,7 @@ cat << EOF > inlet.yaml
 tcp-inlet:
   from: 0.0.0.0:8080
   via: bigquery
-  allow: '(= subject.bigquery-outlet "true")'
+  allow: bigquery-outlet
 EOF
 
 ockam node create inlet.yaml
diff --git a/examples/command/portals/databases/bigquery/metrics_corp/run.sh b/examples/command/portals/databases/bigquery/metrics_corp/run.sh
index c6f6d52b059..80351aabfe4 100755
--- a/examples/command/portals/databases/bigquery/metrics_corp/run.sh
+++ b/examples/command/portals/databases/bigquery/metrics_corp/run.sh
@@ -1,20 +1,50 @@
 #!/usr/bin/env bash
 
+
 run() {
     enrollment_ticket="$1"
+    private_endpoint_address="10.200.0.5"
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # CREATE PRIVATE GOOGLE API ENDPOINT (PRIVATE SERVICE CONNECT API)
+
+    # Create a new VPC.
+    gcloud compute networks create "${name}-vpc" --subnet-mode=custom --project="$GOOGLE_CLOUD_PROJECT_ID"
+
+    # Create a subnet in the VPC.
+    gcloud compute networks subnets create "${name}-subnet" --network="${name}-vpc" --project="$GOOGLE_CLOUD_PROJECT_ID" \
+        --range=10.0.0.0/24 --region=us-central1
+
+    # Enable Private Google Access for the subnet.
+    gcloud compute networks subnets update "${name}-subnet" --project="$GOOGLE_CLOUD_PROJECT_ID" \
+        --region=us-central1 --enable-private-ip-google-access
+
+    # Reserve an internal IP address for the private service connect (psc).
+    gcloud compute addresses create "${name}-psc-address" --global --project="$GOOGLE_CLOUD_PROJECT_ID" \
+        --purpose=PRIVATE_SERVICE_CONNECT --addresses="$private_endpoint_address" --network="${name}-vpc"
+
+    # Create a forwarding rule to connect to BigQuery using the reserved IP address.
+    gcloud compute forwarding-rules create "$PRIVATE_ENDPOINT_NAME" --global --project="$GOOGLE_CLOUD_PROJECT_ID" \
+        --network="${name}-vpc" --address="${name}-psc-address" --target-google-apis-bundle=all-apis
+
+    # Allow Egress traffic to the internet.
+    gcloud compute firewall-rules create allow-all-egress \
+        --network="${name}-vpc" --allow=all --direction=EGRESS --priority=1000 --destination-ranges=0.0.0.0/0 --target-tags=allow-egress
+
 
     # ----------------------------------------------------------------------------------------------------------------
-    # CREATE INSTANCE AND START RELAY
+    # CREATE INSTANCE USING THE PRIVATE GOOGLE API ENDPOINT
     sed "s/\$ENROLLMENT_TICKET/${enrollment_ticket}/g" run_ockam.sh > user_data1.sh
-    sed "s/\$OCKAM_VERSION/${OCKAM_VERSION}/g" user_data1.sh > user_data.sh
+    sed "s/\$OCKAM_VERSION/${OCKAM_VERSION}/g" user_data1.sh > user_data2.sh
+    sed "s/\$PRIVATE_ENDPOINT_NAME/${PRIVATE_ENDPOINT_NAME}/g" user_data2.sh > user_data.sh
 
-    gcloud compute instances create "${name}-key" \
+    gcloud compute instances create "${name}-vm-instance" \
         --project="$GOOGLE_CLOUD_PROJECT_ID" \
-        --zone="us-central1-c" \
-        --create-disk=auto-delete=yes,boot=yes,device-name="${name}-key",image=projects/debian-cloud/global/images/debian-12-bookworm-v20240815,mode=rw,size=10,type=pd-balanced \
+        --zone="us-central1-a" \
+        --create-disk=auto-delete=yes,boot=yes,device-name="${name}-vm-instance",image=projects/debian-cloud/global/images/debian-12-bookworm-v20240815,mode=rw,size=10,type=pd-balanced \
         --machine-type=e2-medium \
-        --network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default \
-        --tags="${name}-key" \
+        --subnet="${name}-subnet" \
+        --tags=allow-egress \
         --metadata-from-file=startup-script=user_data.sh
 
     rm -rf user_data*.sh
@@ -22,8 +52,26 @@ run() {
 
 cleanup() {
     # ----------------------------------------------------------------------------------------------------------------
-    # DELETE INSTANCE
-    gcloud compute instances delete "${name}-key" --zone="us-central1-c" --project="$GOOGLE_CLOUD_PROJECT_ID" --quiet || true
+    # DELETE NETWORK
+
+    # Delete forwarding rule
+    gcloud compute forwarding-rules delete "$PRIVATE_ENDPOINT_NAME" --global --quiet
+
+    # Delete reserved endpoint address
+    gcloud compute addresses delete "${name}-psc-address" --global --quiet
+
+    # Delete rule to allow egress
+    gcloud compute firewall-rules delete allow-all-egress --quiet
+
+    # ----------------------------------------------------------------------------------------------------------------
+    # DELETE INSTANCE RESOURCES
+    gcloud compute instances delete "${name}-vm-instance" --zone="us-central1-a" --project="$GOOGLE_CLOUD_PROJECT_ID" --quiet
+
+    # Delete subnet
+    gcloud compute networks subnets delete "${name}-subnet" --region=us-central1 --quiet
+    # Delete VPC
+    gcloud compute networks delete "${name}-vpc" --quiet
+    
     rm -rf user_data*.sh
 }
 
diff --git a/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh b/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
index 02b4fa1fa3d..b83630b52fd 100755
--- a/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
+++ b/examples/command/portals/databases/bigquery/metrics_corp/run_ockam.sh
@@ -30,23 +30,23 @@ ockam project enroll "$ENROLLMENT_TICKET"
 
 # Create an ockam node.
 #
-# Create an encrypted relay to this node in the project at address: bigquery.googleapis.com.
+# Create an encrypted relay to this node in the project at address: bigquery-$PRIVATE_ENDPOINT_NAME.p.googleapis.com 
+# where PRIVATE_ENDPOINT_NAME is our private endpoint name default know as ockamendpoint.
 # The relay makes this node reachable by other project members.
 #
 # Create an access control policy that only allows project members that possesses a credential with
 # attribute bigquery-inlet="true" to connect to TCP Portal Outlets on this node.
 #
-# Create a TCP Portal Outlet to BigQuery API at at - bigquery.googleapis.com:443.
+# Create a TCP Portal Outlet to BigQuery API at at - bigquery-$PRIVATE_ENDPOINT_NAME.p.googleapis.com:443.
 cat << EOF > outlet.yaml
 tcp-outlet:
-  to: bigquery.googleapis.com:443
+  to: bigquery-$PRIVATE_ENDPOINT_NAME.p.googleapis.com:443
   tls: true
-  allow: '(= subject.bigquery-inlet "true")'
+  allow: bigquery-inlet
 
 relay: bigquery
 EOF
 
+cat outlet.yaml
 ockam node create outlet.yaml
-rm outlet.yaml
-
 EOS
diff --git a/examples/command/portals/databases/bigquery/run.sh b/examples/command/portals/databases/bigquery/run.sh
index ec70cbd9be1..c077562cadd 100755
--- a/examples/command/portals/databases/bigquery/run.sh
+++ b/examples/command/portals/databases/bigquery/run.sh
@@ -3,8 +3,8 @@ set -e
 
 # This script, `./run.sh ...` is invoked on a developer’s work machine.
 #
-# This hands-on example uses Ockam to create an end-to-end encrypted portal to BigQuery. We host a reverse proxy in GCP cloud
-# and access through an Amazon VPC.
+# This hands-on example uses Ockam to create an end-to-end encrypted portal to BigQuery. We connect a BigQuery private endpoint
+# in a GCP VPC and access through an Amazon VPC.
 #
 # The example uses AWS CLI and gcloud CLI to create these VPCs.
 #
@@ -49,7 +49,6 @@ run() {
         export OCKAM_VERSION="v${OCKAM_VERSION}";
     fi
 
-    # Ensure that a project ID is set
     if [[ -z "$GOOGLE_CLOUD_PROJECT_ID" ]]; then
         echo "ERROR: Please set the GOOGLE_CLOUD_PROJECT_ID environment variable"
         exit 1
@@ -60,8 +59,6 @@ run() {
         exit 1
     fi
 
-    echo "$GOOGLE_APPLICATION_CREDENTIALS"
-
     # Invoke `metrics_corp/run.sh` in the directory that has metrics_corp's configuration. Pass the above enrollment ticket
     # as the first argument to run.sh script. Read metrics_corp/run.sh to understand the parts that are provisioned in
     # metrics_corp's virtual private cloud.
@@ -88,10 +85,13 @@ if ! type ockam &>/dev/null; then
 fi
 
 # Check that tools we need are installed.
-for c in aws curl; do
+for c in aws curl gcloud; do
     if ! type "$c" &>/dev/null; then echo "ERROR: Please install: $c" && exit 1; fi
 done
 
+# It is required to use alphanumeric characters for the private endpoint name.
+export PRIVATE_ENDPOINT_NAME="ockamendpoint"
+
 # Check if the first argument is "cleanup"
 # If it is, call the cleanup function. If not, call the run function.
 if [ "$1" = "cleanup" ]; then cleanup; else run; fi