Henrykie/perforce (#38)

assets/packer/perforce/helix-core/README.md

@@ -0,0 +1,54 @@
+# Helix Core Server Deployment Script README
+
+## Overview
+
+This Bash script automates the deployment and configuration of a Helix Core Server (P4D) on a Linux environment, specifically tailored for use with SELinux and systemd. It performs various tasks such as checking and setting up the necessary user and group, handling SELinux context, installing and configuring Perforce Software's Server Deployment Package (SDP), and setting up the Helix Core service with systemd.
+
+### What it Does:
+
+1. **Pre-Flight Checks**: Ensures the script is run with root privileges.
+2. **Environment Setup**: Defines paths and necessary constants for the installation.
+3. **SELinux Handling**: Checks if SELinux is enabled and installs required packages.
+4. **User and Group Verification**: Ensures the 'perforce' user and group exist.
+5. **Directory Creation and Ownership**: Ensures necessary directories exist and have correct ownership.
+6. **Helix Binaries and SDP Installation**: Downloads and extracts SDP, checks for Helix binaries, and downloads them if missing.
+7. **Systemd Service Configuration**: Sets up a systemd service for the p4d server.
+8. **SSL Configuration**: Updates SSL certificate configuration with the EC2 instance DNS name.
+9. **SELinux Context Management**: Updates SELinux context for p4d.
+10. **Crontab Initialization**: Sets up crontab for the 'perforce' user.
+11. **SDP Verification**: Runs a script to verify the SDP installation.
+
+## Prerequisites
+
+- A Linux system with DNF package manager (e.g., Fedora, RHEL, CentOS).
+- Root access to the system.
+- SELinux in Enforcing or Permissive mode (optional but recommended).
+- Access to the internet for downloading necessary packages and binaries.
+
+## How to Use
+
+1. **Download the Script**: Clone or download this repository to your system.
+2. **Provide Execution Permission**: Give execute permission to the script using `chmod +x <script_name>.sh`.
+3. **Run the Script**: Execute the script as root:
+   ```
+   sudo ./<script_name>.sh
+   ```
+4. **Follow the On-Screen Instructions**: The script is mostly automated, but monitor the output for any errors or required manual inputs.
+
+## Important Notes
+
+- This script is designed for a specific use-case and might require modifications for different environments or requirements.
+- Ensure you have a backup of your system before running the script, as it makes significant changes to users, groups, and services.
+- The script assumes an internet connection for downloading packages and binaries.
+
+## Contributing
+
+Contributions to improve the script or documentation are welcome. Please submit pull requests or raise issues as needed.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+---
+
+_This README provides a basic overview. Please refer to the script comments and documentation for detailed understanding of each component and its function._

assets/packer/perforce/helix-core/example.pkrvars.hcl

@@ -0,0 +1,5 @@
+region = "us-west-2"
+vpc_id = "<VPCID FOR BUILDER INSTANCE>"
+# You will need access to this subnet from the provisioning machine
+subnet_id = "<SUBNETID FOR BUILDER INSTANCE>"
+profile = "DEFAULT"

assets/packer/perforce/helix-core/p4_configure.sh

@@ -0,0 +1,305 @@
+#!/bin/bash
+
+
+#Currently this needs proper EBS volume locations from /dev with proper nvme names $1 is a hxlogs $2 hxmetadata $3 hxdepots $4 perforce server type  p4d_master/ p4d_replica
+
+# Log file location
+LOG_FILE="/var/log/p4_configure.log"
+
+# Ensure the script runs only once
+FLAG_FILE="/var/run/p4_configure_ran.flag"
+
+if [ -f "$FLAG_FILE" ]; then
+    echo "Script has already run. Exiting."
+    exit 0
+fi
+
+# Function to log messages
+log_message() {
+    echo "$(date) - $1" >> $LOG_FILE
+}
+
+# Function to check if path is an FSx mount point
+is_fsx_mount() {
+    echo "$1" | grep -qE 'fs-[0-9a-f]{17}\.fsx\.[a-z0-9-]+\.amazonaws\.com:/' #to be verified if catches all fsxes
+    return $?
+}
+
+# Function to create and mount XFS on EBS
+prepare_ebs_volume() {
+    local ebs_volume=$1
+    local mount_point=$2
+
+    # Check if the EBS volume has a file system
+    local fs_type=$(lsblk -no FSTYPE "$ebs_volume")
+
+    if [ -z "$fs_type" ]; then
+        log_message "Creating XFS file system on $ebs_volume."
+        mkfs.xfs "$ebs_volume"
+    fi
+
+    log_message "Mounting $ebs_volume on $mount_point."
+    mount "$ebs_volume" "$mount_point"
+}
+
+# Function to copy SiteTags template and update with AWS regions -> This file will be updated by Ansible with replica AWS regions.
+
+prepare_site_tags() {
+  log_message "Setting up SiteTags for installation"
+  local source="/hxdepots/sdp/Server/Unix/p4/common/config/SiteTags.cfg.sample"
+  local target="/hxdepots/p4/common/config/SiteTags.cfg"
+  local region="$1"
+
+  # Ensure the source file exists before attempting to copy
+  if [ ! -f "$source" ]; then
+    log_message "Error: Source file $source does not exist"
+    return 1  # Exit the function with an error status
+  fi
+
+  # Attempt to copy the file and check if the copy operation was successful
+  if ! cp "$source" "$target"; then
+    log_message "Error: Failed to copy $source to $target"
+    return 1  # Exit the function with an error status
+  fi
+
+  # Remove hyphens from the region string for aws_info
+  local aws_info="aws${region//-/}"
+
+  # Append the AWS info to the target file with the original region format
+  # Using printf to handle the new line correctly across different shells
+  printf "\n%s: AWS %s\n" "$aws_info" "$region" >> "$target"
+
+  log_message "Added $aws_info as a site tag"
+}
+
+
+
+
+# Starting the script
+log_message "Starting the p4 configure script."
+
+# Check if the script received three arguments
+if [ "$#" -ne 4 ]; then
+    log_message "Incorrect usage. Expected 4 arguments, got $#."
+    log_message "Usage: $0 <EBS path or FSx for hxlogs> <EBS path or FSx for hxmetadata> <EBS path or FSx for hxdepots> <p4d_master or p4d_replica>"
+    exit 1
+fi
+
+# Assigning arguments to variables
+EBS_LOGS=$1
+EBS_METADATA=$2
+EBS_DEPOTS=$3
+P4D_TYPE=$4
+
+# Function to perform operations
+perform_operations() {
+    log_message "Performing operations for mounting and syncing directories."
+
+    # Check each mount type and mount accordingly
+    mount_fs_or_ebs() {
+        local mount_point=$1
+        local dest_dir=$2
+        if is_fsx_mount "$mount_point"; then
+            # Mount as FSx
+            mount -t nfs -o nconnect=16,rsize=1048576,wsize=1048576,timeo=600 "$mount_point" "$dest_dir"
+        else
+            # Mount as EBS the called function also creates XFS on EBS
+
+            prepare_ebs_volume "$mount_point" "$dest_dir"
+        fi
+    }
+
+    # Create temporary directories and mount
+    mkdir -p /mnt/temp_hxlogs
+    mkdir -p /mnt/temp_hxmetadata
+    mkdir -p /mnt/temp_hxdepots
+
+    mount_fs_or_ebs $EBS_LOGS /mnt/temp_hxlogs
+    mount_fs_or_ebs $EBS_METADATA /mnt/temp_hxmetadata
+    mount_fs_or_ebs $EBS_DEPOTS /mnt/temp_hxdepots
+
+    # Syncing directories
+    rsync -av /hxlogs/ /mnt/temp_hxlogs/
+    rsync -av /hxmetadata/ /mnt/temp_hxmetadata/
+    rsync -av /hxdepots/ /mnt/temp_hxdepots/
+
+    # Unmount temporary mounts
+    umount /mnt/temp_hxlogs
+    umount /mnt/temp_hxmetadata
+    umount /mnt/temp_hxdepots
+
+    # Clear destination directories
+    rm -rf /hxlogs/*
+    rm -rf /hxmetadata/*
+    rm -rf /hxdepots/*
+
+    # Mount EBS volumes or FSx to final destinations
+    mount_fs_or_ebs $EBS_LOGS /hxlogs
+    mount_fs_or_ebs $EBS_METADATA /hxmetadata
+    mount_fs_or_ebs $EBS_DEPOTS /hxdepots
+
+    log_message "Operation completed successfully."
+}
+
+# Check if EBS volumes or FSx mount points are provided for all required paths
+if ( [ -e "$EBS_LOGS" ] || is_fsx_mount "$EBS_LOGS" ) && \
+   ( [ -e "$EBS_METADATA" ] || is_fsx_mount "$EBS_METADATA" ) && \
+   ( [ -e "$EBS_DEPOTS" ] || is_fsx_mount "$EBS_DEPOTS" ); then
+    perform_operations
+else
+    log_message "One or more required paths are not valid EBS volumes or FSx mount points. No operations performed. Will continue with single disk setup"
+fi
+
+log_message "Starting the configuration part after mounting was done later will configure the commit or replica depending on configuration."
+
+SDP_Setup_Script=/hxdepots/sdp/Server/Unix/setup/mkdirs.sh 
+SDP_New_Server_Script=/p4/sdp/Server/setup/configure_new_server.sh 
+SDP_Live_Checkpoint=/p4/sdp/Server/Unix/p4/common/bin/live_checkpoint.sh 
+SDP_Offline_Recreate=/p4/sdp/Server/Unix/p4/common/bin/recreate_offline_db.sh 
+SDP_Client_Binary=/hxdepots/sdp/helix_binaries/p4 
+SDP=/hxdepots/sdp
+TOKEN=$(curl --request PUT "http://169.254.169.254/latest/api/token" --header "X-aws-ec2-metadata-token-ttl-seconds: 3600") # This is only for the metadata V2 need to check go and try the V1 with no token and see which one works. 
+EC2_DNS_PRIVATE=$(curl -s http://169.254.169.254/latest/meta-data/hostname --header "X-aws-ec2-metadata-token: $TOKEN") # same need to check for V2 vs V1
+SDP_Setup_Script_Config=/hxdepots/sdp/Server/Unix/setup/mkdirs.cfg # Config to the new script needed for mkdirs.sh
+AWS_REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region --header "X-aws-ec2-metadata-token: $TOKEN") # Get AWS region for SiteTags
+
+cd /hxdepots/sdp/Server/Unix/setup # need to cd other
+
+
+#update the mkdirs.cfg so it has proper hostname a private DNS form EC2 otherwise adding replica is not possible due to wrong P4TARGET settings.
+
+if [ ! -f "$SDP_Setup_Script_Config" ]; then
+    log_message "Error: Configuration file not found at $SDP_Setup_Script_Config."
+    exit 1
+fi
+
+# Check if p4d_master server and update sitetags
+
+# Update P4MASTERHOST value in the configuration file
+sed -i "s/^P4MASTERHOST=.*/P4MASTERHOST=$EC2_DNS_PRIVATE/" "$SDP_Setup_Script_Config"
+
+log_message "Updated P4MASTERHOST to $EC2_DNS_PRIVATE in $SDP_Setup_Script_Config."
+
+
+log_message "Mounting done ok - continue to the install"
+
+# Execute mkdirs.sh from the package
+if [ -f "$SDP_Setup_Script" ] && [ -n $P4D_TYPE ]; then
+  chmod +x "$SDP_Setup_Script"
+  "$SDP_Setup_Script" 1 -t $P4D_TYPE
+else
+  log_message "Setup script (mkdirs.sh) not found or P4D Type: $P4D_TYPE not provided."
+fi
+
+# update cert config with ec2 DNS name
+FILE_PATH="/p4/ssl/config.txt"
+
+# Retrieve the EC2 instance DNS name
+EC2_DNS_NAME=$(curl -s http://169.254.169.254/latest/meta-data/public-hostname --header "X-aws-ec2-metadata-token: $TOKEN")
+
+
+# Check if the DNS name was successfully retrieved
+if [ -z "$EC2_DNS_NAME" ]; then
+  echo "Failed to retrieve EC2 instance DNS name."
+  exit 1
+fi
+
+# Replace REPL_DNSNAME with the EC2 instance DNS name for ssl certificate generation
+sed -i "s/REPL_DNSNAME/$EC2_DNS_NAME/" "$FILE_PATH"
+
+echo "File updated successfully."
+
+I=1
+# generate certificate 
+
+/p4/common/bin/p4master_run ${I} /p4/${I}/bin/p4d_${I} -Gc
+
+# Configure systemd service to start p4d
+
+
+cd /etc/systemd/system
+sed -e "s:__INSTANCE__:$I:g" -e "s:__OSUSER__:perforce:g" $SDP/Server/Unix/p4/common/etc/systemd/system/p4d_N.service.t > p4d_${I}.service
+chmod 644 p4d_${I}.service
+systemctl daemon-reload
+
+
+# update label for selinux
+semanage fcontext -a -t bin_t /p4/1/bin/p4d_1_init
+restorecon -vF /p4/1/bin/p4d_1_init
+
+# start service
+systemctl start p4d_1
+
+# Wait for the p4d service to start before continuing
+wait_for_service "p4d_1"
+
+P4PORT=ssl:1666
+P4USER=perforce
+
+#probably need to copy p4 binary to the /usr/bin or add to the path variable to avoid running with a full path adding:
+#permissions for lal users:
+
+
+chmod +x /hxdepots/sdp/helix_binaries/p4
+ln -s $SDP_Client_Binary /usr/bin/p4
+
+# now can test:
+p4 -p ssl:$HOSTNAME:1666 trust -y
+
+
+# Execute new server setup from the extracted package
+if [ -f "$SDP_New_Server_Script" ]; then
+  chmod +x "$SDP_New_Server_Script"
+  "$SDP_New_Server_Script" 1
+else
+  echo "Setup script (configure_new_server.sh) not found."
+fi
+
+
+
+# create a live checkpoint and restore offline db
+# switching to user perforce
+
+
+if [ -f "$SDP_Live_Checkpoint" ]; then
+  chmod +x "$SDP_Live_Checkpoint"
+  sudo -u perforce "$SDP_Live_Checkpoint" 1
+else
+  echo "Setup script (SDP_Live_Checkpoint) not found."
+fi
+
+if [ -f "$SDP_Offline_Recreate" ]; then
+  chmod +x "$SDP_Offline_Recreate"
+  sudo -u perforce "$SDP_Offline_Recreate" 1
+else
+  echo "Setup script (SDP_Offline_Recreate) not found."
+fi
+
+# initialize crontab for user perforce
+
+sudo -u perforce crontab /p4/p4.crontab.1
+
+# verify sdp installation should warn about missing license only:
+/hxdepots/p4/common/bin/verify_sdp.sh 1
+
+
+# Check if the AWS_REGION variable is empty if not prepare for replication.
+if [ -z "$AWS_REGION" ] && [ "$P4D_TYPE" = "p4d_master" ]; then
+    log_message "Error: Not able to get the AWS Region from instance Metadata"
+    exit 1
+else
+    prepare_site_tags "$AWS_REGION"
+    log_message "Created SiteTags file appended AWS Region of this instance"
+fi
+
+
+
+# Create the flag file to prevent re-run
+touch "$FLAG_FILE"
+
+
+
+
+# Ending the script
+log_message "EC2 mount script finished."
+