Better formatting for output

Dockerfile

@@ -28,6 +28,7 @@ RUN apt-get update && \
     	apt-utils \
         rsync \
     	file \
+        dos2unix \
     	gettext && \
         apt-get clean && \
         ln -s /usr/bin/python3.8 /usr/bin/python && \
@@ -60,20 +61,22 @@ RUN mkdir -p ${CODEQL_HOME} \
     ${CODEQL_HOME}/codeql-go-repo \
     /opt/codeql
 
-RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
-    wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
-    unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME} && \
-    rm /tmp/codeql_linux.zip
-
 # get the latest codeql queries and record the HEAD
 RUN git clone https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \
     git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit
 RUN git clone https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \
     git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit
 
+RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \
+    wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \
+    unzip /tmp/codeql_linux.zip -d ${CODEQL_HOME} && \
+    rm /tmp/codeql_linux.zip
+
 ENV PATH="${CODEQL_HOME}/codeql:${PATH}"
 
 # Pre-compile our queries to save time later
-#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-repo/*/ql/src/codeql-suites/*-.qls
-#RUN codeql query compile --threads=0 ${CODEQL_HOME}/codelq-go-repo/ql/src/codeql-suites/*-.qls
+RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql-suites/*.qls
+RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-go-repo/ql/src/codeql-suites/*.qls
+
+ENV PYTHONIOENCODING=utf-8
 ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"]

container/libs/codeql.py

@@ -24,8 +24,8 @@ class CodeQL:
     ERROR_UNKNOWN_OS = 3
     ERROR_GIT_COMMAND = 4
 
-    def __init__(self, codeql_base_dir):
-        self.CODEQL_HOME = codeql_base_dir
+    def __init__(self, codeql_base_dir):
+        self.CODEQL_HOME = codeql_base_dir
 
     def download_and_install_latest_codeql(self, github_version):
         """

container/libs/utils.py

@@ -31,8 +31,12 @@ def check_output_wrapper(*args, **kwargs):
     try:
         return check_output(*args, **kwargs)
     except CalledProcessError as msg:
-        logger.warning('Error %s,%s,%s from command.', msg.returncode, msg.output, msg.stderr)
-        logger.debug('Output: %s', msg.output)
+        logger.error(f"Error {msg.returncode} executing from command.")
+        if msg.stderr != None:
+            print("Command Error Output:\n" + msg.stderr.decode('utf-8'))
+        if msg.output != None:
+            print("Command Output: \n" + msg.output.decode('utf-8'))
+        logger.error("Exiting...")
         sys.exit(ERROR_EXECUTING_COMMAND);
 
 def wipe_and_create_dir(dirname):

container/setup.py

@@ -33,10 +33,11 @@ def setup():
     Download and install the latest codeql cli
     Download and install the latest codeql queries
     """
+    logger.info("Starting setup...")
     args = parse_arguments()
     # check version and download the latest version
     get_latest_codeql(args)
-
+    logger.info("End setup...")
 def get_latest_codeql(args):
     # what version do we have?
     codeql = CodeQL(CODEQL_HOME)

container/startup.py

@@ -41,12 +41,11 @@ def main():
     # what command did the user ask to run? 
     if CODEQL_CLI_ARGS == False or CODEQL_CLI_ARGS == None or CODEQL_CLI_ARGS == ' ':
         # nothing to do
-        logger.info("No valid argument passed in for codeql-cli, nothing to do. To perform some task, please set the CODEQL_CLI_ARGS environment variable to a valid argument...")
+        logger.info("No argument passed in for codeql-cli, nothing to do. To perform some task, please set the CODEQL_CLI_ARGS environment variable to a valid argument...")
     else:
         codeql = CodeQL(CODEQL_HOME)
         run_result = codeql.execute_codeql_command(CODEQL_CLI_ARGS)
-        print(run_result)
-
+
     if WAIT_AFTER_EXEC:
         logger.info("Wait forever specified, waiting...")
         while True:

scripts/unix/analyze_security.sh

@@ -9,9 +9,15 @@ if [ "$#" -ne 2 ]; then
     exit 1
 fi
 
-#docker pull codeql/codeql-container
+RED=''
+RESET=''
+#docker pull sargemonkey/codeql-container
+#[ $? -eq 0 ] && echo "Pulled the container" || echo -e "failed to pull container";exit 1
 docker run --rm --name codeql-container -v "${inputfile}:/opt/src" -v "${outputfile}:/opt/results" -e CODEQL_CLI_ARGS=database\ create\ --language=python\ /opt/src/source_db csteosstools.azurecr.io/codeql/codeql-container
+[ $? -eq 0 ] && echo "Created the database" || echo -e "\n${RED}Failed to create the database${RESET}\n";exit 1
 docker run --rm --name codeql-container -v "${inputfile}:/opt/src" -v "${outputfile}:/opt/results" -e CODEQL_CLI_ARGS=database\ upgrade\ /opt/src/source_db csteosstools.azurecr.io/codeql/codeql-container 
+[ $? -eq 0 ] && echo "Upgraded the database" || echo -e "\n${RED}failed to upgrade the database${RESET}\n";exit 2
 docker run --rm --name codeql-container -v ${inputfile}:/opt/src -v ${outputfile}:/opt/results -e CODEQL_CLI_ARGS=database\ analyze\ /opt/src/source_db\ --format=sarifv2\ --output=/opt/results/issues.sarif\ python-security-and-quality.qls csteosstools.azurecr.io/codeql/codeql-container 
+[ $? -eq 0 ] && echo "Query execution successful" || echo -e "\n${RED}Query execution failed${RESET}\n"; exit 3
 
-echo "If there were no errors in the execution, the results file should be located at ${2}/issues.sarif"
+echo "The results file should be located at ${2}/issues.sarif"

scripts/windows/analyze_security.bat

@@ -20,7 +20,27 @@ exit /b 1
 rem docker pull codeql/codeql-container
 echo docker run --rm --name codeql-container -v "%inputfile%:/opt/src" -v "%outputfile%:/opt/results" -e CODEQL_CLI_ARGS="database create --language=python /opt/src/source_db" csteosstools.azurecr.io/codeql/codeql-container
 start /W /B docker run --rm --name codeql-container -v "%inputfile%:/opt/src" -v "%outputfile%:/opt/results" -e CODEQL_CLI_ARGS="database create --language=python /opt/src/source_db" csteosstools.azurecr.io/codeql/codeql-container
-echo docker run --rm --name codeql-container -v "%inputfile%:/opt/src" -v "%outputfile%:/opt/results" -e CODEQL_CLI_ARGS="database upgrade /opt/src/source_db" csteosstools.azurecr.io/codeql/codeql-container 
-echo docker run --rm --name codeql-container -v "%inputfile%:/opt/src" -v "%outputfile%:/opt/results" -e CODEQL_CLI_ARGS="database analyze /opt/src/source_db --format=sarifv2 --output=/opt/results/issues.sarif python-security-and-quality.qls" csteosstools.azurecr.io/codeql/codeql-container 
 
-echo "If there were no errors in the execution, the results file should be located at %2/issues.sarif"
+call :print_status "Failed creating the database" , %errorlevel%
+if %errorlevel% GTR 0 (
+    call :print_exit_error "Failed creating the database"    
+    exit /b %errorlevel%
+)
+start /W /B docker run --rm --name codeql-container -v "%inputfile%:/opt/src" -v "%outputfile%:/opt/results" -e CODEQL_CLI_ARGS="database upgrade /opt/src/source_db" csteosstools.azurecr.io/codeql/codeql-container 
+if %errorlevel% GTR 0 (
+    call :print_exit_error "Failed upgrading the database"    
+    exit /b %errorlevel%
+)
+start /W /B docker run --rm --name codeql-container -v "%inputfile%:/opt/src" -v "%outputfile%:/opt/results" -e CODEQL_CLI_ARGS="database analyze /opt/src/source_db --format=sarifv2 --output=/opt/results/issues.sarif python-security-and-quality.qls" csteosstools.azurecr.io/codeql/codeql-container 
+if %errorlevel% GTR 0 (
+    call :print_exit_error "Failed to run the query on the database"    
+    exit /b %errorlevel%
+)
+echo "The results file should be located at %2\issues.sarif"
+
+
+:print_exit_error
+    echo.
+    echo %~1
+    echo.
+    echo Exiting...