Merge pull request #382 from qilingframework/dev

Dev to 1.1-rc1

.github/workflows/pythonpublish.yml

@@ -1,18 +1,10 @@
-# This workflows will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+name: PyPI 📦 Distribution
 
-name: Upload Python Package
-
-on:
-   push:
-     tags:
-       - '*'
+on: [push]
 
 jobs:
-  deploy:
-
+  build:
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
@@ -21,13 +13,35 @@ jobs:
         python-version: '3.x'
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.pypi_pass }}
+        pip install setuptools wheel
+    - name: Build distribution 📦
       run: |
         pip install -r requirements.txt
         python setup.py sdist bdist_wheel
-        twine upload dist/*
+    - uses: actions/upload-artifact@v2
+      with:
+         path: ${{ github.workspace }}/dist/*
+
+  publish:
+    needs: [build]
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      - name: Publish distribution 📦 to test PyPI
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.testpypi_pass }}
+          repository_url: https://test.pypi.org/legacy/
+
+      - name: Publish distribution 📦 to PyPI
+        if: ${{ success() }}
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.pypi_pass }}

.travis.yml

@@ -77,7 +77,10 @@ matrix:
           if [[ ! -f /c/Python36/python ]]; then 
                choco install python --version=3.6.8
           fi
-        #- choco install python
+        # Prevent worker settings failure 
+        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
+        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
+        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
         #- choco install kb2999226
         #- python -m pip install --upgrade pip
       install:

CREDITS.TXT

@@ -8,6 +8,10 @@ Advisor
 =======
 NGUYEN Anh Quynh <[email protected]>
 
+Travis, Website and Documentations
+==================================
+FOO Kevin (chfl4gs)
+
 Core Developers
 ===============
 DING tianze (D1iv3) <[email protected]>
@@ -18,11 +22,10 @@ Earl MARCUS (klks84) [email protected]
 
 Key Contributors (in no particular order)
 =========================================
-FOO Kevin (chfl4gs)
 kabeor
 0ssigeno
 liba2k
-assaf_carlsba
+assafcarlsbad
 ucgJhe
 jhumble
 
@@ -53,4 +56,4 @@ phdphuc
 sashs
 knownsec
 hwiosec
-iamyeh
+iamyeh

ChangeLog

@@ -1,5 +1,14 @@
 This file details the changelog of Qiling Framework.
 
+------------------------------------
+[Version 1.1-rc1]: July 17th, 2020
+
+- More refactors and bug fixes
+- More detailed debug output
+- Fix MIPS_EB overflow issue
+- Introduce heap sanitizer
+
+
 ------------------------------------
 [Version 1.1-alpha2]: June 26th, 2020
 

MANIFEST.in

@@ -1,2 +1,3 @@
 recursive-include qiling/extensions/debugger/gdbserver/xml *
+recursive-include qiling/extensions/windows_sdk/defs *
 recursive-include qiling/profiles *

README.md

@@ -4,7 +4,7 @@
 ---
 
 <p align="center">
-<img width="150" height="150" src="docs/qiling_small.png">
+<img width="150" height="150" src="https://raw.githubusercontent.com/qilingframework/qiling/master/docs/qiling_small.png">
 </p>
 
 Qiling is an advanced binary emulation framework, with the following features:
@@ -27,7 +27,7 @@ Visit our website https://www.qiling.io for more information.
 
 #### License
 
-This project is released and distributed under [free software license GPLv2](COPYING).
+This project is released and distributed under [free software license GPLv2](https://github.com/qilingframework/qiling/blob/master/COPYING).
 
 ---
 
@@ -199,5 +199,15 @@ Contact us at email [email protected], or via Twitter [@qiling_io](https://twitter.
 - YU tong (sp1ke) <[email protected]>
 - Earl Marcus (klks84) [email protected] 
 
-#### Travis-CI, Docker and Website
+#### Travis-CI, Docker, Website and Documentation
+
 - FOO Kevin (chfl4gs) <[email protected]>
+
+#### Key Contributors (in no particular order)
+
+- kabeor
+- 0ssigeno
+- liba2k
+- assafcarlsbad
+- ucgJhe
+- jhumble

examples/ntQuerySystemInfo_x86.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# 
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+import sys
+sys.path.append("..")
+from qiling import *
+
+if __name__ == "__main__":
+    ql = Qiling(
+        ["rootfs/x86_windows/bin/NtQuerySystemInformation.exe"],
+        "rootfs/x86_windows",
+        output="default",
+        libcache = True
+    )
+    ql.run()

examples/src/windows/NtQuerySystemInfo.c

@@ -0,0 +1,39 @@
+#include <windows.h>
+#include <stdio.h>
+#include <Winternl.h >
+#include <stdlib.h>
+#pragma comment(lib ,"ntdll.lib")
+typedef struct _SYSTEM_BASIC_INFORMATION2
+{
+	ULONG Reserved;
+	ULONG TimerResolution;
+	ULONG PageSize;
+	ULONG NumberOfPhysicalPages;
+	ULONG LowestPhysicalPageNumber;
+	ULONG HighestPhysicalPageNumber;
+	ULONG AllocationGranularity;
+	ULONG_PTR MinimumUserModeAddress;
+	ULONG_PTR MaximumUserModeAddress;
+	ULONG_PTR ActiveProcessorsAffinityMask;
+	CCHAR NumberOfProcessors;
+} SYSTEM_BASIC_INFORMATION2, * PSYSTEM_BASIC_INFORMATION2;
+int main()
+{
+	ULONG whyme=0;
+	SYSTEM_BASIC_INFORMATION2 sbi;
+
+	NTSTATUS status = NtQuerySystemInformation(SystemBasicInformation, &sbi, sizeof(SYSTEM_BASIC_INFORMATION2), &whyme);
+	printf("Size : 0x%x Status: 0x%x\n", whyme,status);
+
+	printf("ActiveProcessorsAffinityMask : 0x%x\n", sbi.ActiveProcessorsAffinityMask);
+	printf("AllocationGranularity : 0x%x\n", sbi.AllocationGranularity);
+	printf("HighestPhysicalPageNumber : 0x%x\n", sbi.HighestPhysicalPageNumber);
+	printf("LowestPhysicalPageNumber : 0x%x\n", sbi.LowestPhysicalPageNumber);
+	printf("MaximumUserModeAddress : 0x%x\n", sbi.MaximumUserModeAddress);
+	printf("MinimumUserModeAddress : 0x%x\n", sbi.MinimumUserModeAddress);
+	printf("NumberOfPhysicalPages : 0x%x\n", sbi.NumberOfPhysicalPages);
+	printf("NumberOfProcessors : 0x%x\n", sbi.NumberOfProcessors);
+	printf("PageSize : 0x%x\n", sbi.PageSize);
+	printf("TimerResolution : 0x%x\n", sbi.TimerResolution);
+
+}

examples/uefi_sanitized_heap.py

@@ -0,0 +1,48 @@
+import os
+import sys
+from qiling import Qiling
+from qiling.extensions.sanitizers.heap import QlSanitizedMemoryHeap
+
+def my_abort(msg):
+    print(f"\n*** {msg} ***\n")
+    os.abort()
+
+def enable_sanitized_heap(ql, fault_rate=0):
+    ql.loader.heap = QlSanitizedMemoryHeap(ql, ql.loader.heap)
+    ql.loader.heap.pool_fault_rate = fault_rate
+    ql.loader.heap.oob_handler = lambda *args: my_abort("Out-of-bounds read detected")
+    ql.loader.heap.bo_handler = lambda *args: my_abort("Buffer overflow/underflow detected")
+    ql.loader.heap.bad_free_handler = lambda *args: my_abort("Double free or bad free detected")
+    ql.loader.heap.uaf_handler = lambda *args: my_abort("Use-after-free detected")
+
+def sanitized_emulate(path, rootfs, fault_type, output="debug", enable_trace=False):
+    ql = Qiling([path], rootfs, output=output)
+    ql.env['FaultType'] = fault_type
+    enable_sanitized_heap(ql)
+    ql.run()
+    if not ql.loader.heap.validate():
+        my_abort("Canary corruption detected")
+
+def usage():
+    print("""
+Usage: ./uefi_santizied_heap.py <fault-type>
+Valid fault types:
+0 - POOL_OVERFLOW_MEMCPY
+1 - POOL_UNDERFLOW_MEMCPY
+2 - POOL_OVERFLOW_USER,
+3 - POOL_UNDERFLOW_USER
+4 - POOL_OOB_READ_AHEAD
+5 - POOL_OOB_READ_BEHIND
+6 - POOL_DOUBLE_FREE
+7 - POOL_INVALID_FREE
+""")
+    sys.exit(0)
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        usage()
+
+    fault_type = bytes([int(sys.argv[1])])
+    rootfs = os.path.join(os.getcwd(), 'rootfs', 'x8664_efi')
+    path = os.path.join(rootfs, 'bin', 'EfiPoolFault.efi')
+    sanitized_emulate(path, rootfs, fault_type, output='debug', enable_trace=True)

qiling/const.py

@@ -51,6 +51,7 @@ class QL_INTERCEPT(IntEnum):
 D_PROT = 2 # Protocol level debug, print out open file flag
 D_CTNT = 3 # Print out content. File content or content of a tcp stream
 D_RPRT = 4 # Reporting output, main summarizing purposes
+D_DRPT = 5 # Detailed Report, with address
 
 QL_DEBUGGER_ALL = [QL_DEBUGGER.IDAPRO, QL_DEBUGGER.GDB]
 QL_ARCH_ALL = [QL_ARCH.X86, QL_ARCH.X8664, QL_ARCH.ARM, QL_ARCH.ARM64, QL_ARCH.MIPS]
@@ -80,4 +81,108 @@ class QL_INTERCEPT(IntEnum):
         "freebsd": QL_OS.FREEBSD,
         "windows": QL_OS.WINDOWS,
         "uefi": QL_OS.UEFI,
+}
+
+reptypedict = {
+            "BSTR": "POINTER",
+            "DLGPROC": "POINTER",
+            "DWORDLONG": "ULONGLONG",
+            "DWORD_PTR": "POINTER",
+            "GROUP": "INT",
+            "HDC": "POINTER",
+            "HEAP_INFORMATION_CLASS": "UINT",
+            "HGLOBAL": "POINTER",
+            "HHOOK": "POINTER",
+            "HINSTANCE": "HANDLE",
+            "HINTERNET": "POINTER",
+            "HKEY": "HANDLE",
+            "HLOCAL": "POINTER",
+            "HMODULE": "HANDLE",
+            "HOOKPROC": "POINTER",
+            "HRSRC": "POINTER",
+            "HWND": "HANDLE",
+            "INSTALLSTATE": "POINTER",
+            "INTERNET_PORT": "DWORD",
+            "INT_PTR": "POINTER",
+            "LARGE_INTEGER": "POINTER",
+            "LCID": "POINTER",
+            "LONG": "ULONGLONG",
+            "LPARAM": "POINTER",
+            "LPBOOL": "POINTER",
+            "LPBYTE": "POINTER",
+            "LPCCH": "POINTER",
+            "LPCONTEXT": "POINTER",
+            "LPCPINFO": "POINTER",
+            "LPCRITICAL_SECTION": "POINTER",
+            "LPCSTR": "STRING",
+            "LPCVOID": "POINTER",
+            "LPCWCH": "POINTER",
+            "LPCWSTR": "WSTRING",
+            "LPDWORD": "POINTER",
+            "LPFILETIME": "POINTER",
+            "LPINTERNET_BUFFERSA": "POINT",
+            "LPMESSAGEFILTER": "POINTER",
+            "LPMODULEINFO": "POINTER",
+            "LPNLSVERSIONINFO": "POINTER",
+            "LPOSVERSIONINFOA": "STRING",
+            "LPOSVERSIONINFOEXW": "POINTER",
+            "LPOSVERSIONINFOW": "WSTRING",
+            "LPOVERLAPPED": "POINTER",
+            "LPPOINT": "POINTER",
+            "LPPROCESSENTRY32W": "POINTER",
+            "LPSECURITY_ATTRIBUTES": "POINTER",
+            "LPSTARTUPINFOA": "POINTER",
+            "LPSTARTUPINFOW": "POINTER",
+            "LPSTR": "POINTER",
+            "LPSYSTEMTIME": "POINTER",
+            "LPSYSTEM_INFO": "POINTER",
+            "LPTHREAD_START_ROUTINE": "POINTER",
+            "LPTOP_LEVEL_EXCEPTION_FILTER": "DWORD",
+            "LPUNKNOWN": "POINTER",
+            "LPVOID": "POINTER",
+            "LPWCH": "POINTER",
+            "LPWIN32_FIND_DATAA": "POINTER",
+            "LPWORD": "POINTER",
+            "LPWSADATA": "STRING",
+            "LPWSAPROTOCOL_INFOA": "POINTER",
+            "LPWSTR": "POINTER",
+            "MSIHANDLE": "POINTER",
+            "OBJECT_INFORMATION_CLASS": "INT",
+            "OLECHAR": "WSTRING",
+            "PBOOL": "POINTER",
+            "PCACTCTXW": "POINTER",
+            "PCNZCH": "STRING",
+            "PDWORD": "POINTER",
+            "PFLS_CALLBACK_FUNCTION": "POINTER",
+            "PHKEY": "POINTER",
+            "PMEMORY_BASIC_INFORMATION": "POINTER",
+            "PROCESSINFOCLASS": "INT",
+            "PSECURITY_DESCRIPTOR": "POINTER",
+            "PSID": "HANDLE",
+            "PSID_IDENTIFIER_AUTHORITY": "POINTER",
+            "PSLIST_HEADER": "POINTER",
+            "PSRWLOCK": "POINTER",
+            "PULONG": "POINTER",
+            "PVECTORED_EXCEPTION_HANDLER": "HANDLE",
+            "PVOID": "POINTER",
+            "REFCLSID": "POINTER",
+            "REFIID": "POINTER",
+            "REGSAM": "POINTER",
+            "SHELLEXECUTEINFOA": "POINTER",
+            "SHELLEXECUTEINFOW": "POINTER",
+            "SHFILEINFOW": "POINTER",
+            "SOCKET": "INT",
+            "SOLE_AUTHENTICATION_SERVICE": "POINTER",
+            "TOKEN_INFORMATION_CLASS": "DWORD",
+            "UINT_PTR": "POINTER",
+            "ULONG": "UINT",
+            "ULONG_PTR": "POINTER",
+            "WORD": "DWORD",
+            "WPARAM": "UINT",
+            "_EXCEPTION_POINTERS": "POINTER",
+            "int": "INT",
+            "size_t": "UINT",
+            "sockaddr": "POINTER",
+            "unsigned int": "UINT",
+            "void": "POINTER"
 }

qiling/core.py

@@ -18,7 +18,7 @@
 from .core_utils import QLCoreUtils
 from .extensions.debugger import ql_debugger_init
 
-__version__ = "1.1" + "-alpha2"
+__version__ = "1.1" + "-rc1"
 
 class Qiling(QLCoreStructs, QLCoreHooks, QLCoreUtils):    
     def __init__(