Merge pull request #684 from qilingframework/dev

Dev
qilingframework · Feb 8, 2021 · d0e5cdc · d0e5cdc
2 parents d3df704 + d942c5d
commit d0e5cdc
Show file tree

Hide file tree

Showing 303 changed files with 4,864 additions and 4,499 deletions.
diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml
@@ -5,19 +5,30 @@ env:
 jobs:
   tests:
     runs-on: ${{ matrix.os }}
-    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    name: Python ${{ matrix.python-version }} on ${{ matrix.os }} ${{ matrix.container }}
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-2019, macos-10.15, ubuntu-18.04, ubuntu-20.04]
-        python-version: [3.6.8, 3.7.6]
+        os: [ubuntu-18.04, ubuntu-20.04]
+        python-version: [3.6.8, 3.7.6, 3.8.5]
         exclude:
-          - os: windows-2019
-            python-version: 3.7.6
-          - os: macos-10.15
-            python-version: 3.7.6
+ #         - os: windows-2019
+ #           python-version: 3.7.6
+ #         - os: macos-10.15
+ #           python-version: 3.7.6
           - os: ubuntu-20.04
             python-version: 3.7.6
+#          - os: windows-2019
+#            python-version: 3.8.5
+#          - os: macos-10.15
+#            python-version: 3.8.5
+          - os: ubuntu-20.04
+            python-version: 3.8.5
+        include:
+          - os: ubuntu-20.04
+            python-version: 3.6.8
+            container: Docker
+
     steps:
     - uses: actions/checkout@v2
 
@@ -26,41 +37,41 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: win setup MSVC
-      if: contains(matrix.os, 'windows')
-      uses: microsoft/setup-msbuild@v1
+#    - name: win setup MSVC
+#      if: contains(matrix.os, 'windows')
+#      uses: microsoft/setup-msbuild@v1
 
-    - name: win run tests
-      if: contains(matrix.os, 'windows')
-      shell: bash
-      run: |
-         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
-         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
-         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
-         powershell Add-MpPreference -ExclusionPath $GITHUB_WORKSPACE
-         pip3 install setuptools wheel
-         pip3 install .
-         cmd.exe //C 'examples\scripts\dllscollector.bat'
-         cd $GITHUB_WORKSPACE/examples/rootfs/x86_windows/bin
-         unzip -Pinfected wannacry.bin.zip
-         unzip -Pinfected UselessDisk.bin.zip
-         unzip -Pinfected GandCrab502.bin.zip
-         unzip -Pinfected al-khaser.bin.zip
-         unzip -Pinfected sality.dll.zip
-         cd $GITHUB_WORKSPACE/tests
-         cmd.exe //C '.\test_pe.bat'
-    - name: mac run tests
-      if: contains(matrix.os, 'macos')
-      continue-on-error: true
-      shell: bash
-      run: |
-         pip3 install setuptools wheel
-         pip3 install .
-         ./examples/scripts/dylibcollector.sh
-         cd $GITHUB_WORKSPACE/examples/rootfs/x8664_macos/kext
-         unzip -Pinfected SuperRootkit.kext.zip
-         cd $GITHUB_WORKSPACE/tests
-         ./test_macho.sh
+#    - name: win run tests
+#      if: contains(matrix.os, 'windows')
+#      shell: bash
+#      run: |
+#         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
+#         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
+#         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
+#         powershell Add-MpPreference -ExclusionPath $GITHUB_WORKSPACE
+#         pip3 install setuptools wheel
+#         pip3 install .
+#         cmd.exe //C 'examples\scripts\dllscollector.bat'
+#         cd $GITHUB_WORKSPACE/examples/rootfs/x86_windows/bin
+#         unzip -Pinfected wannacry.bin.zip
+#         unzip -Pinfected UselessDisk.bin.zip
+#         unzip -Pinfected GandCrab502.bin.zip
+#         unzip -Pinfected al-khaser.bin.zip
+#         unzip -Pinfected sality.dll.zip
+#         cd $GITHUB_WORKSPACE/tests
+#         cmd.exe //C '.\test_pe.bat'
+#    - name: mac run tests
+#      if: contains(matrix.os, 'macos')
+#      continue-on-error: true
+#      shell: bash
+#      run: |
+#         pip3 install setuptools wheel
+#         pip3 install .
+#         ./examples/scripts/dylibcollector.sh
+#         cd $GITHUB_WORKSPACE/examples/rootfs/x8664_macos/kext
+#         unzip -Pinfected SuperRootkit.kext.zip
+#         cd $GITHUB_WORKSPACE/tests
+#         ./test_macho.sh
     - name: linux run tests
       if: contains(matrix.os, 'ubuntu')
       shell: 'script -q -e -c "bash {0}"'
@@ -69,11 +80,13 @@ jobs:
               pip3 install setuptools wheel flake8
               flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
               pip3 install .
-              cd tests && ./test_elf.sh
+              cd examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
+              cd ../../../../tests && ./test_elf.sh
         elif [ ${{ matrix.os }} == 'ubuntu-20.04' ]; then
-              docker run -it --rm -v ${GITHUB_WORKSPACE}:/qiling qilingframework/qiling:dev bash -c "pip3 install . && cd tests && ./test_elf.sh"
+              docker run -it --rm -v ${GITHUB_WORKSPACE}:/qiling qilingframework/qiling:dev bash -c "pip3 install . && cd examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip && cd ../../../../tests && ./test_elf.sh"
         else
               pip3 install setuptools wheel
               pip3 install .
-              cd tests && ./test_elf.sh
+              cd examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
+              cd ../../../../tests && ./test_elf.sh
         fi
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@
 .idea
 *.pyc
 *.cache
+*.cache2
 .*.swp
 *.raw
 

diff --git a/.travis.yml b/.travis.yml
@@ -33,3 +33,58 @@ matrix:
       script:
         - ./test_macho.sh
 
+    - name: "Python 3.6.8 on Windows"
+      os: windows
+      language: shell
+      env:
+        - PATH="/c/Python36:/c/Python36/Scripts:$PATH"
+      cache:
+        directories:
+#          - $HOME/AppData/Local/Temp/chocolatey
+          - $HOME/AppData/Local/pip/Cache
+          - /c/Python36
+#          - $HOME/AppData/Local/NuGet/Cache
+      before_install:
+        - | 
+          if [[ ! -f /c/Python36/python ]]; then 
+               choco install python --version=3.6.8
+          fi
+        # Prevent worker settings failure 
+        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
+        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
+        - powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
+        #- choco install kb2999226
+        #- python -m pip install --upgrade pip
+      install:
+#        - pip3 install wheel 'capstone>=4.0.1' 'pefile>=2019.4.18' 'python-registry>=1.3.1' 'unicorn>=1.0.2rc3'
+#        - |
+#          if [ ! -f $HOME/dist/keystone*.zip ]; then
+#               git clone https://github.com/keystone-engine/keystone && \
+#               cd keystone && \
+#               mkdir build && \
+#               cd build && \
+#               cmd.exe //C 'C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat' amd64 '&' cmd.exe //C '..\nmake-dll.bat' '&&' nmake instalL && \
+#               cd ../bindings/python && \
+#               python setup.py install && \
+#               python setup.py bdist --formats=zip && \
+#               cp dist/*.zip $HOME/dist/ && \
+#               cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll /c/Python36/Lib/site-packages/keystone/ && \
+#               cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll $HOME/dist/;
+#          else
+#              unzip $HOME/dist/*.zip -d /c && \
+#              cp $HOME/dist/keystone.dll /c/Python36/Lib/site-packages/keystone/;
+#          fi
+        - cd $TRAVIS_BUILD_DIR
+        - pip3 install .
+#        - cp /c/Program\ Files\ \(x86\)/keystone/lib/keystone.dll /c/Python36/Lib/site-packages/keystone/
+      before_script:
+        - cmd.exe //C 'examples\scripts\dllscollector.bat'
+        - cd $TRAVIS_BUILD_DIR/examples/rootfs/x86_windows/bin
+        - unzip -Pinfected wannacry.bin.zip
+        - unzip -Pinfected UselessDisk.bin.zip
+        - unzip -Pinfected GandCrab502.bin.zip
+        - unzip -Pinfected al-khaser.bin.zip
+        - unzip -Pinfected sality.dll.zip
+        - cd $TRAVIS_BUILD_DIR/tests
+      script:
+        - cmd.exe //C '.\test_pe.bat'         
diff --git a/CREDITS.TXT b/CREDITS.TXT
@@ -1,17 +1,21 @@
 This file credits all the contributors of the Qiling Framework project.
 
+
 Project Leader
 ==============
 LAU kaijern (xwings) <[email protected]>
 
+
 Advisor
 =======
 NGUYEN Anh Quynh <[email protected]>
 
+
 Travis, Website and Documentations
 ==================================
 FOO Kevin (chfl4gs) <[email protected]>
 
+
 Core Developers
 ===============
 DING tianze (D1iv3) <[email protected]>
@@ -22,11 +26,6 @@ Earl MARCUS (klks84) [email protected]
 WU chenxu (kabeor) <[email protected]>
 KONG ziqiao (lazymio) <[email protected]>
 
-Demigod team (https://groundx.io/demigod)
-=========================================
-NGUYEN Anh Quynh
-NGUYEN Hong Quang
-DO Minh Tuan
 
 Key Contributors (in no particular order)
 =========================================
@@ -35,6 +34,18 @@ liba2k
 assafcarlsbad
 ucgJhe
 jhumble
+Mark Jansen (learn-more)
+cq674350529
+elicn
+bkerler (viperbjk)
+
+
+Demigod team (https://groundx.io/demigod)
+=========================================
+NGUYEN Anh Quynh
+NGUYEN Hong Quang
+DO Minh Tuan
+
 
 Contributors (in no particular order)
 =====================================
@@ -48,9 +59,8 @@ danielhenrymantilla
 iamyeh
 alfink
 bambu
-bkerler (viperbjk)
-Mark Jansen (learn-more)
-cq674350529
+
+
 
 Alpha testers (in no particular order, named by github id)
 ==========================================================

diff --git a/ChangeLog b/ChangeLog
@@ -7,9 +7,30 @@ BREAK CHANGE
 - ql.filename is renamed to ql.argv.
 - ql.output and ql.verbose now has slightly different meanings and can be adjusted runtime. See their docstring for details.
 - ql.filter now accepts a regular expression.
+- Remove ql.log_dir, ql.log_split, ql.append but add ql.log_file instead.
+
+
+------------------------------------
+[Version 1.2.3]: March [SOMETHING], 2021
+- 
+
 
 ------------------------------------
-[Version 1.3]: January [SOMETHING], 2021
+[Version 1.2.2]: February 8, 2021
+
+- Fix _acmdln and _wcmdln handling
+- More UEFI refactor
+- Refactor common OS space
+- Bring sality test to work again
+- Clean up more test case
+- First stage multithread rewrite done
+- Updated Qiling(shellcode=) to Qiling(code=), still keeping Qiling(shellcode=) for legacy purpose
+- Added support for SMM_RUNTIME_SERVICES_TABLE
+- Fixed regression in code coverage collection
+- Added generic ql.mem.read_ptr helper function
+- merged UEFI, windows, linux and macos print_function
+- merged UEFI, windows, linux and macos fncc
+- make MacOS uses more Qiling API
 
 
 ------------------------------------

diff --git a/Dockerfile b/Dockerfile
@@ -8,8 +8,9 @@ RUN apt-get update \
   && apt-get -y upgrade \
   && apt-get install -y --no-install-recommends cmake build-essential gcc git
 
-RUN git clone -b dev https://github.com/qilingframework/qiling.git \
-  && cd qiling \
+COPY . /qiling
+
+RUN cd /qiling \
   && pip wheel . -w wheels
 
 FROM python:3.6-slim AS base
@@ -19,6 +20,7 @@ COPY --from=builder /qiling /qiling
 WORKDIR /qiling
 
 RUN apt-get update \
+  && apt-get install -y --no-install-recommends unzip \
   && rm -rf /var/lib/apt/lists/* \
   && pip3 install wheels/*.whl \
   && rm -rf wheels

diff --git a/README.md b/README.md
@@ -233,6 +233,10 @@ Contact us at email [email protected], or via Twitter [@qiling_io](https://twitter.
 - assafcarlsbad
 - ucgJhe
 - jhumble
+- Mark Jansen (learn-more)
+- cq674350529
+- elicn
+- bkerler (viperbjk)
 
 ---
 

diff --git a/docs/bg_page.png b/docs/bg_page.png
diff --git a/examples/adcache_x86_windows_debug.py b/examples/adcache_x86_windows_debug.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org)
+#
 
 import sys
 from zipfile import ZipFile
@@ -13,6 +13,6 @@
     with ZipFile("shellcodes/win32_https_download.zip") as zip_reader:
         with zip_reader.open('win32_https_download.bin', 'r', b'infected') as f:
             sc = f.read()
-            ql = Qiling(shellcoder=sc, archtype="x86", ostype="windows",
+            ql = Qiling(code=sc, archtype="x86", ostype="windows",
                         rootfs="rootfs/x86_windows", output="debug")
             ql.run()
diff --git a/examples/cachedlls_x8664_windows.py b/examples/cachedlls_x8664_windows.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+#
 import sys
 sys.path.append("..")
 from qiling import *

diff --git a/examples/crackme_x86_linux.py b/examples/crackme_x86_linux.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+#
 import os
 
 from unicorn import *

diff --git a/examples/crackme_x86_windows.py b/examples/crackme_x86_windows.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+#
 from unicorn import *
 
 from unicorn.x86_const import *

diff --git a/examples/crackme_x86_windows_auto.py b/examples/crackme_x86_windows_auto.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+#
 
 from unicorn.x86_const import *
 

diff --git a/examples/crackme_x86_windows_setcallback.py b/examples/crackme_x86_windows_setcallback.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+#
 import sys
 sys.path.append("..")
 from qiling import *

diff --git a/examples/crackme_x86_windows_unpatch.py b/examples/crackme_x86_windows_unpatch.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
-# Built on top of Unicorn emulator (www.unicorn-engine.org) 
+#
 from unicorn.x86_const import *
 
 import sys
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ @@
     .idea
     *.pyc
     *.cache
+    *.cache2
     .*.swp
     *.raw
@@ Expand Down @@