Merge pull request #749 from qilingframework/dev

Getting ready for 1.2.3
qilingframework · Mar 29, 2021 · 4ba05d0 · 4ba05d0
2 parents 88a76ed + 53c7d58
commit 4ba05d0
Show file tree

Hide file tree

Showing 358 changed files with 10,114 additions and 5,456 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,51 @@
+version: 2.1
+
+jobs:
+  macos:
+    macos:
+      xcode: 10.1
+    environment:
+      HOMEBREW_NO_AUTO_UPDATE: 1
+      MACOSX_DEPLOYMENT_TARGET: 10.13.6
+    steps:
+      - checkout
+      - run:
+          name: "Install wget"
+          command: |
+            brew install wget cmake
+
+      - restore_cache:
+          keys:
+            - python-{{ .Environment.CIRCLE_JOB }}-3.7.0-macos-10.13.6
+
+      - run: 
+          name: "Install qiling framework"
+          command: |
+            pip3 install --upgrade pip
+            pip3 install wheel setuptools
+            pip3 install .
+            cd examples
+            rm -rf rootfs
+            wget https://github.com/qilingframework/rootfs/archive/refs/heads/master.zip 
+            unzip master.zip && mv rootfs-master rootfs
+            cd .. && ./examples/scripts/dylibcollector.sh
+            cd examples/rootfs/x8664_macos/kext
+            unzip -Pinfected SuperRootkit.kext.zip
+
+      - save_cache:
+          paths:
+            - ~/Library/Caches/pip
+          key: python-{{ .Environment.CIRCLE_JOB }}-3.7.0-macos-10.13.6
+
+      - run:
+          name: "Run macos test"
+          command: |
+             cd tests
+             ./test_macho.sh
+
+
+workflows:
+  version: 2
+  run-tests:
+    jobs:
+      - macos
diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml
@@ -9,21 +9,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, ubuntu-20.04]
+        #os: [windows-2019, macos-10.15, ubuntu-18.04, ubuntu-20.04]
+        os: [windows-2019, ubuntu-18.04, ubuntu-20.04]
         python-version: [3.6.8, 3.7.6, 3.8.5]
         exclude:
- #         - os: windows-2019
- #           python-version: 3.7.6
- #         - os: macos-10.15
- #           python-version: 3.7.6
           - os: ubuntu-20.04
-            python-version: 3.7.6
-#          - os: windows-2019
-#            python-version: 3.8.5
-#          - os: macos-10.15
-#            python-version: 3.8.5
+            python-version: 3.7.6       
           - os: ubuntu-20.04
             python-version: 3.8.5
+          - os: windows-2019
+            python-version: 3.6.8
+          - os: windows-2019
+            python-version: 3.7.6                 
+          # - os: macos-10.15
+          #   python-version: 3.6.8
+          # - os: macos-10.15
+          #   python-version: 3.7.6                       
         include:
           - os: ubuntu-20.04
             python-version: 3.6.8
@@ -37,41 +38,47 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-#    - name: win setup MSVC
-#      if: contains(matrix.os, 'windows')
-#      uses: microsoft/setup-msbuild@v1
 
-#    - name: win run tests
-#      if: contains(matrix.os, 'windows')
-#      shell: bash
-#      run: |
-#         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
-#         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
-#         powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
-#         powershell Add-MpPreference -ExclusionPath $GITHUB_WORKSPACE
-#         pip3 install setuptools wheel
-#         pip3 install .
-#         cmd.exe //C 'examples\scripts\dllscollector.bat'
-#         cd $GITHUB_WORKSPACE/examples/rootfs/x86_windows/bin
-#         unzip -Pinfected wannacry.bin.zip
-#         unzip -Pinfected UselessDisk.bin.zip
-#         unzip -Pinfected GandCrab502.bin.zip
-#         unzip -Pinfected al-khaser.bin.zip
-#         unzip -Pinfected sality.dll.zip
-#         cd $GITHUB_WORKSPACE/tests
-#         cmd.exe //C '.\test_pe.bat'
-#    - name: mac run tests
-#      if: contains(matrix.os, 'macos')
-#      continue-on-error: true
-#      shell: bash
-#      run: |
-#         pip3 install setuptools wheel
-#         pip3 install .
-#         ./examples/scripts/dylibcollector.sh
-#         cd $GITHUB_WORKSPACE/examples/rootfs/x8664_macos/kext
-#         unzip -Pinfected SuperRootkit.kext.zip
-#         cd $GITHUB_WORKSPACE/tests
-#         ./test_macho.sh
+    - name: Win setup MSVC
+      if: contains(matrix.os, 'windows')
+      uses: microsoft/setup-msbuild@v1
+
+
+    - name: Win configure Pagefile
+      if: contains(matrix.os, 'windows')
+      uses: al-cheb/[email protected]
+      with:
+          minimum-size: 16GB
+          maximum-size: 16GB
+          disk-root: "C:"
+
+
+    - name: win run tests
+      if: contains(matrix.os, 'windows')
+      shell: bash
+      run: |
+          powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableArchiveScanning \$true'"
+          powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableBehaviorMonitoring \$true'"
+          powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command Set-MpPreference -DisableRealtimeMonitoring \$true'"
+          powershell Start-Process -PassThru -Wait PowerShell -ArgumentList "'-Command  Add-MpPreference -ExclusionPath $GITHUB_WORKSPACE'"
+          pip3 install setuptools wheel
+          pip3 install .
+          cd examples
+          rm -rf rootfs
+          curl -LJk -o master.zip https://github.com/qilingframework/rootfs/archive/refs/heads/master.zip && unzip master.zip
+          mv rootfs-master rootfs
+          cd $GITHUB_WORKSPACE
+          cmd.exe //C 'examples\scripts\dllscollector.bat'
+          cd $GITHUB_WORKSPACE/examples/rootfs/x86_windows/bin
+          unzip -Pinfected wannacry.bin.zip
+          unzip -Pinfected UselessDisk.bin.zip
+          unzip -Pinfected GandCrab502.bin.zip
+          unzip -Pinfected al-khaser.bin.zip
+          unzip -Pinfected sality.dll.zip
+          cd $GITHUB_WORKSPACE/tests
+          cmd.exe //C '.\test_pe.bat'
+
+
     - name: linux run tests
       if: contains(matrix.os, 'ubuntu')
       shell: 'script -q -e -c "bash {0}"'
@@ -80,13 +87,38 @@ jobs:
               pip3 install setuptools wheel flake8
               flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
               pip3 install .
-              cd examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
+              cd examples
+              rm -rf rootfs
+              wget https://github.com/qilingframework/rootfs/archive/refs/heads/master.zip
+              unzip master.zip && mv rootfs-master rootfs
+              cd rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
               cd ../../../../tests && ./test_elf.sh
         elif [ ${{ matrix.os }} == 'ubuntu-20.04' ]; then
+              cd examples
+              rm -rf rootfs
+              wget https://github.com/qilingframework/rootfs/archive/refs/heads/master.zip
+              unzip master.zip && mv rootfs-master rootfs
               docker run -it --rm -v ${GITHUB_WORKSPACE}:/qiling qilingframework/qiling:dev bash -c "pip3 install . && cd examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip && cd ../../../../tests && ./test_elf.sh"
         else
               pip3 install setuptools wheel
               pip3 install .
-              cd examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
+              cd examples
+              rm -rf rootfs
+              wget https://github.com/qilingframework/rootfs/archive/refs/heads/master.zip
+              unzip master.zip && mv rootfs-master rootfs
+              cd rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
               cd ../../../../tests && ./test_elf.sh
         fi
+
+
+    # - name: mac run tests
+    #   if: contains(matrix.os, 'macos')
+    #   shell: bash
+    #   run: |
+    #       pip3 install setuptools wheel
+    #       pip3 install .
+    #       ./examples/scripts/dylibcollector.sh
+    #       cd $GITHUB_WORKSPACE/examples/rootfs/x8664_macos/kext
+    #       unzip -Pinfected SuperRootkit.kext.zip
+    #       cd $GITHUB_WORKSPACE/tests
+    #       ./test_macho.sh        
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,7 @@ build
 dist
 
 # test and logs
+tests/mac_test_elf.sh
 jexamples/
 logs/
 log/
@@ -33,5 +34,3 @@ test.file
 *.o
 core
 *.perf
-examples/rootfs/x86_windows/Windows/registry
-examples/rootfs/x8664_windows/Windows/registry
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "examples/rootfs"]
+	path = examples/rootfs
+	url = https://github.com/qilingframework/rootfs.git
diff --git a/.travis.yml b/.travis.yml
diff --git a/ChangeLog b/ChangeLog
@@ -1,22 +1,26 @@
 This file details the changelog of Qiling Framework.
 
 ------------------------------------
-BREAK CHANGE
-- ql.multithread can be only set during Qiling.__init__ now.
-- ql.nprint and ql.dpring is depreciated. Please use logging directly instead.
-- ql.filename is renamed to ql.argv.
-- ql.output and ql.verbose now has slightly different meanings and can be adjusted runtime. See their docstring for details.
-- ql.filter now accepts a regular expression.
-- Remove ql.log_dir, ql.log_split, ql.append but add ql.log_file instead.
+[Version 1.2.4]: April [SOMETHING], 2021
 
+-
 
 ------------------------------------
-[Version 1.2.3]: March [SOMETHING], 2021
-- 
+[Version 1.2.3]: March 30th, 2021
+
+- Improved PR #689, Android syscall and test fix
+- GDB speed optimization
+- Fixed return value for uid/gid related syscall
+- Resolved multilevel symbolic links
+- Demigod set.api implementation
+- Major refactor, see commit 4aa8e59e04d5a8a5520e4e1e2595ecc78a80beba
+- Clean and remove rootfs
+- ql.filter now accepts a regular expression
+- consolidate output into verbose
 
 
 ------------------------------------
-[Version 1.2.2]: February 8, 2021
+[Version 1.2.2]: February 8th, 2021
 
 - Fix _acmdln and _wcmdln handling
 - More UEFI refactor

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,3 +1,4 @@
 recursive-include qiling/debugger/gdb/xml *
 recursive-include qiling/extensions/windows_sdk/defs *
 recursive-include qiling/profiles *
+include qiling/os/uefi/guids.csv
diff --git a/README.md b/README.md
@@ -24,6 +24,9 @@ Qiling is an advanced binary emulation framework, with the following features:
 
 Qiling also made its way to various international conferences.
 
+2021:
+- [Black Hat, Asia](https://www.blackhat.com/asia-21/arsenal/schedule/index.html#qiling-smart-analysis-for-smart-contract-22643)
+
 2020:
 - [Black Hat, Europe](https://www.blackhat.com/eu-20/arsenal/schedule/index.html#qiling-framework-deep-dive-into-obfuscated-binary-analysis-21781)
 - [Black Hat, USA](https://www.blackhat.com/us-20/arsenal/schedule/index.html#qiling-framework-from-dark-to-dawn-----enlightening-the-analysis-of-the-most-mysterious-iot-firmware--21062)