Merge pull request #98 from microsoft/dilan/2.19.0-upgrade

2.19.0 upgrade

.bazelversion

@@ -1 +1 @@
-7.2.1
+5f5d70b6c4d2fb1a889479569107f1692239e8a7

.github/labeler.yml

@@ -30,6 +30,10 @@ Ruby:
   - ruby/**/*
   - change-notes/**/*ruby*
 
+Rust:
+  - rust/**/*
+  - change-notes/**/*rust*
+
 Swift:
   - swift/**/*
   - change-notes/**/*swift*

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+### Pull Request checklist
+
+#### All query authors
+
+- [ ] A change note is added if necessary. See [the documentation](https://github.com/github/codeql/blob/main/docs/change-notes.md) in this repository.
+- [ ] All new queries have appropriate `.qhelp`. See [the documentation](https://github.com/github/codeql/blob/main/docs/query-help-style-guide.md) in this repository.
+- [ ] QL tests are added if necessary. See [Testing custom queries](https://docs.github.com/en/code-security/codeql-cli/using-the-advanced-functionality-of-the-codeql-cli/testing-custom-queries) in the GitHub documentation.
+- [ ] New and changed queries have correct query metadata. See [the documentation](https://github.com/github/codeql/blob/main/docs/query-metadata-style-guide.md) in this repository.
+
+#### Internal query authors only
+
+- [ ] Autofixes generated based on these changes are valid, only needed if this PR makes significant changes to `.ql`, `.qll`, or `.qhelp` files. See [the documentation](https://github.com/github/codeql-team/blob/main/docs/best-practices/validating-autofix-for-query-changes.md) (internal access required).
+- [ ] Changes are validated [at scale](https://github.com/github/codeql-dca/) (internal access required).
+- [ ] Adding a new query? Consider also [adding the query to autofix](https://github.com/github/codeml-autofix/blob/main/docs/updating-query-support.md#adding-a-new-query-to-the-query-suite).

.github/workflows/buildifier.yml

@@ -24,5 +24,5 @@ jobs:
           extra_args: >
             buildifier --all-files 2>&1 ||
             (
-              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel:buildifier"; exit 1
+              echo -e "In order to format all bazel files, please run:\n  bazel run //misc/bazel/buildifier"; exit 1
             )

.github/workflows/check-change-note.yml

@@ -16,11 +16,12 @@ on:
       - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!rust/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
-    env: 
+    env:
       REPO: ${{ github.repository }}
       PULL_REQUEST_NUMBER: ${{ github.event.number }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -33,7 +34,7 @@ jobs:
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
         run: |
           change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
-          
+
           if [ -z "$change_note_files" ]; then
             echo "No change note found. Either add one, or add the 'no-change-note-required' label."
             exit 1

.github/workflows/cpp-swift-analysis.yml

@@ -37,7 +37,7 @@ jobs:
       with:
         languages: cpp
         config-file: ./.github/codeql/codeql-config.yml
-  
+
     - name: "[Ubuntu] Remove GCC 13 from runner image"
       shell: bash
       run: |
@@ -48,7 +48,7 @@ jobs:
     - name: "Build Swift extractor using Bazel"
       run: |
         bazel clean --expunge
-        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local --features=-layering_check
+        bazel run //swift:create-extractor-pack --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results --spawn_strategy=local
         bazel shutdown
 
     - name: Perform CodeQL Analysis

.github/workflows/csharp-qltest.yml

@@ -29,45 +29,6 @@ permissions:
   contents: read
 
 jobs:
-  qlupgrade:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/fetch-codeql
-      - name: Check DB upgrade scripts
-        run: |
-          echo >empty.trap
-          codeql dataset import -S ql/lib/upgrades/initial/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql dataset upgrade testdb --additional-packs ql/lib
-          diff -q testdb/semmlecode.csharp.dbscheme ql/lib/semmlecode.csharp.dbscheme
-      - name: Check DB downgrade scripts
-        run: |
-          echo >empty.trap
-          rm -rf testdb; codeql dataset import -S ql/lib/semmlecode.csharp.dbscheme testdb empty.trap
-          codeql resolve upgrades --format=lines --allow-downgrades --additional-packs downgrades \
-           --dbscheme=ql/lib/semmlecode.csharp.dbscheme --target-dbscheme=downgrades/initial/semmlecode.csharp.dbscheme |
-           xargs codeql execute upgrades testdb
-          diff -q testdb/semmlecode.csharp.dbscheme downgrades/initial/semmlecode.csharp.dbscheme
-  qltest:
-    if: github.repository_owner == 'github'
-    runs-on: ubuntu-latest-xl
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./csharp/actions/create-extractor-pack
-      - name: Cache compilation cache
-        id: query-cache
-        uses: ./.github/actions/cache-query-compilation
-        with:
-          key: csharp-qltest-${{ matrix.slice }}
-      - name: Run QL tests
-        run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
     strategy:
       matrix:

.github/workflows/ruby-build.yml

@@ -65,8 +65,8 @@ jobs:
         id: cache-extractor
         with:
           path: |
-            ruby/extractor/target/release/codeql-extractor-ruby
-            ruby/extractor/target/release/codeql-extractor-ruby.exe
+            target/release/codeql-extractor-ruby
+            target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
@@ -75,7 +75,7 @@ jobs:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
-            ruby/target
+            target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-rust-cargo-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/**/Cargo.lock') }}
       - name: Check formatting
         if: steps.cache-extractor.outputs.cache-hit != 'true'
@@ -91,7 +91,7 @@ jobs:
         run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        run: ../target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -106,8 +106,8 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/extractor/target/release/codeql-extractor-ruby
-            ruby/extractor/target/release/codeql-extractor-ruby.exe
+            target/release/codeql-extractor-ruby
+            target/release/codeql-extractor-ruby.exe
           retention-days: 1
   compile-queries:
     if: github.repository_owner == 'github'
@@ -140,6 +140,7 @@ jobs:
           path: |
             ${{ runner.temp }}/query-packs/*
           retention-days: 1
+          include-hidden-files: true
 
   package:
     runs-on: ubuntu-latest
@@ -176,6 +177,7 @@ jobs:
           name: codeql-ruby-pack
           path: ruby/codeql-ruby.zip
           retention-days: 1
+          include-hidden-files: true
       - uses: actions/download-artifact@v3
         with:
           name: codeql-ruby-queries
@@ -193,6 +195,7 @@ jobs:
           name: codeql-ruby-bundle
           path: ruby/codeql-ruby-bundle.zip
           retention-days: 1
+          include-hidden-files: true
 
   test:
     defaults:

.github/workflows/rust.yml

@@ -0,0 +1,58 @@
+name: "Rust"
+
+on:
+  pull_request:
+    paths:
+      - "rust/**"
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "shared/**"
+      - "MODULE.bazel"
+      - .github/workflows/rust.yml
+      - .github/actions/**
+      - codeql-workspace.yml
+      - "!**/*.md"
+      - "!**/*.qhelp"
+    branches:
+      - rust-experiment
+      - main
+      - rc/*
+      - codeql-cli-*
+
+permissions:
+  contents: read
+
+jobs:
+  rust-code:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Format
+        working-directory: rust/extractor
+        shell: bash
+        run: |
+          cargo fmt --check
+      - name: Compilation
+        working-directory: rust/extractor
+        shell: bash
+        run: cargo check
+      - name: Clippy
+        working-directory: rust/extractor
+        shell: bash
+        run: |
+          cargo clippy --fix
+          git diff --exit-code
+  rust-codegen:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install CodeQL
+        uses: ./.github/actions/fetch-codeql
+      - name: Code generation
+        shell: bash
+        run: |
+          bazel run //rust/codegen
+          git add .
+          git diff --exit-code HEAD

.gitignore

@@ -7,8 +7,8 @@
 .cache
 
 # qltest projects and artifacts
+*.actual
 */ql/test/**/*.testproj
-*/ql/test/**/*.actual
 */ql/test/**/go.sum
 
 # Visual studio temporaries, except a file used by QL4VS
@@ -65,3 +65,9 @@ node_modules/
 
 # bazel-built in-tree extractor packs
 /*/extractor-pack
+
+# Jetbrains IDE files
+.idea
+
+# cargo build directory
+/target

.pre-commit-config.yaml

@@ -5,9 +5,9 @@ repos:
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
+        exclude: /test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
       - id: end-of-file-fixer
-        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)|.*\.patch
+        exclude: /test/.*$(?<!\.qlref)|.*\.patch$|.*\.qll?$
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v17.0.6
@@ -26,7 +26,7 @@ repos:
         name: Format bazel files
         files: \.(bazel|bzl)
         language: system
-        entry: bazel run //misc/bazel:buildifier
+        entry: bazel run //misc/bazel/buildifier
         pass_filenames: false
 
 #      DISABLED: can be enabled by copying this config and installing `pre-commit` with `--config` on the copy
@@ -45,7 +45,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
-        files: \.(qll?|qhelp|swift)$|^config/identical-files\.json$
+        files: \.(qll?|qhelp|swift|toml)$|^config/identical-files\.json$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -58,7 +58,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
+        files: ^misc/codegen/|^swift/(schema.py$|codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements)|ql/\.generated.list)
         language: system
         entry: bazel run //swift/codegen -- --quiet
         pass_filenames: false
@@ -69,3 +69,17 @@ repos:
         language: system
         entry: bazel test //misc/codegen/test
         pass_filenames: false
+
+      - id: rust-codegen
+        name: Run Rust checked in code generation
+        files: ^misc/codegen/|^rust/(schema.py$|codegen/|.*/generated/|ql/lib/(rust\.dbscheme$|codeql/rust/elements)|\.generated.list)
+        language: system
+        entry: bazel run //rust/codegen -- --quiet
+        pass_filenames: false
+
+      - id: rust-lint
+        name: Run fmt and clippy on Rust code
+        files: ^rust/extractor/(.*rs|Cargo.toml)$
+        language: system
+        entry: python3 rust/lint.py
+        pass_filenames: false