Merge branch 'master' into ci-android-riscv

Tencent · Nov 30, 2024 · de0cf29 · de0cf29
2 parents c8add51 + 0611ca7
commit de0cf29
Show file tree

Hide file tree

Showing 238 changed files with 5,407 additions and 2,768 deletions.
diff --git a/.ci/pnnx.yml b/.ci/pnnx.yml
@@ -31,39 +31,51 @@ jobs:
         include:
           - torch-version: 1.8.1
             torchvision-version: 0.9.1
+            torchaudio-version: 0.8.1
 
           - torch-version: 1.9.1
             torchvision-version: 0.10.1
+            torchaudio-version: 0.9.1
 
           - torch-version: 1.10.0
             torchvision-version: 0.11.1
+            torchaudio-version: '0.10.0+cpu'
 
           - torch-version: 1.11.0
             torchvision-version: 0.12.0
+            torchaudio-version: '0.11.0+cpu'
 
           - torch-version: 1.12.0
             torchvision-version: 0.13.0
+            torchaudio-version: '0.12.0+cpu'
 
           - torch-version: 1.13.0
             torchvision-version: 0.14.0
+            torchaudio-version: '0.13.0+cpu'
 
           - torch-version: 2.0.0
             torchvision-version: 0.15.1
+            torchaudio-version: '2.0.0+cpu'
 
           - torch-version: 2.1.0
             torchvision-version: 0.16.0
+            torchaudio-version: '2.1.0+cpu'
 
           - torch-version: 2.2.1
             torchvision-version: 0.17.1
+            torchaudio-version: '2.2.1+cpu'
 
           - torch-version: 2.3.0
             torchvision-version: 0.18.0
+            torchaudio-version: '2.3.0+cpu'
 
           - torch-version: 2.4.0
             torchvision-version: 0.19.0
+            torchaudio-version: '2.4.0+cpu'
 
           - torch-version: 2.5.0
             torchvision-version: 0.20.0
+            torchaudio-version: '2.5.0+cpu'
 
     runs-on:
       pool-name: docker
@@ -169,7 +181,7 @@ jobs:
     - name: setup-pytorch
       run: |
         export PYTHONUSERBASE=${{ci.workspace}}/torch-${{matrix.torch-version}}
-        pip3 install --user torch==${{matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu --index-url https://download.pytorch.org/whl/cpu
+        pip3 install --user torch==${{matrix.torch-version}}+cpu torchvision==${{matrix.torchvision-version}}+cpu torchaudio==${{matrix.torchaudio-version}} --index-url https://download.pytorch.org/whl/cpu
         pip3 install --user onnx
         pip3 install --user onnxscript
 

diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
@@ -87,7 +87,7 @@ jobs:
     # build wheels for ubuntu-20.04
     - name: Build wheels for ubuntu
       if: matrix.os == 'ubuntu-20.04'
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_LINUX: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build }}
@@ -99,7 +99,7 @@ jobs:
     # build wheels for windows-2019
     - name: Build wheels for windows
       if: matrix.os == 'windows-2019' && (matrix.arch == 'AMD64' || matrix.arch == 'x86')
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_WINDOWS: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build }}
@@ -112,7 +112,7 @@ jobs:
 
     - name: Build wheels for windows ARM64
       if: matrix.os == 'windows-2019' && matrix.arch == 'ARM64'
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_WINDOWS: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build }}
@@ -190,7 +190,7 @@ jobs:
 
     - name: Build wheels for macos x86_64
       if: matrix.os == 'macos-13' && matrix.arch == 'x86_64'
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_MACOS: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build }}
@@ -208,7 +208,7 @@ jobs:
 
     - name: Build wheels for macos arm64
       if: matrix.os == 'macos-13' && matrix.arch == 'arm64'
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_MACOS: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build }}
@@ -264,7 +264,7 @@ jobs:
         platforms: all
 
     - name: Build wheels for manylinux with qemu
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_LINUX: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build_cp }}-${{ matrix.build_sub }}*
@@ -312,7 +312,7 @@ jobs:
         platforms: all
 
     - name: Build wheels for manylinux with qemu
-      uses: pypa/cibuildwheel@v2.20.0
+      uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_ARCHS_LINUX: ${{ matrix.arch }}
         CIBW_BUILD: ${{ matrix.build_pp }}-*

diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml
@@ -56,47 +56,47 @@ jobs:
     - name: codecov
       id: codecov
       continue-on-error: true
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build/lcov.info
     - name: codecov-vlen256-retry-1
       continue-on-error: true
       id: codecov-vlen256-retry-1
       if: steps.codecov.outcome=='failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build/lcov.info
     - name: codecov-vlen256-retry-2
       continue-on-error: true
       id: codecov-vlen256-retry-2
       if: steps.codecov-vlen256-retry-1.outcome=='failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build/lcov.info
     - name: codecov-vlen256-retry-3
       continue-on-error: true
       id: codecov-vlen256-retry-3
       if: steps.codecov-vlen256-retry-2.outcome=='failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build/lcov.info
     - name: codecov-vlen256-retry-4
       continue-on-error: true
       id: codecov-vlen256-retry-4
       if: steps.codecov-vlen256-retry-3.outcome=='failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build/lcov.info
     - name: codecov-vlen256-retry-5
       continue-on-error: true
       id: codecov-vlen256-retry-5
       if: steps.codecov-vlen256-retry-4.outcome=='failure'
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build/lcov.info
@@ -141,7 +141,7 @@ jobs:
         lcov -r lcov.info '*/build-avx512-spr/*' -o lcov.info
         lcov --list lcov.info
     - name: codecov-avx512-spr
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: build-avx512-spr/lcov.info
diff --git a/cmake/ncnn_add_layer.cmake b/cmake/ncnn_add_layer.cmake
@@ -345,7 +345,8 @@ macro(ncnn_add_layer class)
                 ncnn_add_arch_opt_source(${class} asimddp "-march=armv8.2-a+fp16+dotprod")
             endif()
             if(NCNN_RUNTIME_CPU AND NCNN_ARM82FP16FML)
-                ncnn_add_arch_opt_source(${class} asimdfhm "-march=armv8.2-a+fp16+fp16fml")
+                # clang 9.0.9 shipped with android ndk-r21 is missing __ARM_FEATURE_FP16_FML macro for asimdfhm target
+                ncnn_add_arch_opt_source(${class} asimdfhm "-march=armv8.2-a+fp16+fp16fml -D__ARM_FEATURE_FP16_FML")
             endif()
             if(NCNN_RUNTIME_CPU AND NCNN_ARM84BF16)
                 ncnn_add_arch_opt_source(${class} bf16 "-march=armv8.4-a+fp16+dotprod+bf16")

diff --git a/docs/developer-guide/operators.md b/docs/developer-guide/operators.md
@@ -46,6 +46,7 @@
 * [Input](#input)
 * [InstanceNorm](#instancenorm)
 * [Interp](#interp)
+* [InverseSpectrogram](#inversespectrogram)
 * [LayerNorm](#layernorm)
 * [Log](#log)
 * [LRN](#lrn)
@@ -81,6 +82,7 @@
 * [Slice](#slice)
 * [Softmax](#softmax)
 * [Softplus](#softplus)
+* [Spectrogram](#spectrogram)
 * [Split](#split)
 * [Swish](#swish)
 * [TanH](#tanh)
@@ -1141,6 +1143,30 @@ Resize type:
 - 2 = Bilinear
 - 3 = Bicubic
 
+# InverseSpectrogram
+```
+x1 = x as complex
+x1 = x1 * sqrt(norm) if normalized
+y = istft(x1)
+y1 = unpad(y) if center
+
+if returns == 0 return y1 as complex
+if returns == 1 return y1 real
+if returns == 2 return y1 imag
+```
+
+* one_blob_only
+
+| param id  | name          | type  | default   | description       |
+| --------- | ------------- | ----- | --------- | ----------------- |
+| 0         | n_fft         | int   | 0         |                   |
+| 1         | returns       | int   | 1         |                   |
+| 2         | hoplen        | int   | n_fft / 4 |                   |
+| 3         | winlen        | int   | n_fft     |                   |
+| 4         | window_type   | int   | 0         | 0=ones 1=hann 2=hamming |
+| 5         | center        | int   | 1         |                   |
+| 7         | normalized    | int   | 0         | 0=no 1=n_fft 2=window-l2-energy |
+
 # LayerNorm
 ```
 split x along outmost axis into part x0, x1 ...
@@ -1829,6 +1855,31 @@ y = log(exp(x) + 1)
 * one_blob_only
 * support_inplace
 
+# Spectrogram
+```
+x1 = pad(x) if center
+y = stft(x1)
+y = y / sqrt(norm) if normalized
+
+if power == 0 return y as real
+if power == 1 return magnitude
+if power == 2 return square of magnitude
+```
+
+* one_blob_only
+
+| param id  | name          | type  | default   | description       |
+| --------- | ------------- | ----- | --------- | ----------------- |
+| 0         | n_fft         | int   | 0         |                   |
+| 1         | power         | int   | 0         |                   |
+| 2         | hoplen        | int   | n_fft / 4 |                   |
+| 3         | winlen        | int   | n_fft     |                   |
+| 4         | window_type   | int   | 0         | 0=ones 1=hann 2=hamming |
+| 5         | center        | int   | 1         |                   |
+| 6         | pad_type      | int   | 2         | 0=CONSTANT 1=REPLICATE 2=REFLECT |
+| 7         | normalized    | int   | 0         | 0=no 1=n_fft 2=window-l2-energy |
+| 8         | onesided      | int   | 1         |                   |
+
 # Split
 ```
 y0, y1 ... = x

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -167,6 +167,8 @@ ncnn_add_layer(Diag)
 ncnn_add_layer(CELU)
 ncnn_add_layer(Shrink)
 ncnn_add_layer(RMSNorm)
+ncnn_add_layer(Spectrogram)
+ncnn_add_layer(InverseSpectrogram)
 
 if(NCNN_VULKAN)
     ncnn_add_shader(${CMAKE_CURRENT_SOURCE_DIR}/convert_ycbcr.comp)
@@ -632,6 +634,8 @@ if(NCNN_TARGET_ARCH STREQUAL "arm" AND (CMAKE_SIZEOF_VOID_P EQUAL 8 OR NCNN_TARG
             endif()
             if(NCNN_ARM82FP16FML)
                 set(ARM_MARCH_FLAG "${ARM_MARCH_FLAG}+fp16fml")
+                # clang 9.0.9 shipped with android ndk-r21 is missing __ARM_FEATURE_FP16_FML macro for asimdfhm target
+                target_compile_options(ncnn PRIVATE -D__ARM_FEATURE_FP16_FML)
             endif()
         endif()
     endif()