Skip to content

Commit

Permalink
Added functionality to report result compared to a set of studies. Ad…
Browse files Browse the repository at this point in the history 
…ded a first report.
  • Loading branch information
@zond
Martin Bruse authored and zond committed May 7, 2024
1 parent b13c5a0 commit c369663
Show file tree
Hide file tree
Showing 2 changed files with 308 additions and 1 deletion.
253 changes: 253 additions & 0 deletions COMPARISON.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,253 @@
# Zimtohrli correlation report

Created at 2024-05-07

## PEASS_DB

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.47 |0.11 |0.47 |0.85 |0.78 |0.65 |0.56 |0.65 |0.68 |
|DPAM |0.47 |1.00 |0.09 |0.40 |0.57 |0.50 |0.33 |0.36 |0.34 |0.40 |
|GVPMOS |0.11 |0.09 |1.00 |0.09 |0.08 |0.01 |0.28 |0.24 |0.15 |0.32 |
|MOS |0.47 |0.40 |0.09 |1.00 |0.45 |0.36 |0.34 |0.52 |0.25 |0.47 |
|PARLAQ |0.85 |0.57 |0.08 |0.45 |1.00 |0.83 |0.68 |0.49 |0.78 |0.59 |
|PEAQB |0.78 |0.50 |0.01 |0.36 |0.83 |1.00 |0.70 |0.40 |0.71 |0.60 |
|PESQ |0.65 |0.33 |0.28 |0.34 |0.68 |0.70 |1.00 |0.43 |0.59 |0.66 |
|ViSQOL |0.56 |0.36 |0.24 |0.52 |0.49 |0.40 |0.43 |1.00 |0.35 |0.66 |
|WARP-Q |0.65 |0.34 |0.15 |0.25 |0.78 |0.71 |0.59 |0.35 |1.00 |0.41 |
|Zimtohrli |0.68 |0.40 |0.32 |0.47 |0.59 |0.60 |0.66 |0.66 |0.41 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|ViSQOL |0.52 |
|CDPAM |0.47 |
|Zimtohrli |0.47 |
|PARLAQ |0.45 |
|DPAM |0.40 |
|PEAQB |0.36 |
|PESQ |0.34 |
|WARP-Q |0.25 |
|GVPMOS |0.09 |

## SAOC

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.62 |0.38 |0.61 |0.87 |0.62 |0.61 |0.73 |0.71 |0.63 |
|DPAM |0.62 |1.00 |0.51 |0.50 |0.55 |0.55 |0.48 |0.47 |0.66 |0.55 |
|GVPMOS |0.38 |0.51 |1.00 |0.27 |0.34 |0.17 |0.27 |0.17 |0.36 |0.26 |
|MOS |0.61 |0.50 |0.27 |1.00 |0.73 |0.59 |0.76 |0.74 |0.62 |0.71 |
|PARLAQ |0.87 |0.55 |0.34 |0.73 |1.00 |0.67 |0.76 |0.85 |0.75 |0.75 |
|PEAQB |0.62 |0.55 |0.17 |0.59 |0.67 |1.00 |0.60 |0.71 |0.70 |0.73 |
|PESQ |0.61 |0.48 |0.27 |0.76 |0.76 |0.60 |1.00 |0.80 |0.65 |0.79 |
|ViSQOL |0.73 |0.47 |0.17 |0.74 |0.85 |0.71 |0.80 |1.00 |0.70 |0.74 |
|WARP-Q |0.71 |0.66 |0.36 |0.62 |0.75 |0.70 |0.65 |0.70 |1.00 |0.61 |
|Zimtohrli |0.63 |0.55 |0.26 |0.71 |0.75 |0.73 |0.79 |0.74 |0.61 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|PESQ |0.76 |
|ViSQOL |0.74 |
|PARLAQ |0.73 |
|Zimtohrli |0.71 |
|WARP-Q |0.62 |
|CDPAM |0.61 |
|PEAQB |0.59 |
|DPAM |0.50 |
|GVPMOS |0.27 |

## SASSEC

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.56 |0.41 |0.67 |0.88 |0.79 |0.73 |0.82 |0.74 |0.73 |
|DPAM |0.56 |1.00 |0.47 |0.50 |0.52 |0.56 |0.46 |0.56 |0.66 |0.52 |
|GVPMOS |0.41 |0.47 |1.00 |0.38 |0.38 |0.31 |0.30 |0.29 |0.41 |0.34 |
|MOS |0.67 |0.50 |0.38 |1.00 |0.74 |0.68 |0.76 |0.73 |0.65 |0.65 |
|PARLAQ |0.88 |0.52 |0.38 |0.74 |1.00 |0.75 |0.82 |0.88 |0.73 |0.74 |
|PEAQB |0.79 |0.56 |0.31 |0.68 |0.75 |1.00 |0.78 |0.81 |0.72 |0.78 |
|PESQ |0.73 |0.46 |0.30 |0.76 |0.82 |0.78 |1.00 |0.83 |0.67 |0.77 |
|ViSQOL |0.82 |0.56 |0.29 |0.73 |0.88 |0.81 |0.83 |1.00 |0.76 |0.77 |
|WARP-Q |0.74 |0.66 |0.41 |0.65 |0.73 |0.72 |0.67 |0.76 |1.00 |0.59 |
|Zimtohrli |0.73 |0.52 |0.34 |0.65 |0.74 |0.78 |0.77 |0.77 |0.59 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|PESQ |0.76 |
|PARLAQ |0.74 |
|ViSQOL |0.73 |
|PEAQB |0.68 |
|CDPAM |0.67 |
|WARP-Q |0.65 |
|Zimtohrli |0.65 |
|DPAM |0.50 |
|GVPMOS |0.38 |

## SiSEC08

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.53 |0.33 |0.70 |0.86 |0.72 |0.80 |0.79 |0.75 |0.65 |
|DPAM |0.53 |1.00 |0.51 |0.54 |0.53 |0.56 |0.46 |0.49 |0.66 |0.37 |
|GVPMOS |0.33 |0.51 |1.00 |0.29 |0.31 |0.18 |0.18 |0.15 |0.41 |0.14 |
|MOS |0.70 |0.54 |0.29 |1.00 |0.78 |0.66 |0.73 |0.78 |0.75 |0.56 |
|PARLAQ |0.86 |0.53 |0.31 |0.78 |1.00 |0.72 |0.84 |0.86 |0.82 |0.67 |
|PEAQB |0.72 |0.56 |0.18 |0.66 |0.72 |1.00 |0.70 |0.78 |0.79 |0.67 |
|PESQ |0.80 |0.46 |0.18 |0.73 |0.84 |0.70 |1.00 |0.86 |0.79 |0.64 |
|ViSQOL |0.79 |0.49 |0.15 |0.78 |0.86 |0.78 |0.86 |1.00 |0.81 |0.61 |
|WARP-Q |0.75 |0.66 |0.41 |0.75 |0.82 |0.79 |0.79 |0.81 |1.00 |0.56 |
|Zimtohrli |0.65 |0.37 |0.14 |0.56 |0.67 |0.67 |0.64 |0.61 |0.56 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|PARLAQ |0.78 |
|ViSQOL |0.78 |
|WARP-Q |0.75 |
|PESQ |0.73 |
|CDPAM |0.70 |
|PEAQB |0.66 |
|Zimtohrli |0.56 |
|DPAM |0.54 |
|GVPMOS |0.29 |

## coresvnet

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.49 |0.12 |0.59 |0.68 |0.60 |0.67 |0.54 |0.19 |0.76 |
|DPAM |0.49 |1.00 |0.19 |0.21 |0.58 |0.13 |0.38 |0.17 |0.23 |0.55 |
|GVPMOS |0.12 |0.19 |1.00 |0.01 |0.31 |0.01 |0.08 |0.10 |0.33 |0.19 |
|MOS |0.59 |0.21 |0.01 |1.00 |0.54 |0.85 |0.57 |0.80 |0.10 |0.66 |
|PARLAQ |0.68 |0.58 |0.31 |0.54 |1.00 |0.55 |0.67 |0.51 |0.29 |0.81 |
|PEAQB |0.60 |0.13 |0.01 |0.85 |0.55 |1.00 |0.63 |0.86 |0.14 |0.66 |
|PESQ |0.67 |0.38 |0.08 |0.57 |0.67 |0.63 |1.00 |0.51 |0.06 |0.76 |
|ViSQOL |0.54 |0.17 |0.10 |0.80 |0.51 |0.86 |0.51 |1.00 |0.04 |0.63 |
|WARP-Q |0.19 |0.23 |0.33 |0.10 |0.29 |0.14 |0.06 |0.04 |1.00 |0.24 |
|Zimtohrli |0.76 |0.55 |0.19 |0.66 |0.81 |0.66 |0.76 |0.63 |0.24 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|PEAQB |0.85 |
|ViSQOL |0.80 |
|Zimtohrli |0.66 |
|CDPAM |0.59 |
|PESQ |0.57 |
|PARLAQ |0.54 |
|DPAM |0.21 |
|WARP-Q |0.10 |
|GVPMOS |0.01 |

## furball_mushra

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.61 |0.77 |0.66 |0.67 |0.63 |0.85 |0.62 |0.77 |0.81 |
|DPAM |0.61 |1.00 |0.55 |0.54 |0.71 |0.61 |0.67 |0.66 |0.57 |0.67 |
|GVPMOS |0.77 |0.55 |1.00 |0.69 |0.63 |0.56 |0.89 |0.47 |0.64 |0.88 |
|MOS |0.66 |0.54 |0.69 |1.00 |0.56 |0.54 |0.74 |0.52 |0.58 |0.72 |
|PARLAQ |0.67 |0.71 |0.63 |0.56 |1.00 |0.79 |0.71 |0.56 |0.65 |0.68 |
|PEAQB |0.63 |0.61 |0.56 |0.54 |0.79 |1.00 |0.67 |0.58 |0.65 |0.65 |
|PESQ |0.85 |0.67 |0.89 |0.74 |0.71 |0.67 |1.00 |0.64 |0.75 |0.94 |
|ViSQOL |0.62 |0.66 |0.47 |0.52 |0.56 |0.58 |0.64 |1.00 |0.73 |0.60 |
|WARP-Q |0.77 |0.57 |0.64 |0.58 |0.65 |0.65 |0.75 |0.73 |1.00 |0.67 |
|Zimtohrli |0.81 |0.67 |0.88 |0.72 |0.68 |0.65 |0.94 |0.60 |0.67 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|PESQ |0.74 |
|Zimtohrli |0.72 |
|GVPMOS |0.69 |
|CDPAM |0.66 |
|WARP-Q |0.58 |
|PARLAQ |0.56 |
|PEAQB |0.54 |
|DPAM |0.54 |
|ViSQOL |0.52 |

## minimodal_cmaudio_mushra

### Spearman correlation table for all score types

| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli |
|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------|
|CDPAM |1.00 |0.19 |0.42 |0.44 |0.20 |0.21 |0.35 |0.41 |0.08 |0.41 |
|DPAM |0.19 |1.00 |0.11 |0.19 |0.28 |0.18 |0.27 |0.28 |0.02 |0.26 |
|GVPMOS |0.42 |0.11 |1.00 |0.78 |0.30 |0.22 |0.44 |0.54 |0.00 |0.55 |
|MOS |0.44 |0.19 |0.78 |1.00 |0.47 |0.28 |0.60 |0.61 |0.07 |0.65 |
|PARLAQ |0.20 |0.28 |0.30 |0.47 |1.00 |0.55 |0.81 |0.49 |0.26 |0.63 |
|PEAQB |0.21 |0.18 |0.22 |0.28 |0.55 |1.00 |0.66 |0.28 |0.22 |0.39 |
|PESQ |0.35 |0.27 |0.44 |0.60 |0.81 |0.66 |1.00 |0.57 |0.22 |0.74 |
|ViSQOL |0.41 |0.28 |0.54 |0.61 |0.49 |0.28 |0.57 |1.00 |0.02 |0.80 |
|WARP-Q |0.08 |0.02 |0.00 |0.07 |0.26 |0.22 |0.22 |0.02 |1.00 |0.08 |
|Zimtohrli |0.41 |0.26 |0.55 |0.65 |0.63 |0.39 |0.74 |0.80 |0.08 |1.00 |

### Score type MOS Spearman correlation in order

|Score type |Spearman correlation |
|-----------|---------------------|
|GVPMOS |0.78 |
|Zimtohrli |0.65 |
|ViSQOL |0.61 |
|PESQ |0.60 |
|PARLAQ |0.47 |
|CDPAM |0.44 |
|PEAQB |0.28 |
|DPAM |0.19 |
|WARP-Q |0.07 |

## perceptual_audio

### Maximal audibility classification accuracy and threshold per score type

|Score type |Accuracy |Threshold |
|-----------|---------|----------|
|Zimtohrli |0.74 |31 |
|CDPAM |0.71 |0.049 |
|ViSQOL |0.71 |4.2 |
|DPAM |0.69 |0.73 |
|GVPMOS |0.51 |3 |
|WARP-Q |0.49 |1.3 |
|PEAQB |0.49 |-3.9 |
|PARLAQ |0.48 |30 |
|PESQ |0.48 |1 |

## Global leaderboard across all studies

### Mean square error (1 - Spearman correlation, or 1 - accuracy) per score type

|Score type |MSE |Min score |Max score |Mean score |
|-----------|-----|----------|----------|-----------|
|ViSQOL |0.10 |0.52 |0.80 |0.59 |
|Zimtohrli |0.13 |0.47 |0.72 |0.55 |
|PESQ |0.13 |0.34 |0.76 |0.56 |
|PARLAQ |0.15 |0.45 |0.78 |0.53 |
|CDPAM |0.15 |0.44 |0.70 |0.52 |
|PEAQB |0.19 |0.28 |0.85 |0.50 |
|DPAM |0.32 |0.19 |0.54 |0.36 |
|WARP-Q |0.34 |0.07 |0.75 |0.38 |
|GVPMOS |0.42 |0.01 |0.78 |0.31 |

56 changes: 55 additions & 1 deletion go/bin/score/score.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@ import (
"encoding/json"
"flag"
"fmt"
"io"
"log"
"os"
"path/filepath"
"runtime"
"sort"
"time"

"github.com/google/zimtohrli/go/data"
"github.com/google/zimtohrli/go/goohrli"
Expand All @@ -47,16 +49,68 @@ func main() {
zimtohrliPerceptualSampleRate := flag.Float64("zimtohrli_perceptual_sample_rate", goohrli.DefaultPerceptualSampleRate(), "Sample rate of the Zimtohrli spectrograms.")
correlate := flag.String("correlate", "", "Path to a database directory with a study to correlate scores for.")
leaderboard := flag.String("leaderboard", "", "Glob to directories with databases to compute leaderboard for.")
report := flag.String("report", "", "Glob to directories with databases to generate a report for.")
accuracy := flag.String("accuracy", "", "Path to a database directory with a study to provide JND accuracy for.")
workers := flag.Int("workers", runtime.NumCPU(), "Number of concurrent workers for tasks.")
failFast := flag.Bool("fail_fast", false, "Whether to panic immediately on any error.")
flag.Parse()

if *details == "" && *calculate == "" && *correlate == "" && *accuracy == "" && *leaderboard == "" {
if *details == "" && *calculate == "" && *correlate == "" && *accuracy == "" && *leaderboard == "" && *report == "" {
flag.Usage()
os.Exit(1)
}

if *report != "" {
databases, err := filepath.Glob(*report)
if err != nil {
log.Fatal(err)
}
fmt.Printf(`# Zimtohrli correlation report
Created at %s
`, time.Now().Format(time.DateOnly))
studies := make(data.Studies, len(databases))
for index, path := range databases {
fmt.Printf("## %s\n\n", filepath.Base(path))
if studies[index], err = data.OpenStudy(path); err != nil {
log.Fatal(err)
}
isJND := false
if err := studies[index].ViewEachReference(func(ref *data.Reference) error {
for _, dist := range ref.Distortions {
if _, found := dist.Scores[data.JND]; found {
isJND = true
}
}
return io.EOF
}); err != nil {
log.Fatal(err)
}
if isJND {
accuracy, err := studies[index].Accuracy()
if err != nil {
log.Fatal(err)
}
fmt.Println(accuracy)
} else {
corrTable, err := studies[index].Correlate()
if err != nil {
log.Fatal(err)
}
fmt.Println(corrTable)
}
}

fmt.Println("## Global leaderboard across all studies\n")

board, err := studies.Leaderboard()
if err != nil {
log.Fatal(err)
}
fmt.Println(board)
}

if *leaderboard != "" {
databases, err := filepath.Glob(*leaderboard)
if err != nil {
Expand Down

0 comments on commit c369663

Please sign in to comment.