diff --git a/COMPARISON.md b/COMPARISON.md new file mode 100644 index 0000000..9fe701a --- /dev/null +++ b/COMPARISON.md @@ -0,0 +1,253 @@ +# Zimtohrli correlation report + +Created at 2024-05-07 + +## PEASS_DB + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.47 |0.11 |0.47 |0.85 |0.78 |0.65 |0.56 |0.65 |0.68 | +|DPAM |0.47 |1.00 |0.09 |0.40 |0.57 |0.50 |0.33 |0.36 |0.34 |0.40 | +|GVPMOS |0.11 |0.09 |1.00 |0.09 |0.08 |0.01 |0.28 |0.24 |0.15 |0.32 | +|MOS |0.47 |0.40 |0.09 |1.00 |0.45 |0.36 |0.34 |0.52 |0.25 |0.47 | +|PARLAQ |0.85 |0.57 |0.08 |0.45 |1.00 |0.83 |0.68 |0.49 |0.78 |0.59 | +|PEAQB |0.78 |0.50 |0.01 |0.36 |0.83 |1.00 |0.70 |0.40 |0.71 |0.60 | +|PESQ |0.65 |0.33 |0.28 |0.34 |0.68 |0.70 |1.00 |0.43 |0.59 |0.66 | +|ViSQOL |0.56 |0.36 |0.24 |0.52 |0.49 |0.40 |0.43 |1.00 |0.35 |0.66 | +|WARP-Q |0.65 |0.34 |0.15 |0.25 |0.78 |0.71 |0.59 |0.35 |1.00 |0.41 | +|Zimtohrli |0.68 |0.40 |0.32 |0.47 |0.59 |0.60 |0.66 |0.66 |0.41 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|ViSQOL |0.52 | +|CDPAM |0.47 | +|Zimtohrli |0.47 | +|PARLAQ |0.45 | +|DPAM |0.40 | +|PEAQB |0.36 | +|PESQ |0.34 | +|WARP-Q |0.25 | +|GVPMOS |0.09 | + +## SAOC + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.62 |0.38 |0.61 |0.87 |0.62 |0.61 |0.73 |0.71 |0.63 | +|DPAM |0.62 |1.00 |0.51 |0.50 |0.55 |0.55 |0.48 |0.47 |0.66 |0.55 | +|GVPMOS |0.38 |0.51 |1.00 |0.27 |0.34 |0.17 |0.27 |0.17 |0.36 |0.26 | +|MOS |0.61 |0.50 |0.27 |1.00 |0.73 |0.59 |0.76 |0.74 |0.62 |0.71 | +|PARLAQ |0.87 |0.55 |0.34 |0.73 |1.00 |0.67 |0.76 |0.85 |0.75 |0.75 | +|PEAQB |0.62 |0.55 |0.17 |0.59 |0.67 |1.00 |0.60 |0.71 |0.70 |0.73 | +|PESQ |0.61 |0.48 |0.27 |0.76 |0.76 |0.60 |1.00 |0.80 |0.65 |0.79 | +|ViSQOL |0.73 |0.47 |0.17 |0.74 |0.85 |0.71 |0.80 |1.00 |0.70 |0.74 | +|WARP-Q |0.71 |0.66 |0.36 |0.62 |0.75 |0.70 |0.65 |0.70 |1.00 |0.61 | +|Zimtohrli |0.63 |0.55 |0.26 |0.71 |0.75 |0.73 |0.79 |0.74 |0.61 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|PESQ |0.76 | +|ViSQOL |0.74 | +|PARLAQ |0.73 | +|Zimtohrli |0.71 | +|WARP-Q |0.62 | +|CDPAM |0.61 | +|PEAQB |0.59 | +|DPAM |0.50 | +|GVPMOS |0.27 | + +## SASSEC + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.56 |0.41 |0.67 |0.88 |0.79 |0.73 |0.82 |0.74 |0.73 | +|DPAM |0.56 |1.00 |0.47 |0.50 |0.52 |0.56 |0.46 |0.56 |0.66 |0.52 | +|GVPMOS |0.41 |0.47 |1.00 |0.38 |0.38 |0.31 |0.30 |0.29 |0.41 |0.34 | +|MOS |0.67 |0.50 |0.38 |1.00 |0.74 |0.68 |0.76 |0.73 |0.65 |0.65 | +|PARLAQ |0.88 |0.52 |0.38 |0.74 |1.00 |0.75 |0.82 |0.88 |0.73 |0.74 | +|PEAQB |0.79 |0.56 |0.31 |0.68 |0.75 |1.00 |0.78 |0.81 |0.72 |0.78 | +|PESQ |0.73 |0.46 |0.30 |0.76 |0.82 |0.78 |1.00 |0.83 |0.67 |0.77 | +|ViSQOL |0.82 |0.56 |0.29 |0.73 |0.88 |0.81 |0.83 |1.00 |0.76 |0.77 | +|WARP-Q |0.74 |0.66 |0.41 |0.65 |0.73 |0.72 |0.67 |0.76 |1.00 |0.59 | +|Zimtohrli |0.73 |0.52 |0.34 |0.65 |0.74 |0.78 |0.77 |0.77 |0.59 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|PESQ |0.76 | +|PARLAQ |0.74 | +|ViSQOL |0.73 | +|PEAQB |0.68 | +|CDPAM |0.67 | +|WARP-Q |0.65 | +|Zimtohrli |0.65 | +|DPAM |0.50 | +|GVPMOS |0.38 | + +## SiSEC08 + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.53 |0.33 |0.70 |0.86 |0.72 |0.80 |0.79 |0.75 |0.65 | +|DPAM |0.53 |1.00 |0.51 |0.54 |0.53 |0.56 |0.46 |0.49 |0.66 |0.37 | +|GVPMOS |0.33 |0.51 |1.00 |0.29 |0.31 |0.18 |0.18 |0.15 |0.41 |0.14 | +|MOS |0.70 |0.54 |0.29 |1.00 |0.78 |0.66 |0.73 |0.78 |0.75 |0.56 | +|PARLAQ |0.86 |0.53 |0.31 |0.78 |1.00 |0.72 |0.84 |0.86 |0.82 |0.67 | +|PEAQB |0.72 |0.56 |0.18 |0.66 |0.72 |1.00 |0.70 |0.78 |0.79 |0.67 | +|PESQ |0.80 |0.46 |0.18 |0.73 |0.84 |0.70 |1.00 |0.86 |0.79 |0.64 | +|ViSQOL |0.79 |0.49 |0.15 |0.78 |0.86 |0.78 |0.86 |1.00 |0.81 |0.61 | +|WARP-Q |0.75 |0.66 |0.41 |0.75 |0.82 |0.79 |0.79 |0.81 |1.00 |0.56 | +|Zimtohrli |0.65 |0.37 |0.14 |0.56 |0.67 |0.67 |0.64 |0.61 |0.56 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|PARLAQ |0.78 | +|ViSQOL |0.78 | +|WARP-Q |0.75 | +|PESQ |0.73 | +|CDPAM |0.70 | +|PEAQB |0.66 | +|Zimtohrli |0.56 | +|DPAM |0.54 | +|GVPMOS |0.29 | + +## coresvnet + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.49 |0.12 |0.59 |0.68 |0.60 |0.67 |0.54 |0.19 |0.76 | +|DPAM |0.49 |1.00 |0.19 |0.21 |0.58 |0.13 |0.38 |0.17 |0.23 |0.55 | +|GVPMOS |0.12 |0.19 |1.00 |0.01 |0.31 |0.01 |0.08 |0.10 |0.33 |0.19 | +|MOS |0.59 |0.21 |0.01 |1.00 |0.54 |0.85 |0.57 |0.80 |0.10 |0.66 | +|PARLAQ |0.68 |0.58 |0.31 |0.54 |1.00 |0.55 |0.67 |0.51 |0.29 |0.81 | +|PEAQB |0.60 |0.13 |0.01 |0.85 |0.55 |1.00 |0.63 |0.86 |0.14 |0.66 | +|PESQ |0.67 |0.38 |0.08 |0.57 |0.67 |0.63 |1.00 |0.51 |0.06 |0.76 | +|ViSQOL |0.54 |0.17 |0.10 |0.80 |0.51 |0.86 |0.51 |1.00 |0.04 |0.63 | +|WARP-Q |0.19 |0.23 |0.33 |0.10 |0.29 |0.14 |0.06 |0.04 |1.00 |0.24 | +|Zimtohrli |0.76 |0.55 |0.19 |0.66 |0.81 |0.66 |0.76 |0.63 |0.24 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|PEAQB |0.85 | +|ViSQOL |0.80 | +|Zimtohrli |0.66 | +|CDPAM |0.59 | +|PESQ |0.57 | +|PARLAQ |0.54 | +|DPAM |0.21 | +|WARP-Q |0.10 | +|GVPMOS |0.01 | + +## furball_mushra + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.61 |0.77 |0.66 |0.67 |0.63 |0.85 |0.62 |0.77 |0.81 | +|DPAM |0.61 |1.00 |0.55 |0.54 |0.71 |0.61 |0.67 |0.66 |0.57 |0.67 | +|GVPMOS |0.77 |0.55 |1.00 |0.69 |0.63 |0.56 |0.89 |0.47 |0.64 |0.88 | +|MOS |0.66 |0.54 |0.69 |1.00 |0.56 |0.54 |0.74 |0.52 |0.58 |0.72 | +|PARLAQ |0.67 |0.71 |0.63 |0.56 |1.00 |0.79 |0.71 |0.56 |0.65 |0.68 | +|PEAQB |0.63 |0.61 |0.56 |0.54 |0.79 |1.00 |0.67 |0.58 |0.65 |0.65 | +|PESQ |0.85 |0.67 |0.89 |0.74 |0.71 |0.67 |1.00 |0.64 |0.75 |0.94 | +|ViSQOL |0.62 |0.66 |0.47 |0.52 |0.56 |0.58 |0.64 |1.00 |0.73 |0.60 | +|WARP-Q |0.77 |0.57 |0.64 |0.58 |0.65 |0.65 |0.75 |0.73 |1.00 |0.67 | +|Zimtohrli |0.81 |0.67 |0.88 |0.72 |0.68 |0.65 |0.94 |0.60 |0.67 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|PESQ |0.74 | +|Zimtohrli |0.72 | +|GVPMOS |0.69 | +|CDPAM |0.66 | +|WARP-Q |0.58 | +|PARLAQ |0.56 | +|PEAQB |0.54 | +|DPAM |0.54 | +|ViSQOL |0.52 | + +## minimodal_cmaudio_mushra + +### Spearman correlation table for all score types + +| |CDPAM |DPAM |GVPMOS |MOS |PARLAQ |PEAQB |PESQ |ViSQOL |WARP-Q |Zimtohrli | +|----------|------|-----|-------|-----|-------|------|-----|-------|-------|----------| +|CDPAM |1.00 |0.19 |0.42 |0.44 |0.20 |0.21 |0.35 |0.41 |0.08 |0.41 | +|DPAM |0.19 |1.00 |0.11 |0.19 |0.28 |0.18 |0.27 |0.28 |0.02 |0.26 | +|GVPMOS |0.42 |0.11 |1.00 |0.78 |0.30 |0.22 |0.44 |0.54 |0.00 |0.55 | +|MOS |0.44 |0.19 |0.78 |1.00 |0.47 |0.28 |0.60 |0.61 |0.07 |0.65 | +|PARLAQ |0.20 |0.28 |0.30 |0.47 |1.00 |0.55 |0.81 |0.49 |0.26 |0.63 | +|PEAQB |0.21 |0.18 |0.22 |0.28 |0.55 |1.00 |0.66 |0.28 |0.22 |0.39 | +|PESQ |0.35 |0.27 |0.44 |0.60 |0.81 |0.66 |1.00 |0.57 |0.22 |0.74 | +|ViSQOL |0.41 |0.28 |0.54 |0.61 |0.49 |0.28 |0.57 |1.00 |0.02 |0.80 | +|WARP-Q |0.08 |0.02 |0.00 |0.07 |0.26 |0.22 |0.22 |0.02 |1.00 |0.08 | +|Zimtohrli |0.41 |0.26 |0.55 |0.65 |0.63 |0.39 |0.74 |0.80 |0.08 |1.00 | + +### Score type MOS Spearman correlation in order + +|Score type |Spearman correlation | +|-----------|---------------------| +|GVPMOS |0.78 | +|Zimtohrli |0.65 | +|ViSQOL |0.61 | +|PESQ |0.60 | +|PARLAQ |0.47 | +|CDPAM |0.44 | +|PEAQB |0.28 | +|DPAM |0.19 | +|WARP-Q |0.07 | + +## perceptual_audio + +### Maximal audibility classification accuracy and threshold per score type + +|Score type |Accuracy |Threshold | +|-----------|---------|----------| +|Zimtohrli |0.74 |31 | +|CDPAM |0.71 |0.049 | +|ViSQOL |0.71 |4.2 | +|DPAM |0.69 |0.73 | +|GVPMOS |0.51 |3 | +|WARP-Q |0.49 |1.3 | +|PEAQB |0.49 |-3.9 | +|PARLAQ |0.48 |30 | +|PESQ |0.48 |1 | + +## Global leaderboard across all studies + +### Mean square error (1 - Spearman correlation, or 1 - accuracy) per score type + +|Score type |MSE |Min score |Max score |Mean score | +|-----------|-----|----------|----------|-----------| +|ViSQOL |0.10 |0.52 |0.80 |0.59 | +|Zimtohrli |0.13 |0.47 |0.72 |0.55 | +|PESQ |0.13 |0.34 |0.76 |0.56 | +|PARLAQ |0.15 |0.45 |0.78 |0.53 | +|CDPAM |0.15 |0.44 |0.70 |0.52 | +|PEAQB |0.19 |0.28 |0.85 |0.50 | +|DPAM |0.32 |0.19 |0.54 |0.36 | +|WARP-Q |0.34 |0.07 |0.75 |0.38 | +|GVPMOS |0.42 |0.01 |0.78 |0.31 | + diff --git a/go/bin/score/score.go b/go/bin/score/score.go index 5b146fa..1086e14 100644 --- a/go/bin/score/score.go +++ b/go/bin/score/score.go @@ -19,11 +19,13 @@ import ( "encoding/json" "flag" "fmt" + "io" "log" "os" "path/filepath" "runtime" "sort" + "time" "github.com/google/zimtohrli/go/data" "github.com/google/zimtohrli/go/goohrli" @@ -47,16 +49,68 @@ func main() { zimtohrliPerceptualSampleRate := flag.Float64("zimtohrli_perceptual_sample_rate", goohrli.DefaultPerceptualSampleRate(), "Sample rate of the Zimtohrli spectrograms.") correlate := flag.String("correlate", "", "Path to a database directory with a study to correlate scores for.") leaderboard := flag.String("leaderboard", "", "Glob to directories with databases to compute leaderboard for.") + report := flag.String("report", "", "Glob to directories with databases to generate a report for.") accuracy := flag.String("accuracy", "", "Path to a database directory with a study to provide JND accuracy for.") workers := flag.Int("workers", runtime.NumCPU(), "Number of concurrent workers for tasks.") failFast := flag.Bool("fail_fast", false, "Whether to panic immediately on any error.") flag.Parse() - if *details == "" && *calculate == "" && *correlate == "" && *accuracy == "" && *leaderboard == "" { + if *details == "" && *calculate == "" && *correlate == "" && *accuracy == "" && *leaderboard == "" && *report == "" { flag.Usage() os.Exit(1) } + if *report != "" { + databases, err := filepath.Glob(*report) + if err != nil { + log.Fatal(err) + } + fmt.Printf(`# Zimtohrli correlation report + +Created at %s + +`, time.Now().Format(time.DateOnly)) + studies := make(data.Studies, len(databases)) + for index, path := range databases { + fmt.Printf("## %s\n\n", filepath.Base(path)) + if studies[index], err = data.OpenStudy(path); err != nil { + log.Fatal(err) + } + isJND := false + if err := studies[index].ViewEachReference(func(ref *data.Reference) error { + for _, dist := range ref.Distortions { + if _, found := dist.Scores[data.JND]; found { + isJND = true + } + } + return io.EOF + }); err != nil { + log.Fatal(err) + } + if isJND { + accuracy, err := studies[index].Accuracy() + if err != nil { + log.Fatal(err) + } + fmt.Println(accuracy) + } else { + corrTable, err := studies[index].Correlate() + if err != nil { + log.Fatal(err) + } + fmt.Println(corrTable) + } + } + + fmt.Println("## Global leaderboard across all studies\n") + + board, err := studies.Leaderboard() + if err != nil { + log.Fatal(err) + } + fmt.Println(board) + } + if *leaderboard != "" { databases, err := filepath.Glob(*leaderboard) if err != nil {