..

go.mod

@@ -5,16 +5,26 @@ go 1.21
 require (
 	github.com/PuerkitoBio/goquery v1.9.1
 	github.com/dgryski/go-onlinestats v0.0.0-20170612111826-1c7d19468768
+	github.com/nutsdb/nutsdb v1.0.4
 	github.com/youpy/go-wav v0.3.2
 	gonum.org/v1/gonum v0.15.0
 )
 
 require (
 	github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/antlabs/stl v0.0.1 // indirect
+	github.com/antlabs/timer v0.0.11 // indirect
+	github.com/bwmarrin/snowflake v0.3.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
+	github.com/xujiajun/mmap-go v1.0.1 // indirect
+	github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect
 	github.com/youpy/go-riff v0.1.0 // indirect
 	github.com/zaf/g711 v0.0.0-20190814101024-76a4a538f52b // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/tools v0.15.0 // indirect
 )

go.sum

@@ -4,20 +4,38 @@ github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTN
 github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/antlabs/stl v0.0.1 h1:TRD3csCrjREeLhLoQ/supaoCvFhNLBTNIwuRGrDIs6Q=
+github.com/antlabs/stl v0.0.1/go.mod h1:wvVwP1loadLG3cRjxUxK8RL4Co5xujGaZlhbztmUEqQ=
+github.com/antlabs/timer v0.0.11 h1:z75oGFLeTqJHMOcWzUPBKsBbQAz4Ske3AfqJ7bsdcwU=
+github.com/antlabs/timer v0.0.11/go.mod h1:JNV8J3yGvMKhCavGXgj9HXrVZkfdQyKCcqXBT8RdyuU=
+github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
+github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-onlinestats v0.0.0-20170612111826-1c7d19468768 h1:Xzl7CSuSnGsyU+9xmSU2h8w3d7Tnis66xeoNN207tLo=
 github.com/dgryski/go-onlinestats v0.0.0-20170612111826-1c7d19468768/go.mod h1:alfmlCqcg4uw9jaoIU1nOp9RFdJLMuu8P07BCEgpgoo=
+github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
+github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/nutsdb/nutsdb v1.0.4 h1:BurzkxijXJY1/AkIXe1ek+U1ta3WGi6nJt4nCLqkxQ8=
+github.com/nutsdb/nutsdb v1.0.4/go.mod h1:jIbbpBXajzTMZ0o33Yn5zoYIo3v0Dz4WstkVce+sYuQ=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg=
+github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
+github.com/xujiajun/mmap-go v1.0.1 h1:7Se7ss1fLPPRW+ePgqGpCkfGIZzJV6JPq9Wq9iv/WHc=
+github.com/xujiajun/mmap-go v1.0.1/go.mod h1:CNN6Sw4SL69Sui00p0zEzcZKbt+5HtEnYUsc6BKKRMg=
+github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 h1:w0si+uee0iAaCJO9q86T6yrhdadgcsoNuh47LrUykzg=
+github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235/go.mod h1:MR4+0R6A9NS5IABnIM3384FfOq8QFVnm7WDrBOhIaMU=
 github.com/youpy/go-riff v0.1.0 h1:vZO/37nI4tIET8tQI0Qn0Y79qQh99aEpponTPiPut7k=
 github.com/youpy/go-riff v0.1.0/go.mod h1:83nxdDV4Z9RzrTut9losK7ve4hUnxUR8ASSz4BsKXwQ=
 github.com/youpy/go-wav v0.3.2 h1:NLM8L/7yZ0Bntadw/0h95OyUsen+DQIVf9gay+SUsMU=
@@ -41,13 +59,16 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -67,8 +88,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
 gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

go/bin/compare/compare.go

@@ -18,45 +18,16 @@ package main
 import (
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"os"
 
 	"github.com/google/zimtohrli/go/goohrli"
-	"github.com/youpy/go-wav"
+	"github.com/google/zimtohrli/go/io"
 )
 
-func readWAV(path string) ([][]float32, float64, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, 0, err
-	}
-	defer f.Close()
-	r := wav.NewReader(f)
-	format, err := r.Format()
-	if err != nil {
-		return nil, 0, err
-	}
-	samples := []wav.Sample{}
-	var buf []wav.Sample
-	for buf, err = r.ReadSamples(32768); err == nil; buf, err = r.ReadSamples(32768) {
-		samples = append(samples, buf...)
-	}
-	if err != io.EOF {
-		return nil, 0, err
-	}
-	result := make([][]float32, format.NumChannels)
-	for _, sample := range samples {
-		for channelIndex := 0; channelIndex < int(format.NumChannels); channelIndex++ {
-			result[channelIndex] = append(result[channelIndex], float32(sample.Values[channelIndex])/float32(int(1)<<(format.BitsPerSample-1)))
-		}
-	}
-	return result, float64(format.SampleRate), nil
-}
-
 func main() {
-	pathA := flag.String("path_a", "", "Path to WAV file with signal A.")
-	pathB := flag.String("path_b", "", "Path to WAV file with signal B.")
+	pathA := flag.String("path_a", "", "Path to ffmpeg-decodable file with signal A.")
+	pathB := flag.String("path_b", "", "Path to ffmpeg-decodable file with signal B.")
 	frequencyResolution := flag.Float64("frequency_resolution", 1.0, "Band width of smallest filter, i.e. expected frequency resolution of human hearing.")
 	flag.Parse()
 
@@ -65,25 +36,25 @@ func main() {
 		os.Exit(1)
 	}
 
-	signalA, sampleRateA, err := readWAV(*pathA)
+	signalA, err := io.Load(*pathA)
 	if err != nil {
 		log.Panic(err)
 	}
-	signalB, sampleRateB, err := readWAV(*pathB)
+	signalB, err := io.Load(*pathB)
 	if err != nil {
 		log.Panic(err)
 	}
 
-	if sampleRateA != sampleRateB {
-		log.Panic(fmt.Errorf("sample rate of %q is %v, and sample rate of %q is %v", *pathA, sampleRateA, *pathB, sampleRateB))
+	if signalA.Rate != signalB.Rate {
+		log.Panic(fmt.Errorf("sample rate of %q is %v, and sample rate of %q is %v", *pathA, signalA.Rate, *pathB, signalB.Rate))
 	}
 
-	if len(signalA) != len(signalB) {
-		log.Panic(fmt.Errorf("%q has %v channels, and %q has %v channels", *pathA, len(signalA), *pathB, len(signalB)))
+	if len(signalA.Samples) != len(signalB.Samples) {
+		log.Panic(fmt.Errorf("%q has %v channels, and %q has %v channels", *pathA, len(signalA.Samples), *pathB, len(signalB.Samples)))
 	}
 
-	g := goohrli.New(sampleRateA, *frequencyResolution)
-	for channelIndex := 0; channelIndex < len(signalA); channelIndex++ {
-		fmt.Println(g.Distance(signalA[channelIndex], signalB[channelIndex]))
+	g := goohrli.New(signalA.Rate, *frequencyResolution)
+	for channelIndex := range signalA.Samples {
+		fmt.Println(g.Distance(signalA.Samples[channelIndex], signalB.Samples[channelIndex]))
 	}
 }

go/dataset/coresvnet/coresvnet.go

@@ -12,46 +12,53 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package coresvnet provides functions to fetch the listening test at https://listening-test.coresv.net/results.htm.
-package coresvnet
+// coresvnet downloads the listening test at https://listening-test.coresv.net/results.htm.
+package main
 
 import (
+	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	"net/url"
+	"os"
+	"runtime"
 	"strconv"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/google/zimtohrli/go/dataset"
+	"github.com/google/zimtohrli/go/io"
+	"github.com/google/zimtohrli/go/progress"
+	"github.com/google/zimtohrli/go/scores"
+	"github.com/google/zimtohrli/go/worker"
 )
 
-// Fetch returns the metadata for the listening test at https://listening-test.coresv.net/results.htm.
-func Fetch() (*dataset.Dataset, error) {
-	data := &dataset.Dataset{
-		ScoreType: dataset.Mos,
+func populate(dest string) error {
+	study, err := scores.OpenStudy(dest)
+	if err != nil {
+		return err
 	}
 
 	rootURL, err := url.Parse("https://listening-test.coresv.net/results.htm")
 	if err != nil {
-		return nil, err
+		return err
 	}
 	res, err := http.Get(rootURL.String())
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer res.Body.Close()
 	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
+		return fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
 	}
 	doc, err := goquery.NewDocumentFromReader(res.Body)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	resultTable := doc.Find("h2#list3:contains(\"All sets of tracks (5 sets, each 8 tracks)\")").Next().Find("table.table")
 
-	formats := []dataset.Format{"", "", dataset.Opus, dataset.Aac, dataset.Ogg, dataset.Mp3, dataset.Faac, dataset.Faac}
-
+	references := []*scores.Reference{}
 	err = nil
+	audioFiles := 0
 	resultTable.Find("tbody > tr").Each(func(index int, sel *goquery.Selection) {
 		columns := sel.Find("td")
 		if columns.Length() != 8 {
@@ -62,31 +69,71 @@ func Fetch() (*dataset.Dataset, error) {
 			err = parseErr
 			return
 		}
-		ref := &dataset.Reference{
-			Name:     u.String(),
-			Provider: dataset.URLPathProvider(u),
-			Format:   dataset.Wav,
+		ref := &scores.Reference{
+			Name: u.String(),
 		}
+		audioFiles++
 		for columnIndex := 2; columnIndex < columns.Length(); columnIndex++ {
 			u, parseErr := rootURL.Parse(columns.Eq(columnIndex).Find("a").AttrOr("href", ""))
 			if parseErr != nil {
 				err = parseErr
 				return
 			}
-			dist := &dataset.Distortion{
-				Name:     u.String(),
-				Provider: dataset.URLPathProvider(u),
-				Format:   formats[columnIndex],
+			dist := &scores.Distortion{
+				Name:   u.String(),
+				Scores: map[scores.ScoreType]float64{},
 			}
 			score, parseErr := strconv.ParseFloat(columns.Eq(columnIndex).Text(), 64)
 			if parseErr != nil {
 				err = parseErr
 				return
 			}
-			dist.Score = score
+			dist.Scores[scores.MOS] = score
 			ref.Distortions = append(ref.Distortions, dist)
+			audioFiles++
 		}
-		data.References = append(data.References, ref)
+		references = append(references, ref)
 	})
-	return data, err
+	bar := progress.New("Downloading")
+	pool := worker.Pool[any]{
+		Workers:    runtime.NumCPU(),
+		OnComplete: bar.Update,
+	}
+	for _, loopRef := range references {
+		ref := loopRef
+		pool.Submit(func(f func(any)) error {
+			ref.Path, err = io.Recode(ref.Name, dest)
+			if err != nil {
+				return err
+			}
+			for _, dist := range ref.Distortions {
+				dist.Path, err = io.Recode(dist.Name, dest)
+				if err != nil {
+					return err
+				}
+			}
+			if err := study.Put(ref); err != nil {
+				return err
+			}
+			return nil
+		})
+	}
+	if err := pool.Error(); err != nil {
+		return err
+	}
+	fmt.Println()
+	return nil
+}
+
+func main() {
+	destination := flag.String("dest", "", "Destination directory.")
+	flag.Parse()
+	if *destination == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	if err := populate(*destination); err != nil {
+		log.Fatal(err)
+	}
 }

go/dataset/runner/runner.go

@@ -161,7 +161,7 @@ func (s Setup) getOptimizeComparisons() (map[string]comparisonSlice, error) {
 	pool := &worker.Pool[prepareResult]{
 		Workers: safeInt(s.MaxWorkers),
 		OnComplete: func(submitted, completed int) {
-			bar.Update(completed, submitted)
+			bar.Update(submitted, completed)
 		},
 	}
 	for _, loopDatasetDir := range datasetDirs {
@@ -256,7 +256,7 @@ func (s Setup) optimize() error {
 			pool := &worker.Pool[compareResult]{
 				Workers: safeInt(s.MaxWorkers),
 				OnComplete: func(submitted, completed int) {
-					bar.Update(completed, submitted)
+					bar.Update(submitted, completed)
 				},
 			}
 
@@ -462,7 +462,7 @@ func (s Setup) Run() error {
 		}
 		bar := progress.New(fmt.Sprintf("Calculating for %v references", len(data.References)))
 		result, err := data.Calculate(metrics, func(submitted, completed int) {
-			bar.Update(completed, submitted)
+			bar.Update(submitted, completed)
 		}, safeInt(s.MaxWorkers), safeString(s.ProgressDirectory))
 		if err != nil {
 			return err