Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update to use gzip-compressed precompute features #55

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

update to use gzip-compressed precompute features #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ test_onecore_precomp: precompdir
${AUDFPRINT} new --dbase fpdbase0.pklz precompdir/Nine_Lives/0*
${AUDFPRINT} new --dbase fpdbase1.pklz precompdir/Nine_Lives/1*
${AUDFPRINT} merge --dbase fpdbase1.pklz fpdbase0.pklz
${AUDFPRINT} match --dbase fpdbase1.pklz precompdir/query.afpt
${AUDFPRINT} match --dbase fpdbase1.pklz precompdir/query.afptz

test_onecore_newmerge: precompdir
${AUDFPRINT} new --dbase fpdbase0.pklz precompdir/Nine_Lives/0*
${AUDFPRINT} new --dbase fpdbase1.pklz precompdir/Nine_Lives/1*
rm -f fpdbase2.pklz
${AUDFPRINT} newmerge --dbase fpdbase2.pklz fpdbase0.pklz fpdbase1.pklz
${AUDFPRINT} match --dbase fpdbase2.pklz precompdir/query.afpt
${AUDFPRINT} match --dbase fpdbase2.pklz precompdir/query.afptz

precompdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py
rm -rf precompdir
Expand All @@ -52,7 +52,7 @@ test_onecore_precomppk: precomppkdir
${AUDFPRINT} new --dbase fpdbase0.pklz precomppkdir/Nine_Lives/0*
${AUDFPRINT} new --dbase fpdbase1.pklz precomppkdir/Nine_Lives/1*
${AUDFPRINT} merge --dbase fpdbase1.pklz fpdbase0.pklz
${AUDFPRINT} match --dbase fpdbase1.pklz precomppkdir/query.afpk
${AUDFPRINT} match --dbase fpdbase1.pklz precomppkdir/query.afpkz
rm -rf precomppkdir

precomppkdir: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py
Expand All @@ -72,7 +72,7 @@ test_mucore_precomp: precompdir_mu
${AUDFPRINT} new --dbase fpdbase_mu0.pklz --ncores 4 precompdir_mu/Nine_Lives/0*
${AUDFPRINT} new --dbase fpdbase_mu.pklz --ncores 4 precompdir_mu/Nine_Lives/1*
${AUDFPRINT} merge --dbase fpdbase_mu.pklz fpdbase_mu0.pklz
${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt precompdir_mu/query.afpt
${AUDFPRINT} match --dbase fpdbase_mu.pklz --ncores 4 precompdir_mu/query.afptz precompdir_mu/query.afptz precompdir_mu/query.afptz precompdir_mu/query.afptz precompdir_mu/query.afptz precompdir_mu/query.afptz precompdir_mu/query.afptz

precompdir_mu: audfprint.py audfprint_analyze.py audfprint_match.py hash_table.py
rm -rf precompdir_mu
Expand Down
16 changes: 10 additions & 6 deletions audfprint_analyze.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import os
import struct # For reading/writing hashes to file
import time # For glob2hashtable, localtester
import gzip # for compressed file writing

import numpy as np
import scipy.signal
Expand All @@ -22,11 +23,12 @@
import stft



# ############### Globals ############### #
# Special extension indicating precomputed fingerprint
PRECOMPEXT = '.afpt'
PRECOMPEXT = '.afptz'
# A different precomputed fingerprint is just the peaks
PRECOMPPKEXT = '.afpk'
PRECOMPPKEXT = '.afpkz'


def locmax(vec, indices=False):
Expand Down Expand Up @@ -458,10 +460,12 @@ def ingest(self, hashtable, filename):
PEAK_FMT = '<2i'
PEAK_MAGIC = b'audfprintpeakV00' # 16 chars, FWIW

def is_compressed_ext(filename):
return filename[-1]=='z'

def hashes_save(hashfilename, hashes):
""" Write out a list of (time, hash) pairs as 32 bit ints """
with open(hashfilename, 'wb') as f:
with gzip.open(hashfilename, 'wb') if is_compressed_ext(hashfilename) else open(hashfilename, 'wb') as f:
f.write(HASH_MAGIC)
for time_, hash_ in hashes:
f.write(struct.pack(HASH_FMT, time_, hash_))
Expand All @@ -471,7 +475,7 @@ def hashes_load(hashfilename):
""" Read back a set of hashes written by hashes_save """
hashes = []
fmtsize = struct.calcsize(HASH_FMT)
with open(hashfilename, 'rb') as f:
with gzip.open(hashfilename, 'rb') if is_compressed_ext(hashfilename) else open(hashfilename, 'wb') as f:
magic = f.read(len(HASH_MAGIC))
if magic != HASH_MAGIC:
raise IOError('%s is not a hash file (magic %s)'
Expand All @@ -485,7 +489,7 @@ def hashes_load(hashfilename):

def peaks_save(peakfilename, peaks):
""" Write out a list of (time, bin) pairs as 32 bit ints """
with open(peakfilename, 'wb') as f:
with gzip.open(peakfilename, 'wb') if is_compressed_ext(peakfilename) else open(peakfilename, 'wb') as f:
f.write(PEAK_MAGIC)
for time_, bin_ in peaks:
f.write(struct.pack(PEAK_FMT, time_, bin_))
Expand All @@ -495,7 +499,7 @@ def peaks_load(peakfilename):
""" Read back a set of (time, bin) pairs written by peaks_save """
peaks = []
fmtsize = struct.calcsize(PEAK_FMT)
with open(peakfilename, 'rb') as f:
with gzip.open(peakfilename, 'rb') if is_compressed_ext(peakfilename) else open(peakfilename, 'wb') as f:
magic = f.read(len(PEAK_MAGIC))
if magic != PEAK_MAGIC:
raise IOError('%s is not a peak file (magic %s)'
Expand Down