Skip to content

Commit

Permalink
Turn CandidateCompression into CandidateTemplateWithCompression
Browse files Browse the repository at this point in the history 
... and add CandidateWithLocation and CandidateWithUnknownLocation ,
so that the BIC implementations only need to deal with one value
instead of carrying around three; we will want to add one more,
and encapsulating them all into a single template will make it
transparent to the cache implementations.

Signed-off-by: Miloslav Trmač <[email protected]>
  • Loading branch information
@mtrmac
mtrmac committed Jul 27, 2024
1 parent ca8655f commit b7e61fe
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 71 deletions.
25 changes: 4 additions & 21 deletions pkg/blobinfocache/boltdb/boltdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,8 +373,8 @@ func (bdc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
compressorName = string(compressorNameValue)
}
}
ok, compressionOp, compressionAlgo := prioritize.CandidateCompression(v2Options, digest, compressorName)
if !ok {
template := prioritize.CandidateTemplateWithCompression(v2Options, digest, compressorName)
if template == nil {
return candidates
}

Expand All @@ -388,28 +388,11 @@ func (bdc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if err := t.UnmarshalBinary(v); err != nil {
return err
}
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
Location: types.BICLocationReference{Opaque: string(k)},
},
LastSeen: t,
})
candidates = append(candidates, prioritize.CandidateWithLocation(template, types.BICLocationReference{Opaque: string(k)}, t))
return nil
}) // FIXME? Log error (but throttle the log volume on repeated accesses)?
} else if v2Options != nil {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: time.Time{},
})
candidates = append(candidates, prioritize.CandidateWithUnknownLocation(template))
}
return candidates
}
Expand Down
47 changes: 39 additions & 8 deletions pkg/blobinfocache/internal/prioritize/prioritize.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,21 @@ const replacementAttempts = 5
// This is a heuristic/guess, and could well use a different value.
const replacementUnknownLocationAttempts = 2

// CandidateCompression returns (true, compressionOp, compressionAlgo) if a blob
// CandidateTemplateWithCompression returns a partially-filled BICReplacementCandidate2 if a blob
// with compressionName (which can be Uncompressed or UnknownCompression) is acceptable for a CandidateLocations* call with v2Options.
//
// v2Options can be set to nil if the call is CandidateLocations (i.e. compression is not required to be known);
// if not nil, the call is assumed to be CandidateLocations2.
//
// The (compressionOp, compressionAlgo) values are suitable for BICReplacementCandidate2
func CandidateCompression(v2Options *blobinfocache.CandidateLocations2Options, digest digest.Digest, compressorName string) (bool, types.LayerCompression, *compression.Algorithm) {
// The returned value is suitable for CandidateWithLocation and CandidateWithUnknownLocation.
func CandidateTemplateWithCompression(v2Options *blobinfocache.CandidateLocations2Options, digest digest.Digest, compressorName string) *blobinfocache.BICReplacementCandidate2 {
if v2Options == nil {
return true, types.PreserveOriginal, nil // Anything goes. The (compressionOp, compressionAlgo) values are not used.
return &blobinfocache.BICReplacementCandidate2{ // Anything goes. The CompressionOperation, CompressionAlgorithm values are not used.
Digest: digest,
CompressionOperation: types.PreserveOriginal,
CompressionAlgorithm: nil,
CompressionAnnotations: nil,
}
}

var op types.LayerCompression
Expand All @@ -45,14 +50,14 @@ func CandidateCompression(v2Options *blobinfocache.CandidateLocations2Options, d
algo = nil
case blobinfocache.UnknownCompression:
logrus.Debugf("Ignoring BlobInfoCache record of digest %q with unknown compression", digest.String())
return false, types.PreserveOriginal, nil // Not allowed with CandidateLocations2
return nil // Not allowed with CandidateLocations2
default:
op = types.Compress
algo_, err := compression.AlgorithmByName(compressorName)
if err != nil {
logrus.Debugf("Ignoring BlobInfoCache record of digest %q with unrecognized compression %q: %v",
digest.String(), compressorName, err)
return false, types.PreserveOriginal, nil // The BICReplacementCandidate2.CompressionAlgorithm field is required
return nil // The BICReplacementCandidate2.CompressionAlgorithm field is required
}
algo = &algo_
}
Expand All @@ -66,10 +71,15 @@ func CandidateCompression(v2Options *blobinfocache.CandidateLocations2Options, d
}
logrus.Debugf("Ignoring BlobInfoCache record of digest %q, compression %q does not match required %s or MIME types %#v",
digest.String(), compressorName, requiredCompresssion, v2Options.PossibleManifestFormats)
return false, types.PreserveOriginal, nil
return nil
}

return true, op, algo
return &blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: op,
CompressionAlgorithm: algo,
CompressionAnnotations: nil,
}
}

// CandidateWithTime is the input to types.BICReplacementCandidate prioritization.
Expand All @@ -78,6 +88,27 @@ type CandidateWithTime struct {
LastSeen time.Time // Time the candidate was last known to exist (either read or written) (not set for Candidate.UnknownLocation)
}

// CandidateWithLocation returns a complete CandidateWithTime combining (template from CandidateTemplateWithCompression, location, lastSeen)
func CandidateWithLocation(template *blobinfocache.BICReplacementCandidate2, location types.BICLocationReference, lastSeen time.Time) CandidateWithTime {
res := CandidateWithTime{
Candidate: *template,
LastSeen: lastSeen,
}
res.Candidate.Location = location
return res
}

// CandidateWithUnknownLocation returns a complete CandidateWithTime for a template from CandidateTemplateWithCompression and an unknown location.
func CandidateWithUnknownLocation(template *blobinfocache.BICReplacementCandidate2) CandidateWithTime {
res := CandidateWithTime{
Candidate: *template,
LastSeen: time.Time{},
}
res.Candidate.UnknownLocation = true
res.Candidate.Location = types.BICLocationReference{Opaque: ""}
return res
}

// candidateSortState is a closure for a comparison used by slices.SortFunc on candidates to prioritize,
// along with the specially-treated digest values relevant to the ordering.
type candidateSortState struct {
Expand Down
25 changes: 4 additions & 21 deletions pkg/blobinfocache/memory/memory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,34 +179,17 @@ func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if v, ok := mem.compressors[digest]; ok {
compressorName = v
}
ok, compressionOp, compressionAlgo := prioritize.CandidateCompression(v2Options, digest, compressorName)
if !ok {
template := prioritize.CandidateTemplateWithCompression(v2Options, digest, compressorName)
if template == nil {
return candidates
}
locations := mem.knownLocations[locationKey{transport: transport.Name(), scope: scope, blobDigest: digest}] // nil if not present
if len(locations) > 0 {
for l, t := range locations {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
Location: l,
},
LastSeen: t,
})
candidates = append(candidates, prioritize.CandidateWithLocation(template, l, t))
}
} else if v2Options != nil {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: time.Time{},
})
candidates = append(candidates, prioritize.CandidateWithUnknownLocation(template))
}
return candidates
}
Expand Down
25 changes: 4 additions & 21 deletions pkg/blobinfocache/sqlite/sqlite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,8 +507,8 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
compressorName = compressor
}
}
ok, compressionOp, compressionAlgo := prioritize.CandidateCompression(v2Options, digest, compressorName)
if !ok {
template := prioritize.CandidateTemplateWithCompression(v2Options, digest, compressorName)
if template == nil {
return candidates, nil
}

Expand All @@ -527,32 +527,15 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if err := rows.Scan(&location, &time); err != nil {
return nil, fmt.Errorf("scanning candidate: %w", err)
}
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
Location: types.BICLocationReference{Opaque: location},
},
LastSeen: time,
})
candidates = append(candidates, prioritize.CandidateWithLocation(template, types.BICLocationReference{Opaque: location}, time))
rowAdded = true
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterating through locations: %w", err)
}

if !rowAdded && v2Options != nil {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: time.Time{},
})
candidates = append(candidates, prioritize.CandidateWithUnknownLocation(template))
}
return candidates, nil
}
Expand Down

0 comments on commit b7e61fe

Please sign in to comment.