diff --git a/_DEPRECATED/v4/src/0.Base/utils.jl b/_DEPRECATED/v4/src/0.Base/utils.jl index 59bddca..122b441 100644 --- a/_DEPRECATED/v4/src/0.Base/utils.jl +++ b/_DEPRECATED/v4/src/0.Base/utils.jl @@ -56,7 +56,6 @@ function _quoted_join(col, sep) return join(strs, sep) end - function _canonical_bytes(bytes) bytes < 1024 && return (bytes, "bytes") bytes /= 1024 diff --git a/src/2.BloberiaBase/base.jl b/src/2.BloberiaBase/base.jl index 5985179..58fab91 100644 --- a/src/2.BloberiaBase/base.jl +++ b/src/2.BloberiaBase/base.jl @@ -14,30 +14,6 @@ Base.copy(B::Bloberia) = Bloberia(B.root) # obj struct interface bloberia(bo::Bloberia) = bo -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# import Base.show -# function Base.show(io::IO, B::Bloberia) -# print(io, "Bloberia") -# _isdir = isdir(B.root) -# _pretty_print_pairs(io, -# "\n filesys", -# hasfilesys(B) ? B.root : "" -# ) -# _pretty_print_pairs(io, -# "\n batch(es)", -# _isdir ? batchcount(B) : 0 -# ) -# _pretty_print_pairs(io, -# "\n blob(s)", -# _isdir ? vblobcount(B) : 0 -# ) -# val, unit = _isdir ? _canonical_bytes(filesize(B)) : (0.0, "bytes") -# _pretty_print_pairs(io, -# "\n disk usage", -# _isdir ? string(round(val; digits = 3), " ", unit) : 0.0 -# ) -# end - # ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # # import Base.getindex # # # uuid indexing @@ -59,14 +35,14 @@ bloberia(bo::Bloberia) = bo # # Base.getindex(B::Bloberia) = blob!(B, BLOBERIA_DEFAULT_RABLOB_ID) # random access blob! -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# # Use, uuids -# function vblobcount(B::Bloberia, bbid_pt = nothing) -# count = 0 -# bbs = eachbatch(B, bbid_pt) -# for bb in bbs -# count += vblobcount(bb) -# end -# return count -# end +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# Use, uuids +function vblobcount(B::Bloberia, bbid_pt = nothing) + count = 0 + bbs = eachbatch(B, bbid_pt) + for bb in bbs + count += vblobcount(bb) + end + return count +end diff --git a/src/2.BloberiaBase/show.jl b/src/2.BloberiaBase/show.jl new file mode 100644 index 0000000..45c59c3 --- /dev/null +++ b/src/2.BloberiaBase/show.jl @@ -0,0 +1,23 @@ +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +import Base.show +function Base.show(io::IO, B::Bloberia) + print(io, "Bloberia") + _isdir = isdir(B.root) + _pretty_print_pairs(io, + "\n filesys", + hasfilesys(B) ? B.root : "" + ) + _pretty_print_pairs(io, + "\n batch(es)", + _isdir ? batchcount(B) : 0 + ) + _pretty_print_pairs(io, + "\n blob(s)", + _isdir ? vblobcount(B) : 0 + ) + val, unit = _isdir ? _canonical_bytes(filesize(B)) : (0.0, "bytes") + _pretty_print_pairs(io, + "\n disk usage", + _isdir ? string(round(val; digits = 3), " ", unit) : 0.0 + ) +end \ No newline at end of file diff --git a/src/3.BlobBatchBase/base.jl b/src/3.BlobBatchBase/base.jl index 42ab51b..e522e1a 100644 --- a/src/3.BlobBatchBase/base.jl +++ b/src/3.BlobBatchBase/base.jl @@ -34,67 +34,6 @@ function close!(bb::BlobBatch) setindex!(getmeta(bb), false, "bb.isopen") end -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# function _bb_show_file_sortby(ph) -# name = basename(ph) -# name == "uuid.blobs.jls" && return "." -# name == "meta.jls" && return ".." -# return name -# end - -# import Base.show -# function Base.show(io::IO, bb::BlobBatch) -# print(io, "BlobBatch(", repr(bb.uuid), ")") -# hasfilesys(bb) || return -# _pretty_print_pairs(io, -# "\n filesys", -# basename(batchpath(bb)) -# ) -# _pretty_print_pairs(io, -# "\n blob(s)", -# vblobcount(bb) -# ) -# if !isempty(bb.frames) -# print(io, "\nRam frames: ") -# for (frame, _bb_frame) in bb.frames -# isempty(_bb_frame) && continue -# kT_pairs = Set() -# for (_, _b_frame) in _bb_frame -# for (key, val) in _b_frame -# push!(kT_pairs, string(key) => typeof(val)) -# end -# end -# print(io, "\n \"", frame, "\" ") -# _kv_print_type(io, kT_pairs; _typeof = identity) -# end -# end - -# if isdir(bb) -# _bb_filesize = 0.0 -# print(io, "\nDisk frames: ") -# b_files = readdir(batchpath(bb); join = true) -# sort!(b_files; by = _bb_show_file_sortby) -# for path in b_files -# endswith(path, ".frame.jls") || continue -# _filesize = filesize(path) -# val, unit = _canonical_bytes(_filesize) -# print(io, "\n \"", basename(path), "\" ") -# print(io, "[") -# printstyled(io, string(round(val; digits = 3), " ", unit); -# color = :blue -# ) -# print(io, "]") -# _bb_filesize += _filesize -# end -# val, unit = _canonical_bytes(_bb_filesize) -# print(io, "\ndisk usage: ") -# printstyled(io, string(round(val; digits = 3), " ", unit); -# color = :blue -# ) -# end - -# end - # ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # # getindex @@ -102,28 +41,32 @@ end # Base.getindex(bb::BlobBatch, uuid::UInt128) = blob(bb, uuid) # Base.getindex(bb::BlobBatch, i::Int) = blob(bb, i) -# # isempty -# function Base.isempty(bb::BlobBatch) -# ondemand_loadvuuids!(bb) -# return isempty(bb.vuuids) -# end - -# # isempty -# import Base.empty! -# function Base.empty!(bb::BlobBatch) -# empty!(bb.meta) -# empty!(bb.vuuids) -# empty!(bb.temp) -# empty!(bb.frames) -# return nothing -# end +# isempty (ram only) +function Base.isempty(bb::BlobBatch) + isempty(bb.vuuids) || return false + isempty(bb.dframes) || return false + isempty(bb.vframes) || return false + return true +end -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# # Set the batch limit +# isempty (ram only) +import Base.empty! +function Base.empty!(bb::BlobBatch) + empty!(bb.meta) + empty!(bb.vuuids) + empty!(bb.temp) + empty!(bb.dframes) + empty!(bb.vframes) + return nothing +end -# function isfullbatch(bb::BlobBatch) -# B_lim = getmeta(bb.B, "batches.blobs.lim", typemax(Int))::Int -# bb_lim = getmeta(bb, "blobs.lim", typemax(Int))::Int -# lim = min(B_lim, bb_lim) -# return vblobcount(bb) >= lim -# end +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# Set the batch vframes limit +function isfullbatch(bb::BlobBatch) + B_meta = getmeta(bb.B) + B_lim = get(B_meta, "config.batches.vblobs.lim", typemax(Int))::Int + bb_meta = getmeta(bb) + bb_lim = get(bb_meta, "config.vblobs.lim", typemax(Int))::Int + lim = min(B_lim, bb_lim) + return vblobcount(bb) >= lim +end diff --git a/src/3.BlobBatchBase/dblob.jl b/src/3.BlobBatchBase/dblob.jl index 502d8fa..8f784fa 100644 --- a/src/3.BlobBatchBase/dblob.jl +++ b/src/3.BlobBatchBase/dblob.jl @@ -1,18 +1,4 @@ -# # random blobs +# random blobs -# # return blob if exist in DISK -# function blob(B::Bloberia, id::String) -# rb = dBlob(B, id) -# path = rablobpath(rb) -# isdir(path) || error("dBlob(\"", id, "\") not found.") -# return rb -# end -# blob(B::Bloberia) = blob(B, BLOBERIA_DEFAULT_RABLOB_ID) - -# # existing or new blob -# # in practice an unchecked blob -# function blob!(B::Bloberia, id::String) -# rb = dBlob(B, id) -# return rb -# end -# blob!(B::Bloberia) = blob!(B, BLOBERIA_DEFAULT_RABLOB_ID) +# return the only dblob +dblob(bb::BlobBatch) = dBlob(bb) \ No newline at end of file diff --git a/src/3.BlobBatchBase/getframe.jl b/src/3.BlobBatchBase/getframe.jl index 22a0c96..a072ae5 100644 --- a/src/3.BlobBatchBase/getframe.jl +++ b/src/3.BlobBatchBase/getframe.jl @@ -18,25 +18,29 @@ function getdframe!(bb::BlobBatch, framekey::AbstractString) return get!(OrderedDict, bb.dframes, framekey) end -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# # hasframe -# function hasframe_ram(bb::BlobBatch, frame) -# # check ram -# frame == "temp" && return true -# frame == "meta" && return true -# haskey(bb.frames, frame) && return true -# return false -# end +function getvuuids(bb::BlobBatch) + ondemand_loadvuuids!(bb) + return bb.vuuids +end + +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# hasframe +hasvframe_ram(bb::BlobBatch, frame) = + haskey(bb.vframes, frame) +hasdframe_ram(bb::BlobBatch, frame) = + haskey(bb.dframes, frame) +hasframe_ram(bb::BlobBatch, frame) = + hasvframe_ram(bb, frame) || hasdframe_ram(bb, frame) -# function hasframe_disk(bb::BlobBatch, frame) -# frame == "temp" && return false -# frame == "meta" && return isfile(meta_jlspath(bb)) -# isfile(vframe_jlspath(bb, frame)) && return true -# return false -# end +hasvframe_disk(bb::BlobBatch, frame) = + isfile(vframe_jlspath(bb, frame)) +hasdframe_disk(bb::BlobBatch, frame) = + isfile(dframe_jlspath(bb, frame)) +hasframe_disk(bb::BlobBatch, frame) = + hasvframe_disk(bb, frame) || hasdframe_ram(bb, frame) -# function hasframe(bb::BlobBatch, frame) -# hasframe_ram(bb, frame) && return true -# hasframe_disk(bb, frame) && return true -# return false -# end \ No newline at end of file +function hasframe(bb::BlobBatch, frame) + hasframe_ram(bb, frame) && return true + hasframe_disk(bb, frame) && return true + return false +end \ No newline at end of file diff --git a/src/3.BlobBatchBase/iterator.jl b/src/3.BlobBatchBase/iterator.jl index 01a7990..23ab2ba 100644 --- a/src/3.BlobBatchBase/iterator.jl +++ b/src/3.BlobBatchBase/iterator.jl @@ -1,37 +1,39 @@ -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# function eachblob(bb::BlobBatch) -# return Channel{vBlob}(0) do _ch -# ondemand_loadvuuids!(bb) -# for uuid in bb.vuuids -# b = vBlob(bb, uuid) # I do not need to check if blob exist -# put!(_ch, b) -# end -# end -# end +# The blobbatch iterate accross the blobs -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# # Iterator -# function _bb_iterate_next(ch::Channel, ch_next) -# isnothing(ch_next) && return nothing -# item, ch_state = ch_next -# bb_state = (ch, ch_state) -# return (item, bb_state) -# end +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +function eachblob(bb::BlobBatch) + return Channel{vBlob}(0) do _ch + vuuids = getvuuids(bb) + for uuid in vuuids + # I do not need to check if blob exist + b = vBlob(bb, uuid) + put!(_ch, b) + end + end +end -# import Base.iterate -# function Base.iterate(bb::BlobBatch) -# ch = eachblob(bb) -# ch_next = iterate(ch) -# return _bb_iterate_next(ch, ch_next) -# end +# --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# Iterator +function _bb_iterate_next(ch::Channel, ch_next) + isnothing(ch_next) && return nothing + item, ch_state = ch_next + bb_state = (ch, ch_state) + return (item, bb_state) +end -# function Base.iterate(::BlobBatch, bb_state) -# isnothing(bb_state) && return nothing -# ch, ch_state = bb_state -# ch_next = iterate(ch, ch_state) -# return _bb_iterate_next(ch, ch_next) -# end +import Base.iterate +function Base.iterate(bb::BlobBatch) + ch = eachblob(bb) + ch_next = iterate(ch) + return _bb_iterate_next(ch, ch_next) +end -# import Base.length -# Base.length(bb::BlobBatch) = vblobcount(bb) +function Base.iterate(::BlobBatch, bb_state) + isnothing(bb_state) && return nothing + ch, ch_state = bb_state + ch_next = iterate(ch, ch_state) + return _bb_iterate_next(ch, ch_next) +end +import Base.length +Base.length(bb::BlobBatch) = vblobcount(bb) \ No newline at end of file diff --git a/src/3.BlobBatchBase/serialize.jl b/src/3.BlobBatchBase/serialize.jl index b66761d..0f3a9c8 100644 --- a/src/3.BlobBatchBase/serialize.jl +++ b/src/3.BlobBatchBase/serialize.jl @@ -28,10 +28,15 @@ function serialize_dframe!(bb::BlobBatch, frame) end ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# serialize all or a particular frame depending on 'id' function serialize!(bb::BlobBatch, id = nothing; ignoreempty = false) + # TODO: Go back to isfull interface better + # TODO: This might be to restrictive (Think about it) + # - Maybe close individual files + # isopen(bb) || error("This batch is closed!!!") + onserialize!(bb) - mkpath(bb) # meta @@ -63,35 +68,3 @@ function serialize!(bb::BlobBatch, id = nothing; ignoreempty = false) return bb end - -## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# import Serialization.serialize -# function Serialization.serialize(bb::BlobBatch, frame::AbstractString; ignoreempty = true) - -# onserialize!(bb, frame) - -# dir = batchpath(bb) -# mkpath(dir) - -# # meta -# if frame == "meta" -# ignore = isempty(bb.meta) -# ignore = ignore && ignoreempty -# ignore || serialize_meta!(bb) -# return -# end - -# # uuids -# if frame == "uuids" -# ignore = isempty(bb.vuuids) -# ignore = ignore && ignoreempty -# ignore || serialize_vuuids!(bb) -# end - -# # frames -# ignore = isempty(bb.frames) | isempty(bb.frames[frame]) -# ignore = ignore && ignoreempty -# ignore || serialize_vframe!(bb, frame) - -# return bb -# end \ No newline at end of file diff --git a/src/3.BlobBatchBase/show.jl b/src/3.BlobBatchBase/show.jl new file mode 100644 index 0000000..fffa22c --- /dev/null +++ b/src/3.BlobBatchBase/show.jl @@ -0,0 +1,85 @@ +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +function _bb_show_file_sortby(ph) + name = basename(ph) + name == "meta.jls" && return "." + name == "vuuids.jls" && return ".." + return name +end + +function _show_vframes(io::IO, bb::BlobBatch) + isempty(bb.vframes) && return + print(io, "\nRam vframes: ") + for (frame, _bb_frame) in bb.vframes + isempty(_bb_frame) && continue + kT_pairs = Set() + for (_, _b_frame) in _bb_frame + for (key, val) in _b_frame + push!(kT_pairs, string(key) => typeof(val)) + end + end + print(io, "\n \"", frame, "\" ") + _kv_print_type(io, kT_pairs; _typeof = identity) + end +end + +function _show_dframes(io::IO, bb::BlobBatch) + isempty(bb.dframes) && return + print(io, "\nRam dframes: ") + for (frame, _bb_frame) in bb.dframes + isempty(_bb_frame) && continue + kT_pairs = Set() + for (key, val) in _bb_frame + push!(kT_pairs, string(key) => typeof(val)) + end + print(io, "\n \"", frame, "\" ") + _kv_print_type(io, kT_pairs; _typeof = identity) + end +end + +function _show_disk_files(io::IO, bb::BlobBatch) + _bb_filesize = 0.0 + print(io, "\nDisk files: ") + b_files = isdir(bb) ? readdir(batchpath(bb); join = true) : [] + sort!(b_files; by = _bb_show_file_sortby) + for path in b_files + # endswith(path, hint) || continue + _filesize = filesize(path) + val, unit = _canonical_bytes(_filesize) + print(io, "\n \"", basename(path), "\" ") + print(io, "[") + printstyled(io, string(round(val; digits = 3), " ", unit); + color = :blue + ) + print(io, "]") + _bb_filesize += _filesize + end + val, unit = _canonical_bytes(_bb_filesize) + print(io, "\ndisk usage: ") + printstyled(io, string(round(val; digits = 3), " ", unit); + color = :blue + ) +end + +import Base.show +function Base.show(io::IO, bb::BlobBatch) + print(io, "BlobBatch(", repr(bb.id), ")") + hasfilesys(bb) || return + _pretty_print_pairs(io, + "\n filesys", + string("@./", basename(batchpath(bb))) + ) + _pretty_print_pairs(io, + "\n vblob(s)", vblobcount(bb) + ) + # vframes + _show_vframes(io, bb) + + # dframes + _show_dframes(io, bb) + + # disk + _show_disk_files(io, bb) + + nothing + +end \ No newline at end of file diff --git a/src/3.BlobBatchBase/vblob.jl b/src/3.BlobBatchBase/vblob.jl index 55f82fc..d896471 100644 --- a/src/3.BlobBatchBase/vblob.jl +++ b/src/3.BlobBatchBase/vblob.jl @@ -1,22 +1,23 @@ # ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # get an existing vblob function vblob(bb::BlobBatch, i::Integer) - ondemand_loadvuuids!(bb) - return vBlob(bb, _getindex(bb.vuuids, i)) + vuuids = getvuuids(bb) + return vBlob(bb, _getindex(vuuids, i)) end function vblob(bb::BlobBatch, uuid::UInt128) - ondemand_loadvuuids!(bb) - uuid ∈ bb.vuuids || error("Uuid ", repr(uuid), " not present") + vuuids = getvuuids(bb) + uuid ∈ vuuids || error("Uuid ", repr(uuid), " not present") return vBlob(bb, uuid) end # registrer a new vblob if it does not exist function vblob!(bb::BlobBatch, uuid::Integer) uuid = UInt128(uuid) - ondemand_loadvuuids!(bb) b = vBlob(bb, uuid) - uuid ∈ bb.vuuids && return b + vuuids = getvuuids(bb) + uuid ∈ vuuids && return b + #TODO Use isfull interface isopen(bb) || error("bb is closed!!!") push!(bb.vuuids, uuid) return b @@ -30,6 +31,6 @@ rvblob!(bb::BlobBatch) = vblob!(bb, uuid_int()) # ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # Use, uuids # RAM STATE function vblobcount(bb::BlobBatch) - ondemand_loadvuuids!(bb) - return length(bb.vuuids) + vuuids = getvuuids(bb) + return length(vuuids) end diff --git a/src/5.vBlobBase/base.jl b/src/5.vBlobBase/base.jl index 121e7c8..57c6776 100644 --- a/src/5.vBlobBase/base.jl +++ b/src/5.vBlobBase/base.jl @@ -19,6 +19,17 @@ function _lock_obj_identity_hash(vb::vBlob, h0)::UInt64 return h end +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# ram only +import Base.empty! +function empty!(vb::vBlob) + bb = vb.batch + for (_, framedat) in bb.vframes + haskey(framedat, vb.uuid) || continue + empty!(framedat[vb,uuid]) + end +end + # ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # import Base.show # function Base.show(io::IO, b::vBlob) @@ -33,7 +44,7 @@ end # end # end -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # # setindex # function Base.setindex!(b::vBlob, value, frame::AbstractString, key) # _b_frame = getframe!(b, frame) # add frame if required diff --git a/src/6.dBlobBase/base.jl b/src/6.dBlobBase/base.jl index f8a7a0f..dd27402 100644 --- a/src/6.dBlobBase/base.jl +++ b/src/6.dBlobBase/base.jl @@ -3,7 +3,7 @@ # default is ok import Base.copy -Base.copy(rb::dBlob) = dBlob(rb.batch) +Base.copy(db::dBlob) = dBlob(db.batch) ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # parents @@ -33,12 +33,8 @@ end # # end # end -# ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# # setindex -# function Base.setindex!(b::dBlob, value, frame::AbstractString, key) -# _b_frame = getframe!(b, frame) # add frame if required -# return setindex!(_b_frame, value, key) -# end +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- + # # import Base.keys # # Base.keys(b::dBlob) = keys(_rablob_dict!(b)) @@ -49,10 +45,6 @@ end # # import Base.haskey # # Base.haskey(b::dBlob, key) = keys(_rablob_dict!(b), key) -# # isempty -# Base.isempty(rb::dBlob) = isempty(rb.frames) - -# function Base.empty!(rb::dBlob) -# empty!(rb.meta) -# empty!(rb.frames) -# end \ No newline at end of file +# isempty +Base.isempty(db::dBlob) = isempty(db.batch.dframes) +Base.empty!(db::dBlob) = empty!(db.batch.dframes) \ No newline at end of file diff --git a/src/7.BlobyRefBase/base.jl b/src/7.BlobyRefBase/base.jl index 92a0bb5..ea777e8 100644 --- a/src/7.BlobyRefBase/base.jl +++ b/src/7.BlobyRefBase/base.jl @@ -1,6 +1,6 @@ ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # Interface to implement -_merge_link!(::BlobyRef, rb) = error("Not implemented") +_merge_link!(::BlobyRef, db) = error("Not implemented") readref(::BlobyRef) = error("Not implemented") ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- @@ -85,41 +85,41 @@ readref(ref::BlobyRef{:vBlob, rT}) where rT = _v_blob(ref) readref(ref::BlobyRef{:vBlobVal, rT}) where rT = getindex(_v_blob(ref), ref.link["vb.frame"], ref.link["vb.key"]) -## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# # dBlob -# function _merge_link!(ref::BlobyRef, rb::dBlob, frame, key) -# ref.link["rb.id"] = rb.id -# ref.link["rb.frame"] = frame -# ref.link["rb.key"] = key -# end - -# function blobyref(rb::dBlob) -# ref = BlobyRef(:dBlob, dBlob) -# _merge_link!(ref, rb.B) -# _merge_link!(ref, rb, nothing, nothing) -# return ref -# end - -# function blobyref(rb::dBlob, frame, key; rT = nothing) -# if isnothing(rT) -# val = getindex(rb, frame, key) -# rT = typeof(val) -# end -# ref = BlobyRef(:raBlobVal, rT) -# _merge_link!(ref, rb.B) -# _merge_link!(ref, rb, frame, key) -# return ref -# end - -# _ra_blob(ref::BlobyRef) = -# blob!(_bloberia(ref), ref.link["rb.id"]) - -# bloberia(ref::BlobyRef{:dBlob, dBlob}) = _bloberia(ref) -# bloberia(ref::BlobyRef{:raBlobVal, rT}) where rT = _bloberia(ref) -# blob(ref::BlobyRef{:dBlob, dBlob}) = _ra_blob(ref) -# blob(ref::BlobyRef{:raBlobVal, rT}) where rT = _ra_blob(ref) - -# readref(ref::BlobyRef{:dBlob, rT}) where rT = _ra_blob(ref) -# readref(ref::BlobyRef{:raBlobVal, rT}) where rT = -# getindex(_ra_blob(ref), ref.link["rb.frame"], ref.link["rb.key"]) +# --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# dBlob +function _merge_link!(ref::BlobyRef, ::dBlob, frame, key) + ref.link["db.frame"] = frame + ref.link["db.key"] = key +end + +function blobyref(db::dBlob) + ref = BlobyRef(:dBlob, dBlob) + _merge_link!(ref, db.batch.B) + _merge_link!(ref, db.batch) + _merge_link!(ref, db, nothing, nothing) + return ref +end + +function blobyref(db::dBlob, frame, key; rT = nothing) + if isnothing(rT) + val = getindex(db, frame, key) + rT = typeof(val) + end + ref = BlobyRef(:dBlobVal, rT) + _merge_link!(ref, db.batch.B) + _merge_link!(ref, db.batch) + _merge_link!(ref, db, frame, key) + return ref +end + +_ra_blob(ref::BlobyRef) = dblob(_blobbatch(ref)) + +bloberia(ref::BlobyRef{:dBlob, dBlob}) = _bloberia(ref) +bloberia(ref::BlobyRef{:dBlobVal, rT}) where rT = _bloberia(ref) +blob(ref::BlobyRef{:dBlob, dBlob}) = _ra_blob(ref) +blob(ref::BlobyRef{:dBlobVal, rT}) where rT = _ra_blob(ref) + +readref(ref::BlobyRef{:dBlob, rT}) where rT = _ra_blob(ref) +readref(ref::BlobyRef{:dBlobVal, rT}) where rT = + getindex(_ra_blob(ref), ref.link["db.frame"], ref.link["db.key"]) \ No newline at end of file diff --git a/src/Bloberias.jl b/src/Bloberias.jl index 815ac86..b24b0d1 100644 --- a/src/Bloberias.jl +++ b/src/Bloberias.jl @@ -32,6 +32,7 @@ module Bloberias include("2.BloberiaBase/iterator.jl") include("2.BloberiaBase/loading.jl") include("2.BloberiaBase/lock.jl") + include("2.BloberiaBase/show.jl") include("2.BloberiaBase/meta.jl") include("2.BloberiaBase/serialize.jl") @@ -45,6 +46,7 @@ module Bloberias include("3.BlobBatchBase/lock.jl") include("3.BlobBatchBase/meta.jl") include("3.BlobBatchBase/serialize.jl") + include("3.BlobBatchBase/show.jl") include("3.BlobBatchBase/vblob.jl") #! include 4.AbstractBlobBase diff --git a/test/runtests.jl b/test/runtests.jl index 553e7f7..85b8b63 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -11,73 +11,97 @@ using Test B_ROOT = joinpath(tempname(), "db") atexit(() -> rm(B_ROOT; force = true, recursive = true)) - try; for testi in 1:10 # repeat many times + try; for testi in 1:1 # repeat many times println("-"^30) @show testi + ## .-- . -. - .--..- -- .- - --..-.-.- .- -.-- + # Test full interface + let + B = Bloberia(B_ROOT) + rm(B) + bb = blobbatch!(B, "globals") + + _lim = 11 + bb_meta = getmeta(bb) + bb_meta["config.vblobs.lim"] = _lim + while !isfullbatch(bb) + b = rvblob!(bb) + b["0", "rand"] = rand() + end + @test vblobcount(bb) == _lim + end + ## .-- . -. - .--..- -- .- - --..-.-.- .- -.-- # withblob! interface let B = Bloberia(B_ROOT) rm(B.root; force = true, recursive = true) - bb = blobbatch!(B) - rb = blob!(B) - vb = vblob!(bb) - + bb = blobbatch!(B, "0") + db = dblob(bb) + vb = vblob!(bb, 0) + _dat0 = [1,2,3] frame = string(hash(_dat0)) - for b in [rb, vb] + for b in [db, vb] withblob!(b, :get!, frame, "+1") do return _dat0 .+ 1 end @test b[frame, "+1"] == _dat0 .+ 1 - serialize(b) + serialize!(bb) end - + # check again but from loading # empty! - empty!(rb) - @test isempty(rb.frames) - _ref1 = withblob!(rb, :get!, frame, "+1") do + empty!(db) + @test isempty(db.batch.dframes) + _ref1 = withblob!(db, :get!, frame, "+1") do return "not to load" end @test _ref1[] == _dat0 .+ 1 empty!(bb) - @test isempty(bb.frames) - vb = blob(bb, 1) # first blob + @test isempty(bb) + @test isempty(bb.vuuids) + @test isempty(bb.dframes) + @test isempty(bb.vframes) + vb = vblob(bb, 1) # first blob _ref1 = withblob!(vb, :get!, frame, "+1") do return "not to load" end @test _ref1[] == _dat0 .+ 1 # Shadow copy - rb = dBlob(rb) # shadow copy - @test isempty(rb.frames) - _ref1 = withblob!(rb, :get!, frame, "+1") do + bb = copy(bb) # shadow copy + db = dblob(bb) + @test isempty(bb) + _ref1 = withblob!(db, :get!, frame, "+1") do return "not to load" end @test _ref1[] == _dat0 .+ 1 - - bb = BlobBatch(bb) # shadow copy - @test isempty(bb.frames) - vb = blob(bb, 1) # first blob + + bb = copy(bb) # shadow copy + @test isempty(bb) + vb = vblob(bb, 1) # first blob _ref1 = withblob!(vb, :get!, frame, "+1") do return "not to load" end @test _ref1[] == _dat0 .+ 1 - end + end + ## .-- . -. - .--..- -- .- - --..-.-.- .- -.-- # blobbatches interface let B = Bloberia(B_ROOT) rm(B; force = true) + break # DEV HEAD + bb0 = headbatch!(B) bb_lim = 100 setmeta!(B, "batches.blobs.lim", bb_lim) - + # write to ram _token = rand() for it in 1:(bb_lim ÷ 2) @@ -90,7 +114,7 @@ using Test end @test !isfullbatch(bb0) @test hasframe(bb0, "1") - serialize(bb0) + serialize!(bb0) _bbs = collect(eachbatch(B)) @test length(_bbs) == 1 @@ -107,7 +131,7 @@ using Test @test b["1", "_token"] == _token end - bb2 = blobbatch!(B) # new batch + bb2 = blobbatch!(B, "0") # new batch @test length(bb2) == 0 @test bb0.uuid != bb2.uuid @test length(bb2) != length(bb1) @@ -120,7 +144,7 @@ using Test end @test isfullbatch(bb0) @test length(bb0) == bb_lim - serialize(bb0) + serialize!(bb0) bb3 = headbatch!(B) @test length(bb3) == 0 @@ -136,17 +160,17 @@ using Test rm(B; force = true) rb1 = blob!(B, "test") - serialize(rb1) + serialize!(rb1) rb2 = blob(B, "test") # default frame rb1["bla"] = rand(5,5) - serialize(rb1) + serialize!(rb1) @test rb1["bla"] == rb2["bla"] # custom frame ("blo") rb1["blo", "bla"] = rand(5,5) - serialize(rb1) + serialize!(rb1) @test rb2["blo", "bla"] == rb1["blo", "bla"] end @@ -157,7 +181,7 @@ using Test B = Bloberia(B_ROOT) rm(B; force = true) b = blob!(B) - bb = blobbatch!(B) + bb = blobbatch!(B, "0") for obj in [B, b, bb] println(typeof(obj)) @@ -176,7 +200,7 @@ using Test @test isempty(meta1) # no disk copy yet setmeta!(obj, "bla", _dat0) - serialize(obj; ignoreempty = false) # create disk copy + serialize!(obj; ignoreempty = false) # create disk copy empty!(getmeta(obj)) @test all(getmeta(obj, "bla") .== _dat0) # data is loaded on demand end @@ -193,7 +217,7 @@ using Test @test bloberia(bref).root == B.root # BlobBatch - bb = blobbatch!(B) + bb = blobbatch!(B, "0") bref = blobyref(bb) @test bloberia(bref).root == B.root @test bref[].uuid == bb.uuid @@ -209,7 +233,7 @@ using Test # btBatchVal tb["val"] = 1 - serialize(tb) + serialize!(tb) @test bloberia(bref).root == B.root @test blobbatch(bref).uuid == bb.uuid @test blob(bref).uuid == tb.uuid @@ -217,19 +241,19 @@ using Test @test bref[] == tb["val"] # raBatch - rb = blob!(B) - bref = blobyref(rb) + db = blob!(B) + bref = blobyref(db) @test bloberia(bref).root == B.root - @test bref[].id == rb.id - @test blob(bref).id == rb.id + @test bref[].id == db.id + @test blob(bref).id == db.id # raBatchVal - rb["val"] = 1 - serialize(rb) + db["val"] = 1 + serialize!(db) @test bloberia(bref).root == B.root - @test blob(bref).id == rb.id - bref = blobyref(rb, "val") - @test bref[] == rb["val"] + @test blob(bref).id == db.id + bref = blobyref(db, "val") + @test bref[] == db["val"] nothing end @@ -239,11 +263,11 @@ using Test testi == 1 && let B = Bloberia(B_ROOT) rm(B.root; force = true, recursive = true) - bb = blobbatch!(B) - rb = blob!(B) + bb = blobbatch!(B, "0") + db = blob!(B) vb = vblob!(bb) - _bos = [B, bb, rb, vb] + _bos = [B, bb, db, vb] _dt = 1 # @show _dt # no lock