diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8db96a8..b19c747 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -24,18 +24,24 @@ jobs: matrix: version: - '1.10' - - '1.6' - - 'nightly' os: - ubuntu-latest + # - macOS-latest + # - windows-latest arch: - x64 + nthreads: + - 3 steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} + - name: Install Custom Registry + run: julia -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(;url="https://github.com/FF-UH/CSC_Registry.jl"))' + - name: Install Custom Registry + run: julia -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(;url="https://github.com/MetabolicXploration/MetX_Registry_jl"))' - uses: julia-actions/cache@v1 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 diff --git a/Project.toml b/Project.toml index 59ebe34..abbede0 100644 --- a/Project.toml +++ b/Project.toml @@ -3,15 +3,15 @@ uuid = "c7313018-426c-4109-b567-6f433e2229c2" authors = ["josePereiro and contributors"] version = "1.0.0-DEV" -[compat] -julia = "1.6" - [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +SimpleLockFiles = "0c975e61-21d9-4454-8de5-ff08fd2c478f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +[compat] +julia = "1.6" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/Base/lock.jl b/src/Base/lock.jl new file mode 100644 index 0000000..3809f4b --- /dev/null +++ b/src/Base/lock.jl @@ -0,0 +1,25 @@ +# this just requere to implement a _getlock + +function _lock(f::Function, B::Bloberia; kwargs...) + lk = _getlock(B) + isnothing(lk) && return f() # ignore locking + lock(f, lk, kwargs...) +end +function _lock(B::Bloberia; kwargs...) + lk = _getlock(B) + isnothing(lk) && return # ignore locking + lock(lk, kwargs...) + return B +end + +function _islocked(B::Bloberia) + lk = _getlock(B) + isnothing(lk) && return false + return islocked(lk) +end + +function _unlock(B::Bloberia; force = false) + lk = _getlock(B) + isnothing(lk) && return # ignore + return unlock(lk; force) +end \ No newline at end of file diff --git a/src/BlobBatchBase/base.jl b/src/BlobBatchBase/base.jl index 773194a..cd355ae 100644 --- a/src/BlobBatchBase/base.jl +++ b/src/BlobBatchBase/base.jl @@ -110,9 +110,3 @@ Base.iterate(bb::BlobBatch, state) = iterate(bb.uuids, state) import Base.length Base.length(bb::BlobBatch) = blobcount(bb) -## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- -# Use, uuids -function blobcount(bb::BlobBatch) - _ondemand_loaduuids!(bb) - return length(bb.uuids) -end \ No newline at end of file diff --git a/src/BlobBatchBase/blob.jl b/src/BlobBatchBase/blob.jl index c09f989..5663a35 100644 --- a/src/BlobBatchBase/blob.jl +++ b/src/BlobBatchBase/blob.jl @@ -5,4 +5,11 @@ function blob(bb::BlobBatch, uuid = uuid_int()) b = Blob(bb, uuid) push!(bb.uuids, b.uuid) return b +end + +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +# Use, uuids +function blobcount(bb::BlobBatch) + _ondemand_loaduuids!(bb) + return length(bb.uuids) end \ No newline at end of file diff --git a/src/BlobBatchBase/lock.jl b/src/BlobBatchBase/lock.jl new file mode 100644 index 0000000..dae01b2 --- /dev/null +++ b/src/BlobBatchBase/lock.jl @@ -0,0 +1,14 @@ +_pidfile(bb::BlobBatches) = joinpath(batchpath(bb), "bb.pidfile") +_getlock(bb::BlobBatches) = get!(() -> SimpleLockFile(_pidfile(bb)), bb.temp, "_lock") +_setlock!(bb::BlobBatches, lk) = setindex!(bb["temp"], lk, "_lock") + +import Base.lock +Base.lock(f::Function, bb::BlobBatches; kwargs...) = _lock(f, bb; kwargs...) +Base.lock(bb::BlobBatches; kwargs...) = _lock(bb; kwargs...) + +import Base.islocked +Base.islocked(bb::BlobBatches) = _islocked(bb) + +import Base.unlock +Base.unlock(bb::BlobBatches; force = false) = _unlock(bb; force) + \ No newline at end of file diff --git a/src/BloberiaBase/base.jl b/src/BloberiaBase/base.jl index 866f4c4..b25a51c 100644 --- a/src/BloberiaBase/base.jl +++ b/src/BloberiaBase/base.jl @@ -7,6 +7,20 @@ const BLOBERIA_DEFAULT_FRAME_NAME = "0" Bloberia(root) = Bloberia(root, OrderedDict(), OrderedDict()) Bloberia() = Bloberia("") +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +import Base.show +function Base.show(io::IO, B::Bloberia) + if _hasfilesys(B) + count = batchcount(B) + print(io, "Bloberia with ", count, " batch(es)") + print(io, "\nfilesys: ", B.root) + else + print(io, "Bloberia: filesys not found...") + end +end + + + ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # filesys function _hasfilesys(B::Bloberia) diff --git a/src/BloberiaBase/blobbatch.jl b/src/BloberiaBase/blobbatch.jl index 8df7b17..4466c34 100644 --- a/src/BloberiaBase/blobbatch.jl +++ b/src/BloberiaBase/blobbatch.jl @@ -19,4 +19,15 @@ function headbatch(B::Bloberia, group::AbstractString = BLOBERIA_DEFAULT_BATCH_G end # If non found, return new return isnothing(bb) ? BlobBatch(B, group) : bb +end + +## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- +function batchcount(B::Bloberia) + count = 0 + isdir(B.root) || return count + for path in readdir(B.root; join = true) + _isbatchdir(path) || continue + count += 1 + end + return count end \ No newline at end of file diff --git a/src/BloberiaBase/iterator.jl b/src/BloberiaBase/iterator.jl index 132747c..94fb090 100644 --- a/src/BloberiaBase/iterator.jl +++ b/src/BloberiaBase/iterator.jl @@ -12,7 +12,7 @@ function batches(B::Bloberia, group_pt = nothing; paths = sortfun(readdir(B.root; join = true)) n = ceil(Int, length(paths) / ch_size) ts = map(Iterators.partition(paths, n)) do t_paths - @spawn for path in t_paths + Threads.@spawn for path in t_paths _isbatchdir(path) || continue group, uuid = _split_batchname(path) # filter @@ -31,7 +31,7 @@ end ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- # foreach_batch -# TODO: meke all this multithreading and race safe +# TODO: make all this multithreading and race safe function foreach_batch(f::Function, B::Bloberia, group_pt = nothing; sortfun = identity, ch_size::Int = nthreads(), diff --git a/src/BloberiaBase/lock.jl b/src/BloberiaBase/lock.jl new file mode 100644 index 0000000..6eeda98 --- /dev/null +++ b/src/BloberiaBase/lock.jl @@ -0,0 +1,14 @@ +_pidfile(B::Bloberia) = joinpath(B.root, "B.pidfile") +_getlock(B::Bloberia) = get!(() -> SimpleLockFile(_pidfile(B)), B.temp, "_lock") +_setlock!(B::Bloberia, lk) = setindex!(B["temp"], lk, "_lock") + +import Base.lock +Base.lock(f::Function, B::Bloberia; kwargs...) = _lock(f, B; kwargs...) +Base.lock(B::Bloberia; kwargs...) = _lock(B; kwargs...) + +import Base.islocked +Base.islocked(B::Bloberia) = _islocked(B) + +import Base.unlock +Base.unlock(B::Bloberia; force = false) = _unlock(B; force) + \ No newline at end of file diff --git a/src/Bloberias.jl b/src/Bloberias.jl index 08c7bd8..7a712ed 100644 --- a/src/Bloberias.jl +++ b/src/Bloberias.jl @@ -9,6 +9,7 @@ module Bloberias using Serialization using UUIDs using Base.Threads + using SimpleLockFiles #! include . @@ -23,6 +24,7 @@ module Bloberias include("BloberiaBase/base.jl") include("BloberiaBase/blobbatch.jl") include("BloberiaBase/iterator.jl") + include("BloberiaBase/lock.jl") #! include BlobBase include("BlobBase/base.jl") @@ -39,4 +41,4 @@ module Bloberias @_exportall_words() -end +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index db56ad9..147badd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,5 +2,6 @@ using Bloberias using Test @testset "Bloberias.jl" begin - # Write your tests here. + B = Bloberia() + @test true end