WIP: CI

.github/workflows/CI.yml

@@ -24,18 +24,24 @@ jobs:
       matrix:
         version:
           - '1.10'
-          - '1.6'
-          - 'nightly'
         os:
           - ubuntu-latest
+          # - macOS-latest
+          # - windows-latest
         arch:
           - x64
+        nthreads:
+          - 3
     steps:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
+      - name: Install Custom Registry
+        run: julia -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(;url="https://github.com/FF-UH/CSC_Registry.jl"))'
+      - name: Install Custom Registry
+        run: julia -e 'import Pkg; Pkg.Registry.add(Pkg.RegistrySpec(;url="https://github.com/MetabolicXploration/MetX_Registry_jl"))'
       - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1

Project.toml

@@ -3,15 +3,15 @@ uuid = "c7313018-426c-4109-b567-6f433e2229c2"
 authors = ["josePereiro <[email protected]> and contributors"]
 version = "1.0.0-DEV"
 
-[compat]
-julia = "1.6"
-
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+SimpleLockFiles = "0c975e61-21d9-4454-8de5-ff08fd2c478f"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
+[compat]
+julia = "1.6"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/Base/lock.jl

@@ -0,0 +1,25 @@
+# this just requere to implement a _getlock
+
+function _lock(f::Function, B::Bloberia; kwargs...) 
+    lk = _getlock(B)
+    isnothing(lk) && return f() # ignore locking
+    lock(f, lk, kwargs...)
+end
+function _lock(B::Bloberia; kwargs...) 
+    lk = _getlock(B)
+    isnothing(lk) && return # ignore locking 
+    lock(lk, kwargs...)
+    return B
+end
+
+function _islocked(B::Bloberia) 
+    lk = _getlock(B)
+    isnothing(lk) && return false
+    return islocked(lk)
+end
+
+function _unlock(B::Bloberia; force = false) 
+    lk = _getlock(B)
+    isnothing(lk) && return # ignore
+    return unlock(lk; force)
+end

src/BlobBatchBase/base.jl

@@ -110,9 +110,3 @@ Base.iterate(bb::BlobBatch, state) = iterate(bb.uuids, state)
 import Base.length
 Base.length(bb::BlobBatch) = blobcount(bb)
 
-## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- 
-# Use, uuids
-function blobcount(bb::BlobBatch)
-    _ondemand_loaduuids!(bb)
-    return length(bb.uuids)
-end

src/BlobBatchBase/blob.jl

@@ -5,4 +5,11 @@ function blob(bb::BlobBatch, uuid = uuid_int())
     b = Blob(bb, uuid)
     push!(bb.uuids, b.uuid)
     return b
+end
+
+## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- 
+# Use, uuids
+function blobcount(bb::BlobBatch)
+    _ondemand_loaduuids!(bb)
+    return length(bb.uuids)
 end

src/BlobBatchBase/lock.jl

@@ -0,0 +1,14 @@
+_pidfile(bb::BlobBatches) = joinpath(batchpath(bb), "bb.pidfile")
+_getlock(bb::BlobBatches) = get!(() -> SimpleLockFile(_pidfile(bb)), bb.temp, "_lock")
+_setlock!(bb::BlobBatches, lk) = setindex!(bb["temp"], lk, "_lock")
+
+import Base.lock
+Base.lock(f::Function, bb::BlobBatches; kwargs...) = _lock(f, bb; kwargs...)
+Base.lock(bb::BlobBatches; kwargs...) = _lock(bb; kwargs...) 
+
+import Base.islocked
+Base.islocked(bb::BlobBatches) = _islocked(bb) 
+
+import Base.unlock
+Base.unlock(bb::BlobBatches; force = false) = _unlock(bb; force) 
+

src/BloberiaBase/base.jl

@@ -7,6 +7,20 @@ const BLOBERIA_DEFAULT_FRAME_NAME = "0"
 Bloberia(root) = Bloberia(root, OrderedDict(), OrderedDict())
 Bloberia() = Bloberia("")
 
+## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- 
+import Base.show
+function Base.show(io::IO, B::Bloberia)
+    if _hasfilesys(B)
+        count = batchcount(B)
+        print(io, "Bloberia with ", count, " batch(es)")
+        print(io, "\nfilesys: ", B.root)
+    else
+        print(io, "Bloberia: filesys not found...")
+    end
+end
+
+
+
 ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- 
 # filesys
 function _hasfilesys(B::Bloberia)

src/BloberiaBase/blobbatch.jl

@@ -19,4 +19,15 @@ function headbatch(B::Bloberia, group::AbstractString = BLOBERIA_DEFAULT_BATCH_G
     end
     # If non found, return new
     return isnothing(bb) ? BlobBatch(B, group) : bb
+end
+
+## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- 
+function batchcount(B::Bloberia)
+    count = 0
+    isdir(B.root) || return count
+    for path in readdir(B.root; join = true)
+        _isbatchdir(path) || continue
+        count += 1
+    end
+    return count
 end

src/BloberiaBase/iterator.jl

@@ -12,7 +12,7 @@ function batches(B::Bloberia, group_pt = nothing;
         paths = sortfun(readdir(B.root; join = true))
         n = ceil(Int, length(paths) / ch_size)
         ts = map(Iterators.partition(paths, n)) do t_paths
-            @spawn for path in t_paths
+            Threads.@spawn for path in t_paths
                 _isbatchdir(path) || continue
                 group, uuid = _split_batchname(path)
                 # filter
@@ -31,7 +31,7 @@ end
 
 ## --.--. - .-. .- .--.-.- .- .---- ... . .-.-.-.- 
 # foreach_batch
-# TODO: meke all this multithreading and race safe
+# TODO: make all this multithreading and race safe
 function foreach_batch(f::Function, B::Bloberia, group_pt = nothing; 
         sortfun = identity, 
         ch_size::Int = nthreads(), 

src/BloberiaBase/lock.jl

@@ -0,0 +1,14 @@
+_pidfile(B::Bloberia) = joinpath(B.root, "B.pidfile")
+_getlock(B::Bloberia) = get!(() -> SimpleLockFile(_pidfile(B)), B.temp, "_lock")
+_setlock!(B::Bloberia, lk) = setindex!(B["temp"], lk, "_lock")
+
+import Base.lock
+Base.lock(f::Function, B::Bloberia; kwargs...) = _lock(f, B; kwargs...)
+Base.lock(B::Bloberia; kwargs...) = _lock(B; kwargs...) 
+
+import Base.islocked
+Base.islocked(B::Bloberia) = _islocked(B) 
+
+import Base.unlock
+Base.unlock(B::Bloberia; force = false) = _unlock(B; force) 
+

src/Bloberias.jl

@@ -9,6 +9,7 @@ module Bloberias
     using Serialization
     using UUIDs
     using Base.Threads
+    using SimpleLockFiles
 
     #! include .
 
@@ -23,6 +24,7 @@ module Bloberias
     include("BloberiaBase/base.jl")
     include("BloberiaBase/blobbatch.jl")
     include("BloberiaBase/iterator.jl")
+    include("BloberiaBase/lock.jl")
 
     #! include BlobBase
     include("BlobBase/base.jl")
@@ -39,4 +41,4 @@ module Bloberias
 
     @_exportall_words()
 
-end
+end

test/runtests.jl

@@ -2,5 +2,6 @@ using Bloberias
 using Test
 
 @testset "Bloberias.jl" begin
-    # Write your tests here.
+    B = Bloberia()
+    @test true
 end