diff --git a/CHANGELOG.md b/CHANGELOG.md index 0931c895..89bf05b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.5.6 + - Add Aqua tests and eliminate method ambiguities + ## 0.5.5 - Experimental support for writing to and reading from `IO` objects e.g. `jldopen(io, "r")` diff --git a/Project.toml b/Project.toml index e2669ac2..05be4ca1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "JLD2" uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.5.5" +version = "0.5.6" [deps] FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" @@ -13,7 +13,7 @@ TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" [compat] FileIO = "1.5" -MacroTools = "0.5" +MacroTools = "0.5.10" OrderedCollections = "1" PrecompileTools = "1" Requires = "1.3" diff --git a/README.md b/README.md index b693a303..d1cfaa8a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # JLD2 -| **Documentation** | **Tests** | **CodeCov** | **Downloads** | -|:--------:|:---------------:|:-------:|:---:| -|[![](https://img.shields.io/badge/docs-online-blue.svg)](https://JuliaIO.github.io/JLD2.jl/dev)| [![CI](https://github.com/JuliaIO/JLD2.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/JuliaIO/JLD2.jl/actions/workflows/ci.yml) | [![codecov.io](https://codecov.io/github/JuliaIO/JLD2.jl/coverage.svg?branch=master)](https://codecov.io/github/JuliaIO/JLD2.jl?branch=master) | [![JLD2 Downloads](https://img.shields.io/badge/dynamic/json?url=http%3A%2F%2Fjuliapkgstats.com%2Fapi%2Fv1%2Fmonthly_downloads%2FJLD2&query=total_requests&suffix=%2Fmonth&label=Downloads)](http://juliapkgstats.com/pkg/JLD2) | +[![](https://img.shields.io/badge/docs-online-blue.svg)](https://JuliaIO.github.io/JLD2.jl/dev) +[![CI](https://github.com/JuliaIO/JLD2.jl/actions/workflows/ci.yml/badge.svg)](https://github.com/JuliaIO/JLD2.jl/actions/workflows/ci.yml) +[![codecov.io](https://codecov.io/github/JuliaIO/JLD2.jl/coverage.svg?branch=master)](https://codecov.io/github/JuliaIO/JLD2.jl?branch=master) +[![JLD2 Downloads](https://img.shields.io/badge/dynamic/json?url=http%3A%2F%2Fjuliapkgstats.com%2Fapi%2Fv1%2Fmonthly_downloads%2FJLD2&query=total_requests&suffix=%2Fmonth&label=Downloads)](http://juliapkgstats.com/pkg/JLD2) +[![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl) + JLD2 saves and loads Julia data structures in a format comprising a subset of HDF5, without any dependency on the HDF5 C library. JLD2 is able to read most HDF5 files created by other HDF5 implementations supporting HDF5 File Format Specification Version 3.0 (i.e. libhdf5 1.10 or later) and similarly those should be able to read the files that JLD2 produces. JLD2 provides read-only support for files created with the JLD package. diff --git a/src/backwards_compatibility.jl b/src/backwards_compatibility.jl index 5692e855..26f81929 100644 --- a/src/backwards_compatibility.jl +++ b/src/backwards_compatibility.jl @@ -1,15 +1,15 @@ # The following block allows reading Union types written prior to v0.2 const LEGACY_H5TYPE_UNION = VariableLengthDatatype(H5TYPE_DATATYPE) -function jlconvert(::ReadRepresentation{Union, Vlen{DataTypeODR()}}, f::JLDFile, +function jlconvert(::MappedRepr{Union, Vlen{DataTypeODR}}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) - v = Union{jlconvert(ReadRepresentation{DataType,Vlen{DataTypeODR()}}(), f, ptr, NULL_REFERENCE)...} + v = Union{jlconvert(MappedRepr{DataType,Vlen{DataTypeODR}}(), f, ptr, NULL_REFERENCE)...} track_weakref!(f, header_offset, v) v end constructrr(::JLDFile, ::Type{T}, dt::VariableLengthDatatype, ::Vector{ReadAttribute}) where {T<:Union} = - dt == LEGACY_H5TYPE_UNION ? (ReadRepresentation{Union,Vlen{DataTypeODR()}}(), true) : + dt == LEGACY_H5TYPE_UNION ? (MappedRepr{Union,Vlen{DataTypeODR}}(), true) : throw(UnsupportedFeatureException()) # The following definition is needed to correctly load Strings written @@ -18,7 +18,7 @@ function read_array(f::JLDFile, dataspace::ReadDataspace, rr::FixedLengthString{String}, layout::DataLayout, filters::FilterPipeline, header_offset::RelOffset, attributes::Union{Vector{ReadAttribute},Nothing}) - rrv = ReadRepresentation{UInt8,odr(UInt8)}() + rrv = SameRepr{UInt8}() v = read_array(f, dataspace, rrv, layout, filters, NULL_REFERENCE, attributes) String(v) end diff --git a/src/committed_datatype_introspection.jl b/src/committed_datatype_introspection.jl index 4554e2b7..29785ee8 100644 --- a/src/committed_datatype_introspection.jl +++ b/src/committed_datatype_introspection.jl @@ -41,7 +41,7 @@ function stringify_committed_datatype(f, cdt; showfields=false) else # These are normal julia types dtrr = jltype(f, dt.members[i]) - fieldtype = string(typeof(dtrr).parameters[1]) + fieldtype = string(julia_type(dtrr)) end push!(field_strs, "$(dt.names[i])::$(fieldtype)") end @@ -97,7 +97,7 @@ function typestring_from_refs(f::JLDFile, ptr::Ptr) # Test for a potential null pointer indicating an empty array isinit = jlunsafe_load(convert(Ptr{UInt32}, ptr)) != 0 if isinit - refs = jlconvert(ReadRepresentation{RelOffset, Vlen{RelOffset}}(), f, ptr, NULL_REFERENCE) + refs = jlconvert(MappedRepr{RelOffset, Vlen{RelOffset}}(), f, ptr, NULL_REFERENCE) #println("datatypes at refs $(Int.(getproperty.(refs,:offset)))") params = Any[let # If the reference is to a committed datatype, read the datatype @@ -127,10 +127,10 @@ function jlconvert_string_wrap(rr, f, offset) end end -function jlconvert_string(rr::ReadRepresentation{T,DataTypeODR()}, +function jlconvert_string(::MappedRepr{T,DataTypeODR}, f::JLDFile, ptr::Ptr) where T - mypath = String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) + mypath = String(jlconvert(MappedRepr{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) params = typestring_from_refs(f, ptr+odr_sizeof(Vlen{UInt8})) if startswith(mypath, r"Core|Main") mypath = last(split(mypath, "."; limit=2)) @@ -143,7 +143,7 @@ function jlconvert_string(rr::ReadRepresentation{T,DataTypeODR()}, end end -function jlconvert_string(::ReadRepresentation{Union, UnionTypeODR()}, f::JLDFile, +function jlconvert_string(::MappedRepr{Union, UnionTypeODR}, f::JLDFile, ptr::Ptr)#, header_offset::RelOffset) # Skip union type description in the beginning ptr += odr_sizeof(Vlen{String}) diff --git a/src/data/custom_serialization.jl b/src/data/custom_serialization.jl index 380784ba..2da69ad6 100644 --- a/src/data/custom_serialization.jl +++ b/src/data/custom_serialization.jl @@ -47,12 +47,12 @@ h5convert!(out::Pointers, odr::Type{CustomSerialization{T,RelOffset}}, h5convert_uninitialized!(out::Pointers, odr::Type{CustomSerialization{T,ODR}}) where {T,ODR} = h5convert_uninitialized!(out, ODR) -jlconvert_canbeuninitialized(::ReadRepresentation{T,CustomSerialization{S,ODR}}) where {T,S,ODR} = - jlconvert_canbeuninitialized(ReadRepresentation{S,ODR}()) -jlconvert_isinitialized(::ReadRepresentation{T,CustomSerialization{S,ODR}}, ptr::Ptr) where {T,S,ODR} = - jlconvert_isinitialized(ReadRepresentation{S,ODR}(), ptr) +jlconvert_canbeuninitialized(::MappedRepr{T,CustomSerialization{S,ODR}}) where {T,S,ODR} = + jlconvert_canbeuninitialized(MappedRepr{S,ODR}()) +jlconvert_isinitialized(::MappedRepr{T,CustomSerialization{S,ODR}}, ptr::Ptr) where {T,S,ODR} = + jlconvert_isinitialized(MappedRepr{S,ODR}(), ptr) -function jlconvert(::ReadRepresentation{T,CustomSerialization{S,RelOffset}}, +function jlconvert(::MappedRepr{T,CustomSerialization{S,RelOffset}}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where {T,S} # Concerns objects whose custom serialization is itself only referenced by a RelOffset # This be important when the original object is mutable @@ -60,7 +60,7 @@ function jlconvert(::ReadRepresentation{T,CustomSerialization{S,RelOffset}}, return rconvert(T, load_dataset(f, offset)) end -function jlconvert(::ReadRepresentation{T,CustomSerialization{S,ODR}}, +function jlconvert(::MappedRepr{T,CustomSerialization{S,ODR}}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where {T,S,ODR} if ismutabletype(T) && !(T <: Core.SimpleVector) @@ -70,7 +70,7 @@ function jlconvert(::ReadRepresentation{T,CustomSerialization{S,ODR}}, track_weakref!(f, header_offset, obj) # actually load the data - v = rconvert(T, jlconvert(ReadRepresentation{S,ODR}(), f, ptr, header_offset))::T + v = rconvert(T, jlconvert(ReadRepresentation(S,ODR), f, ptr, header_offset))::T # copy fields to initial struct for i in 0:nfields(obj)-1 fieldval = ccall(:jl_get_nth_field, Any, (Any, Csize_t), v, i) @@ -78,7 +78,7 @@ function jlconvert(::ReadRepresentation{T,CustomSerialization{S,ODR}}, end return obj else - v = rconvert(T, jlconvert(ReadRepresentation{S,ODR}(), f, ptr, header_offset))::T + v = rconvert(T, jlconvert(ReadRepresentation(S,ODR), f, ptr, header_offset))::T track_weakref!(f, header_offset, v) return v end diff --git a/src/data/number_types.jl b/src/data/number_types.jl index f4a3f858..176c0785 100644 --- a/src/data/number_types.jl +++ b/src/data/number_types.jl @@ -22,7 +22,7 @@ struct BENumber{T} x::T end -jlconvert(::ReadRepresentation{T,BENumber{T}}, ::JLDFile, ptr::Ptr, ::RelOffset) where {T} = +jlconvert(::MappedRepr{T,BENumber{T}}, ::JLDFile, ptr::Ptr, ::RelOffset) where {T} = bswap(jlunsafe_load(pconvert(Ptr{T}, ptr))) function jltype(f::JLDFile, dt::FixedPointDatatype) @@ -33,29 +33,29 @@ function jltype(f::JLDFile, dt::FixedPointDatatype) throw(UnsupportedFeatureException()) if endianness == 0 if dt.size == 8 - return signed ? ReadRepresentation{Int64,Int64}() : ReadRepresentation{UInt64,UInt64}() + return signed ? SameRepr{Int64}() : SameRepr{UInt64}() elseif dt.size == 1 - return signed ? ReadRepresentation{Int8,Int8}() : ReadRepresentation{UInt8,UInt8}() + return signed ? SameRepr{Int8}() : SameRepr{UInt8}() elseif dt.size == 4 - return signed ? ReadRepresentation{Int32,Int32}() : ReadRepresentation{UInt32,UInt32}() + return signed ? SameRepr{Int32}() : SameRepr{Int32}() elseif dt.size == 2 - return signed ? ReadRepresentation{Int16,Int16}() : ReadRepresentation{UInt16,UInt16}() + return signed ? SameRepr{Int16}() : SameRepr{UInt16}() elseif dt.size == 16 - return signed ? ReadRepresentation{Int128,Int128}() : ReadRepresentation{UInt128,UInt128}() + return signed ? SameRepr{Int128}() : SameRepr{UInt128}() else throw(UnsupportedFeatureException()) end else if dt.size == 8 - return signed ? ReadRepresentation{Int64,BENumber{Int64}}() : ReadRepresentation{UInt64,BENumber{UInt64}}() + return signed ? MappedRepr{Int64,BENumber{Int64}}() : MappedRepr{UInt64,BENumber{UInt64}}() elseif dt.size == 1 - return signed ? ReadRepresentation{Int8,BENumber{Int8}}() : ReadRepresentation{UInt8,BENumber{UInt8}}() + return signed ? MappedRepr{Int8,BENumber{Int8}}() : MappedRepr{UInt8,BENumber{UInt8}}() elseif dt.size == 4 - return signed ? ReadRepresentation{Int32,BENumber{Int32}}() : ReadRepresentation{UInt32,BENumber{UInt32}}() + return signed ? MappedRepr{Int32,BENumber{Int32}}() : MappedRepr{UInt32,BENumber{UInt32}}() elseif dt.size == 2 - return signed ? ReadRepresentation{Int16,BENumber{Int16}}() : ReadRepresentation{UInt16,BENumber{UInt16}}() + return signed ? MappedRepr{Int16,BENumber{Int16}}() : MappedRepr{UInt16,BENumber{UInt16}}() elseif dt.size == 16 - return signed ? ReadRepresentation{Int128,BENumber{Int128}}() : ReadRepresentation{UInt128,BENumber{UInt128}}() + return signed ? MappedRepr{Int128,BENumber{Int128}}() : MappedRepr{UInt128,BENumber{UInt128}}() else throw(UnsupportedFeatureException()) end @@ -64,7 +64,7 @@ end # Special handling for booleans as they are not considered <: Integer in HDF5 h5fieldtype(::JLDFile, ::Type{Bool}, ::Type{Bool}, ::Initialized) =BitFieldDatatype(1) -jltype(::JLDFile, ::BitFieldDatatype) = ReadRepresentation{Bool, Bool}() +jltype(::JLDFile, ::BitFieldDatatype) = SameRepr{Bool}() h5fieldtype(::JLDFile, ::Type{Float16}, ::Type{Float16}, ::Initialized) = FloatingPointDatatype(UInt8(DT_FLOATING_POINT) + 0x3<<4, 0x20, 0x0f, 0x00, 2, 0, 16, 10, 5, 0, 10, 0x0000000f) @@ -82,17 +82,17 @@ h5fieldtype(::JLDFile, ::Type{BENumber{Float64}}, ::Type{Float64}, ::Initialized function jltype(f::JLDFile, dt::FloatingPointDatatype) if dt == h5fieldtype(f, Float64, Float64, Val{true}) - return ReadRepresentation{Float64,Float64}() + return SameRepr{Float64}() elseif dt == h5fieldtype(f, Float32, Float32, Val{true}) - return ReadRepresentation{Float32,Float32}() + return SameRepr{Float32}() elseif dt == h5fieldtype(f, Float16, Float16, Val{true}) - return ReadRepresentation{Float16,Float16}() + return SameRepr{Float16}() elseif dt == h5fieldtype(f, BENumber{Float64}, Float64, Val{true}) - return ReadRepresentation{Float32,BENumber{Float32}}() + return MappedRepr{Float32,BENumber{Float32}}() elseif dt == h5fieldtype(f, BENumber{Float32}, Float32, Val{true}) - return ReadRepresentation{Float32,BENumber{Float32}}() + return MappedRepr{Float32,BENumber{Float32}}() elseif dt == h5fieldtype(f, BENumber{Float16}, Float16, Val{true}) - return ReadRepresentation{Float32,BENumber{Float32}}() + return MappedRepr{Float32,BENumber{Float32}}() else throw(UnsupportedFeatureException()) end @@ -108,5 +108,5 @@ h5type(f::JLDFile, writeas::PrimitiveTypeTypes, x) = # Used only for custom serialization constructrr(f::JLDFile, T::PrimitiveTypeTypes, dt::Union{FixedPointDatatype,FloatingPointDatatype}, ::Vector{ReadAttribute}) = - dt == h5fieldtype(f, T, T, Val{true}) ? (ReadRepresentation{T,T}(), true) : + dt == h5fieldtype(f, T, T, Val{true}) ? (SameRepr{T}(), true) : throw(UnsupportedFeatureException()) diff --git a/src/data/reconstructing_datatypes.jl b/src/data/reconstructing_datatypes.jl index ebb0fd87..044d75b6 100644 --- a/src/data/reconstructing_datatypes.jl +++ b/src/data/reconstructing_datatypes.jl @@ -4,14 +4,14 @@ function read_field_datatypes(f::JLDFile, dt::CompoundDatatype, attrs::Vector{Re for attr in attrs if attr.name == :field_types offsets = read_attr_data(f, attr, ReferenceDatatype(), - ReadRepresentation{RelOffset,RelOffset}()) + SameRepr{RelOffset}()) elseif attr.name == :field_names namevec = read_attr_data(f, attr, H5TYPE_VLEN_UTF8, - ReadRepresentation{String,Vlen{String}}()) + MappedRepr{String,Vlen{String}}()) elseif attr.name == :field_datatypes # Legacy: Files written before JLD2 v0.4.54 offsets = read_attr_data(f, attr, ReferenceDatatype(), - ReadRepresentation{RelOffset,RelOffset}()) + SameRepr{RelOffset}()) end end isnothing(namevec) && (namevec = string.(dt.names)) @@ -93,7 +93,7 @@ function jltype(f::JLDFile, sdt::Union{SharedDatatype,CommittedDatatype}) end f.jlh5type[DataType] = cdt f.datatypes[cdt.index] = dt - return (f.h5jltype[cdt] = ReadRepresentation{DataType, DataTypeODR()}()) + return (f.h5jltype[cdt] = MappedRepr{DataType, DataTypeODR}()) end f.plain && return f.h5jltype[cdt] = jltype(f, dt) @@ -111,7 +111,7 @@ function jltype(f::JLDFile, sdt::Union{SharedDatatype,CommittedDatatype}) canonical = false else rr, canonical = constructrr(f, custom_datatype, dt, attrs) - rr = ReadRepresentation{read_as, CustomSerialization{typeof(rr).parameters...}}() + rr = MappedRepr{read_as, CustomSerialization{julia_repr(rr), file_repr(rr)}}() canonical &= writeas(read_as) === custom_datatype end else @@ -129,11 +129,11 @@ end function constructrr(::JLDFile, T::DataType, dt::BasicDatatype, attrs::Vector{ReadAttribute}) dt.class == DT_OPAQUE || throw(UnsupportedFeatureException()) if sizeof(T) == dt.size && isempty(T.types) - return (ReadRepresentation{T,T}(), true) + return (SameRepr{T}(), true) end empty = any(a->a.name==:empty, attrs) if empty - !hasdata(T) && return (ReadRepresentation{T,nothing}(), true) + !hasdata(T) && return (MappedRepr{T,nothing}(), true) @warn("$T has $(sizeof(T)*8) bytes, but written type was empty; reconstructing") elseif isempty(T.types) @warn("primitive type $T has $(sizeof(T)*8) bits, but written type has $(dt.size*8) bits; reconstructing") @@ -146,7 +146,7 @@ end struct TypeMappingException <: Exception end -unpack_odr(::OnDiskRepresentation{Offsets,JLTypes,H5Types}) where {Offsets,JLTypes,H5Types} = +unpack_odr(::Type{<:OnDiskRepresentation{Offsets,JLTypes,H5Types}}) where {Offsets,JLTypes,H5Types} = (Offsets, JLTypes.parameters, H5Types.parameters) @@ -201,7 +201,8 @@ will have a matching memory layout without first inspecting the memory layout. dtrr = jltype(f, dt.members[dtindex]) end - readtype, odrtype = typeof(dtrr).parameters + readtype = julia_repr(dtrr) + odrtype = file_repr(dtrr) if typeintersect(readtype, wstype) === Union{} && !hasmethod(convert, Tuple{Type{wstype}, readtype}) @@ -226,7 +227,7 @@ will have a matching memory layout without first inspecting the memory layout. types[i] === wstype && # An OnDiskRepresentation as odr means that something "fixable" went wrong # for this field - !(odrs[i] isa OnDiskRepresentation) && + !(odrs[i] <: OnDiskRepresentation) && !(odrs[i] <: CustomSerialization) mapped[dtindex] = true end @@ -239,9 +240,9 @@ will have a matching memory layout without first inspecting the memory layout. "\n\nData in these fields will not be accessible") end - samelayout && return (ReadRepresentation{T,T}(), true) + samelayout && return (SameRepr{T}(), true) offsets = (offsets...,) - if (wodr = odr(T)) isa OnDiskRepresentation + if (wodr = odr(T)) <: OnDiskRepresentation odr_offsets, odr_types, odr_h5types = unpack_odr(wodr) tequal = length(odr_types) == length(types) for i = 1:length(types) @@ -250,9 +251,9 @@ will have a matching memory layout without first inspecting the memory layout. tequal &= odr_h5types[i] == odrs[i] end tequal &= odr_offsets == offsets - tequal && return (ReadRepresentation{T,wodr}(), true) + tequal && return (MappedRepr{T,wodr}(), true) end - return (ReadRepresentation{T,OnDiskRepresentation{offsets, Tuple{types...}, Tuple{odrs...}, Int(offsets[end]+odr_sizeof(odrs[end]))}()}(), false) + return (MappedRepr{T,OnDiskRepresentation{offsets, Tuple{types...}, Tuple{odrs...}, Int(offsets[end]+odr_sizeof(odrs[end]))}}(), false) end function constructrr(f::JLDFile, u::Upgrade, dt::CompoundDatatype, @@ -261,14 +262,14 @@ function constructrr(f::JLDFile, u::Upgrade, dt::CompoundDatatype, field_datatypes = read_field_datatypes(f, dt, attrs) rodr = reconstruct_odr(f, dt, field_datatypes) fnames = tuple((Symbol(k) for k in keys(field_datatypes))...) - T2 = NamedTuple{fnames, typeof(rodr).parameters[2]} - return (ReadRepresentation{u.target, CustomSerialization{T2, rodr}}(), false) + T2 = NamedTuple{fnames, rodr.parameters[2]} + return (MappedRepr{u.target, CustomSerialization{T2, rodr}}(), false) end function constructrr(f::JLDFile, u::Upgrade, dt::BasicDatatype, attrs::Vector{ReadAttribute}, hard_failure::Bool=false) - return (ReadRepresentation{u.target, CustomSerialization{NamedTuple{(), Tuple{}},nothing}}(), false) + return (MappedRepr{u.target, CustomSerialization{NamedTuple{(), Tuple{}},nothing}}(), false) end function constructrr(f::JLDFile, T::UnionAll, dt::CompoundDatatype, @@ -280,7 +281,7 @@ end # Find types in modules # returns the result of searching for the type in the specified module m -function _resolve_type_singlemodule(::ReadRepresentation{T,DataTypeODR()}, +function _resolve_type_singlemodule(::MappedRepr{T,DataTypeODR}, m, parts, mypath, @@ -299,7 +300,7 @@ isunknowntype(x) = false isunknowntype(::Type{Union{}}) = false isunknowntype(x::Type) = x <: UnknownType ? true : false -function _resolve_type(rr::ReadRepresentation{T,DataTypeODR()}, +function _resolve_type(rr::MappedRepr{T,DataTypeODR}, f::JLDFile, ptr::Ptr, header_offset::RelOffset, @@ -328,12 +329,12 @@ function types_from_refs(f::JLDFile, ptr::Ptr) isinit = jlunsafe_load(pconvert(Ptr{UInt32}, ptr)) != 0 unknown_params = false if isinit - refs = jlconvert(ReadRepresentation{RelOffset, Vlen{RelOffset}}(), f, ptr, NULL_REFERENCE) + refs = jlconvert(MappedRepr{RelOffset, Vlen{RelOffset}}(), f, ptr, NULL_REFERENCE) params = Any[let # If the reference is to a committed datatype, read the datatype nulldt = CommittedDatatype(UNDEFINED_ADDRESS, 0) cdt = get(f.datatype_locations, ref, nulldt) - res = cdt !== nulldt ? eltype(jltype(f, cdt)) : load_dataset(f, ref) + res = cdt !== nulldt ? julia_repr(jltype(f, cdt)) : load_dataset(f, ref) unknown_params |= isunknowntype(res) || isreconstructed(res) res end for ref in refs] @@ -344,10 +345,10 @@ end # Read a type. Returns an instance of UnknownType if the type or parameters # could not be resolved. -function jlconvert(rr::ReadRepresentation{T,DataTypeODR()}, +function jlconvert(rr::MappedRepr{<:Type,DataTypeODR}, f::JLDFile, ptr::Ptr, - header_offset::RelOffset) where T + header_offset::RelOffset) params, unknown_params = types_from_refs(f, ptr+odr_sizeof(Vlen{UInt8})) # For cross-platform compatibility convert integer type parameters to system precision @@ -361,7 +362,7 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()}, end end hasparams = !isempty(params) - mypath = String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) + mypath = String(jlconvert(MappedRepr{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) if mypath in keys(f.typemap) m = f.typemap[mypath] @@ -390,7 +391,7 @@ function jlconvert(rr::ReadRepresentation{T,DataTypeODR()}, end constructrr(::JLDFile, ::Type{T}, dt::CompoundDatatype, ::Vector{ReadAttribute}) where {T<:DataType} = - dt == H5TYPE_DATATYPE ? (ReadRepresentation{DataType,DataTypeODR()}(), true) : + dt == H5TYPE_DATATYPE ? (MappedRepr{DataType,DataTypeODR}(), true) : throw(UnsupportedFeatureException()) @@ -448,10 +449,10 @@ isreconstructed(x::Type{Union{}}) = false function reconstruct_bitstype(name::Union{Symbol,String}, size::Integer, empty::Bool) if empty - return (ReadRepresentation{ReconstructedSingleton{Symbol(name)}, nothing}(), false) + return (MappedRepr{ReconstructedSingleton{Symbol(name)}, nothing}(), false) else T = ReconstructedPrimitive{Symbol(name), uintofsize(size)} - return (ReadRepresentation{T, T}(), false) + return (SameRepr{T}(), false) end end @@ -534,7 +535,7 @@ function constructrr(f::JLDFile, unk::Type{UnknownType{T,P}}, dt::CompoundDataty rodr = reconstruct_odr(f, dt, field_datatypes) # This is a "pseudo-RR" since the tuple is not fully parametrized, but # the parameters must depend on the types actually encoded in the file - return (ReadRepresentation{Tuple,rodr}(), false) + return (MappedRepr{Tuple,rodr}(), false) else @warn("read type $(typestring(unk)) was parametrized, but type " * "$(T) in workspace is not; reconstructing") @@ -601,12 +602,12 @@ function reconstruct_odr(f::JLDFile, dt::CompoundDatatype, else throw(InternalError("Field $k not found in datatype")) end - push!(types, typeof(dtrr).parameters[1]) - push!(h5types, typeof(dtrr).parameters[2]) + push!(types, julia_repr(dtrr)) + push!(h5types, file_repr(dtrr)) push!(offsets, offset) offset += odr_sizeof(dtrr) end - OnDiskRepresentation{(offsets...,), Tuple{types...}, Tuple{h5types...},Int(dt.size)}() + OnDiskRepresentation{(offsets...,), Tuple{types...}, Tuple{h5types...},Int(dt.size)} end # Reconstruct type that is a "lost cause": either we were not able to resolve @@ -617,26 +618,26 @@ function reconstruct_compound(f::JLDFile, T::String, dt::H5Datatype, rodr = reconstruct_odr(f, dt, field_datatypes) _, types, odrs = unpack_odr(rodr) fnames = tuple((Symbol(k) for k in keys(field_datatypes))...,) - if !any(jlconvert_canbeuninitialized(ReadRepresentation{types[i], odrs[i]}()) for i = 1:length(types)) + if !any(jlconvert_canbeuninitialized(ReadRepresentation(types[i], odrs[i])) for i = 1:length(types)) rt = ReconstructedStatic{Symbol(T), fnames, Tuple{types...}} - odr = OnDiskRepresentation{(0,), Tuple{NamedTuple{fnames,Tuple{types...}}}, Tuple{rodr}, Int(dt.size)}() - return (ReadRepresentation{rt, odr}(), false) + odr = OnDiskRepresentation{(0,), Tuple{NamedTuple{fnames,Tuple{types...}}}, Tuple{rodr}, Int(dt.size)} + return (MappedRepr{rt, odr}(), false) end T = ReconstructedMutable{Symbol(T), fnames, Tuple{types...}} - return ReadRepresentation{T, rodr}(), false + return MappedRepr{T, rodr}(), false end # At present, we write Union{} as an object of Core.TypeofBottom. The method above # basically works, but `Expr(:new, Type{Union{}})` is a bit weird and causes problems for # inference. Better to define a separate method. -jlconvert(::ReadRepresentation{Core.TypeofBottom,nothing}, f::JLDFile, ptr::Ptr, +jlconvert(::MappedRepr{Core.TypeofBottom,nothing}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) = Union{} -function jlconvert(::ReadRepresentation{T, S}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where {T<:ReconstructedMutable, S} +function jlconvert(::MappedRepr{T,S}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where {T<:ReconstructedMutable, S<:OnDiskRepresentation} offsets, types, odrs = unpack_odr(S) res = Vector{Any}(undef, length(types)) for i = 1:length(types) - rr = ReadRepresentation{types[i],odrs[i]}() + rr = ReadRepresentation(types[i],odrs[i]) if !(jlconvert_canbeuninitialized(rr)) || jlconvert_isinitialized(rr, ptr+offsets[i]) res[i] = jlconvert(rr, f, ptr+offsets[i], NULL_REFERENCE) end @@ -644,11 +645,12 @@ function jlconvert(::ReadRepresentation{T, S}, f::JLDFile, ptr::Ptr, header_offs return T(res) end +jlconvert(::ReadRepresentation{T, S}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where {T,S} = + rconvert(T, jlunsafe_load(pconvert(Ptr{S}, ptr))) + # This jlconvert method handles compound types with padding or references -@generated function jlconvert(::ReadRepresentation{T,S}, f::JLDFile, ptr::Ptr, - header_offset::RelOffset) where {T,S} - isa(S, DataType) && return :(convert(T, jlunsafe_load(pconvert(Ptr{S}, ptr)))) - @assert isa(S, OnDiskRepresentation) +@generated function jlconvert(::MappedRepr{T,S}, f::JLDFile, ptr::Ptr, + header_offset::RelOffset) where {T,S<:OnDiskRepresentation} offsets, types, odrs = unpack_odr(S) fn = T === Tuple ? [Symbol(i) for i = 1:length(types)] : fieldnames(T) @@ -659,7 +661,7 @@ end end for i = 1:length(types) rtype = types[i] - rr = ReadRepresentation{rtype,odrs[i]}() + rr = ReadRepresentation(rtype,odrs[i]) ttype = T.types[i] if isnothing(odrs[i]) # Type is not stored or single instance @@ -685,7 +687,7 @@ end rtype = types[i] fsym = Symbol("field_", fn[i]) push!(fsyms, fsym) - rr = ReadRepresentation{rtype,odrs[i]}() + rr = ReadRepresentation(rtype,odrs[i]) if isnothing(odrs[i]) push!(blk.args, :($fsym = $(Expr(:new, rtype)))) diff --git a/src/data/specialcased_types.jl b/src/data/specialcased_types.jl index 740fb39c..d7e4fe3c 100644 --- a/src/data/specialcased_types.jl +++ b/src/data/specialcased_types.jl @@ -4,7 +4,7 @@ struct OpaqueData{N} OpaqueData(data) = new{length(data)}(data) end -function jlconvert(rr::ReadRepresentation{OpaqueData{N}, NTuple{N,UInt8}}, ::JLDFile, ptr::Ptr, ::RelOffset) where N +function jlconvert(::MappedRepr{OpaqueData{N}, NTuple{N,UInt8}}, ::JLDFile, ptr::Ptr, ::RelOffset) where N data = Vector{UInt8}(undef, N) unsafe_copyto!(pointer(data), pconvert(Ptr{UInt8}, ptr), N) OpaqueData(data) @@ -50,20 +50,20 @@ function jltype(f::JLDFile, dt::BasicDatatype) class = dt.class if class >> 4 == 1 if class%16 == DT_REFERENCE - return ReadRepresentation{Any,RelOffset}() + return MappedRepr{Any,RelOffset}() elseif class%16 == DT_STRING if dt.bitfield1 == 0x00 && dt.bitfield2 == 0x00 && dt.bitfield3 == 0x00 #return AsciiString{NullTerminated}(dt.size) - return ReadRepresentation{String, FixedLengthAsciiString{NullTerminated, dt.size}}() + return MappedRepr{String, FixedLengthAsciiString{NullTerminated, dt.size}}() elseif dt.bitfield1 == 0x10 && dt.bitfield2 == 0x00 && dt.bitfield3 == 0x00 return FixedLengthString{String}(dt.size) elseif dt.bitfield1 == 0x02 && dt.bitfield2 == 0x00 && dt.bitfield3 == 0x00 - return ReadRepresentation{String, FixedLengthAsciiString{SpacePadded, dt.size}}() + return MappedRepr{String, FixedLengthAsciiString{SpacePadded, dt.size}}() else throw(UnsupportedFeatureException("Encountered an unsupported string type. $dt")) end elseif class%16 == DT_OPAQUE - return ReadRepresentation{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}() + return MappedRepr{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}() else throw(UnsupportedFeatureException("Encountered an unsupported type.")) @@ -78,9 +78,9 @@ function jltype(f::JLDFile, dt::BasicDatatype) throw(UnsupportedFeatureException("Encountered an unsupported string type.")) end elseif class%16 == DT_OPAQUE - return ReadRepresentation{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}() + return MappedRepr{OpaqueData{Int(dt.size)},NTuple{Int(dt.size),UInt8}}() elseif class%16 == DT_REFERENCE - return ReadRepresentation{Any,RelOffset}() + return MappedRepr{Any,RelOffset}() else throw(UnsupportedFeatureException()) end @@ -89,20 +89,20 @@ end function jltype(f::JLDFile, dt::VariableLengthDatatype) if dt == H5TYPE_VLEN_UTF8 # this is the fully supported JLD2 string - return ReadRepresentation{String,Vlen{String}}() + return MappedRepr{String,Vlen{String}}() elseif dt.bitfield1 & 0x1 == 0x1 # it's some kind of string. Let's try - return ReadRepresentation{String,Vlen{String}}() + return MappedRepr{String,Vlen{String}}() else#if dt.bitfield1 & 0x1 == 0x0 # it's a sequence rr = jltype(f, dt.basetype) - T = typeof(rr).parameters[1] - odr = typeof(rr).parameters[2] - return ReadRepresentation{Vector{T}, Vlen{odr}}() + T = julia_repr(rr) + odr = file_repr(rr) + return MappedRepr{Vector{T}, Vlen{odr}}() end end -jlconvert(::ReadRepresentation{Vector{T},Vlen{ODR}}, f::JLDFile, ptr::Ptr, ::RelOffset) where {T, ODR} = - jlconvert(ReadRepresentation{T,Vlen{ODR}}(), f, ptr, UNDEFINED_ADDRESS) +jlconvert(::MappedRepr{Vector{T},Vlen{ODR}}, f::JLDFile, ptr::Ptr, ::RelOffset) where {T, ODR} = + jlconvert(MappedRepr{T,Vlen{ODR}}(), f, ptr, UNDEFINED_ADDRESS) function h5convert!(out::Pointers, fls::FixedLengthString, f::JLDFile, x, ::JLDWriteSession) @@ -112,8 +112,8 @@ end h5convert!(out::Pointers, ::Type{Vlen{String}}, f::JLDFile, x, wsession::JLDWriteSession) = store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, x), wsession) -jlconvert(::ReadRepresentation{String,Vlen{String}}, f::JLDFile, ptr::Ptr, ::RelOffset) = - String(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) +jlconvert(::MappedRepr{String,Vlen{String}}, f::JLDFile, ptr::Ptr, ::RelOffset) = + String(jlconvert(MappedRepr{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) function jlconvert(rr::FixedLengthString{String}, ::JLDFile, ptr::Ptr, ::RelOffset) data = Vector{UInt8}(undef, rr.length) unsafe_copyto!(pointer(data), pconvert(Ptr{UInt8}, ptr), rr.length) @@ -126,19 +126,19 @@ function jlconvert(rr::AsciiString{NullTerminated}, ::JLDFile, ptr::Ptr, ::RelOf unsafe_copyto!(pointer(data), pconvert(Ptr{UInt8}, ptr), rr.length) String(data[1:end-1]) end -function jlconvert(rr::ReadRepresentation{String, FixedLengthAsciiString{NullTerminated,N}}, ::JLDFile, ptr::Ptr, ::RelOffset) where {N} +function jlconvert(::MappedRepr{String, FixedLengthAsciiString{NullTerminated,N}}, ::JLDFile, ptr::Ptr, ::RelOffset) where {N} data = Vector{UInt8}(undef, N) unsafe_copyto!(pointer(data), pconvert(Ptr{UInt8}, ptr), N) String(data) end -function jlconvert(rr::ReadRepresentation{String, FixedLengthAsciiString{SpacePadded,N}}, ::JLDFile, ptr::Ptr, ::RelOffset) where {N} +function jlconvert(::MappedRepr{String, FixedLengthAsciiString{SpacePadded,N}}, ::JLDFile, ptr::Ptr, ::RelOffset) where {N} data = Vector{UInt8}(undef, N) unsafe_copyto!(pointer(data), pconvert(Ptr{UInt8}, ptr), N) rstrip(String(data)) end odr_sizeof(x::AsciiString) = Int(x.length) -odr_sizeof(x::Type{FixedLengthAsciiString{TERM, N}}) where {TERM, N} = Int(N)::Int +odr_sizeof(::Type{FixedLengthAsciiString{TERM, N}}) where {TERM, N} = Int(N)::Int @@ -147,7 +147,7 @@ odr_sizeof(x::Type{FixedLengthAsciiString{TERM, N}}) where {TERM, N} = Int(N)::I # Used only for custom serialization constructrr(::JLDFile, ::Type{String}, dt::VariableLengthDatatype{FixedPointDatatype}, ::Vector{ReadAttribute}) = dt == H5TYPE_VLEN_UTF8 ? - (ReadRepresentation{String,Vlen{String}}(), true) : + (MappedRepr{String,Vlen{String}}(), true) : throw(UnsupportedFeatureException()) ## Symbols @@ -167,11 +167,11 @@ function h5convert!(out::Pointers, ::Type{Vlen{String}}, f::JLDFile, x::Symbol, end constructrr(::JLDFile, ::Type{Symbol}, dt::VariableLengthDatatype{FixedPointDatatype}, ::Vector{ReadAttribute}) = - dt == H5TYPE_VLEN_UTF8 ? (ReadRepresentation{Symbol,Vlen{String}}(), true) : + dt == H5TYPE_VLEN_UTF8 ? (MappedRepr{Symbol,Vlen{String}}(), true) : throw(UnsupportedFeatureException()) -jlconvert(::ReadRepresentation{Symbol,Vlen{String}}, f::JLDFile, ptr::Ptr, ::RelOffset) = - Symbol(jlconvert(ReadRepresentation{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) +jlconvert(::MappedRepr{Symbol,Vlen{String}}, f::JLDFile, ptr::Ptr, ::RelOffset) = + Symbol(jlconvert(MappedRepr{UInt8,Vlen{UInt8}}(), f, ptr, NULL_REFERENCE)) ## BigInts and BigFloats @@ -183,9 +183,9 @@ rconvert(::Type{BigFloat}, x::String) = parse(BigFloat, x) # BigInts and BigFloats are defined as mutable structs but should be reconstructed like an # immutable one. This overrides the default behavior. -function jlconvert(::ReadRepresentation{BN,CustomSerialization{String,Vlen{String}}}, +function jlconvert(::MappedRepr{BN,CustomSerialization{String,Vlen{String}}}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where BN <: Union{BigInt, BigFloat} - rconvert(BN, jlconvert(ReadRepresentation{String, Vlen{String}}(), f, ptr, header_offset)) + rconvert(BN, jlconvert(MappedRepr{String, Vlen{String}}(), f, ptr, header_offset)) end ## Pointers @@ -244,7 +244,7 @@ _odr(writtenas::Type{T}, readas::DataType, odr) where {T<:Array} = function constructrr(::JLDFile, ::Type{T}, dt::BasicDatatype, attrs::Vector{ReadAttribute}) where T<:Array dt.class == DT_REFERENCE || throw(UnsupportedFeatureException()) - (ReadRepresentation{Array, RelOffset}(), true) + (MappedRepr{Array, RelOffset}(), true) end ## SimpleVectors @@ -286,8 +286,8 @@ end # convert to an array instead. const NTUPLE_INLINE_THRESHOLD = 10 -function writeas(NT::Type{NTuple{N,T}}) where {N,T} - if N > NTUPLE_INLINE_THRESHOLD +function writeas(NT::Type{Tuple{T, Vararg{T,N}}}) where {N,T} + if (N+1) > NTUPLE_INLINE_THRESHOLD return Vector{T} else return NT @@ -317,7 +317,7 @@ function rconvert(::Type{Module}, x::String) end end -function jlconvert(::ReadRepresentation{Module,CustomSerialization{String,Vlen{String}}}, +function jlconvert(::MappedRepr{Module,CustomSerialization{String,Vlen{String}}}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) -rconvert(Module, jlconvert(ReadRepresentation{String, Vlen{String}}(), f, ptr, header_offset)) +rconvert(Module, jlconvert(MappedRepr{String, Vlen{String}}(), f, ptr, header_offset)) end diff --git a/src/data/writing_datatypes.jl b/src/data/writing_datatypes.jl index 1f60b49d..cf80bfb4 100644 --- a/src/data/writing_datatypes.jl +++ b/src/data/writing_datatypes.jl @@ -44,7 +44,7 @@ end end # Gets the size of an on-disk representation -function odr_sizeof(::OnDiskRepresentation{Offsets,JLTypes,H5Types,Size}) where {Offsets,JLTypes,H5Types,Size} +function odr_sizeof(::Type{OnDiskRepresentation{Offsets,JLTypes,H5Types,Size}}) where {Offsets,JLTypes,H5Types,Size} Size::Int end @@ -241,10 +241,10 @@ end wrtypeattr = WrittenAttribute(:written_type, WriteDataspace(f, DataType, odr(DataType)), h5type(f, DataType, DataType), writeas) - f.h5jltype[cdt] = ReadRepresentation{readas,CustomSerialization{writeas, odr(writeas)}}() + f.h5jltype[cdt] = MappedRepr{readas,CustomSerialization{writeas, odr(writeas)}}() commit(f, dtype, tuple(typeattr, wrtypeattr, attributes...)) else - f.h5jltype[cdt] = ReadRepresentation{writeas,odr(writeas)}() + f.h5jltype[cdt] = ReadRepresentation(writeas,odr(writeas)) commit(f, dtype, tuple(typeattr, attributes...)) end @@ -261,7 +261,7 @@ h5convert!(out::Pointers, ::Type{T}, ::JLDFile, x, ::JLDWriteSession) where {T} # We pack types that have padding using a staged h5convert! method @generated function h5convert!(out::Pointers, - ::OnDiskRepresentation{Offsets,Types,H5Types,Size}, + ::Type{OnDiskRepresentation{Offsets,Types,H5Types,Size}}, file::JLDFile, x, wsession::JLDWriteSession) where {Offsets,Types,H5Types,Size} T = x types = Types.parameters @@ -299,8 +299,7 @@ jlconvert_canbeuninitialized(::Any) = false # jlconvert converts data from a pointer into a Julia object. This method # handles types where this is just a simple load -jlconvert(::ReadRepresentation{T,T}, ::JLDFile, ptr::Ptr, - ::RelOffset) where {T} = +jlconvert(::SameRepr{T}, ::JLDFile, ptr::Ptr, ::RelOffset) where {T} = jlunsafe_load(pconvert(Ptr{T}, ptr)) # When fields are undefined in the file but can't be in the workspace, we need @@ -329,20 +328,19 @@ h5convert_uninitialized!(out::Pointers, odr::Type{RelOffset}) = (jlunsafe_store!(pconvert(Ptr{RelOffset}, out), NULL_REFERENCE); nothing) # Reading references as references -jlconvert(::ReadRepresentation{RelOffset,RelOffset}, f::JLDFile, ptr::Ptr, - ::RelOffset) = +jlconvert(::SameRepr{RelOffset}, f::JLDFile, ptr::Ptr, ::RelOffset) = jlunsafe_load(pconvert(Ptr{RelOffset}, ptr)) -jlconvert_canbeuninitialized(::ReadRepresentation{RelOffset,RelOffset}) = false +jlconvert_canbeuninitialized(::SameRepr{RelOffset}) = false # Reading references as other types -function jlconvert(::ReadRepresentation{T,RelOffset}, f::JLDFile, ptr::Ptr, +function jlconvert(::MappedRepr{T,RelOffset}, f::JLDFile, ptr::Ptr, ::RelOffset) where T x = load_dataset(f, jlunsafe_load(pconvert(Ptr{RelOffset}, ptr))) (isa(x, T) ? x : rconvert(T, x))::T end -jlconvert_canbeuninitialized(::ReadRepresentation{T,RelOffset}) where {T} = true -jlconvert_isinitialized(::ReadRepresentation{T,RelOffset}, ptr::Ptr) where {T} = +jlconvert_canbeuninitialized(::MappedRepr{T,RelOffset}) where {T} = true +jlconvert_isinitialized(::MappedRepr{T,RelOffset}, ptr::Ptr) where {T} = jlunsafe_load(pconvert(Ptr{RelOffset}, ptr)) != NULL_REFERENCE ## Routines for variable-length datatypes @@ -364,10 +362,10 @@ h5convert_uninitialized!(out::Pointers, odr::Type{T}) where {T<:Vlen} = (jlunsafe_store!(pconvert(Ptr{Int128}, out), 0); nothing) # Read variable-length data given offset and length in ptr -jlconvert(::ReadRepresentation{T,Vlen{S}}, f::JLDFile, ptr::Ptr, ::RelOffset) where {T,S} = - read_heap_object(f, jlunsafe_load(pconvert(Ptr{GlobalHeapID}, ptr+4)), ReadRepresentation{T, S}()) -jlconvert_canbeuninitialized(::ReadRepresentation{T,Vlen{S}}) where {T,S} = true -jlconvert_isinitialized(::ReadRepresentation{T,Vlen{S}}, ptr::Ptr) where {T,S} = +jlconvert(::MappedRepr{T,Vlen{S}}, f::JLDFile, ptr::Ptr, ::RelOffset) where {T,S} = + read_heap_object(f, jlunsafe_load(pconvert(Ptr{GlobalHeapID}, ptr+4)), ReadRepresentation(T, S)) +jlconvert_canbeuninitialized(::MappedRepr{<: Type, <: Vlen}) = true +jlconvert_isinitialized(::MappedRepr{T,Vlen{S}}, ptr::Ptr) where {T,S} = jlunsafe_load(pconvert(Ptr{GlobalHeapID}, ptr+4)) != GlobalHeapID(RelOffset(0), 0) @@ -400,7 +398,7 @@ function h5fieldtype(f::JLDFile, ::Type{T}, readas::Type, ::Initialized) where T cdt = CommittedDatatype(h5o, id) f.datatype_locations[h5o] = cdt f.jlh5type[DataType] = cdt - f.h5jltype[cdt] = ReadRepresentation{DataType,DataTypeODR()}() + f.h5jltype[cdt] = MappedRepr{DataType,DataTypeODR}() push!(f.datatypes, H5TYPE_DATATYPE) f.types_group[string(id, pad=8)] = h5o @@ -409,11 +407,11 @@ function h5fieldtype(f::JLDFile, ::Type{T}, readas::Type, ::Initialized) where T cdt end -fieldodr(::Type{T}, ::Bool) where {T<:DataType} = DataTypeODR() +fieldodr(::Type{T}, ::Bool) where {T<:DataType} = DataTypeODR h5type(f::JLDFile, ::Type{T}, x) where {T<:DataType} = h5fieldtype(f, DataType, typeof(x), Val{true}) -odr(::Type{T}) where {T<:DataType} = DataTypeODR() +odr(::Type{T}) where {T<:DataType} = DataTypeODR function typename(T::DataType) s = IOBuffer() @@ -443,7 +441,7 @@ function refs_from_types(f::JLDFile, types, wsession::JLDWriteSession) for x in types] end -function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsession::JLDWriteSession) +function h5convert!(out::Pointers, ::Type{DataTypeODR}, f::JLDFile, T::DataType, wsession::JLDWriteSession) t = typename(T) if T <: Function && isgensym(Symbol(T)) @warn LazyString("Attempting to store ", T, ".\n", @@ -463,7 +461,7 @@ end # This is a trick to compactly write long NTuple # This uses that NTuple{N,T} === Tuple{T,T,T,T,...,T} -function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{<: NTuple}, wsession::JLDWriteSession) +function h5convert!(out::Pointers, ::Type{DataTypeODR}, f::JLDFile, T::Type{<: NTuple}, wsession::JLDWriteSession) params = T.parameters N = length(params) if N ≤ 1 || !(reduce(==, params)) @@ -509,7 +507,7 @@ function h5fieldtype(f::JLDFile, ::Type{T}, readas::Type, ::Initialized) where T commit(f, H5TYPE_UNION, Union, readas) end -fieldodr(::Type{T}, ::Bool) where {T<:Union} = UnionTypeODR() +fieldodr(::Type{T}, ::Bool) where {T<:Union} = UnionTypeODR h5fieldtype(f::JLDFile, ::Type{Union{}}, ::Initialized) = nothing fieldodr(::Type{Union{}}, initialized::Bool) = nothing @@ -517,7 +515,7 @@ h5type(f::JLDFile, ::Type{T}, x::Any) where {T<:Union} = h5fieldtype(f, Union, typeof(x), Val{true}) odr(::Type{Union}) = fieldodr(Union, true) -function h5convert!(out::Pointers, ::UnionTypeODR, f::JLDFile, x::Union, wsession::JLDWriteSession) +function h5convert!(out::Pointers, ::Type{UnionTypeODR}, f::JLDFile, x::Union, wsession::JLDWriteSession) # Write a description of the union type # This is not needed for loading the file but makes h5dump output clearer t = string(x) @@ -539,7 +537,7 @@ function h5convert!(out::Pointers, ::UnionTypeODR, f::JLDFile, x::Union, wsessio end end -function jlconvert(::ReadRepresentation{Union, UnionTypeODR()}, f::JLDFile, +function jlconvert(::MappedRepr{Union, UnionTypeODR}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) # Skip union type description in the beginning ptr += odr_sizeof(Vlen{String}) @@ -555,7 +553,7 @@ end function constructrr(::JLDFile, ::Type{T}, dt::CompoundDatatype, ::Vector{ReadAttribute}) where {T<:Union} - dt == H5TYPE_UNION ? (ReadRepresentation{Union,UnionTypeODR()}(), true) : + dt == H5TYPE_UNION ? (MappedRepr{Union,UnionTypeODR}(), true) : throw(UnsupportedFeatureException()) end @@ -567,7 +565,7 @@ const UnionAllODR = OnDiskRepresentation{(0, 8),Tuple{TypeVar,Any},Tuple{RelOffs # generic h5convert! method for the specific UnionAll type rather than for UnionAll # more generally. function h5convert!(out::Pointers, - odr::UnionAllODR, + odr::Type{UnionAllODR}, f::JLDFile, x::UnionAll, wsession::JLDWriteSession) h5convert!(out, RelOffset, f, x.var, f.datatype_wsession) h5convert!(out+odr_sizeof(RelOffset), RelOffset, f, x.body, f.datatype_wsession) @@ -599,9 +597,9 @@ end # jlconvert for empty objects -function jlconvert(@nospecialize(rr::ReadRepresentation{T,nothing} where T), f::JLDFile, ptr::Ptr, - header_offset::RelOffset)::eltype(rr) - T = eltype(rr) +function jlconvert(@nospecialize(rr::MappedRepr{T,nothing} where T), f::JLDFile, ptr::Ptr, + header_offset::RelOffset)::julia_repr(rr) + T = julia_repr(rr) sizeof(T) == 0 && return newstruct(T)::T # In this case, T is a non-empty object, but the written data was empty @@ -613,10 +611,10 @@ function jlconvert(@nospecialize(rr::ReadRepresentation{T,nothing} where T), f:: if writtenas === ty # This will usually equal `ty()` unless ty does not have a # constructor without arguments - jlconvert(ReadRepresentation{ty,nothing}(), f, ptr, header_offset) + jlconvert(MappedRepr{ty,nothing}(), f, ptr, header_offset) else rconvert(ty, - jlconvert(ReadRepresentation{writtenas,nothing}(), f, ptr, header_offset) + jlconvert(MappedRepr{writtenas,nothing}(), f, ptr, header_offset) ) end end @@ -656,7 +654,7 @@ end offset += odr_sizeof(fodr) end - OnDiskRepresentation{(offsets...,), Tuple{T.types...}, Tuple{odrs...}, Int(offset)}() + OnDiskRepresentation{(offsets...,), Tuple{T.types...}, Tuple{odrs...}, Int(offset)} end abstract type DataMode end @@ -669,7 +667,7 @@ datamode(::DataType) = ReferenceFree() datamode(::FixedLengthString) = ReferenceFree() datamode(::AsciiString) = ReferenceFree() datamode(::Nothing) = ReferenceFree() -function datamode(::OnDiskRepresentation{Offsets,JLTypes,H5Types,Size} where {Offsets,JLTypes,Size}) where H5Types +function datamode(::Type{OnDiskRepresentation{Offsets,JLTypes,H5Types,Size}}) where {Offsets,JLTypes,H5Types,Size} for ty in H5Types.parameters datamode(ty) == HasReferences() && return HasReferences() end diff --git a/src/datasets.jl b/src/datasets.jl index c8186e5a..30d7cdf6 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -68,9 +68,9 @@ Otherwise, `datatype_offset` points to the offset of the datatype attribute. if layout.data_offset == -1 # There was no layout message. # That means, this dataset is just a datatype - return typeof(rr).parameters[1] + return julia_repr(rr) elseif layout.data_offset == typemax(Int64) - T,_ = typeof(rr).parameters + T = julia_repr(rr) if layout.data_length > -1 # TODO: this could use the fill value message to populate the array @warn "This array should be populated by a fill value. This is not (yet) implemented." @@ -109,7 +109,7 @@ end # Reference arrays can only be arrays or null dataspace (for Union{} case) function read_data(f::JLDFile, - rr::ReadRepresentation{Any,RelOffset}, + ::MappedRepr{Any,RelOffset}, read_dataspace::Tuple{ReadDataspace,RelOffset,DataLayout,FilterPipeline}, attributes::Vector{ReadAttribute}) @@ -126,11 +126,11 @@ function read_data(f::JLDFile, if isunknowntype(T) str = typestring(T) @warn "type $(str) does not exist in workspace; interpreting Array{$str} as Array{Any}" maxlog=1 - rr = ReadRepresentation{Any,RelOffset}() + rr = MappedRepr{Any,RelOffset}() elseif T isa Upgrade - rr = ReadRepresentation{T.target, RelOffset}() + rr = MappedRepr{T.target, RelOffset}() else - rr = ReadRepresentation{T, RelOffset}() + rr = ReadRepresentation(T, RelOffset) end seek(io, startpos) return read_array(f, dataspace, rr, @@ -138,9 +138,9 @@ function read_data(f::JLDFile, end end elseif dataspace.dataspace_type == DS_NULL - return read_empty(f, ReadRepresentation{Union{},nothing}(), dataspace, attributes, header_offset) + return read_empty(f, MappedRepr{Union{},nothing}(), dataspace, attributes, header_offset) elseif dataspace.dataspace_type == DS_V1 - return read_array(f, dataspace, ReadRepresentation{Any,RelOffset}(), + return read_array(f, dataspace, MappedRepr{Any,RelOffset}(), layout, FilterPipeline(), header_offset, attributes) end throw(UnsupportedFeatureException("Dataspace type $(dataspace.dataspace_type) not implemented")) @@ -223,7 +223,7 @@ end @nospecialize(rr::ReadRepresentation), layout::DataLayout, filters::FilterPipeline, header_offset::RelOffset, attributes::Union{Vector{ReadAttribute},Nothing}) - T = eltype(rr) + T = julia_repr(rr) io = f.io data_offset = layout.data_offset if !ischunked(layout) || (layout.chunk_indexing_type == 1) diff --git a/src/datatypes.jl b/src/datatypes.jl index ef163df5..92d35c02 100644 --- a/src/datatypes.jl +++ b/src/datatypes.jl @@ -154,8 +154,8 @@ end function jltype(f::JLDFile, dt::CompoundDatatype) field_datatypes = OrderedDict{String, RelOffset}(string.(dt.names) .=> fill(NULL_REFERENCE, length(dt.names))) odr = reconstruct_odr(f, dt, field_datatypes) - T = NamedTuple{tuple(dt.names...), typeof(odr).parameters[2]} - return ReadRepresentation{T, odr}() + T = NamedTuple{tuple(dt.names...), odr.parameters[2]} + return MappedRepr{T, odr}() end Base.:(==)(x::CompoundDatatype, y::CompoundDatatype) = @@ -359,17 +359,17 @@ odr_sizeof(::Type{ArrayPlaceHolder{T,D}}) where {T,D} = Int(odr_sizeof(T)*prod(D function jltype(f::JLDFile, dt::ArrayDatatype) rr = jltype(f, dt.base_type) - T = typeof(rr).parameters[1] - ReadRepresentation{Array{T, Int(dt.dimensionality)}, ArrayPlaceHolder{rr, tuple(dt.dims...)}}() + T = julia_repr(rr) + MappedRepr{Array{T, Int(dt.dimensionality)}, ArrayPlaceHolder{rr, tuple(dt.dims...)}}() end -function jlconvert(::ReadRepresentation{Array{T,D}, ArrayPlaceHolder{RR, DIMS}}, f::JLDFile, ptr::Ptr, +function jlconvert(::MappedRepr{Array{T,D}, ArrayPlaceHolder{RR, DIMS}}, f::JLDFile, ptr::Ptr, header_offset::RelOffset) where {T, D, RR, DIMS} v = Array{T, D}(undef, reverse(DIMS)...) for i=1:prod(DIMS) v[i] = jlconvert(RR, f, ptr, header_offset) - ptr += jlsizeof(typeof(RR).parameters[2]) + ptr += jlsizeof(file_repr(RR)) end return v end @@ -404,7 +404,7 @@ function jlread(io::IO, ::Type{EnumerationDatatype}) end function jltype(f::JLDFile, dt::EnumerationDatatype) - ReadRepresentation{dt.base_type, dt.base_type}() + SameRepr{dt.base_type}() end # Can't read big endian ints diff --git a/src/explicit_datasets.jl b/src/explicit_datasets.jl index d61a2704..02569aac 100644 --- a/src/explicit_datasets.jl +++ b/src/explicit_datasets.jl @@ -62,7 +62,7 @@ function Base.show(io::IO, ::MIME"text/plain", dset::Dataset) print(io, prefix*"datatype: $(typeof(dt))", iscommitted ? " (committed)\n" : "\n") iscommitted && println(io, prefix*"\tcommitted at: $(dt.header_offset)") rr = jltype(dset.parent.f, dt) - jt = typeof(rr).parameters[1] + jt = julia_repr(rr) println(io, prefix*"\twritten structure: $jt") if iscommitted juliatype, writtentype, fields = stringify_committed_datatype(f, f.datatype_locations[dt.header_offset], showfields=true) @@ -393,7 +393,7 @@ function ismmappable(dset::Dataset) f = dset.parent.f dt = dset.datatype rr = jltype(f, dt) - T = typeof(rr).parameters[1] + T = julia_repr(rr) !(samelayout(T)) && return false !isempty(dset.filters.filters) && return false ret = false @@ -421,7 +421,7 @@ function readmmap(dset::Dataset) # figure out the element type dt = dset.datatype rr = jltype(f, dt) - T = typeof(rr).parameters[1] + T = julia_repr(rr) ndims, offset = get_ndims_offset(f, ReadDataspace(f, dset.dataspace), collect(values(dset.attributes))) io = f.io @@ -512,12 +512,12 @@ function allocate_early(dset::Dataset, T::DataType) end end -struct ArrayDataset{T, N, ODR, io} <: AbstractArray{T, N} +struct ArrayDataset{T, N, RR<:ReadRepresentation{T}, io} <: AbstractArray{T, N} f::JLDFile{io} dset::Dataset dims::NTuple{N, Int} data_address::Int64 - rr::ReadRepresentation{T, ODR} + rr::RR writable::Bool end function ArrayDataset(dset::Dataset) @@ -526,11 +526,12 @@ function ArrayDataset(dset::Dataset) f = dset.parent.f dt = dset.datatype writable = f.writable && (dset.layout.layout_class == LcContiguous) + rr = jltype(f, !(f.plain) && dt isa SharedDatatype ? get(f.datatype_locations, dt.header_offset, dt) : dt) return ArrayDataset( f, dset, Int.(reverse(dset.dataspace.dimensions)), fileoffset(f, dset.layout.data_address), - jltype(f, !(f.plain) && dt isa SharedDatatype ? get(f.datatype_locations, dt.header_offset, dt) : dt), + rr, writable ) end diff --git a/src/global_heaps.jl b/src/global_heaps.jl index 7f365ad7..2faf39c7 100644 --- a/src/global_heaps.jl +++ b/src/global_heaps.jl @@ -125,7 +125,7 @@ function jlread(io::IO, ::Type{GlobalHeap}) GlobalHeap(offset, heapsz, free, objects) end -function read_heap_object(f::JLDFile, hid::GlobalHeapID, rr::ReadRepresentation{T,RR}) where {T,RR} +function read_heap_object(f::JLDFile, hid::GlobalHeapID, rr::ReadRepresentation) io = f.io if haskey(f.global_heaps, hid.heap_offset) gh = f.global_heaps[hid.heap_offset] @@ -135,8 +135,8 @@ function read_heap_object(f::JLDFile, hid::GlobalHeapID, rr::ReadRepresentation{ end seek(io, gh.objects[hid.index]+8) len = Int(jlread(io, Length)) - n = div(len, odr_sizeof(RR)) - len == n * odr_sizeof(RR) || throw(InvalidDataException()) + n = div(len, odr_sizeof(rr)) + len == n * odr_sizeof(rr) || throw(InvalidDataException()) - read_array!(Vector{T}(undef, n), f, rr) + read_array!(Vector{julia_repr(rr)}(undef, n), f, rr) end diff --git a/src/inlineunion.jl b/src/inlineunion.jl index e84debdb..cba8aa84 100644 --- a/src/inlineunion.jl +++ b/src/inlineunion.jl @@ -52,7 +52,7 @@ end # except for the ReadRepresentation and the very last line where the data is # converted back into a Union Array function read_array(f::JLDFile, dataspace::ReadDataspace, - rr::ReadRepresentation{InlineUnionEl{T1,T2},RR}, layout::DataLayout, + rr::MappedRepr{InlineUnionEl{T1,T2},RR}, layout::DataLayout, filters::FilterPipeline, header_offset::RelOffset, attributes::Union{Vector{ReadAttribute},Nothing}) where {T1, T2,RR} io = f.io diff --git a/src/io/dataio.jl b/src/io/dataio.jl index 58b6556b..3799d4ab 100644 --- a/src/io/dataio.jl +++ b/src/io/dataio.jl @@ -109,7 +109,7 @@ const MMAP_CUTOFF = 1048576 obj end -function read_array!(v::Array{T}, f::JLDFile{<:MemoryBackedIO}, ::ReadRepresentation{T,T}) where T +function read_array!(v::Array{T}, f::JLDFile{<:MemoryBackedIO}, ::SameRepr{T}) where T inptr = f.io.curptr n = length(v) unsafe_copyto!(pointer(v), pconvert(Ptr{T}, inptr), n) @@ -117,20 +117,19 @@ function read_array!(v::Array{T}, f::JLDFile{<:MemoryBackedIO}, ::ReadRepresenta v end -function read_array!(v::Array{T}, f::JLDFile{<:MemoryBackedIO}, rr::ReadRepresentation{T,RR}) where {T,RR} +function read_array!(v::Array{T}, f::JLDFile{<:MemoryBackedIO}, rr::ReadRepresentation{T}) where T cp0 = f.io.curptr @simd for i in eachindex(v) - cp = cp0 + (i-1)*odr_sizeof(RR) + cp = cp0 + (i-1)*odr_sizeof(rr) if !jlconvert_canbeuninitialized(rr) || jlconvert_isinitialized(rr, cp) v[i] = jlconvert(rr, f, cp, NULL_REFERENCE) end end - f.io.curptr = cp0 + odr_sizeof(RR) * length(v) + f.io.curptr = cp0 + odr_sizeof(rr) * length(v) v end -function write_data(io::MemoryBackedIO, f::JLDFile, data, odr::S, ::ReferenceFree, - wsession::JLDWriteSession) where S +function write_data(io::MemoryBackedIO, f::JLDFile, data, odr, ::ReferenceFree, wsession::JLDWriteSession) ensureroom(io, odr_sizeof(odr)) cp = io.curptr h5convert!(cp, odr, f, data, wsession) @@ -139,8 +138,7 @@ function write_data(io::MemoryBackedIO, f::JLDFile, data, odr::S, ::ReferenceFre nothing end -function write_data(io::MemoryBackedIO, f::JLDFile, data, odr::S, ::HasReferences, - wsession::JLDWriteSession) where S +function write_data(io::MemoryBackedIO, f::JLDFile, data, odr, ::HasReferences, wsession::JLDWriteSession) ensureroom(io, odr_sizeof(odr)) cp = IndirectPointer(io) h5convert!(cp, odr, f, data, wsession) @@ -148,8 +146,7 @@ function write_data(io::MemoryBackedIO, f::JLDFile, data, odr::S, ::HasReference nothing end -function write_data(io::MemoryBackedIO, f::JLDFile, data::Array{T}, odr::S, ::ReferenceFree, - wsession::JLDWriteSession) where {T,S} +function write_data(io::MemoryBackedIO, f::JLDFile, data::Array, odr, ::ReferenceFree, wsession::JLDWriteSession) ensureroom(io, odr_sizeof(odr) * length(data)) cp0 = io.curptr @simd for i = 1:length(data) @@ -161,8 +158,7 @@ function write_data(io::MemoryBackedIO, f::JLDFile, data::Array{T}, odr::S, ::Re nothing end -function write_data(io::MemoryBackedIO, f::JLDFile, data::Array{T}, odr::S, ::HasReferences, - wsession::JLDWriteSession) where {T,S} +function write_data(io::MemoryBackedIO, f::JLDFile, data::Array, odr, ::HasReferences, wsession::JLDWriteSession) ensureroom(io, odr_sizeof(odr) * length(data)) cp = IndirectPointer(io) @@ -195,19 +191,19 @@ function read_scalar(f::JLDFile, rr, header_offset::RelOffset) end -function read_array!(v::Array{T}, f::JLDFile, rr::ReadRepresentation{T,T}) where {T} +function read_array!(v::Array{T}, f::JLDFile, ::SameRepr{T}) where {T} unsafe_read(f.io, pointer(v), odr_sizeof(T)*length(v)) v end -function read_array!(v::Array{T}, f::JLDFile, rr::ReadRepresentation{T,RR}) where {T,RR} +function read_array!(v::Array{T}, f::JLDFile, rr::ReadRepresentation{T}) where {T} n = length(v) - nb = odr_sizeof(RR)*n + nb = odr_sizeof(rr)*n data = read!(f.io, Vector{UInt8}(undef, nb)) @GC.preserve data begin p0 = Ptr{Cvoid}(pointer(data)) @simd for i = eachindex(v) - dataptr = p0 + odr_sizeof(RR)*(i-1) + dataptr = p0 + odr_sizeof(rr)*(i-1) if !jlconvert_canbeuninitialized(rr) || jlconvert_isinitialized(rr, dataptr) v[i] = jlconvert(rr, f, dataptr, NULL_REFERENCE) end @@ -233,8 +229,7 @@ function write_data(io::IO, f::JLDFile, data, odr, _, wsession::JLDWriteSession) nothing end -function write_data(io::IO, f::JLDFile, data::Array{T}, odr::S, wm::DataMode, - wsession::JLDWriteSession) where {T,S} +function write_data(io::IO, f::JLDFile, data::Array, odr, wm::DataMode, wsession::JLDWriteSession) nb = odr_sizeof(odr) * length(data) buf = Vector{UInt8}(undef, nb) pos = position(io) @@ -260,7 +255,7 @@ end read_bytestring(io::Union{IOStream, IOBuffer}) = String(readuntil(io, 0x00)) # Late addition for MmapIO that can't be defined in mmapio.jl due to include ordering -function read_array!(v::Array{T}, f::JLDFile{MmapIO}, ::ReadRepresentation{T,T}) where T +function read_array!(v::Array{T}, f::JLDFile{MmapIO}, ::SameRepr{T}) where T io = f.io inptr = io.curptr n = length(v) diff --git a/src/io/io_wrappers.jl b/src/io/io_wrappers.jl index 50dbc8b1..75e0b589 100644 --- a/src/io/io_wrappers.jl +++ b/src/io/io_wrappers.jl @@ -142,7 +142,7 @@ function _read(io::RWBuffer, T::DataType) end # needed explicitly for avoiding ambiguities Base.read(io::RWBuffer, T::Type{UInt8}) = _read(io, T) -Base.read(io::RWBuffer, T::PlainType) = _read(io, T) +#Base.read(io::RWBuffer, T::PlainType) = _read(io, T) Base.write(io::RWBuffer, x::UInt8) = _write(io, x) jlwrite(io::RWBuffer, x::String) = _write(io, x) diff --git a/src/misc.jl b/src/misc.jl index db4a3142..bafd11f2 100644 --- a/src/misc.jl +++ b/src/misc.jl @@ -198,7 +198,7 @@ function flag2uint(flag::UInt8) end -jlread(io::IO, ::Type{NTuple{N,T}}) where {N,T} = ntuple(_->jlread(io, T), Val{N}()) +jlread(io::IO, ::Type{Tuple{T, Vararg{T,N}}}) where {N,T} = ntuple(_->jlread(io, T), Val{N+1}()) jlread(io::IO, ::Type{Tuple{}}) = () function write_nb_int(io::IO, sz::Integer, nb::Integer) diff --git a/src/object_headers.jl b/src/object_headers.jl index e4d07e2b..c6620676 100644 --- a/src/object_headers.jl +++ b/src/object_headers.jl @@ -22,7 +22,7 @@ struct Hmessage{IO} m::Message{IO} end -function write_header_message(io, vtype::Val{HMT}, hflags=0x00, size=0; kwargs...) where HMT +function write_header_message(io, vtype::Val{HMT}, hflags::Integer=0x00, size=0; kwargs...) where HMT kw = (; kwargs...) size = compute_size(vtype, hflags, size, kw) jlwrite(io, UInt8(HMT)) diff --git a/src/types.jl b/src/types.jl index 01bf4f02..b69e6773 100644 --- a/src/types.jl +++ b/src/types.jl @@ -242,8 +242,15 @@ SharedDatatype(dt::CommittedDatatype) = SharedDatatype(dt.header_offset) A type encoding both the Julia type `T` and the on-disk (HDF5) representation `ODR`. """ -struct ReadRepresentation{T,ODR} end -Base.eltype(::Type{<:ReadRepresentation{T}}) where T = T +abstract type ReadRepresentation{T, ODR} end +julia_repr(::ReadRepresentation{T}) where T = T +file_repr(::ReadRepresentation{T,ODR}) where {T,ODR} = ODR + +struct SameRepr{T} <: ReadRepresentation{T,T} end +struct MappedRepr{T,ODR} <: ReadRepresentation{T,ODR} end + +ReadRepresentation(::Type{T}, ::Type{T}) where T = SameRepr{T}() +ReadRepresentation(T, ODR) = MappedRepr{T,ODR}() """ CustomSerialization{T,S} diff --git a/test/Project.toml b/test/Project.toml index 7ea9f235..e3e523a0 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,4 +1,5 @@ [deps] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" @@ -10,4 +11,8 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" + +[compat] +Aqua = "0.8" \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index c5424344..89a11958 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -34,3 +34,16 @@ include("unpack_test.jl") include("dataset_api.jl") include("mmap_test.jl") include("wrapped_io.jl") + +using TestItemRunner + +@run_package_tests + +@testitem "Aqua Testing" begin + using Aqua + Aqua.test_all(JLD2; + deps_compat = (; + ignore = [:Mmap,], + ), + ) +end \ No newline at end of file