diff --git a/Cargo.toml b/Cargo.toml index 3eca038e054..42dad19ed8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.23.1" +version = "0.24.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" diff --git a/Contributing.md b/Contributing.md index 76af08325fb..2615028a1ba 100644 --- a/Contributing.md +++ b/Contributing.md @@ -109,7 +109,24 @@ You can run these checks yourself with `nox`. Use `nox -l` to list the full set `nox -s clippy-all` #### Tests -`nox -s test` or `cargo test` for Rust tests only, `nox -f pytests/noxfile.py -s test` for Python tests only +`nox -s test` or `cargo test` or `cargo nextest` for Rust tests only, `nox -f pytests/noxfile.py -s test` for +Python tests only. + +Configuring the python interpreter version (`Py_*` cfg options) when running `cargo test` is the same as for regular +packages which is explained in [the docs](https://pyo3.rs/v0.22.5/building-and-distribution). +The easiest way to configure the python version is to install with the system package manager or +[pyenv](https://github.com/pyenv/pyenv) then set `PYO3_PYTHON`. +[uv python install](https://docs.astral.sh/uv/concepts/python-versions/) cannot currently be used as it sets some +[incorrect sysconfig values](https://github.com/astral-sh/uv/issues/8429). + +`Py_LIMITED_API` can be controlled with the `abi3` feature of the `pyo3` crate: + +``` +LD_LIBRARY_PATH=/lib PYO3_PYTHON=/bin/python \ + cargo nextest run --package pyo3 --features abi3 ... +``` + +use the `PYO3_PRINT_CONFIG=1` to check the identified configuration. #### Check all conditional compilation `nox -s check-feature-powerset` @@ -183,7 +200,11 @@ PyO3 supports all officially supported Python versions, as well as the latest Py PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible. -The minimum Rust version supported will be decided when the release which bumps Python and Rust versions is made. At the time, the minimum Rust version will be set no higher than the lowest Rust version shipped in the current Debian, RHEL and Alpine Linux distributions. +The minimum Rust version supported will be decided when the release which bumps Python and Rust versions is made. +At the time, the minimum Rust version will be set no higher than the lowest Rust version shipped in the current +[Debian](https://packages.debian.org/search?keywords=rustc), +[RHEL](https://docs.redhat.com/en/documentation/red_hat_developer_tools/1) and +[Alpine Linux](https://pkgs.alpinelinux.org/package/edge/main/x86/rust) distributions. CI tests both the most recent stable Rust version and the minimum supported Rust version. Because of Rust's stability guarantees this is sufficient to confirm support for all Rust versions in between. diff --git a/guide/pyclass-parameters.md b/guide/pyclass-parameters.md index 7ebca2ec821..cfd28fc144e 100644 --- a/guide/pyclass-parameters.md +++ b/guide/pyclass-parameters.md @@ -23,6 +23,7 @@ | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. | | `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a thread-safe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. | | `weakref` | Allows this class to be [weakly referenceable][params-6]. | +| `opaque` | Forces the use of the 'opaque layout' ([PEP 697](https://peps.python.org/pep-0697/)) for this class and any subclasses that extend it. Primarily used for internal testing. | All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or more accompanying `#[pyo3(...)]` annotations, e.g.: diff --git a/guide/src/class.md b/guide/src/class.md index 5d2c8435416..b4a9854d880 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -1378,12 +1378,20 @@ impl pyo3::types::DerefToPyAny for MyClass {} unsafe impl pyo3::type_object::PyTypeInfo for MyClass { const NAME: &'static str = "MyClass"; const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None; + const OPAQUE: bool = false; + #[inline] fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject { ::lazy_type_object() .get_or_init(py) .as_type_ptr() } + + #[inline] + fn try_get_type_object_raw() -> ::std::option::Option<*mut pyo3::ffi::PyTypeObject> { + ::lazy_type_object() + .try_get_raw() + } } impl pyo3::PyClass for MyClass { diff --git a/guide/src/class/metaclass.md b/guide/src/class/metaclass.md new file mode 100644 index 00000000000..b7843e77c36 --- /dev/null +++ b/guide/src/class/metaclass.md @@ -0,0 +1,72 @@ +# Creating a Metaclass +A [metaclass](https://docs.python.org/3/reference/datamodel.html#metaclasses) is a class that derives `type` and can +be used to influence the construction of other classes. + +Some examples of where metaclasses can be used: + +- [`ABCMeta`](https://docs.python.org/3/library/abc.html) for defining abstract classes +- [`EnumType`](https://docs.python.org/3/library/enum.html) for defining enums +- [`NamedTuple`](https://docs.python.org/3/library/typing.html#typing.NamedTuple) for defining tuples with elements + that can be accessed by name in addition to index. +- singleton classes +- automatic registration of classes +- ORM +- serialization / deserialization / validation (e.g. [pydantic](https://docs.pydantic.dev/latest/api/base_model/)) + +### Example: A Simple Metaclass + +Note: Creating metaclasses is only possible with python 3.12+ + +```rust +#[pyclass(subclass, extends=PyType)] +#[derive(Default)] +struct MyMetaclass { + counter: u64, +}; + +#[pymethods] +impl MyMetaclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + slf: Bound<'_, Metaclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + slf.borrow_mut().counter = 5; + } + + fn __getitem__(&self, item: u64) -> u64 { + item + 1 + } + + fn increment_counter(&mut self) { + self.counter += 1; + } + + fn get_counter(&self) -> u64 { + self.counter + } +} +``` + +Used like so: +```python +class Foo(metaclass=MyMetaclass): + def __init__() -> None: + ... + +assert type(Foo) is MyMetaclass +assert Foo.some_var == 123 +assert Foo[100] == 101 +Foo.increment_counter() +assert Foo.get_counter() == 1 +``` + +In the example above `MyMetaclass` extends `PyType` (making it a metaclass). It does not define `#[new]` as +[this is not supported](https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass). Instead `__init__` is +defined which is called whenever a class is created that uses `MyMetaclass` as its metaclass. +The arguments to `__init__` are the same as the arguments to `type(name, bases, kwds)`. A `Default` impl is required +in order to define `__init__`. The data in the struct is initialised to `Default` before `__init__` is called. + +When special methods like `__getitem__` are defined for a metaclass they apply to the classes they construct, so +`Foo[123]` calls `MyMetaclass.__getitem__(Foo, 123)`. diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 8a361a1442e..e929571f2a9 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -141,6 +141,11 @@ given signatures should be interpreted as follows: Determines the "truthyness" of an object. + - `__init__(, ...) -> ()` - the arguments can be defined as for + normal `pymethods`. The pyclass struct must implement `Default`. + If the class defines `__new__` and `__init__` the values set in + `__new__` are overridden by `Default` before `__init__` is called. + - `__call__(, ...) -> object` - here, any argument list can be defined as for normal `pymethods` diff --git a/newsfragments/4678.added.md b/newsfragments/4678.added.md new file mode 100644 index 00000000000..1d8f3293133 --- /dev/null +++ b/newsfragments/4678.added.md @@ -0,0 +1 @@ +Add support for opaque PyObjects allowing extending variable/unknown sized base classes (including `type` to create metaclasses) \ No newline at end of file diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index 6fe75e44302..7e7674634ca 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -20,16 +20,17 @@ pub mod kw { syn::custom_keyword!(eq_int); syn::custom_keyword!(extends); syn::custom_keyword!(freelist); + syn::custom_keyword!(from_item_all); syn::custom_keyword!(from_py_with); syn::custom_keyword!(frozen); syn::custom_keyword!(get); syn::custom_keyword!(get_all); syn::custom_keyword!(hash); syn::custom_keyword!(item); - syn::custom_keyword!(from_item_all); syn::custom_keyword!(mapping); syn::custom_keyword!(module); syn::custom_keyword!(name); + syn::custom_keyword!(opaque); syn::custom_keyword!(ord); syn::custom_keyword!(pass_module); syn::custom_keyword!(rename_all); diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 93596611f18..5eadcd8ac04 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -72,6 +72,7 @@ pub struct PyClassPyO3Options { pub module: Option, pub name: Option, pub ord: Option, + pub opaque: Option, pub rename_all: Option, pub sequence: Option, pub set_all: Option, @@ -95,6 +96,7 @@ pub enum PyClassPyO3Option { Module(ModuleAttribute), Name(NameAttribute), Ord(kw::ord), + Opaque(kw::opaque), RenameAll(RenameAllAttribute), Sequence(kw::sequence), SetAll(kw::set_all), @@ -133,6 +135,8 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Name) } else if lookahead.peek(attributes::kw::ord) { input.parse().map(PyClassPyO3Option::Ord) + } else if lookahead.peek(attributes::kw::opaque) { + input.parse().map(PyClassPyO3Option::Opaque) } else if lookahead.peek(kw::rename_all) { input.parse().map(PyClassPyO3Option::RenameAll) } else if lookahead.peek(attributes::kw::sequence) { @@ -205,6 +209,7 @@ impl PyClassPyO3Options { PyClassPyO3Option::Module(module) => set_option!(module), PyClassPyO3Option::Name(name) => set_option!(name), PyClassPyO3Option::Ord(ord) => set_option!(ord), + PyClassPyO3Option::Opaque(opaque) => set_option!(opaque), PyClassPyO3Option::RenameAll(rename_all) => set_option!(rename_all), PyClassPyO3Option::Sequence(sequence) => set_option!(sequence), PyClassPyO3Option::SetAll(set_all) => set_option!(set_all), @@ -1823,10 +1828,25 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre quote! { ::core::option::Option::None } }; + let opaque = if attr.options.opaque.is_some() { + quote! { + const OPAQUE: bool = true; + + #[cfg(not(Py_3_12))] + ::core::compile_error!("#[pyclass(opaque)] requires python 3.12 or later"); + } + } else { + // if opaque is not supported an error will be raised at construction + quote! { + const OPAQUE: bool = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE; + } + }; + quote! { unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls { const NAME: &'static str = #cls_name; const MODULE: ::std::option::Option<&'static str> = #module; + #opaque #[inline] fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject { @@ -1835,6 +1855,12 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre .get_or_init(py) .as_type_ptr() } + + #[inline] + fn try_get_type_object_raw() -> ::std::option::Option<*mut #pyo3_path::ffi::PyTypeObject> { + <#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object() + .try_get_raw() + } } } } @@ -2202,7 +2228,7 @@ impl<'a> PyClassImplsBuilder<'a> { let dict_offset = if self.attr.options.dict.is_some() { quote! { - fn dict_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + fn dict_offset() -> ::std::option::Option<#pyo3_path::impl_::pyclass::PyObjectOffset> { ::std::option::Option::Some(#pyo3_path::impl_::pyclass::dict_offset::()) } } @@ -2210,10 +2236,9 @@ impl<'a> PyClassImplsBuilder<'a> { TokenStream::new() }; - // insert space for weak ref let weaklist_offset = if self.attr.options.weakref.is_some() { quote! { - fn weaklist_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> { + fn weaklist_offset() -> ::std::option::Option<#pyo3_path::impl_::pyclass::PyObjectOffset> { ::std::option::Option::Some(#pyo3_path::impl_::pyclass::weaklist_offset::()) } } @@ -2298,8 +2323,9 @@ impl<'a> PyClassImplsBuilder<'a> { let pyclass_base_type_impl = attr.options.subclass.map(|subclass| { quote_spanned! { subclass.span() => impl #pyo3_path::impl_::pyclass::PyClassBaseType for #cls { - type LayoutAsBase = #pyo3_path::impl_::pycell::PyClassObject; + type StaticLayout = #pyo3_path::impl_::pycell::PyStaticClassLayout; type BaseNativeType = ::BaseNativeType; + type RecursiveOperations = #pyo3_path::impl_::pycell::PyClassRecursiveOperations; type Initializer = #pyo3_path::pyclass_init::PyClassInitializer; type PyClassMutability = ::PyClassMutability; } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 560c3c9dcc1..fda1b360e40 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -143,6 +143,7 @@ impl PyMethodKind { "__gt__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GT__)), "__ge__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GE__)), // Some tricky protocols which don't fit the pattern of the rest + "__init__" => PyMethodKind::Proto(PyMethodProtoKind::Init), "__call__" => PyMethodKind::Proto(PyMethodProtoKind::Call), "__traverse__" => PyMethodKind::Proto(PyMethodProtoKind::Traverse), "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Clear), @@ -154,6 +155,7 @@ impl PyMethodKind { enum PyMethodProtoKind { Slot(&'static SlotDef), + Init, Call, Traverse, Clear, @@ -212,6 +214,9 @@ pub fn gen_py_method( let slot = slot_def.generate_type_slot(cls, spec, &method.method_name, ctx)?; GeneratedPyMethod::Proto(slot) } + PyMethodProtoKind::Init => { + GeneratedPyMethod::Proto(impl_init_slot(cls, method.spec, ctx)?) + } PyMethodProtoKind::Call => { GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec, ctx)?) } @@ -303,8 +308,11 @@ fn ensure_no_forbidden_protocol_attributes( method_name: &str, ) -> syn::Result<()> { if let Some(signature) = &spec.signature.attribute { - // __call__ is allowed to have a signature, but nothing else is. - if !matches!(proto_kind, PyMethodProtoKind::Call) { + // __call__ and __init__ are allowed to have a signature, but nothing else is. + if !matches!( + proto_kind, + PyMethodProtoKind::Call | PyMethodProtoKind::Init + ) { bail_spanned!(signature.kw.span() => format!("`signature` cannot be used with magic method `{}`", method_name)); } } @@ -394,6 +402,41 @@ pub fn impl_py_method_def_new( }) } +fn impl_init_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { + let Ctx { pyo3_path, .. } = ctx; + + spec.convention = CallingConvention::Varargs; + + let wrapper_ident = syn::Ident::new("__pymethod___init____", Span::call_site()); + let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?; + let slot_def = quote! { + #pyo3_path::ffi::PyType_Slot { + slot: #pyo3_path::ffi::Py_tp_init, + pfunc: { + unsafe extern "C" fn trampoline( + slf: *mut #pyo3_path::ffi::PyObject, + args: *mut #pyo3_path::ffi::PyObject, + kwargs: *mut #pyo3_path::ffi::PyObject, + ) -> ::std::os::raw::c_int + { + #pyo3_path::impl_::trampoline::initproc( + slf, + args, + kwargs, + #pyo3_path::impl_::pyclass_init::initialize_with_default::<#cls>, + #cls::#wrapper_ident + ) + } + trampoline + } as #pyo3_path::ffi::initproc as _ + } + }; + Ok(MethodAndSlotDef { + associated_method, + slot_def, + }) +} + fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result { let Ctx { pyo3_path, .. } = ctx; @@ -831,8 +874,8 @@ pub fn impl_py_getter_def( struct Offset; unsafe impl #pyo3_path::impl_::pyclass::OffsetCalculator<#cls, #ty> for Offset { - fn offset() -> usize { - #pyo3_path::impl_::pyclass::class_offset::<#cls>() + + fn offset() -> #pyo3_path::impl_::pyclass::PyObjectOffset { + #pyo3_path::impl_::pyclass::subclass_offset::<#cls>() + #pyo3_path::impl_::pyclass::offset_of!(#cls, #field) } } diff --git a/src/exceptions.rs b/src/exceptions.rs index 6f0fa3e674c..d3766ad61f4 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -87,18 +87,13 @@ macro_rules! import_exception { $crate::pyobject_native_type_core!( $name, - $name::type_object_raw, #module=::std::option::Option::Some(stringify!($module)) ); - - impl $name { - fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::types::PyTypeMethods; - static TYPE_OBJECT: $crate::impl_::exceptions::ImportedExceptionTypeObject = - $crate::impl_::exceptions::ImportedExceptionTypeObject::new(stringify!($module), stringify!($name)); - TYPE_OBJECT.get(py).as_type_ptr() - } - } + $crate::pyobject_native_type_object_methods!( + $name, + #import_module=$module, + #import_name=$name + ); }; } @@ -123,23 +118,16 @@ macro_rules! import_exception_bound { $crate::pyobject_native_type_info!( $name, - $name::type_object_raw, - ::std::option::Option::Some(stringify!($module)) + ::std::option::Option::Some(stringify!($module)), + false + ); + $crate::pyobject_native_type_object_methods!( + $name, + #import_module=$module, + #import_name=$name ); impl $crate::types::DerefToPyAny for $name {} - - impl $name { - fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::types::PyTypeMethods; - static TYPE_OBJECT: $crate::impl_::exceptions::ImportedExceptionTypeObject = - $crate::impl_::exceptions::ImportedExceptionTypeObject::new( - stringify!($module), - stringify!($name), - ); - TYPE_OBJECT.get(py).as_type_ptr() - } - } }; } @@ -246,28 +234,20 @@ macro_rules! create_exception_type_object { ($module: expr, $name: ident, $base: ty, $doc: expr) => { $crate::pyobject_native_type_core!( $name, - $name::type_object_raw, #module=::std::option::Option::Some(stringify!($module)) ); - - impl $name { - fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - use $crate::sync::GILOnceCell; - static TYPE_OBJECT: GILOnceCell<$crate::Py<$crate::types::PyType>> = - GILOnceCell::new(); - - TYPE_OBJECT - .get_or_init(py, || - $crate::PyErr::new_type( - py, - $crate::ffi::c_str!(concat!(stringify!($module), ".", stringify!($name))), - $doc, - ::std::option::Option::Some(&py.get_type::<$base>()), - ::std::option::Option::None, - ).expect("Failed to initialize new exception type.") - ).as_ptr() as *mut $crate::ffi::PyTypeObject + $crate::pyobject_native_type_object_methods!( + $name, + #create=|py| { + $crate::PyErr::new_type( + py, + $crate::ffi::c_str!(concat!(stringify!($module), ".", stringify!($name))), + $doc, + ::std::option::Option::Some(&py.get_type::<$base>()), + ::std::option::Option::None, + ).expect("Failed to initialize new exception type.") } - } + ); }; } @@ -278,7 +258,8 @@ macro_rules! impl_native_exception ( pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, |_py| unsafe { $crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject } $(, #checkfunction=$checkfunction)?); + $crate::pyobject_native_type!($name, $layout $(, #checkfunction=$checkfunction)?); + $crate::pyobject_native_type_object_methods!($name, #global_ptr=$crate::ffi::$exc_name); $crate::pyobject_subclassable_native_type!($name, $layout); ); ($name:ident, $exc_name:ident, $doc:expr) => ( @@ -295,7 +276,8 @@ macro_rules! impl_windows_native_exception ( pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, |_py| unsafe { $crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject }); + $crate::pyobject_native_type!($name, $layout); + $crate::pyobject_native_type_object_methods!($name, #global_ptr=$crate::ffi::$exc_name); ); ($name:ident, $exc_name:ident, $doc:expr) => ( impl_windows_native_exception!($name, $exc_name, $doc, $crate::ffi::PyBaseExceptionObject); @@ -377,6 +359,7 @@ impl_native_exception!( ffi::PyBaseExceptionObject, #checkfunction=ffi::PyExceptionInstance_Check ); + impl_native_exception!(PyException, PyExc_Exception, native_doc!("Exception")); impl_native_exception!( PyStopAsyncIteration, diff --git a/src/impl_/coroutine.rs b/src/impl_/coroutine.rs index f893a2c2fe9..6d53d11ff1c 100644 --- a/src/impl_/coroutine.rs +++ b/src/impl_/coroutine.rs @@ -6,12 +6,17 @@ use std::{ use crate::{ coroutine::{cancel::ThrowCallback, Coroutine}, instance::Bound, - pycell::impl_::PyClassBorrowChecker, + pycell::{ + borrow_checker::PyClassBorrowChecker, + layout::{PyObjectLayout, TypeObjectStrategy}, + }, pyclass::boolean_struct::False, types::{PyAnyMethods, PyString}, IntoPyObject, Py, PyAny, PyClass, PyErr, PyResult, Python, }; +use super::pycell::GetBorrowChecker; + pub fn new_coroutine<'py, F, T, E>( name: &Bound<'py, PyString>, qualname_prefix: Option<&'static str>, @@ -26,16 +31,15 @@ where Coroutine::new(Some(name.clone()), qualname_prefix, throw_callback, future) } -fn get_ptr(obj: &Py) -> *mut T { - obj.get_class_object().get_ptr() -} - pub struct RefGuard(Py); impl RefGuard { pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { let bound = obj.downcast::()?; - bound.get_class_object().borrow_checker().try_borrow()?; + // SAFETY: can assume the type object for `T` is initialized because an instance (`obj`) has been created. + let strategy = unsafe { TypeObjectStrategy::assume_init() }; + let borrow_checker = T::PyClassMutability::borrow_checker(obj.as_raw_ref(), strategy); + borrow_checker.try_borrow()?; Ok(RefGuard(bound.clone().unbind())) } } @@ -44,18 +48,19 @@ impl Deref for RefGuard { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: `RefGuard` has been built from `PyRef` and provides the same guarantees - unsafe { &*get_ptr(&self.0) } + unsafe { + PyObjectLayout::get_data::(self.0.as_raw_ref(), TypeObjectStrategy::assume_init()) + } } } impl Drop for RefGuard { fn drop(&mut self) { - Python::with_gil(|gil| { - self.0 - .bind(gil) - .get_class_object() - .borrow_checker() - .release_borrow() + Python::with_gil(|py| { + // SAFETY: `self.0` contains an object that is an instance of `T` + let borrow_checker = + unsafe { PyObjectLayout::get_borrow_checker::(py, self.0.as_raw_ref()) }; + borrow_checker.release_borrow(); }) } } @@ -65,7 +70,10 @@ pub struct RefMutGuard>(Py); impl> RefMutGuard { pub fn new(obj: &Bound<'_, PyAny>) -> PyResult { let bound = obj.downcast::()?; - bound.get_class_object().borrow_checker().try_borrow_mut()?; + // SAFETY: can assume the type object for `T` is initialized because an instance (`obj`) has been created. + let strategy = unsafe { TypeObjectStrategy::assume_init() }; + let borrow_checker = T::PyClassMutability::borrow_checker(obj.as_raw_ref(), strategy); + borrow_checker.try_borrow_mut()?; Ok(RefMutGuard(bound.clone().unbind())) } } @@ -74,25 +82,31 @@ impl> Deref for RefMutGuard { type Target = T; fn deref(&self) -> &Self::Target { // SAFETY: `RefMutGuard` has been built from `PyRefMut` and provides the same guarantees - unsafe { &*get_ptr(&self.0) } + unsafe { + PyObjectLayout::get_data::(self.0.as_raw_ref(), TypeObjectStrategy::assume_init()) + } } } impl> DerefMut for RefMutGuard { fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: `RefMutGuard` has been built from `PyRefMut` and provides the same guarantees - unsafe { &mut *get_ptr(&self.0) } + unsafe { + &mut *PyObjectLayout::get_data_ptr::( + self.0.as_ptr(), + TypeObjectStrategy::assume_init(), + ) + } } } impl> Drop for RefMutGuard { fn drop(&mut self) { - Python::with_gil(|gil| { - self.0 - .bind(gil) - .get_class_object() - .borrow_checker() - .release_borrow_mut() + Python::with_gil(|py| { + // SAFETY: `self.0` contains an object that is an instance of `T` + let borrow_checker = + unsafe { PyObjectLayout::get_borrow_checker::(py, self.0.as_raw_ref()) }; + borrow_checker.release_borrow_mut(); }) } } diff --git a/src/impl_/pycell.rs b/src/impl_/pycell.rs index 93514c7bb29..fe4378ddc1d 100644 --- a/src/impl_/pycell.rs +++ b/src/impl_/pycell.rs @@ -1,4 +1,7 @@ //! Externally-accessible implementation of pycell -pub use crate::pycell::impl_::{ - GetBorrowChecker, PyClassMutability, PyClassObject, PyClassObjectBase, PyClassObjectLayout, +pub use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassMutability}; +pub use crate::pycell::layout::{ + static_layout::InvalidStaticLayout, static_layout::PyStaticClassLayout, + static_layout::PyStaticNativeLayout, PyClassRecursiveOperations, + PyNativeTypeRecursiveOperations, PyObjectRecursiveOperations, }; diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 7bb61442ec5..468ddd8866c 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -1,13 +1,19 @@ +#[cfg(Py_3_12)] +use crate::pycell::layout::TypeObjectStrategy; use crate::{ exceptions::{PyAttributeError, PyNotImplementedError, PyRuntimeError, PyValueError}, ffi, impl_::{ freelist::FreeList, - pycell::{GetBorrowChecker, PyClassMutability, PyClassObjectLayout}, + pycell::{GetBorrowChecker, PyClassMutability}, pyclass_init::PyObjectInit, pymethods::{PyGetterDef, PyMethodDefType}, }, - pycell::PyBorrowError, + pycell::{ + layout::{usize_to_py_ssize, PyObjectLayout, PyObjectRecursiveOperations}, + PyBorrowError, + }, + type_object::PyLayout, types::{any::PyAnyMethods, PyBool}, Borrowed, BoundObject, IntoPyObject, IntoPyObjectExt, Py, PyAny, PyClass, PyErr, PyRef, PyResult, PyTypeInfo, Python, @@ -33,14 +39,14 @@ pub use probes::*; /// Gets the offset of the dictionary from the start of the object in bytes. #[inline] -pub fn dict_offset() -> ffi::Py_ssize_t { - PyClassObject::::dict_offset() +pub fn dict_offset() -> PyObjectOffset { + PyObjectLayout::dict_offset::() } /// Gets the offset of the weakref list from the start of the object in bytes. #[inline] -pub fn weaklist_offset() -> ffi::Py_ssize_t { - PyClassObject::::weaklist_offset() +pub fn weaklist_offset() -> PyObjectOffset { + PyObjectLayout::weaklist_offset::() } /// Represents the `__dict__` field for `#[pyclass]`. @@ -160,19 +166,19 @@ unsafe impl Sync for PyClassItems {} /// Users are discouraged from implementing this trait manually; it is a PyO3 implementation detail /// and may be changed at any time. pub trait PyClassImpl: Sized + 'static { - /// #[pyclass(subclass)] + /// `#[pyclass(subclass)]` const IS_BASETYPE: bool = false; - /// #[pyclass(extends=...)] + /// `#[pyclass(extends=...)]` const IS_SUBCLASS: bool = false; - /// #[pyclass(mapping)] + /// `#[pyclass(mapping)]` const IS_MAPPING: bool = false; - /// #[pyclass(sequence)] + /// `#[pyclass(sequence)]` const IS_SEQUENCE: bool = false; - /// Base class + /// Base class (the direct parent configured via `#[pyclass(extends=...)]`) type BaseType: PyTypeInfo + PyClassBaseType; /// Immutable or mutable @@ -205,13 +211,17 @@ pub trait PyClassImpl: Sized + 'static { fn items_iter() -> PyClassItemsIter; + /// Used to provide the `__dictoffset__` slot + /// (equivalent to [tp_dictoffset](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dictoffset)) #[inline] - fn dict_offset() -> Option { + fn dict_offset() -> Option { None } + /// Used to provide the `__weaklistoffset__` slot + /// (equivalent to [tp_weaklistoffset](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_weaklistoffset) #[inline] - fn weaklist_offset() -> Option { + fn weaklist_offset() -> Option { None } @@ -903,7 +913,7 @@ macro_rules! generate_pyclass_richcompare_slot { } pub use generate_pyclass_richcompare_slot; -use super::{pycell::PyClassObject, pymethods::BoundRef}; +use super::pymethods::BoundRef; /// Implements a freelist. /// @@ -1116,8 +1126,8 @@ impl PyClassThreadChecker for ThreadCheckerImpl { private_impl! {} } -/// Trait denoting that this class is suitable to be used as a base type for PyClass. - +/// Trait denoting that this class is suitable to be used as a base type for PyClass +/// (meaning it can be used with `#[pyclass(extends=...)]`). #[cfg_attr( all(diagnostic_namespace, Py_LIMITED_API), diagnostic::on_unimplemented( @@ -1136,24 +1146,28 @@ impl PyClassThreadChecker for ThreadCheckerImpl { ) )] pub trait PyClassBaseType: Sized { - type LayoutAsBase: PyClassObjectLayout; - type BaseNativeType; + /// A struct that describes the memory layout of a `ffi:PyObject` with the type of `Self`. + /// Only valid when `::OPAQUE == false`. + type StaticLayout: PyLayout; + /// The nearest ancestor in the inheritance tree that is a native type (not a `#[pyclass]` annotated struct). + type BaseNativeType: PyTypeInfo; + /// The implementation for recursive operations that walk the inheritance tree back to the `BaseNativeType`. + /// (two implementations: one for native type, one for pyclass) + type RecursiveOperations: PyObjectRecursiveOperations; + /// The implementation for constructing new a new `ffi::PyObject` of this type. + /// (two implementations: one for native type, one for pyclass) type Initializer: PyObjectInit; type PyClassMutability: PyClassMutability; } /// Implementation of tp_dealloc for pyclasses without gc pub(crate) unsafe extern "C" fn tp_dealloc(obj: *mut ffi::PyObject) { - crate::impl_::trampoline::dealloc(obj, PyClassObject::::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, PyObjectLayout::deallocate::) } /// Implementation of tp_dealloc for pyclasses with gc pub(crate) unsafe extern "C" fn tp_dealloc_with_gc(obj: *mut ffi::PyObject) { - #[cfg(not(PyPy))] - { - ffi::PyObject_GC_UnTrack(obj.cast()); - } - crate::impl_::trampoline::dealloc(obj, PyClassObject::::tp_dealloc) + crate::impl_::trampoline::dealloc(obj, PyObjectLayout::deallocate_with_gc::) } pub(crate) unsafe extern "C" fn get_sequence_item_from_mapping( @@ -1187,6 +1201,29 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( result } +/// Offset of a field within a PyObject in bytes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PyObjectOffset { + /// An offset relative to the start of the object + Absolute(ffi::Py_ssize_t), + /// An offset relative to the start of the subclass-specific data. + /// Only allowed when basicsize is negative (which is only allowed for python >=3.12). + /// + Relative(ffi::Py_ssize_t), +} + +impl std::ops::Add for PyObjectOffset { + type Output = PyObjectOffset; + + fn add(self, rhs: usize) -> Self::Output { + let rhs = usize_to_py_ssize(rhs); + match self { + PyObjectOffset::Absolute(offset) => PyObjectOffset::Absolute(offset + rhs), + PyObjectOffset::Relative(offset) => PyObjectOffset::Relative(offset + rhs), + } + } +} + /// Helper trait to locate field within a `#[pyclass]` for a `#[pyo3(get)]`. /// /// Below MSRV 1.77 we can't use `std::mem::offset_of!`, and the replacement in @@ -1196,13 +1233,13 @@ pub(crate) unsafe extern "C" fn assign_sequence_item_from_mapping( /// /// The trait is unsafe to implement because producing an incorrect offset will lead to UB. pub unsafe trait OffsetCalculator { - /// Offset to the field within a `PyClassObject`, in bytes. - fn offset() -> usize; + /// Offset to the field within a PyObject + fn offset() -> PyObjectOffset; } // Used in generated implementations of OffsetCalculator -pub fn class_offset() -> usize { - offset_of!(PyClassObject, contents) +pub fn subclass_offset() -> PyObjectOffset { + PyObjectLayout::contents_offset::() } // Used in generated implementations of OffsetCalculator @@ -1281,11 +1318,22 @@ impl< pub fn generate(&self, name: &'static CStr, doc: &'static CStr) -> PyMethodDefType { use crate::pyclass::boolean_struct::private::Boolean; if ClassT::Frozen::VALUE { + let (offset, flags) = match Offset::offset() { + PyObjectOffset::Absolute(offset) => (offset, ffi::Py_READONLY), + #[cfg(Py_3_12)] + PyObjectOffset::Relative(offset) => { + (offset, ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET) + } + #[cfg(not(Py_3_12))] + PyObjectOffset::Relative(_) => { + panic!("relative offsets not valid before python 3.12"); + } + }; PyMethodDefType::StructMember(ffi::PyMemberDef { name: name.as_ptr(), type_code: ffi::Py_T_OBJECT_EX, - offset: Offset::offset() as ffi::Py_ssize_t, - flags: ffi::Py_READONLY, + offset, + flags, doc: doc.as_ptr(), }) } else { @@ -1438,12 +1486,29 @@ unsafe fn ensure_no_mutable_alias<'py, ClassT: PyClass>( /// calculates the field pointer from an PyObject pointer #[inline] -fn field_from_object(obj: *mut ffi::PyObject) -> *mut FieldT +fn field_from_object(py: Python<'_>, obj: *mut ffi::PyObject) -> *mut FieldT where ClassT: PyClass, Offset: OffsetCalculator, { - unsafe { obj.cast::().add(Offset::offset()).cast::() } + let (base, offset) = match Offset::offset() { + PyObjectOffset::Absolute(offset) => (obj.cast::(), offset), + #[cfg(Py_3_12)] + PyObjectOffset::Relative(offset) => { + // Safety: obj must be a valid `PyObject` whose type is a subtype of `ClassT` + let contents = unsafe { + PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)) + }; + (contents.cast::(), offset) + } + #[cfg(not(Py_3_12))] + PyObjectOffset::Relative(_) => { + let _ = py; + panic!("relative offsets not valid before python 3.12"); + } + }; + // Safety: conditions for pointer addition must be met + unsafe { base.add(offset as usize) }.cast::() } #[allow(deprecated)] @@ -1456,7 +1521,7 @@ fn pyo3_get_value_topyobject< obj: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1473,7 +1538,7 @@ where Offset: OffsetCalculator, { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1493,7 +1558,7 @@ where Offset: OffsetCalculator, { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1514,7 +1579,7 @@ fn pyo3_get_value< obj: *mut ffi::PyObject, ) -> PyResult<*mut ffi::PyObject> { let _holder = unsafe { ensure_no_mutable_alias::(py, &obj)? }; - let value = field_from_object::(obj); + let value = field_from_object::(py, obj); // SAFETY: Offset is known to describe the location of the value, and // _holder is preventing mutable aliasing @@ -1593,6 +1658,8 @@ impl Deprecation { #[cfg(test)] #[cfg(feature = "macros")] mod tests { + use crate::pycell::layout::PyClassObjectContents; + use super::*; #[test] @@ -1621,9 +1688,14 @@ mod tests { Some(PyMethodDefType::StructMember(member)) => { assert_eq!(unsafe { CStr::from_ptr(member.name) }, ffi::c_str!("value")); assert_eq!(member.type_code, ffi::Py_T_OBJECT_EX); + #[repr(C)] + struct ExpectedLayout { + ob_base: ffi::PyObject, + contents: PyClassObjectContents, + } assert_eq!( member.offset, - (memoffset::offset_of!(PyClassObject, contents) + (memoffset::offset_of!(ExpectedLayout, contents) + memoffset::offset_of!(FrozenClass, value)) as ffi::Py_ssize_t ); diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index d3bede7b2f3..6b7832bbf74 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -4,6 +4,8 @@ use std::{ thread::{self, ThreadId}, }; +use pyo3_ffi::PyTypeObject; + use crate::{ exceptions::PyRuntimeError, ffi, @@ -61,6 +63,10 @@ impl LazyTypeObject { self.0 .get_or_try_init(py, create_type_object::, T::NAME, T::items_iter()) } + + pub fn try_get_raw(&self) -> Option<*mut PyTypeObject> { + self.0.try_get_raw() + } } impl LazyTypeObjectInner { @@ -92,6 +98,14 @@ impl LazyTypeObjectInner { }) } + pub fn try_get_raw(&self) -> Option<*mut ffi::PyTypeObject> { + unsafe { + self.value + .get_raw() + .map(|obj| (*obj).type_object.as_ptr().cast::()) + } + } + fn ensure_init( &self, type_object: &Bound<'_, PyType>, diff --git a/src/impl_/pyclass_init.rs b/src/impl_/pyclass_init.rs index 7242b6186d9..b31657867f3 100644 --- a/src/impl_/pyclass_init.rs +++ b/src/impl_/pyclass_init.rs @@ -1,11 +1,29 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; use crate::internal::get_slot::TP_ALLOC; +use crate::pycell::layout::{PyClassObjectContents, PyObjectLayout, TypeObjectStrategy}; use crate::types::PyType; -use crate::{ffi, Borrowed, PyErr, PyResult, Python}; +use crate::{ffi, Borrowed, PyClass, PyErr, PyResult, Python}; use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo}; +use std::any::TypeId; use std::marker::PhantomData; +pub unsafe fn initialize_with_default( + py: Python<'_>, + obj: *mut ffi::PyObject, +) { + // only sets the PyClassContents of the 'most derived type' + // so any parent pyclasses would remain uninitialized. + assert!( + TypeId::of::() == TypeId::of::(), + "initialize_with_default does not currently support multi-level inheritance" + ); + std::ptr::write( + PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)), + PyClassObjectContents::new(T::default()), + ); +} + /// Initializer for Python types. /// /// This trait is intended to use internally for distinguishing `#[pyclass]` and @@ -37,14 +55,16 @@ impl PyObjectInit for PyNativeTypeInitializer { type_object: *mut PyTypeObject, subtype: *mut PyTypeObject, ) -> PyResult<*mut ffi::PyObject> { - // HACK (due to FIXME below): PyBaseObject_Type's tp_new isn't happy with NULL arguments + // HACK (due to FIXME below): PyBaseObject_Type and PyType_Type tp_new aren't happy with NULL arguments let is_base_object = type_object == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type); - let subtype_borrowed: Borrowed<'_, '_, PyType> = subtype - .cast::() - .assume_borrowed_unchecked(py) - .downcast_unchecked(); + let is_metaclass = type_object == std::ptr::addr_of_mut!(ffi::PyType_Type); + + if is_base_object || is_metaclass { + let subtype_borrowed: Borrowed<'_, '_, PyType> = subtype + .cast::() + .assume_borrowed_unchecked(py) + .downcast_unchecked(); - if is_base_object { let alloc = subtype_borrowed .get_slot(TP_ALLOC) .unwrap_or(ffi::PyType_GenericAlloc); diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 58d0c93c240..44f26c4d252 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -2,9 +2,9 @@ use crate::exceptions::PyStopAsyncIteration; use crate::gil::LockGIL; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; -use crate::impl_::pycell::{PyClassObject, PyClassObjectLayout}; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; -use crate::pycell::impl_::PyClassBorrowChecker as _; +use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; +use crate::pycell::layout::{PyObjectLayout, TypeObjectStrategy}; use crate::pycell::{PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; @@ -20,6 +20,8 @@ use std::os::raw::{c_int, c_void}; use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr::null_mut; +use super::pycell::{PyClassRecursiveOperations, PyObjectRecursiveOperations}; +use super::pyclass::PyClassImpl; use super::trampoline; /// Python 3.8 and up - __ipow__ has modulo argument correctly populated. @@ -301,25 +303,34 @@ where // SAFETY: `slf` is a valid Python object pointer to a class object of type T, and // traversal is running so no mutations can occur. - let class_object: &PyClassObject = &*slf.cast(); + let raw_obj = &*slf; + + // SAFETY: type objects for `T` and all ancestors of `T` are created the first time an + // instance of `T` is created. Since `slf` is an instance of `T` the type objects must + // have been created. + let strategy = TypeObjectStrategy::assume_init(); let retval = // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so // do not traverse them if not on their owning thread :( - if class_object.check_threadsafe().is_ok() + if PyClassRecursiveOperations::::check_threadsafe(raw_obj, strategy).is_ok() // ... and we cannot traverse a type which might be being mutated by a Rust thread - && class_object.borrow_checker().try_borrow().is_ok() { - struct TraverseGuard<'a, T: PyClass>(&'a PyClassObject); - impl Drop for TraverseGuard<'_, T> { + && T::PyClassMutability::borrow_checker(raw_obj, strategy).try_borrow().is_ok() { + struct TraverseGuard<'a, Cls: PyClassImpl>(&'a ffi::PyObject, PhantomData); + impl Drop for TraverseGuard<'_, Cls> { fn drop(&mut self) { - self.0.borrow_checker().release_borrow() + let borrow_checker = Cls::PyClassMutability::borrow_checker( + self.0, + unsafe { TypeObjectStrategy::assume_init() } + ); + borrow_checker.release_borrow(); } } // `.try_borrow()` above created a borrow, we need to release it when we're done // traversing the object. This allows us to read `instance` safely. - let _guard = TraverseGuard(class_object); - let instance = &*class_object.contents.value.get(); + let _guard: TraverseGuard<'_, T> = TraverseGuard(raw_obj, PhantomData); + let instance = PyObjectLayout::get_data::(raw_obj, strategy); let visit = PyVisit { visit, arg, _guard: PhantomData }; diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index 7ffad8abdcd..32416af6a02 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -122,6 +122,29 @@ trampolines!( pub fn unaryfunc(slf: *mut ffi::PyObject) -> *mut ffi::PyObject; ); +/// `tp_init` should return 0 on success and -1 on error. +/// [docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_init) +#[inline] +pub unsafe fn initproc( + slf: *mut ffi::PyObject, + args: *mut ffi::PyObject, + kwargs: *mut ffi::PyObject, + // initializes the object to a valid state before running the user-defined init function + initialize: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject), + f: for<'py> unsafe fn( + Python<'py>, + *mut ffi::PyObject, + *mut ffi::PyObject, + *mut ffi::PyObject, + ) -> PyResult<*mut ffi::PyObject>, +) -> c_int { + // the map() discards the success value of `f` and converts to the success return value for tp_init (0) + trampoline(|py| { + initialize(py, slf); + f(py, slf, args, kwargs).map(|_| 0) + }) +} + #[cfg(any(not(Py_LIMITED_API), Py_3_11))] trampoline! { pub fn getbufferproc(slf: *mut ffi::PyObject, buf: *mut ffi::Py_buffer, flags: c_int) -> c_int; diff --git a/src/instance.rs b/src/instance.rs index 840416116f3..a7b138a2a8b 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,8 +1,8 @@ use crate::conversion::IntoPyObject; use crate::err::{self, PyErr, PyResult}; -use crate::impl_::pycell::PyClassObject; use crate::internal_tricks::ptr_from_ref; -use crate::pycell::{PyBorrowError, PyBorrowMutError}; +use crate::pycell::layout::TypeObjectStrategy; +use crate::pycell::{layout::PyObjectLayout, PyBorrowError, PyBorrowMutError}; use crate::pyclass::boolean_struct::{False, True}; use crate::types::{any::PyAnyMethods, string::PyStringMethods, typeobject::PyTypeMethods}; use crate::types::{DerefToPyAny, PyDict, PyString, PyTuple}; @@ -462,8 +462,8 @@ where } #[inline] - pub(crate) fn get_class_object(&self) -> &PyClassObject { - self.1.get_class_object() + pub(crate) fn get_raw_object(&self) -> &ffi::PyObject { + self.1.as_raw_ref() } } @@ -553,6 +553,17 @@ impl<'py, T> Bound<'py, T> { self.1.as_ptr() } + /// Returns the raw FFI object represented by self. + /// + /// # Safety + /// + /// The reference is borrowed; callers should not decrease the reference count + /// when they are finished with the object. + #[inline] + pub fn as_raw_ref(&self) -> &ffi::PyObject { + self.1.as_raw_ref() + } + /// Returns an owned raw FFI pointer represented by self. /// /// # Safety @@ -1116,6 +1127,17 @@ impl Py { self.0.as_ptr() } + /// Returns the raw FFI object represented by self. + /// + /// # Safety + /// + /// The reference is borrowed; callers should not decrease the reference count + /// when they are finished with the object. + #[inline] + pub fn as_raw_ref(&self) -> &ffi::PyObject { + unsafe { &*self.0.as_ptr() } + } + /// Returns an owned raw FFI pointer represented by self. /// /// # Safety @@ -1289,17 +1311,10 @@ where where T: PyClass + Sync, { - // Safety: The class itself is frozen and `Sync` - unsafe { &*self.get_class_object().get_ptr() } - } - - /// Get a view on the underlying `PyClass` contents. - #[inline] - pub(crate) fn get_class_object(&self) -> &PyClassObject { - let class_object = self.as_ptr().cast::>(); - // Safety: Bound is known to contain an object which is laid out in memory as a - // PyClassObject. - unsafe { &*class_object } + // Safety: the PyTypeObject for T will have been created when the first instance of T was created. + // Since Py contains an instance of T the type object must have already been created. + let strategy = unsafe { TypeObjectStrategy::assume_init() }; + unsafe { PyObjectLayout::get_data::(self.as_raw_ref(), strategy) } } } @@ -2338,7 +2353,6 @@ a = A() for i in 0..10 { let instance = Py::new(py, FrozenClass(i)).unwrap(); assert_eq!(instance.get().0, i); - assert_eq!(instance.bind(py).get().0, i); } }) diff --git a/src/internal/get_slot.rs b/src/internal/get_slot.rs index 260893d4204..55a4bb240fe 100644 --- a/src/internal/get_slot.rs +++ b/src/internal/get_slot.rs @@ -126,6 +126,7 @@ impl_slots! { TP_BASE: (Py_tp_base, tp_base) -> *mut ffi::PyTypeObject, TP_CLEAR: (Py_tp_clear, tp_clear) -> Option, TP_DESCR_GET: (Py_tp_descr_get, tp_descr_get) -> Option, + TP_DEALLOC: (Py_tp_dealloc, tp_dealloc) -> Option, TP_FREE: (Py_tp_free, tp_free) -> Option, TP_TRAVERSE: (Py_tp_traverse, tp_traverse) -> Option, } diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 97b13aff2a8..d2e67326c11 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -47,3 +47,15 @@ pub(crate) const fn ptr_from_ref(t: &T) -> *const T { pub(crate) fn ptr_from_mut(t: &mut T) -> *mut T { t as *mut T } + +// TODO: use ptr::cast_mut on MSRV 1.65 +#[inline] +pub(crate) fn cast_mut(t: *const T) -> *mut T { + t as *mut T +} + +// TODO: use ptr::cast_const on MSRV 1.65 +#[inline] +pub(crate) fn cast_const(t: *mut T) -> *const T { + t as *const T +} diff --git a/src/pycell.rs b/src/pycell.rs index c7e5226a292..0102babb3b2 100644 --- a/src/pycell.rs +++ b/src/pycell.rs @@ -207,8 +207,10 @@ use std::fmt; use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; -pub(crate) mod impl_; -use impl_::{PyClassBorrowChecker, PyClassObjectLayout}; +pub(crate) mod borrow_checker; +pub(crate) mod layout; +use borrow_checker::PyClassBorrowChecker; +use layout::{PyObjectLayout, TypeObjectStrategy}; /// A wrapper type for an immutably borrowed value from a [`Bound<'py, T>`]. /// @@ -308,9 +310,11 @@ impl<'py, T: PyClass> PyRef<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_class_object(); - cell.ensure_threadsafe(); - cell.borrow_checker() + let raw_obj = obj.as_raw_ref(); + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; + borrow_checker .try_borrow() .map(|_| Self { inner: obj.clone() }) } @@ -431,21 +435,23 @@ where } } -impl Deref for PyRef<'_, T> { +impl<'py, T: PyClass> Deref for PyRef<'py, T> { type Target = T; #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_class_object().get_ptr() } + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let obj = self.inner.as_raw_ref(); + unsafe { PyObjectLayout::get_data::(obj, TypeObjectStrategy::lazy(py)) } } } -impl Drop for PyRef<'_, T> { +impl<'py, T: PyClass> Drop for PyRef<'py, T> { fn drop(&mut self) { - self.inner - .get_class_object() - .borrow_checker() - .release_borrow() + let obj = self.inner.as_raw_ref(); + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; + borrow_checker.release_borrow(); } } @@ -564,16 +570,18 @@ impl<'py, T: PyClass> PyRefMut<'py, T> { } pub(crate) fn try_borrow(obj: &Bound<'py, T>) -> Result { - let cell = obj.get_class_object(); - cell.ensure_threadsafe(); - cell.borrow_checker() + let raw_obj = obj.as_raw_ref(); + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + unsafe { PyObjectLayout::ensure_threadsafe::(py, raw_obj) }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, raw_obj) }; + borrow_checker .try_borrow_mut() .map(|_| Self { inner: obj.clone() }) } - pub(crate) fn downgrade(slf: &Self) -> &PyRef<'py, T> { + pub(crate) fn downgrade(&self) -> &PyRef<'py, T> { // `PyRefMut` and `PyRef` have the same layout - unsafe { &*ptr_from_ref(slf).cast() } + unsafe { &*ptr_from_ref(self).cast() } } } @@ -615,28 +623,32 @@ where } } -impl> Deref for PyRefMut<'_, T> { +impl<'py, T: PyClass> Deref for PyRefMut<'py, T> { type Target = T; #[inline] fn deref(&self) -> &T { - unsafe { &*self.inner.get_class_object().get_ptr() } + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let obj = self.inner.as_raw_ref(); + unsafe { PyObjectLayout::get_data::(obj, TypeObjectStrategy::lazy(py)) } } } -impl> DerefMut for PyRefMut<'_, T> { +impl<'py, T: PyClass> DerefMut for PyRefMut<'py, T> { #[inline] fn deref_mut(&mut self) -> &mut T { - unsafe { &mut *self.inner.get_class_object().get_ptr() } + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let obj = self.inner.as_ptr(); + unsafe { &mut *PyObjectLayout::get_data_ptr::(obj, TypeObjectStrategy::lazy(py)) } } } -impl> Drop for PyRefMut<'_, T> { +impl<'py, T: PyClass> Drop for PyRefMut<'py, T> { fn drop(&mut self) { - self.inner - .get_class_object() - .borrow_checker() - .release_borrow_mut() + let obj = self.inner.get_raw_object(); + let py: Python<'py> = unsafe { Python::assume_gil_acquired() }; + let borrow_checker = unsafe { PyObjectLayout::get_borrow_checker::(py, obj) }; + borrow_checker.release_borrow_mut(); } } diff --git a/src/pycell/impl_.rs b/src/pycell/borrow_checker.rs similarity index 69% rename from src/pycell/impl_.rs rename to src/pycell/borrow_checker.rs index 1b0724d8481..045b8c8b931 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/borrow_checker.rs @@ -1,20 +1,14 @@ #![allow(missing_docs)] //! Crate-private implementation of PyClassObject -use std::cell::UnsafeCell; use std::marker::PhantomData; -use std::mem::ManuallyDrop; use std::sync::atomic::{AtomicUsize, Ordering}; -use crate::impl_::pyclass::{ - PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, -}; -use crate::internal::get_slot::TP_FREE; -use crate::type_object::{PyLayout, PySizedLayout}; -use crate::types::{PyType, PyTypeMethods}; -use crate::{ffi, PyClass, PyTypeInfo, Python}; +use crate::impl_::pyclass::PyClassImpl; +use crate::{ffi, PyTypeInfo}; -use super::{PyBorrowError, PyBorrowMutError}; +use super::layout::TypeObjectStrategy; +use super::{PyBorrowError, PyBorrowMutError, PyObjectLayout}; pub trait PyClassMutability { // The storage for this inheritance layer. Only the first mutable class in @@ -100,15 +94,13 @@ pub struct BorrowChecker(BorrowFlag); pub trait PyClassBorrowChecker { /// Initial value for self fn new() -> Self; - /// Increments immutable borrow count, if possible fn try_borrow(&self) -> Result<(), PyBorrowError>; - /// Decrements immutable borrow count fn release_borrow(&self); /// Increments mutable borrow count, if possible fn try_borrow_mut(&self) -> Result<(), PyBorrowMutError>; - /// Decremements mutable borrow count + /// Decrements mutable borrow count fn release_borrow_mut(&self); } @@ -176,181 +168,45 @@ impl PyClassBorrowChecker for BorrowChecker { } pub trait GetBorrowChecker { - fn borrow_checker( - class_object: &PyClassObject, - ) -> &::Checker; + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a ::Checker; } -impl> GetBorrowChecker for MutableClass { - fn borrow_checker(class_object: &PyClassObject) -> &BorrowChecker { - &class_object.contents.borrow_checker +impl + PyTypeInfo> GetBorrowChecker for MutableClass { + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a BorrowChecker { + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; + &contents.borrow_checker } } -impl> GetBorrowChecker for ImmutableClass { - fn borrow_checker(class_object: &PyClassObject) -> &EmptySlot { - &class_object.contents.borrow_checker +impl + PyTypeInfo> GetBorrowChecker for ImmutableClass { + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a EmptySlot { + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; + &contents.borrow_checker } } -impl, M: PyClassMutability> GetBorrowChecker - for ExtendsMutableAncestor +impl GetBorrowChecker for ExtendsMutableAncestor where - T::BaseType: PyClassImpl + PyClassBaseType>, + T: PyClassImpl, + M: PyClassMutability, + T::BaseType: PyClassImpl, ::PyClassMutability: PyClassMutability, { - fn borrow_checker(class_object: &PyClassObject) -> &BorrowChecker { - <::PyClassMutability as GetBorrowChecker>::borrow_checker(&class_object.ob_base) - } -} - -/// Base layout of PyClassObject. -#[doc(hidden)] -#[repr(C)] -pub struct PyClassObjectBase { - ob_base: T, -} - -unsafe impl PyLayout for PyClassObjectBase where U: PySizedLayout {} - -#[doc(hidden)] -pub trait PyClassObjectLayout: PyLayout { - fn ensure_threadsafe(&self); - fn check_threadsafe(&self) -> Result<(), PyBorrowError>; - /// Implementation of tp_dealloc. - /// # Safety - /// - slf must be a valid pointer to an instance of a T or a subclass. - /// - slf must not be used after this call (as it will be freed). - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject); -} - -impl PyClassObjectLayout for PyClassObjectBase -where - U: PySizedLayout, - T: PyTypeInfo, -{ - fn ensure_threadsafe(&self) {} - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - Ok(()) - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - // FIXME: there is potentially subtle issues here if the base is overwritten - // at runtime? To be investigated. - let type_obj = T::type_object(py); - let type_ptr = type_obj.as_type_ptr(); - let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(slf)); - - // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free - if type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { - let tp_free = actual_type - .get_slot(TP_FREE) - .expect("PyBaseObject_Type should have tp_free"); - return tp_free(slf.cast()); - } - - // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. - #[cfg(not(Py_LIMITED_API))] - { - // FIXME: should this be using actual_type.tp_dealloc? - if let Some(dealloc) = (*type_ptr).tp_dealloc { - // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which - // assumes the exception is currently GC tracked, so we have to re-track - // before calling the dealloc so that it can safely call Py_GC_UNTRACK. - #[cfg(not(any(Py_3_11, PyPy)))] - if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { - ffi::PyObject_GC_Track(slf.cast()); - } - dealloc(slf); - } else { - (*actual_type.as_type_ptr()) - .tp_free - .expect("type missing tp_free")(slf.cast()); - } - } - - #[cfg(Py_LIMITED_API)] - unreachable!("subclassing native types is not possible with the `abi3` feature"); - } -} - -/// The layout of a PyClass as a Python object -#[repr(C)] -pub struct PyClassObject { - pub(crate) ob_base: ::LayoutAsBase, - pub(crate) contents: PyClassObjectContents, -} - -#[repr(C)] -pub(crate) struct PyClassObjectContents { - pub(crate) value: ManuallyDrop>, - pub(crate) borrow_checker: ::Storage, - pub(crate) thread_checker: T::ThreadChecker, - pub(crate) dict: T::Dict, - pub(crate) weakref: T::WeakRef, -} - -impl PyClassObject { - pub(crate) fn get_ptr(&self) -> *mut T { - self.contents.value.get() - } - - /// Gets the offset of the dictionary from the start of the struct in bytes. - pub(crate) fn dict_offset() -> ffi::Py_ssize_t { - use memoffset::offset_of; - - let offset = - offset_of!(PyClassObject, contents) + offset_of!(PyClassObjectContents, dict); - - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - offset.try_into().expect("offset should fit in Py_ssize_t") - } - - /// Gets the offset of the weakref list from the start of the struct in bytes. - pub(crate) fn weaklist_offset() -> ffi::Py_ssize_t { - use memoffset::offset_of; - - let offset = - offset_of!(PyClassObject, contents) + offset_of!(PyClassObjectContents, weakref); - - // Py_ssize_t may not be equal to isize on all platforms - #[allow(clippy::useless_conversion)] - offset.try_into().expect("offset should fit in Py_ssize_t") - } -} - -impl PyClassObject { - pub(crate) fn borrow_checker(&self) -> &::Checker { - T::PyClassMutability::borrow_checker(self) - } -} - -unsafe impl PyLayout for PyClassObject {} -impl PySizedLayout for PyClassObject {} - -impl PyClassObjectLayout for PyClassObject -where - ::LayoutAsBase: PyClassObjectLayout, -{ - fn ensure_threadsafe(&self) { - self.contents.thread_checker.ensure(); - self.ob_base.ensure_threadsafe(); - } - fn check_threadsafe(&self) -> Result<(), PyBorrowError> { - if !self.contents.thread_checker.check() { - return Err(PyBorrowError { _private: () }); - } - self.ob_base.check_threadsafe() - } - unsafe fn tp_dealloc(py: Python<'_>, slf: *mut ffi::PyObject) { - // Safety: Python only calls tp_dealloc when no references to the object remain. - let class_object = &mut *(slf.cast::>()); - if class_object.contents.thread_checker.can_drop(py) { - ManuallyDrop::drop(&mut class_object.contents.value); - } - class_object.contents.dict.clear_dict(py); - class_object.contents.weakref.clear_weakrefs(slf, py); - ::LayoutAsBase::tp_dealloc(py, slf) + fn borrow_checker<'a>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a BorrowChecker { + // the same PyObject pointer can be re-interpreted as the base/parent type + <::PyClassMutability as GetBorrowChecker>::borrow_checker(obj, strategy) } } @@ -359,8 +215,8 @@ where mod tests { use super::*; - use crate::prelude::*; use crate::pyclass::boolean_struct::{False, True}; + use crate::{prelude::*, PyClass}; #[pyclass(crate = "crate", subclass)] struct MutableBase; @@ -588,6 +444,8 @@ mod tests { #[test] #[cfg(not(target_arch = "wasm32"))] fn test_thread_safety_2() { + use std::cell::UnsafeCell; + struct SyncUnsafeCell(UnsafeCell); unsafe impl Sync for SyncUnsafeCell {} diff --git a/src/pycell/layout.rs b/src/pycell/layout.rs new file mode 100644 index 00000000000..1948f3a5d94 --- /dev/null +++ b/src/pycell/layout.rs @@ -0,0 +1,1740 @@ +#![allow(missing_docs)] +//! Crate-private implementation of how PyClassObjects are laid out in memory and how to access data from raw PyObjects + +use std::cell::UnsafeCell; +use std::marker::PhantomData; +use std::mem::ManuallyDrop; +use std::ptr::addr_of_mut; + +use memoffset::offset_of; + +use crate::impl_::pyclass::{ + PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset, +}; +use crate::internal::get_slot::{TP_DEALLOC, TP_FREE}; +use crate::internal_tricks::{cast_const, cast_mut}; +use crate::pycell::borrow_checker::{GetBorrowChecker, PyClassBorrowChecker}; +use crate::type_object::PyNativeType; +use crate::types::PyType; +use crate::{ffi, PyTypeInfo, Python}; + +#[cfg(not(Py_LIMITED_API))] +use crate::types::PyTypeMethods; + +use super::borrow_checker::PyClassMutability; +use super::{ptr_from_ref, PyBorrowError}; + +/// The layout of the region of a [ffi::PyObject] specifically relating to type `T`. +/// +/// In an inheritance hierarchy where `#[pyclass(extends=PyDict)] struct A;` and `#[pyclass(extends=A)] struct B;` +/// a [ffi::PyObject] of type `B` has separate memory for [ffi::PyDictObject] (the base native type) and +/// `PyClassObjectContents` and `PyClassObjectContents`. The memory associated with `A` or `B` can be obtained +/// using `PyObjectLayout::get_contents::()` (where `T=A` or `T=B`). +#[repr(C)] +pub(crate) struct PyClassObjectContents { + /// The data associated with the user-defined struct annotated with `#[pyclass]` + pub(crate) value: ManuallyDrop>, + pub(crate) borrow_checker: ::Storage, + pub(crate) thread_checker: T::ThreadChecker, + /// A pointer to a [ffi::PyObject] if `T` is annotated with `#[pyclass(dict)]` and a zero-sized field otherwise. + pub(crate) dict: T::Dict, + /// A pointer to a [ffi::PyObject] if `T` is annotated with `#[pyclass(weakref)]` and a zero-sized field otherwise. + pub(crate) weakref: T::WeakRef, +} + +impl PyClassObjectContents { + pub(crate) fn new(init: T) -> Self { + PyClassObjectContents { + value: ManuallyDrop::new(UnsafeCell::new(init)), + borrow_checker: ::Storage::new(), + thread_checker: T::ThreadChecker::new(), + dict: T::Dict::INIT, + weakref: T::WeakRef::INIT, + } + } + + unsafe fn dealloc(&mut self, py: Python<'_>, py_object: *mut ffi::PyObject) { + if self.thread_checker.can_drop(py) { + ManuallyDrop::drop(&mut self.value); + } + self.dict.clear_dict(py); + self.weakref.clear_weakrefs(py_object, py); + } +} + +/// Functions for working with [ffi::PyObject]s recursively by re-interpreting the object +/// as being an instance of the most derived class through each base class until +/// the `BaseNativeType` is reached. +/// +/// E.g. if `#[pyclass(extends=PyDict)] struct A;` and `#[pyclass(extends=A)] struct B;` +/// then calling a method on a PyObject of type `B` will call the method for `B`, then `A`, then `PyDict`. +#[doc(hidden)] +pub trait PyObjectRecursiveOperations { + /// [PyTypeInfo::type_object_raw()] may create type objects lazily. + /// This method ensures that the type objects for all ancestor types of the provided object. + fn ensure_type_objects_initialized(py: Python<'_>); + + /// Call [PyClassThreadChecker::ensure()] on all ancestor types of the provided object. + fn ensure_threadsafe(obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>); + + /// Call [PyClassThreadChecker::check()] on all ancestor types of the provided object. + fn check_threadsafe( + obj: &ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> Result<(), PyBorrowError>; + + /// Cleanup then free the memory for `obj`. + /// + /// # Safety + /// - `obj` must be a valid pointer to an instance of a `T` or a subclass. + /// - `obj` must not be used after this call (as it will be freed). + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject); +} + +/// Used to fill out [PyClassBaseType::RecursiveOperations] for instances of `PyClass` +pub struct PyClassRecursiveOperations(PhantomData); + +impl PyObjectRecursiveOperations for PyClassRecursiveOperations { + fn ensure_type_objects_initialized(py: Python<'_>) { + let _ = ::type_object_raw(py); + ::RecursiveOperations::ensure_type_objects_initialized(py); + } + + fn ensure_threadsafe(obj: &ffi::PyObject, strategy: TypeObjectStrategy<'_>) { + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; + contents.thread_checker.ensure(); + ::RecursiveOperations::ensure_threadsafe(obj, strategy); + } + + fn check_threadsafe( + obj: &ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> Result<(), PyBorrowError> { + let contents = unsafe { PyObjectLayout::get_contents::(obj, strategy) }; + if !contents.thread_checker.check() { + return Err(PyBorrowError { _private: () }); + } + ::RecursiveOperations::check_threadsafe(obj, strategy) + } + + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + // Safety: Python only calls tp_dealloc when no references to the object remain. + let contents = + &mut *PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)); + contents.dealloc(py, obj); + ::RecursiveOperations::deallocate(py, obj); + } +} + +/// Used to fill out [PyClassBaseType::RecursiveOperations] for native types +pub struct PyNativeTypeRecursiveOperations(PhantomData); + +impl PyObjectRecursiveOperations + for PyNativeTypeRecursiveOperations +{ + fn ensure_type_objects_initialized(py: Python<'_>) { + let _ = ::type_object_raw(py); + } + + fn ensure_threadsafe(_obj: &ffi::PyObject, _strategy: TypeObjectStrategy<'_>) {} + + fn check_threadsafe( + _obj: &ffi::PyObject, + _strategy: TypeObjectStrategy<'_>, + ) -> Result<(), PyBorrowError> { + Ok(()) + } + + /// Call the destructor (`tp_dealloc`) of an object which is an instance of a + /// subclass of the native type `T`. + /// + /// Does not clear up any data from subtypes of `type_ptr` so it is assumed that those + /// destructors have been called first. + /// + /// [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + /// + /// # Safety + /// - `obj` must be a valid pointer to an instance of the type at `type_ptr` or a subclass. + /// - `obj` must not be used after this call (as it will be freed). + unsafe fn deallocate(py: Python<'_>, obj: *mut ffi::PyObject) { + // the `BaseNativeType` of the object + let type_ptr = ::type_object_raw(py); + + // FIXME: there is potentially subtle issues here if the base is overwritten at runtime? To be investigated. + + // the 'most derived class' of `obj`. i.e. the result of calling `type(obj)`. + let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(obj)); + + if type_ptr == std::ptr::addr_of_mut!(ffi::PyBaseObject_Type) { + // the `PyBaseObject_Type` destructor (tp_dealloc) just calls tp_free so we can do this directly + let tp_free = actual_type + .get_slot(TP_FREE) + .expect("base type should have tp_free"); + return tp_free(obj.cast()); + } + + if type_ptr == std::ptr::addr_of_mut!(ffi::PyType_Type) { + let tp_dealloc = PyType::from_borrowed_type_ptr(py, type_ptr) + .get_slot(TP_DEALLOC) + .expect("PyType_Type should have tp_dealloc"); + // `PyType_Type::dealloc` calls `Py_GC_UNTRACK` so we have to re-track before deallocating + #[cfg(not(PyPy))] + ffi::PyObject_GC_Track(obj.cast()); + return tp_dealloc(obj.cast()); + } + + // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. + #[cfg(not(Py_LIMITED_API))] + { + // FIXME: should this be using actual_type.tp_dealloc? + if let Some(dealloc) = (*type_ptr).tp_dealloc { + // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which + // assumes the exception is currently GC tracked, so we have to re-track + // before calling the dealloc so that it can safely call Py_GC_UNTRACK. + #[cfg(not(any(Py_3_11, PyPy)))] + if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 { + ffi::PyObject_GC_Track(obj.cast()); + } + dealloc(obj); + } else { + (*actual_type.as_type_ptr()) + .tp_free + .expect("type missing tp_free")(obj.cast()); + } + } + + #[cfg(Py_LIMITED_API)] + unreachable!("subclassing native types is not possible with the `abi3` feature"); + } +} + +/// Utilities for working with [ffi::PyObject] objects that utilise [PEP 697](https://peps.python.org/pep-0697/). +#[doc(hidden)] +pub(crate) mod opaque_layout { + #[cfg(Py_3_12)] + use super::{PyClassObjectContents, TypeObjectStrategy}; + #[cfg(Py_3_12)] + use crate::ffi; + use crate::{impl_::pyclass::PyClassImpl, PyTypeInfo}; + + /// Obtain a pointer to the region of `obj` that relates to `T` + /// + /// # Safety + /// - `obj` must be a valid `ffi::PyObject` of type `T` or a subclass of `T` that uses the opaque layout + #[cfg(Py_3_12)] + pub(crate) unsafe fn get_contents_ptr( + obj: *mut ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> *mut PyClassObjectContents { + let type_obj = match strategy { + TypeObjectStrategy::Lazy(py) => T::type_object_raw(py), + TypeObjectStrategy::AssumeInit(_) => { + T::try_get_type_object_raw().unwrap_or_else(|| { + panic!( + "type object for {} not initialized", + std::any::type_name::() + ) + }) + } + }; + assert!(!type_obj.is_null(), "type object is NULL"); + debug_assert!( + unsafe { ffi::PyType_IsSubtype(ffi::Py_TYPE(obj), type_obj) } == 1, + "the object is not an instance of {}", + std::any::type_name::() + ); + let pointer = unsafe { ffi::PyObject_GetTypeData(obj, type_obj) }; + assert!(!pointer.is_null(), "pointer to pyclass data returned NULL"); + pointer.cast() + } + + #[inline(always)] + #[cfg(not(Py_3_12))] + pub fn panic_unsupported() -> ! { + assert!(T::OPAQUE); + panic!( + "The opaque object layout (used by {}) is not supported until python 3.12", + std::any::type_name::() + ); + } +} + +/// Utilities for working with [ffi::PyObject] objects that utilise the standard layout for python extensions, +/// where the base class is placed at the beginning of a `repr(C)` struct. +#[doc(hidden)] +pub(crate) mod static_layout { + use crate::{ + impl_::pyclass::{PyClassBaseType, PyClassImpl}, + type_object::{PyLayout, PySizedLayout}, + }; + + use super::PyClassObjectContents; + + // The layout of a [ffi::PyObject] that uses the static layout + #[repr(C)] + pub struct PyStaticClassLayout { + pub(crate) ob_base: ::StaticLayout, + pub(crate) contents: PyClassObjectContents, + } + + unsafe impl PyLayout for PyStaticClassLayout {} + + /// Layout of a native type `T` with a known size (not opaque) + /// Corresponds to [PyObject](https://docs.python.org/3/c-api/structures.html#c.PyObject) from the C API. + #[doc(hidden)] + #[repr(C)] + pub struct PyStaticNativeLayout { + ob_base: T, + } + + unsafe impl PyLayout for PyStaticNativeLayout where U: PySizedLayout {} + + /// a struct for use with opaque native types to indicate that they + /// cannot be used as part of a static layout. + #[repr(C)] + pub struct InvalidStaticLayout; + + /// This is valid insofar as casting a `*mut ffi::PyObject` to `*mut InvalidStaticLayout` is valid + /// since `InvalidStaticLayout` has no fields to read. + unsafe impl PyLayout for InvalidStaticLayout {} +} + +/// The method to use for obtaining a [ffi::PyTypeObject] pointer describing `T: PyTypeInfo` for +/// use with [PyObjectLayout] functions. +/// +/// [PyTypeInfo::type_object_raw()] requires the GIL to be held because it may lazily construct the type object. +/// Some situations require that the GIL is not held so [PyObjectLayout] cannot call this method directly. +/// The different solutions to this have different trade-offs. +#[derive(Clone, Copy)] +pub enum TypeObjectStrategy<'a> { + Lazy(Python<'a>), + AssumeInit(PhantomData<&'a ()>), +} + +impl<'a> TypeObjectStrategy<'a> { + /// Hold the GIL and only obtain/construct type objects lazily when required. + pub fn lazy(py: Python<'a>) -> Self { + TypeObjectStrategy::Lazy(py) + } + + /// Assume that [PyTypeInfo::type_object_raw()] has been called for any of the required type objects. + /// + /// Once initialized, the type objects are cached and can be obtained without holding the GIL. + /// + /// # Safety + /// + /// - Ensure that any `T` that may be used with this strategy has already been initialized + /// by calling [PyTypeInfo::type_object_raw()]. + /// - Only [PyTypeInfo::OPAQUE] classes require type objects for traversal so if this strategy is only + /// used with non-opaque classes then no action is required. + /// - When used with [PyClassRecursiveOperations] or [GetBorrowChecker], the strategy may be used with + /// base classes as well as the most derived type. + /// [PyClassRecursiveOperations::ensure_type_objects_initialized()] can be used to initialize + /// all base classes above the given type. + pub unsafe fn assume_init() -> Self { + TypeObjectStrategy::AssumeInit(PhantomData) + } +} + +/// Functions for working with [ffi::PyObject]s +pub(crate) struct PyObjectLayout {} + +impl PyObjectLayout { + /// Obtain a pointer to the portion of `obj` relating to the type `T` + /// + /// # Safety + /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_contents_ptr( + obj: *mut ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> *mut PyClassObjectContents { + debug_assert!(!obj.is_null(), "get_contents_ptr of null object"); + if T::OPAQUE { + #[cfg(Py_3_12)] + { + opaque_layout::get_contents_ptr(obj, strategy) + } + + #[cfg(not(Py_3_12))] + { + let _ = strategy; + opaque_layout::panic_unsupported::(); + } + } else { + let obj: *mut static_layout::PyStaticClassLayout = obj.cast(); + // indicates `ob_base` has type [static_layout::InvalidStaticLayout] + debug_assert_ne!( + offset_of!(static_layout::PyStaticClassLayout, contents), + 0, + "invalid ob_base found" + ); + addr_of_mut!((*obj).contents) + } + } + + /// Obtain a reference to the portion of `obj` relating to the type `T` + /// + /// # Safety + /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_contents<'a, T: PyClassImpl + PyTypeInfo>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a PyClassObjectContents { + &*cast_const(PyObjectLayout::get_contents_ptr::( + cast_mut(ptr_from_ref(obj)), + strategy, + )) + } + + /// Obtain a pointer to the portion of `obj` containing the user data for `T` + /// + /// # Safety + /// `obj` must point to a valid `PyObject` whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_data_ptr( + obj: *mut ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> *mut T { + let contents = PyObjectLayout::get_contents_ptr::(obj, strategy); + (*contents).value.get() + } + + /// Obtain a reference to the portion of `obj` containing the user data for `T` + /// + /// # Safety + /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_data<'a, T: PyClassImpl + PyTypeInfo>( + obj: &'a ffi::PyObject, + strategy: TypeObjectStrategy<'_>, + ) -> &'a T { + &*PyObjectLayout::get_data_ptr::(cast_mut(ptr_from_ref(obj)), strategy) + } + + /// Obtain a reference to the borrow checker for `obj` + /// + /// Note: this method is for convenience. The implementation is in [GetBorrowChecker]. + /// + /// # Safety + /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn get_borrow_checker<'a, T: PyClassImpl + PyTypeInfo>( + py: Python<'_>, + obj: &'a ffi::PyObject, + ) -> &'a ::Checker { + T::PyClassMutability::borrow_checker(obj, TypeObjectStrategy::lazy(py)) + } + + /// Ensure that `obj` is thread safe. + /// + /// Note: this method is for convenience. The implementation is in [PyClassRecursiveOperations]. + /// + /// # Safety + /// `obj` must point to a valid [ffi::PyObject] whose type is `T` or a subclass of `T`. + pub(crate) unsafe fn ensure_threadsafe( + py: Python<'_>, + obj: &ffi::PyObject, + ) { + PyClassRecursiveOperations::::ensure_threadsafe(obj, TypeObjectStrategy::lazy(py)); + } + + /// Clean up then free the memory associated with `obj`. + /// + /// Note: this method is for convenience. The implementation is in [PyClassRecursiveOperations]. + /// + /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + pub(crate) unsafe fn deallocate( + py: Python<'_>, + obj: *mut ffi::PyObject, + ) { + PyClassRecursiveOperations::::deallocate(py, obj); + } + + /// Clean up then free the memory associated with `obj`. + /// + /// Use instead of `deallocate()` if `T` has the `Py_TPFLAGS_HAVE_GC` flag set. + /// + /// See [tp_dealloc docs](https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_dealloc) + pub(crate) unsafe fn deallocate_with_gc( + py: Python<'_>, + obj: *mut ffi::PyObject, + ) { + #[cfg(not(PyPy))] + { + ffi::PyObject_GC_UnTrack(obj.cast()); + } + PyClassRecursiveOperations::::deallocate(py, obj); + } + + /// Used to set `PyType_Spec::basicsize` when creating a `PyTypeObject` for `T` + /// ([docs](https://docs.python.org/3/c-api/type.html#c.PyType_Spec.basicsize)) + pub(crate) fn basicsize() -> ffi::Py_ssize_t { + if T::OPAQUE { + #[cfg(Py_3_12)] + { + // negative to indicate 'extra' space that python will allocate + // specifically for `T` excluding the base class. + -usize_to_py_ssize(std::mem::size_of::>()) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported::(); + } else { + usize_to_py_ssize(std::mem::size_of::>()) + } + } + + /// Gets the offset of the contents from the start of the struct in bytes. + pub(crate) fn contents_offset() -> PyObjectOffset { + if T::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(0) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported::(); + } else { + PyObjectOffset::Absolute(usize_to_py_ssize(memoffset::offset_of!( + static_layout::PyStaticClassLayout, + contents + ))) + } + } + + /// Gets the offset of the dictionary from the start of the struct in bytes. + pub(crate) fn dict_offset() -> PyObjectOffset { + if T::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + dict + ))) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported::(); + } else { + let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + + memoffset::offset_of!(PyClassObjectContents, dict); + + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) + } + } + + /// Gets the offset of the weakref list from the start of the struct in bytes. + pub(crate) fn weaklist_offset() -> PyObjectOffset { + if T::OPAQUE { + #[cfg(Py_3_12)] + { + PyObjectOffset::Relative(usize_to_py_ssize(memoffset::offset_of!( + PyClassObjectContents, + weakref + ))) + } + + #[cfg(not(Py_3_12))] + opaque_layout::panic_unsupported::(); + } else { + let offset = memoffset::offset_of!(static_layout::PyStaticClassLayout, contents) + + memoffset::offset_of!(PyClassObjectContents, weakref); + + PyObjectOffset::Absolute(usize_to_py_ssize(offset)) + } + } +} + +/// Py_ssize_t may not be equal to isize on all platforms +pub(crate) fn usize_to_py_ssize(value: usize) -> ffi::Py_ssize_t { + #[allow(clippy::useless_conversion)] + value.try_into().expect("value should fit in Py_ssize_t") +} + +/// Tests specific to the static layout +#[cfg(all(test, feature = "macros"))] +#[allow(clippy::bool_comparison)] // `== false` is harder to miss than ! +mod static_tests { + use static_assertions::const_assert; + + #[cfg(not(Py_LIMITED_API))] + use super::test_utils::get_pyobject_size; + use super::*; + + use crate::prelude::*; + use memoffset::offset_of; + use std::mem::size_of; + + /// Test the functions calculate properties about the static layout without requiring an instance. + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_type_properties_no_inheritance() { + #[pyclass(crate = "crate", extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + /// typically called `ob_base`. In C it is defined using the `PyObject_HEAD` macro + /// [docs](https://docs.python.org/3/c-api/structures.html) + native_base: ffi::PyObject, + contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + let expected_contents_offset = offset_of!(ExpectedLayout, contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_contents_offset), + ); + + let dict_size = size_of::<::Dict>(); + assert_eq!(dict_size, 0); + let expected_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Absolute(expected_contents_offset + expected_dict_offset_in_contents), + ); + + let weakref_size = size_of::<::WeakRef>(); + assert_eq!(weakref_size, 0); + let expected_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Absolute( + expected_contents_offset + expected_weakref_offset_in_contents + ), + ); + + assert_eq!( + expected_dict_offset_in_contents, + expected_weakref_offset_in_contents + ); + } + + /// Test the functions calculate properties about the static layout without requiring an instance. + /// The class in this test requires extra space for the `dict` and `weaklist` fields + #[test] + #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] + fn test_layout_properties_no_inheritance_optional_fields() { + #[pyclass(crate = "crate", dict, weakref, extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyObject, + contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + let expected_contents_offset = offset_of!(ExpectedLayout, contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_contents_offset), + ); + + let dict_size = size_of::<::Dict>(); + assert!(dict_size > 0); + let expected_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Absolute(expected_contents_offset + expected_dict_offset_in_contents), + ); + + let weakref_size = size_of::<::WeakRef>(); + assert!(weakref_size > 0); + let expected_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Absolute( + expected_contents_offset + expected_weakref_offset_in_contents + ), + ); + + assert!(expected_dict_offset_in_contents < expected_weakref_offset_in_contents); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_type_properties_with_inheritance() { + use std::any::TypeId; + + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_field: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_field: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyDictObject, + parent_contents: PyClassObjectContents, + child_contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + Python::with_gil(|py| { + let typ_size = get_pyobject_size::(py) as isize; + assert_eq!(typ_size, expected_size); + }); + + let expected_parent_contents_offset = + offset_of!(ExpectedLayout, parent_contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_parent_contents_offset), + ); + + let expected_child_contents_offset = + offset_of!(ExpectedLayout, child_contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_child_contents_offset), + ); + + let child_dict_size = size_of::<::Dict>(); + assert_eq!(child_dict_size, 0); + let expected_child_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Absolute( + expected_child_contents_offset + expected_child_dict_offset_in_contents + ), + ); + + let child_weakref_size = size_of::<::WeakRef>(); + assert_eq!(child_weakref_size, 0); + let expected_child_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Absolute( + expected_child_contents_offset + expected_child_weakref_offset_in_contents + ), + ); + } + + /// Test the functions that operate on pyclass instances + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_contents_access_no_inheritance() { + #[pyclass(crate = "crate", extends=PyAny)] + struct MyClass { + my_value: u64, + } + + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyObject, + contents: PyClassObjectContents, + } + + Python::with_gil(|py| { + let obj = Py::new(py, MyClass { my_value: 123 }).unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let contents_ptr_int = contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let contents_ptr = contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(contents_ptr, contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let casted_obj = obj_ptr.cast::(); + let expected_contents_ptr = unsafe { addr_of_mut!((*casted_obj).contents) }; + assert_eq!(contents_ptr, expected_contents_ptr); + + // test getting contents by reference + let contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(contents), expected_contents_ptr); + + // test getting data pointer + let data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_data_ptr = unsafe { (*expected_contents_ptr).value.get() }; + assert_eq!(data_ptr, expected_data_ptr); + assert_eq!(unsafe { (*data_ptr).my_value }, 123); + + // test getting data by reference + let data = unsafe { + PyObjectLayout::get_data::(obj.as_raw_ref(), TypeObjectStrategy::lazy(py)) + }; + assert_eq!(ptr_from_ref(data), expected_data_ptr); + }); + } + + /// Test the functions that operate on pyclass instances. + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_contents_access_with_inheritance() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + parent_value: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + child_value: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayout { + native_base: ffi::PyDictObject, + parent_contents: PyClassObjectContents, + child_contents: PyClassObjectContents, + } + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass { parent_value: 123 }).add_subclass( + ChildClass { + child_value: "foo".to_owned(), + }, + ), + ) + .unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let parent_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let parent_contents_ptr_int = parent_contents_ptr as usize; + let child_contents_ptr_int = child_contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let parent_contents_ptr = + parent_contents_ptr_int as *mut PyClassObjectContents; + let child_contents_ptr = + child_contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let parent_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + let child_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(parent_contents_ptr, parent_contents_ptr_without_gil); + assert_eq!(child_contents_ptr, child_contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let casted_obj = obj_ptr.cast::(); + let expected_parent_contents_ptr = + unsafe { addr_of_mut!((*casted_obj).parent_contents) }; + let expected_child_contents_ptr = unsafe { addr_of_mut!((*casted_obj).child_contents) }; + assert_eq!(parent_contents_ptr, expected_parent_contents_ptr); + assert_eq!(child_contents_ptr, expected_child_contents_ptr); + + // test getting contents by reference + let parent_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_contents), expected_parent_contents_ptr); + assert_eq!(ptr_from_ref(child_contents), expected_child_contents_ptr); + + // test getting data pointer + let parent_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let child_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_parent_data_ptr = unsafe { (*expected_parent_contents_ptr).value.get() }; + let expected_child_data_ptr = unsafe { (*expected_child_contents_ptr).value.get() }; + assert_eq!(parent_data_ptr, expected_parent_data_ptr); + assert_eq!(unsafe { (*parent_data_ptr).parent_value }, 123); + assert_eq!(child_data_ptr, expected_child_data_ptr); + assert_eq!(unsafe { &(*child_data_ptr).child_value }, "foo"); + + // test getting data by reference + let parent_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_data), expected_parent_data_ptr); + assert_eq!(ptr_from_ref(child_data), expected_child_data_ptr); + }); + } + + #[test] + fn test_inherited_size() { + #[pyclass(crate = "crate", subclass)] + struct ParentClass; + + #[pyclass(crate = "crate", extends = ParentClass)] + struct ChildClass(#[allow(unused)] u64); + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + + #[repr(C)] + struct ExpectedLayoutWithData { + native_base: ffi::PyObject, + parent_class: PyClassObjectContents, + child_class: PyClassObjectContents, + } + let expected_size = std::mem::size_of::() as ffi::Py_ssize_t; + + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + } + + /// Test that `Drop::drop` is called for pyclasses + #[test] + fn test_destructor_called() { + use std::sync::{Arc, Mutex}; + + let deallocations: Arc>> = Arc::new(Mutex::new(Vec::new())); + + #[pyclass(crate = "crate", subclass)] + struct ParentClass(Arc>>); + + impl Drop for ParentClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ParentClass".to_owned()); + } + } + + #[pyclass(crate = "crate", extends = ParentClass)] + struct ChildClass(Arc>>); + + impl Drop for ChildClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ChildClass".to_owned()); + } + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == false); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass(deallocations.clone())) + .add_subclass(ChildClass(deallocations.clone())), + ) + .unwrap(); + assert!(deallocations.lock().unwrap().is_empty()); + drop(obj); + }); + + assert_eq!( + deallocations.lock().unwrap().as_slice(), + &["ChildClass", "ParentClass"] + ); + } + + #[test] + fn test_empty_class() { + #[pyclass(crate = "crate")] + struct EmptyClass; + + // even if the user class has no data some additional space is required + assert!(size_of::>() > 0); + } + + /// It is essential that `InvalidStaticLayout` has 0 size so that it can be distinguished from a valid layout + #[test] + fn test_invalid_base() { + assert_eq!(std::mem::size_of::(), 0); + + #[repr(C)] + struct InvalidLayout { + ob_base: static_layout::InvalidStaticLayout, + contents: u8, + } + + assert_eq!(offset_of!(InvalidLayout, contents), 0); + } +} + +/// Tests specific to the opaque layout +#[cfg(all(test, Py_3_12, feature = "macros"))] +#[allow(clippy::bool_comparison)] // `== false` is harder to miss than ! +mod opaque_tests { + use memoffset::offset_of; + use static_assertions::const_assert; + use std::mem::size_of; + use std::ops::Range; + + #[cfg(not(Py_LIMITED_API))] + use super::test_utils::get_pyobject_size; + use super::*; + + use crate::{prelude::*, PyClass}; + + /// Check that all the type properties are as expected for the given class `T`. + /// Unlike the static layout, the properties of a type in the opaque layout are + /// derived entirely from `T`, not the whole [ffi::PyObject]. + fn check_opaque_type_properties(has_dict: bool, has_weakref: bool) { + let contents_size = size_of::>() as ffi::Py_ssize_t; + // negative indicates 'in addition to the base class' + assert!(PyObjectLayout::basicsize::() == -contents_size); + + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Relative(0), + ); + + let dict_size = size_of::<::Dict>(); + if has_dict { + assert!(dict_size > 0); + } else { + assert_eq!(dict_size, 0); + } + let expected_dict_offset_in_contents = + offset_of!(PyClassObjectContents, dict) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::dict_offset::(), + PyObjectOffset::Relative(expected_dict_offset_in_contents), + ); + + let weakref_size = size_of::<::WeakRef>(); + if has_weakref { + assert!(weakref_size > 0); + } else { + assert_eq!(weakref_size, 0); + } + let expected_weakref_offset_in_contents = + offset_of!(PyClassObjectContents, weakref) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::weaklist_offset::(), + PyObjectOffset::Relative(expected_weakref_offset_in_contents), + ); + + if has_dict { + assert!(expected_dict_offset_in_contents < expected_weakref_offset_in_contents); + } else { + assert_eq!( + expected_dict_offset_in_contents, + expected_weakref_offset_in_contents + ); + } + } + + /// Test the functions calculate properties about the opaque layout without requiring an instance. + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_type_properties_no_inheritance() { + #[pyclass(crate = "crate", opaque, extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + const_assert!(::OPAQUE == true); + + check_opaque_type_properties::(false, false); + } + + /// Test the functions calculate properties about the opaque layout without requiring an instance. + /// The class in this test requires extra space for the `dict` and `weaklist` fields + #[test] + fn test_layout_properties_no_inheritance_optional_fields() { + #[pyclass(crate = "crate", dict, weakref, opaque, extends=PyAny)] + struct MyClass(#[allow(unused)] u64); + const_assert!(::OPAQUE == true); + + check_opaque_type_properties::(true, true); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_type_properties_with_inheritance_opaque_base() { + use std::any::TypeId; + + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, opaque, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_field: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_field: String, + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + + check_opaque_type_properties::(false, false); + check_opaque_type_properties::(false, false); + } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_type_properties_with_inheritance_static_base() { + use std::any::TypeId; + + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_field: u64, + } + + #[pyclass(crate = "crate", opaque, extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_field: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == true); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + assert_eq!( + TypeId::of::<::BaseType>(), + TypeId::of::() + ); + + check_opaque_type_properties::(false, false); + + #[repr(C)] + struct ParentExpectedLayout { + native_base: ffi::PyDictObject, + parent_contents: PyClassObjectContents, + } + + let expected_size = size_of::() as ffi::Py_ssize_t; + assert_eq!(PyObjectLayout::basicsize::(), expected_size); + + let expected_parent_contents_offset = + offset_of!(ParentExpectedLayout, parent_contents) as ffi::Py_ssize_t; + assert_eq!( + PyObjectLayout::contents_offset::(), + PyObjectOffset::Absolute(expected_parent_contents_offset), + ); + } + + /// Test the functions that operate on pyclass instances + /// The class in this test extends the default base class `PyAny` so there is 'no inheritance'. + #[test] + fn test_contents_access_no_inheritance() { + #[pyclass(crate = "crate", opaque, extends=PyAny)] + struct MyClass { + my_value: u64, + } + + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new(py, MyClass { my_value: 123 }).unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let contents_ptr_int = contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let contents_ptr = contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(contents_ptr, contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + // the `MyClass` data has to be between the base type and the end of the PyObject. + #[cfg(not(Py_LIMITED_API))] + { + let pyobject_size = get_pyobject_size::(py); + let contents_range = bytes_range( + contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(contents_range.start >= size_of::()); + assert!(contents_range.end <= pyobject_size); + } + + // test getting contents by reference + let contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(contents), contents_ptr); + + // test getting data pointer + let data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_data_ptr = unsafe { (*contents_ptr).value.get() }; + assert_eq!(data_ptr, expected_data_ptr); + assert_eq!(unsafe { (*data_ptr).my_value }, 123); + + // test getting data by reference + let data = unsafe { + PyObjectLayout::get_data::(obj.as_raw_ref(), TypeObjectStrategy::lazy(py)) + }; + assert_eq!(ptr_from_ref(data), expected_data_ptr); + }); + } + + /// Test the functions that operate on pyclass instances. + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_contents_access_with_inheritance_opaque_base() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, opaque, extends=PyDict)] + struct ParentClass { + parent_value: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + child_value: String, + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass { parent_value: 123 }).add_subclass( + ChildClass { + child_value: "foo".to_owned(), + }, + ), + ) + .unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let parent_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let parent_contents_ptr_int = parent_contents_ptr as usize; + let child_contents_ptr_int = child_contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let parent_contents_ptr = + parent_contents_ptr_int as *mut PyClassObjectContents; + let child_contents_ptr = + child_contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let parent_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + let child_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(parent_contents_ptr, parent_contents_ptr_without_gil); + assert_eq!(child_contents_ptr, child_contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let parent_pyobject_size = get_pyobject_size::(py); + let child_pyobject_size = get_pyobject_size::(py); + assert!(child_pyobject_size > parent_pyobject_size); + let parent_range = bytes_range( + parent_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + let child_range = bytes_range( + child_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(parent_range.start >= size_of::()); + assert!(parent_range.end <= parent_pyobject_size); + assert!(child_range.start >= parent_range.end); + assert!(child_range.end <= child_pyobject_size); + + // test getting contents by reference + let parent_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_contents), parent_contents_ptr); + assert_eq!(ptr_from_ref(child_contents), child_contents_ptr); + + // test getting data pointer + let parent_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let child_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_parent_data_ptr = unsafe { (*parent_contents_ptr).value.get() }; + let expected_child_data_ptr = unsafe { (*child_contents_ptr).value.get() }; + assert_eq!(parent_data_ptr, expected_parent_data_ptr); + assert_eq!(unsafe { (*parent_data_ptr).parent_value }, 123); + assert_eq!(child_data_ptr, expected_child_data_ptr); + assert_eq!(unsafe { &(*child_data_ptr).child_value }, "foo"); + + // test getting data by reference + let parent_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_data), expected_parent_data_ptr); + assert_eq!(ptr_from_ref(child_data), expected_child_data_ptr); + }); + } + + /// Test the functions that operate on pyclass instances. + #[test] + #[cfg(not(Py_LIMITED_API))] + fn test_contents_access_with_inheritance_static_base() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, extends=PyDict)] + struct ParentClass { + parent_value: u64, + } + + #[pyclass(crate = "crate", opaque, extends=ParentClass)] + struct ChildClass { + child_value: String, + } + + const_assert!(::OPAQUE == false); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass { parent_value: 123 }).add_subclass( + ChildClass { + child_value: "foo".to_owned(), + }, + ), + ) + .unwrap(); + let obj_ptr = obj.as_ptr(); + + // test obtaining contents pointer normally (with GIL held) + let parent_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents_ptr = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + + // work around the fact that pointers are not `Send` + let obj_ptr_int = obj_ptr as usize; + let parent_contents_ptr_int = parent_contents_ptr as usize; + let child_contents_ptr_int = child_contents_ptr as usize; + + // test that the contents pointer can be obtained without the GIL held + py.allow_threads(move || { + let obj_ptr = obj_ptr_int as *mut ffi::PyObject; + let parent_contents_ptr = + parent_contents_ptr_int as *mut PyClassObjectContents; + let child_contents_ptr = + child_contents_ptr_int as *mut PyClassObjectContents; + + // Safety: type object was created when `obj` was constructed + let parent_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + let child_contents_ptr_without_gil = unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::assume_init(), + ) + }; + assert_eq!(parent_contents_ptr, parent_contents_ptr_without_gil); + assert_eq!(child_contents_ptr, child_contents_ptr_without_gil); + }); + + // test that contents pointer matches expecations + let parent_pyobject_size = get_pyobject_size::(py); + let child_pyobject_size = get_pyobject_size::(py); + assert!(child_pyobject_size > parent_pyobject_size); + let parent_range = bytes_range( + parent_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + let child_range = bytes_range( + child_contents_ptr_int - obj_ptr_int, + size_of::>(), + ); + assert!(parent_range.start >= size_of::()); + assert!(parent_range.end <= parent_pyobject_size); + assert!(child_range.start >= parent_range.end); + assert!(child_range.end <= child_pyobject_size); + + // test getting contents by reference + let parent_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_contents = unsafe { + PyObjectLayout::get_contents::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_contents), parent_contents_ptr); + assert_eq!(ptr_from_ref(child_contents), child_contents_ptr); + + // test getting data pointer + let parent_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let child_data_ptr = unsafe { + PyObjectLayout::get_data_ptr::(obj_ptr, TypeObjectStrategy::lazy(py)) + }; + let expected_parent_data_ptr = unsafe { (*parent_contents_ptr).value.get() }; + let expected_child_data_ptr = unsafe { (*child_contents_ptr).value.get() }; + assert_eq!(parent_data_ptr, expected_parent_data_ptr); + assert_eq!(unsafe { (*parent_data_ptr).parent_value }, 123); + assert_eq!(child_data_ptr, expected_child_data_ptr); + assert_eq!(unsafe { &(*child_data_ptr).child_value }, "foo"); + + // test getting data by reference + let parent_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + let child_data = unsafe { + PyObjectLayout::get_data::( + obj.as_raw_ref(), + TypeObjectStrategy::lazy(py), + ) + }; + assert_eq!(ptr_from_ref(parent_data), expected_parent_data_ptr); + assert_eq!(ptr_from_ref(child_data), expected_child_data_ptr); + }); + } + + /// Test that `Drop::drop` is called for pyclasses + #[test] + fn test_destructor_called() { + use std::sync::{Arc, Mutex}; + + let deallocations: Arc>> = Arc::new(Mutex::new(Vec::new())); + + #[pyclass(crate = "crate", subclass, opaque)] + struct ParentClass(Arc>>); + + impl Drop for ParentClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ParentClass".to_owned()); + } + } + + #[pyclass(crate = "crate", extends = ParentClass)] + struct ChildClass(Arc>>); + + impl Drop for ChildClass { + fn drop(&mut self) { + self.0.lock().unwrap().push("ChildClass".to_owned()); + } + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new( + py, + PyClassInitializer::from(ParentClass(deallocations.clone())) + .add_subclass(ChildClass(deallocations.clone())), + ) + .unwrap(); + assert!(deallocations.lock().unwrap().is_empty()); + drop(obj); + }); + + assert_eq!( + deallocations.lock().unwrap().as_slice(), + &["ChildClass", "ParentClass"] + ); + } + + #[test] + #[should_panic(expected = "OpaqueClassNeverConstructed not initialized")] + fn test_panic_when_incorrectly_assume_initialized() { + #[pyclass(crate = "crate", opaque)] + struct OpaqueClassNeverConstructed; + + const_assert!(::OPAQUE); + + let obj = Python::with_gil(|py| py.None()); + + assert!(OpaqueClassNeverConstructed::try_get_type_object_raw().is_none()); + unsafe { + PyObjectLayout::get_contents_ptr::( + obj.as_ptr(), + TypeObjectStrategy::assume_init(), + ); + } + } + + #[test] + #[cfg(all(debug_assertions, not(Py_LIMITED_API)))] + #[should_panic(expected = "the object is not an instance of")] + fn test_panic_when_incorrect_type() { + use crate::types::PyDict; + + #[pyclass(crate = "crate", subclass, opaque, extends=PyDict)] + struct ParentClass { + #[allow(unused)] + parent_value: u64, + } + + #[pyclass(crate = "crate", extends=ParentClass)] + struct ChildClass { + #[allow(unused)] + child_value: String, + } + + const_assert!(::OPAQUE == true); + const_assert!(::OPAQUE == true); + + Python::with_gil(|py| { + let obj = Py::new(py, ParentClass { parent_value: 123 }).unwrap(); + let obj_ptr = obj.as_ptr(); + + unsafe { + PyObjectLayout::get_contents_ptr::( + obj_ptr, + TypeObjectStrategy::lazy(py), + ) + }; + }); + } + + /// Create a range from a start and size instead of a start and end + #[allow(unused)] + fn bytes_range(start: usize, size: usize) -> Range { + Range { + start, + end: start + size, + } + } +} + +#[cfg(all(test, not(Py_3_12), feature = "macros"))] +mod opaque_fail_tests { + use crate::{ + prelude::*, + types::{PyDict, PyTuple, PyType}, + PyTypeInfo, + }; + + #[pyclass(crate = "crate", extends=PyType)] + #[derive(Default)] + struct Metaclass; + + #[pymethods(crate = "crate")] + impl Metaclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__(&mut self, _args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) {} + } + + /// PyType uses the opaque layout. While explicitly using `#[pyclass(opaque)]` can be caught at compile time, + /// it is also possible to create a pyclass that uses the opaque layout by extending an opaque native type. + #[test] + #[should_panic( + expected = "The opaque object layout (used by pyo3::pycell::layout::opaque_fail_tests::Metaclass) is not supported until python 3.12" + )] + fn test_panic_at_construction_inherit_opaque() { + Python::with_gil(|py| { + Py::new(py, Metaclass).unwrap(); + }); + } + + #[test] + #[should_panic( + expected = "The opaque object layout (used by pyo3::pycell::layout::opaque_fail_tests::Metaclass) is not supported until python 3.12" + )] + fn test_panic_at_type_construction_inherit_opaque() { + Python::with_gil(|py| { + ::type_object(py); + }); + } +} + +#[cfg(test)] +mod test_utils { + #[cfg(not(Py_LIMITED_API))] + use crate::{ffi, PyClass, PyTypeInfo, Python}; + + /// The size in bytes of a [ffi::PyObject] of the type `T` + #[cfg(not(Py_LIMITED_API))] + #[allow(unused)] + pub fn get_pyobject_size(py: Python<'_>) -> usize { + let typ = ::type_object(py); + let raw_typ = typ.as_ptr().cast::(); + let size = unsafe { (*raw_typ).tp_basicsize }; + usize::try_from(size).expect("size should be a valid usize") + } +} diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 8a02baa8ad1..9ae504bbe09 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -2,15 +2,15 @@ use crate::{ exceptions::PyTypeError, ffi, impl_::{ - pycell::PyClassObject, pyclass::{ assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, - tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassItemsIter, + tp_dealloc_with_gc, MaybeRuntimePyMethodDef, PyClassItemsIter, PyObjectOffset, }, pymethods::{Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter, _call_clear}, trampoline::trampoline, }, internal_tricks::ptr_from_ref, + pycell::layout::PyObjectLayout, types::{typeobject::PyTypeMethods, PyType}, Py, PyClass, PyResult, PyTypeInfo, Python, }; @@ -18,7 +18,7 @@ use std::{ collections::HashMap, ffi::{CStr, CString}, os::raw::{c_char, c_int, c_ulong, c_void}, - ptr, + ptr::{self, addr_of_mut}, }; pub(crate) struct PyClassTypeObject { @@ -41,13 +41,13 @@ where is_mapping: bool, is_sequence: bool, doc: &'static CStr, - dict_offset: Option, - weaklist_offset: Option, + dict_offset: Option, + weaklist_offset: Option, is_basetype: bool, items_iter: PyClassItemsIter, name: &'static str, module: Option<&'static str>, - size_of: usize, + basicsize: ffi::Py_ssize_t, ) -> PyResult { PyTypeBuilder { slots: Vec::new(), @@ -61,6 +61,7 @@ where is_mapping, is_sequence, has_new: false, + has_init: false, has_dealloc: false, has_getitem: false, has_setitem: false, @@ -75,7 +76,7 @@ where .offsets(dict_offset, weaklist_offset) .set_is_basetype(is_basetype) .class_items(items_iter) - .build(py, name, module, size_of) + .build(py, name, module, basicsize) } unsafe { @@ -93,7 +94,7 @@ where T::items_iter(), T::NAME, T::MODULE, - std::mem::size_of::>(), + PyObjectLayout::basicsize::(), ) } } @@ -115,12 +116,13 @@ struct PyTypeBuilder { is_mapping: bool, is_sequence: bool, has_new: bool, + has_init: bool, has_dealloc: bool, has_getitem: bool, has_setitem: bool, has_traverse: bool, has_clear: bool, - dict_offset: Option, + dict_offset: Option, class_flags: c_ulong, // Before Python 3.9, need to patch in buffer methods manually (they don't work in slots) #[cfg(all(not(Py_3_9), not(Py_LIMITED_API)))] @@ -133,6 +135,7 @@ impl PyTypeBuilder { unsafe fn push_slot(&mut self, slot: c_int, pfunc: *mut T) { match slot { ffi::Py_tp_new => self.has_new = true, + ffi::Py_tp_init => self.has_init = true, ffi::Py_tp_dealloc => self.has_dealloc = true, ffi::Py_mp_subscript => self.has_getitem = true, ffi::Py_mp_ass_subscript => self.has_setitem = true, @@ -256,7 +259,11 @@ impl PyTypeBuilder { } get_dict = get_dict_impl; - closure = dict_offset as _; + if let PyObjectOffset::Absolute(offset) = dict_offset { + closure = offset as _; + } else { + unreachable!("PyObjectOffset::Relative requires >=3.12"); + } } property_defs.push(ffi::PyGetSetDef { @@ -357,20 +364,31 @@ impl PyTypeBuilder { fn offsets( mut self, - dict_offset: Option, - #[allow(unused_variables)] weaklist_offset: Option, + dict_offset: Option, + #[allow(unused_variables)] weaklist_offset: Option, ) -> Self { self.dict_offset = dict_offset; #[cfg(Py_3_9)] { #[inline(always)] - fn offset_def(name: &'static CStr, offset: ffi::Py_ssize_t) -> ffi::PyMemberDef { + fn offset_def(name: &'static CStr, offset: PyObjectOffset) -> ffi::PyMemberDef { + let (offset, flags) = match offset { + PyObjectOffset::Absolute(offset) => (offset, ffi::Py_READONLY), + #[cfg(Py_3_12)] + PyObjectOffset::Relative(offset) => { + (offset, ffi::Py_READONLY | ffi::Py_RELATIVE_OFFSET) + } + #[cfg(not(Py_3_12))] + PyObjectOffset::Relative(_) => { + panic!("relative offsets not valid before python 3.12"); + } + }; ffi::PyMemberDef { name: name.as_ptr().cast(), type_code: ffi::Py_T_PYSSIZET, offset, - flags: ffi::Py_READONLY, + flags, doc: std::ptr::null_mut(), } } @@ -400,12 +418,23 @@ impl PyTypeBuilder { (*(*type_object).tp_as_buffer).bf_releasebuffer = builder.buffer_procs.bf_releasebuffer; - if let Some(dict_offset) = dict_offset { - (*type_object).tp_dictoffset = dict_offset; + match dict_offset { + Some(PyObjectOffset::Absolute(offset)) => { + (*type_object).tp_dictoffset = offset; + } + Some(PyObjectOffset::Relative(_)) => { + panic!("PyObjectOffset::Relative requires >=3.12") + } + None => {} } - - if let Some(weaklist_offset) = weaklist_offset { - (*type_object).tp_weaklistoffset = weaklist_offset; + match weaklist_offset { + Some(PyObjectOffset::Absolute(offset)) => { + (*type_object).tp_weaklistoffset = offset; + } + Some(PyObjectOffset::Relative(_)) => { + panic!("PyObjectOffset::Relative requires >=3.12") + } + None => {} } })); } @@ -417,7 +446,7 @@ impl PyTypeBuilder { py: Python<'_>, name: &'static str, module_name: Option<&'static str>, - basicsize: usize, + basicsize: ffi::Py_ssize_t, ) -> PyResult { // `c_ulong` and `c_uint` have the same size // on some platforms (like windows) @@ -427,8 +456,21 @@ impl PyTypeBuilder { unsafe { self.push_slot(ffi::Py_tp_base, self.tp_base) } - if !self.has_new { - // Safety: This is the correct slot type for Py_tp_new + // Safety: self.tp_base must be a valid PyTypeObject + let is_metaclass = + unsafe { ffi::PyType_IsSubtype(self.tp_base, addr_of_mut!(ffi::PyType_Type)) } != 0; + if is_metaclass { + // if the pyclass derives from `type` (is a metaclass) then `tp_new` must not be set. + // Metaclasses that override tp_new are not supported. + // https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass + assert!( + !self.has_new, + "Metaclasses must not specify __new__ (use __init__ instead)" + ); + // To avoid uninitialized memory, __init__ must be defined instead + assert!(self.has_init, "Metaclasses must specify __init__"); + } else if !self.has_new { + // Safety: The default constructor is a valid value of tp_new unsafe { self.push_slot(ffi::Py_tp_new, no_constructor_defined as *mut c_void) } } diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 6dc6ec12c6b..d936c995ac0 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,19 +1,13 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::ffi_ptr_ext::FfiPtrExt; use crate::impl_::callback::IntoPyCallbackOutput; -use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; +use crate::impl_::pyclass::PyClassBaseType; use crate::impl_::pyclass_init::{PyNativeTypeInitializer, PyObjectInit}; +use crate::pycell::layout::{PyObjectLayout, TypeObjectStrategy}; use crate::types::PyAnyMethods; use crate::{ffi, Bound, Py, PyClass, PyResult, Python}; -use crate::{ - ffi::PyTypeObject, - pycell::impl_::{PyClassBorrowChecker, PyClassMutability, PyClassObjectContents}, -}; -use std::{ - cell::UnsafeCell, - marker::PhantomData, - mem::{ManuallyDrop, MaybeUninit}, -}; +use crate::{ffi::PyTypeObject, pycell::layout::PyClassObjectContents}; +use std::marker::PhantomData; /// Initializer for our `#[pyclass]` system. /// @@ -165,14 +159,6 @@ impl PyClassInitializer { where T: PyClass, { - /// Layout of a PyClassObject after base new has been called, but the contents have not yet been - /// written. - #[repr(C)] - struct PartiallyInitializedClassObject { - _ob_base: ::LayoutAsBase, - contents: MaybeUninit>, - } - let (init, super_init) = match self.0 { PyClassInitializerImpl::Existing(value) => return Ok(value.into_bound(py)), PyClassInitializerImpl::New { init, super_init } => (init, super_init), @@ -180,16 +166,9 @@ impl PyClassInitializer { let obj = super_init.into_new_object(py, target_type)?; - let part_init: *mut PartiallyInitializedClassObject = obj.cast(); std::ptr::write( - (*part_init).contents.as_mut_ptr(), - PyClassObjectContents { - value: ManuallyDrop::new(UnsafeCell::new(init)), - borrow_checker: ::Storage::new(), - thread_checker: T::ThreadChecker::new(), - dict: T::Dict::INIT, - weakref: T::WeakRef::INIT, - }, + PyObjectLayout::get_contents_ptr::(obj, TypeObjectStrategy::lazy(py)), + PyClassObjectContents::new(init), ); // Safety: obj is a valid pointer to an object of type `target_type`, which` is a known diff --git a/src/sync.rs b/src/sync.rs index 0845eaf8cec..5b9ce826d98 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -179,6 +179,17 @@ impl GILOnceCell { } } + /// Get a pointer to the contained value, or `None` if the cell has not yet been written. + #[inline] + pub fn get_raw(&self) -> Option<*const T> { + if self.once.is_completed() { + // SAFETY: the cell has been written. + Some(unsafe { (*self.data.get()).as_ptr() }) + } else { + None + } + } + /// Get a reference to the contained value, initializing it if needed using the provided /// closure. /// diff --git a/src/type_object.rs b/src/type_object.rs index b7cad4ab3b2..eacf9e04cf0 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -5,9 +5,18 @@ use crate::types::any::PyAnyMethods; use crate::types::{PyAny, PyType}; use crate::{ffi, Bound, Python}; +/// `T: PyNativeType` represents that `T` is a struct representing a 'native python class'. +/// a 'native class' is a wrapper around a [ffi::PyTypeObject] that is defined by the python +/// API such as `PyDict` for `dict`. +/// +/// This trait is intended to be used internally. +/// +/// # Safety +/// +/// This trait must only be implemented for types which represent native python classes. +pub unsafe trait PyNativeType {} + /// `T: PyLayout` represents that `T` is a concrete representation of `U` in the Python heap. -/// E.g., `PyClassObject` is a concrete representation of all `pyclass`es, and `ffi::PyObject` -/// is of `PyAny`. /// /// This trait is intended to be used internally. /// @@ -27,14 +36,14 @@ pub trait PySizedLayout: PyLayout + Sized {} /// /// This trait is marked unsafe because: /// - specifying the incorrect layout can lead to memory errors -/// - the return value of type_object must always point to the same PyTypeObject instance +/// - the return value of type_object must always point to the same `PyTypeObject` instance /// /// It is safely implemented by the `pyclass` macro. /// /// # Safety /// -/// Implementations must provide an implementation for `type_object_raw` which infallibly produces a -/// non-null pointer to the corresponding Python type object. +/// Implementations must return the correct non-null `PyTypeObject` pointer corresponding to the type of `Self` +/// from `type_object_raw` and `try_get_type_object_raw`. pub unsafe trait PyTypeInfo: Sized { /// Class name. const NAME: &'static str; @@ -42,9 +51,22 @@ pub unsafe trait PyTypeInfo: Sized { /// Module name, if any. const MODULE: Option<&'static str>; - /// Returns the PyTypeObject instance for this type. + /// Whether classes that extend from this type must use the 'opaque type' extension mechanism + /// rather than using the standard mechanism of placing the data for this type at the end + /// of a new `repr(C)` struct + const OPAQUE: bool; + + /// Returns the [ffi::PyTypeObject] instance for this type. fn type_object_raw(py: Python<'_>) -> *mut ffi::PyTypeObject; + /// Returns the [ffi::PyTypeObject] instance for this type if it is known statically or has already + /// been initialized (by calling [PyTypeInfo::type_object_raw()]). + /// + /// # Safety + /// - It is valid to always return Some. + /// - It is not valid to return None once [PyTypeInfo::type_object_raw()] has been called. + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject>; + /// Returns the safe abstraction over the type object. #[inline] fn type_object(py: Python<'_>) -> Bound<'_, PyType> { diff --git a/src/types/any.rs b/src/types/any.rs index d060c187631..4e4e1b6e216 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -39,18 +39,20 @@ fn PyObject_Check(_: *mut ffi::PyObject) -> c_int { pyobject_native_type_info!( PyAny, - pyobject_native_static_type_object!(ffi::PyBaseObject_Type), Some("builtins"), + false, #checkfunction=PyObject_Check ); - +pyobject_native_type_object_methods!(PyAny, #global=ffi::PyBaseObject_Type); +pyobject_native_type_marker!(PyAny); pyobject_native_type_sized!(PyAny, ffi::PyObject); // We cannot use `pyobject_subclassable_native_type!()` because it cfgs out on `Py_LIMITED_API`. impl crate::impl_::pyclass::PyClassBaseType for PyAny { - type LayoutAsBase = crate::impl_::pycell::PyClassObjectBase; + type StaticLayout = crate::impl_::pycell::PyStaticNativeLayout; type BaseNativeType = PyAny; + type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; - type PyClassMutability = crate::pycell::impl_::ImmutableClass; + type PyClassMutability = crate::pycell::borrow_checker::ImmutableClass; } /// This trait represents the Python APIs which are usable on all Python objects. diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 53043fa798c..c67d4a2e146 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -22,7 +22,8 @@ use std::convert::Infallible; #[repr(transparent)] pub struct PyBool(PyAny); -pyobject_native_type!(PyBool, ffi::PyObject, pyobject_native_static_type_object!(ffi::PyBool_Type), #checkfunction=ffi::PyBool_Check); +pyobject_native_type!(PyBool, ffi::PyObject, #checkfunction=ffi::PyBool_Check); +pyobject_native_type_object_methods!(PyBool, #global=ffi::PyBool_Type); impl PyBool { /// Depending on `val`, returns `true` or `false`. diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index d1bbd0ac7e4..d7fe64b3b8a 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -16,7 +16,8 @@ use std::slice; #[repr(transparent)] pub struct PyByteArray(PyAny); -pyobject_native_type_core!(PyByteArray, pyobject_native_static_type_object!(ffi::PyByteArray_Type), #checkfunction=ffi::PyByteArray_Check); +pyobject_native_type_core!(PyByteArray, #checkfunction=ffi::PyByteArray_Check); +pyobject_native_type_object_methods!(PyByteArray, #global=ffi::PyByteArray_Type); impl PyByteArray { /// Creates a new Python bytearray object. diff --git a/src/types/bytes.rs b/src/types/bytes.rs index 77b1d2b735d..195e7f25884 100644 --- a/src/types/bytes.rs +++ b/src/types/bytes.rs @@ -47,7 +47,8 @@ use std::str; #[repr(transparent)] pub struct PyBytes(PyAny); -pyobject_native_type_core!(PyBytes, pyobject_native_static_type_object!(ffi::PyBytes_Type), #checkfunction=ffi::PyBytes_Check); +pyobject_native_type_core!(PyBytes, #checkfunction=ffi::PyBytes_Check); +pyobject_native_type_object_methods!(PyBytes, #global=ffi::PyBytes_Type); impl PyBytes { /// Creates a new Python bytestring object. diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 9d9e6e4eb72..494597eafd7 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -47,7 +47,8 @@ use std::os::raw::{c_char, c_int, c_void}; #[repr(transparent)] pub struct PyCapsule(PyAny); -pyobject_native_type_core!(PyCapsule, pyobject_native_static_type_object!(ffi::PyCapsule_Type), #checkfunction=ffi::PyCapsule_CheckExact); +pyobject_native_type_core!(PyCapsule, #checkfunction=ffi::PyCapsule_CheckExact); +pyobject_native_type_object_methods!(PyCapsule, #global=ffi::PyCapsule_Type); impl PyCapsule { /// Constructs a new capsule whose contents are `value`, associated with `name`. diff --git a/src/types/code.rs b/src/types/code.rs index 0c1683c75be..412314ba571 100644 --- a/src/types/code.rs +++ b/src/types/code.rs @@ -8,11 +8,8 @@ use crate::PyAny; #[repr(transparent)] pub struct PyCode(PyAny); -pyobject_native_type_core!( - PyCode, - pyobject_native_static_type_object!(ffi::PyCode_Type), - #checkfunction=ffi::PyCode_Check -); +pyobject_native_type_core!(PyCode, #checkfunction=ffi::PyCode_Check); +pyobject_native_type_object_methods!(PyCode, #global=ffi::PyCode_Type); #[cfg(test)] mod tests { diff --git a/src/types/complex.rs b/src/types/complex.rs index 58651569b47..105ed67aeaf 100644 --- a/src/types/complex.rs +++ b/src/types/complex.rs @@ -20,13 +20,8 @@ use std::os::raw::c_double; pub struct PyComplex(PyAny); pyobject_subclassable_native_type!(PyComplex, ffi::PyComplexObject); - -pyobject_native_type!( - PyComplex, - ffi::PyComplexObject, - pyobject_native_static_type_object!(ffi::PyComplex_Type), - #checkfunction=ffi::PyComplex_Check -); +pyobject_native_type!(PyComplex, ffi::PyComplexObject, #checkfunction=ffi::PyComplex_Check); +pyobject_native_type_object_methods!(PyComplex, #global=ffi::PyComplex_Type); impl PyComplex { /// Creates a new `PyComplex` from the given real and imaginary values. diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 8ab512ac466..ad77bfae630 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -30,6 +30,8 @@ use std::os::raw::c_int; #[cfg(feature = "chrono")] use std::ptr; +use super::PyType; + fn ensure_datetime_api(py: Python<'_>) -> PyResult<&'static PyDateTime_CAPI> { if let Some(api) = unsafe { pyo3_ffi::PyDateTimeAPI().as_ref() } { Ok(api) @@ -195,10 +197,15 @@ pub struct PyDate(PyAny); pyobject_native_type!( PyDate, crate::ffi::PyDateTime_Date, - |py| expect_datetime_api(py).DateType, #module=Some("datetime"), #checkfunction=PyDate_Check ); +pyobject_native_type_object_methods!( + PyDate, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).DateType).unbind() + } +); pyobject_subclassable_native_type!(PyDate, crate::ffi::PyDateTime_Date); impl PyDate { @@ -266,10 +273,15 @@ pub struct PyDateTime(PyAny); pyobject_native_type!( PyDateTime, crate::ffi::PyDateTime_DateTime, - |py| expect_datetime_api(py).DateTimeType, #module=Some("datetime"), #checkfunction=PyDateTime_Check ); +pyobject_native_type_object_methods!( + PyDateTime, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).DateTimeType).unbind() + } +); pyobject_subclassable_native_type!(PyDateTime, crate::ffi::PyDateTime_DateTime); impl PyDateTime { @@ -512,10 +524,15 @@ pub struct PyTime(PyAny); pyobject_native_type!( PyTime, crate::ffi::PyDateTime_Time, - |py| expect_datetime_api(py).TimeType, #module=Some("datetime"), #checkfunction=PyTime_Check ); +pyobject_native_type_object_methods!( + PyTime, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).TimeType).unbind() + } +); pyobject_subclassable_native_type!(PyTime, crate::ffi::PyDateTime_Time); impl PyTime { @@ -668,10 +685,15 @@ pub struct PyTzInfo(PyAny); pyobject_native_type!( PyTzInfo, crate::ffi::PyObject, - |py| expect_datetime_api(py).TZInfoType, #module=Some("datetime"), #checkfunction=PyTZInfo_Check ); +pyobject_native_type_object_methods!( + PyTzInfo, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).TZInfoType).unbind() + } +); pyobject_subclassable_native_type!(PyTzInfo, crate::ffi::PyObject); /// Equivalent to `datetime.timezone.utc` @@ -720,10 +742,15 @@ pub struct PyDelta(PyAny); pyobject_native_type!( PyDelta, crate::ffi::PyDateTime_Delta, - |py| expect_datetime_api(py).DeltaType, #module=Some("datetime"), #checkfunction=PyDelta_Check ); +pyobject_native_type_object_methods!( + PyDelta, + #create=|py| unsafe { + PyType::from_borrowed_type_ptr(py, expect_datetime_api(py).DeltaType).unbind() + } +); pyobject_subclassable_native_type!(PyDelta, crate::ffi::PyDateTime_Delta); impl PyDelta { diff --git a/src/types/dict.rs b/src/types/dict.rs index b3c8e37962b..56df259031d 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -18,12 +18,8 @@ pub struct PyDict(PyAny); pyobject_subclassable_native_type!(PyDict, crate::ffi::PyDictObject); -pyobject_native_type!( - PyDict, - ffi::PyDictObject, - pyobject_native_static_type_object!(ffi::PyDict_Type), - #checkfunction=ffi::PyDict_Check -); +pyobject_native_type!(PyDict, ffi::PyDictObject, #checkfunction=ffi::PyDict_Check); +pyobject_native_type_object_methods!(PyDict, #global=ffi::PyDict_Type); /// Represents a Python `dict_keys`. #[cfg(not(any(PyPy, GraalPy)))] @@ -31,11 +27,9 @@ pyobject_native_type!( pub struct PyDictKeys(PyAny); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type_core!( - PyDictKeys, - pyobject_native_static_type_object!(ffi::PyDictKeys_Type), - #checkfunction=ffi::PyDictKeys_Check -); +pyobject_native_type_core!(PyDictKeys, #checkfunction=ffi::PyDictKeys_Check); +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PyDictKeys, #global=ffi::PyDictKeys_Type); /// Represents a Python `dict_values`. #[cfg(not(any(PyPy, GraalPy)))] @@ -43,11 +37,9 @@ pyobject_native_type_core!( pub struct PyDictValues(PyAny); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type_core!( - PyDictValues, - pyobject_native_static_type_object!(ffi::PyDictValues_Type), - #checkfunction=ffi::PyDictValues_Check -); +pyobject_native_type_core!(PyDictValues, #checkfunction=ffi::PyDictValues_Check); +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PyDictValues, #global=ffi::PyDictValues_Type); /// Represents a Python `dict_items`. #[cfg(not(any(PyPy, GraalPy)))] @@ -55,11 +47,9 @@ pyobject_native_type_core!( pub struct PyDictItems(PyAny); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type_core!( - PyDictItems, - pyobject_native_static_type_object!(ffi::PyDictItems_Type), - #checkfunction=ffi::PyDictItems_Check -); +pyobject_native_type_core!(PyDictItems, #checkfunction=ffi::PyDictItems_Check); +#[cfg(not(any(PyPy, GraalPy)))] +pyobject_native_type_object_methods!(PyDictItems, #global=ffi::PyDictItems_Type); impl PyDict { /// Creates a new empty dictionary. diff --git a/src/types/ellipsis.rs b/src/types/ellipsis.rs index ee5898c9013..54e78e556df 100644 --- a/src/types/ellipsis.rs +++ b/src/types/ellipsis.rs @@ -29,13 +29,17 @@ impl PyEllipsis { unsafe impl PyTypeInfo for PyEllipsis { const NAME: &'static str = "ellipsis"; - const MODULE: Option<&'static str> = None; + const OPAQUE: bool = false; fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_Ellipsis()) } } + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject> { + Some(unsafe { ffi::Py_TYPE(ffi::Py_Ellipsis()) }) + } + #[inline] fn is_type_of(object: &Bound<'_, PyAny>) -> bool { // ellipsis is not usable as a base type diff --git a/src/types/float.rs b/src/types/float.rs index 3c2d6643d18..51b2d150fba 100644 --- a/src/types/float.rs +++ b/src/types/float.rs @@ -26,13 +26,8 @@ use std::os::raw::c_double; pub struct PyFloat(PyAny); pyobject_subclassable_native_type!(PyFloat, crate::ffi::PyFloatObject); - -pyobject_native_type!( - PyFloat, - ffi::PyFloatObject, - pyobject_native_static_type_object!(ffi::PyFloat_Type), - #checkfunction=ffi::PyFloat_Check -); +pyobject_native_type!(PyFloat, ffi::PyFloatObject, #checkfunction=ffi::PyFloat_Check); +pyobject_native_type_object_methods!(PyFloat, #global=ffi::PyFloat_Type); impl PyFloat { /// Creates a new Python `float` object. diff --git a/src/types/frame.rs b/src/types/frame.rs index 8d88d4754ae..3b1528f4f0c 100644 --- a/src/types/frame.rs +++ b/src/types/frame.rs @@ -8,8 +8,5 @@ use crate::PyAny; #[repr(transparent)] pub struct PyFrame(PyAny); -pyobject_native_type_core!( - PyFrame, - pyobject_native_static_type_object!(ffi::PyFrame_Type), - #checkfunction=ffi::PyFrame_Check -); +pyobject_native_type_core!(PyFrame, #checkfunction=ffi::PyFrame_Check); +pyobject_native_type_object_methods!(PyFrame, #global=ffi::PyFrame_Type); diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 954c49b5902..da60960996e 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -71,19 +71,11 @@ pub struct PyFrozenSet(PyAny); #[cfg(not(any(PyPy, GraalPy)))] pyobject_subclassable_native_type!(PyFrozenSet, crate::ffi::PySetObject); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type!( - PyFrozenSet, - ffi::PySetObject, - pyobject_native_static_type_object!(ffi::PyFrozenSet_Type), - #checkfunction=ffi::PyFrozenSet_Check -); - +pyobject_native_type!(PyFrozenSet, ffi::PySetObject, #checkfunction=ffi::PyFrozenSet_Check); #[cfg(any(PyPy, GraalPy))] -pyobject_native_type_core!( - PyFrozenSet, - pyobject_native_static_type_object!(ffi::PyFrozenSet_Type), - #checkfunction=ffi::PyFrozenSet_Check -); +pyobject_native_type_core!(PyFrozenSet, #checkfunction=ffi::PyFrozenSet_Check); + +pyobject_native_type_object_methods!(PyFrozenSet, #global=ffi::PyFrozenSet_Type); impl PyFrozenSet { /// Creates a new frozenset. diff --git a/src/types/function.rs b/src/types/function.rs index 039e2774546..9ce6595f2b3 100644 --- a/src/types/function.rs +++ b/src/types/function.rs @@ -18,7 +18,8 @@ use std::ffi::CStr; #[repr(transparent)] pub struct PyCFunction(PyAny); -pyobject_native_type_core!(PyCFunction, pyobject_native_static_type_object!(ffi::PyCFunction_Type), #checkfunction=ffi::PyCFunction_Check); +pyobject_native_type_core!(PyCFunction, #checkfunction=ffi::PyCFunction_Check); +pyobject_native_type_object_methods!(PyCFunction, #global=ffi::PyCFunction_Type); impl PyCFunction { /// Create a new built-in function with keywords (*args and/or **kwargs). @@ -226,4 +227,6 @@ unsafe impl Send for ClosureDestructor {} pub struct PyFunction(PyAny); #[cfg(not(Py_LIMITED_API))] -pyobject_native_type_core!(PyFunction, pyobject_native_static_type_object!(ffi::PyFunction_Type), #checkfunction=ffi::PyFunction_Check); +pyobject_native_type_core!(PyFunction, #checkfunction=ffi::PyFunction_Check); +#[cfg(not(Py_LIMITED_API))] +pyobject_native_type_object_methods!(PyFunction, #global=ffi::PyFunction_Type); diff --git a/src/types/list.rs b/src/types/list.rs index af2b557cba9..a27e1350b93 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -22,7 +22,8 @@ use crate::types::sequence::PySequenceMethods; #[repr(transparent)] pub struct PyList(PyAny); -pyobject_native_type_core!(PyList, pyobject_native_static_type_object!(ffi::PyList_Type), #checkfunction=ffi::PyList_Check); +pyobject_native_type_core!(PyList, #checkfunction=ffi::PyList_Check); +pyobject_native_type_object_methods!(PyList, #global=ffi::PyList_Type); #[inline] #[track_caller] diff --git a/src/types/mappingproxy.rs b/src/types/mappingproxy.rs index fc28687c561..437986632fe 100644 --- a/src/types/mappingproxy.rs +++ b/src/types/mappingproxy.rs @@ -19,11 +19,8 @@ unsafe fn dict_proxy_check(op: *mut ffi::PyObject) -> c_int { ffi::Py_IS_TYPE(op, std::ptr::addr_of_mut!(ffi::PyDictProxy_Type)) } -pyobject_native_type_core!( - PyMappingProxy, - pyobject_native_static_type_object!(ffi::PyDictProxy_Type), - #checkfunction=dict_proxy_check -); +pyobject_native_type_core!(PyMappingProxy, #checkfunction=dict_proxy_check); +pyobject_native_type_object_methods!(PyMappingProxy, #global=ffi::PyDictProxy_Type); impl PyMappingProxy { /// Creates a mappingproxy from an object. diff --git a/src/types/memoryview.rs b/src/types/memoryview.rs index 81acc5cbb2a..c05b344cf28 100644 --- a/src/types/memoryview.rs +++ b/src/types/memoryview.rs @@ -10,7 +10,8 @@ use crate::{ffi, Bound, PyAny}; #[repr(transparent)] pub struct PyMemoryView(PyAny); -pyobject_native_type_core!(PyMemoryView, pyobject_native_static_type_object!(ffi::PyMemoryView_Type), #checkfunction=ffi::PyMemoryView_Check); +pyobject_native_type_core!(PyMemoryView, #checkfunction=ffi::PyMemoryView_Check); +pyobject_native_type_object_methods!(PyMemoryView, #global=ffi::PyMemoryView_Type); impl PyMemoryView { /// Creates a new Python `memoryview` object from another Python object that diff --git a/src/types/mod.rs b/src/types/mod.rs index d84f099e773..e38d1dd8c6a 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -139,29 +139,25 @@ macro_rules! pyobject_native_type_named ( }; ); -#[doc(hidden)] -#[macro_export] -macro_rules! pyobject_native_static_type_object( - ($typeobject:expr) => { - |_py| { - #[allow(unused_unsafe)] // https://github.com/rust-lang/rust/pull/125834 - unsafe { ::std::ptr::addr_of_mut!($typeobject) } - } - }; -); - #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_info( - ($name:ty, $typeobject:expr, $module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, $module:expr, $opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { unsafe impl<$($generics,)*> $crate::type_object::PyTypeInfo for $name { const NAME: &'static str = stringify!($name); const MODULE: ::std::option::Option<&'static str> = $module; + const OPAQUE: bool = $opaque; #[inline] - #[allow(clippy::redundant_closure_call)] fn type_object_raw(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { - $typeobject(py) + // provided by pyobject_native_type_object_methods!() + Self::type_object_raw_impl(py) + } + + #[inline] + fn try_get_type_object_raw() -> ::std::option::Option<*mut $crate::ffi::PyTypeObject> { + // provided by pyobject_native_type_object_methods!() + Self::try_get_type_object_raw_impl() } $( @@ -180,16 +176,46 @@ macro_rules! pyobject_native_type_info( }; ); +#[doc(hidden)] +#[macro_export] +macro_rules! pyobject_native_type_marker( + ($name:ty) => { + unsafe impl $crate::type_object::PyNativeType for $name {} + } +); + /// Declares all of the boilerplate for Python types. #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type_core { - ($name:ty, $typeobject:expr, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + ($name:ty, #module=$module:expr, #opaque=$opaque:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { $crate::pyobject_native_type_named!($name $(;$generics)*); - $crate::pyobject_native_type_info!($name, $typeobject, $module $(, #checkfunction=$checkfunction)? $(;$generics)*); + $crate::pyobject_native_type_marker!($name); + $crate::pyobject_native_type_info!( + $name, + $module, + $opaque + $(, #checkfunction=$checkfunction)? + $(;$generics)* + ); }; - ($name:ty, $typeobject:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject, #module=::std::option::Option::Some("builtins") $(, #checkfunction=$checkfunction)? $(;$generics)*); + ($name:ty, #module=$module:expr $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!( + $name, + #module=$module, + #opaque=false + $(, #checkfunction=$checkfunction)? + $(;$generics)* + ); + }; + ($name:ty $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!( + $name, + #module=::std::option::Option::Some("builtins"), + #opaque=false + $(, #checkfunction=$checkfunction)? + $(;$generics)* + ); }; } @@ -199,10 +225,11 @@ macro_rules! pyobject_subclassable_native_type { ($name:ty, $layout:path $(;$generics:ident)*) => { #[cfg(not(Py_LIMITED_API))] impl<$($generics,)*> $crate::impl_::pyclass::PyClassBaseType for $name { - type LayoutAsBase = $crate::impl_::pycell::PyClassObjectBase<$layout>; + type StaticLayout = $crate::impl_::pycell::PyStaticNativeLayout<$layout>; type BaseNativeType = $name; + type RecursiveOperations = $crate::impl_::pycell::PyNativeTypeRecursiveOperations; type Initializer = $crate::impl_::pyclass_init::PyNativeTypeInitializer; - type PyClassMutability = $crate::pycell::impl_::ImmutableClass; + type PyClassMutability = $crate::pycell::borrow_checker::ImmutableClass; } } } @@ -221,14 +248,83 @@ macro_rules! pyobject_native_type_sized { #[doc(hidden)] #[macro_export] macro_rules! pyobject_native_type { - ($name:ty, $layout:path, $typeobject:expr $(, #module=$module:expr)? $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { - $crate::pyobject_native_type_core!($name, $typeobject $(, #module=$module)? $(, #checkfunction=$checkfunction)? $(;$generics)*); + ($name:ty, $layout:path $(, #module=$module:expr)? $(, #checkfunction=$checkfunction:path)? $(;$generics:ident)*) => { + $crate::pyobject_native_type_core!($name $(, #module=$module)? $(, #checkfunction=$checkfunction)? $(;$generics)*); // To prevent inheriting native types with ABI3 #[cfg(not(Py_LIMITED_API))] $crate::pyobject_native_type_sized!($name, $layout $(;$generics)*); }; } +/// Implement methods for obtaining the type object associated with a native type. +/// These methods are referred to in `pyobject_native_type_info` for implementing `PyTypeInfo`. +#[doc(hidden)] +#[macro_export] +macro_rules! pyobject_native_type_object_methods { + // the type object is not known statically and so must be created (once) with the GIL held + ($name:ty, #create=$create_type_object:expr) => { + impl $name { + fn type_object_cell() -> &'static $crate::sync::GILOnceCell<$crate::Py<$crate::types::PyType>> { + static TYPE_OBJECT: $crate::sync::GILOnceCell<$crate::Py<$crate::types::PyType>> = + $crate::sync::GILOnceCell::new(); + &TYPE_OBJECT + } + + #[allow(clippy::redundant_closure_call)] + fn type_object_raw_impl(py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { + Self::type_object_cell() + .get_or_init(py, || $create_type_object(py)) + .as_ptr() + .cast::<$crate::ffi::PyTypeObject>() + } + + fn try_get_type_object_raw_impl() -> ::std::option::Option<*mut $crate::ffi::PyTypeObject> { + unsafe { + Self::type_object_cell().get_raw().map(|obj| { (*obj).as_ptr().cast() }) + } + } + } + }; + // the type object can be created without holding the GIL + ($name:ty, #get=$get_type_object:expr) => { + impl $name { + fn type_object_raw_impl(_py: $crate::Python<'_>) -> *mut $crate::ffi::PyTypeObject { + Self::try_get_type_object_raw_impl().expect("type object is None when it should be Some") + } + + #[allow(clippy::redundant_closure_call)] + fn try_get_type_object_raw_impl() -> ::std::option::Option<*mut $crate::ffi::PyTypeObject> { + Some($get_type_object()) + } + } + }; + // the type object is imported from a module + ($name:ty, #import_module=$import_module:expr, #import_name=$import_name:expr) => { + $crate::pyobject_native_type_object_methods!($name, #create=|py: $crate::Python<'_>| { + let module = stringify!($import_module); + let name = stringify!($import_name); + || -> $crate::PyResult<$crate::Py<$crate::types::PyType>> { + use $crate::types::PyAnyMethods; + $crate::PyResult::Ok(py.import(module)?.getattr(name)?.downcast_into()?.unbind()) + }() + .unwrap_or_else(|e| ::std::panic!("failed to import {}.{}: {}", module, name, e)) + }); + }; + // the type object is known statically + ($name:ty, #global=$ffi_type_object:path) => { + $crate::pyobject_native_type_object_methods!($name, #get=|| { + #[allow(unused_unsafe)] // https://github.com/rust-lang/rust/pull/125834 + unsafe { ::std::ptr::addr_of_mut!($ffi_type_object) } + }); + }; + // the type object is known statically + ($name:ty, #global_ptr=$ffi_type_object:path) => { + $crate::pyobject_native_type_object_methods!($name, #get=|| { + unsafe { $ffi_type_object.cast::<$crate::ffi::PyTypeObject>() } + }); + }; +} + pub(crate) mod any; pub(crate) mod boolobject; pub(crate) mod bytearray; diff --git a/src/types/module.rs b/src/types/module.rs index fd7299cb084..ac8c76af907 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -31,7 +31,8 @@ use std::str; #[repr(transparent)] pub struct PyModule(PyAny); -pyobject_native_type_core!(PyModule, pyobject_native_static_type_object!(ffi::PyModule_Type), #checkfunction=ffi::PyModule_Check); +pyobject_native_type_core!(PyModule, #checkfunction=ffi::PyModule_Check); +pyobject_native_type_object_methods!(PyModule, #global=ffi::PyModule_Type); impl PyModule { /// Creates a new module object with the `__name__` attribute set to `name`. diff --git a/src/types/none.rs b/src/types/none.rs index 1ec12d3f5b0..9b66cb8e482 100644 --- a/src/types/none.rs +++ b/src/types/none.rs @@ -29,13 +29,17 @@ impl PyNone { unsafe impl PyTypeInfo for PyNone { const NAME: &'static str = "NoneType"; - const MODULE: Option<&'static str> = None; + const OPAQUE: bool = false; fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_None()) } } + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject> { + Some(unsafe { ffi::Py_TYPE(ffi::Py_None()) }) + } + #[inline] fn is_type_of(object: &Bound<'_, PyAny>) -> bool { // NoneType is not usable as a base type diff --git a/src/types/notimplemented.rs b/src/types/notimplemented.rs index d93ab466d2d..9c0371cb998 100644 --- a/src/types/notimplemented.rs +++ b/src/types/notimplemented.rs @@ -34,11 +34,16 @@ impl PyNotImplemented { unsafe impl PyTypeInfo for PyNotImplemented { const NAME: &'static str = "NotImplementedType"; const MODULE: Option<&'static str> = None; + const OPAQUE: bool = false; fn type_object_raw(_py: Python<'_>) -> *mut ffi::PyTypeObject { unsafe { ffi::Py_TYPE(ffi::Py_NotImplemented()) } } + fn try_get_type_object_raw() -> Option<*mut ffi::PyTypeObject> { + Some(unsafe { ffi::Py_TYPE(ffi::Py_NotImplemented()) }) + } + #[inline] fn is_type_of(object: &Bound<'_, PyAny>) -> bool { // NotImplementedType is not usable as a base type diff --git a/src/types/num.rs b/src/types/num.rs index 0e377f66d48..b31dbe74892 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -14,7 +14,8 @@ use crate::{ffi, instance::Bound, PyAny}; #[repr(transparent)] pub struct PyInt(PyAny); -pyobject_native_type_core!(PyInt, pyobject_native_static_type_object!(ffi::PyLong_Type), #checkfunction=ffi::PyLong_Check); +pyobject_native_type_core!(PyInt, #checkfunction=ffi::PyLong_Check); +pyobject_native_type_object_methods!(PyInt, #global=ffi::PyLong_Type); /// Deprecated alias for [`PyInt`]. #[deprecated(since = "0.23.0", note = "use `PyInt` instead")] diff --git a/src/types/pysuper.rs b/src/types/pysuper.rs index 81db8cea869..c9de1a3a189 100644 --- a/src/types/pysuper.rs +++ b/src/types/pysuper.rs @@ -11,10 +11,8 @@ use crate::{PyAny, PyResult}; #[repr(transparent)] pub struct PySuper(PyAny); -pyobject_native_type_core!( - PySuper, - pyobject_native_static_type_object!(ffi::PySuper_Type) -); +pyobject_native_type_core!(PySuper); +pyobject_native_type_object_methods!(PySuper, #global=ffi::PySuper_Type); impl PySuper { /// Constructs a new super object. More read about super object: [docs](https://docs.python.org/3/library/functions.html#super) diff --git a/src/types/set.rs b/src/types/set.rs index d5e39ebc83d..51ced5fac5b 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -25,19 +25,11 @@ pub struct PySet(PyAny); pyobject_subclassable_native_type!(PySet, crate::ffi::PySetObject); #[cfg(not(any(PyPy, GraalPy)))] -pyobject_native_type!( - PySet, - ffi::PySetObject, - pyobject_native_static_type_object!(ffi::PySet_Type), - #checkfunction=ffi::PySet_Check -); - +pyobject_native_type!(PySet, ffi::PySetObject, #checkfunction=ffi::PySet_Check); #[cfg(any(PyPy, GraalPy))] -pyobject_native_type_core!( - PySet, - pyobject_native_static_type_object!(ffi::PySet_Type), - #checkfunction=ffi::PySet_Check -); +pyobject_native_type_core!(PySet, #checkfunction=ffi::PySet_Check); + +pyobject_native_type_object_methods!(PySet, #global=ffi::PySet_Type); impl PySet { /// Creates a new set with elements from the given slice. diff --git a/src/types/slice.rs b/src/types/slice.rs index 9ca2aa4ec43..a0ec89d549d 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -19,12 +19,8 @@ use std::convert::Infallible; #[repr(transparent)] pub struct PySlice(PyAny); -pyobject_native_type!( - PySlice, - ffi::PySliceObject, - pyobject_native_static_type_object!(ffi::PySlice_Type), - #checkfunction=ffi::PySlice_Check -); +pyobject_native_type!(PySlice, ffi::PySliceObject, #checkfunction=ffi::PySlice_Check); +pyobject_native_type_object_methods!(PySlice, #global=ffi::PySlice_Type); /// Return value from [`PySliceMethods::indices`]. #[derive(Debug, Eq, PartialEq)] diff --git a/src/types/string.rs b/src/types/string.rs index 65a9e85fa3e..f6c4143c160 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -158,7 +158,8 @@ impl<'a> PyStringData<'a> { #[repr(transparent)] pub struct PyString(PyAny); -pyobject_native_type_core!(PyString, pyobject_native_static_type_object!(ffi::PyUnicode_Type), #checkfunction=ffi::PyUnicode_Check); +pyobject_native_type_core!(PyString, #checkfunction=ffi::PyUnicode_Check); +pyobject_native_type_object_methods!(PyString, #global=ffi::PyUnicode_Type); impl PyString { /// Creates a new Python string object. diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 885c0f67031..32d791b599b 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -12,11 +12,8 @@ use crate::{ffi, Bound, PyAny}; #[repr(transparent)] pub struct PyTraceback(PyAny); -pyobject_native_type_core!( - PyTraceback, - pyobject_native_static_type_object!(ffi::PyTraceBack_Type), - #checkfunction=ffi::PyTraceBack_Check -); +pyobject_native_type_core!(PyTraceback, #checkfunction=ffi::PyTraceBack_Check); +pyobject_native_type_object_methods!(PyTraceback, #global=ffi::PyTraceBack_Type); /// Implementation of functionality for [`PyTraceback`]. /// diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 216a376d833..b12a2122e50 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -60,7 +60,8 @@ fn try_new_from_iter<'py>( #[repr(transparent)] pub struct PyTuple(PyAny); -pyobject_native_type_core!(PyTuple, pyobject_native_static_type_object!(ffi::PyTuple_Type), #checkfunction=ffi::PyTuple_Check); +pyobject_native_type_core!(PyTuple, #checkfunction=ffi::PyTuple_Check); +pyobject_native_type_object_methods!(PyTuple, #global=ffi::PyTuple_Type); impl PyTuple { /// Constructs a new tuple with the given elements. diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 7a66b7ad0df..020c90126fb 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,3 +1,4 @@ +use super::PyString; use crate::err::{self, PyResult}; use crate::instance::Borrowed; #[cfg(not(Py_3_13))] @@ -6,8 +7,6 @@ use crate::types::any::PyAnyMethods; use crate::types::PyTuple; use crate::{ffi, Bound, PyAny, PyTypeInfo, Python}; -use super::PyString; - /// Represents a reference to a Python `type` object. /// /// Values of this type are accessed via PyO3's smart pointers, e.g. as @@ -18,7 +17,23 @@ use super::PyString; #[repr(transparent)] pub struct PyType(PyAny); -pyobject_native_type_core!(PyType, pyobject_native_static_type_object!(ffi::PyType_Type), #checkfunction=ffi::PyType_Check); +pyobject_native_type_core!( + PyType, + #module=Some("builtins"), + #opaque=true, + #checkfunction=ffi::PyType_Check +); +pyobject_native_type_object_methods!(PyType, #global=ffi::PyType_Type); + +impl crate::impl_::pyclass::PyClassBaseType for PyType { + /// [ffi::PyType_Type] has a variable size and private fields even when using the unlimited API, it therefore + /// cannot be used with the static layout. Attempts to do so will panic when accessed. + type StaticLayout = crate::impl_::pycell::InvalidStaticLayout; + type BaseNativeType = PyType; + type RecursiveOperations = crate::impl_::pycell::PyNativeTypeRecursiveOperations; + type Initializer = crate::impl_::pyclass_init::PyNativeTypeInitializer; + type PyClassMutability = crate::pycell::borrow_checker::ImmutableClass; +} impl PyType { /// Creates a new type object. diff --git a/src/types/weakref/reference.rs b/src/types/weakref/reference.rs index edabb6da935..59522493641 100644 --- a/src/types/weakref/reference.rs +++ b/src/types/weakref/reference.rs @@ -22,10 +22,11 @@ pyobject_subclassable_native_type!(PyWeakrefReference, crate::ffi::PyWeakReferen pyobject_native_type!( PyWeakrefReference, ffi::PyWeakReference, - pyobject_native_static_type_object!(ffi::_PyWeakref_RefType), #module=Some("weakref"), #checkfunction=ffi::PyWeakref_CheckRefExact ); +#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] +pyobject_native_type_object_methods!(PyWeakrefReference, #global=ffi::_PyWeakref_RefType); // When targetting alternative or multiple interpreters, it is better to not use the internal API. #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 4a687a89eea..92afc2cfd1c 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -426,6 +426,52 @@ fn test_tuple_struct_class() { }); } +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pyclass] +struct NoDunderDictSupport { + _pad: [u8; 32], +} + +#[test] +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[should_panic(expected = "inst.a = 1")] +fn no_dunder_dict_support_assignment() { + Python::with_gil(|py| { + let inst = Py::new( + py, + NoDunderDictSupport { + _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", + }, + ) + .unwrap(); + // should panic as this class has no __dict__ + py_run!(py, inst, r#"inst.a = 1"#); + }); +} + +#[test] +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +fn no_dunder_dict_support_setattr() { + Python::with_gil(|py| { + let inst = Py::new( + py, + NoDunderDictSupport { + _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", + }, + ) + .unwrap(); + let err = inst + .into_bound(py) + .as_any() + .setattr("a", 1) + .unwrap_err() + .to_string(); + assert!(err.contains( + "AttributeError: 'builtins.NoDunderDictSupport' object has no attribute 'a'" + )); + }); +} + #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] #[pyclass(dict, subclass)] struct DunderDictSupport { diff --git a/tests/test_class_init.rs b/tests/test_class_init.rs new file mode 100644 index 00000000000..dfee17f99f0 --- /dev/null +++ b/tests/test_class_init.rs @@ -0,0 +1,322 @@ +#![cfg(feature = "macros")] + +use pyo3::{ + exceptions::PyValueError, + prelude::*, + types::{IntoPyDict, PyDict, PyTuple}, +}; + +#[pyclass] +#[derive(Default)] +struct EmptyClassWithInit {} + +#[pymethods] +impl EmptyClassWithInit { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { + EmptyClassWithInit {} + } + + fn __init__(&self) {} +} + +#[test] +fn empty_class_with_init() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + assert!(typeobj + .call((), None) + .unwrap() + .downcast::() + .is_ok()); + + // Calling with arbitrary args or kwargs is not ok + assert!(typeobj.call(("some", "args"), None).is_err()); + assert!(typeobj + .call((), Some(&[("some", "kwarg")].into_py_dict(py).unwrap())) + .is_err()); + }); +} + +#[pyclass] +struct SimpleInit { + pub number: u64, +} + +impl Default for SimpleInit { + fn default() -> Self { + Self { number: 2 } + } +} + +#[pymethods] +impl SimpleInit { + #[new] + fn new() -> SimpleInit { + SimpleInit { number: 1 } + } + + fn __init__(&mut self) { + assert_eq!(self.number, 2); + self.number = 3; + } +} + +#[test] +fn simple_init() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + let obj = typeobj.call((), None).unwrap(); + let obj = obj.downcast::().unwrap(); + assert_eq!(obj.borrow().number, 3); + + // Calling with arbitrary args or kwargs is not ok + assert!(typeobj.call(("some", "args"), None).is_err()); + assert!(typeobj + .call((), Some(&[("some", "kwarg")].into_py_dict(py).unwrap())) + .is_err()); + }); +} + +#[pyclass] +struct InitWithTwoArgs { + data1: i32, + data2: i32, +} + +impl Default for InitWithTwoArgs { + fn default() -> Self { + Self { + data1: 123, + data2: 234, + } + } +} + +#[pymethods] +impl InitWithTwoArgs { + #[new] + fn new(arg1: i32, _arg2: i32) -> Self { + InitWithTwoArgs { + data1: arg1, + data2: 0, + } + } + + fn __init__(&mut self, _arg1: i32, arg2: i32) { + assert_eq!(self.data1, 123); + assert_eq!(self.data2, 234); + self.data2 = arg2; + } +} + +#[test] +fn init_with_two_args() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + let wrp = typeobj + .call((10, 20), None) + .map_err(|e| e.display(py)) + .unwrap(); + let obj = wrp.downcast::().unwrap(); + let obj_ref = obj.borrow(); + assert_eq!(obj_ref.data1, 123); + assert_eq!(obj_ref.data2, 20); + + assert!(typeobj.call(("a", "b", "c"), None).is_err()); + }); +} + +#[pyclass] +#[derive(Default)] +struct InitWithVarArgs { + args: Option, + kwargs: Option, +} + +#[pymethods] +impl InitWithVarArgs { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn new(_args: &Bound<'_, PyTuple>, _kwargs: Option<&Bound<'_, PyDict>>) -> Self { + InitWithVarArgs { + args: None, + kwargs: None, + } + } + + #[pyo3(signature = (*args, **kwargs))] + fn __init__(&mut self, args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>) { + self.args = Some(args.to_string()); + self.kwargs = Some(kwargs.map(|kwargs| kwargs.to_string()).unwrap_or_default()); + } +} + +#[test] +fn init_with_var_args() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + let kwargs = [("a", 1), ("b", 42)].into_py_dict(py).unwrap(); + let wrp = typeobj + .call((10, 20), Some(&kwargs)) + .map_err(|e| e.display(py)) + .unwrap(); + let obj = wrp.downcast::().unwrap(); + let obj_ref = obj.borrow(); + assert_eq!(obj_ref.args, Some("(10, 20)".to_owned())); + assert_eq!(obj_ref.kwargs, Some("{'a': 1, 'b': 42}".to_owned())); + }); +} + +#[pyclass(subclass)] +struct SuperClass { + #[pyo3(get)] + rust_new: bool, + #[pyo3(get)] + rust_default: bool, + #[pyo3(get)] + rust_init: bool, +} + +impl Default for SuperClass { + fn default() -> Self { + Self { + rust_new: false, + rust_default: true, + rust_init: false, + } + } +} + +#[pymethods] +impl SuperClass { + #[new] + fn new() -> Self { + SuperClass { + rust_new: true, + rust_default: false, + rust_init: false, + } + } + + fn __init__(&mut self) { + assert!(!self.rust_new); + assert!(self.rust_default); + assert!(!self.rust_init); + self.rust_init = true; + } +} + +#[test] +fn subclass_init() { + Python::with_gil(|py| { + let super_cls = py.get_type::(); + let source = pyo3_ffi::c_str!(pyo3::indoc::indoc!( + r#" + class Class(SuperClass): + pass + c = Class() + assert c.rust_new is False # overridden because __init__ called + assert c.rust_default is True + assert c.rust_init is True + + class Class(SuperClass): + def __init__(self): + self.py_init = True + c = Class() + assert c.rust_new is True # not overridden because __init__ not called + assert c.rust_default is False + assert c.rust_init is False + assert c.py_init is True + + class Class(SuperClass): + def __init__(self): + super().__init__() + self.py_init = True + c = Class() + assert c.rust_new is False # overridden because __init__ called + assert c.rust_default is True + assert c.rust_init is True + assert c.py_init is True + "# + )); + let globals = PyModule::import(py, "__main__").unwrap().dict(); + globals.set_item("SuperClass", super_cls).unwrap(); + py.run(source, Some(&globals), None) + .map_err(|e| e.display(py)) + .unwrap(); + }); +} + +#[pyclass(extends=SuperClass)] +#[derive(Default)] +struct SubClass {} + +#[pymethods] +impl SubClass { + #[new] + fn new() -> (Self, SuperClass) { + (SubClass {}, SuperClass::new()) + } + + fn __init__(&mut self) {} +} + +#[test] +#[should_panic( + expected = "initialize_with_default does not currently support multi-level inheritance" +)] +fn subclass_pyclass_init() { + Python::with_gil(|py| { + let sub_cls = py.get_type::(); + let source = pyo3_ffi::c_str!(pyo3::indoc::indoc!( + r#" + c = SubClass() + "# + )); + let globals = PyModule::import(py, "__main__").unwrap().dict(); + globals.set_item("SubClass", sub_cls).unwrap(); + py.run(source, Some(&globals), None) + .map_err(|e| e.display(py)) + .unwrap(); + }); +} + +#[pyclass] +#[derive(Debug, Default)] +struct InitWithCustomError {} + +struct CustomError; + +impl From for PyErr { + fn from(_error: CustomError) -> PyErr { + PyValueError::new_err("custom error") + } +} + +#[pymethods] +impl InitWithCustomError { + #[new] + fn new(_should_raise: bool) -> InitWithCustomError { + InitWithCustomError {} + } + + fn __init__(&self, should_raise: bool) -> Result<(), CustomError> { + if should_raise { + Err(CustomError) + } else { + Ok(()) + } + } +} + +#[test] +fn init_with_custom_error() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + typeobj.call((false,), None).unwrap(); + let err = typeobj.call((true,), None).unwrap_err(); + assert_eq!(err.to_string(), "ValueError: custom error"); + }); +} diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index e4e80e90263..b6f09adb601 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -23,6 +23,8 @@ fn test_compile_errors() { t.compile_fail("tests/ui/reject_generics.rs"); t.compile_fail("tests/ui/deprecations.rs"); t.compile_fail("tests/ui/invalid_closure.rs"); + #[cfg(not(Py_3_12))] + t.compile_fail("tests/ui/invalid_opaque.rs"); t.compile_fail("tests/ui/pyclass_send.rs"); t.compile_fail("tests/ui/invalid_argument_attributes.rs"); t.compile_fail("tests/ui/invalid_intopy_derive.rs"); @@ -46,6 +48,7 @@ fn test_compile_errors() { t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); t.compile_fail("tests/ui/get_set_all.rs"); + t.compile_fail("tests/ui/init_without_default.rs"); t.compile_fail("tests/ui/traverse.rs"); t.compile_fail("tests/ui/invalid_pymodule_in_root.rs"); t.compile_fail("tests/ui/invalid_pymodule_glob.rs"); diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 7190dd49555..7fca4138c32 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -175,6 +175,217 @@ except Exception as e: }); } +#[cfg(Py_3_12)] +mod inheriting_type { + use super::*; + use pyo3::types::PyType; + use pyo3::types::{PyDict, PyTuple}; + + #[pyclass(subclass, extends=PyType)] + #[derive(Debug)] + struct Metaclass { + counter: u64, + } + + impl Default for Metaclass { + fn default() -> Self { + Self { counter: 999 } + } + } + + #[pymethods] + impl Metaclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + slf: Bound<'_, Metaclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + let mut slf = slf.borrow_mut(); + assert_eq!(slf.counter, 999); + slf.counter = 5; + } + + fn __getitem__(&self, item: u64) -> u64 { + item + 1 + } + + fn increment_counter(&mut self) { + self.counter += 1; + } + + fn get_counter(&self) -> u64 { + self.counter + } + } + + #[test] + fn test_metaclass() { + Python::with_gil(|py| { + #[allow(non_snake_case)] + let Metaclass = py.get_type::(); + + // check base type + py_run!(py, Metaclass, r#"assert Metaclass.__bases__ == (type,)"#); + + // check can be used as a metaclass + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + value = "foo_value" + assert type(Foo) is Metaclass + assert isinstance(Foo, Metaclass) + assert Foo.value == "foo_value" + assert Foo[100] == 101 + FooDynamic = Metaclass("FooDynamic", (), {}) + assert type(FooDynamic) is Metaclass + assert FooDynamic[100] == 101 + "# + ); + + // can hold data + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + pass + + assert Foo.get_counter() == 5 + Foo.increment_counter() + assert Foo.get_counter() == 6 + "# + ); + + // can be subclassed + py_run!( + py, + Metaclass, + r#" + class Foo(Metaclass): + value = "foo_value" + + class Bar(metaclass=Foo): + value = "bar_value" + + assert isinstance(Bar, Foo) + assert Bar.get_counter() == 5 + Bar.increment_counter() + assert Bar.get_counter() == 6 + assert Bar.value == "bar_value" + assert Bar[100] == 101 + "# + ); + }); + } + + #[pyclass(subclass, extends=Metaclass)] + #[derive(Debug, Default)] + struct MetaclassSubclass {} + + #[pymethods] + impl MetaclassSubclass { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + _slf: Bound<'_, MetaclassSubclass>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + } + } + + #[test] + #[should_panic( + expected = "initialize_with_default does not currently support multi-level inheritance" + )] + fn test_metaclass_subclass() { + Python::with_gil(|py| { + #[allow(non_snake_case)] + let MetaclassSubclass = py.get_type::(); + py_run!( + py, + MetaclassSubclass, + r#" + class Foo(metaclass=MetaclassSubclass): + pass + "# + ); + }); + } + + #[test] + #[should_panic(expected = "Metaclasses must specify __init__")] + fn inherit_type_missing_init() { + use pyo3::types::PyType; + + #[pyclass(subclass, extends=PyType)] + #[derive(Debug, Default)] + struct MetaclassMissingInit; + + #[pymethods] + impl MetaclassMissingInit {} + + Python::with_gil(|py| { + #[allow(non_snake_case)] + let Metaclass = py.get_type::(); + + // panics when used + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + pass + "# + ); + }); + } + + #[test] + #[should_panic(expected = "Metaclasses must not specify __new__ (use __init__ instead)")] + fn inherit_type_with_new() { + use pyo3::types::PyType; + + #[pyclass(subclass, extends=PyType)] + #[derive(Debug, Default)] + struct MetaclassWithNew; + + #[pymethods] + impl MetaclassWithNew { + #[new] + #[pyo3(signature = (*_args, **_kwargs))] + fn new(_args: Bound<'_, PyTuple>, _kwargs: Option>) -> Self { + MetaclassWithNew {} + } + + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + _slf: Bound<'_, MetaclassWithNew>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + } + } + + Python::with_gil(|py| { + #[allow(non_snake_case)] + let Metaclass = py.get_type::(); + + // panics when used + py_run!( + py, + Metaclass, + r#" + class Foo(metaclass=Metaclass): + pass + "# + ); + }); + } +} + // Subclassing builtin types is not allowed in the LIMITED API. #[cfg(not(Py_LIMITED_API))] mod inheriting_native_type { diff --git a/tests/test_variable_sized_class_basics.rs b/tests/test_variable_sized_class_basics.rs new file mode 100644 index 00000000000..9adde34d849 --- /dev/null +++ b/tests/test_variable_sized_class_basics.rs @@ -0,0 +1,51 @@ +#![cfg(all(Py_3_12, feature = "macros"))] + +use pyo3::types::{PyDict, PyInt, PyTuple}; +use pyo3::{prelude::*, types::PyType}; +use pyo3::{py_run, PyTypeInfo}; +use static_assertions::const_assert; + +#[path = "../src/tests/common.rs"] +mod common; + +#[pyclass(extends=PyType)] +#[derive(Default)] +struct ClassWithObjectField { + #[pyo3(get, set)] + value: Option, +} + +#[pymethods] +impl ClassWithObjectField { + #[pyo3(signature = (*_args, **_kwargs))] + fn __init__( + _slf: Bound<'_, ClassWithObjectField>, + _args: Bound<'_, PyTuple>, + _kwargs: Option>, + ) { + } +} + +#[test] +fn class_with_object_field() { + Python::with_gil(|py| { + let ty = py.get_type::(); + const_assert!(::OPAQUE); + py_run!( + py, + ty, + "x = ty('X', (), {}); x.value = 5; assert x.value == 5" + ); + py_run!( + py, + ty, + "x = ty('X', (), {}); x.value = None; assert x.value == None" + ); + + let obj = Bound::new(py, ClassWithObjectField { value: None }).unwrap(); + py_run!(py, obj, "obj.value = 5"); + let obj_ref = obj.borrow(); + let value = obj_ref.value.as_ref().unwrap(); + assert_eq!(*value.downcast_bound::(py).unwrap(), 5); + }); +} diff --git a/tests/ui/abi3_inheritance.stderr b/tests/ui/abi3_inheritance.stderr index 309b67a633d..110092cb0a8 100644 --- a/tests/ui/abi3_inheritance.stderr +++ b/tests/ui/abi3_inheritance.stderr @@ -7,7 +7,9 @@ error[E0277]: pyclass `PyException` cannot be subclassed = help: the trait `PyClassBaseType` is not implemented for `PyException` = note: `PyException` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType note: required by a bound in `PyClassImpl::BaseType` --> src/impl_/pyclass.rs | @@ -23,5 +25,7 @@ error[E0277]: pyclass `PyException` cannot be subclassed = help: the trait `PyClassBaseType` is not implemented for `PyException` = note: `PyException` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/abi3_nativetype_inheritance.stderr b/tests/ui/abi3_nativetype_inheritance.stderr index 872de60b244..6a8d9f29c1c 100644 --- a/tests/ui/abi3_nativetype_inheritance.stderr +++ b/tests/ui/abi3_nativetype_inheritance.stderr @@ -7,7 +7,9 @@ error[E0277]: pyclass `PyDict` cannot be subclassed = help: the trait `PyClassBaseType` is not implemented for `PyDict` = note: `PyDict` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType note: required by a bound in `PyClassImpl::BaseType` --> src/impl_/pyclass.rs | @@ -23,5 +25,7 @@ error[E0277]: pyclass `PyDict` cannot be subclassed = help: the trait `PyClassBaseType` is not implemented for `PyDict` = note: `PyDict` must have `#[pyclass(subclass)]` to be eligible for subclassing = note: with the `abi3` feature enabled, PyO3 does not support subclassing native types - = help: the trait `PyClassBaseType` is implemented for `PyAny` + = help: the following other types implement trait `PyClassBaseType`: + PyAny + PyType = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/init_without_default.rs b/tests/ui/init_without_default.rs new file mode 100644 index 00000000000..0d69a3e0a4f --- /dev/null +++ b/tests/ui/init_without_default.rs @@ -0,0 +1,11 @@ +use pyo3::prelude::*; + +#[pyclass] +struct MyClass {} + +#[pymethods] +impl MyClass { + fn __init__(&self) {} +} + +fn main() {} diff --git a/tests/ui/init_without_default.stderr b/tests/ui/init_without_default.stderr new file mode 100644 index 00000000000..44bc1ab8cd8 --- /dev/null +++ b/tests/ui/init_without_default.stderr @@ -0,0 +1,16 @@ +error[E0277]: the trait bound `MyClass: Default` is not satisfied + --> tests/ui/init_without_default.rs:7:6 + | +7 | impl MyClass { + | ^^^^^^^ the trait `Default` is not implemented for `MyClass` + | +note: required by a bound in `initialize_with_default` + --> src/impl_/pyclass_init.rs + | + | pub unsafe fn initialize_with_default( + | ^^^^^^^ required by this bound in `initialize_with_default` +help: consider annotating `MyClass` with `#[derive(Default)]` + | +4 + #[derive(Default)] +5 | struct MyClass {} + | diff --git a/tests/ui/invalid_opaque.rs b/tests/ui/invalid_opaque.rs new file mode 100644 index 00000000000..8f4d72bb0d6 --- /dev/null +++ b/tests/ui/invalid_opaque.rs @@ -0,0 +1,6 @@ +use pyo3::prelude::*; + +#[pyclass(opaque)] +struct MyClass; + +fn main() {} diff --git a/tests/ui/invalid_opaque.stderr b/tests/ui/invalid_opaque.stderr new file mode 100644 index 00000000000..9ca79b46f72 --- /dev/null +++ b/tests/ui/invalid_opaque.stderr @@ -0,0 +1,7 @@ +error: #[pyclass(opaque)] requires python 3.12 or later + --> tests/ui/invalid_opaque.rs:3:1 + | +3 | #[pyclass(opaque)] + | ^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/tests/ui/invalid_pyclass_args.stderr b/tests/ui/invalid_pyclass_args.stderr index d1335e0f1a1..26286ad217b 100644 --- a/tests/ui/invalid_pyclass_args.stderr +++ b/tests/ui/invalid_pyclass_args.stderr @@ -1,4 +1,4 @@ -error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` +error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `opaque`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` --> tests/ui/invalid_pyclass_args.rs:4:11 | 4 | #[pyclass(extend=pyo3::types::PyDict)] @@ -46,7 +46,7 @@ error: expected string literal 25 | #[pyclass(module = my_module)] | ^^^^^^^^^ -error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` +error: expected one of: `crate`, `dict`, `eq`, `eq_int`, `extends`, `freelist`, `frozen`, `get_all`, `hash`, `mapping`, `module`, `name`, `ord`, `opaque`, `rename_all`, `sequence`, `set_all`, `str`, `subclass`, `unsendable`, `weakref` --> tests/ui/invalid_pyclass_args.rs:28:11 | 28 | #[pyclass(weakrev)]