Opaque Object Layout

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyo3"
-version = "0.23.1"
+version = "0.24.0"
 description = "Bindings to Python interpreter"
 authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
 readme = "README.md"

Contributing.md

@@ -109,7 +109,24 @@ You can run these checks yourself with `nox`. Use  `nox -l` to list the full set
 `nox -s clippy-all`
 
 #### Tests
-`nox -s test` or `cargo test` for Rust tests only, `nox -f pytests/noxfile.py -s test` for Python tests only
+`nox -s test` or `cargo test` or `cargo nextest` for Rust tests only, `nox -f pytests/noxfile.py -s test` for
+Python tests only.
+
+Configuring the python interpreter version (`Py_*` cfg options) when running `cargo test` is the same as for regular
+packages which is explained in [the docs](https://pyo3.rs/v0.22.5/building-and-distribution).
+The easiest way to configure the python version is to install with the system package manager or
+[pyenv](https://github.com/pyenv/pyenv) then set `PYO3_PYTHON`.
+[uv python install](https://docs.astral.sh/uv/concepts/python-versions/) cannot currently be used as it sets some
+[incorrect sysconfig values](https://github.com/astral-sh/uv/issues/8429).
+
+`Py_LIMITED_API` can be controlled with the `abi3` feature of the `pyo3` crate:
+
+```
+LD_LIBRARY_PATH=<python_path>/lib PYO3_PYTHON=<python_path>/bin/python \
+    cargo nextest run --package pyo3 --features abi3 ...
+```
+
+use the `PYO3_PRINT_CONFIG=1` to check the identified configuration.
 
 #### Check all conditional compilation
 `nox -s check-feature-powerset`
@@ -183,7 +200,11 @@ PyO3 supports all officially supported Python versions, as well as the latest Py
 
 PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible.
 
-The minimum Rust version supported will be decided when the release which bumps Python and Rust versions is made. At the time, the minimum Rust version will be set no higher than the lowest Rust version shipped in the current Debian, RHEL and Alpine Linux distributions.
+The minimum Rust version supported will be decided when the release which bumps Python and Rust versions is made.
+At the time, the minimum Rust version will be set no higher than the lowest Rust version shipped in the current
+[Debian](https://packages.debian.org/search?keywords=rustc),
+[RHEL](https://docs.redhat.com/en/documentation/red_hat_developer_tools/1) and
+[Alpine Linux](https://pkgs.alpinelinux.org/package/edge/main/x86/rust) distributions.
 
 CI tests both the most recent stable Rust version and the minimum supported Rust version. Because of Rust's stability guarantees this is sufficient to confirm support for all Rust versions in between.
 

guide/pyclass-parameters.md

@@ -23,6 +23,7 @@
 | `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. |
 | `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a thread-safe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. |
 | `weakref` | Allows this class to be [weakly referenceable][params-6]. |
+| `opaque` | Forces the use of the 'opaque layout' ([PEP 697](https://peps.python.org/pep-0697/)) for this class and any subclasses that extend it. Primarily used for internal testing. |
 
 All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or
 more accompanying `#[pyo3(...)]` annotations, e.g.:

guide/src/class.md

@@ -1378,12 +1378,20 @@ impl pyo3::types::DerefToPyAny for MyClass {}
 unsafe impl pyo3::type_object::PyTypeInfo for MyClass {
     const NAME: &'static str = "MyClass";
     const MODULE: ::std::option::Option<&'static str> = ::std::option::Option::None;
+    const OPAQUE: bool = false;
+
     #[inline]
     fn type_object_raw(py: pyo3::Python<'_>) -> *mut pyo3::ffi::PyTypeObject {
         <Self as pyo3::impl_::pyclass::PyClassImpl>::lazy_type_object()
             .get_or_init(py)
             .as_type_ptr()
     }
+
+    #[inline]
+    fn try_get_type_object_raw() -> ::std::option::Option<*mut pyo3::ffi::PyTypeObject> {
+        <Self as pyo3::impl_::pyclass::PyClassImpl>::lazy_type_object()
+            .try_get_raw()
+    }
 }
 
 impl pyo3::PyClass for MyClass {

guide/src/class/metaclass.md

@@ -0,0 +1,72 @@
+# Creating a Metaclass
+A [metaclass](https://docs.python.org/3/reference/datamodel.html#metaclasses) is a class that derives `type` and can
+be used to influence the construction of other classes.
+
+Some examples of where metaclasses can be used:
+
+- [`ABCMeta`](https://docs.python.org/3/library/abc.html) for defining abstract classes
+- [`EnumType`](https://docs.python.org/3/library/enum.html) for defining enums
+- [`NamedTuple`](https://docs.python.org/3/library/typing.html#typing.NamedTuple) for defining tuples with elements
+  that can be accessed by name in addition to index.
+- singleton classes
+- automatic registration of classes
+- ORM
+- serialization / deserialization / validation (e.g. [pydantic](https://docs.pydantic.dev/latest/api/base_model/))
+
+### Example: A Simple Metaclass
+
+Note: Creating metaclasses is only possible with python 3.12+
+
+```rust
+#[pyclass(subclass, extends=PyType)]
+#[derive(Default)]
+struct MyMetaclass {
+    counter: u64,
+};
+
+#[pymethods]
+impl MyMetaclass {
+    #[pyo3(signature = (*_args, **_kwargs))]
+    fn __init__(
+        slf: Bound<'_, Metaclass>,
+        _args: Bound<'_, PyTuple>,
+        _kwargs: Option<Bound<'_, PyDict>>,
+    ) {
+        slf.borrow_mut().counter = 5;
+    }
+
+    fn __getitem__(&self, item: u64) -> u64 {
+        item + 1
+    }
+
+    fn increment_counter(&mut self) {
+        self.counter += 1;
+    }
+
+    fn get_counter(&self) -> u64 {
+        self.counter
+    }
+}
+```
+
+Used like so:
+```python
+class Foo(metaclass=MyMetaclass):
+    def __init__() -> None:
+        ...
+
+assert type(Foo) is MyMetaclass
+assert Foo.some_var == 123
+assert Foo[100] == 101
+Foo.increment_counter()
+assert Foo.get_counter() == 1
+```
+
+In the example above `MyMetaclass` extends `PyType` (making it a metaclass). It does not define `#[new]` as
+[this is not supported](https://docs.python.org/3/c-api/type.html#c.PyType_FromMetaclass). Instead `__init__` is
+defined which is called whenever a class is created that uses `MyMetaclass` as its metaclass.
+The arguments to `__init__` are the same as the arguments to `type(name, bases, kwds)`. A `Default` impl is required
+in order to define `__init__`. The data in the struct is initialised to `Default` before `__init__` is called.
+
+When special methods like `__getitem__` are defined for a metaclass they apply to the classes they construct, so
+`Foo[123]` calls `MyMetaclass.__getitem__(Foo, 123)`.

guide/src/class/protocols.md

@@ -141,6 +141,11 @@ given signatures should be interpreted as follows:
 
     Determines the "truthyness" of an object.
 
+  - `__init__(<self>, ...) -> ()` - the arguments can be defined as for
+    normal `pymethods`. The pyclass struct must implement `Default`.
+    If the class defines `__new__` and `__init__` the values set in
+    `__new__` are overridden by `Default` before `__init__` is called.
+
   - `__call__(<self>, ...) -> object` - here, any argument list can be defined
     as for normal `pymethods`
 

newsfragments/4678.added.md

@@ -0,0 +1 @@
+Add support for opaque PyObjects allowing extending variable/unknown sized base classes (including `type` to create metaclasses)

pyo3-macros-backend/src/attributes.rs

@@ -20,16 +20,17 @@ pub mod kw {
     syn::custom_keyword!(eq_int);
     syn::custom_keyword!(extends);
     syn::custom_keyword!(freelist);
+    syn::custom_keyword!(from_item_all);
     syn::custom_keyword!(from_py_with);
     syn::custom_keyword!(frozen);
     syn::custom_keyword!(get);
     syn::custom_keyword!(get_all);
     syn::custom_keyword!(hash);
     syn::custom_keyword!(item);
-    syn::custom_keyword!(from_item_all);
     syn::custom_keyword!(mapping);
     syn::custom_keyword!(module);
     syn::custom_keyword!(name);
+    syn::custom_keyword!(opaque);
     syn::custom_keyword!(ord);
     syn::custom_keyword!(pass_module);
     syn::custom_keyword!(rename_all);

pyo3-macros-backend/src/pyclass.rs

@@ -72,6 +72,7 @@ pub struct PyClassPyO3Options {
     pub module: Option<ModuleAttribute>,
     pub name: Option<NameAttribute>,
     pub ord: Option<kw::ord>,
+    pub opaque: Option<kw::opaque>,
     pub rename_all: Option<RenameAllAttribute>,
     pub sequence: Option<kw::sequence>,
     pub set_all: Option<kw::set_all>,
@@ -95,6 +96,7 @@ pub enum PyClassPyO3Option {
     Module(ModuleAttribute),
     Name(NameAttribute),
     Ord(kw::ord),
+    Opaque(kw::opaque),
     RenameAll(RenameAllAttribute),
     Sequence(kw::sequence),
     SetAll(kw::set_all),
@@ -133,6 +135,8 @@ impl Parse for PyClassPyO3Option {
             input.parse().map(PyClassPyO3Option::Name)
         } else if lookahead.peek(attributes::kw::ord) {
             input.parse().map(PyClassPyO3Option::Ord)
+        } else if lookahead.peek(attributes::kw::opaque) {
+            input.parse().map(PyClassPyO3Option::Opaque)
         } else if lookahead.peek(kw::rename_all) {
             input.parse().map(PyClassPyO3Option::RenameAll)
         } else if lookahead.peek(attributes::kw::sequence) {
@@ -205,6 +209,7 @@ impl PyClassPyO3Options {
             PyClassPyO3Option::Module(module) => set_option!(module),
             PyClassPyO3Option::Name(name) => set_option!(name),
             PyClassPyO3Option::Ord(ord) => set_option!(ord),
+            PyClassPyO3Option::Opaque(opaque) => set_option!(opaque),
             PyClassPyO3Option::RenameAll(rename_all) => set_option!(rename_all),
             PyClassPyO3Option::Sequence(sequence) => set_option!(sequence),
             PyClassPyO3Option::SetAll(set_all) => set_option!(set_all),
@@ -1823,10 +1828,25 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre
         quote! { ::core::option::Option::None }
     };
 
+    let opaque = if attr.options.opaque.is_some() {
+        quote! {
+            const OPAQUE: bool = true;
+
+            #[cfg(not(Py_3_12))]
+            ::core::compile_error!("#[pyclass(opaque)] requires python 3.12 or later");
+        }
+    } else {
+        // if opaque is not supported an error will be raised at construction
+        quote! {
+            const OPAQUE: bool = <<#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseType as #pyo3_path::type_object::PyTypeInfo>::OPAQUE;
+        }
+    };
+
     quote! {
         unsafe impl #pyo3_path::type_object::PyTypeInfo for #cls {
             const NAME: &'static str = #cls_name;
             const MODULE: ::std::option::Option<&'static str> = #module;
+            #opaque
 
             #[inline]
             fn type_object_raw(py: #pyo3_path::Python<'_>) -> *mut #pyo3_path::ffi::PyTypeObject {
@@ -1835,6 +1855,12 @@ fn impl_pytypeinfo(cls: &syn::Ident, attr: &PyClassArgs, ctx: &Ctx) -> TokenStre
                     .get_or_init(py)
                     .as_type_ptr()
             }
+
+            #[inline]
+            fn try_get_type_object_raw() -> ::std::option::Option<*mut #pyo3_path::ffi::PyTypeObject> {
+                <#cls as #pyo3_path::impl_::pyclass::PyClassImpl>::lazy_type_object()
+                    .try_get_raw()
+            }
         }
     }
 }
@@ -2202,18 +2228,17 @@ impl<'a> PyClassImplsBuilder<'a> {
 
         let dict_offset = if self.attr.options.dict.is_some() {
             quote! {
-                fn dict_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> {
+                fn dict_offset() -> ::std::option::Option<#pyo3_path::impl_::pyclass::PyObjectOffset> {
                     ::std::option::Option::Some(#pyo3_path::impl_::pyclass::dict_offset::<Self>())
                 }
             }
         } else {
             TokenStream::new()
         };
 
-        // insert space for weak ref
         let weaklist_offset = if self.attr.options.weakref.is_some() {
             quote! {
-                fn weaklist_offset() -> ::std::option::Option<#pyo3_path::ffi::Py_ssize_t> {
+                fn weaklist_offset() -> ::std::option::Option<#pyo3_path::impl_::pyclass::PyObjectOffset> {
                     ::std::option::Option::Some(#pyo3_path::impl_::pyclass::weaklist_offset::<Self>())
                 }
             }
@@ -2298,8 +2323,9 @@ impl<'a> PyClassImplsBuilder<'a> {
         let pyclass_base_type_impl = attr.options.subclass.map(|subclass| {
             quote_spanned! { subclass.span() =>
                 impl #pyo3_path::impl_::pyclass::PyClassBaseType for #cls {
-                    type LayoutAsBase = #pyo3_path::impl_::pycell::PyClassObject<Self>;
+                    type StaticLayout = #pyo3_path::impl_::pycell::PyStaticClassLayout<Self>;
                     type BaseNativeType = <Self as #pyo3_path::impl_::pyclass::PyClassImpl>::BaseNativeType;
+                    type RecursiveOperations = #pyo3_path::impl_::pycell::PyClassRecursiveOperations<Self>;
                     type Initializer = #pyo3_path::pyclass_init::PyClassInitializer<Self>;
                     type PyClassMutability = <Self as #pyo3_path::impl_::pyclass::PyClassImpl>::PyClassMutability;
                 }

pyo3-macros-backend/src/pymethod.rs

@@ -143,6 +143,7 @@ impl PyMethodKind {
             "__gt__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GT__)),
             "__ge__" => PyMethodKind::Proto(PyMethodProtoKind::SlotFragment(&__GE__)),
             // Some tricky protocols which don't fit the pattern of the rest
+            "__init__" => PyMethodKind::Proto(PyMethodProtoKind::Init),
             "__call__" => PyMethodKind::Proto(PyMethodProtoKind::Call),
             "__traverse__" => PyMethodKind::Proto(PyMethodProtoKind::Traverse),
             "__clear__" => PyMethodKind::Proto(PyMethodProtoKind::Clear),
@@ -154,6 +155,7 @@ impl PyMethodKind {
 
 enum PyMethodProtoKind {
     Slot(&'static SlotDef),
+    Init,
     Call,
     Traverse,
     Clear,
@@ -212,6 +214,9 @@ pub fn gen_py_method(
                     let slot = slot_def.generate_type_slot(cls, spec, &method.method_name, ctx)?;
                     GeneratedPyMethod::Proto(slot)
                 }
+                PyMethodProtoKind::Init => {
+                    GeneratedPyMethod::Proto(impl_init_slot(cls, method.spec, ctx)?)
+                }
                 PyMethodProtoKind::Call => {
                     GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec, ctx)?)
                 }
@@ -303,8 +308,11 @@ fn ensure_no_forbidden_protocol_attributes(
     method_name: &str,
 ) -> syn::Result<()> {
     if let Some(signature) = &spec.signature.attribute {
-        // __call__ is allowed to have a signature, but nothing else is.
-        if !matches!(proto_kind, PyMethodProtoKind::Call) {
+        // __call__ and __init__ are allowed to have a signature, but nothing else is.
+        if !matches!(
+            proto_kind,
+            PyMethodProtoKind::Call | PyMethodProtoKind::Init
+        ) {
             bail_spanned!(signature.kw.span() => format!("`signature` cannot be used with magic method `{}`", method_name));
         }
     }
@@ -394,6 +402,41 @@ pub fn impl_py_method_def_new(
     })
 }
 
+fn impl_init_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result<MethodAndSlotDef> {
+    let Ctx { pyo3_path, .. } = ctx;
+
+    spec.convention = CallingConvention::Varargs;
+
+    let wrapper_ident = syn::Ident::new("__pymethod___init____", Span::call_site());
+    let associated_method = spec.get_wrapper_function(&wrapper_ident, Some(cls), ctx)?;
+    let slot_def = quote! {
+        #pyo3_path::ffi::PyType_Slot {
+            slot: #pyo3_path::ffi::Py_tp_init,
+            pfunc: {
+                unsafe extern "C" fn trampoline(
+                    slf: *mut #pyo3_path::ffi::PyObject,
+                    args: *mut #pyo3_path::ffi::PyObject,
+                    kwargs: *mut #pyo3_path::ffi::PyObject,
+                ) -> ::std::os::raw::c_int
+                {
+                    #pyo3_path::impl_::trampoline::initproc(
+                        slf,
+                        args,
+                        kwargs,
+                        #pyo3_path::impl_::pyclass_init::initialize_with_default::<#cls>,
+                        #cls::#wrapper_ident
+                    )
+                }
+                trampoline
+            } as #pyo3_path::ffi::initproc as _
+        }
+    };
+    Ok(MethodAndSlotDef {
+        associated_method,
+        slot_def,
+    })
+}
+
 fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>, ctx: &Ctx) -> Result<MethodAndSlotDef> {
     let Ctx { pyo3_path, .. } = ctx;
 
@@ -831,8 +874,8 @@ pub fn impl_py_getter_def(
 
                     struct Offset;
                     unsafe impl #pyo3_path::impl_::pyclass::OffsetCalculator<#cls, #ty> for Offset {
-                        fn offset() -> usize {
-                            #pyo3_path::impl_::pyclass::class_offset::<#cls>() +
+                        fn offset() -> #pyo3_path::impl_::pyclass::PyObjectOffset {
+                            #pyo3_path::impl_::pyclass::subclass_offset::<#cls>() +
                             #pyo3_path::impl_::pyclass::offset_of!(#cls, #field)
                         }
                     }