Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support 64 bits and prefetchable BARs #792

Merged
merged 2 commits into from
Mar 29, 2024
Merged

Support 64 bits and prefetchable BARs #792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9e9a648
Support 64 bits and prefetchable BARs
jfgd Jan 29, 2024
e89963e
Add simple tests for 64bits and prefetch region flag
jfgd Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion include/libvfio-user.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,11 @@ typedef ssize_t (vfu_region_access_cb_t)(vfu_ctx_t *vfu_ctx, char *buf,
/* If unset, this is an IO region. */
#define VFU_REGION_FLAG_MEM (1 << 2)
#define VFU_REGION_FLAG_ALWAYS_CB (1 << 3)
#define VFU_REGION_FLAG_64_BITS (1 << 4)
#define VFU_REGION_FLAG_PREFETCH (1 << 5)
#define VFU_REGION_FLAG_MASK (VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM | \
VFU_REGION_FLAG_ALWAYS_CB)
VFU_REGION_FLAG_ALWAYS_CB | VFU_REGION_FLAG_64_BITS | \
VFU_REGION_FLAG_PREFETCH)

/**
* Set up a device region.
Expand Down
4 changes: 4 additions & 0 deletions include/pci_defs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ extern "C" {
* instead?
*/

#define PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_32 (PCI_BASE_ADDRESS_MEM_TYPE_32 >> 1)
#define PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_1M (PCI_BASE_ADDRESS_MEM_TYPE_1M >> 1)
#define PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_64 (PCI_BASE_ADDRESS_MEM_TYPE_64 >> 1)

typedef union {
uint32_t raw;
struct {
Expand Down
11 changes: 11 additions & 0 deletions lib/libvfio-user.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1729,6 +1729,17 @@ vfu_realize_ctx(vfu_ctx_t *vfu_ctx)
if (!(vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_MEM)) {
vfu_ctx->pci.config_space->hdr.bars[i].io.region_type |= 0x1;
}
if ((vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_64_BITS)) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jlevon in vfu_setup_region() we don't check whether the size is > 2^32 for 32-bit BARs, I think we should fail if VFU_REGION_FLAG_64_BITS isn't set.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't support bars of that size anyway I think due to the region size uint32_t in private.h

vfu_ctx->pci.config_space->hdr.bars[i].mem.locatable
= PCI_BASE_ADDRESS_MEM_TYPE_LOCATABLE_64;
}
if ((vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_PREFETCH)) {
jlevon marked this conversation as resolved.
Show resolved Hide resolved
vfu_ctx->pci.config_space->hdr.bars[i].mem.prefetchable = 1;
if (!(vfu_ctx->reg_info[i].flags & VFU_REGION_FLAG_64_BITS)) {
vfu_log(vfu_ctx, LOG_WARNING,
"Region %d has prefetchable flag set, but not 64bits flag", i);
}
}
}

if (vfu_ctx->irqs == NULL) {
Expand Down
9 changes: 9 additions & 0 deletions test/py/libvfio_user.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,13 @@

PCI_EXT_CAP_VNDR_HDR_SIZEOF = 8

PCI_BASE_ADDRESS_SPACE_IO = 0x01
PCI_BASE_ADDRESS_SPACE_MEMORY = 0x00
PCI_BASE_ADDRESS_MEM_TYPE_32 = 0x00
PCI_BASE_ADDRESS_MEM_TYPE_1M = 0x02
PCI_BASE_ADDRESS_MEM_TYPE_64 = 0x04
PCI_BASE_ADDRESS_MEM_PREFETCH = 0x08

# MSI registers
PCI_MSI_FLAGS = 2 # Message Control offset
PCI_MSI_ADDRESS_LO = 4 # Message Address offset
Expand Down Expand Up @@ -201,6 +208,8 @@ def is_32bit():
VFU_REGION_FLAG_RW = (VFU_REGION_FLAG_READ | VFU_REGION_FLAG_WRITE)
VFU_REGION_FLAG_MEM = 4
VFU_REGION_FLAG_ALWAYS_CB = 8
VFU_REGION_FLAG_64_BITS = 16
VFU_REGION_FLAG_PREFETCH = 32

VFIO_USER_F_DMA_REGION_READ = (1 << 0)
VFIO_USER_F_DMA_REGION_WRITE = (1 << 1)
Expand Down
20 changes: 18 additions & 2 deletions test/py/test_vfu_realize_ctx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,30 @@ def test_vfu_realize_ctx_pci_bars():
ret = vfu_setup_region(ctx, index=VFU_PCI_DEV_BAR1_REGION_IDX, size=4096,
flags=(VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM))
assert ret == 0
ret = vfu_setup_region(ctx, index=VFU_PCI_DEV_BAR2_REGION_IDX,
size=1048576,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a specific reason why you use these values?

flags=(VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM
| VFU_REGION_FLAG_64_BITS))
assert ret == 0
ret = vfu_setup_region(ctx, index=VFU_PCI_DEV_BAR4_REGION_IDX,
size=1073741824,
flags=(VFU_REGION_FLAG_RW | VFU_REGION_FLAG_MEM
| VFU_REGION_FLAG_64_BITS
| VFU_REGION_FLAG_PREFETCH))
assert ret == 0

ret = vfu_realize_ctx(ctx)
assert ret == 0

# region_type should be set non-MEM BAR, unset otherwise
hdr = get_pci_header(ctx)
assert hdr.bars[0].io == 0x1
assert hdr.bars[1].io == 0
assert hdr.bars[0].io == PCI_BASE_ADDRESS_SPACE_IO
assert hdr.bars[1].mem == PCI_BASE_ADDRESS_SPACE_MEMORY
assert hdr.bars[2].mem == (PCI_BASE_ADDRESS_SPACE_MEMORY
| PCI_BASE_ADDRESS_MEM_TYPE_64)
assert hdr.bars[4].mem == (PCI_BASE_ADDRESS_SPACE_MEMORY
| PCI_BASE_ADDRESS_MEM_TYPE_64
| PCI_BASE_ADDRESS_MEM_PREFETCH)

vfu_destroy_ctx(ctx)

Expand Down
Loading