This post is about DOE which is Data Object Exchange which is used by CMA to implement SPDM for PCI/CXL IDE specification(starting Gen5, I think). I can’t download ENC on PCI SIG, So the next best thing is reading the Linux kernel driver that uses DOE. Luckily, CXL driver uses it. win-win!

Entry point the probe Link to heading

Starting with drivers/cxl/pci.c, .probe is called with devices by PCI core.

static struct pci_driver cxl_pci_driver = {
	.name			= KBUILD_MODNAME,
	.id_table		= cxl_mem_pci_tbl,
	.probe			= cxl_pci_probe,
	.driver	= {
		.probe_type	= PROBE_PREFER_ASYNCHRONOUS,
	},
};

module_pci_driver(cxl_pci_driver);

cxl_pci_probe calls devm_cxl_pci_create_doe

	devm_cxl_pci_create_doe(cxlds);

mailbox Link to heading

Still in drivers/cxl/pci.c, pcim_doe_create_mb is called from devm_cxl_pci_create_doe to create mailbox and pushed with xa_insert into &cxlds->doe_mbs

static void devm_cxl_pci_create_doe(struct cxl_dev_state *cxlds)
{
	struct device *dev = cxlds->dev;
	struct pci_dev *pdev = to_pci_dev(dev);
	u16 off = 0;

	xa_init(&cxlds->doe_mbs);

	pci_doe_for_each_off(pdev, off) {
		struct pci_doe_mb *doe_mb;

		doe_mb = pcim_doe_create_mb(pdev, off);
...
...
		}

		if (xa_insert(&cxlds->doe_mbs, off, doe_mb, GFP_KERNEL)) {
		}

	}
}

pcim_doe_create_mb takes us to drivers/pci/doe.c, pci_doe_mb is just a wrapper for mailbox.

/**
 * struct pci_doe_mb - State for a single DOE mailbox
 *
 * This state is used to manage a single DOE mailbox capability.  All fields
 * should be considered opaque to the consumers and the structure passed into
 * the helpers below after being created by devm_pci_doe_create()
 *
 * @pdev: PCI device this mailbox belongs to
 * @cap_offset: Capability offset
 * @prots: Array of protocols supported (encoded as long values)
 * @wq: Wait queue for work item
 * @work_queue: Queue of pci_doe_work items
 * @flags: Bit array of PCI_DOE_FLAG_* flags
 */
struct pci_doe_mb {
	struct pci_dev *pdev;
	u16 cap_offset;
	struct xarray prots;

	wait_queue_head_t wq;
	struct workqueue_struct *work_queue;
	unsigned long flags;
};

For example, pci_doe_write_ctrl uses to offset to configuration space for that PCI(CXL device)

static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
{
	struct pci_dev *pdev = doe_mb->pdev;
	int offset = doe_mb->cap_offset;

	pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
}

At this point, we have mailbox created and linked to pci dev. The next section goes through another data structure pci_doe_task which what actually gets sent to CXL. I think(TODO confirm).

task Link to heading

pci_doe_task is defined as container for DOE protocol

/**
 * struct pci_doe_task - represents a single query/response
 *
 * @prot: DOE Protocol
 * @request_pl: The request payload
 * @request_pl_sz: Size of the request payload (bytes)
 * @response_pl: The response payload
 * @response_pl_sz: Size of the response payload (bytes)
 * @rv: Return value.  Length of received response or error (bytes)
 * @complete: Called when task is complete
 * @private: Private data for the consumer
 * @work: Used internally by the mailbox
 * @doe_mb: Used internally by the mailbox
 *
 * The payload sizes and rv are specified in bytes with the following
 * restrictions concerning the protocol.
 *
 *	1) The request_pl_sz must be a multiple of double words (4 bytes)
 *	2) The response_pl_sz must be >= a single double word (4 bytes)
 *	3) rv is returned as bytes but it will be a multiple of double words
 *
 * NOTE there is no need for the caller to initialize work or doe_mb.
 */
struct pci_doe_task {
	struct pci_doe_protocol prot;
	u32 *request_pl;
	size_t request_pl_sz;
	u32 *response_pl;
	size_t response_pl_sz;
	int rv;
	void (*complete)(struct pci_doe_task *task);
	void *private;

	/* No need for the user to initialize these fields */
	struct work_struct work;
	struct pci_doe_mb *doe_mb;
};

one path to send request is doe_statemachine_work which calls pci_doe_send_req

static void doe_statemachine_work(struct work_struct *work)
{
	struct pci_doe_task *task = container_of(work, struct pci_doe_task,
						 work);
	struct pci_doe_mb *doe_mb = task->doe_mb;
	struct pci_dev *pdev = doe_mb->pdev;
	int offset = doe_mb->cap_offset;
	unsigned long timeout_jiffies;
	u32 val;
	int rc;


	/* Send request */
	rc = pci_doe_send_req(doe_mb, task);

doe_statemachine_work is called from pci_doe_submit_task

int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
{
	if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
		return -EINVAL;

...
...
	task->doe_mb = doe_mb;
	INIT_WORK(&task->work, doe_statemachine_work);

pci_doe_submit_task is called from pci_doe_discovery

static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid,
			     u8 *protocol)
{
	u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
				    *index);
	u32 response_pl;
	DECLARE_COMPLETION_ONSTACK(c);
	struct pci_doe_task task = {
		.prot.vid = PCI_VENDOR_ID_PCI_SIG,
		.prot.type = PCI_DOE_PROTOCOL_DISCOVERY,
		.request_pl = &request_pl,
		.request_pl_sz = sizeof(request_pl),
		.response_pl = &response_pl,
		.response_pl_sz = sizeof(response_pl),
		.complete = pci_doe_task_complete,
		.private = &c,
	};
	int rc;

	rc = pci_doe_submit_task(doe_mb, &task);

static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
{
	u8 index = 0;
	u8 xa_idx = 0;

	do {
		int rc;
		u16 vid;
		u8 prot;

		rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);

pci_doe_discovery is called by pcim_doe_create_mb

struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
{

...
...

	rc = pci_doe_cache_protocols(doe_mb);

pcim_doe_create_mb is called in the probe above section that create traces mailbox creation.