Skip to content

Commit fa62285

Browse files
Jason Chienalistair23
Jason Chien
authored and
alistair23
committed
hw/riscv/riscv-iommu.c: Introduce a translation tag for the page table cache
This commit introduces a translation tag to avoid invalidating an entry that should not be invalidated when IOMMU executes invalidation commands. E.g. IOTINVAL.VMA with GV=0, AV=0, PSCV=1 invalidates both a mapping of single stage translation and a mapping of nested translation with the same PSCID, but only the former one should be invalidated. Signed-off-by: Jason Chien <jason.chien@sifive.com> Reviewed-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com> Message-ID: <20241108110147.11178-1-jason.chien@sifive.com> Signed-off-by: Alistair Francis <alistair.francis@wdc.com>
1 parent 2d8e825 commit fa62285

File tree

1 file changed

+153
-52
lines changed

1 file changed

+153
-52
lines changed

‎hw/riscv/riscv-iommu.c‎

Lines changed: 153 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,16 @@ struct RISCVIOMMUContext {
6464
uint64_t msiptp; /* MSI redirection page table pointer */
6565
};
6666

67+
typedef enum RISCVIOMMUTransTag {
68+
RISCV_IOMMU_TRANS_TAG_BY, /* Bypass */
69+
RISCV_IOMMU_TRANS_TAG_SS, /* Single Stage */
70+
RISCV_IOMMU_TRANS_TAG_VG, /* G-stage only */
71+
RISCV_IOMMU_TRANS_TAG_VN, /* Nested translation */
72+
} RISCVIOMMUTransTag;
73+
6774
/* Address translation cache entry */
6875
struct RISCVIOMMUEntry {
76+
RISCVIOMMUTransTag tag; /* Translation Tag */
6977
uint64_t iova:44; /* IOVA Page Number */
7078
uint64_t pscid:20; /* Process Soft-Context identifier */
7179
uint64_t phys:44; /* Physical Page Number */
@@ -1227,7 +1235,7 @@ static gboolean riscv_iommu_iot_equal(gconstpointer v1, gconstpointer v2)
12271235
RISCVIOMMUEntry *t1 = (RISCVIOMMUEntry *) v1;
12281236
RISCVIOMMUEntry *t2 = (RISCVIOMMUEntry *) v2;
12291237
return t1->gscid == t2->gscid && t1->pscid == t2->pscid &&
1230-
t1->iova == t2->iova;
1238+
t1->iova == t2->iova && t1->tag == t2->tag;
12311239
}
12321240

12331241
static guint riscv_iommu_iot_hash(gconstpointer v)
@@ -1236,67 +1244,115 @@ static guint riscv_iommu_iot_hash(gconstpointer v)
12361244
return (guint)t->iova;
12371245
}
12381246

1239-
/* GV: 1 PSCV: 1 AV: 1 */
1247+
/* GV: 0 AV: 0 PSCV: 0 GVMA: 0 */
1248+
/* GV: 0 AV: 0 GVMA: 1 */
1249+
static
1250+
void riscv_iommu_iot_inval_all(gpointer key, gpointer value, gpointer data)
1251+
{
1252+
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
1253+
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1254+
if (iot->tag == arg->tag) {
1255+
iot->perm = IOMMU_NONE;
1256+
}
1257+
}
1258+
1259+
/* GV: 0 AV: 0 PSCV: 1 GVMA: 0 */
1260+
static
1261+
void riscv_iommu_iot_inval_pscid(gpointer key, gpointer value, gpointer data)
1262+
{
1263+
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
1264+
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1265+
if (iot->tag == arg->tag &&
1266+
iot->pscid == arg->pscid) {
1267+
iot->perm = IOMMU_NONE;
1268+
}
1269+
}
1270+
1271+
/* GV: 0 AV: 1 PSCV: 0 GVMA: 0 */
1272+
static
1273+
void riscv_iommu_iot_inval_iova(gpointer key, gpointer value, gpointer data)
1274+
{
1275+
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
1276+
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1277+
if (iot->tag == arg->tag &&
1278+
iot->iova == arg->iova) {
1279+
iot->perm = IOMMU_NONE;
1280+
}
1281+
}
1282+
1283+
/* GV: 0 AV: 1 PSCV: 1 GVMA: 0 */
12401284
static void riscv_iommu_iot_inval_pscid_iova(gpointer key, gpointer value,
12411285
gpointer data)
12421286
{
12431287
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
12441288
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1245-
if (iot->gscid == arg->gscid &&
1289+
if (iot->tag == arg->tag &&
12461290
iot->pscid == arg->pscid &&
12471291
iot->iova == arg->iova) {
12481292
iot->perm = IOMMU_NONE;
12491293
}
12501294
}
12511295

1252-
/* GV: 1 PSCV: 1 AV: 0 */
1253-
static void riscv_iommu_iot_inval_pscid(gpointer key, gpointer value,
1254-
gpointer data)
1296+
/* GV: 1 AV: 0 PSCV: 0 GVMA: 0 */
1297+
/* GV: 1 AV: 0 GVMA: 1 */
1298+
static
1299+
void riscv_iommu_iot_inval_gscid(gpointer key, gpointer value, gpointer data)
12551300
{
12561301
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
12571302
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1258-
if (iot->gscid == arg->gscid &&
1259-
iot->pscid == arg->pscid) {
1303+
if (iot->tag == arg->tag &&
1304+
iot->gscid == arg->gscid) {
12601305
iot->perm = IOMMU_NONE;
12611306
}
12621307
}
12631308

1264-
/* GV: 1 GVMA: 1 */
1265-
static void riscv_iommu_iot_inval_gscid_gpa(gpointer key, gpointer value,
1266-
gpointer data)
1309+
/* GV: 1 AV: 0 PSCV: 1 GVMA: 0 */
1310+
static void riscv_iommu_iot_inval_gscid_pscid(gpointer key, gpointer value,
1311+
gpointer data)
12671312
{
12681313
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
12691314
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1270-
if (iot->gscid == arg->gscid) {
1271-
/* simplified cache, no GPA matching */
1315+
if (iot->tag == arg->tag &&
1316+
iot->gscid == arg->gscid &&
1317+
iot->pscid == arg->pscid) {
12721318
iot->perm = IOMMU_NONE;
12731319
}
12741320
}
12751321

1276-
/* GV: 1 GVMA: 0 */
1277-
static void riscv_iommu_iot_inval_gscid(gpointer key, gpointer value,
1278-
gpointer data)
1322+
/* GV: 1 AV: 1 PSCV: 0 GVMA: 0 */
1323+
/* GV: 1 AV: 1 GVMA: 1 */
1324+
static void riscv_iommu_iot_inval_gscid_iova(gpointer key, gpointer value,
1325+
gpointer data)
12791326
{
12801327
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
12811328
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1282-
if (iot->gscid == arg->gscid) {
1329+
if (iot->tag == arg->tag &&
1330+
iot->gscid == arg->gscid &&
1331+
iot->iova == arg->iova) {
12831332
iot->perm = IOMMU_NONE;
12841333
}
12851334
}
12861335

1287-
/* GV: 0 */
1288-
static void riscv_iommu_iot_inval_all(gpointer key, gpointer value,
1289-
gpointer data)
1336+
/* GV: 1 AV: 1 PSCV: 1 GVMA: 0 */
1337+
static void riscv_iommu_iot_inval_gscid_pscid_iova(gpointer key, gpointer value,
1338+
gpointer data)
12901339
{
12911340
RISCVIOMMUEntry *iot = (RISCVIOMMUEntry *) value;
1292-
iot->perm = IOMMU_NONE;
1341+
RISCVIOMMUEntry *arg = (RISCVIOMMUEntry *) data;
1342+
if (iot->tag == arg->tag &&
1343+
iot->gscid == arg->gscid &&
1344+
iot->pscid == arg->pscid &&
1345+
iot->iova == arg->iova) {
1346+
iot->perm = IOMMU_NONE;
1347+
}
12931348
}
12941349

12951350
/* caller should keep ref-count for iot_cache object */
12961351
static RISCVIOMMUEntry *riscv_iommu_iot_lookup(RISCVIOMMUContext *ctx,
1297-
GHashTable *iot_cache, hwaddr iova)
1352+
GHashTable *iot_cache, hwaddr iova, RISCVIOMMUTransTag transtag)
12981353
{
12991354
RISCVIOMMUEntry key = {
1355+
.tag = transtag,
13001356
.gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID),
13011357
.pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID),
13021358
.iova = PPN_DOWN(iova),
@@ -1322,10 +1378,11 @@ static void riscv_iommu_iot_update(RISCVIOMMUState *s,
13221378
}
13231379

13241380
static void riscv_iommu_iot_inval(RISCVIOMMUState *s, GHFunc func,
1325-
uint32_t gscid, uint32_t pscid, hwaddr iova)
1381+
uint32_t gscid, uint32_t pscid, hwaddr iova, RISCVIOMMUTransTag transtag)
13261382
{
13271383
GHashTable *iot_cache;
13281384
RISCVIOMMUEntry key = {
1385+
.tag = transtag,
13291386
.gscid = gscid,
13301387
.pscid = pscid,
13311388
.iova = PPN_DOWN(iova),
@@ -1336,9 +1393,24 @@ static void riscv_iommu_iot_inval(RISCVIOMMUState *s, GHFunc func,
13361393
g_hash_table_unref(iot_cache);
13371394
}
13381395

1396+
static RISCVIOMMUTransTag riscv_iommu_get_transtag(RISCVIOMMUContext *ctx)
1397+
{
1398+
uint64_t satp = get_field(ctx->satp, RISCV_IOMMU_ATP_MODE_FIELD);
1399+
uint64_t gatp = get_field(ctx->gatp, RISCV_IOMMU_ATP_MODE_FIELD);
1400+
1401+
if (satp == RISCV_IOMMU_DC_FSC_MODE_BARE) {
1402+
return (gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) ?
1403+
RISCV_IOMMU_TRANS_TAG_BY : RISCV_IOMMU_TRANS_TAG_VG;
1404+
} else {
1405+
return (gatp == RISCV_IOMMU_DC_IOHGATP_MODE_BARE) ?
1406+
RISCV_IOMMU_TRANS_TAG_SS : RISCV_IOMMU_TRANS_TAG_VN;
1407+
}
1408+
}
1409+
13391410
static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
13401411
IOMMUTLBEntry *iotlb, bool enable_cache)
13411412
{
1413+
RISCVIOMMUTransTag transtag = riscv_iommu_get_transtag(ctx);
13421414
RISCVIOMMUEntry *iot;
13431415
IOMMUAccessFlags perm;
13441416
bool enable_pid;
@@ -1364,7 +1436,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
13641436
}
13651437
}
13661438

1367-
iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova);
1439+
iot = riscv_iommu_iot_lookup(ctx, iot_cache, iotlb->iova, transtag);
13681440
perm = iot ? iot->perm : IOMMU_NONE;
13691441
if (perm != IOMMU_NONE) {
13701442
iotlb->translated_addr = PPN_PHYS(iot->phys);
@@ -1395,6 +1467,7 @@ static int riscv_iommu_translate(RISCVIOMMUState *s, RISCVIOMMUContext *ctx,
13951467
iot->gscid = get_field(ctx->gatp, RISCV_IOMMU_DC_IOHGATP_GSCID);
13961468
iot->pscid = get_field(ctx->ta, RISCV_IOMMU_DC_TA_PSCID);
13971469
iot->perm = iotlb->perm;
1470+
iot->tag = transtag;
13981471
riscv_iommu_iot_update(s, iot_cache, iot);
13991472
}
14001473

@@ -1602,44 +1675,72 @@ static void riscv_iommu_process_cq_tail(RISCVIOMMUState *s)
16021675

16031676
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA,
16041677
RISCV_IOMMU_CMD_IOTINVAL_OPCODE):
1605-
if (cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV) {
1678+
{
1679+
bool gv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV);
1680+
bool av = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV);
1681+
bool pscv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV);
1682+
uint32_t gscid = get_field(cmd.dword0,
1683+
RISCV_IOMMU_CMD_IOTINVAL_GSCID);
1684+
uint32_t pscid = get_field(cmd.dword0,
1685+
RISCV_IOMMU_CMD_IOTINVAL_PSCID);
1686+
hwaddr iova = (cmd.dword1 << 2) & TARGET_PAGE_MASK;
1687+
1688+
if (pscv) {
16061689
/* illegal command arguments IOTINVAL.GVMA & PSCV == 1 */
16071690
goto cmd_ill;
1608-
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV)) {
1609-
/* invalidate all cache mappings */
1610-
func = riscv_iommu_iot_inval_all;
1611-
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV)) {
1612-
/* invalidate cache matching GSCID */
1613-
func = riscv_iommu_iot_inval_gscid;
1614-
} else {
1615-
/* invalidate cache matching GSCID and ADDR (GPA) */
1616-
func = riscv_iommu_iot_inval_gscid_gpa;
16171691
}
1618-
riscv_iommu_iot_inval(s, func,
1619-
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOTINVAL_GSCID), 0,
1620-
cmd.dword1 << 2 & TARGET_PAGE_MASK);
1692+
1693+
func = riscv_iommu_iot_inval_all;
1694+
1695+
if (gv) {
1696+
func = (av) ? riscv_iommu_iot_inval_gscid_iova :
1697+
riscv_iommu_iot_inval_gscid;
1698+
}
1699+
1700+
riscv_iommu_iot_inval(
1701+
s, func, gscid, pscid, iova, RISCV_IOMMU_TRANS_TAG_VG);
1702+
1703+
riscv_iommu_iot_inval(
1704+
s, func, gscid, pscid, iova, RISCV_IOMMU_TRANS_TAG_VN);
16211705
break;
1706+
}
16221707

16231708
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA,
16241709
RISCV_IOMMU_CMD_IOTINVAL_OPCODE):
1625-
if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV)) {
1626-
/* invalidate all cache mappings, simplified model */
1627-
func = riscv_iommu_iot_inval_all;
1628-
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV)) {
1629-
/* invalidate cache matching GSCID, simplified model */
1630-
func = riscv_iommu_iot_inval_gscid;
1631-
} else if (!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV)) {
1632-
/* invalidate cache matching GSCID and PSCID */
1633-
func = riscv_iommu_iot_inval_pscid;
1710+
{
1711+
bool gv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_GV);
1712+
bool av = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_AV);
1713+
bool pscv = !!(cmd.dword0 & RISCV_IOMMU_CMD_IOTINVAL_PSCV);
1714+
uint32_t gscid = get_field(cmd.dword0,
1715+
RISCV_IOMMU_CMD_IOTINVAL_GSCID);
1716+
uint32_t pscid = get_field(cmd.dword0,
1717+
RISCV_IOMMU_CMD_IOTINVAL_PSCID);
1718+
hwaddr iova = (cmd.dword1 << 2) & TARGET_PAGE_MASK;
1719+
RISCVIOMMUTransTag transtag;
1720+
1721+
if (gv) {
1722+
transtag = RISCV_IOMMU_TRANS_TAG_VN;
1723+
if (pscv) {
1724+
func = (av) ? riscv_iommu_iot_inval_gscid_pscid_iova :
1725+
riscv_iommu_iot_inval_gscid_pscid;
1726+
} else {
1727+
func = (av) ? riscv_iommu_iot_inval_gscid_iova :
1728+
riscv_iommu_iot_inval_gscid;
1729+
}
16341730
} else {
1635-
/* invalidate cache matching GSCID and PSCID and ADDR (IOVA) */
1636-
func = riscv_iommu_iot_inval_pscid_iova;
1731+
transtag = RISCV_IOMMU_TRANS_TAG_SS;
1732+
if (pscv) {
1733+
func = (av) ? riscv_iommu_iot_inval_pscid_iova :
1734+
riscv_iommu_iot_inval_pscid;
1735+
} else {
1736+
func = (av) ? riscv_iommu_iot_inval_iova :
1737+
riscv_iommu_iot_inval_all;
1738+
}
16371739
}
1638-
riscv_iommu_iot_inval(s, func,
1639-
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOTINVAL_GSCID),
1640-
get_field(cmd.dword0, RISCV_IOMMU_CMD_IOTINVAL_PSCID),
1641-
cmd.dword1 << 2 & TARGET_PAGE_MASK);
1740+
1741+
riscv_iommu_iot_inval(s, func, gscid, pscid, iova, transtag);
16421742
break;
1743+
}
16431744

16441745
case RISCV_IOMMU_CMD(RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT,
16451746
RISCV_IOMMU_CMD_IODIR_OPCODE):

0 commit comments

Comments
 (0)