--- /usr/src/sys/amd64/amd64/pmap.c +++ /usr/src/sys/amd64/amd64/pmap.c @@ -1792,45 +1792,0 @@ -static int pmap_allow_2m_x_ept; -SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN | CTLFLAG_NOFETCH, - &pmap_allow_2m_x_ept, 0, - "Allow executable superpage mappings in EPT"); - -void -pmap_allow_2m_x_ept_recalculate(void) -{ - /* - * SKL002, SKL012S. Since the EPT format is only used by - * Intel CPUs, the vendor check is merely a formality. - */ - if (!(cpu_vendor_id != CPU_VENDOR_INTEL || - (cpu_ia32_arch_caps & IA32_ARCH_CAP_IF_PSCHANGE_MC_NO) != 0 || - (CPUID_TO_FAMILY(cpu_id) == 0x6 && - (CPUID_TO_MODEL(cpu_id) == 0x26 || /* Atoms */ - CPUID_TO_MODEL(cpu_id) == 0x27 || - CPUID_TO_MODEL(cpu_id) == 0x35 || - CPUID_TO_MODEL(cpu_id) == 0x36 || - CPUID_TO_MODEL(cpu_id) == 0x37 || - CPUID_TO_MODEL(cpu_id) == 0x86 || - CPUID_TO_MODEL(cpu_id) == 0x1c || - CPUID_TO_MODEL(cpu_id) == 0x4a || - CPUID_TO_MODEL(cpu_id) == 0x4c || - CPUID_TO_MODEL(cpu_id) == 0x4d || - CPUID_TO_MODEL(cpu_id) == 0x5a || - CPUID_TO_MODEL(cpu_id) == 0x5c || - CPUID_TO_MODEL(cpu_id) == 0x5d || - CPUID_TO_MODEL(cpu_id) == 0x5f || - CPUID_TO_MODEL(cpu_id) == 0x6e || - CPUID_TO_MODEL(cpu_id) == 0x7a || - CPUID_TO_MODEL(cpu_id) == 0x57 || /* Knights */ - CPUID_TO_MODEL(cpu_id) == 0x85)))) - pmap_allow_2m_x_ept = 1; - TUNABLE_INT_FETCH("hw.allow_2m_x_ept", &pmap_allow_2m_x_ept); -} - -static bool -pmap_allow_2m_x_page(pmap_t pmap, bool executable) -{ - - return (pmap->pm_type != PT_EPT || !executable || - !pmap_allow_2m_x_ept); -} - @@ -1881,3 +1836,0 @@ - /* IFU */ - pmap_allow_2m_x_ept_recalculate(); - @@ -5453,9 +5405,0 @@ -static bool -pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde) -{ - - if (pmap->pm_type != PT_EPT) - return (false); - return ((pde & EPT_PG_EXECUTE) != 0); -} - @@ -5497,3 +5440,1 @@ - if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V) || - !pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, - newpde))) { --- + if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) { @@ -5929,6 +5870,0 @@ - if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap, - newpde))) { - CTR2(KTR_PMAP, "pmap_enter_pde: 2m x blocked for va %#lx" - " in pmap %p", va, pmap); - return (KERN_FAILURE); - } @@ -6081,1 +6016,0 @@ - pmap_allow_2m_x_page(pmap, (prot & VM_PROT_EXECUTE) != 0) && --- /usr/src/sys/amd64/include/pmap.h +++ /usr/src/sys/amd64/include/pmap.h @@ -427,1 +427,0 @@ -void pmap_allow_2m_x_ept_recalculate(void); --- /usr/src/sys/conf/files +++ /usr/src/sys/conf/files @@ -3481,3 +3481,0 @@ -dev/virtio/pci/virtio_pci_if.m optional virtio_pci -dev/virtio/pci/virtio_pci_legacy.c optional virtio_pci -dev/virtio/pci/virtio_pci_modern.c optional virtio_pci --- /usr/src/sys/conf/newvers.sh +++ /usr/src/sys/conf/newvers.sh @@ -49,1 +49,1 @@ -BRANCH="RELEASE-p1" --- +BRANCH="RELEASE" --- /usr/src/sys/dev/cpuctl/cpuctl.c +++ /usr/src/sys/dev/cpuctl/cpuctl.c @@ -53,4 +53,0 @@ -#include -#include -#include - @@ -546,1 +542,0 @@ - pmap_allow_2m_x_ept_recalculate(); --- /usr/src/sys/dev/netmap/if_ptnet.c +++ /usr/src/sys/dev/netmap/if_ptnet.c @@ -90,2 +90,0 @@ -#ifdef WITH_PTNETMAP - @@ -2296,1 +2294,0 @@ -#endif /* WITH_PTNETMAP */ --- /usr/src/sys/dev/virtio/balloon/virtio_balloon.c +++ /usr/src/sys/dev/virtio/balloon/virtio_balloon.c @@ -32,1 +32,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/balloon/virtio_balloon.c 326255 2017-11-27 14:52:40Z pfg $"); @@ -83,1 +83,0 @@ - { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" }, @@ -93,2 +92,1 @@ -static int vtballoon_negotiate_features(struct vtballoon_softc *); -static int vtballoon_setup_features(struct vtballoon_softc *); --- +static void vtballoon_negotiate_features(struct vtballoon_softc *); @@ -114,1 +112,1 @@ -static void vtballoon_setup_sysctl(struct vtballoon_softc *); --- +static void vtballoon_add_sysctl(struct vtballoon_softc *); @@ -116,3 +114,0 @@ -#define vtballoon_modern(_sc) \ - (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0) - @@ -120,1 +115,1 @@ -#define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST --- +#define VTBALLOON_FEATURES 0 @@ -161,1 +156,1 @@ -DRIVER_MODULE(virtio_balloon, vtpcil, vtballoon_driver, --- +DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver, @@ -163,2 +158,0 @@ -DRIVER_MODULE(virtio_balloon, vtpcim, vtballoon_driver, - vtballoon_devclass, 0, 0); @@ -188,1 +181,0 @@ - virtio_set_feature_desc(dev, vtballoon_feature_desc); @@ -193,1 +185,1 @@ - vtballoon_setup_sysctl(sc); --- + vtballoon_add_sysctl(sc); @@ -195,5 +187,2 @@ - error = vtballoon_setup_features(sc); - if (error) { - device_printf(dev, "cannot setup features\n"); - goto fail; - } --- + virtio_set_feature_desc(dev, vtballoon_feature_desc); + vtballoon_negotiate_features(sc); @@ -285,1 +274,1 @@ -static int --- +static void @@ -292,4 +281,2 @@ - features = VTBALLOON_FEATURES; - - sc->vtballoon_features = virtio_negotiate_features(dev, features); - return (virtio_finalize_features(dev)); --- + features = virtio_negotiate_features(dev, VTBALLOON_FEATURES); + sc->vtballoon_features = features; @@ -299,12 +286,0 @@ -vtballoon_setup_features(struct vtballoon_softc *sc) -{ - int error; - - error = vtballoon_negotiate_features(sc); - if (error) - return (error); - - return (0); -} - -static int @@ -468,2 +443,1 @@ - m = vm_page_alloc(NULL, 0, - VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP); --- + m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ); @@ -492,4 +466,1 @@ - if (vtballoon_modern(sc)) - return (desired); - else - return (le32toh(desired)); --- + return (le32toh(desired)); @@ -501,1 +472,0 @@ - uint32_t npages; @@ -503,4 +473,0 @@ - npages = sc->vtballoon_current_npages; - if (!vtballoon_modern(sc)) - npages = htole32(npages); - @@ -508,1 +474,2 @@ - offsetof(struct virtio_balloon_config, actual), npages); --- + offsetof(struct virtio_balloon_config, actual), + htole32(sc->vtballoon_current_npages)); @@ -580,1 +547,1 @@ -vtballoon_setup_sysctl(struct vtballoon_softc *sc) --- +vtballoon_add_sysctl(struct vtballoon_softc *sc) --- /usr/src/sys/dev/virtio/balloon/virtio_balloon.h +++ /usr/src/sys/dev/virtio/balloon/virtio_balloon.h @@ -30,1 +30,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/balloon/virtio_balloon.h 326022 2017-11-20 19:36:21Z pfg $ @@ -39,1 +39,0 @@ -#define VIRTIO_BALLOON_F_DEFLATE_ON_OOM 0x4 /* Deflate balloon on OOM */ @@ -58,3 +57,1 @@ -#define VIRTIO_BALLOON_S_AVAIL 6 /* Available memory as in /proc */ -#define VIRTIO_BALLOON_S_CACHES 7 /* Disk caches */ -#define VIRTIO_BALLOON_S_NR 8 --- +#define VIRTIO_BALLOON_S_NR 6 @@ -62,23 +59,0 @@ -/* - * Memory statistics structure. - * Driver fills an array of these structures and passes to device. - * - * NOTE: fields are laid out in a way that would make compiler add padding - * between and after fields, so we have to use compiler-specific attributes to - * pack it, to disable this padding. This also often causes compiler to - * generate suboptimal code. - * - * We maintain this statistics structure format for backwards compatibility, - * but don't follow this example. - * - * If implementing a similar structure, do something like the below instead: - * struct virtio_balloon_stat { - * __virtio16 tag; - * __u8 reserved[6]; - * __virtio64 val; - * }; - * - * In other words, add explicit reserved fields to align field and - * structure boundaries at field size, avoiding compiler padding - * without the packed attribute. - */ --- /usr/src/sys/dev/virtio/block/virtio_blk.c +++ /usr/src/sys/dev/virtio/block/virtio_blk.c @@ -32,1 +32,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/block/virtio_blk.c 326255 2017-11-27 14:52:40Z pfg $"); @@ -79,4 +79,5 @@ -#define VTBLK_FLAG_DETACH 0x0002 -#define VTBLK_FLAG_SUSPEND 0x0004 -#define VTBLK_FLAG_BARRIER 0x0008 -#define VTBLK_FLAG_WCE_CONFIG 0x0010 --- +#define VTBLK_FLAG_READONLY 0x0002 +#define VTBLK_FLAG_DETACH 0x0004 +#define VTBLK_FLAG_SUSPEND 0x0008 +#define VTBLK_FLAG_BARRIER 0x0010 +#define VTBLK_FLAG_WC_CONFIG 0x0020 @@ -111,1 +112,1 @@ - { VIRTIO_BLK_F_FLUSH, "FlushCmd" }, --- + { VIRTIO_BLK_F_WCE, "WriteCache" }, @@ -114,1 +115,0 @@ - { VIRTIO_BLK_F_MQ, "Multiqueue" }, @@ -136,2 +136,2 @@ -static int vtblk_negotiate_features(struct vtblk_softc *); -static int vtblk_setup_features(struct vtblk_softc *); --- +static void vtblk_negotiate_features(struct vtblk_softc *); +static void vtblk_setup_features(struct vtblk_softc *); @@ -196,8 +196,0 @@ -#define vtblk_modern(_sc) (((_sc)->vtblk_features & VIRTIO_F_VERSION_1) != 0) -#define vtblk_htog16(_sc, _val) virtio_htog16(vtblk_modern(_sc), _val) -#define vtblk_htog32(_sc, _val) virtio_htog32(vtblk_modern(_sc), _val) -#define vtblk_htog64(_sc, _val) virtio_htog64(vtblk_modern(_sc), _val) -#define vtblk_gtoh16(_sc, _val) virtio_gtoh16(vtblk_modern(_sc), _val) -#define vtblk_gtoh32(_sc, _val) virtio_gtoh32(vtblk_modern(_sc), _val) -#define vtblk_gtoh64(_sc, _val) virtio_gtoh64(vtblk_modern(_sc), _val) - @@ -210,2 +202,4 @@ -#define VTBLK_COMMON_FEATURES \ - (VIRTIO_BLK_F_SIZE_MAX | \ --- +/* Features desired/implemented by this driver. */ +#define VTBLK_FEATURES \ + (VIRTIO_BLK_F_BARRIER | \ + VIRTIO_BLK_F_SIZE_MAX | \ @@ -216,1 +210,1 @@ - VIRTIO_BLK_F_FLUSH | \ --- + VIRTIO_BLK_F_WCE | \ @@ -221,3 +215,0 @@ -#define VTBLK_MODERN_FEATURES (VTBLK_COMMON_FEATURES) -#define VTBLK_LEGACY_FEATURES (VIRTIO_BLK_F_BARRIER | VTBLK_COMMON_FEATURES) - @@ -268,1 +259,1 @@ -DRIVER_MODULE(virtio_blk, vtpcil, vtblk_driver, vtblk_devclass, --- +DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass, @@ -270,2 +261,0 @@ -DRIVER_MODULE(virtio_blk, vtpcim, vtblk_driver, vtblk_devclass, - vtblk_modevent, 0); @@ -315,2 +304,0 @@ - sc = device_get_softc(dev); - sc->vtblk_dev = dev; @@ -319,0 +306,2 @@ + sc = device_get_softc(dev); + sc->vtblk_dev = dev; @@ -326,0 +315,1 @@ + vtblk_setup_features(sc); @@ -327,6 +317,0 @@ - error = vtblk_setup_features(sc); - if (error) { - device_printf(dev, "cannot setup features\n"); - goto fail; - } - @@ -560,0 +544,10 @@ + /* + * Fail any write if RO. Unfortunately, there does not seem to + * be a better way to report our readonly'ness to GEOM above. + */ + if (sc->vtblk_flags & VTBLK_FLAG_READONLY && + (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) { + vtblk_bio_done(sc, bp, EROFS); + return; + } + @@ -574,1 +568,1 @@ -static int --- +static void @@ -581,2 +575,1 @@ - features = virtio_bus_is_modern(dev) ? VTBLK_MODERN_FEATURES : - VTBLK_LEGACY_FEATURES; --- + features = VTBLK_FEATURES; @@ -585,1 +578,0 @@ - return (virtio_finalize_features(dev)); @@ -588,1 +580,1 @@ -static int --- +static void @@ -592,1 +584,0 @@ - int error; @@ -596,3 +587,1 @@ - error = vtblk_negotiate_features(sc); - if (error) - return (error); --- + vtblk_negotiate_features(sc); @@ -602,2 +591,2 @@ - if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE)) - sc->vtblk_flags |= VTBLK_FLAG_WCE_CONFIG; --- + if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) + sc->vtblk_flags |= VTBLK_FLAG_READONLY; @@ -605,3 +594,3 @@ - sc->vtblk_flags |= VTBLK_FLAG_BARRIER; /* Legacy. */ - - return (0); --- + sc->vtblk_flags |= VTBLK_FLAG_BARRIER; + if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE)) + sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG; @@ -686,1 +675,2 @@ - dp->d_flags = DISKFLAG_UNMAPPED_BIO | DISKFLAG_DIRECT_COMPLETION; --- + dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO | + DISKFLAG_DIRECT_COMPLETION; @@ -692,5 +682,1 @@ - if (virtio_with_feature(dev, VIRTIO_BLK_F_RO)) - dp->d_flags |= DISKFLAG_WRITE_PROTECT; - else { - if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH)) - dp->d_flags |= DISKFLAG_CANFLUSHCACHE; --- + if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0) @@ -698,1 +684,0 @@ - } @@ -882,1 +867,1 @@ - req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); --- + req->vbr_hdr.ioprio = 1; @@ -886,2 +871,1 @@ - req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH); - req->vbr_hdr.sector = 0; --- + req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; @@ -890,2 +874,2 @@ - req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN); - req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / 512); --- + req->vbr_hdr.type = VIRTIO_BLK_T_IN; + req->vbr_hdr.sector = bp->bio_offset / 512; @@ -894,2 +878,2 @@ - req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT); - req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / 512); --- + req->vbr_hdr.type = VIRTIO_BLK_T_OUT; + req->vbr_hdr.sector = bp->bio_offset / 512; @@ -902,1 +886,1 @@ - req->vbr_hdr.type |= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER); --- + req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER; @@ -933,2 +917,1 @@ - req->vbr_hdr.type &= vtblk_gtoh32(sc, - ~VIRTIO_BLK_T_BARRIER); --- + req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER; @@ -1038,0 +1021,1 @@ + struct bio_queue queue; @@ -1043,0 +1027,1 @@ + TAILQ_INIT(&queue); @@ -1045,3 +1030,0 @@ - struct bio_queue queue; - - TAILQ_INIT(&queue); @@ -1138,6 +1120,1 @@ - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, - geometry.cylinders, blkcfg); - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, - geometry.heads, blkcfg); - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, - geometry.sectors, blkcfg); --- + VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg); @@ -1145,9 +1122,2 @@ - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, - topology.physical_block_exp, blkcfg); - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, - topology.alignment_offset, blkcfg); - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, - topology.min_io_size, blkcfg); - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, - topology.opt_io_size, blkcfg); - VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, wce, blkcfg); --- + VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg); + VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg); @@ -1177,2 +1147,2 @@ - req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID); - req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); --- + req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID; + req->vbr_hdr.ioprio = 1; @@ -1309,3 +1279,3 @@ - req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT); - req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); - req->vbr_hdr.sector = vtblk_gtoh64(sc, offset / 512); --- + req->vbr_hdr.type = VIRTIO_BLK_T_OUT; + req->vbr_hdr.ioprio = 1; + req->vbr_hdr.sector = offset / 512; @@ -1331,2 +1301,2 @@ - req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH); - req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1); --- + req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH; + req->vbr_hdr.ioprio = 1; @@ -1360,1 +1330,1 @@ - offsetof(struct virtio_blk_config, wce), wc); --- + offsetof(struct virtio_blk_config, writeback), wc); @@ -1369,1 +1339,1 @@ - if (sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) { --- + if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) { @@ -1375,1 +1345,1 @@ - wc = blkcfg->wce; --- + wc = blkcfg->writeback; @@ -1377,1 +1347,1 @@ - wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_FLUSH); --- + wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE); @@ -1394,1 +1364,1 @@ - if ((sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) == 0) --- + if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0) --- /usr/src/sys/dev/virtio/block/virtio_blk.h +++ /usr/src/sys/dev/virtio/block/virtio_blk.h @@ -30,1 +30,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/block/virtio_blk.h 326022 2017-11-20 19:36:21Z pfg $ @@ -37,0 +37,1 @@ +#define VIRTIO_BLK_F_BARRIER 0x0001 /* Does host support barriers? */ @@ -42,1 +43,2 @@ -#define VIRTIO_BLK_F_FLUSH 0x0200 /* Flush command supported */ --- +#define VIRTIO_BLK_F_SCSI 0x0080 /* Supports scsi command passthru */ +#define VIRTIO_BLK_F_WCE 0x0200 /* Writeback mode enabled after reset */ @@ -45,1 +47,0 @@ -#define VIRTIO_BLK_F_MQ 0x1000 /* Support more than one vq */ @@ -47,7 +48,0 @@ -/* Legacy feature bits */ -#define VIRTIO_BLK_F_BARRIER 0x0001 /* Does host support barriers? */ -#define VIRTIO_BLK_F_SCSI 0x0080 /* Supports scsi command passthru */ - -/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */ -#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH - @@ -75,1 +69,0 @@ - /* exponent for physical block per logical block. */ @@ -77,1 +70,0 @@ - /* alignment offset in logical blocks. */ @@ -79,2 +71,0 @@ - /* minimum I/O size without performance penalty in logical - * blocks. */ @@ -82,1 +72,0 @@ - /* optimal sustained I/O size in logical blocks. */ @@ -87,2 +76,1 @@ - uint8_t wce; - uint8_t unused; --- + uint8_t writeback; @@ -90,2 +78,0 @@ - /* Number of vqs, only available when VIRTIO_BLK_F_MQ is set */ - uint16_t num_queues; @@ -124,5 +110,1 @@ -/* - * This comes first in the read scatter-gather list. - * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated, - * this is the first element of the read scatter-gather list. - */ --- +/* This is the first element of the read scatter-gather list. */ --- /usr/src/sys/dev/virtio/console/virtio_console.c +++ /usr/src/sys/dev/virtio/console/virtio_console.c @@ -30,1 +30,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/console/virtio_console.c 328218 2018-01-21 15:42:36Z pfg $"); @@ -161,2 +161,2 @@ -static int vtcon_setup_features(struct vtcon_softc *); -static int vtcon_negotiate_features(struct vtcon_softc *); --- +static void vtcon_setup_features(struct vtcon_softc *); +static void vtcon_negotiate_features(struct vtcon_softc *); @@ -230,8 +230,0 @@ -#define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0) -#define vtcon_htog16(_sc, _val) virtio_htog16(vtcon_modern(_sc), _val) -#define vtcon_htog32(_sc, _val) virtio_htog32(vtcon_modern(_sc), _val) -#define vtcon_htog64(_sc, _val) virtio_htog64(vtcon_modern(_sc), _val) -#define vtcon_gtoh16(_sc, _val) virtio_gtoh16(vtcon_modern(_sc), _val) -#define vtcon_gtoh32(_sc, _val) virtio_gtoh32(vtcon_modern(_sc), _val) -#define vtcon_gtoh64(_sc, _val) virtio_gtoh64(vtcon_modern(_sc), _val) - @@ -267,1 +259,1 @@ -DRIVER_MODULE(virtio_console, vtpcil, vtcon_driver, vtcon_devclass, --- +DRIVER_MODULE(virtio_console, virtio_pci, vtcon_driver, vtcon_devclass, @@ -269,2 +261,0 @@ -DRIVER_MODULE(virtio_console, vtpcim, vtcon_driver, vtcon_devclass, - vtcon_modevent, 0); @@ -336,1 +326,0 @@ - virtio_set_feature_desc(dev, vtcon_feature_desc); @@ -341,5 +330,2 @@ - error = vtcon_setup_features(sc); - if (error) { - device_printf(dev, "cannot setup features\n"); - goto fail; - } --- + virtio_set_feature_desc(dev, vtcon_feature_desc); + vtcon_setup_features(sc); @@ -437,1 +423,1 @@ -static int --- +static void @@ -447,1 +433,0 @@ - return (virtio_finalize_features(dev)); @@ -450,1 +435,1 @@ -static int --- +static void @@ -454,1 +439,0 @@ - int error; @@ -458,3 +442,1 @@ - error = vtcon_negotiate_features(sc); - if (error) - return (error); --- + vtcon_negotiate_features(sc); @@ -466,2 +448,0 @@ - - return (0); @@ -870,2 +850,1 @@ - uint32_t id; - uint16_t event; --- + int id; @@ -874,2 +853,1 @@ - id = vtcon_htog32(sc, control->id); - event = vtcon_htog16(sc, control->event); --- + id = control->id; @@ -877,3 +855,2 @@ - if (id >= sc->vtcon_max_ports) { - device_printf(dev, "%s: event %d invalid port ID %d\n", - __func__, event, id); --- + if (id < 0 || id >= sc->vtcon_max_ports) { + device_printf(dev, "%s: invalid port ID %d\n", __func__, id); @@ -883,1 +860,1 @@ - switch (event) { --- + switch (control->event) { @@ -1011,3 +988,3 @@ - control.id = vtcon_gtoh32(sc, portid); - control.event = vtcon_gtoh16(sc, event); - control.value = vtcon_gtoh16(sc, value); --- + control.id = portid; + control.event = event; + control.value = value; --- /usr/src/sys/dev/virtio/console/virtio_console.h +++ /usr/src/sys/dev/virtio/console/virtio_console.h @@ -31,1 +31,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/console/virtio_console.h 273515 2014-10-23 04:47:32Z bryanv $ --- /usr/src/sys/dev/virtio/mmio/virtio_mmio.c +++ /usr/src/sys/dev/virtio/mmio/virtio_mmio.c @@ -51,1 +51,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/mmio/virtio_mmio.c 335211 2018-06-15 16:19:10Z br $"); @@ -428,4 +428,0 @@ - break; - case VIRTIO_IVAR_SUBVENDOR: - case VIRTIO_IVAR_MODERN: - *result = 0; --- /usr/src/sys/dev/virtio/mmio/virtio_mmio.h +++ /usr/src/sys/dev/virtio/mmio/virtio_mmio.h @@ -30,1 +30,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/mmio/virtio_mmio.h 284544 2015-06-18 10:33:04Z br $ --- /usr/src/sys/dev/virtio/mmio/virtio_mmio_if.m +++ /usr/src/sys/dev/virtio/mmio/virtio_mmio_if.m @@ -30,1 +30,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/dev/virtio/mmio/virtio_mmio_if.m 285091 2015-07-03 14:13:16Z br $ --- /usr/src/sys/dev/virtio/pci/virtio_pci.c +++ /usr/src/sys/dev/virtio/pci/virtio_pci.c @@ -4,1 +4,1 @@ - * Copyright (c) 2017, Bryan Venteicher --- + * Copyright (c) 2011, Bryan Venteicher @@ -29,0 +29,2 @@ +/* Driver for the VirtIO PCI interface. */ + @@ -30,1 +32,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.c 328218 2018-01-21 15:42:36Z pfg $"); @@ -36,2 +38,0 @@ -#include -#include @@ -52,1 +52,0 @@ -#include @@ -54,1 +53,1 @@ -#include "virtio_pci_if.h" --- +#include "virtio_bus_if.h" @@ -57,1 +56,73 @@ -static void vtpci_describe_features(struct vtpci_common *, const char *, --- +struct vtpci_interrupt { + struct resource *vti_irq; + int vti_rid; + void *vti_handler; +}; + +struct vtpci_virtqueue { + struct virtqueue *vtv_vq; + int vtv_no_intr; +}; + +struct vtpci_softc { + device_t vtpci_dev; + struct resource *vtpci_res; + struct resource *vtpci_msix_res; + uint64_t vtpci_features; + uint32_t vtpci_flags; +#define VTPCI_FLAG_NO_MSI 0x0001 +#define VTPCI_FLAG_NO_MSIX 0x0002 +#define VTPCI_FLAG_LEGACY 0x1000 +#define VTPCI_FLAG_MSI 0x2000 +#define VTPCI_FLAG_MSIX 0x4000 +#define VTPCI_FLAG_SHARED_MSIX 0x8000 +#define VTPCI_FLAG_ITYPE_MASK 0xF000 + + /* This "bus" will only ever have one child. */ + device_t vtpci_child_dev; + struct virtio_feature_desc *vtpci_child_feat_desc; + + int vtpci_nvqs; + struct vtpci_virtqueue *vtpci_vqs; + + /* + * Ideally, each virtqueue that the driver provides a callback for will + * receive its own MSIX vector. If there are not sufficient vectors + * available, then attempt to have all the VQs share one vector. For + * MSIX, the configuration changed notifications must be on their own + * vector. + * + * If MSIX is not available, we will attempt to have the whole device + * share one MSI vector, and then, finally, one legacy interrupt. + */ + struct vtpci_interrupt vtpci_device_interrupt; + struct vtpci_interrupt *vtpci_msix_vq_interrupts; + int vtpci_nmsix_resources; +}; + +static int vtpci_probe(device_t); +static int vtpci_attach(device_t); +static int vtpci_detach(device_t); +static int vtpci_suspend(device_t); +static int vtpci_resume(device_t); +static int vtpci_shutdown(device_t); +static void vtpci_driver_added(device_t, driver_t *); +static void vtpci_child_detached(device_t, device_t); +static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); +static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); + +static uint64_t vtpci_negotiate_features(device_t, uint64_t); +static int vtpci_with_feature(device_t, uint64_t); +static int vtpci_alloc_virtqueues(device_t, int, int, + struct vq_alloc_info *); +static int vtpci_setup_intr(device_t, enum intr_type); +static void vtpci_stop(device_t); +static int vtpci_reinit(device_t, uint64_t); +static void vtpci_reinit_complete(device_t); +static void vtpci_notify_virtqueue(device_t, uint16_t); +static uint8_t vtpci_get_status(device_t); +static void vtpci_set_status(device_t, uint8_t); +static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); +static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); + +static void vtpci_describe_features(struct vtpci_softc *, const char *, @@ -59,7 +130,9 @@ -static int vtpci_alloc_msix(struct vtpci_common *, int); -static int vtpci_alloc_msi(struct vtpci_common *); -static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *); -static int vtpci_alloc_intr_msix_shared(struct vtpci_common *); -static int vtpci_alloc_intr_msi(struct vtpci_common *); -static int vtpci_alloc_intr_intx(struct vtpci_common *); -static int vtpci_alloc_interrupt(struct vtpci_common *, int, int, --- +static void vtpci_probe_and_attach_child(struct vtpci_softc *); + +static int vtpci_alloc_msix(struct vtpci_softc *, int); +static int vtpci_alloc_msi(struct vtpci_softc *); +static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *); +static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *); +static int vtpci_alloc_intr_msi(struct vtpci_softc *); +static int vtpci_alloc_intr_legacy(struct vtpci_softc *); +static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int, @@ -67,2 +140,1 @@ -static void vtpci_free_interrupt(struct vtpci_common *, - struct vtpci_interrupt *); --- +static int vtpci_alloc_intr_resources(struct vtpci_softc *); @@ -70,5 +142,1 @@ -static void vtpci_free_interrupts(struct vtpci_common *); -static void vtpci_free_virtqueues(struct vtpci_common *); -static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *); -static int vtpci_alloc_intr_resources(struct vtpci_common *); -static int vtpci_setup_intx_interrupt(struct vtpci_common *, --- +static int vtpci_setup_legacy_interrupt(struct vtpci_softc *, @@ -76,1 +144,1 @@ -static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *, --- +static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *, @@ -78,2 +146,1 @@ -static int vtpci_set_host_msix_vectors(struct vtpci_common *); -static int vtpci_setup_msix_interrupts(struct vtpci_common *, --- +static int vtpci_setup_msix_interrupts(struct vtpci_softc *, @@ -81,3 +148,18 @@ -static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type); -static int vtpci_reinit_virtqueue(struct vtpci_common *, int); -static void vtpci_intx_intr(void *); --- +static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type); + +static int vtpci_register_msix_vector(struct vtpci_softc *, int, + struct vtpci_interrupt *); +static int vtpci_set_host_msix_vectors(struct vtpci_softc *); +static int vtpci_reinit_virtqueue(struct vtpci_softc *, int); + +static void vtpci_free_interrupt(struct vtpci_softc *, + struct vtpci_interrupt *); +static void vtpci_free_interrupts(struct vtpci_softc *); +static void vtpci_free_virtqueues(struct vtpci_softc *); +static void vtpci_release_child_resources(struct vtpci_softc *); +static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *); +static void vtpci_reset(struct vtpci_softc *); + +static void vtpci_select_virtqueue(struct vtpci_softc *, int); + +static void vtpci_legacy_intr(void *); @@ -90,1 +172,1 @@ -static void vtpci_setup_sysctl(struct vtpci_common *); --- +#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt @@ -92,1 +174,2 @@ -#define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt --- +#define VIRTIO_PCI_CONFIG(_sc) \ + VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0) @@ -95,3 +178,1 @@ - * This module contains two drivers: - * - virtio_pci_legacy (vtpcil) for pre-V1 support - * - virtio_pci_modern (vtpcim) for V1 support --- + * I/O port read/write wrappers. @@ -99,3 +180,6 @@ -MODULE_VERSION(virtio_pci, 1); -MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); -MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); --- +#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) +#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) +#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) +#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) +#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) +#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) @@ -103,1 +187,2 @@ -int vtpci_disable_msix = 0; --- +/* Tunables. */ +static int vtpci_disable_msix = 0; @@ -106,5 +191,8 @@ -static uint8_t -vtpci_read_isr(struct vtpci_common *cn) -{ - return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev)); -} --- +static device_method_t vtpci_methods[] = { + /* Device interface. */ + DEVMETHOD(device_probe, vtpci_probe), + DEVMETHOD(device_attach, vtpci_attach), + DEVMETHOD(device_detach, vtpci_detach), + DEVMETHOD(device_suspend, vtpci_suspend), + DEVMETHOD(device_resume, vtpci_resume), + DEVMETHOD(device_shutdown, vtpci_shutdown), @@ -112,5 +200,5 @@ -static uint16_t -vtpci_get_vq_size(struct vtpci_common *cn, int idx) -{ - return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx)); -} --- + /* Bus interface. */ + DEVMETHOD(bus_driver_added, vtpci_driver_added), + DEVMETHOD(bus_child_detached, vtpci_child_detached), + DEVMETHOD(bus_read_ivar, vtpci_read_ivar), + DEVMETHOD(bus_write_ivar, vtpci_write_ivar), @@ -118,5 +206,11 @@ -static bus_size_t -vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx) -{ - return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx)); -} --- + /* VirtIO bus interface. */ + DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), + DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), + DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), + DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), + DEVMETHOD(virtio_bus_stop, vtpci_stop), + DEVMETHOD(virtio_bus_reinit, vtpci_reinit), + DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), + DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), + DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), + DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), @@ -124,5 +218,2 @@ -static void -vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq) -{ - VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq); -} --- + DEVMETHOD_END +}; @@ -130,5 +221,5 @@ -static void -vtpci_disable_vq(struct vtpci_common *cn, int idx) -{ - VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx); -} --- +static driver_t vtpci_driver = { + "virtio_pci", + vtpci_methods, + sizeof(struct vtpci_softc) +}; @@ -136,0 +227,7 @@ +devclass_t vtpci_devclass; + +DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); +MODULE_VERSION(virtio_pci, 1); +MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); +MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); + @@ -137,1 +235,1 @@ -vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr) --- +vtpci_probe(device_t dev) @@ -139,1 +237,21 @@ - return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr)); --- + char desc[36]; + const char *name; + + if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) + return (ENXIO); + + if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || + pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) + return (ENXIO); + + if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) + return (ENXIO); + + name = virtio_device_name(pci_get_subdevice(dev)); + if (name == NULL) + name = "Unknown"; + + snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); + device_set_desc_copy(dev, desc); + + return (BUS_PROBE_DEFAULT); @@ -143,2 +261,1 @@ -vtpci_register_vq_msix(struct vtpci_common *cn, int idx, - struct vtpci_interrupt *intr) --- +vtpci_attach(device_t dev) @@ -146,2 +263,3 @@ - return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr)); -} --- + struct vtpci_softc *sc; + device_t child; + int rid; @@ -149,3 +267,2 @@ -void -vtpci_init(struct vtpci_common *cn, device_t dev, bool modern) -{ --- + sc = device_get_softc(dev); + sc->vtpci_dev = dev; @@ -153,2 +270,0 @@ - cn->vtpci_dev = dev; - @@ -157,2 +272,8 @@ - if (modern) - cn->vtpci_flags |= VTPCI_FLAG_MODERN; --- + rid = PCIR_BAR(0); + sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, + RF_ACTIVE); + if (sc->vtpci_res == NULL) { + device_printf(dev, "cannot map I/O space\n"); + return (ENXIO); + } + @@ -160,3 +281,1 @@ - cn->vtpci_flags |= VTPCI_FLAG_NO_MSI; - if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0) - cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX; --- + sc->vtpci_flags |= VTPCI_FLAG_NO_MSI; @@ -164,2 +283,5 @@ - vtpci_setup_sysctl(cn); -} --- + if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) { + rid = PCIR_BAR(1); + sc->vtpci_msix_res = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + } @@ -167,4 +289,2 @@ -int -vtpci_add_child(struct vtpci_common *cn) -{ - device_t dev, child; --- + if (sc->vtpci_msix_res == NULL) + sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX; @@ -172,1 +292,1 @@ - dev = cn->vtpci_dev; --- + vtpci_reset(sc); @@ -174,2 +294,4 @@ - child = device_add_child(dev, NULL, -1); - if (child == NULL) { --- + /* Tell the host we've noticed this device. */ + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); + + if ((child = device_add_child(dev, NULL, -1)) == NULL) { @@ -177,0 +299,2 @@ + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); + vtpci_detach(dev); @@ -180,1 +304,2 @@ - cn->vtpci_child_dev = child; --- + sc->vtpci_child_dev = child; + vtpci_probe_and_attach_child(sc); @@ -185,2 +310,2 @@ -int -vtpci_delete_child(struct vtpci_common *cn) --- +static int +vtpci_detach(device_t dev) @@ -188,1 +313,2 @@ - device_t dev, child; --- + struct vtpci_softc *sc; + device_t child; @@ -191,1 +317,1 @@ - dev = cn->vtpci_dev; --- + sc = device_get_softc(dev); @@ -193,2 +319,1 @@ - child = cn->vtpci_child_dev; - if (child != NULL) { --- + if ((child = sc->vtpci_child_dev) != NULL) { @@ -198,1 +323,1 @@ - cn->vtpci_child_dev = NULL; --- + sc->vtpci_child_dev = NULL; @@ -201,0 +326,14 @@ + vtpci_reset(sc); + + if (sc->vtpci_msix_res != NULL) { + bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), + sc->vtpci_msix_res); + sc->vtpci_msix_res = NULL; + } + + if (sc->vtpci_res != NULL) { + bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), + sc->vtpci_res); + sc->vtpci_res = NULL; + } + @@ -204,2 +343,2 @@ -void -vtpci_child_detached(struct vtpci_common *cn) --- +static int +vtpci_suspend(device_t dev) @@ -208,5 +347,1 @@ - vtpci_release_child_resources(cn); - - cn->vtpci_child_feat_desc = NULL; - cn->vtpci_host_features = 0; - cn->vtpci_features = 0; --- + return (bus_generic_suspend(dev)); @@ -215,2 +350,2 @@ -int -vtpci_reinit(struct vtpci_common *cn) --- +static int +vtpci_resume(device_t dev) @@ -218,1 +353,0 @@ - int idx, error; @@ -220,5 +354,2 @@ - for (idx = 0; idx < cn->vtpci_nvqs; idx++) { - error = vtpci_reinit_virtqueue(cn, idx); - if (error) - return (error); - } --- + return (bus_generic_resume(dev)); +} @@ -226,5 +357,3 @@ - if (vtpci_is_msix_enabled(cn)) { - error = vtpci_set_host_msix_vectors(cn); - if (error) - return (error); - } --- +static int +vtpci_shutdown(device_t dev) +{ @@ -232,0 +361,4 @@ + (void) bus_generic_shutdown(dev); + /* Forcibly stop the host device. */ + vtpci_stop(dev); + @@ -236,2 +369,1 @@ -vtpci_describe_features(struct vtpci_common *cn, const char *msg, - uint64_t features) --- +vtpci_driver_added(device_t dev, driver_t *driver) @@ -239,1 +371,1 @@ - device_t dev, child; --- + struct vtpci_softc *sc; @@ -241,2 +373,1 @@ - dev = cn->vtpci_dev; - child = cn->vtpci_child_dev; --- + sc = device_get_softc(dev); @@ -244,4 +375,1 @@ - if (device_is_attached(child) || bootverbose == 0) - return; - - virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc); --- + vtpci_probe_and_attach_child(sc); @@ -250,3 +378,2 @@ -uint64_t -vtpci_negotiate_features(struct vtpci_common *cn, - uint64_t child_features, uint64_t host_features) --- +static void +vtpci_child_detached(device_t dev, device_t child) @@ -254,1 +381,1 @@ - uint64_t features; --- + struct vtpci_softc *sc; @@ -256,2 +383,1 @@ - cn->vtpci_host_features = host_features; - vtpci_describe_features(cn, "host", host_features); --- + sc = device_get_softc(dev); @@ -259,11 +385,2 @@ - /* - * Limit negotiated features to what the driver, virtqueue, and - * host all support. - */ - features = host_features & child_features; - features = virtio_filter_transport_features(features); - - cn->vtpci_features = features; - vtpci_describe_features(cn, "negotiated", features); - - return (features); --- + vtpci_reset(sc); + vtpci_release_child_resources(sc); @@ -272,2 +389,2 @@ -int -vtpci_with_feature(struct vtpci_common *cn, uint64_t feature) --- +static int +vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) @@ -275,2 +392,1 @@ - return ((cn->vtpci_features & feature) != 0); -} --- + struct vtpci_softc *sc; @@ -278,5 +394,1 @@ -int -vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result) -{ - device_t dev; - int error; --- + sc = device_get_softc(dev); @@ -284,2 +396,2 @@ - dev = cn->vtpci_dev; - error = 0; --- + if (sc->vtpci_child_dev != child) + return (ENOENT); @@ -288,0 +400,1 @@ + case VIRTIO_IVAR_DEVTYPE: @@ -300,3 +413,0 @@ - case VIRTIO_IVAR_MODERN: - *result = vtpci_is_modern(cn); - break; @@ -304,1 +414,1 @@ - error = ENOENT; --- + return (ENOENT); @@ -307,1 +417,1 @@ - return (error); --- + return (0); @@ -310,2 +420,2 @@ -int -vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value) --- +static int +vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) @@ -313,1 +423,1 @@ - int error; --- + struct vtpci_softc *sc; @@ -315,1 +425,1 @@ - error = 0; --- + sc = device_get_softc(dev); @@ -317,0 +427,3 @@ + if (sc->vtpci_child_dev != child) + return (ENOENT); + @@ -319,1 +432,1 @@ - cn->vtpci_child_feat_desc = (void *) value; --- + sc->vtpci_child_feat_desc = (void *) value; @@ -322,1 +435,1 @@ - error = ENOENT; --- + return (ENOENT); @@ -325,1 +438,1 @@ - return (error); --- + return (0); @@ -328,3 +441,2 @@ -int -vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs, - struct vq_alloc_info *vq_info) --- +static uint64_t +vtpci_negotiate_features(device_t dev, uint64_t child_features) @@ -332,2 +444,2 @@ - device_t dev; - int idx, align, error; --- + struct vtpci_softc *sc; + uint64_t host_features, features; @@ -335,1 +447,1 @@ - dev = cn->vtpci_dev; --- + sc = device_get_softc(dev); @@ -337,0 +449,3 @@ + host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); + vtpci_describe_features(sc, "host", host_features); + @@ -338,3 +453,2 @@ - * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO, - * the tables do not have to be allocated contiguously, but we do so - * anyways. --- + * Limit negotiated features to what the driver, virtqueue, and + * host all support. @@ -342,1 +456,3 @@ - align = 4096; --- + features = host_features & child_features; + features = virtqueue_filter_features(features); + sc->vtpci_features = features; @@ -344,1 +460,30 @@ - if (cn->vtpci_nvqs != 0) --- + vtpci_describe_features(sc, "negotiated", features); + vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); + + return (features); +} + +static int +vtpci_with_feature(device_t dev, uint64_t feature) +{ + struct vtpci_softc *sc; + + sc = device_get_softc(dev); + + return ((sc->vtpci_features & feature) != 0); +} + +static int +vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, + struct vq_alloc_info *vq_info) +{ + struct vtpci_softc *sc; + struct virtqueue *vq; + struct vtpci_virtqueue *vqx; + struct vq_alloc_info *info; + int idx, error; + uint16_t size; + + sc = device_get_softc(dev); + + if (sc->vtpci_nvqs != 0) @@ -349,1 +494,1 @@ - cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), --- + sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue), @@ -351,1 +496,1 @@ - if (cn->vtpci_vqs == NULL) --- + if (sc->vtpci_vqs == NULL) @@ -355,7 +500,1 @@ - struct vtpci_virtqueue *vqx; - struct vq_alloc_info *info; - struct virtqueue *vq; - bus_size_t notify_offset; - uint16_t size; - - vqx = &cn->vtpci_vqs[idx]; --- + vqx = &sc->vtpci_vqs[idx]; @@ -364,2 +503,2 @@ - size = vtpci_get_vq_size(cn, idx); - notify_offset = vtpci_get_vq_notify_off(cn, idx); --- + vtpci_select_virtqueue(sc, idx); + size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); @@ -367,1 +506,1 @@ - error = virtqueue_alloc(dev, idx, size, notify_offset, align, --- + error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN, @@ -375,1 +514,2 @@ - vtpci_set_vq(cn, vq); --- + vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, + virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); @@ -380,1 +520,1 @@ - cn->vtpci_nvqs++; --- + sc->vtpci_nvqs++; @@ -384,1 +524,1 @@ - vtpci_free_virtqueues(cn); --- + vtpci_free_virtqueues(sc); @@ -390,1 +530,1 @@ -vtpci_alloc_msix(struct vtpci_common *cn, int nvectors) --- +vtpci_setup_intr(device_t dev, enum intr_type type) @@ -392,0 +532,242 @@ + struct vtpci_softc *sc; + int attempt, error; + + sc = device_get_softc(dev); + + for (attempt = 0; attempt < 5; attempt++) { + /* + * Start with the most desirable interrupt configuration and + * fallback towards less desirable ones. + */ + switch (attempt) { + case 0: + error = vtpci_alloc_intr_msix_pervq(sc); + break; + case 1: + error = vtpci_alloc_intr_msix_shared(sc); + break; + case 2: + error = vtpci_alloc_intr_msi(sc); + break; + case 3: + error = vtpci_alloc_intr_legacy(sc); + break; + default: + device_printf(dev, + "exhausted all interrupt allocation attempts\n"); + return (ENXIO); + } + + if (error == 0 && vtpci_setup_interrupts(sc, type) == 0) + break; + + vtpci_cleanup_setup_intr_attempt(sc); + } + + if (bootverbose) { + if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) + device_printf(dev, "using legacy interrupt\n"); + else if (sc->vtpci_flags & VTPCI_FLAG_MSI) + device_printf(dev, "using MSI interrupt\n"); + else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) + device_printf(dev, "using shared MSIX interrupts\n"); + else + device_printf(dev, "using per VQ MSIX interrupts\n"); + } + + return (0); +} + +static void +vtpci_stop(device_t dev) +{ + + vtpci_reset(device_get_softc(dev)); +} + +static int +vtpci_reinit(device_t dev, uint64_t features) +{ + struct vtpci_softc *sc; + int idx, error; + + sc = device_get_softc(dev); + + /* + * Redrive the device initialization. This is a bit of an abuse of + * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to + * play nice. + * + * We do not allow the host device to change from what was originally + * negotiated beyond what the guest driver changed. MSIX state should + * not change, number of virtqueues and their size remain the same, etc. + * This will need to be rethought when we want to support migration. + */ + + if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) + vtpci_stop(dev); + + /* + * Quickly drive the status through ACK and DRIVER. The device + * does not become usable again until vtpci_reinit_complete(). + */ + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); + + vtpci_negotiate_features(dev, features); + + for (idx = 0; idx < sc->vtpci_nvqs; idx++) { + error = vtpci_reinit_virtqueue(sc, idx); + if (error) + return (error); + } + + if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { + error = vtpci_set_host_msix_vectors(sc); + if (error) + return (error); + } + + return (0); +} + +static void +vtpci_reinit_complete(device_t dev) +{ + + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); +} + +static void +vtpci_notify_virtqueue(device_t dev, uint16_t queue) +{ + struct vtpci_softc *sc; + + sc = device_get_softc(dev); + + vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); +} + +static uint8_t +vtpci_get_status(device_t dev) +{ + struct vtpci_softc *sc; + + sc = device_get_softc(dev); + + return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); +} + +static void +vtpci_set_status(device_t dev, uint8_t status) +{ + struct vtpci_softc *sc; + + sc = device_get_softc(dev); + + if (status != VIRTIO_CONFIG_STATUS_RESET) + status |= vtpci_get_status(dev); + + vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); +} + +static void +vtpci_read_dev_config(device_t dev, bus_size_t offset, + void *dst, int length) +{ + struct vtpci_softc *sc; + bus_size_t off; + uint8_t *d; + int size; + + sc = device_get_softc(dev); + off = VIRTIO_PCI_CONFIG(sc) + offset; + + for (d = dst; length > 0; d += size, off += size, length -= size) { + if (length >= 4) { + size = 4; + *(uint32_t *)d = vtpci_read_config_4(sc, off); + } else if (length >= 2) { + size = 2; + *(uint16_t *)d = vtpci_read_config_2(sc, off); + } else { + size = 1; + *d = vtpci_read_config_1(sc, off); + } + } +} + +static void +vtpci_write_dev_config(device_t dev, bus_size_t offset, + void *src, int length) +{ + struct vtpci_softc *sc; + bus_size_t off; + uint8_t *s; + int size; + + sc = device_get_softc(dev); + off = VIRTIO_PCI_CONFIG(sc) + offset; + + for (s = src; length > 0; s += size, off += size, length -= size) { + if (length >= 4) { + size = 4; + vtpci_write_config_4(sc, off, *(uint32_t *)s); + } else if (length >= 2) { + size = 2; + vtpci_write_config_2(sc, off, *(uint16_t *)s); + } else { + size = 1; + vtpci_write_config_1(sc, off, *s); + } + } +} + +static void +vtpci_describe_features(struct vtpci_softc *sc, const char *msg, + uint64_t features) +{ + device_t dev, child; + + dev = sc->vtpci_dev; + child = sc->vtpci_child_dev; + + if (device_is_attached(child) || bootverbose == 0) + return; + + virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); +} + +static void +vtpci_probe_and_attach_child(struct vtpci_softc *sc) +{ + device_t dev, child; + + dev = sc->vtpci_dev; + child = sc->vtpci_child_dev; + + if (child == NULL) + return; + + if (device_get_state(child) != DS_NOTPRESENT) + return; + + if (device_probe(child) != 0) + return; + + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); + if (device_attach(child) != 0) { + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); + vtpci_reset(sc); + vtpci_release_child_resources(sc); + /* Reset status for future attempt. */ + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); + } else { + vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); + VIRTIO_ATTACH_COMPLETED(child); + } +} + +static int +vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) +{ @@ -395,1 +777,1 @@ - dev = cn->vtpci_dev; --- + dev = sc->vtpci_dev; @@ -406,1 +788,1 @@ - cn->vtpci_nmsix_resources = required; --- + sc->vtpci_nmsix_resources = required; @@ -416,1 +798,1 @@ -vtpci_alloc_msi(struct vtpci_common *cn) --- +vtpci_alloc_msi(struct vtpci_softc *sc) @@ -421,1 +803,1 @@ - dev = cn->vtpci_dev; --- + dev = sc->vtpci_dev; @@ -438,1 +820,1 @@ -vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn) --- +vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc) @@ -442,1 +824,2 @@ - if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) --- + if (vtpci_disable_msix != 0 || + sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) @@ -445,2 +828,2 @@ - for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) { - if (cn->vtpci_vqs[i].vtv_no_intr == 0) --- + for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) { + if (sc->vtpci_vqs[i].vtv_no_intr == 0) @@ -450,1 +833,1 @@ - error = vtpci_alloc_msix(cn, nvectors); --- + error = vtpci_alloc_msix(sc, nvectors); @@ -454,1 +837,1 @@ - cn->vtpci_flags |= VTPCI_FLAG_MSIX; --- + sc->vtpci_flags |= VTPCI_FLAG_MSIX; @@ -460,1 +843,1 @@ -vtpci_alloc_intr_msix_shared(struct vtpci_common *cn) --- +vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc) @@ -464,1 +847,2 @@ - if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) --- + if (vtpci_disable_msix != 0 || + sc->vtpci_flags & VTPCI_FLAG_NO_MSIX) @@ -467,1 +851,1 @@ - error = vtpci_alloc_msix(cn, 1); --- + error = vtpci_alloc_msix(sc, 1); @@ -471,1 +855,1 @@ - cn->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; --- + sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX; @@ -477,1 +861,1 @@ -vtpci_alloc_intr_msi(struct vtpci_common *cn) --- +vtpci_alloc_intr_msi(struct vtpci_softc *sc) @@ -482,1 +866,1 @@ - if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI) --- + if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI) @@ -485,1 +869,1 @@ - error = vtpci_alloc_msi(cn); --- + error = vtpci_alloc_msi(sc); @@ -489,1 +873,1 @@ - cn->vtpci_flags |= VTPCI_FLAG_MSI; --- + sc->vtpci_flags |= VTPCI_FLAG_MSI; @@ -495,1 +879,1 @@ -vtpci_alloc_intr_intx(struct vtpci_common *cn) --- +vtpci_alloc_intr_legacy(struct vtpci_softc *sc) @@ -498,1 +882,1 @@ - cn->vtpci_flags |= VTPCI_FLAG_INTX; --- + sc->vtpci_flags |= VTPCI_FLAG_LEGACY; @@ -504,1 +888,1 @@ -vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags, --- +vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags, @@ -509,1 +893,1 @@ - irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags); --- + irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags); @@ -519,89 +903,0 @@ -static void -vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr) -{ - device_t dev; - - dev = cn->vtpci_dev; - - if (intr->vti_handler != NULL) { - bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); - intr->vti_handler = NULL; - } - - if (intr->vti_irq != NULL) { - bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, - intr->vti_irq); - intr->vti_irq = NULL; - intr->vti_rid = -1; - } -} - -static void -vtpci_free_interrupts(struct vtpci_common *cn) -{ - struct vtpci_interrupt *intr; - int i, nvq_intrs; - - vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt); - - if (cn->vtpci_nmsix_resources != 0) { - nvq_intrs = cn->vtpci_nmsix_resources - 1; - cn->vtpci_nmsix_resources = 0; - - if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) { - for (i = 0; i < nvq_intrs; i++, intr++) - vtpci_free_interrupt(cn, intr); - - free(cn->vtpci_msix_vq_interrupts, M_DEVBUF); - cn->vtpci_msix_vq_interrupts = NULL; - } - } - - if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) - pci_release_msi(cn->vtpci_dev); - - cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; -} - -static void -vtpci_free_virtqueues(struct vtpci_common *cn) -{ - struct vtpci_virtqueue *vqx; - int idx; - - for (idx = 0; idx < cn->vtpci_nvqs; idx++) { - vtpci_disable_vq(cn, idx); - - vqx = &cn->vtpci_vqs[idx]; - virtqueue_free(vqx->vtv_vq); - vqx->vtv_vq = NULL; - } - - free(cn->vtpci_vqs, M_DEVBUF); - cn->vtpci_vqs = NULL; - cn->vtpci_nvqs = 0; -} - -void -vtpci_release_child_resources(struct vtpci_common *cn) -{ - - vtpci_free_interrupts(cn); - vtpci_free_virtqueues(cn); -} - -static void -vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn) -{ - int idx; - - if (cn->vtpci_flags & VTPCI_FLAG_MSIX) { - vtpci_register_cfg_msix(cn, NULL); - - for (idx = 0; idx < cn->vtpci_nvqs; idx++) - vtpci_register_vq_msix(cn, idx, NULL); - } - - vtpci_free_interrupts(cn); -} - @@ -609,1 +904,1 @@ -vtpci_alloc_intr_resources(struct vtpci_common *cn) --- +vtpci_alloc_intr_resources(struct vtpci_softc *sc) @@ -614,0 +909,1 @@ + rid = 0; @@ -616,2 +912,1 @@ - if (cn->vtpci_flags & VTPCI_FLAG_INTX) { - rid = 0; --- + if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) @@ -619,1 +914,1 @@ - } else --- + else @@ -623,3 +918,3 @@ - * When using INTX or MSI interrupts, this resource handles all - * interrupts. When using MSIX, this resource handles just the - * configuration changed interrupt. --- + * For legacy and MSI interrupts, this single resource handles all + * interrupts. For MSIX, this resource is used for the configuration + * changed interrupt. @@ -627,4 +922,3 @@ - intr = &cn->vtpci_device_interrupt; - - error = vtpci_alloc_interrupt(cn, rid, flags, intr); - if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI)) --- + intr = &sc->vtpci_device_interrupt; + error = vtpci_alloc_interrupt(sc, rid, flags, intr); + if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI)) @@ -633,6 +927,2 @@ - /* - * Now allocate the interrupts for the virtqueues. This may be one - * for all the virtqueues, or one for each virtqueue. Subtract one - * below for because of the configuration changed interrupt. - */ - nvq_intrs = cn->vtpci_nmsix_resources - 1; --- + /* Subtract one for the configuration changed interrupt. */ + nvq_intrs = sc->vtpci_nmsix_resources - 1; @@ -640,1 +930,1 @@ - cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs * --- + intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs * @@ -642,1 +932,1 @@ - if (cn->vtpci_msix_vq_interrupts == NULL) --- + if (sc->vtpci_msix_vq_interrupts == NULL) @@ -645,2 +935,0 @@ - intr = cn->vtpci_msix_vq_interrupts; - @@ -648,1 +936,1 @@ - error = vtpci_alloc_interrupt(cn, rid, flags, intr); --- + error = vtpci_alloc_interrupt(sc, rid, flags, intr); @@ -657,1 +945,1 @@ -vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type) --- +vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type) @@ -662,1 +950,3 @@ - intr = &cn->vtpci_device_interrupt; --- + intr = &sc->vtpci_device_interrupt; + error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL, + vtpci_legacy_intr, sc, &intr->vti_handler); @@ -664,3 +954,0 @@ - error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, - vtpci_intx_intr, cn, &intr->vti_handler); - @@ -671,1 +958,1 @@ -vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type) --- +vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) @@ -677,1 +964,1 @@ - intr = cn->vtpci_msix_vq_interrupts; --- + intr = sc->vtpci_msix_vq_interrupts; @@ -679,2 +966,2 @@ - for (i = 0; i < cn->vtpci_nvqs; i++) { - vqx = &cn->vtpci_vqs[i]; --- + for (i = 0; i < sc->vtpci_nvqs; i++) { + vqx = &sc->vtpci_vqs[i]; @@ -685,1 +972,1 @@ - error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, --- + error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, @@ -698,1 +985,1 @@ -vtpci_set_host_msix_vectors(struct vtpci_common *cn) --- +vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type) @@ -700,0 +987,76 @@ + device_t dev; + struct vtpci_interrupt *intr; + int error; + + dev = sc->vtpci_dev; + intr = &sc->vtpci_device_interrupt; + + error = bus_setup_intr(dev, intr->vti_irq, type, NULL, + vtpci_config_intr, sc, &intr->vti_handler); + if (error) + return (error); + + if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { + intr = sc->vtpci_msix_vq_interrupts; + error = bus_setup_intr(dev, intr->vti_irq, type, + vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc, + &intr->vti_handler); + } else + error = vtpci_setup_pervq_msix_interrupts(sc, type); + + return (error ? error : vtpci_set_host_msix_vectors(sc)); +} + +static int +vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type) +{ + int error; + + type |= INTR_MPSAFE; + KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, + ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags)); + + error = vtpci_alloc_intr_resources(sc); + if (error) + return (error); + + if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) + error = vtpci_setup_legacy_interrupt(sc, type); + else if (sc->vtpci_flags & VTPCI_FLAG_MSI) + error = vtpci_setup_msi_interrupt(sc, type); + else + error = vtpci_setup_msix_interrupts(sc, type); + + return (error); +} + +static int +vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, + struct vtpci_interrupt *intr) +{ + device_t dev; + uint16_t vector; + + dev = sc->vtpci_dev; + + if (intr != NULL) { + /* Map from guest rid to host vector. */ + vector = intr->vti_rid - 1; + } else + vector = VIRTIO_MSI_NO_VECTOR; + + vtpci_write_config_2(sc, offset, vector); + + /* Read vector to determine if the host had sufficient resources. */ + if (vtpci_read_config_2(sc, offset) != vector) { + device_printf(dev, + "insufficient host resources for MSIX interrupts\n"); + return (ENODEV); + } + + return (0); +} + +static int +vtpci_set_host_msix_vectors(struct vtpci_softc *sc) +{ @@ -701,1 +1064,1 @@ - int idx, error; --- + int idx, offset, error; @@ -703,2 +1066,4 @@ - intr = &cn->vtpci_device_interrupt; - error = vtpci_register_cfg_msix(cn, intr); --- + intr = &sc->vtpci_device_interrupt; + offset = VIRTIO_MSI_CONFIG_VECTOR; + + error = vtpci_register_msix_vector(sc, offset, intr); @@ -708,3 +1073,7 @@ - intr = cn->vtpci_msix_vq_interrupts; - for (idx = 0; idx < cn->vtpci_nvqs; idx++) { - if (cn->vtpci_vqs[idx].vtv_no_intr) --- + intr = sc->vtpci_msix_vq_interrupts; + offset = VIRTIO_MSI_QUEUE_VECTOR; + + for (idx = 0; idx < sc->vtpci_nvqs; idx++) { + vtpci_select_virtqueue(sc, idx); + + if (sc->vtpci_vqs[idx].vtv_no_intr) @@ -715,1 +1084,1 @@ - error = vtpci_register_vq_msix(cn, idx, tintr); --- + error = vtpci_register_msix_vector(sc, offset, tintr); @@ -723,2 +1092,2 @@ - if (!cn->vtpci_vqs[idx].vtv_no_intr && - (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) --- + if (!sc->vtpci_vqs[idx].vtv_no_intr && + (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0) @@ -732,1 +1101,1 @@ -vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type) --- +vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx) @@ -734,1 +1103,2 @@ - struct vtpci_interrupt *intr; --- + struct vtpci_virtqueue *vqx; + struct virtqueue *vq; @@ -736,0 +1106,1 @@ + uint16_t size; @@ -737,1 +1108,2 @@ - intr = &cn->vtpci_device_interrupt; --- + vqx = &sc->vtpci_vqs[idx]; + vq = vqx->vtv_vq; @@ -739,2 +1111,6 @@ - error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL, - vtpci_config_intr, cn, &intr->vti_handler); --- + KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); + + vtpci_select_virtqueue(sc, idx); + size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); + + error = virtqueue_reinit(vq, size); @@ -744,2 +1120,2 @@ - if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) { - intr = &cn->vtpci_msix_vq_interrupts[0]; --- + vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, + virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); @@ -747,7 +1123,1 @@ - error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, - vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn, - &intr->vti_handler); - } else - error = vtpci_setup_pervq_msix_interrupts(cn, type); - - return (error ? error : vtpci_set_host_msix_vectors(cn)); --- + return (0); @@ -756,2 +1126,2 @@ -static int -vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type) --- +static void +vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr) @@ -759,1 +1129,1 @@ - int error; --- + device_t dev; @@ -761,3 +1131,1 @@ - type |= INTR_MPSAFE; - KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK, - ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags)); --- + dev = sc->vtpci_dev; @@ -765,3 +1133,4 @@ - error = vtpci_alloc_intr_resources(cn); - if (error) - return (error); --- + if (intr->vti_handler != NULL) { + bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler); + intr->vti_handler = NULL; + } @@ -769,8 +1138,6 @@ - if (cn->vtpci_flags & VTPCI_FLAG_INTX) - error = vtpci_setup_intx_interrupt(cn, type); - else if (cn->vtpci_flags & VTPCI_FLAG_MSI) - error = vtpci_setup_msi_interrupt(cn, type); - else - error = vtpci_setup_msix_interrupts(cn, type); - - return (error); --- + if (intr->vti_irq != NULL) { + bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid, + intr->vti_irq); + intr->vti_irq = NULL; + intr->vti_rid = -1; + } @@ -779,2 +1146,2 @@ -int -vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type) --- +static void +vtpci_free_interrupts(struct vtpci_softc *sc) @@ -782,2 +1149,2 @@ - device_t dev; - int attempt, error; --- + struct vtpci_interrupt *intr; + int i, nvq_intrs; @@ -785,1 +1152,1 @@ - dev = cn->vtpci_dev; --- + vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt); @@ -787,23 +1154,3 @@ - for (attempt = 0; attempt < 5; attempt++) { - /* - * Start with the most desirable interrupt configuration and - * fallback towards less desirable ones. - */ - switch (attempt) { - case 0: - error = vtpci_alloc_intr_msix_pervq(cn); - break; - case 1: - error = vtpci_alloc_intr_msix_shared(cn); - break; - case 2: - error = vtpci_alloc_intr_msi(cn); - break; - case 3: - error = vtpci_alloc_intr_intx(cn); - break; - default: - device_printf(dev, - "exhausted all interrupt allocation attempts\n"); - return (ENXIO); - } --- + if (sc->vtpci_nmsix_resources != 0) { + nvq_intrs = sc->vtpci_nmsix_resources - 1; + sc->vtpci_nmsix_resources = 0; @@ -811,2 +1158,4 @@ - if (error == 0 && vtpci_setup_intrs(cn, type) == 0) - break; --- + intr = sc->vtpci_msix_vq_interrupts; + if (intr != NULL) { + for (i = 0; i < nvq_intrs; i++, intr++) + vtpci_free_interrupt(sc, intr); @@ -814,1 +1163,3 @@ - vtpci_cleanup_setup_intr_attempt(cn); --- + free(sc->vtpci_msix_vq_interrupts, M_DEVBUF); + sc->vtpci_msix_vq_interrupts = NULL; + } @@ -817,10 +1168,2 @@ - if (bootverbose) { - if (cn->vtpci_flags & VTPCI_FLAG_INTX) - device_printf(dev, "using legacy interrupt\n"); - else if (cn->vtpci_flags & VTPCI_FLAG_MSI) - device_printf(dev, "using MSI interrupt\n"); - else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) - device_printf(dev, "using shared MSIX interrupts\n"); - else - device_printf(dev, "using per VQ MSIX interrupts\n"); - } --- + if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX)) + pci_release_msi(sc->vtpci_dev); @@ -828,1 +1171,1 @@ - return (0); --- + sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK; @@ -831,2 +1174,2 @@ -static int -vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx) --- +static void +vtpci_free_virtqueues(struct vtpci_softc *sc) @@ -835,2 +1178,1 @@ - struct virtqueue *vq; - int error; --- + int idx; @@ -838,2 +1180,2 @@ - vqx = &cn->vtpci_vqs[idx]; - vq = vqx->vtv_vq; --- + for (idx = 0; idx < sc->vtpci_nvqs; idx++) { + vqx = &sc->vtpci_vqs[idx]; @@ -841,1 +1183,2 @@ - KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx)); --- + vtpci_select_virtqueue(sc, idx); + vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0); @@ -843,3 +1186,3 @@ - error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx)); - if (error == 0) - vtpci_set_vq(cn, vq); --- + virtqueue_free(vqx->vtv_vq); + vqx->vtv_vq = NULL; + } @@ -847,1 +1190,3 @@ - return (error); --- + free(sc->vtpci_vqs, M_DEVBUF); + sc->vtpci_vqs = NULL; + sc->vtpci_nvqs = 0; @@ -851,1 +1196,1 @@ -vtpci_intx_intr(void *xcn) --- +vtpci_release_child_resources(struct vtpci_softc *sc) @@ -853,1 +1198,46 @@ - struct vtpci_common *cn; --- + + vtpci_free_interrupts(sc); + vtpci_free_virtqueues(sc); +} + +static void +vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc) +{ + int idx; + + if (sc->vtpci_flags & VTPCI_FLAG_MSIX) { + vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR, + VIRTIO_MSI_NO_VECTOR); + + for (idx = 0; idx < sc->vtpci_nvqs; idx++) { + vtpci_select_virtqueue(sc, idx); + vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR, + VIRTIO_MSI_NO_VECTOR); + } + } + + vtpci_free_interrupts(sc); +} + +static void +vtpci_reset(struct vtpci_softc *sc) +{ + + /* + * Setting the status to RESET sets the host device to + * the original, uninitialized state. + */ + vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); +} + +static void +vtpci_select_virtqueue(struct vtpci_softc *sc, int idx) +{ + + vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx); +} + +static void +vtpci_legacy_intr(void *xsc) +{ + struct vtpci_softc *sc; @@ -858,2 +1248,2 @@ - cn = xcn; - isr = vtpci_read_isr(cn); --- + sc = xsc; + vqx = &sc->vtpci_vqs[0]; @@ -861,0 +1251,3 @@ + /* Reading the ISR also clears it. */ + isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); + @@ -862,1 +1255,1 @@ - vtpci_config_intr(cn); --- + vtpci_config_intr(sc); @@ -865,2 +1258,1 @@ - vqx = &cn->vtpci_vqs[0]; - for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { --- + for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { @@ -874,1 +1266,1 @@ -vtpci_vq_shared_intr_filter(void *xcn) --- +vtpci_vq_shared_intr_filter(void *xsc) @@ -876,1 +1268,1 @@ - struct vtpci_common *cn; --- + struct vtpci_softc *sc; @@ -880,2 +1272,0 @@ - cn = xcn; - vqx = &cn->vtpci_vqs[0]; @@ -883,0 +1273,2 @@ + sc = xsc; + vqx = &sc->vtpci_vqs[0]; @@ -884,1 +1276,1 @@ - for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { --- + for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { @@ -893,1 +1285,1 @@ -vtpci_vq_shared_intr(void *xcn) --- +vtpci_vq_shared_intr(void *xsc) @@ -895,1 +1287,1 @@ - struct vtpci_common *cn; --- + struct vtpci_softc *sc; @@ -899,2 +1291,2 @@ - cn = xcn; - vqx = &cn->vtpci_vqs[0]; --- + sc = xsc; + vqx = &sc->vtpci_vqs[0]; @@ -902,1 +1294,1 @@ - for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) { --- + for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) { @@ -930,1 +1322,1 @@ -vtpci_config_intr(void *xcn) --- +vtpci_config_intr(void *xsc) @@ -932,1 +1324,1 @@ - struct vtpci_common *cn; --- + struct vtpci_softc *sc; @@ -935,2 +1327,2 @@ - cn = xcn; - child = cn->vtpci_child_dev; --- + sc = xsc; + child = sc->vtpci_child_dev; @@ -940,61 +1332,0 @@ -} - -static int -vtpci_feature_sysctl(struct sysctl_req *req, struct vtpci_common *cn, - uint64_t features) -{ - struct sbuf *sb; - int error; - - sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); - if (sb == NULL) - return (ENOMEM); - - error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc); - sbuf_delete(sb); - - return (error); -} - -static int -vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS) -{ - struct vtpci_common *cn; - - cn = arg1; - - return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features)); -} - -static int -vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS) -{ - struct vtpci_common *cn; - - cn = arg1; - - return (vtpci_feature_sysctl(req, cn, cn->vtpci_features)); -} - -static void -vtpci_setup_sysctl(struct vtpci_common *cn) -{ - device_t dev; - struct sysctl_ctx_list *ctx; - struct sysctl_oid *tree; - struct sysctl_oid_list *child; - - dev = cn->vtpci_dev; - ctx = device_get_sysctl_ctx(dev); - tree = device_get_sysctl_tree(dev); - child = SYSCTL_CHILDREN(tree); - - SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs", - CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues"); - - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features", - CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, - vtpci_host_features_sysctl, "A", "Features supported by the host"); - SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features", - CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0, - vtpci_negotiated_features_sysctl, "A", "Features negotiated"); --- /usr/src/sys/dev/virtio/pci/virtio_pci.h +++ /usr/src/sys/dev/virtio/pci/virtio_pci.h @@ -2,1 +2,1 @@ - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD --- + * SPDX-License-Identifier: BSD-3-Clause @@ -4,2 +4,1 @@ - * Copyright (c) 2017, Bryan Venteicher - * All rights reserved. --- + * Copyright IBM Corp. 2007 @@ -7,0 +6,6 @@ + * Authors: + * Anthony Liguori + * + * This header is BSD licensed so anyone can use the definitions to implement + * compatible drivers/servers. + * @@ -11,2 +16,1 @@ - * notice unmodified, this list of conditions, and the following - * disclaimer. --- + * notice, this list of conditions and the following disclaimer. @@ -16,0 +20,14 @@ + * 3. Neither the name of IBM nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. @@ -17,12 +35,1 @@ - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.h 326022 2017-11-20 19:36:21Z pfg $ @@ -34,5 +41,4 @@ -struct vtpci_interrupt { - struct resource *vti_irq; - int vti_rid; - void *vti_handler; -}; --- +/* VirtIO PCI vendor/device ID. */ +#define VIRTIO_PCI_VENDORID 0x1AF4 +#define VIRTIO_PCI_DEVICEID_MIN 0x1000 +#define VIRTIO_PCI_DEVICEID_MAX 0x103F @@ -40,5 +46,2 @@ -struct vtpci_virtqueue { - struct virtqueue *vtv_vq; - int vtv_no_intr; - int vtv_notify_offset; -}; --- +/* VirtIO ABI version, this must match exactly. */ +#define VIRTIO_PCI_ABI_VERSION 0 @@ -46,6 +49,16 @@ -struct vtpci_common { - device_t vtpci_dev; - uint64_t vtpci_host_features; - uint64_t vtpci_features; - struct vtpci_virtqueue *vtpci_vqs; - int vtpci_nvqs; --- +/* + * VirtIO Header, located in BAR 0. + */ +#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/ +#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */ +#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */ +#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */ +#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */ +#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */ +#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */ +#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading + * also clears the register (8, RO) */ +/* Only if MSIX is enabled: */ +#define VIRTIO_MSI_CONFIG_VECTOR 20 /* configuration change vector (16, RW) */ +#define VIRTIO_MSI_QUEUE_VECTOR 22 /* vector for selected VQ notifications + (16, RW) */ @@ -53,9 +66,6 @@ - uint32_t vtpci_flags; -#define VTPCI_FLAG_NO_MSI 0x0001 -#define VTPCI_FLAG_NO_MSIX 0x0002 -#define VTPCI_FLAG_MODERN 0x0004 -#define VTPCI_FLAG_INTX 0x1000 -#define VTPCI_FLAG_MSI 0x2000 -#define VTPCI_FLAG_MSIX 0x4000 -#define VTPCI_FLAG_SHARED_MSIX 0x8000 -#define VTPCI_FLAG_ITYPE_MASK 0xF000 --- +/* The bit of the ISR which indicates a device has an interrupt. */ +#define VIRTIO_PCI_ISR_INTR 0x1 +/* The bit of the ISR which indicates a device configuration change. */ +#define VIRTIO_PCI_ISR_CONFIG 0x2 +/* Vector value used to disable MSI for queue. */ +#define VIRTIO_MSI_NO_VECTOR 0xFFFF @@ -63,3 +73,5 @@ - /* The VirtIO PCI "bus" will only ever have one child. */ - device_t vtpci_child_dev; - struct virtio_feature_desc *vtpci_child_feat_desc; --- +/* + * The remaining space is defined by each driver as the per-driver + * configuration space. + */ +#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20) @@ -67,14 +79,5 @@ - /* - * Ideally, each virtqueue that the driver provides a callback for will - * receive its own MSIX vector. If there are not sufficient vectors - * available, then attempt to have all the VQs share one vector. For - * MSIX, the configuration changed notifications must be on their own - * vector. - * - * If MSIX is not available, attempt to have the whole device share - * one MSI vector, and then, finally, one intx interrupt. - */ - struct vtpci_interrupt vtpci_device_interrupt; - struct vtpci_interrupt *vtpci_msix_vq_interrupts; - int vtpci_nmsix_resources; -}; --- +/* + * How many bits to shift physical queue address written to QUEUE_PFN. + * 12 is historical, and due to x86 page size. + */ +#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12 @@ -82,49 +85,2 @@ -extern int vtpci_disable_msix; - -static inline device_t -vtpci_child_device(struct vtpci_common *cn) -{ - return (cn->vtpci_child_dev); -} - -static inline bool -vtpci_is_msix_available(struct vtpci_common *cn) -{ - return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0); -} - -static inline bool -vtpci_is_msix_enabled(struct vtpci_common *cn) -{ - return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0); -} - -static inline bool -vtpci_is_modern(struct vtpci_common *cn) -{ - return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0); -} - -static inline int -vtpci_virtqueue_count(struct vtpci_common *cn) -{ - return (cn->vtpci_nvqs); -} - -void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern); -int vtpci_add_child(struct vtpci_common *cn); -int vtpci_delete_child(struct vtpci_common *cn); -void vtpci_child_detached(struct vtpci_common *cn); -int vtpci_reinit(struct vtpci_common *cn); - -uint64_t vtpci_negotiate_features(struct vtpci_common *cn, - uint64_t child_features, uint64_t host_features); -int vtpci_with_feature(struct vtpci_common *cn, uint64_t feature); - -int vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result); -int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value); - -int vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs, - struct vq_alloc_info *vq_info); -int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type); -void vtpci_release_child_resources(struct vtpci_common *cn); --- +/* The alignment to use between consumer and producer parts of vring. */ +#define VIRTIO_PCI_VRING_ALIGN 4096 --- /usr/src/sys/dev/virtio/random/virtio_random.c +++ /usr/src/sys/dev/virtio/random/virtio_random.c @@ -32,1 +32,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/random/virtio_random.c 338324 2018-08-26 12:51:46Z markm $"); @@ -61,2 +61,1 @@ -static int vtrnd_negotiate_features(struct vtrnd_softc *); -static int vtrnd_setup_features(struct vtrnd_softc *); --- +static void vtrnd_negotiate_features(struct vtrnd_softc *); @@ -89,1 +88,1 @@ -DRIVER_MODULE(virtio_random, vtpcil, vtrnd_driver, vtrnd_devclass, --- +DRIVER_MODULE(virtio_random, virtio_pci, vtrnd_driver, vtrnd_devclass, @@ -91,2 +90,0 @@ -DRIVER_MODULE(virtio_random, vtpcim, vtrnd_driver, vtrnd_devclass, - vtrnd_modevent, 0); @@ -136,1 +133,0 @@ - virtio_set_feature_desc(dev, vtrnd_feature_desc); @@ -140,5 +136,2 @@ - error = vtrnd_setup_features(sc); - if (error) { - device_printf(dev, "cannot setup features\n"); - goto fail; - } --- + virtio_set_feature_desc(dev, vtrnd_feature_desc); + vtrnd_negotiate_features(sc); @@ -173,1 +166,1 @@ -static int --- +static void @@ -183,1 +176,0 @@ - return (virtio_finalize_features(dev)); @@ -187,12 +179,0 @@ -vtrnd_setup_features(struct vtrnd_softc *sc) -{ - int error; - - error = vtrnd_negotiate_features(sc); - if (error) - return (error); - - return (0); -} - -static int @@ -240,2 +220,0 @@ -// random_harvest_queue(&value, sizeof(value), sizeof(value) * NBBY / 2, -// RANDOM_PURE_VIRTIO); --- /usr/src/sys/dev/virtio/scsi/virtio_scsi.c +++ /usr/src/sys/dev/virtio/scsi/virtio_scsi.c @@ -32,1 +32,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/scsi/virtio_scsi.c 349691 2019-07-03 19:52:24Z vangyzen $"); @@ -79,2 +79,1 @@ -static int vtscsi_negotiate_features(struct vtscsi_softc *); -static int vtscsi_setup_features(struct vtscsi_softc *); --- +static void vtscsi_negotiate_features(struct vtscsi_softc *); @@ -139,4 +138,4 @@ -static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *, - struct ccb_scsiio *, struct virtio_scsi_cmd_req *); -static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *, struct ccb_hdr *, - uint32_t, uintptr_t, struct virtio_scsi_ctrl_tmf_req *); --- +static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *, + struct virtio_scsi_cmd_req *); +static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t, + uintptr_t, struct virtio_scsi_ctrl_tmf_req *); @@ -188,1 +187,1 @@ -static void vtscsi_setup_sysctl(struct vtscsi_softc *); --- +static void vtscsi_add_sysctl(struct vtscsi_softc *); @@ -193,8 +192,0 @@ -#define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0) -#define vtscsi_htog16(_sc, _val) virtio_htog16(vtscsi_modern(_sc), _val) -#define vtscsi_htog32(_sc, _val) virtio_htog32(vtscsi_modern(_sc), _val) -#define vtscsi_htog64(_sc, _val) virtio_htog64(vtscsi_modern(_sc), _val) -#define vtscsi_gtoh16(_sc, _val) virtio_gtoh16(vtscsi_modern(_sc), _val) -#define vtscsi_gtoh32(_sc, _val) virtio_gtoh32(vtscsi_modern(_sc), _val) -#define vtscsi_gtoh64(_sc, _val) virtio_gtoh64(vtscsi_modern(_sc), _val) - @@ -218,2 +209,0 @@ - { VIRTIO_SCSI_F_CHANGE, "ChangeEvent" }, - { VIRTIO_SCSI_F_T10_PI, "T10PI" }, @@ -242,1 +231,1 @@ -DRIVER_MODULE(virtio_scsi, vtpcil, vtscsi_driver, vtscsi_devclass, --- +DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass, @@ -244,2 +233,0 @@ -DRIVER_MODULE(virtio_scsi, vtpcim, vtscsi_driver, vtscsi_devclass, - vtscsi_modevent, 0); @@ -291,1 +278,0 @@ - virtio_set_feature_desc(dev, vtscsi_feature_desc); @@ -297,1 +283,1 @@ - vtscsi_setup_sysctl(sc); --- + vtscsi_add_sysctl(sc); @@ -299,5 +285,2 @@ - error = vtscsi_setup_features(sc); - if (error) { - device_printf(dev, "cannot setup features\n"); - goto fail; - } --- + virtio_set_feature_desc(dev, vtscsi_feature_desc); + vtscsi_negotiate_features(sc); @@ -305,0 +288,7 @@ + if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) + sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; + if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) + sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; + if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) + sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; + @@ -416,1 +406,1 @@ -static int --- +static void @@ -423,4 +413,2 @@ - features = VTSCSI_FEATURES; - - sc->vtscsi_features = virtio_negotiate_features(dev, features); - return (virtio_finalize_features(dev)); --- + features = virtio_negotiate_features(dev, VTSCSI_FEATURES); + sc->vtscsi_features = features; @@ -429,22 +417,0 @@ -static int -vtscsi_setup_features(struct vtscsi_softc *sc) -{ - device_t dev; - int error; - - dev = sc->vtscsi_dev; - - error = vtscsi_negotiate_features(sc); - if (error) - return (error); - - if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) - sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT; - if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT)) - sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL; - if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG)) - sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG; - - return (0); -} - @@ -568,1 +534,0 @@ - virtio_reinit_complete(dev); @@ -570,0 +535,1 @@ + virtio_reinit_complete(dev); @@ -1123,1 +1089,1 @@ - vtscsi_init_scsi_cmd_req(sc, csio, cmd_req); --- + vtscsi_init_scsi_cmd_req(csio, cmd_req); @@ -1243,1 +1209,1 @@ - vtscsi_init_ctrl_tmf_req(sc, to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, --- + vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, @@ -1351,1 +1317,0 @@ - uint32_t resp_sense_length; @@ -1355,1 +1320,1 @@ - csio->resid = vtscsi_htog32(sc, cmd_resp->resid); --- + csio->resid = cmd_resp->resid; @@ -1362,3 +1327,1 @@ - resp_sense_length = vtscsi_htog32(sc, cmd_resp->sense_len); - - if (resp_sense_length > 0) { --- + if (cmd_resp->sense_len > 0) { @@ -1367,2 +1330,3 @@ - if (resp_sense_length < csio->sense_len) - csio->sense_resid = csio->sense_len - resp_sense_length; --- + if (cmd_resp->sense_len < csio->sense_len) + csio->sense_resid = csio->sense_len - + cmd_resp->sense_len; @@ -1372,2 +1336,1 @@ - bzero(&csio->sense_data, sizeof(csio->sense_data)); - memcpy(cmd_resp->sense, &csio->sense_data, --- + memcpy(&csio->sense_data, cmd_resp->sense, @@ -1534,1 +1497,1 @@ - vtscsi_init_ctrl_tmf_req(sc, ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, --- + vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK, @@ -1603,1 +1566,1 @@ - vtscsi_init_ctrl_tmf_req(sc, ccbh, subtype, 0, tmf_req); --- + vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req); @@ -1640,1 +1603,1 @@ -vtscsi_init_scsi_cmd_req(struct vtscsi_softc *sc, struct ccb_scsiio *csio, --- +vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio, @@ -1661,1 +1624,1 @@ - cmd_req->tag = vtscsi_gtoh64(sc, (uintptr_t) csio); --- + cmd_req->tag = (uintptr_t) csio; @@ -1671,2 +1634,2 @@ -vtscsi_init_ctrl_tmf_req(struct vtscsi_softc *sc, struct ccb_hdr *ccbh, - uint32_t subtype, uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) --- +vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype, + uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req) @@ -1677,3 +1640,3 @@ - tmf_req->type = vtscsi_gtoh32(sc, VIRTIO_SCSI_T_TMF); - tmf_req->subtype = vtscsi_gtoh32(sc, subtype); - tmf_req->tag = vtscsi_gtoh64(sc, tag); --- + tmf_req->type = VIRTIO_SCSI_T_TMF; + tmf_req->subtype = subtype; + tmf_req->tag = tag; @@ -2313,1 +2276,1 @@ -vtscsi_setup_sysctl(struct vtscsi_softc *sc) --- +vtscsi_add_sysctl(struct vtscsi_softc *sc) --- /usr/src/sys/dev/virtio/scsi/virtio_scsi.h +++ /usr/src/sys/dev/virtio/scsi/virtio_scsi.h @@ -28,1 +28,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/scsi/virtio_scsi.h 326255 2017-11-27 14:52:40Z pfg $ @@ -34,1 +34,7 @@ -/* Default values of the CDB and sense data size configuration fields */ --- +/* Feature bits */ +#define VIRTIO_SCSI_F_INOUT 0x0001 /* Single request can contain both + * read and write buffers */ +#define VIRTIO_SCSI_F_HOTPLUG 0x0002 /* Host should enable hot plug/unplug + * of new LUNs and targets. + */ + @@ -43,1 +49,1 @@ - uint8_t prio; /* SAM command priority field */ --- + uint8_t prio; @@ -48,12 +54,0 @@ -/* SCSI command request, followed by protection information */ -struct virtio_scsi_cmd_req_pi { - uint8_t lun[8]; /* Logical Unit Number */ - uint64_t tag; /* Command identifier */ - uint8_t task_attr; /* Task attribute */ - uint8_t prio; /* SAM command priority field */ - uint8_t crn; - uint32_t pi_bytesout; /* DataOUT PI Number of bytes */ - uint32_t pi_bytesin; /* DataIN PI Number of bytes */ - uint8_t cdb[VIRTIO_SCSI_CDB_SIZE]; -} __packed; - @@ -113,16 +107,0 @@ -/* Feature bits */ -#define VIRTIO_SCSI_F_INOUT 0x0001 /* Single request can contain both - * read and write buffers. - */ -#define VIRTIO_SCSI_F_HOTPLUG 0x0002 /* Host should enable hot plug/unplug - * of new LUNs and targets. - */ -#define VIRTIO_SCSI_F_CHANGE 0x0004 /* Host will report changes to LUN - * parameters via a - * VIRTIO_SCSI_T_PARAM_CHANGE event. - */ -#define VIRTIO_SCSI_F_T10_PI 0x0008 /* Extended fields for T10 protection - * information (DIF/DIX) are included - * in the SCSI request header. - */ - @@ -165,1 +143,0 @@ -#define VIRTIO_SCSI_T_PARAM_CHANGE 3 --- /usr/src/sys/dev/virtio/scsi/virtio_scsivar.h +++ /usr/src/sys/dev/virtio/scsi/virtio_scsivar.h @@ -28,1 +28,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/scsi/virtio_scsivar.h 326255 2017-11-27 14:52:40Z pfg $ --- /usr/src/sys/dev/virtio/virtio.c +++ /usr/src/sys/dev/virtio/virtio.c @@ -30,1 +30,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/virtio.c 329601 2018-02-19 19:28:24Z bryanv $"); @@ -63,1 +63,1 @@ - { VIRTIO_ID_RPMSG, "Remote Processor Messaging" }, --- + { VIRTIO_ID_RPMSG, "Remote Processor Messaging" }, @@ -69,3 +69,3 @@ - { VIRTIO_ID_INPUT, "Input" }, - { VIRTIO_ID_VSOCK, "VSOCK Transport" }, - { VIRTIO_ID_CRYPTO, "Crypto" }, --- + { VIRTIO_ID_INPUT, "Input" }, + { VIRTIO_ID_VSOCK, "VSOCK Transport" }, + { VIRTIO_ID_CRYPTO, "Crypto" }, @@ -78,7 +78,4 @@ - { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, /* Legacy */ - { VIRTIO_F_ANY_LAYOUT, "AnyLayout" }, /* Legacy */ - { VIRTIO_RING_F_INDIRECT_DESC, "RingIndirectDesc" }, - { VIRTIO_RING_F_EVENT_IDX, "RingEventIdx" }, - { VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */ - { VIRTIO_F_VERSION_1, "Version1" }, - { VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" }, --- + { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, + { VIRTIO_RING_F_INDIRECT_DESC, "RingIndirect" }, + { VIRTIO_RING_F_EVENT_IDX, "EventIdx" }, + { VIRTIO_F_BAD_FEATURE, "BadFeature" }, @@ -122,3 +119,3 @@ -int -virtio_describe_sbuf(struct sbuf *sb, uint64_t features, - struct virtio_feature_desc *desc) --- +void +virtio_describe(device_t dev, const char *msg, + uint64_t features, struct virtio_feature_desc *desc) @@ -126,1 +123,1 @@ - const char *name; --- + struct sbuf sb; @@ -128,0 +125,2 @@ + char *buf; + const char *name; @@ -130,1 +129,4 @@ - sbuf_printf(sb, "%#jx", (uintmax_t) features); --- + if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) { + device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features); + return; + } @@ -132,0 +134,3 @@ + sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN); + sbuf_printf(&sb, "%s features: %#jx", msg, (uintmax_t) features); + @@ -141,1 +146,1 @@ - sbuf_cat(sb, " <"); --- + sbuf_cat(&sb, " <"); @@ -143,1 +148,1 @@ - sbuf_cat(sb, ","); --- + sbuf_cat(&sb, ","); @@ -147,1 +152,1 @@ - sbuf_printf(sb, "%#jx", (uintmax_t) val); --- + sbuf_printf(&sb, "%#jx", (uintmax_t) val); @@ -149,1 +154,1 @@ - sbuf_cat(sb, name); --- + sbuf_cat(&sb, name); @@ -153,1 +158,1 @@ - sbuf_cat(sb, ">"); --- + sbuf_cat(&sb, ">"); @@ -155,21 +160,6 @@ - return (sbuf_finish(sb)); -} - -void -virtio_describe(device_t dev, const char *msg, uint64_t features, - struct virtio_feature_desc *desc) -{ - struct sbuf sb; - char *buf; - int error; - - if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) { - error = ENOMEM; - goto out; - } - - sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN); - sbuf_printf(&sb, "%s features: ", msg); - - error = virtio_describe_sbuf(&sb, features, desc); - if (error == 0) --- +#if __FreeBSD_version < 900020 + sbuf_finish(&sb); + if (sbuf_overflowed(&sb) == 0) +#else + if (sbuf_finish(&sb) == 0) +#endif @@ -180,6 +170,0 @@ - -out: - if (error != 0) { - device_printf(dev, "%s features: %#jx\n", msg, - (uintmax_t) features); - } @@ -188,42 +172,0 @@ -uint64_t -virtio_filter_transport_features(uint64_t features) -{ - uint64_t transport, mask; - - transport = (1ULL << - (VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1; - transport <<= VIRTIO_TRANSPORT_F_START; - - mask = -1ULL & ~transport; - mask |= VIRTIO_RING_F_INDIRECT_DESC; - mask |= VIRTIO_RING_F_EVENT_IDX; - mask |= VIRTIO_F_VERSION_1; - - return (features & mask); -} - -int -virtio_bus_is_modern(device_t dev) -{ - uintptr_t modern; - - virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern); - return (modern != 0); -} - -void -virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst, - int size, int count) -{ - int i, gen; - - do { - gen = virtio_config_generation(dev); - - for (i = 0; i < count; i++) { - virtio_read_device_config(dev, offset + i * size, - (uint8_t *) dst + i * size, size); - } - } while (gen != virtio_config_generation(dev)); -} - @@ -255,7 +197,0 @@ -} - -int -virtio_finalize_features(device_t dev) -{ - - return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev))); --- /usr/src/sys/dev/virtio/virtio.h +++ /usr/src/sys/dev/virtio/virtio.h @@ -28,1 +28,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/virtio.h 329601 2018-02-19 19:28:24Z bryanv $ @@ -34,1 +34,0 @@ -#include @@ -38,1 +37,0 @@ -struct sbuf; @@ -62,1 +60,0 @@ -#define VIRTIO_IVAR_MODERN 7 @@ -71,7 +68,1 @@ - uint64_t features, struct virtio_feature_desc *desc); -int virtio_describe_sbuf(struct sbuf *sb, uint64_t features, - struct virtio_feature_desc *desc); -uint64_t virtio_filter_transport_features(uint64_t features); -int virtio_bus_is_modern(device_t dev); -void virtio_read_device_config_array(device_t dev, bus_size_t offset, - void *dst, int size, int count); --- + uint64_t features, struct virtio_feature_desc *feature_desc); @@ -85,1 +76,0 @@ -int virtio_finalize_features(device_t dev); @@ -143,1 +133,0 @@ -VIRTIO_READ_IVAR(modern, VIRTIO_IVAR_MODERN); --- /usr/src/sys/dev/virtio/virtio_bus_if.m +++ /usr/src/sys/dev/virtio/virtio_bus_if.m @@ -26,1 +26,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/dev/virtio/virtio_bus_if.m 329601 2018-02-19 19:28:24Z bryanv $ @@ -39,6 +39,0 @@ - virtio_bus_default_finalize_features(device_t dev) - { - return (0); - } - - static int @@ -56,4 +50,0 @@ -METHOD int finalize_features { - device_t dev; -} DEFAULT virtio_bus_default_finalize_features; - @@ -93,1 +83,0 @@ - bus_size_t offset; --- /usr/src/sys/dev/virtio/virtio_config.h +++ /usr/src/sys/dev/virtio/virtio_config.h @@ -28,1 +28,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/virtio_config.h 335305 2018-06-17 20:45:48Z bryanv $ --- /usr/src/sys/dev/virtio/virtio_ids.h +++ /usr/src/sys/dev/virtio/virtio_ids.h @@ -28,1 +28,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/virtio_ids.h 327958 2018-01-14 06:03:40Z bryanv $ --- /usr/src/sys/dev/virtio/virtio_if.m +++ /usr/src/sys/dev/virtio/virtio_if.m @@ -26,1 +26,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/dev/virtio/virtio_if.m 255110 2013-09-01 04:20:23Z bryanv $ --- /usr/src/sys/dev/virtio/virtio_ring.h +++ /usr/src/sys/dev/virtio/virtio_ring.h @@ -32,1 +32,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/virtio_ring.h 335305 2018-06-17 20:45:48Z bryanv $ --- /usr/src/sys/dev/virtio/virtqueue.c +++ /usr/src/sys/dev/virtio/virtqueue.c @@ -35,1 +35,1 @@ -__FBSDID("$FreeBSD$"); --- +__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/virtqueue.c 329602 2018-02-19 19:31:18Z bryanv $"); @@ -60,0 +60,1 @@ + char vq_name[VIRTQUEUE_MAX_NAME_SZ]; @@ -63,3 +64,2 @@ -#define VIRTQUEUE_FLAG_MODERN 0x0001 -#define VIRTQUEUE_FLAG_INDIRECT 0x0002 -#define VIRTQUEUE_FLAG_EVENT_IDX 0x0004 --- +#define VIRTQUEUE_FLAG_INDIRECT 0x0001 +#define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 @@ -67,0 +67,3 @@ + int vq_alignment; + int vq_ring_size; + void *vq_ring_mem; @@ -68,1 +71,1 @@ - bus_size_t vq_notify_offset; --- + int vq_indirect_mem_size; @@ -87,6 +90,0 @@ - void *vq_ring_mem; - int vq_indirect_mem_size; - int vq_alignment; - int vq_ring_size; - char vq_name[VIRTQUEUE_MAX_NAME_SZ]; - @@ -140,8 +137,0 @@ -#define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0) -#define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val) -#define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val) -#define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val) -#define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val) -#define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val) -#define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val) - @@ -161,3 +150,2 @@ -virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, - bus_size_t notify_offset, int align, vm_paddr_t highaddr, - struct vq_alloc_info *info, struct virtqueue **vqp) --- +virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, + vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) @@ -199,1 +187,0 @@ - vq->vq_notify_offset = notify_offset; @@ -206,2 +193,0 @@ - if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0) - vq->vq_flags |= VIRTQUEUE_FLAG_MODERN; @@ -312,2 +297,2 @@ - indirect[i].next = vq_gtoh16(vq, i + 1); - indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END); --- + indirect[i].next = i + 1; + indirect[i].next = VQ_RING_DESC_CHAIN_END; @@ -411,1 +396,0 @@ - @@ -460,1 +444,1 @@ - used_idx = vq_htog16(vq, vq->vq_ring.used->idx); --- + used_idx = vq->vq_ring.used->idx; @@ -472,1 +456,1 @@ - if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx)) --- + if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) @@ -499,1 +483,1 @@ - avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx); --- + avail_idx = vq->vq_ring.avail->idx; @@ -524,6 +508,4 @@ - vring_used_event(&vq->vq_ring) = vq_gtoh16(vq, - vq->vq_used_cons_idx - vq->vq_nentries - 1); - return; - } - - vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT); --- + vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - + vq->vq_nentries - 1; + } else + vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; @@ -592,1 +574,1 @@ - if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx)) --- + if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) @@ -599,1 +581,1 @@ - desc_idx = (uint16_t) vq_htog32(vq, uep->id); --- + desc_idx = (uint16_t) uep->id; @@ -601,1 +583,1 @@ - *len = vq_htog32(vq, uep->len); --- + *len = uep->len; @@ -659,7 +641,7 @@ - vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq), - vq->vq_queued_cnt, vq->vq_desc_head_idx, - vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx, - vq_htog16(vq, vq->vq_ring.used->idx), - vq_htog16(vq, vring_used_event(&vq->vq_ring)), - vq_htog16(vq, vq->vq_ring.avail->flags), - vq_htog16(vq, vq->vq_ring.used->flags)); --- + vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, + virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx, + vq->vq_ring.avail->idx, vq->vq_used_cons_idx, + vq->vq_ring.used->idx, + vring_used_event(&vq->vq_ring), + vq->vq_ring.avail->flags, + vq->vq_ring.used->flags); @@ -682,2 +664,2 @@ - vr->desc[i].next = vq_gtoh16(vq, i + 1); - vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END); --- + vr->desc[i].next = i + 1; + vr->desc[i].next = VQ_RING_DESC_CHAIN_END; @@ -689,1 +671,1 @@ - uint16_t avail_idx, avail_ring_idx; --- + uint16_t avail_idx; @@ -698,3 +680,2 @@ - avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx); - avail_ring_idx = avail_idx & (vq->vq_nentries - 1); - vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx); --- + avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1); + vq->vq_ring.avail->ring[avail_idx] = desc_idx; @@ -703,1 +684,1 @@ - vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1); --- + vq->vq_ring.avail->idx++; @@ -722,1 +703,1 @@ - i++, idx = vq_htog16(vq, dp->next), seg++) { --- + i++, idx = dp->next, seg++) { @@ -727,2 +708,2 @@ - dp->addr = vq_gtoh64(vq, seg->ss_paddr); - dp->len = vq_gtoh32(vq, seg->ss_len); --- + dp->addr = seg->ss_paddr; + dp->len = seg->ss_len; @@ -732,1 +713,1 @@ - dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT); --- + dp->flags |= VRING_DESC_F_NEXT; @@ -734,1 +715,1 @@ - dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE); --- + dp->flags |= VRING_DESC_F_WRITE; @@ -779,3 +760,3 @@ - dp->addr = vq_gtoh64(vq, dxp->indirect_paddr); - dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc)); - dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT); --- + dp->addr = dxp->indirect_paddr; + dp->len = needed * sizeof(struct vring_desc); + dp->flags = VRING_DESC_F_INDIRECT; @@ -786,1 +767,1 @@ - vq->vq_desc_head_idx = vq_htog16(vq, dp->next); --- + vq->vq_desc_head_idx = dp->next; @@ -804,7 +785,4 @@ - if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { - vring_used_event(&vq->vq_ring) = - vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc); - } else { - vq->vq_ring.avail->flags &= - vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT); - } --- + if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) + vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; + else + vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; @@ -828,1 +806,1 @@ - uint16_t new_idx, prev_idx, event_idx, flags; --- + uint16_t new_idx, prev_idx, event_idx; @@ -831,1 +809,1 @@ - new_idx = vq_htog16(vq, vq->vq_ring.avail->idx); --- + new_idx = vq->vq_ring.avail->idx; @@ -833,1 +811,1 @@ - event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring)); --- + event_idx = vring_avail_event(&vq->vq_ring); @@ -838,2 +816,1 @@ - flags = vq->vq_ring.used->flags; - return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0); --- + return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0); @@ -846,2 +823,1 @@ - VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index, - vq->vq_notify_offset); --- + VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); @@ -866,5 +842,4 @@ - if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) { - while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) { - uint16_t next_idx = vq_htog16(vq, dp->next); - VQ_RING_ASSERT_VALID_IDX(vq, next_idx); - dp = &vq->vq_ring.desc[next_idx]; --- + if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { + while (dp->flags & VRING_DESC_F_NEXT) { + VQ_RING_ASSERT_VALID_IDX(vq, dp->next); + dp = &vq->vq_ring.desc[dp->next]; @@ -883,1 +858,1 @@ - dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx); --- + dp->next = vq->vq_desc_head_idx; --- /usr/src/sys/dev/virtio/virtqueue.h +++ /usr/src/sys/dev/virtio/virtqueue.h @@ -28,1 +28,1 @@ - * $FreeBSD$ --- + * $FreeBSD: releng/12.1/sys/dev/virtio/virtqueue.h 329602 2018-02-19 19:31:18Z bryanv $ @@ -73,2 +73,2 @@ - bus_size_t notify_offset, int align, vm_paddr_t highaddr, - struct vq_alloc_info *info, struct virtqueue **vqp); --- + int align, vm_paddr_t highaddr, struct vq_alloc_info *info, + struct virtqueue **vqp); --- /usr/src/sys/modules/virtio/Makefile +++ /usr/src/sys/modules/virtio/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/Makefile 273515 2014-10-23 04:47:32Z bryanv $ --- /usr/src/sys/modules/virtio/balloon/Makefile +++ /usr/src/sys/modules/virtio/balloon/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/balloon/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/modules/virtio/block/Makefile +++ /usr/src/sys/modules/virtio/block/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/block/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/modules/virtio/console/Makefile +++ /usr/src/sys/modules/virtio/console/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/console/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/modules/virtio/network/Makefile +++ /usr/src/sys/modules/virtio/network/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/network/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/modules/virtio/pci/Makefile +++ /usr/src/sys/modules/virtio/pci/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/pci/Makefile 314651 2017-03-04 10:10:17Z ngie $ @@ -29,3 +29,2 @@ -SRCS= virtio_pci.c virtio_pci_legacy.c virtio_pci_modern.c -SRCS+= virtio_pci_if.c virtio_pci_if.h -SRCS+= virtio_bus_if.h virtio_if.h --- +SRCS= virtio_pci.c +SRCS+= virtio_bus_if.h virtio_if.h --- /usr/src/sys/modules/virtio/random/Makefile +++ /usr/src/sys/modules/virtio/random/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/random/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/modules/virtio/scsi/Makefile +++ /usr/src/sys/modules/virtio/scsi/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/scsi/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/modules/virtio/virtio/Makefile +++ /usr/src/sys/modules/virtio/virtio/Makefile @@ -2,1 +2,1 @@ -# $FreeBSD$ --- +# $FreeBSD: releng/12.1/sys/modules/virtio/virtio/Makefile 314651 2017-03-04 10:10:17Z ngie $ --- /usr/src/sys/net/netmap_virt.h +++ /usr/src/sys/net/netmap_virt.h @@ -47,2 +47,2 @@ -#define PTNETMAP_PCI_DEVICE_ID 0xcccc /* memory device */ -#define PTNETMAP_PCI_NETIF_ID 0xcccd /* ptnet network interface */ --- +#define PTNETMAP_PCI_DEVICE_ID 0x000c /* memory device */ +#define PTNETMAP_PCI_NETIF_ID 0x000d /* ptnet network interface */ --- /usr/src/sys/x86/include/specialreg.h +++ /usr/src/sys/x86/include/specialreg.h @@ -450,1 +450,0 @@ -#define IA32_ARCH_CAP_IF_PSCHANGE_MC_NO 0x00000040