#!/usr/local/bin/cbsd
#v10.1.0
MYARG="mode"
MYOPTARG="jname ppt rom"
MYDESC="Manage bhyve ppt devices"
CBSDMODULE="bhyve"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

CBSD allows you to configure bhyve arguments to throw devices into the guest, 
if your hardware supports it. 'bhyve-ppt' can manage PPT devices list.

Before 'bhyve-ppt', you need to find a device for passthru, e.g.:

'pciconv -vl'

And use /boot/loader.conf to configure PPT, e.g. 'pciconf -vl' for GPU/display passthru:

vgapci0@pci0:0:2:0:     class=0x030000 rev=0x06 hdr=0x00 vendor=0x8086 device=0x0412 subvendor=0x1462 subdevice=0x7816
    vendor     = 'Intel Corporation'
    device     = 'Xeon E3-1200 v3/4th Gen Core Processor Integrated Graphics Controller'
    class      = display
    subclass   = VGA

Add into /boot/loader.conf:

pptdevs=\"0/2/0\"

PPT requirements:

Keep in mind that IOMMU (DMAR for Intel-based processors and IVRS for AMD-based processors) must be 
enabled in the BIOS.  Make sure you are loading the vmm.ko module through the loader.conf and not kld.
Also for AMD processors you need to add to loader.conf:

vmm_load="YES"
hw.vmm.amdvi.enable=1

All PPT guests should use '-S' flags, wired memory. To set:

cbsd bset jname=XXX bhyve_wire_memory=1

For Intel/AMD VM in GOP mode you may need an alternative UEFI/EDK2 ROM:

/usr/local/cbsd/upgrade/patch/efigop.fd  ( use it as 'efi_firmware' in CBSD bhyve options ).

Additional notes for GOP/rom options:

 The VBIOS can be extracted on different ways. Sadly not many on FreeBSD.
 - On Linux you can try something like: https://01.org/linuxgraphics/documentation/development/how-dump-video-bios;
 - On Windows you can try GPU-Z: https://nvidia.custhelp.com/rnt/rnw/img/enduser/gpu-z.png;
 - You can download the proper VBIOS for your card;

Warning! Using the wrong VBIOS ( 'rom=' ) or a wrong version might be dangerous!


${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}mode=${N0_COLOR} - action, available:
         - 'attach' - attach PPT to jname
         - 'detach' - detach PPT from jname;
 ${N2_COLOR}ppt=${N0_COLOR}  - target PPT;
 ${N2_COLOR}rom=${N0_COLOR}  - path to rom, e.g: /path/to/gop.rom (see https://reviews.freebsd.org/D26209);

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd bhyve-ppt mode=list
 # cbsd bhyve-ppt mode=attach ppt=0/2/0 jname=vmname
 # cbsd bhyve-ppt mode=detach ppt=0/2/0 jname=vmname

"

. ${subrdir}/nc.subr
oppt=
orom=
rom=
ppt=
. ${cbsdinit}
[ -z "${rom}" ] && rom="0"
[ -n "${ppt}" ] && oppt="${ppt}"
[ -n "${rom}" ] && orom="${rom}"

. ${subrdir}/virtual.subr
. ${system}

device=$( ${PCICONF_CMD} -l | ${EGREP_CMD} -E "^ppt[0-9]+@" | ${WC_CMD} -l | ${AWK_CMD} '{printf $1}' )

[ -z "${device}" ] && err 1 "${N1_COLOR}No such ppt devices${N0_COLOR}"

for id in $( ${SEQ_CMD} 0 ${device} ); do
	eval $( ${PCICONF_CMD} -vl | ${GREP_CMD} -A4 ^ppt${id}@pci | ${GREP_CMD} -E "^ppt${id}@|vendor|device" | while read a; do
		prefix=$( substr --pos=0 --len=6 --str="$a" )
		case "${prefix}" in
			vendor)
				value=$( echo "${a}" | ${CUT_CMD} -d = -f 2 | ${TR_CMD} -d "'" )
				echo "vendor${id}=\"${value}\""
				;;
			device)
				value=$( echo "${a}" | ${CUT_CMD} -d = -f 2 | ${TR_CMD} -d "'" )
				echo "device${id}=\"${value}\""
				;;
			ppt*)
				value=$( echo  "${a}" | ${AWK_CMD} '{printf $1}' | ${TR_CMD} -d "a-z@" | ${CUT_CMD} -d : -f 2-4 | ${TR_CMD} ":" "/" )
				echo "ppt${id}=\"${value}\""
				;;
			*)
				;;
		esac
	done )
done

# export $device, $ppt, $vendor variable
# $1 - as ppt, e.g: 5/1/0
load_ppt()
{
	my_ppt="${1}"

	[ -z "${my_ppt}" ] && return 1

	device=
	ppt=
	vendor=

	local tmp_ppt

	for id in $( ${SEQ_CMD} 0 ${device} ); do
		tmp_ppt=

		eval tmp_ppt="\$ppt${id}"
		[ -z "${tmp_ppt}" ] && continue

		if [ "${my_ppt}" = "${tmp_ppt}" ]; then
			eval device="\$device${id}"
			eval vendor="\$vendor${id}"
		fi
	done

	export ppt="${my_ppt}"
}

# export tmp_jname as linked jail to $ppt
init_ppt_action()
{
	[ -z "${ppt}" ] && err 1 "${N1_COLOR}Please set: ${N2_COLOR}ppt=${N0_COLOR}"
	[ -z "${jname}" ] && err 1 "${N1_COLOR}Please set: ${N2_COLOR}jname=${N0_COLOR}"

	# check for PCI passthrough
	if check_dmar; then
		err 1 "${N1_COLOR}DMAR or IVRS not found via acpidump: IOMMU/VT-d not enabled ? Please Check you hardware and/or BIOS setting${N0_COLOR}"
	fi

	. ${subrdir}/rcconf.subr
	[ $? -eq 1 ] && err 1 "${N1_COLOR}No such jail: ${N2_COLOR}${jname}${N0_COLOR}"
	[ "${emulator}" != "bhyve" ] && err 1 "${N1_COLOR}Only for bhyve type VMs${N0_COLOR}"

	load_ppt "${ppt}"
	[ -z "${ppt}" ] && err 1 "${N1_COLOR}No such ppt: ${N2_COLOR}${ppt}${N0_COLOR}"

	tmp_jname=$( cbsdsqlro local SELECT jname FROM bhyveppt WHERE ppt=\"${ppt}\" 2>/dev/null )
}

case "${mode}" in
	list)
		for id in $( ${SEQ_CMD} 0 ${device} ); do
			ppt=
			device=
			vendor=
			rom=
			jname=0

			eval ppt="\$ppt${id}"
			[ -z "${ppt}" ] && continue

			eval device="\$device${id}"
			eval vendor="\$vendor${id}"
			eval rom="\$rom${id}"

			jname=$( cbsdsqlro local SELECT jname FROM bhyveppt WHERE ppt=\"${ppt}\" 2>/dev/null )
			[ -z "${jname}" -o "${jname}" = "0" ] && jname="\-"
			[ -z "${rom}" -o "${rom}" = "0" ] && rom="\-"
			[ -n "${ppt}" ] && echo "${ppt} : ${device} : ${vendor} : ${jname}"
		done
		;;
	attach)
		init_ppt_action
		[ -n "${tmp_jname}" -a "${tmp_jname}" != "0" ] && err 1 "${N1_COLOR}ppt already used by: ${N2_COLOR}${tmp_jname}${N1_COLOR}. Please detach first${N0_COLOR}"
		[ -n "${orom}" ] && rom="${orom}"
		cbsdsqlrw local "INSERT INTO bhyveppt ( ppt, device, vendor, jname, rom ) VALUES ( \"${ppt}\", \"device\", \"${vendor}\", \"${jname}\", \"${rom}\" )"
		err 0 "${N1_COLOR}Attached${N0_COLOR}"
		;;
	detach)
		init_ppt_action

		[ -z "${tmp_jname}" -a "${tmp_jname}" = "0" ] && err 1 "${N1_COLOR}ppt is not attached: ${N2_COLOR}${ppt}${N1_COLOR}"

		cbsdsqlrw local "DELETE FROM bhyveppt WHERE ppt=\"${ppt}\""
		err 0 "${N1_COLOR}Detached${N0_COLOR}"
		;;
	*)
		err 1 "${N1_COLOR}Unknown mode: ${N2_COLOR}${mode}${N0_COLOR}"
esac

exit 0
