#!/usr/local/bin/cbsd
#v12.0.5
MYARG=""
MYOPTARG="cfg_only debug debug_engine delay jname lm"
MYDESC="Start QEMU domain"
ADDHELP="
${H3_COLOR}Description${N0_COLOR}:

Start the qemu domain. You can start the offline machine, or continue the work
of the frozen machine from the checkpoint.

${H3_COLOR}Options${N0_COLOR}:

 ${N2_COLOR}cfg_only=${N0_COLOR}         - when set (e.g:'1'), find/create tap/vnc and
                     generate qemu args but without run, e.g for debugging.
 ${N2_COLOR}debug=${N0_COLOR}            - more debug messages.
 ${N2_COLOR}debug_engine=${N0_COLOR}     - overwrite debug_engine settings: use gdb or lldb
                     as debugger when launch qemu ( mostly for inherits debug
                     with live migration ).
 ${N2_COLOR}delay=${N0_COLOR}            - <sec>, delay N secbefore start, mainly to smooth
                     the astart, default is: '0', no delay.
 ${N2_COLOR}inter=${N0_COLOR}            - set 1 to prevent any questions and to accept answers by default.
 ${N2_COLOR}jname=${N0_COLOR}            - Target VM. If jail='*' or jail='vm*' then start  all
                     qemu or VM whose names begin with 'vm', e.g. 'vm1', 'vm2'...
 ${N2_COLOR}lm=${N0_COLOR}               - 0,1: when set to 1: prepare for acceptance of this
                     domain via live migration request ( used by 'bmigrate').
 ${N2_COLOR}lm_dport=${N0_COLOR}         - (optional) for live migration, port for migration.
                     data exchange. Can be '0' for auto port, by default.
 ${N2_COLOR}lm_rnodename=${N0_COLOR}     - (optional) for live migration, remote/source
                     CBSD nodename.

${H3_COLOR}Examples${N0_COLOR}:

 # cbsd qstart

${H3_COLOR}See also${N0_COLOR}:

 cbsd qstop --help

"

CBSDMODULE="qemu"
EXTHELP="wf_xstop_qstart"

# by default - no live-migrated domain
lm="0"
lm_dport=
lm_rnodename=
odebug_engine=

cfg_only=		# create a configuration and run immediately

. ${subrdir}/nc.subr
. ${tools}		# for select_jail_by_list

readconf buildworld.conf
readconf jail-freebsd-default.conf

jname=

# check for cloud function when CBSDfile exist
Makefile="${CBSD_PWD}/CBSDfile"
if [ ! -r ${Makefile} ]; then
	[ -z "${1}" ] && select_jail_by_list -s "List of offline VMs" -a "Off" -e qls -r ${sqlreplica}
fi

odebug_engine=		# overwrite debug_engine variable
debug_engine=		# reset debug_engine before init

delay=0
. ${cbsdinit}
ojname="${jname}"

# live migration
if [ "${lm}" = "1" ]; then
	exec /usr/local/sbin/xl -t migrate-receive
	exit 0
fi

. ${system}
. ${subrdir}/universe.subr
. ${subrdir}/qemu.subr
. ${subrdir}/vnet.subr 		# get_vm_uplink_interface

if [ -r ${Makefile} ]; then
	[ -z "${CBSDFILE_RECURSIVE}" ] && ${ECHO} "${N1_COLOR}found CBSDfile: ${N2_COLOR}${Makefile}${N0_COLOR}" 1>&2
	. ${Makefile}
	all_qemu_list=$( ${EGREP_CMD} '^qemu_[a-zA-Z0-9_@%:][-a-zA-Z0-9_@%:]*\(\)$' ${Makefile} | ${XARGS_CMD} | ${TR_CMD} -d "()" | ${SED_CMD} s#qemu_##g )
	jname="${all_qemu_list}"

	if [ -n "${ojname}" ]; then
		jname="${ojname}"
	fi

	[ -z "${jname}" ] && err 1 "${N1_COLOR}${CBSD_APP}: give me jname${N0_COLOR}"

	if [ -n "${CLOUD_URL}" -a -n "${CLOUD_KEY}" ]; then
		cbsd_api=1
	else
		cbsd_api=0
	fi
else
	cbsd_api=0
fi

# trim args from "$*"
ARGS=
for i in $*; do
	prefix6=$( substr --pos=0 --len=6 --str="${i}" )

	[ "${prefix6}" = "delay=" ] && shift && continue
	[ "${prefix6}" = "jname=" ] && shift && continue
	[ "${prefix6}" = "quiet=" ] && shift && continue
	[ "${prefix6}" = "inter=" ] && shift && continue

	if [ -n "${ARGS}" ]; then
		ARGS="${ARGS} ${i}"
	else
		ARGS="${i}"
	fi
done

[ -z "${jname}" -a -z "${ARGS}" ] && err 1 "${N1_COLOR}no qemu specified${N0_COLOR}"
[ -n "${debug_engine}" ] && odebug_engine="${debug_engine}"	# store overwrite debug_engine

. ${subrdir}/fetch.subr
. ${subrdir}/jcreate.subr       # for external_exec_master_script
. ${subrdir}/virtual.subr		# for init_systap

start_qemu()
{
	local QEMUCFG
	local xvm_ram _qemu_bin

	# profile
	readconf vm-${vm_os_type}-${vm_os_profile}.conf
	if [ -z "${vm_profile}" ]; then
		${ECHO} "${N1_COLOR}No such profile: ${N2_COLOR}vm-${vm_os_type}-${vm_os_profile}.conf${N0_COLOR}"
		sleep 2
	fi
	# re-read jail params and apply personal after profile
	. ${subrdir}/rcconf.subr

	# CBSD QUEUE
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_qemu_queue_name=${cbsd_qemu_queue_name} id=${jname} cmd=qstart status=1 workdir="${workdir}"
	fi

	vm_boot=$( cbsdsqlro ${main_sqlite_local} SELECT vm_boot FROM settings 2>/dev/null )

	# todo: shared qemu/qemu virtual
	# Cloud-init init. Cloud init only for empty disk
	if check_for_empty_hdd path=${data}/dsk1.vhd; then
		if [ -n "${vm_iso_path}" ]; then
			local prefix=
			local prefix6=$( substr --pos=0 --len=6 --str="${vm_iso_path}" )
			if [ "${prefix6}" = "cloud-" ]; then
				local _orig_vm_iso_path="${vm_iso_path}"
				vm_boot="cd"
				init_iso
				if [ $? -eq 1 ]; then
					err 1 "${N1_COLOR}No such cloud source: ${N2_COLOR}${vm_iso_path}/${iso_img}${N0_COLOR}"
				fi
				vm_boot="hdd"
				${ECHO} "${N1_COLOR}cloud init image initialization..${N0_COLOR}"
				local _myfile="${iso_img}"
				[ ! -r "${_myfile}" ] && err 1 "${N1_COLOR}Error: qstart: not readable: ${N2_COLOR}${_myfile}${N0_COLOR}"

				. ${subrdir}/zfs.subr

				if is_getzvol ${_myfile}; then
					_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_cloud_source_is_zvol=1
					_cloud_source_zvol="${is_zvol}"
				else
					_cloud_source_is_zvol=0
					_cloud_source_zvol=
					_ebytes=$( ${STAT_CMD} -f "%z" ${_myfile} 2>/dev/null )
				fi

				# store original disk size to restore them after replace by cloud image
				if is_getzvol ${data}/dsk1.vhd; then
					_dsk_ebytes=$( ${ZFS_CMD} get -Hp -o value volsize ${is_zvol} )
					_dsk_source_zvol="${is_zvol}"
					_cloud_truncate=0
				else
					_dsk_ebytes=$( ${STAT_CMD} -f "%z" ${data}/dsk1.vhd 2>/dev/null )
					_cloud_truncate=1
					_dsk_source_zvol=
				fi

				# print some warning about not optimal when zfsfeat=1 but for some 
				# reason cloning not available?

				# if source image is not ZVOl, use dd method for cloning
				[ ${_cloud_source_is_zvol} -eq 0 ] && _cloud_truncate=1

				if [ ${_cloud_truncate} -eq 1 ]; then
					${ECHO} "${N1_COLOR}Clone cloud image into first/system vm disk (${W1_COLOR}dd${N1_COLOR} method)${N0_COLOR}"
					# to generic clonedata (add dd method)?
					/usr/bin/nice -n 20 ${DD_CMD} if=${_myfile} bs=4m | ${miscdir}/cbsdtee -e ${_ebytes} > ${data}/dsk1.vhd
					echo
					# adjust original image size
					${TRUNCATE_CMD} -s${_dsk_ebytes} ${data}/dsk1.vhd
				else
					${ECHO} "${N1_COLOR}Clone cloud image into first/system vm disk (zfs clone method)${N0_COLOR}"
					# to generic clonedata ?
						_cloud_snapshot_name="${_cloud_source_zvol}@boot-${jname}"
						echo "${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name}"
						${ZFS_CMD} get -Ht snapshot userrefs ${_cloud_snapshot_name} > /dev/null 2>&1
						_ret=$?
						if [ ${_ret} -eq 1 ]; then
							# create cloud snapshot for $jname
							${ZFS_CMD} snapshot ${_cloud_source_zvol}@boot-${jname}
							# destory original zvol disk for vm
							${ZFS_CMD} destroy ${_dsk_source_zvol}
							#${ZFS_CMD} clone ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							${ZFS_CMD} clone -o volsize=${_dsk_ebytes} ${_cloud_source_zvol}@boot-${jname} ${_dsk_source_zvol}
							cbsdlogger NOTICE ${CBSD_APP}: zfs clone method cloud image: gpart commit zvol/${_dsk_source_zvol}
							${GPART_CMD} commit zvol/${_dsk_source_zvol}
							# restore original size
							# ${ZFS_CMD} set volsize=${_dsk_ebytes} ${_dsk_source_zvol}
							# bug here, need for atomic ops ^^ in clone action
							${ZFS_CMD} set cbsdsnap:jname=${jname} ${_cloud_source_zvol}@boot-${jname}
							${ZFS_CMD} set cbsdsnap:snapname=cloud ${_cloud_source_zvol}@boot-${jname}
						else
							err 1 "${N1_COLOR}snapshot already exist ${_cloud_snapshot_name}${N0_COLOR}"
						fi
				fi
				echo "Eject cloud source: media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname}"
				media mode=detach name=${_orig_vm_iso_path} path=${_myfile} type=iso jname=${jname}
			else
				manage_boot_by_empty_hdd
			fi
		fi
	else
		manage_boot_by_empty_hdd
	fi

	if [ "${vm_boot}" = "cd" ]; then
		init_iso
		if [ $? -eq 1 ]; then
			printf "${N1_COLOR}Continue without ${iso_img}. Hope this is ok, sleep for 5 seconds ${N0_COLOR}"
				for i in $( ${JOT_CMD} 5 ); do
					printf "."
					sleep 1
				done
				echo
		fi
	fi

	case "${vm_boot}" in
		"hdd")
			boot_arg='-boot c'
			;;
		"cd")
			boot_arg='-boot d'
			;;
	esac

	# for vnet we can make another action
	. ${subrdir}/vnet.subr

	#unset zero-value
	[ "${qemu_flags}" = "0" ] && unset qemu_flags
	[ "${vm_os_profile}" = "0" ] && unset vm_os_profile

	# reset global qemu_pci_id_busy_list (used by compile_ func via next_pci_id()
	# and remove buffer file with old $qemu_pci_id_busy_list ( used by add_qemu_pci_id_busy )
	qemu_pci_id_busy_list=

	qemu_pci_index=1
	[ -f ${jailsysdir}/${jname}/qemu_pciid ] && ${RM_CMD} -f ${jailsysdir}/${jname}/qemu_pciid

	# mark modified field to FALSE in pcibus table to find unmodified entries for pciid cleanup
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE pcibus SET modified=false

	# truncate pcibus_run table
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite DELETE FROM pcibus_run

	if ! compile_machine_args; then
		${ECHO} "${N1_COLOR}No such machine_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset machine_args
	fi
	if ! compile_cpu_args; then
		${ECHO} "${N1_COLOR}No such cpu_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset cpu_args
	fi
	if ! compile_kernel_args; then
		${ECHO} "${N1_COLOR}No such kernel_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset kernel_args
	fi
	if ! compile_bios_args; then
		${ECHO} "${N1_COLOR}No such bios_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset bios_args
	fi
	if ! compile_vga_args; then
		${ECHO} "${N1_COLOR}No such vga_args for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset vga_args
	fi
	if ! compile_dsk_args; then
		${ECHO} "${N1_COLOR}No such disk for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset dsk_args
	fi

	if ! compile_cd_args; then
		unset cd_args
	fi

	# init nic_args
	if ! compile_nic_args ; then
		${ECHO} "${N1_COLOR}No such nic for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
		unset nic_args
	fi

	# init console_args
	#if ! compile_console_args; then
	#	${ECHO} "${N1_COLOR}No such console for VMs: ${N2_COLOR}${jname}${N0_COLOR}"
	#	unset console_args
	#fi

	if [ "${spice_default}" != "1" ]; then
		# init vnc_args
		if ! compile_vnc_args ; then
			unset vnc_args
		fi
	else
		# init vnc_args
		if ! compile_spice_args ; then
			unset vnc_spice
		fi
	fi

	xvm_ram=$(( vm_ram / 1024 / 1024  ))

	# Poehali!

	export_qemu_data_for_external_hook
	external_exec_master_script "master_prestart.d"

	vm_logfile=$( ${MKTEMP_CMD} )

	QEMUCFG="${jailsysdir}/${jname}/qemu.cfg"
	${TRUNCATE_CMD} -s0 ${QEMUCFG}

	# restore overwrite debug_engine
	[ -n "${odebug_engine}" ] && debug_engine="${odebug_engine}"

	# todo: loop for multiple devices
	eval mybridge="\$nic0"

	case ${spice_default} in
		0)
			# init args
			;;
		1)
			# init args
			;;
	esac

#	if [ -n "${soundhw}" -a "${soundhw}" != "none" ]; then
#		${CAT_CMD} >> ${QEMUCFG} <<EOF
#soundhw="${soundhw}"
#EOF
#	fi

# UEFI
#bios = 'ovmf'
#nographics = 1
#bios_override = '/usr/share/ovmf/x64/OVMF_CODE.fd'    # Added to try to be explicit, same behavior with or without

	# todo: shared qemu/qemu virtual
	# cloud-init enabled?
	# this section should be AFTER master_prestart
	# cloud_init and CLOUD_FILES was initialized earlier
	if [ ${cloud_init} -eq 1 ]; then
			${RM_CMD} -f ${jailsysdir}/${jname}/seed.iso
			if [ "${vm_os_type}" = "windows" ]; then
				GENISOIMAGE_CMD=$( which /usr/local/bin/genisoimage )
				[ -z "${GENISOIMAGE_CMD}" ] && err 1 "${N1_COLOR}no such genisoimage tool. Windows cloud required it. Please install it first: ${N2_COLOR}pkg install -y sysutils/genisoimage${N0_COLOR}"
				${GENISOIMAGE_CMD} -joliet-long -R -V config-2 -o ${jailsysdir}/${jname}/seed.iso ${jailsysdir}/${jname}/cloud-init
			else
				GENISOIMAGE_CMD=$( which makefs )
				${GENISOIMAGE_CMD} -t cd9660 -o label="cidata" -o isolevel=2 -o rockridge -o publisher="CBSD" ${jailsysdir}/${jname}/seed.iso ${jailsysdir}/${jname}/cloud-init
				# see /usr/src/usr.sbin/makefs/cd9660/cd9660_strings.c + /usr/src/usr.sbin/makefs/cd9660.c (cd9660_valid_a_char)
				# why upper here, whereis spec?
				${SED_CMD} -i '' s:CIDATA:cidata: ${jailsysdir}/${jname}/seed.iso
			fi
	fi

	case "${arch}" in
		riscv64)
			_qemu_bin=$( which qemu-system-${arch} 2>/dev/null )
			[ -z "${_qemu_bin}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such executable: ${N2_COLOR}qemu-system-${arch}${N0_COLOR}"
			;;
		aarch64)
			_qemu_bin=$( which qemu-system-${arch} 2>/dev/null )
			[ -z "${_qemu_bin}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such executable: ${N2_COLOR}qemu-system-${arch}${N0_COLOR}"
			;;
		*)
			_qemu_bin="${QEMU_SYSTEM_X86_64_CMD}"
			[ -z "${_qemu_bin}" ] && err 1 "${N1_COLOR}${CBSD_APP}: no such executable: ${N2_COLOR}qemu-system-${arch}${N0_COLOR}"
			;;
	esac

# compile_rnd
#-object rng-random,id=rng0,filename=/dev/urandom \
#-device virtio-rng-pci,rng=rng0 \

# compile_usb
#-usb -device qemu-xhci,id=usbbus \


	# qemu restore/checkpoint
	if [ -n "${checkpoint}" ]; then
		CHECKPOINT_DIR="${jailsysdir}/${jname}/checkpoints"
		CHECKPOINT="${CHECKPOINT_DIR}/${checkpoint}.ckp"
		if [ -r ${CHECKPOINT} ]; then
			${ECHO} "${N1_COLOR}Checkpoint found, starting from: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
			qemu_cmd="${XL_CMD} -vvv restore ${CHECKPOINT}"
		else
			err 1 "${N1_COLOR}Checkpoint not found: ${N2_COLOR}${CHECKPOINT}${N0_COLOR}"
		fi
	else
		# -machine type=q35 \
		#-cpu host -M pc-i440fx-hirsute
		#-cpu host \
		qemu_cmd="${_qemu_bin} \
${machine_args} \
${cpu_args} \
-smp cpus=${vm_cpus} -m ${xvm_ram} \
${kernel_args} \
${bios_args} \
${cd_args} \
${boot_arg} \
${dsk_args} \
${nic_args} \
${vnc_args} \
-display curses \
${vga_args} \
-name ${jname} \
-pidfile ${jailsysdir}/${jname}/qemu.pid \
-monitor unix:${jailsysdir}/${jname}/qemu-monitor.sock,server,nowait
"
	fi

	echo "[debug]: $qemu_cmd"
	[ -n "${cfg_only}" ] && exit 0

	case "${debug_engine}" in
		gdb)
			if [ -x /usr/local/bin/gdb ]; then
				gdb_cmd="/usr/local/bin/gdb"
			elif [ -x /usr/libexec/gdb ]; then
				gdb_cmd="/usr/libexec/gdb"
			elif [ -x /usr/bin/gdb ]; then
				gdb_cmd="/usr/bin/gdb"
			fi
			# break while loop
			echo
			echo "Warning"
			echo "Run xl throuch GDB. Please execute 'run' to launch QEMU instance"
			echo
			echo "${gdb_cmd} -batch --args ${qemu_cmd}"
			${gdb_cmd} -ex run --args ${qemu_cmd}
			;;
		lldb)
			echo
			echo "Warning"
			echo "Run xl throuch LLDB. Please execute 'run' to launch QEMU instance"
			echo
			echo "/usr/bin/lldb -- ${qemu_cmd}"
			/usr/bin/lldb -- ${qemu_cmd}
			qemu_exit=$?
			${RM_CMD} -f ${tmpdir}/cmds.$$
			;;
		*)
			tmuxcmd=$( which tmux )
			[ -z "${tmuxcmd}" ] && err 1 "${N1_COLOR}bstart: no such tmux cmd${N0_COLOR}"
			${tmuxcmd} -2 -u new -d -s "cbsd-${jname}" "${qemu_cmd}"
			;;
	esac

	[ -n "${dsk_bootable}" ] && ${ECHO} "${N1_COLOR}Boot device: ${N2_COLOR}${dsk_bootable}${N0_COLOR}"
	printf "${N1_COLOR}Waiting for PID"
	vm_pid=0
	for i in $( ${SEQ_CMD} 10 ); do
		if [ -r ${jailsysdir}/${jname}/qemu.pid ]; then
			_test_pid=$( ${CAT_CMD} ${jailsysdir}/${jname}/qemu.pid 2>/dev/null )
			if [ -n "${_test_pid}" ]; then
				${PS_CMD} -p ${_test_pid} > /dev/null 2>&1
				_ret=$?
				if [ ${_ret} -eq 0 ]; then
					vm_pid="${_test_pid}"
					break
				fi
			fi
		fi
		sleep 1
		printf "."
	done

	[ -z "${vm_pid}" ] && vm_pid="0"

	echo

	external_exec_master_script "master_poststart.d"

	# CBSD QUEUE
	if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
		[ -n "${cbsd_qemu_queue_name}" ] && ${cbsd_queue_backend} cbsd_queue_name=${cbsd_qemu_queue_name} id=${jname} cmd=bstart status=2 data_status=1 workdir="${workdir}"
	fi

	${ECHO} "${N1_COLOR}PID: ${N2_COLOR}${vm_pid}${N0_COLOR}"
	cbsdsqlrw local "UPDATE jails SET jid=\"${vm_pid}\" WHERE jname=\"${jname}\""

	# update state_time
	cbsdsqlrw ${jailsysdir}/${jname}/local.sqlite UPDATE settings SET state_time="(strftime('%s','now'))"

	# update state_time, local SQLite for back compatible
	cbsdsqlrw local "UPDATE jails SET state_time=\"(strftime('%s','now'))\" WHERE jname=\"${jname}\""

	exit 0
}


# MAIN for multiple jails
TRAP=""
emulator="qemu"		# for jname_is_multiple
jname_is_multiple
if [ "${mod_cbsd_queue_enabled}" = "YES" -a -z "${MOD_CBSD_QUEUE_DISABLED}" ]; then
	readconf cbsd_queue.conf
	[ -z "${cbsd_queue_backend}" ] && MOD_CBSD_QUEUE_DISABLED="1"
fi

if [ $# -gt 1 -a -z "${jname}" -o -n "${jail_list}" ]; then
	# multiple astart always non interactive
	export inter=0
	# recursive
	if [ -n "${jail_list}" ]; then
		JLIST="${jail_list}"
	else
		JLIST=$*
	fi

	for jname in ${JLIST}; do
		TRAP="${TRAP} ${RM_CMD} -f ${ftmpdir}/qstart.${jname}.$$;"
		trap "${TRAP}" HUP INT ABRT BUS TERM EXIT
		env NOINTER=1 ${DAEMON_CMD} -p ${ftmpdir}/qstart.${jname}.$$ /usr/local/bin/cbsd qstart jname=${jname} delay=1
		#lets save .pid file
		sleep 1
		[ -f "${ftmpdir}/qstart.${jname}.$$" ] && cbsd_pwait --pid=$( ${CAT_CMD} ${ftmpdir}/qstart.${jname}.$$ ) --timeout=${parallel}
		trap "" HUP INT ABRT BUS TERM EXIT
		# Artificial delay to create a sequence (for order compliance)
		# todo: determine VM complete starting
		sleep 12
	done

	wait_for_fpid -a start -t ${parallel}

	err 0 "${N1_COLOR}Multiple qstart: ${N2_COLOR}done${N0_COLOR}"
fi

# MAIN
. ${distsharedir}/qemu.conf		# only for for MYCOL variables: used in exports below
. ${subrdir}/time.subr
[ -z "${jname}" ] && jname=${1}

init_qemu

[ "${create_cbsdsystem_tap}" = "1" ] && init_systap

# mainly to smooth mass start in astart.
if [ ${delay} -ne 0 ]; then
	[ -z "${boot_delay}" ] && boot_delay=1
		cbsdlogger NOTICE ${CBSD_APP}: delayed boot for ${jname}: ${boot_delay} sec.
		sleep ${boot_delay}
fi

# start qemu
st_time=$( ${DATE_CMD} +%s )

# cloud-init enabled?
# we need this section (besides section cloud-init section in start_qemu() to allocate pci bus id via touching seed.iso
# since some helpers may work with runtime (e.g pcibus + cloudinit) config
if [ -d ${jailsysdir}/${jname}/cloud-init ]; then
	CLOUD_FILES=$( ${FIND_CMD} ${jailsysdir}/${jname}/cloud-init/ -type f | ${XARGS_CMD} )
	# gen seed only if files exist
	if [ -n "${CLOUD_FILES}" ]; then
		cloud_init=1
		${ECHO} "${H5_COLOR}cloud-init: ${H3_COLOR}enabled${N0_COLOR}"
		${TOUCH_CMD} ${jailsysdir}/${jname}/seed.iso
	else
		cloud_init=0
		unset CLOUD_FILES
	fi
else
	cloud_init=0
fi


default_profile="qemu-default-default.conf"
readconf vnc.conf
readconf spice.conf
readconf qstart.conf

readconf ${default_profile}

. ${subrdir}/rcconf.subr
[ $? -eq 1 ] && err 1 "${N1_COLOR}No such domain: ${N2_COLOR}${jname}${N0_COLOR}"
[ ${status} -eq 2 ] && err 1 "${N1_COLOR}Domain in slave mode. Please ${N2_COLOR}cbsd jswmode mode=master${N1_COLOR} first${N0_COLOR}"

[ ${jid} -ne 0 ] && err 1 "${N1_COLOR}Jail ${jname} already running, jid: ${N2_COLOR}${jid}${N0_COLOR}"
[ "${emulator}" != "qemu" ] && err 1 "${N1_COLOR}Not qemu mode${N0_COLOR}"

# cleanup old artifacts before start new session
qcleanup jname="${jname}"

[ -z "${vm_ram}" -o -z "${vm_cpus}" -o -z "${vm_os_type}" ] && err 1 "${N1_COLOR}Parameter is mandatory: ${N2_COLOR}vm_ram, vm_cpus, vm_os_type${N0_COLOR}"
[ -z "${iso_auto_fetch}" ] && iso_auto_fetch=0
[ -z "${debug}" ] && debug=0

if [ ${vm_cpus} -gt ${ncpu} -a ${vm_cpus} -lt ${vm_cpus_max} ]; then
	${ECHO} "${N1_COLOR}Warning! Current node cpu: ${N2_COLOR}${ncpu}${N1_COLOR}, guest cpu: ${N2_COLOR}${vm_cpus}. ${N1_COLOR}Overcommitting vCPUs can hurt perfomance.${N0_COLOR}"
elif [ ${vm_cpus} -lt 1 -o ${vm_cpus} -gt ${vm_cpus_max} ]; then
	err 1 "${N1_COLOR}Valid number of guest CPUs within 1 - ${vm_cpus_max} range. Current vm_cpus: ${N2_COLOR}${vm_cpus}${N0_COLOR}"
fi

main_sqlite_local="${jailsysdir}/${jname}/local.sqlite"

# hardcoded first disk path from SQL. Todo: mark bootable disk(s)
MDFILE=$( cbsdsqlro ${jailsysdir}/${jname}/local.sqlite SELECT dsk_path FROM qemudsk WHERE jname=\"${jname}\" AND dsk_type=\"vhd\" LIMIT 1 2>/dev/null )

if [ -z "${MDFILE}" ]; then
	${ECHO} "${N1_COLOR}Warning: no any storage device found for this VM${N0_COLOR}"
else
	if [ ! -f "${data}/${MDFILE}" -a ! -h "${data}/${MDFILE}" ]; then
		${ECHO} "${N1_COLOR}No such ${data}/${MDFILE} but mdsize flags is not null.${N0_COLOR}"

		# if zfsfeat=1, try scan for zvol
		[ "${zfsfeat}" != "1" ] && break

		readconf zfs.conf
		. ${subrdir}/zfs.subr
		DATA=$( ${ZFS_CMD} get -Ho value name ${jaildatadir} 2>/dev/null )

		[ -z "${DATA}" ] && break

		for lunname in $( ${SEQ_CMD} 0 10 ); do
			if [ -r /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ]; then
				${LN_CMD} -sf /dev/zvol/${DATA}/bcbsd-${jname}-dsk${lunname}.vhd ${data}/dsk${lunname}.vhd
				${ECHO} "${N1_COLOR}Found zvol and create symlink: ${data}/dsk${lunname}.vhd -> ${DATA}/bcbsd-${jname}-dsk${lunname}.vhd"
			fi
		done
	fi
fi

# export variables for external hooks
export jname=${jname}

for _i in ${JARG} ${MYCOL}; do
	T=
	eval T="\$$_i"
	export ${_i}="${T}"
done

# test for incorrect state
if [ ${status} -eq 3 ]; then
	cbsdsqlrw local UPDATE jails SET maintenance=\"${comment}\" WHERE jname=\"${jname}\"
	comment="cbsdsqlro local SELECT maintenance FROM jails WHERE jname=\"${jname}\""
	if [ "${comment}" = "Stopping_VM" ]; then
		jswmode jname=${jname} mode=master comment='0'
	else
		${ECHO} "${N1_COLOR}Xen in maintenance: ${N2_COLOR}${comment}${N0_COLOR}"
		err 1 "${N1_COLOR}Please finish maintenance and switch mode via: ${N2_COLOR}jswmode jname=${jname} mode=master comment='0'${N0_COLOR}"
	fi
fi

start_qemu

end_time=$( ${DATE_CMD} +%s )
diff_time=$(( end_time - st_time ))
diff_time=$( displaytime ${diff_time} )
cbsdlogger NOTICE ${CBSD_APP}: QEMU domain ${jname} started in ${diff_time}

exit 0
