shithub: riscv

Download patch

ref: dd79854239a8c434b1d50e29b381cb21c62f713f
parent: 0d9119da0f6e85e4263fd790989b1851dd3d104d
author: Keegan Saunders <keegan@undefinedbehaviour.org>
date: Sat Oct 28 20:41:46 EDT 2023 

add arm64 qemu kernel

This kernel is designed for running on the QEMU "virt"
machine. It implements the QEMU ECAM-based PCIe and
utilizes VirtIO drivers for networking and storage.
USB, screen and so on are umimplemented, because this
kernel is meant to be installed via UART, and then
administered as a CPU server via rcpu. It is booted
using U-Boot, so the QEMU "virt" machine "firmware"
is required, otherwise installation is straightforward.

This is the QEMU command used to run this kernel on
an M1 Mac under Hypervisor.framework:

	qemu-system-aarch64 -M virt-2.12,accel=hvf,gic-version=3 \
		-cpu host -m 4G -smp 4 \
		-bios u-boot.bin \
		-drive file=9front.arm64.qcow2,if=none,id=disk \
		-device virtio-blk-pci-non-transitional,drive=disk \
		-serial stdio

This code is based off of the i.MX 8 kernel.

diff: cannot open b/sys/src/9/arm64//null: file does not exist: 'b/sys/src/9/arm64//null' diff: cannot open b/sys/src/boot/qemu//null: file does not exist: 'b/sys/src/boot/qemu//null' 
--- a/sys/lib/dist/mkfile
+++ b/sys/lib/dist/mkfile
@@ -32,6 +32,19 @@
 	mv $target.$pid.disk $target
 	}
 
+%.arm64.qcow2: /n/src9/sys/src/boot/qemu/boot.scr
+	@{
+	objtype=arm64
+	kernel=/n/src9/$objtype/9qemu.u
+	> /env/plan9.ini {
+		echo 'console=0'
+	}
+	fatfiles=(/n/src9/sys/src/boot/qemu/boot.scr /env/plan9.ini $kernel)
+	mb=3770
+	mk $target.$pid.disk
+	mv $target.$pid.disk $target
+	}
+
 %.pi.img:
 	@{
 	objtype=arm
@@ -148,7 +161,7 @@
 	mk binds
 	rm -f $target
 	s=`{basename $target}
-	if(~ $target *.amd64.qcow2.*){
+	if(~ $target *.qcow2.*){
 		disk/qcowfs -n `{echo $mb '*1048576' | pc} $target
 		disk/partfs -m /n/$s /mnt/qcow/data
 	}
@@ -172,7 +185,7 @@
 		disk/prep -bw -a^(nvram fs) $d/plan9
 		disk/format -d $d/dos $fatfiles
 	}
-	if not if(~ $target *.reform.img.*){
+	if not if(~ $target *.reform.img.* *.arm64.qcow2.*){
 		{
 			echo 'a p1 4M 100M'
 			echo 't p1 FAT32'
--- /dev/null
+++ b/sys/src/9/arm64/cache.v8.s
@@ -1,0 +1,212 @@
+#include "sysreg.h"
+
+#undef	SYSREG
+#define	SYSREG(op0,op1,Cn,Cm,op2)	SPR(((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5))
+
+/*
+ * instruction cache operations
+ */
+TEXT cacheiinvse(SB), 1, $-4
+	MOVWU	len+8(FP), R2
+	ADD	R0, R2
+
+	MRS	DAIF, R11
+	MSR	$0x2, DAIFSet
+	MOVWU	$1, R10
+	MSR	R10, CSSELR_EL1
+	ISB	$SY
+	MRS	CCSIDR_EL1, R4
+
+	ANDW	$7, R4
+	ADDW	$4, R4		// log2(linelen)
+	LSL	R4, R10
+	LSR	R4, R0
+	LSL	R4, R0
+
+_iinvse:
+	IC	R0, 3,7,5,1	// IVAU
+	ADD	R10, R0
+	CMP	R0, R2
+	BGT	_iinvse
+	DSB	$NSH
+	ISB	$SY
+	MSR	R11, DAIF
+	RETURN
+
+TEXT cacheiinv(SB), 1, $-4
+	IC	R0, 0,7,5,0	// IALLU
+	DSB	$NSH
+	ISB	$SY
+	RETURN
+
+TEXT cacheuwbinv(SB), 1, $0
+	BL	cachedwbinv(SB)
+	BL	cacheiinv(SB)
+	RETURN
+
+/*
+ * data cache operations
+ */
+TEXT cachedwbse(SB), 1, $-4
+	MOV	LR, R29
+	BL	cachedva<>(SB)
+TEXT dccvac(SB), 1, $-4
+	DC	R0, 3,7,10,1	// CVAC
+	RETURN
+
+TEXT cacheduwbse(SB), 1, $-4
+	MOV	LR, R29
+	BL	cachedva<>(SB)
+TEXT dccvau(SB), 1, $-4
+	DC	R0, 3,7,11,1	// CVAU
+	RETURN
+
+TEXT cachedinvse(SB), 1, $-4
+	MOV	LR, R29
+	BL	cachedva<>(SB)
+TEXT dcivac(SB), 1, $-4
+	DC	R0, 0,7,6,1	// IVAC
+	RETURN
+
+TEXT cachedwbinvse(SB), 1, $-4
+	MOV	LR, R29
+	BL	cachedva<>(SB)
+TEXT dccivac(SB), 1, $-4
+	DC	R0, 3,7,14,1	// CIVAC
+	RETURN
+
+TEXT cachedva<>(SB), 1, $-4
+	MOV	LR, R1
+	MOVWU	len+8(FP), R2
+	ADD	R0, R2
+
+	MRS	DAIF, R11
+	MSR	$0x2, DAIFSet
+	MOVWU	$0, R10
+	MSR	R10, CSSELR_EL1
+	ISB	$SY
+	MRS	CCSIDR_EL1, R4
+
+	ANDW	$7, R4
+	ADDW	$4, R4		// log2(linelen)
+	MOVWU	$1, R10
+	LSL	R4, R10
+	LSR	R4, R0
+	LSL	R4, R0
+
+	DSB	$SY
+	ISB	$SY
+_cachedva:
+	BL	(R1)
+	ADD	R10, R0
+	CMP	R0, R2
+	BGT	_cachedva
+	DSB	$SY
+	ISB	$SY
+	MSR	R11, DAIF
+	RET	R29
+
+/*
+ * l1 cache operations
+ */
+TEXT cachedwb(SB), 1, $-4
+	MOVWU	$0, R0
+_cachedwb:
+	MOV	LR, R29
+	BL	cachedsw<>(SB)
+TEXT dccsw(SB), 1, $-4
+	DC	R0, 0,7,10,2	// CSW
+	RETURN
+
+TEXT cachedinv(SB), 1, $-4
+	MOVWU	$0, R0
+_cachedinv:
+	MOV	LR, R29
+	BL	cachedsw<>(SB)
+TEXT dcisw(SB), 1, $-4
+	DC	R0, 0,7,6,2	// ISW
+	RETURN
+
+TEXT cachedwbinv(SB), 1, $-4
+	MOVWU	$0, R0
+_cachedwbinv:
+	MOV	LR, R29
+	BL	cachedsw<>(SB)
+TEXT dccisw(SB), 1, $-4
+	DC	R0, 0,7,14,2	// CISW
+	RETURN
+
+/*
+ * l2 cache operations
+ */
+TEXT l2cacheuwb(SB), 1, $-4
+	MOVWU	$1, R0
+	B	_cachedwb
+TEXT l2cacheuinv(SB), 1, $-4
+	MOVWU	$1, R0
+	B	_cachedinv
+TEXT l2cacheuwbinv(SB), 1, $-4
+	MOVWU	$1, R0
+	B	_cachedwbinv
+
+TEXT cachesize(SB), 1, $-4
+	MRS	DAIF, R11
+	MSR	$0x2, DAIFSet
+	MSR	R0, CSSELR_EL1
+	ISB	$SY
+	MRS	CCSIDR_EL1, R0
+	MSR	R11, DAIF
+	RETURN
+
+TEXT cachedsw<>(SB), 1, $-4
+	MOV	LR, R1
+
+	MRS	DAIF, R11
+	MSR	$0x2, DAIFSet
+	ADDW	R0, R0, R8
+	MSR	R8, CSSELR_EL1
+	ISB	$SY
+	MRS	CCSIDR_EL1, R4
+
+	LSR	$3, R4, R7
+	ANDW	$1023, R7	// lastway
+	ADDW	$1, R7, R5	// #ways
+
+	LSR	$13, R4, R2
+	ANDW	$32767, R2	// lastset
+	ADDW	$1, R2		// #sets
+
+	ANDW	$7, R4
+	ADDW	$4, R4		// log2(linelen)
+
+	MOVWU	$32, R3		// wayshift = 32 - log2(#ways)
+_countlog2ways:
+	CBZ	R7, _loop	// lastway == 0?
+	LSR	$1, R7		// lastway >>= 1
+	SUB	$1, R3		// wayshift--
+	B _countlog2ways
+_loop:
+	DSB	$SY
+	ISB	$SY
+_nextway:
+	MOVWU	$0, R6		// set
+_nextset:
+	LSL	R3, R7, R0	// way<<wayshift
+	LSL	R4, R6, R9	// set<<log2(linelen)
+	ORRW	R8, R0		// level
+	ORRW	R9, R0		// setway
+
+	BL	(R1)		// op(setway)
+
+	ADDW	$1, R6		// set++
+	CMPW	R2, R6
+	BLT	_nextset
+
+	ADDW	$1, R7		// way++
+	CMPW	R5, R7
+	BLT	_nextway
+
+	DSB	$SY
+	ISB	$SY
+	MSR	R11, DAIF
+	RET	R29
--- /dev/null
+++ b/sys/src/9/arm64/clock.c
@@ -1,0 +1,120 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "io.h"
+#include "ureg.h"
+#include "sysreg.h"
+
+static uvlong freq;
+
+enum {
+	Enable	= 1<<0,
+	Imask	= 1<<1,
+	Istatus = 1<<2,
+};
+
+void
+clockshutdown(void)
+{
+}
+
+static void
+localclockintr(Ureg *ureg, void *)
+{
+	timerintr(ureg, 0);
+}
+
+void
+clockinit(void)
+{
+	syswr(PMCR_EL0, 1<<6 | 7);
+	syswr(PMCNTENSET, 1<<31);
+	syswr(PMUSERENR_EL0, 1<<2);
+	syswr(CNTKCTL_EL1, 1<<1);
+
+	syswr(CNTV_TVAL_EL0, ~0UL);
+	syswr(CNTV_CTL_EL0, Enable);
+
+	if(m->machno == 0){
+		freq = sysrd(CNTFRQ_EL0);
+		print("timer frequency %lld Hz\n", freq);
+	}
+
+	/*
+	 * we are using virtual counter register CNTVCT_EL0
+	 * instead of the performance counter in userspace.
+	 */
+	m->cyclefreq = freq;
+
+	intrenable(IRQcntvns, localclockintr, nil, BUSUNKNOWN, "clock");
+}
+
+void
+timerset(uvlong next)
+{
+	uvlong now;
+	long period;
+
+	now = fastticks(nil);
+	period = next - now;
+	syswr(CNTV_TVAL_EL0, period);
+}
+
+uvlong
+fastticks(uvlong *hz)
+{
+	if(hz)
+		*hz = freq;
+	return sysrd(CNTVCT_EL0);
+}
+
+ulong
+perfticks(void)
+{
+	return fastticks(nil);
+}
+
+ulong
+µs(void)
+{
+	uvlong hz;
+	uvlong t = fastticks(&hz);
+	return (t * 1000000ULL) / hz;
+}
+
+void
+microdelay(int n)
+{
+	ulong now;
+
+	now = µs();
+	while(µs() - now < n);
+}
+
+void
+delay(int n)
+{
+	while(--n >= 0)
+		microdelay(1000);
+}
+
+void
+synccycles(void)
+{
+	static Ref r1, r2;
+	int s;
+
+	s = splhi();
+	r2.ref = 0;
+	incref(&r1);
+	while(r1.ref != conf.nmach)
+		;
+//	syswr(PMCR_EL0, 1<<6 | 7);
+	incref(&r2);
+	while(r2.ref != conf.nmach)
+		;
+	r1.ref = 0;
+	splx(s);
+}
--- /dev/null
+++ b/sys/src/9/arm64/dat.h
@@ -1,0 +1,229 @@
+/*
+ * Time.
+ *
+ * HZ should divide 1000 evenly, ideally.
+ * 100, 125, 200, 250 and 333 are okay.
+ */
+#define	HZ		100			/* clock frequency */
+#define	MS2HZ		(1000/HZ)		/* millisec per clock tick */
+#define	TK2SEC(t)	((t)/HZ)		/* ticks to seconds */
+
+enum {
+	Mhz	= 1000 * 1000,
+
+	GpioLow = 0,
+	GpioHigh,
+	GpioRising,
+	GpioFalling,
+	GpioEdge,
+};
+
+typedef struct Conf	Conf;
+typedef struct Confmem	Confmem;
+typedef struct FPsave	FPsave;
+typedef struct PFPU	PFPU;
+typedef struct ISAConf	ISAConf;
+typedef struct Label	Label;
+typedef struct Lock	Lock;
+typedef struct Memcache	Memcache;
+typedef struct MMMU	MMMU;
+typedef struct Mach	Mach;
+typedef struct Page	Page;
+typedef struct PhysUart	PhysUart;
+typedef struct Pcidev	Pcidev;
+typedef struct PMMU	PMMU;
+typedef struct Proc	Proc;
+typedef u64int		PTE;
+typedef struct Soc	Soc;
+typedef struct Uart	Uart;
+typedef struct Ureg	Ureg;
+typedef uvlong		Tval;
+typedef void		KMap;
+
+#pragma incomplete Pcidev
+#pragma incomplete Ureg
+
+#define MAXSYSARG	5	/* for mount(fd, mpt, flag, arg, srv) */
+
+/*
+ *  parameters for sysproc.c
+ */
+#define AOUT_MAGIC	(R_MAGIC)
+
+struct Lock
+{
+	ulong	key;
+	u32int	sr;
+	uintptr	pc;
+	Proc*	p;
+	Mach*	m;
+	int	isilock;
+};
+
+struct Label
+{
+	uintptr	sp;
+	uintptr	pc;
+};
+
+struct FPsave
+{
+	uvlong	regs[32][2];
+
+	ulong	control;
+	ulong	status;
+};
+
+#define KFPSTATE
+
+struct PFPU
+{
+	int	fpstate;
+	int	kfpstate;
+	FPsave	*fpsave;
+	FPsave	*kfpsave;
+};
+
+enum
+{
+	FPinit,
+	FPactive,
+	FPinactive,
+	FPprotected,
+
+	/* bits or'd with the state */
+	FPillegal= 0x100,
+};
+
+struct Confmem
+{
+	uintptr	base;
+	ulong	npage;
+	uintptr	limit;
+	uintptr	kbase;
+	uintptr	klimit;
+};
+
+struct Conf
+{
+	ulong	nmach;		/* processors */
+	ulong	nproc;		/* processes */
+	Confmem	mem[3];		/* physical memory */
+	ulong	npage;		/* total physical pages of memory */
+	ulong	upages;		/* user page pool */
+	ulong	copymode;	/* 0 is copy on write, 1 is copy on reference */
+	ulong	ialloc;		/* max interrupt time allocation in bytes */
+	ulong	pipeqsize;	/* size in bytes of pipe queues */
+	ulong	nimage;		/* number of page cache image headers */
+	ulong	nswap;		/* number of swap pages */
+	int	nswppo;		/* max # of pageouts per segment pass */
+	ulong	hz;		/* processor cycle freq */
+	ulong	mhz;
+	int	monitor;	/* flag */
+};
+
+/*
+ *  MMU stuff in Mach.
+ */
+struct MMMU
+{
+	PTE*	mmutop;		/* first level user page table */
+};
+
+/*
+ *  MMU stuff in proc
+ */
+#define NCOLOR	1		/* 1 level cache, don't worry about VCE's */
+
+struct PMMU
+{
+	union {
+	Page	*mmufree;	/* mmuhead[0] is freelist head */
+	Page	*mmuhead[PTLEVELS];
+	};
+	Page	*mmutail[PTLEVELS];
+	int	asid;
+	uintptr	tpidr;
+};
+
+#include "../port/portdat.h"
+
+struct Mach
+{
+	int	machno;			/* physical id of processor */
+	uintptr	splpc;			/* pc of last caller to splhi */
+	Proc*	proc;			/* current process on this processor */
+	/* end of offsets known to asm */
+
+	MMMU;
+
+	PMach;
+
+	int	fpstate;
+	FPsave	*fpsave;
+
+	int	cputype;
+	ulong	delayloop;
+
+	int	stack[1];
+};
+
+struct
+{
+	char	machs[MAXMACH];		/* active CPUs */
+	int	exiting;		/* shutdown */
+}active;
+
+#define MACHP(n)	((Mach*)MACHADDR(n))
+
+extern register Mach* m;			/* R27 */
+extern register Proc* up;			/* R26 */
+extern int normalprint;
+
+/*
+ *  a parsed plan9.ini line
+ */
+#define NISAOPT		8
+
+struct ISAConf {
+	char	*type;
+	uvlong	port;
+	int	irq;
+	ulong	dma;
+	ulong	mem;
+	ulong	size;
+	ulong	freq;
+
+	int	nopt;
+	char	*opt[NISAOPT];
+};
+
+/*
+ * Horrid. But the alternative is 'defined'.
+ */
+#ifdef _DBGC_
+#define DBGFLG		(dbgflg[_DBGC_])
+#else
+#define DBGFLG		(0)
+#endif /* _DBGC_ */
+
+int vflag;
+extern char dbgflg[256];
+
+#define dbgprint	print		/* for now */
+
+/*
+ *  hardware info about a device
+ */
+typedef struct {
+	ulong	port;
+	int	size;
+} Devport;
+
+struct DevConf
+{
+	ulong	intnum;			/* interrupt number */
+	char	*type;			/* card type, malloced */
+	int	nports;			/* Number of ports */
+	Devport	*ports;			/* The ports themselves */
+};
--- /dev/null
+++ b/sys/src/9/arm64/devrtc.c
@@ -1,0 +1,108 @@
+/*
+ *  PL031 RTC driver
+ */
+
+#include	"u.h"
+#include	"../port/lib.h"
+#include	"mem.h"
+#include	"dat.h"
+#include	"fns.h"
+#include	"../port/error.h"
+
+enum{
+	Qdir = 0,
+	Qrtc,
+};
+
+static u32int *regs = (u32int *)(VIRTIO + 0x01010000);
+
+Dirtab rtcdir[]={
+	".",	{Qdir, 0, QTDIR},	0,	0555,
+	"rtc",	{Qrtc, 0},		0,	0664,
+};
+
+static Chan*
+rtcattach(char* spec)
+{
+	return devattach('r', spec);
+}
+
+static Walkqid*	 
+rtcwalk(Chan* c, Chan *nc, char** name, int nname)
+{
+	return devwalk(c, nc, name, nname, rtcdir, nelem(rtcdir), devgen);
+}
+
+static int	 
+rtcstat(Chan* c, uchar* dp, int n)
+{
+	return devstat(c, dp, n, rtcdir, nelem(rtcdir), devgen);
+}
+
+static Chan*
+rtcopen(Chan* c, int omode)
+{
+	omode = openmode(omode);
+	switch((ulong)c->qid.path){
+	case Qrtc:
+		if(strcmp(up->user, eve)!=0 && omode!=OREAD)
+			error(Eperm);
+		break;
+	}
+	return devopen(c, omode, rtcdir, nelem(rtcdir), devgen);
+}
+
+static void	 
+rtcclose(Chan*)
+{
+}
+
+long	 
+rtctime(void)
+{
+	return *regs;
+}
+
+static long	 
+rtcread(Chan* c, void* buf, long n, vlong off)
+{
+	ulong offset = off;
+
+	if(c->qid.type & QTDIR)
+		return devdirread(c, buf, n, rtcdir, nelem(rtcdir), devgen);
+
+	switch((ulong)c->qid.path){
+	case Qrtc:
+		return readnum(offset, buf, n, rtctime(), 12);
+	}
+	error(Ebadarg);
+	return 0;
+}
+
+static long	 
+rtcwrite(Chan*, void*, long, vlong)
+{
+	error(Eperm);
+	return 0;
+}
+
+Dev rtcdevtab = {
+	'r',
+	"rtc",
+
+	devreset,
+	devinit,
+	devshutdown,
+	rtcattach,
+	rtcwalk,
+	rtcstat,
+	rtcopen,
+	devcreate,
+	rtcclose,
+	rtcread,
+	devbread,
+	rtcwrite,
+	devbwrite,
+	devremove,
+	devwstat,
+};
--- /dev/null
+++ b/sys/src/9/arm64/fns.h
@@ -1,0 +1,171 @@
+#include "../port/portfns.h"
+
+/* l.s */
+extern void sev(void);
+extern int tas(void *);
+extern int cmpswap(long*, long, long);
+extern void coherence(void);
+extern void idlehands(void);
+extern uvlong vcycles(void);
+#define cycles(ip) *(ip) = vcycles()
+extern int splfhi(void);
+extern void splflo(void);
+extern void touser(uintptr sp);
+extern void forkret(void);
+extern void noteret(void);
+extern void returnto(void*);
+extern void fpon(void);
+extern void fpoff(void);
+extern void fpsaveregs(void*);
+extern void fploadregs(void*);
+extern void hvccall(Ureg*);
+
+extern void setttbr(uintptr pa);
+extern uintptr getfar(void);
+
+extern void flushasidva(uintptr asidva);
+extern void tlbivae1is(uintptr asidva);
+
+extern void flushasidvall(uintptr asidva);
+extern void tlbivale1is(uintptr asidva);
+
+extern void flushasid(uintptr asid);
+extern void tlbiaside1is(uintptr asid);
+
+extern void flushtlb(void);
+extern void tlbivmalle1(void);
+
+extern void flushlocaltlb(void);
+extern void tlbivmalle1(void);
+
+/* cache */
+extern ulong cachesize(int level);
+
+extern void cacheiinvse(void*, int);
+extern void cacheuwbinv(void);
+extern void cacheiinv(void);
+
+extern void cachedwbse(void*, int);
+extern void cacheduwbse(void*, int);
+extern void cachedinvse(void*, int);
+extern void cachedwbinvse(void*, int);
+
+extern void cachedwb(void);
+extern void cachedinv(void);
+extern void cachedwbinv(void);
+
+extern void l2cacheuwb(void);
+extern void l2cacheuinv(void);
+extern void l2cacheuwbinv(void);
+
+/* mmu */
+#define	getpgcolor(a)	0
+extern uintptr paddr(void*);
+#define PADDR(a) paddr((void*)(a))
+extern uintptr cankaddr(uintptr);
+extern void* kaddr(uintptr);
+#define KADDR(a) kaddr(a)
+extern void kmapinval(void);
+#define	VA(k)	((uintptr)(k))
+extern KMap *kmap(Page*);
+extern void kunmap(KMap*);
+extern uintptr mmukmap(uintptr, uintptr, usize);
+extern void* vmap(uvlong, vlong);
+extern void vunmap(void*, vlong);
+
+extern void mmu0init(uintptr*);
+extern void mmuidmap(uintptr*);
+extern void mmu1init(void);
+extern void meminit(void);
+
+extern void putasid(Proc*);
+
+extern void* ucalloc(usize);
+
+/* clock */
+extern void clockinit(void);
+extern void synccycles(void);
+extern void armtimerset(int);
+extern void clockshutdown(void);
+
+/* fpu */
+extern void fpuinit(void);
+extern void fpuprocsetup(Proc*);
+extern void fpuprocfork(Proc*);
+extern void fpuprocsave(Proc*);
+extern void fpuprocrestore(Proc*);
+extern FPsave* fpukenter(Ureg*);
+extern void fpukexit(Ureg*, FPsave*);
+extern void mathtrap(Ureg*);
+
+/* trap */
+extern void trapinit(void);
+extern int userureg(Ureg*);
+extern void evenaddr(uintptr);
+extern void setkernur(Ureg*, Proc*);
+extern void procfork(Proc*);
+extern void procsetup(Proc*);
+extern void procsave(Proc*);
+extern void procrestore(Proc *);
+extern void trap(Ureg*);
+extern void syscall(Ureg*);
+extern void noted(Ureg*, ulong);
+extern void faultarm64(Ureg*);
+extern void dumpstack(void);
+extern void dumpregs(Ureg*);
+
+/* irq */
+extern void intrinit(void);
+extern void intrcpushutdown(void);
+extern void intrsoff(void);
+extern void intrenable(int, void (*)(Ureg*, void*), void*, int, char*);
+extern void intrdisable(int, void (*)(Ureg*, void*), void*, int, char*);
+extern int irq(Ureg*);
+extern void fiq(Ureg*);
+
+/* sysreg */
+extern uvlong	sysrd(ulong);
+extern void	syswr(ulong, uvlong);
+
+/* uartimx */
+extern void uartconsinit(void);
+
+/* dma */
+extern void dmaflush(int, void*, ulong);
+
+/* main */
+extern char *getconf(char *name);
+extern void setconfenv(void);
+extern void writeconf(void);
+
+extern int isaconfig(char*, int, ISAConf*);
+extern void links(void);
+
+/* ccm */
+extern void setclkgate(char *name, int on);
+extern void setclkrate(char *name, char *source, int freq);
+extern int getclkrate(char *name);
+
+/* gpc */
+extern void powerup(char *dom);
+
+/* lcd */
+extern void lcdinit(void);
+
+/* iomux */
+extern void iomuxpad(char *pads, char *sel, char *cfg);
+extern uint iomuxgpr(int gpr, uint set, uint mask);
+
+/* gpio */
+#define GPIO_PIN(n, m)	((n)<<5 | (m))
+extern void gpioout(uint pin, int set);
+extern int gpioin(uint pin);
+void gpiointrenable(uint pin, int mode, void (*f)(uint pin, void *a), void *a);
+void gpiointrdisable(uint pin);
+
+/* pciimx */
+extern int pcicfgrw8(int tbdf, int rno, int data, int read);
+extern int pcicfgrw16(int tbdf, int rno, int data, int read);
+extern int pcicfgrw32(int tbdf, int rno, int data, int read);
+extern void pciintrenable(int tbdf, void (*f)(Ureg*, void*), void *a);
+extern void pciintrdisable(int tbdf, void (*f)(Ureg*, void*), void *a);
--- /dev/null
+++ b/sys/src/9/arm64/fpu.c
@@ -1,0 +1,289 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+
+#include "ureg.h"
+#include "sysreg.h"
+
+/* libc */
+extern ulong getfcr(void);
+extern void setfcr(ulong fcr);
+extern ulong getfsr(void);
+extern void setfsr(ulong fsr);
+
+static FPsave fpsave0;
+
+static void
+fpsave(FPsave *p)
+{
+	p->control = getfcr();
+	p->status = getfsr();
+	fpsaveregs(p->regs);
+	fpoff();
+}
+
+static void
+fprestore(FPsave *p)
+{
+	fpon();
+	setfcr(p->control);
+	setfsr(p->status);
+	fploadregs(p->regs);
+}
+
+static void
+fpinit(void)
+{
+	fprestore(&fpsave0);
+}
+
+void
+fpuinit(void)
+{
+	m->fpstate = FPinit;
+	m->fpsave = nil;
+	fpoff();
+}
+
+static FPsave*
+fpalloc(void)
+{
+	FPsave *save;
+
+	while((save = mallocalign(sizeof(FPsave), 16, 0, 0)) == nil){
+		spllo();
+		resrcwait("no memory for FPsave");
+		splhi();
+	}
+	return save;
+}
+
+static void
+fpfree(FPsave *save)
+{
+	free(save);
+}
+
+
+/*
+ *  Protect or save FPU state and setup new state
+ *  (lazily in the case of user process) for the kernel.
+ *  All syscalls, traps and interrupts (except mathtrap()!)
+ *  are handled between fpukenter() and fpukexit(),
+ *  so they can use floating point and vector instructions.
+ */
+FPsave*
+fpukenter(Ureg*)
+{
+	if(up == nil){
+		switch(m->fpstate){
+		case FPactive:
+			fpsave(m->fpsave);
+			/* wet floor */
+		case FPinactive:
+			m->fpstate = FPinit;
+			return m->fpsave;
+		}
+		return nil;
+	}
+
+	switch(up->fpstate){
+	case FPactive:
+		up->fpstate = FPprotected;
+		fpoff();
+		/* wet floor */
+	case FPprotected:
+		return nil;
+	}
+
+	switch(up->kfpstate){
+	case FPactive:
+		fpsave(up->kfpsave);
+		/* wet floor */
+	case FPinactive:
+		up->kfpstate = FPinit;
+		return up->kfpsave;
+	}
+	return nil;
+}
+
+void
+fpukexit(Ureg *ureg, FPsave *save)
+{
+	if(up == nil){
+		switch(m->fpstate){
+		case FPactive:
+			fpoff();
+			/* wet floor */
+		case FPinactive:
+			fpfree(m->fpsave);
+			m->fpstate = FPinit;
+		}
+		m->fpsave = save;
+		if(save != nil)
+			m->fpstate = FPinactive;
+		return;
+	}
+
+	if(up->fpstate == FPprotected){
+		if(userureg(ureg)){
+			up->fpstate = FPactive;
+			fpon();
+		}
+		return;
+	}
+
+	switch(up->kfpstate){
+	case FPactive:
+		fpoff();
+		/* wet floor */
+	case FPinactive:
+		fpfree(up->kfpsave);
+		up->kfpstate = FPinit;
+	}
+	up->kfpsave = save;
+	if(save != nil)
+		up->kfpstate = FPinactive;
+}
+
+void
+fpuprocsetup(Proc *p)
+{
+	p->fpstate = FPinit;
+}
+
+void
+fpuprocfork(Proc *p)
+{
+	int s;
+
+	s = splhi();
+	switch(up->fpstate & ~FPillegal){
+	case FPprotected:
+		fpon();
+		/* wet floor */
+	case FPactive:
+		fpsave(up->fpsave);
+		up->fpstate = FPinactive;
+		/* wet floor */
+	case FPinactive:
+		if(p->fpsave == nil)
+			p->fpsave = fpalloc();
+		memmove(p->fpsave, up->fpsave, sizeof(FPsave));
+		p->fpstate = FPinactive;
+	}
+	splx(s);
+}
+
+void
+fpuprocsave(Proc *p)
+{
+	if(p->state == Moribund){
+		if(p->fpstate == FPactive || p->kfpstate == FPactive)
+			fpoff();
+		fpfree(p->fpsave);
+		fpfree(p->kfpsave);
+		p->fpsave = p->kfpsave = nil;
+		p->fpstate = p->kfpstate = FPinit;
+		return;
+	}
+	if(p->kfpstate == FPactive){
+		fpsave(p->kfpsave);
+		p->kfpstate = FPinactive;
+		return;
+	}
+	if(p->fpstate == FPprotected)
+		fpon();
+	else if(p->fpstate != FPactive)
+		return;
+	fpsave(p->fpsave);
+	p->fpstate = FPinactive;
+}
+
+void
+fpuprocrestore(Proc*)
+{
+	/*
+	 * when the scheduler switches,
+	 * we can discard its fp state.
+	 */
+	switch(m->fpstate){
+	case FPactive:
+		fpoff();
+		/* wet floor */
+	case FPinactive:
+		fpfree(m->fpsave);
+		m->fpsave = nil;
+		m->fpstate = FPinit;
+	}
+}
+
+void
+mathtrap(Ureg *ureg)
+{
+	if(!userureg(ureg)){
+		if(up == nil){
+			switch(m->fpstate){
+			case FPinit:
+				m->fpsave = fpalloc();
+				m->fpstate = FPactive;
+				fpinit();
+				break;
+			case FPinactive:
+				fprestore(m->fpsave);
+				m->fpstate = FPactive;
+				break;
+			default:
+				panic("floating point error in irq");
+			}
+			return;
+		}
+
+		if(up->fpstate == FPprotected){
+			fpon();
+			fpsave(up->fpsave);
+			up->fpstate = FPinactive;
+		}
+
+		switch(up->kfpstate){
+		case FPinit:
+			up->kfpsave = fpalloc();
+			up->kfpstate = FPactive;
+			fpinit();
+			break;
+		case FPinactive:
+			fprestore(up->kfpsave);
+			up->kfpstate = FPactive;
+			break;
+		default:
+			panic("floating point error in trap");
+		}
+		return;
+	}
+
+	if(up->fpstate & FPillegal){
+		postnote(up, 1, "sys: floating point in note handler", NDebug);
+		return;
+	}
+	switch(up->fpstate){
+	case FPinit:
+		if(up->fpsave == nil)
+			up->fpsave = fpalloc();
+		up->fpstate = FPactive;
+		fpinit();
+		break;
+	case FPinactive:
+		fprestore(up->fpsave);
+		up->fpstate = FPactive;
+		break;
+	case FPprotected:
+		up->fpstate = FPactive;
+		fpon();
+		break;
+	case FPactive:
+		postnote(up, 1, "sys: floating point error", NDebug);
+		break;
+	}
+}
--- /dev/null
+++ b/sys/src/9/arm64/gic.c
@@ -1,0 +1,320 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "io.h"
+#include "../port/pci.h"
+#include "ureg.h"
+#include "sysreg.h"
+#include "../port/error.h"
+
+enum {
+	GICD_CTLR	= 0x000/4,	/* RW, Distributor Control Register */
+	GICD_TYPER	= 0x004/4,	/* RO, Interrupt Controller Type */
+	GICD_IIDR	= 0x008/4,	/* RO, Distributor Implementer Identification Register */
+
+	GICD_IGROUPR0	= 0x080/4,	/* RW, Interrupt Group Registers (0x80-0xBC) */
+
+	GICD_ISENABLER0	= 0x100/4,	/* RW, Interrupt Set-Enable Registers (0x100-0x13C) */
+	GICD_ICENABLER0	= 0x180/4,	/* RW, Interrupt Clear-Enable Registers (0x180-0x1BC) */
+
+	GICD_ISPENDR0	= 0x200/4,	/* RW, Interrupt Set-Pending Registers (0x200-0x23C) */
+	GICD_ICPENDR0	= 0x280/4,	/* RW, Interrupt Clear-Pending Registers (0x280-0x2BC) */
+
+	GICD_ISACTIVER0	= 0x300/4,	/* RW, Interrupt Set-Active Registers (0x300-0x33C) */
+	GICD_ICACTIVER0 = 0x380/4,	/* RW, Interrupt Clear-Active Registers (0x380-0x3BC) */
+
+	GICD_IPRIORITYR0= 0x400/4,	/* RW, Interrupt Priority Registers (0x400-0x5FC) */
+	GICD_TARGETSR0	= 0x800/4,	/* RW, Interrupt Target Registers (0x800-0x9FC) */
+	GICD_ICFGR0	= 0xC00/4,	/* RW, Interrupt Configuration Registers (0xC00-0xC7C) */
+
+	GICD_ISR0	= 0xD00/4,
+	GICD_PPISR	= GICD_ISR0,	/* RO, Private Peripheral Interrupt Status Register */
+	GICD_SPISR0	= GICD_ISR0+1,	/* RO, Shared Peripheral Interrupt Status Register */
+	GICD_SGIR	= 0xF00/4,	/* WO, Software Generated Interrupt Register */
+
+	GICD_CPENDSGIR0	= 0xF10/4,	/* RW, SGI Clear-Pending Registers (0xF10-0xF1C) */
+	GICD_SPENDSGIR0	= 0xF20/4,	/* RW, SGI Set-Pending Registers (0xF20-0xF2C) */
+
+	GICD_PIDR4	= 0xFD0/4,	/* RO, Perpheral ID Registers */
+	GICD_PIDR5	= 0xFD4/4,
+	GICD_PIDR6	= 0xFD8/4,
+	GICD_PIDR7	= 0xFDC/4,
+	GICD_PIDR0	= 0xFE0/4,
+	GICD_PIDR1	= 0xFE4/4,
+	GICD_PIDR2	= 0xFE8/4,
+	GICD_PIDR3	= 0xFEC/4,
+
+	GICD_CIDR0	= 0xFF0/4,	/* RO, Component ID Registers */
+	GICD_CIDR1	= 0xFF4/4,
+	GICD_CIDR2	= 0xFF8/4,
+	GICD_CIDR3	= 0xFFC/4,
+
+	RD_base		= 0x00000,
+	GICR_CTLR	= (RD_base+0x000)/4,
+	GICR_IIDR	= (RD_base+0x004)/4,
+	GICR_TYPER	= (RD_base+0x008)/4,
+	GICR_STATUSR	= (RD_base+0x010)/4,
+	GICR_WAKER	= (RD_base+0x014)/4,
+	GICR_SETLPIR	= (RD_base+0x040)/4,
+	GICR_CLRLPIR	= (RD_base+0x048)/4,
+	GICR_PROPBASER	= (RD_base+0x070)/4,
+	GICR_PENDBASER	= (RD_base+0x078)/4,
+	GICR_INVLPIR	= (RD_base+0x0A0)/4,
+	GICR_INVALLR	= (RD_base+0x0B0)/4,
+	GICR_SYNCR	= (RD_base+0x0C0)/4,
+
+	SGI_base	= 0x10000,
+	GICR_IGROUPR0	= (SGI_base+0x080)/4,
+	GICR_ISENABLER0	= (SGI_base+0x100)/4,
+	GICR_ICENABLER0	= (SGI_base+0x180)/4,
+	GICR_ISPENDR0	= (SGI_base+0x200)/4,
+	GICR_ICPENDR0	= (SGI_base+0x280)/4,
+	GICR_ISACTIVER0	= (SGI_base+0x300)/4,
+	GICR_ICACTIVER0	= (SGI_base+0x380)/4,
+	GICR_IPRIORITYR0= (SGI_base+0x400)/4,
+	GICR_ICFGR0	= (SGI_base+0xC00)/4,
+	GICR_ICFGR1	= (SGI_base+0xC04)/4,
+	GICR_IGRPMODR0	= (SGI_base+0xD00)/4,
+	GICR_NSACR	= (SGI_base+0xE00)/4,
+};
+
+typedef struct Vctl Vctl;
+struct Vctl {
+	Vctl	*next;
+	void	(*f)(Ureg*, void*);
+	void	*a;
+	int	irq;
+	u32int	intid;
+};
+
+static Lock vctllock;
+static Vctl *vctl[MAXMACH][32], *vfiq;
+static u32int *dregs = (u32int*)VIRTIO;
+
+static u32int*
+getrregs(int machno)
+{
+	u32int *rregs = (u32int*)(VIRTIO + 0xa0000);
+
+	for(;;){
+		if((rregs[GICR_TYPER] & 0xFFFF00) == (machno << 8))
+			return rregs;
+		if(rregs[GICR_TYPER] & (1<<4))
+			break;
+		rregs += (0x20000/4);
+	}
+	panic("getrregs: no re-distributor for cpu %d\n", machno);
+	return nil;
+}
+
+void
+intrcpushutdown(void)
+{
+	/* disable cpu interface */
+	syswr(ICC_IGRPEN0_EL1, 0);
+	syswr(ICC_IGRPEN1_EL1, 0);
+	coherence();
+}
+
+void
+intrsoff(void)
+{
+	/* disable distributor */
+	dregs[GICD_CTLR] = 0;
+	coherence();
+	while(dregs[GICD_CTLR]&(1<<31))
+		;
+}
+
+void
+intrinit(void)
+{
+	u32int *rregs;
+	int i, n;
+
+	if(m->machno == 0){
+		intrsoff();
+
+		/* clear all interrupts */
+		n = ((dregs[GICD_TYPER] & 0x1F)+1) << 5;
+		for(i = 32; i < n; i += 32){
+			dregs[GICD_IGROUPR0 + (i/32)] = -1;
+
+			dregs[GICD_ISENABLER0 + (i/32)] = -1;
+			while(dregs[GICD_CTLR]&(1<<31))
+				;
+			dregs[GICD_ICENABLER0 + (i/32)] = -1;
+			while(dregs[GICD_CTLR]&(1<<31))
+				;
+			dregs[GICD_ICACTIVER0 + (i/32)] = -1;
+		}
+		for(i = 0; i < n; i += 4){
+			dregs[GICD_IPRIORITYR0 + (i/4)] = 0;
+			dregs[GICD_TARGETSR0 + (i/4)] = 0;
+		}
+		for(i = 32; i < n; i += 16){
+			dregs[GICD_ICFGR0 + (i/16)] = 0;
+		}
+		coherence();
+		while(dregs[GICD_CTLR]&(1<<31))
+			;
+		dregs[GICD_CTLR] = (1<<0) | (1<<1) | (1<<4);
+	}
+
+	rregs = getrregs(m->machno);
+	n = 32;
+	for(i = 0; i < n; i += 32){
+		rregs[GICR_IGROUPR0 + (i/32)] = -1;
+
+		rregs[GICR_ISENABLER0 + (i/32)] = -1;
+		while(rregs[GICR_CTLR]&(1<<3))
+			;
+		rregs[GICR_ICENABLER0 + (i/32)] = -1;
+		while(dregs[GICD_CTLR]&(1<<31))
+			;
+		rregs[GICR_ICACTIVER0 + (i/32)] = -1;
+	}
+	for(i = 0; i < n; i += 4){
+		rregs[GICR_IPRIORITYR0 + (i/4)] = 0;
+	}
+	coherence();
+	while(rregs[GICR_CTLR]&(1<<3))
+		;
+
+	coherence();
+
+	/* enable cpu interface */
+	syswr(ICC_CTLR_EL1, 0);
+	syswr(ICC_BPR1_EL1, 7);
+	syswr(ICC_PMR_EL1, 0xFF);
+
+	coherence();
+}
+
+
+/*
+ *  called by trap to handle irq interrupts.
+ *  returns true iff a clock interrupt, thus maybe reschedule.
+ */
+int
+irq(Ureg* ureg)
+{
+	Vctl *v;
+	int clockintr;
+	u32int intid;
+
+	m->intr++;
+	intid = sysrd(ICC_IAR1_EL1) & 0xFFFFFF;
+// iprint("i<%d>", intid);
+	if((intid & ~3) == 1020)
+		return 0; // spurious
+	clockintr = 0;
+	for(v = vctl[m->machno][intid%32]; v != nil; v = v->next)
+		if(v->intid == intid){
+			coherence();
+			v->f(ureg, v->a);
+			coherence();
+			if(v->irq == IRQcntvns)
+				clockintr = 1;
+		}
+	coherence();
+	syswr(ICC_EOIR1_EL1, intid);
+	return clockintr;
+}
+
+/*
+ * called direct from lexception.s to handle fiq interrupt.
+ */
+void
+fiq(Ureg *ureg)
+{
+	Vctl *v;
+	u32int intid;
+
+	m->intr++;
+	intid = sysrd(ICC_IAR1_EL1) & 0xFFFFFF;
+// iprint("f<%d>", intid);
+	if((intid & ~3) == 1020)
+		return;	// spurious
+	v = vfiq;
+	if(v != nil && v->intid == intid && m->machno == 0){
+		coherence();
+		v->f(ureg, v->a);
+		coherence();
+	}
+	syswr(ICC_EOIR1_EL1, intid);
+}
+
+void
+intrenable(int irq, void (*f)(Ureg*, void*), void *a, int tbdf, char *)
+{
+	Vctl *v;
+	u32int intid;
+	int cpu, prio;
+
+	if(BUSTYPE(tbdf) == BusPCI){
+		pciintrenable(tbdf, f, a);
+		return;
+	}
+
+	if(tbdf != BUSUNKNOWN)
+		return;
+
+	prio = 0x80;
+	intid = irq;
+	if((v = xalloc(sizeof(Vctl))) == nil)
+		panic("intrenable: no mem");
+	v->irq = irq;
+	v->intid = intid;
+	v->f = f;
+	v->a = a;
+
+	lock(&vctllock);
+	if(intid < SPI)
+		cpu = m->machno;
+	else
+		cpu = 0;
+	if(irq == IRQfiq){
+		vfiq = v;
+		prio = 0;
+	}else{
+		v->next = vctl[cpu][intid%32];
+		vctl[cpu][intid%32] = v;
+	}
+	syswr(ICC_IGRPEN1_EL1, sysrd(ICC_IGRPEN1_EL1)|1);
+	coherence();
+
+	syswr(ICC_EOIR1_EL1, intid);
+	coherence();
+
+	/* setup */
+	if(intid < 32){
+		u32int *rregs = getrregs(cpu);
+		rregs[GICR_IPRIORITYR0 + (intid/4)] |= prio << ((intid%4) << 3);
+		coherence();
+		rregs[GICR_ISENABLER0] = 1 << (intid%32);
+		coherence();
+		while(rregs[GICR_CTLR]&(1<<3))
+			;
+	} else {
+		dregs[GICD_IPRIORITYR0 + (intid/4)] |= prio << ((intid%4) << 3);
+		dregs[GICD_TARGETSR0 + (intid/4)] |= (1<<cpu) << ((intid%4) << 3);
+		coherence();
+		dregs[GICD_ISENABLER0 + (intid/32)] = 1 << (intid%32);
+		coherence();
+		while(dregs[GICD_CTLR]&(1<<31))
+			;
+	}
+	unlock(&vctllock);
+}
+
+void
+intrdisable(int tbdf, void (*f)(Ureg*, void*), void *a, int, char*)
+{
+	if(BUSTYPE(tbdf) == BusPCI){
+		pciintrdisable(tbdf, f, a);
+		return;
+	}
+}
--- /dev/null
+++ b/sys/src/9/arm64/init9.s
@@ -1,0 +1,4 @@
+TEXT main(SB), 1, $8
+	MOV	$setSB(SB), R28		/* load the SB */
+	MOV	$boot(SB), R0
+	B	startboot(SB)
--- /dev/null
+++ b/sys/src/9/arm64/io.h
@@ -1,0 +1,17 @@
+enum {
+	IRQfiq		= -1,
+
+	PPI		= 16,
+	SPI		= 32,
+
+	IRQcntvns	= PPI+11,
+
+	IRQuart		= SPI+1,
+
+	IRQpci1		= SPI+3,
+	IRQpci2		= SPI+4,
+	IRQpci3		= SPI+5,
+	IRQpci4		= SPI+6,
+};
+
+#define BUSUNKNOWN (-1)
--- /dev/null
+++ b/sys/src/9/arm64/l.s
@@ -1,0 +1,732 @@
+#include "mem.h"
+#include "sysreg.h"
+
+#undef	SYSREG
+#define	SYSREG(op0,op1,Cn,Cm,op2)	SPR(((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5))
+
+TEXT _start(SB), 1, $-4
+	MOV	R0, R26		/* save */
+
+	MOV	$setSB-KZERO(SB), R28
+	BL	svcmode<>(SB)
+
+	/* use dedicated stack pointer per exception level */
+	MOVWU	$1, R1
+	MSR	R1, SPSel
+
+	BL	mmudisable<>(SB)
+
+	/* invalidate local caches */
+	BL	cachedwbinv(SB)
+	BL	l2cacheuwbinv(SB)
+	BL	cacheiinv(SB)
+
+	MOV	$(MACHADDR(0)-KZERO), R27
+	MRS	MPIDR_EL1, R1
+	ANDW	$(MAXMACH-1), R1
+	MOVWU	$MACHSIZE, R2
+	MULW	R1, R2, R2
+	SUB	R2, R27
+
+	ADD	$(MACHSIZE-16), R27, R2
+	MOV	R2, SP
+
+	CBNZ	R1, _startup
+
+	/* clear page table and machs */
+	MOV	$(L1BOT-KZERO), R1
+	MOV	$(MACHADDR(-1)-KZERO), R2
+_zerol1:
+	MOV	ZR, (R1)8!
+	CMP	R1, R2
+	BNE	_zerol1
+
+	/* clear BSS */
+	MOV	$edata-KZERO(SB), R1
+	MOV	$end-KZERO(SB), R2
+_zerobss:
+	MOV	ZR, (R1)8!
+	CMP	R1, R2
+	BNE	_zerobss
+
+	/* setup page tables */
+	MOV	$(L1BOT-KZERO), R0
+	BL	mmuidmap(SB)
+
+	MOV	$(L1-KZERO), R0
+	BL	mmu0init(SB)
+
+	SEVL
+_startup:
+	WFE
+	BL	mmuenable<>(SB)
+
+	MOV	R26, R0
+	MOV	$0, R26
+	ORR	$KZERO, R27
+	MSR	R27, TPIDR_EL1
+	MOV	$setSB(SB), R28
+
+	BL	main(SB)
+
+TEXT	stop<>(SB), 1, $-4
+_stop:
+	WFE
+	B	_stop
+
+TEXT	aaa<>(SB), 1, $-4
+xxx:
+	MOV $(0x860040+VIRTIO), R1
+	MOVW $'A', R2
+	MOVW R2, (R1)
+	B xxx
+
+TEXT sev(SB), 1, $-4
+	SEV
+	WFE
+	RETURN
+
+TEXT svcmode<>(SB), 1, $-4
+	MSR	$0xF, DAIFSet
+	MRS	CurrentEL, R0
+	ANDW	$(3<<2), R0
+	CMPW	$(1<<2), R0
+	BEQ	el1
+	CMPW	$(2<<2), R0
+	BEQ	el2
+	B	stop<>(SB)
+el2:
+	MOV	$0, R0
+	MSR	R0, MDCR_EL2
+	ISB	$SY
+
+	/* set virtual timer offset to zero */
+	MOV	$0, R0
+	MSR	R0, CNTVOFF_EL2
+
+	/* HCR = RW, HCD, SWIO, BSU, FB */
+	MOVWU	$(1<<31 | 1<<29 | 1<<2 | 0<<10 | 0<<9), R0
+	MSR	R0, HCR_EL2
+	ISB	$SY
+
+	/* SCTLR = RES1 */
+	MOVWU	$(3<<4 | 1<<11 | 1<<16 | 1<<18 | 3<<22 | 3<<28), R0
+	ISB	$SY
+	MSR	R0, SCTLR_EL2
+	ISB	$SY
+
+	/* set VMID to zero */
+	MOV	$0, R0
+	MSR	R0, VTTBR_EL2
+	ISB	$SY
+
+	MOVWU	$(0xF<<6 | 4), R0
+	MSR	R0, SPSR_EL2
+	MSR	LR, ELR_EL2
+	ERET
+el1:
+	RETURN
+
+TEXT mmudisable<>(SB), 1, $-4
+#define SCTLRCLR \
+	/* RES0 */	( 3<<30 \
+	/* RES0 */	| 1<<27 \
+	/* UCI */	| 1<<26 \
+	/* EE */	| 1<<25 \
+	/* RES0 */	| 1<<21 \
+	/* E0E */	| 1<<24 \
+	/* WXN */	| 1<<19 \
+	/* nTWE */	| 1<<18 \
+	/* RES0 */	| 1<<17 \
+	/* nTWI */	| 1<<16 \
+	/* UCT */	| 1<<15 \
+	/* DZE */	| 1<<14 \
+	/* RES0 */	| 1<<13 \
+	/* RES0 */	| 1<<10 \
+	/* UMA */	| 1<<9 \
+	/* SA0 */	| 1<<4 \
+	/* SA */	| 1<<3 \
+	/* A */		| 1<<1 )
+#define SCTLRSET \
+	/* RES1 */	( 3<<28 \
+	/* RES1 */	| 3<<22 \
+	/* RES1 */	| 1<<20 \
+	/* RES1 */	| 1<<11 )
+#define SCTLRMMU \
+	/* I */		( 1<<12 \
+	/* C */		| 1<<2 \
+	/* M */		| 1<<0 )
+
+	/* initialise SCTLR, MMU and caches off */
+	ISB	$SY
+	MRS	SCTLR_EL1, R0
+	BIC	$(SCTLRCLR | SCTLRMMU), R0
+	ORR	$SCTLRSET, R0
+	ISB	$SY
+	MSR	R0, SCTLR_EL1
+	ISB	$SY
+
+	B	flushlocaltlb(SB)
+
+TEXT mmuenable<>(SB), 1, $-4
+	/* return to virtual */
+	ORR	$KZERO, LR
+	MOV	LR, -16(RSP)!
+
+	BL	flushlocaltlb(SB)
+
+	/* memory attributes */
+#define MAIRINIT \
+	( 0xFF << MA_MEM_WB*8 \
+	| 0x33 << MA_MEM_WT*8 \
+	| 0x44 << MA_MEM_UC*8 \
+	| 0x00 << MA_DEV_nGnRnE*8 \
+	| 0x04 << MA_DEV_nGnRE*8 \
+	| 0x08 << MA_DEV_nGRE*8 \
+	| 0x0C << MA_DEV_GRE*8 )
+	MOV	$MAIRINIT, R1
+	MSR	R1, MAIR_EL1
+	ISB	$SY
+
+	/* translation control */
+#define TCRINIT \
+	/* TBI1 */	( 0<<38 \
+	/* TBI0 */	| 0<<37 \
+	/* AS */	| 0<<36 \
+	/* TG1 */	| (((3<<16|1<<14|2<<12)>>PGSHIFT)&3)<<30 \
+	/* SH1 */	| SHARE_INNER<<28 \
+	/* ORGN1 */	| CACHE_WB<<26 \
+	/* IRGN1 */	| CACHE_WB<<24 \
+	/* EPD1 */	| 0<<23 \
+	/* A1 */	| 0<<22 \
+	/* T1SZ */	| (64-EVASHIFT)<<16 \
+	/* TG0 */	| (((1<<16|2<<14|0<<12)>>PGSHIFT)&3)<<14 \
+	/* SH0 */	| SHARE_INNER<<12 \
+	/* ORGN0 */	| CACHE_WB<<10 \
+	/* IRGN0 */	| CACHE_WB<<8 \
+	/* EPD0 */	| 0<<7 \
+	/* T0SZ */	| (64-EVASHIFT)<<0 )
+	MOV	$TCRINIT, R1
+	MRS	ID_AA64MMFR0_EL1, R2
+	ANDW	$0x7, R2	// PARange
+	ADD	R2<<32, R1	// IPS
+	MSR	R1, TCR_EL1
+	ISB	$SY
+
+	/* load the page tables */
+	MOV	$(L1BOT-KZERO), R0
+	MOV	$(L1TOP-KZERO), R1
+	ISB	$SY
+	MSR	R0, TTBR0_EL1
+	MSR	R1, TTBR1_EL1
+	ISB	$SY
+
+	/* enable MMU and caches */
+	MRS	SCTLR_EL1, R1
+	ORR	$SCTLRMMU, R1
+	ISB	$SY
+	MSR	R1, SCTLR_EL1
+	ISB	$SY
+
+	MOV	RSP, R1
+	ORR	$KZERO, R1
+	MOV	R1, RSP
+	MOV	(RSP)16!, LR
+	B	cacheiinv(SB)
+
+TEXT touser(SB), 1, $-4
+	MOVWU	$0x10028, R1	// entry
+	MOVWU	$0, R2		// psr
+	MSR	R0, SP_EL0	// sp
+	MSR	R1, ELR_EL1
+	MSR	R2, SPSR_EL1
+	ERET
+
+TEXT cas(SB), 1, $-4
+TEXT cmpswap(SB), 1, $-4
+	MOVWU	ov+8(FP), R1
+	MOVWU	nv+16(FP), R2
+_cas1:
+	LDXRW	(R0), R3
+	CMP	R3, R1
+	BNE	_cas0
+	STXRW	R2, (R0), R4
+	CBNZ	R4, _cas1
+	MOVW	$1, R0
+	DMB	$ISH
+	RETURN
+_cas0:
+	CLREX
+	MOVW	$0, R0
+	RETURN
+
+TEXT tas(SB), 1, $-4
+TEXT _tas(SB), 1, $-4
+	MOVW	$0xdeaddead, R2
+_tas1:
+	LDXRW	(R0), R1
+	STXRW	R2, (R0), R3
+	CBNZ	R3, _tas1
+	MOVW	R1, R0
+
+TEXT coherence(SB), 1, $-4
+	DMB	$ISH
+	RETURN
+
+TEXT islo(SB), 1, $-4
+	MRS	DAIF, R0
+	AND	$(0x2<<6), R0
+	EOR	$(0x2<<6), R0
+	RETURN
+
+TEXT splhi(SB), 1, $-4
+	MRS	DAIF, R0
+	MSR	$0x2, DAIFSet
+	RETURN
+
+TEXT splfhi(SB), 1, $-4
+	MRS	DAIF, R0
+	MSR	$0x3, DAIFSet
+	RETURN
+
+TEXT spllo(SB), 1, $-4
+	MSR	$0x3, DAIFClr
+	RETURN
+
+TEXT splflo(SB), 1, $-4
+	MSR	$0x1, DAIFClr
+	RETURN
+
+TEXT splx(SB), 1, $-4
+	MSR	R0, DAIF
+	RETURN
+
+TEXT idlehands(SB), 1, $-4
+	DMB	$ISH
+	MOVW	nrdy(SB), R0
+	CBNZ	R0, _ready
+	WFI
+_ready:
+	RETURN
+
+TEXT vcycles(SB), 1, $-4
+	MRS	CNTVCT_EL0, R0
+	RETURN
+
+TEXT lcycles(SB), 1, $-4
+	MRS	PMCCNTR_EL0, R0
+	RETURN
+
+TEXT setlabel(SB), 1, $-4
+	MOV	LR, 8(R0)
+	MOV	SP, R1
+	MOV	R1, 0(R0)
+	MOVW	$0, R0
+	RETURN
+
+TEXT gotolabel(SB), 1, $-4
+	MOV	8(R0), LR	/* link */
+	MOV	0(R0), R1	/* sp */
+	MOV	R1, SP
+	MOVW	$1, R0
+	RETURN
+
+TEXT returnto(SB), 1, $-4
+	MOV	R0, 0(SP)
+	RETURN
+
+TEXT getfar(SB), 1, $-4
+	MRS	FAR_EL1, R0
+	RETURN
+
+TEXT setttbr(SB), 1, $-4
+	DSB	$ISHST
+	MSR	R0, TTBR0_EL1
+	DSB	$ISH
+	ISB	$SY
+	RETURN
+
+/*
+ * TLB maintenance operations.
+ * these broadcast to all cpu's in the cluser
+ * (inner sharable domain).
+ */
+TEXT flushasidva(SB), 1, $-4
+TEXT tlbivae1is(SB), 1, $-4
+	DSB	$ISHST
+	TLBI	R0, 0,8,3,1	/* VAE1IS */
+	DSB	$ISH
+	ISB	$SY
+	RETURN
+
+TEXT flushasidvall(SB), 1, $-4
+TEXT tlbivale1is(SB), 1, $-4
+	DSB	$ISHST
+	TLBI	R0, 0,8,3,5	/* VALE1IS */
+	DSB	$ISH
+	ISB	$SY
+	RETURN
+
+TEXT flushasid(SB), 1, $-4
+TEXT tlbiaside1is(SB), 1, $-4
+	DSB	$ISHST
+	TLBI	R0, 0,8,3,2	/* ASIDE1IS */
+	DSB	$ISH
+	ISB	$SY
+	RETURN
+
+TEXT flushtlb(SB), 1, $-4
+TEXT tlbivmalle1is(SB), 1, $-4
+	DSB	$ISHST
+	TLBI	R0, 0,8,3,0	/* VMALLE1IS */
+	DSB	$ISH
+	ISB	$SY
+	RETURN
+
+/*
+ * flush the tlb of this cpu. no broadcast.
+ */
+TEXT flushlocaltlb(SB), 1, $-4
+TEXT tlbivmalle1(SB), 1, $-4
+	DSB	$NSHST
+	TLBI	R0, 0,8,7,0	/* VMALLE1 */
+	DSB	$NSH
+	ISB	$SY
+	RETURN
+
+/*
+ * floating-point support.
+ */
+TEXT fpon(SB), 1, $-4
+	MOVW $(3<<20), R0
+	MSR R0, CPACR_EL1
+	ISB $SY
+	RETURN
+
+TEXT fpoff(SB), 1, $-4
+	MOVW $(0<<20), R0
+	MSR R0, CPACR_EL1
+	ISB $SY
+	RETURN
+
+TEXT fpsaveregs(SB), 1, $-4
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 0)  /* MOV { V0, V1, V2, V3  }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 4)  /* MOV { V4, V5, V6, V7  }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 8)  /* MOV { V8, V9, V10,V11 }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 12) /* MOV { V12,V13,V14,V15 }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 16) /* MOV { V16,V17,V18,V19 }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 20) /* MOV { V20,V21,V22,V23 }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 24) /* MOV { V24,V25,V26,V27 }, (R0)64! */
+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 28) /* MOV { V28,V29,V30,V31 }, (R0)64! */
+	RETURN
+
+TEXT fploadregs(SB), 1, $-4
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 0)  /* MOV (R0)64!, { V0, V1, V2, V3  } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 4)  /* MOV (R0)64!, { V4, V5, V6, V7  } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 8)  /* MOV (R0)64!, { V8, V9, V10,V11 } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 12) /* MOV (R0)64!, { V12,V13,V14,V15 } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 16) /* MOV (R0)64!, { V16,V17,V18,V19 } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 20) /* MOV (R0)64!, { V20,V21,V22,V23 } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 24) /* MOV (R0)64!, { V24,V25,V26,V27 } */
+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 28) /* MOV (R0)64!, { V28,V29,V30,V31 } */
+	RETURN
+
+// syscall or trap from EL0
+TEXT vsys0(SB), 1, $-4
+	LSRW	$26, R0, R17	// ec
+	CMPW	$0x15, R17	// SVC trap?
+	BNE	_itsatrap	// nope.
+
+	MOVP	R26, R27, 224(RSP)
+	MOVP	R28, R29, 240(RSP)
+
+	MRS	SP_EL0, R1
+	MRS	ELR_EL1, R2
+	MRS	SPSR_EL1, R3
+
+	MOV	R0, 288(RSP)	// type
+	MOV	R1, 264(RSP)	// sp
+	MOV	R2, 272(RSP)	// pc
+	MOV	R3, 280(RSP)	// psr
+
+	MOV	$setSB(SB), R28
+	MRS	TPIDR_EL1, R27
+	MOV	16(R27), R26
+
+	ADD	$16, RSP, R0	// ureg
+	BL	syscall(SB)
+
+TEXT forkret(SB), 1, $-4
+	MSR	$0x3, DAIFSet	// interrupts off
+
+	ADD	$16, RSP, R0	// ureg
+
+	MOV	16(RSP), R0	// ret
+	MOV	264(RSP), R1	// sp
+	MOV	272(RSP), R2	// pc
+	MOV	280(RSP), R3	// psr
+
+	MSR	R1, SP_EL0
+	MSR	R2, ELR_EL1
+	MSR	R3, SPSR_EL1
+
+	MOVP	224(RSP), R26, R27
+	MOVP	240(RSP), R28, R29
+
+	MOV	256(RSP), R30	// link
+
+	ADD	$TRAPFRAMESIZE, RSP
+	ERET
+
+TEXT itsatrap<>(SB), 1, $-4
+_itsatrap:
+	MOVP	R1, R2, 24(RSP)
+	MOVP	R3, R4, 40(RSP)
+	MOVP	R5, R6, 56(RSP)
+	MOVP	R7, R8, 72(RSP)
+	MOVP	R9, R10, 88(RSP)
+	MOVP	R11, R12, 104(RSP)
+	MOVP	R13, R14, 120(RSP)
+	MOVP	R15, R16, 136(RSP)
+
+	MOVP	R18, R19, 160(RSP)
+	MOVP	R20, R21, 176(RSP)
+	MOVP	R22, R23, 192(RSP)
+	MOVP	R24, R25, 208(RSP)
+
+// trap/irq/fiq/serr from EL0
+TEXT vtrap0(SB), 1, $-4
+	MOVP	R26, R27, 224(RSP)
+	MOVP	R28, R29, 240(RSP)
+
+	MRS	SP_EL0, R1
+	MRS	ELR_EL1, R2
+	MRS	SPSR_EL1, R3
+
+	MOV	R0, 288(RSP)	// type
+	MOV	R1, 264(RSP)	// sp
+	MOV	R2, 272(RSP)	// pc
+	MOV	R3, 280(RSP)	// psr
+
+	MOV	$setSB(SB), R28
+	MRS	TPIDR_EL1, R27
+	MOV	16(R27), R26
+
+	ADD	$16, RSP, R0	// ureg
+	BL	trap(SB)
+
+TEXT noteret(SB), 1, $-4
+	MSR	$0x3, DAIFSet	// interrupts off
+
+	ADD	$16, RSP, R0	// ureg
+
+	MOV	264(RSP), R1	// sp
+	MOV	272(RSP), R2	// pc
+	MOV	280(RSP), R3	// psr
+
+	MSR	R1, SP_EL0
+	MSR	R2, ELR_EL1
+	MSR	R3, SPSR_EL1
+
+	MOVP	224(RSP), R26, R27
+	MOVP	240(RSP), R28, R29
+
+_intrreturn:
+	MOVP	16(RSP), R0, R1
+	MOVP	32(RSP), R2, R3
+	MOVP	48(RSP), R4, R5
+	MOVP	64(RSP), R6, R7
+	MOVP	80(RSP), R8, R9
+	MOVP	96(RSP), R10, R11
+	MOVP	112(RSP), R12, R13
+	MOVP	128(RSP), R14, R15
+	MOVP	144(RSP), R16, R17
+	MOVP	160(RSP), R18, R19
+	MOVP	176(RSP), R20, R21
+	MOVP	192(RSP), R22, R23
+	MOVP	208(RSP), R24, R25
+
+	MOV	256(RSP), R30	// link
+
+	ADD	$TRAPFRAMESIZE, RSP
+	ERET
+
+// irq/fiq/trap/serr from EL1
+TEXT vtrap1(SB), 1, $-4
+	MOV	R29, 248(RSP)	// special
+
+	ADD	$TRAPFRAMESIZE, RSP, R1
+	MRS	ELR_EL1, R2
+	MRS	SPSR_EL1, R3
+
+	MOV	R0, 288(RSP)	// type
+	MOV	R1, 264(RSP)	// sp
+	MOV	R2, 272(RSP)	// pc
+	MOV	R3, 280(RSP)	// psr
+
+	ADD	$16, RSP, R0	// ureg
+	BL	trap(SB)
+
+	MSR	$0x3, DAIFSet	// interrupts off
+
+	MOV	272(RSP), R2	// pc
+	MOV	280(RSP), R3	// psr
+
+	MSR	R2, ELR_EL1
+	MSR	R3, SPSR_EL1
+
+	MOV	248(RSP), R29	// special
+	B	_intrreturn	
+
+// vector tables
+TEXT vsys(SB), 1, $-4
+	SUB	$TRAPFRAMESIZE, RSP
+
+	MOV	R0, 16(RSP)
+	MOV	R30, 256(RSP)	// link
+
+	MOV	R17, 152(RSP)	// temp
+
+	MRS	ESR_EL1, R0	// type
+
+_vsyspatch:
+	B	_vsyspatch	// branch to vsys0() patched in
+
+TEXT vtrap(SB), 1, $-4
+	SUB	$TRAPFRAMESIZE, RSP
+
+	MOVP	R0, R1, 16(RSP)
+	MOVP	R2, R3, 32(RSP)
+	MOVP	R4, R5, 48(RSP)
+	MOVP	R6, R7, 64(RSP)
+	MOVP	R8, R9, 80(RSP)
+	MOVP	R10, R11, 96(RSP)
+	MOVP	R12, R13, 112(RSP)
+	MOVP	R14, R15, 128(RSP)
+	MOVP	R16, R17, 144(RSP)
+	MOVP	R18, R19, 160(RSP)
+	MOVP	R20, R21, 176(RSP)
+	MOVP	R22, R23, 192(RSP)
+	MOVP	R24, R25, 208(RSP)
+
+	MOV	R30, 256(RSP)	// link
+
+	MRS	ESR_EL1, R0	// type
+
+_vtrappatch:
+	B	_vtrappatch	// branch to vtrapX() patched in
+
+TEXT virq(SB), 1, $-4
+	SUB	$TRAPFRAMESIZE, RSP
+
+	MOVP	R0, R1, 16(RSP)
+	MOVP	R2, R3, 32(RSP)
+	MOVP	R4, R5, 48(RSP)
+	MOVP	R6, R7, 64(RSP)
+	MOVP	R8, R9, 80(RSP)
+	MOVP	R10, R11, 96(RSP)
+	MOVP	R12, R13, 112(RSP)
+	MOVP	R14, R15, 128(RSP)
+	MOVP	R16, R17, 144(RSP)
+	MOVP	R18, R19, 160(RSP)
+	MOVP	R20, R21, 176(RSP)
+	MOVP	R22, R23, 192(RSP)
+	MOVP	R24, R25, 208(RSP)
+
+	MOV	R30, 256(RSP)	// link
+
+	MOV	$(1<<32), R0	// type irq
+
+_virqpatch:
+	B	_virqpatch	// branch to vtrapX() patched in
+
+TEXT vfiq(SB), 1, $-4
+	SUB	$TRAPFRAMESIZE, RSP
+
+	MOVP	R0, R1, 16(RSP)
+	MOVP	R2, R3, 32(RSP)
+	MOVP	R4, R5, 48(RSP)
+	MOVP	R6, R7, 64(RSP)
+	MOVP	R8, R9, 80(RSP)
+	MOVP	R10, R11, 96(RSP)
+	MOVP	R12, R13, 112(RSP)
+	MOVP	R14, R15, 128(RSP)
+	MOVP	R16, R17, 144(RSP)
+	MOVP	R18, R19, 160(RSP)
+	MOVP	R20, R21, 176(RSP)
+	MOVP	R22, R23, 192(RSP)
+	MOVP	R24, R25, 208(RSP)
+
+	MOV	R30, 256(RSP)	// link
+	MOV	$(2<<32), R0	// type fiq
+
+_vfiqpatch:
+	B	_vfiqpatch	// branch to vtrapX() patched in
+
+TEXT vserr(SB), 1, $-4
+	SUB	$TRAPFRAMESIZE, RSP
+
+	MOVP	R0, R1, 16(RSP)
+	MOVP	R2, R3, 32(RSP)
+	MOVP	R4, R5, 48(RSP)
+	MOVP	R6, R7, 64(RSP)
+	MOVP	R8, R9, 80(RSP)
+	MOVP	R10, R11, 96(RSP)
+	MOVP	R12, R13, 112(RSP)
+	MOVP	R14, R15, 128(RSP)
+	MOVP	R16, R17, 144(RSP)
+	MOVP	R18, R19, 160(RSP)
+	MOVP	R20, R21, 176(RSP)
+	MOVP	R22, R23, 192(RSP)
+	MOVP	R24, R25, 208(RSP)
+
+	MOV	R30, 256(RSP)	// link
+
+	MRS	ESR_EL1, R0
+	ORR	$(3<<32), R0	// type
+_vserrpatch:
+	B	_vserrpatch	// branch to vtrapX() patched in
+
+/* fault-proof memcpy */
+TEXT peek(SB), 1, $-4
+	MOV	R0, R1
+	MOV	dst+8(FP), R2
+	MOVWU	len+16(FP), R0
+TEXT _peekinst(SB), 1, $-4
+_peekloop:
+	MOVBU	(R1)1!, R3
+	MOVBU	R3, (R2)1!
+	SUBS	$1, R0
+	BNE	_peekloop
+	RETURN
+
+TEXT hvccall(SB), 1, $32
+	/* save extern registers */
+	MOVP	R26, R27, (RSP)
+
+	/* R0 = Ureg */
+	MOV	R0, R8
+
+	/* save ureg pointer */
+	MOV	R8, 16(RSP)
+
+	MOVP	0(R8), R0, R1
+	MOVP	16(R8), R2, R3
+	MOVP	32(R8), R4, R5
+	MOVP	48(R8), R6, R7
+
+	HVC
+
+	/* restore ureg pointer */
+	MOV	16(RSP), R8
+
+	MOVP	R0, R1, 0(R8)
+	MOVP	R2, R3, 16(R8)
+
+	/* restore extern registers */
+	MOVP	(RSP), R26, R27
+
+	RETURN
+
+
+	
--- /dev/null
+++ b/sys/src/9/arm64/main.c
@@ -1,0 +1,412 @@
+#include "u.h"
+#include "tos.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "../port/error.h"
+#include "pool.h"
+#include "io.h"
+#include "sysreg.h"
+#include "ureg.h"
+
+#include "rebootcode.i"
+
+Conf conf;
+
+#define	MAXCONF 64
+static char *confname[MAXCONF];
+static char *confval[MAXCONF];
+static int nconf = -1;
+
+void
+bootargsinit(void)
+{
+	int i, j, n;
+	char *cp, *line[MAXCONF], *p, *q;
+
+	/*
+	 *  parse configuration args from dos file plan9.ini
+	 */
+	cp = BOOTARGS;
+	cp[BOOTARGSLEN-1] = 0;
+
+	/*
+	 * Strip out '\r', change '\t' -> ' '.
+	 */
+	p = cp;
+	for(q = cp; *q; q++){
+		if(*q == -1)
+			break;
+		if(*q == '\r')
+			continue;
+		if(*q == '\t')
+			*q = ' ';
+		*p++ = *q;
+	}
+	*p = 0;
+
+	n = getfields(cp, line, MAXCONF, 1, "\n");
+	if(n <= 0){
+		/* empty plan9.ini, no configuration passed */
+		return;
+	}
+
+	nconf = 0;
+	for(i = 0; i < n; i++){
+		if(*line[i] == '#')
+			continue;
+		cp = strchr(line[i], '=');
+		if(cp == nil)
+			continue;
+		*cp++ = '\0';
+		for(j = 0; j < nconf; j++){
+			if(cistrcmp(confname[j], line[i]) == 0)
+				break;
+		}
+		confname[j] = line[i];
+		confval[j] = cp;
+		if(j == nconf)
+			nconf++;
+	}
+}
+
+char*
+getconf(char *name)
+{
+	int i;
+
+	for(i = 0; i < nconf; i++)
+		if(cistrcmp(confname[i], name) == 0)
+			return confval[i];
+	return nil;
+}
+
+void
+setconfenv(void)
+{
+	int i;
+
+	if(nconf < 0){
+		/* use defaults when there was no configuration */
+		ksetenv("console", "0", 1);
+		return;
+	}
+
+	for(i = 0; i < nconf; i++){
+		if(confname[i][0] != '*')
+			ksetenv(confname[i], confval[i], 0);
+		ksetenv(confname[i], confval[i], 1);
+	}
+}
+
+void
+writeconf(void)
+{
+	char *p, *q;
+	int n;
+
+	p = getconfenv();
+	if(waserror()) {
+		free(p);
+		nexterror();
+	}
+
+	/* convert to name=value\n format */
+	for(q=p; *q; q++) {
+		q += strlen(q);
+		*q = '=';
+		q += strlen(q);
+		*q = '\n';
+	}
+	n = q - p + 1;
+	if(n >= BOOTARGSLEN)
+		error("kernel configuration too large");
+	memmove(BOOTARGS, p, n);
+	memset(BOOTARGS+n, 0, BOOTARGSLEN-n);
+	poperror();
+	free(p);
+}
+
+int
+isaconfig(char *, int, ISAConf *)
+{
+	return 0;
+}
+
+/*
+ *  starting place for first process
+ */
+void
+init0(void)
+{
+	char buf[2*KNAMELEN], **sp;
+
+	chandevinit();
+
+	if(!waserror()){
+		snprint(buf, sizeof(buf), "%s %s", "ARM64", conffile);
+		ksetenv("terminal", buf, 0);
+		ksetenv("cputype", "arm64", 0);
+		if(cpuserver)
+			ksetenv("service", "cpu", 0);
+		else
+			ksetenv("service", "terminal", 0);
+		setconfenv();
+		poperror();
+	}
+	kproc("alarm", alarmkproc, 0);
+
+	sp = (char**)(USTKTOP-sizeof(Tos) - 8 - sizeof(sp[0])*4);
+	sp[3] = sp[2] = sp[1] = nil;
+	strcpy(sp[1] = (char*)&sp[4], "boot");
+	sp[0] = (void*)&sp[1];
+
+	splhi();
+	fpukexit(nil, nil);
+	touser((uintptr)sp);
+}
+
+void
+confinit(void)
+{
+	int userpcnt;
+	ulong kpages;
+	char *p;
+	int i;
+
+	conf.nmach = 1;
+	if(p = getconf("*ncpu"))
+		conf.nmach = strtol(p, 0, 0);
+	if(conf.nmach > MAXMACH)
+		conf.nmach = MAXMACH;
+
+	if(p = getconf("service")){
+		if(strcmp(p, "cpu") == 0)
+			cpuserver = 1;
+		else if(strcmp(p,"terminal") == 0)
+			cpuserver = 0;
+	}
+
+	if(p = getconf("*kernelpercent"))
+		userpcnt = 100 - strtol(p, 0, 0);
+	else
+		userpcnt = 0;
+
+	if(userpcnt < 10)
+		userpcnt = 60 + cpuserver*10;
+
+	conf.npage = 0;
+	for(i = 0; i < nelem(conf.mem); i++)
+		conf.npage += conf.mem[i].npage;
+
+	kpages = conf.npage - (conf.npage*userpcnt)/100;
+	if(kpages > ((uintptr)-VDRAM)/BY2PG)
+		kpages = ((uintptr)-VDRAM)/BY2PG;
+
+	conf.upages = conf.npage - kpages;
+	conf.ialloc = (kpages/2)*BY2PG;
+
+	/* set up other configuration parameters */
+	conf.nproc = 100 + ((conf.npage*BY2PG)/MB)*5;
+	if(cpuserver)
+		conf.nproc *= 3;
+	if(conf.nproc > 4000)
+		conf.nproc = 4000;
+	conf.nswap = conf.npage*3;
+	conf.nswppo = 4096;
+	conf.nimage = 200;
+
+	conf.copymode = conf.nmach > 1;
+
+	/*
+	 * Guess how much is taken by the large permanent
+	 * datastructures. Mntcache and Mntrpc are not accounted for.
+	 */
+	kpages = conf.npage - conf.upages;
+	kpages *= BY2PG;
+	kpages -= conf.upages*sizeof(Page)
+		+ conf.nproc*sizeof(Proc*)
+		+ conf.nimage*sizeof(Image)
+		+ conf.nswap
+		+ conf.nswppo*sizeof(Page*);
+	mainmem->maxsize = kpages;
+	imagmem->maxsize = kpages;
+}
+
+void
+machinit(void)
+{
+	m->ticks = 1;
+	m->perf.period = 1;
+	active.machs[m->machno] = 1;
+}
+
+void
+mpinit(void)
+{
+	extern void _start(void);
+	int i;
+
+	for(i = 1; i < conf.nmach; i++){
+		Ureg u = {0};
+
+		MACHP(i)->machno = i;
+		cachedwbinvse(MACHP(i), MACHSIZE);
+
+		u.r0 = 0x84000003;	/* CPU_ON */
+		u.r1 = (sysrd(MPIDR_EL1) & ~(0xFF0000FFULL)) | i;
+		u.r2 = PADDR(_start);
+		u.r3 = i;
+		hvccall(&u);
+	}
+	synccycles();
+}
+
+void
+cpuidprint(void)
+{
+	iprint("cpu%d: 1000MHz QEMU\n", m->machno);
+}
+
+void
+main(void)
+{
+	machinit();
+	if(m->machno){
+		trapinit();
+		fpuinit();
+		intrinit();
+		clockinit();
+		cpuidprint();
+		synccycles();
+		timersinit();
+		mmu1init();
+		m->ticks = MACHP(0)->ticks;
+		schedinit();
+		return;
+	}
+	quotefmtinstall();
+	bootargsinit();
+	meminit();
+	confinit();
+	xinit();
+	uartconsinit();
+	printinit();
+	print("\nPlan 9\n");
+	trapinit();
+	fpuinit();
+	intrinit();
+	clockinit();
+	cpuidprint();
+	timersinit();
+	pageinit();
+	procinit0();
+	initseg();
+	links();
+	chandevreset();
+	userinit();
+	mpinit();
+	mmu1init();
+	schedinit();
+}
+
+void
+exit(int)
+{
+	Ureg u = { .r0 = 0x84000002 };	/* CPU_OFF */
+
+	cpushutdown();
+	splfhi();
+
+	if(m->machno == 0){
+		/* clear secrets */
+		zeroprivatepages();
+		poolreset(secrmem);
+
+		u.r0 = 0x84000009;	/* SYSTEM RESET */
+	}
+	hvccall(&u);
+}
+
+static void
+rebootjump(void *entry, void *code, ulong size)
+{
+	void (*f)(void*, void*, ulong);
+
+	intrcpushutdown();
+
+	/* redo identity map */
+	setttbr(PADDR(L1BOT));
+
+	/* setup reboot trampoline function */
+	f = (void*)REBOOTADDR;
+	memmove(f, rebootcode, sizeof(rebootcode));
+
+	cachedwbinvse(f, sizeof(rebootcode));
+	cacheiinvse(f, sizeof(rebootcode));
+
+	(*f)(entry, code, size);
+
+	for(;;);
+}
+
+void
+reboot(void*, void *code, ulong size)
+{
+	writeconf();
+	while(m->machno != 0){
+		procwired(up, 0);
+		sched();
+	}
+
+	cpushutdown();
+	delay(2000);
+
+	splfhi();
+
+	/* turn off buffered serial console */
+	serialoq = nil;
+
+	/* shutdown devices */
+	chandevshutdown();
+
+	/* stop the clock */
+	clockshutdown();
+	intrsoff();
+
+	/* clear secrets */
+	zeroprivatepages();
+	poolreset(secrmem);
+
+	/* off we go - never to return */
+	rebootjump((void*)(KTZERO-KZERO), code, size);
+}
+
+void
+dmaflush(int clean, void *p, ulong len)
+{
+	uintptr s = (uintptr)p;
+	uintptr e = (uintptr)p + len;
+
+	if(clean){
+		s &= ~(BLOCKALIGN-1);
+		e += BLOCKALIGN-1;
+		e &= ~(BLOCKALIGN-1);
+		cachedwbse((void*)s, e - s);
+		return;
+	}
+	if(s & BLOCKALIGN-1){
+		s &= ~(BLOCKALIGN-1);
+		cachedwbinvse((void*)s, BLOCKALIGN);
+		s += BLOCKALIGN;
+	}
+	if(e & BLOCKALIGN-1){
+		e &= ~(BLOCKALIGN-1);
+		if(e < s)
+			return;
+		cachedwbinvse((void*)e, BLOCKALIGN);
+	}
+	if(s < e)
+		cachedinvse((void*)s, e - s);
+}
--- /dev/null
+++ b/sys/src/9/arm64/mem.h
@@ -1,0 +1,147 @@
+/*
+ * Memory and machine-specific definitions.  Used in C and assembler.
+ */
+#define KiB		1024u			/* Kibi 0x0000000000000400 */
+#define MiB		1048576u		/* Mebi 0x0000000000100000 */
+#define GiB		1073741824u		/* Gibi 000000000040000000 */
+
+/*
+ * Sizes:
+ * 	L0	L1	L2	L3
+ *	4K	2M	1G	512G
+ *	16K	32M	64G	128T
+ *	64K	512M	4T	-
+ */
+#define	PGSHIFT		12		/* log(BY2PG) */
+#define	BY2PG		(1ULL<<PGSHIFT)	/* bytes per page */
+#define	ROUND(s, sz)	(((s)+(sz-1))&~(sz-1))
+#define	PGROUND(s)	ROUND(s, BY2PG)
+
+/* effective virtual address space */
+#define EVASHIFT	34
+#define EVAMASK		((1ULL<<EVASHIFT)-1)
+
+#define PTSHIFT		(PGSHIFT-3)
+#define PTLEVELS	(((EVASHIFT-PGSHIFT)+PTSHIFT-1)/PTSHIFT)	
+#define PTLX(v, l)	((((v) & EVAMASK) >> (PGSHIFT + (l)*PTSHIFT)) & ((1 << PTSHIFT)-1))
+#define PGLSZ(l)	(1ULL << (PGSHIFT + (l)*PTSHIFT))
+
+#define PTL1X(v, l)	(L1TABLEX(v, l) | PTLX(v, l))
+#define L1TABLEX(v, l)	(L1TABLE(v, l) << PTSHIFT)
+#define L1TABLES	((-KSEG0+PGLSZ(2)-1)/PGLSZ(2))
+#define L1TABLE(v, l)	(L1TABLES - ((PTLX(v, 2) % L1TABLES) >> (((l)-1)*PTSHIFT)) + (l)-1)
+#define L1TOPSIZE	(1ULL << (EVASHIFT - PTLEVELS*PTSHIFT))
+
+#define	MAXMACH		24			/* max # cpus system can run */
+#define	MACHSIZE	(8*KiB)
+
+#define KSTACK		(8*KiB)
+#define STACKALIGN(sp)	((sp) & ~7)		/* bug: assure with alloc */
+#define TRAPFRAMESIZE	(38*8)
+
+#define VDRAM		(0xFFFFFFFFC0000000ULL)	/* 0x40000000 - 0x80000000 */
+#define	KTZERO		(VDRAM + 0x100000)	/* 0x40100000 - kernel text start */
+
+#define PHYSIO		0x8000000
+#define PHYSIOEND	0x10000000
+
+#define	VIRTIO		(0xFFFFFFFFB0000000ULL)
+
+#define	KZERO		(0xFFFFFFFF80000000ULL)	/* 0x00000000 - kernel address space */
+
+#define VMAP		(0xFFFFFFFF00000000ULL)	/* 0x00000000 - 0x40000000 */
+
+#define KMAPEND		(0xFFFFFFFF00000000ULL)	/* 0x140000000 */
+#define KMAP		(0xFFFFFFFE00000000ULL)	/*  0x40000000 */
+
+#define KSEG0		(0xFFFFFFFE00000000ULL)
+
+/* temporary identity map for TTBR0 (using only top-level) */
+#define L1BOT		((L1-L1TOPSIZE)&-BY2PG)
+
+/* shared kernel page table for TTBR1 */
+#define L1		(L1TOP-L1SIZE)
+#define L1SIZE		((L1TABLES+PTLEVELS-2)*BY2PG)
+#define L1TOP		((MACHADDR(MAXMACH-1)-L1TOPSIZE)&-BY2PG)
+
+#define MACHADDR(n)	(KTZERO-((n)+1)*MACHSIZE)
+
+#define CONFADDR	(VDRAM + 0x10000)	/* 0x40010000 */
+
+#define BOOTARGS	((char*)CONFADDR)
+#define BOOTARGSLEN	0x10000
+
+#define	REBOOTADDR	(VDRAM-KZERO + 0x20000)	/* 0x40020000 */
+
+#define	UZERO		0ULL			/* user segment */
+#define	UTZERO		(UZERO+0x10000)		/* user text start */
+#define	USTKTOP		((EVAMASK>>1)-0xFFFF)	/* user segment end +1 */
+#define	USTKSIZE	(16*1024*1024)		/* user stack size */
+
+#define BLOCKALIGN	64			/* only used in allocb.c */
+
+/*
+ * Sizes
+ */
+#define BI2BY		8			/* bits per byte */
+#define BY2SE		4
+#define BY2WD		8
+#define BY2V		8			/* only used in xalloc.c */
+
+#define	PTEMAPMEM	(1024*1024)
+#define	PTEPERTAB	(PTEMAPMEM/BY2PG)
+#define	SEGMAPSIZE	8192
+#define	SSEGMAPSIZE	16
+#define	PPN(x)		((x)&~(BY2PG-1))
+
+#define SHARE_NONE	0
+#define SHARE_OUTER	2
+#define SHARE_INNER	3
+
+#define CACHE_UC	0
+#define CACHE_WB	1
+#define CACHE_WT	2
+#define CACHE_WB_NA	3
+
+#define MA_MEM_WB	0
+#define MA_MEM_WT	1
+#define MA_MEM_UC	2
+#define MA_DEV_nGnRnE	3
+#define MA_DEV_nGnRE	4
+#define MA_DEV_nGRE	5
+#define MA_DEV_GRE	6
+
+#define	PTEVALID	1
+#define PTEBLOCK	0
+#define PTETABLE	2
+#define PTEPAGE		2
+
+#define PTEMA(x)	((x)<<2)
+#define PTEAP(x)	((x)<<6)
+#define PTESH(x)	((x)<<8)
+
+#define PTEAF		(1<<10)
+#define PTENG		(1<<11)
+#define PTEPXN		(1ULL<<53)
+#define PTEUXN		(1ULL<<54)
+
+#define PTEKERNEL	PTEAP(0)
+#define PTEUSER		PTEAP(1)
+#define PTEWRITE	PTEAP(0)
+#define PTERONLY	PTEAP(2)
+#define PTENOEXEC	(PTEPXN|PTEUXN)
+
+#define PTECACHED	PTEMA(MA_MEM_WB)
+#define PTEWT		PTEMA(MA_MEM_WT)
+#define PTEUNCACHED	PTEMA(MA_MEM_UC)
+#define PTEDEVICE	PTEMA(MA_DEV_nGnRE)
+
+/*
+ * Physical machine information from here on.
+ *	PHYS addresses as seen from the arm cpu.
+ *	BUS  addresses as seen from peripherals
+ */
+#define	PHYSDRAM	0
+
+#define MIN(a, b)	((a) < (b)? (a): (b))
+#define MAX(a, b)	((a) > (b)? (a): (b))
--- /dev/null
+++ b/sys/src/9/arm64/mkfile
@@ -1,0 +1,101 @@
+CONF=qemu
+CONFLIST=qemu
+
+kzero=0xffffffff80000000
+loadaddr=0xffffffffc0100000
+
+objtype=arm64
+</$objtype/mkfile
+p=9
+
+DEVS=`{rc ../port/mkdevlist $CONF}
+
+PORT=\
+	alarm.$O\
+	alloc.$O\
+	allocb.$O\
+	auth.$O\
+	cache.$O\
+	chan.$O\
+	dev.$O\
+	edf.$O\
+	fault.$O\
+	mul64fract.$O\
+	page.$O\
+	parse.$O\
+	pgrp.$O\
+	portclock.$O\
+	print.$O\
+	proc.$O\
+	qio.$O\
+	qlock.$O\
+	rdb.$O\
+	rebootcmd.$O\
+	segment.$O\
+	syscallfmt.$O\
+	sysfile.$O\
+	sysproc.$O\
+	taslock.$O\
+	tod.$O\
+	xalloc.$O\
+	userinit.$O\
+
+OBJ=\
+	l.$O\
+	cache.v8.$O\
+	clock.$O\
+	fpu.$O\
+	main.$O\
+	mmu.$O\
+	sysreg.$O\
+	random.$O\
+	trap.$O\
+	$CONF.root.$O\
+	$CONF.rootc.$O\
+	$DEVS\
+	$PORT\
+
+# HFILES=
+
+LIB=\
+	/$objtype/lib/libmemlayer.a\
+	/$objtype/lib/libmemdraw.a\
+	/$objtype/lib/libdraw.a\
+	/$objtype/lib/libip.a\
+	/$objtype/lib/libsec.a\
+	/$objtype/lib/libmp.a\
+	/$objtype/lib/libc.a\
+#	/$objtype/lib/libdtracy.a\
+
+9:V:	$p$CONF $p$CONF.u
+
+$p$CONF.u:D:	$p$CONF
+	aux/aout2uimage -Z$kzero $p$CONF
+
+$p$CONF:D:	$OBJ $CONF.$O $LIB
+	$LD -o $target -T$loadaddr -l $prereq
+	size $target
+
+$OBJ: $HFILES
+
+install:V: /$objtype/$p$CONF
+
+/$objtype/$p$CONF:D: $p$CONF $p$CONF.u
+	cp -x $p$CONF $p$CONF.u /$objtype/
+
+<../boot/bootmkfile
+<../port/portmkfile
+<|../port/mkbootrules $CONF
+
+main.$O: rebootcode.i
+
+pciqemu.$O: ../port/pci.h
+
+initcode.out:		init9.$O initcode.$O /$objtype/lib/libc.a
+	$LD -l -R1 -s -o $target $prereq
+
+rebootcode.out:		rebootcode.$O cache.v8.$O
+	$LD -l -H6 -R1 -T0x40020000 -s -o $target $prereq
+
+$CONF.clean:
+	rm -rf $p$CONF $p$CONF.u errstr.h $CONF.c boot$CONF.c
--- /dev/null
+++ b/sys/src/9/arm64/mmu.c
@@ -1,0 +1,450 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "sysreg.h"
+
+#define INITMAP	(ROUND((uintptr)end + BY2PG, PGLSZ(1))-KZERO)
+
+/*
+ * Create initial identity map in top-level page table
+ * (L1BOT) for TTBR0. This page table is only used until
+ * mmu1init() loads m->mmutop.
+ */
+void
+mmuidmap(uintptr *l1bot)
+{
+	uintptr pa, pe, attr;
+
+	/* VDRAM */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
+	pe = -KZERO;
+	for(pa = VDRAM - KZERO; pa < pe; pa += PGLSZ(PTLEVELS-1))
+		l1bot[PTLX(pa, PTLEVELS-1)] = pa | PTEVALID | PTEBLOCK | attr;
+}
+
+/*
+ * Create initial shared kernel page table (L1) for TTBR1.
+ * This page table coveres the INITMAP and VIRTIO,
+ * and later we fill the ram mappings in meminit().
+ */
+void
+mmu0init(uintptr *l1)
+{
+	uintptr va, pa, pe, attr;
+
+	/* DRAM - INITMAP */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
+	pe = INITMAP;
+	for(pa = VDRAM - KZERO, va = VDRAM; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+
+	/* VIRTIO */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
+	pe = PHYSIOEND;
+	for(pa = PHYSIO, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
+		if(((pa|va) & PGLSZ(1)-1) != 0){
+			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
+			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
+				assert(l1[PTLX(va, 0)] == 0);
+				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
+			}
+			break;
+		}
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+	}
+
+	if(PTLEVELS > 2)
+	for(va = KSEG0; va != 0; va += PGLSZ(2))
+		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
+
+	if(PTLEVELS > 3)
+	for(va = KSEG0; va != 0; va += PGLSZ(3))
+		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
+}
+
+void
+mmu1init(void)
+{
+	m->mmutop = mallocalign(L1TOPSIZE, BY2PG, 0, 0);
+	if(m->mmutop == nil)
+		panic("mmu1init: no memory for mmutop");
+	memset(m->mmutop, 0, L1TOPSIZE);
+	mmuswitch(nil);
+}
+
+/* KZERO maps the first 1GB of ram */
+uintptr
+paddr(void *va)
+{
+	if((uintptr)va >= KZERO)
+		return (uintptr)va-KZERO;
+	panic("paddr: va=%#p pc=%#p", va, getcallerpc(&va));
+	return 0;
+}
+
+uintptr
+cankaddr(uintptr pa)
+{
+	if(pa < (uintptr)-KZERO)
+		return -KZERO - pa;
+	return 0;
+}
+
+void*
+kaddr(uintptr pa)
+{
+	if(pa < (uintptr)-KZERO)
+		return (void*)(pa + KZERO);
+	panic("kaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
+	return nil;
+}
+
+static void*
+kmapaddr(uintptr pa)
+{
+	if(pa < (uintptr)-KZERO)
+		return (void*)(pa + KZERO);
+	if(pa < (VDRAM - KZERO) || pa >= (VDRAM - KZERO) + (KMAPEND - KMAP))
+		panic("kmapaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
+	return (void*)(pa + KMAP - (VDRAM - KZERO));
+}
+
+KMap*
+kmap(Page *p)
+{
+	return kmapaddr(p->pa);
+}
+
+void
+kunmap(KMap*)
+{
+}
+
+void
+kmapinval(void)
+{
+}
+
+static void*
+rampage(void)
+{
+	uintptr pa;
+
+	if(conf.npage)
+		return mallocalign(BY2PG, BY2PG, 0, 0);
+
+	pa = conf.mem[0].base;
+	assert((pa % BY2PG) == 0);
+	assert(pa < INITMAP);
+	conf.mem[0].base += BY2PG;
+	return KADDR(pa);
+}
+
+static void
+l1map(uintptr va, uintptr pa, uintptr pe, uintptr attr)
+{
+	uintptr *l1, *l0;
+
+	assert(pa < pe);
+
+	va &= -BY2PG;
+	pa &= -BY2PG;
+	pe = PGROUND(pe);
+
+	attr |= PTEKERNEL | PTEAF;
+
+	l1 = (uintptr*)L1;
+
+	while(pa < pe){
+		if(l1[PTL1X(va, 1)] == 0 && (pe-pa) >= PGLSZ(1) && ((va|pa) & PGLSZ(1)-1) == 0){
+			l1[PTL1X(va, 1)] = PTEVALID | PTEBLOCK | pa | attr;
+			va += PGLSZ(1);
+			pa += PGLSZ(1);
+			continue;
+		}
+		if(l1[PTL1X(va, 1)] & PTEVALID) {
+			assert((l1[PTL1X(va, 1)] & PTETABLE) == PTETABLE);
+			l0 = KADDR(l1[PTL1X(va, 1)] & -PGLSZ(0));
+		} else {
+			l0 = rampage();
+			memset(l0, 0, BY2PG);
+			l1[PTL1X(va, 1)] = PTEVALID | PTETABLE | PADDR(l0);
+		}
+		assert(l0[PTLX(va, 0)] == 0);
+		l0[PTLX(va, 0)] = PTEVALID | PTEPAGE | pa | attr;
+		va += BY2PG;
+		pa += BY2PG;
+	}
+}
+
+static void
+kmapram(uintptr base, uintptr limit)
+{
+	if(base < (uintptr)-KZERO && limit > (uintptr)-KZERO){
+		kmapram(base, (uintptr)-KZERO);
+		kmapram((uintptr)-KZERO, limit);
+		return;
+	}
+	if(base < INITMAP)
+		base = INITMAP;
+	if(base >= limit || limit <= INITMAP)
+		return;
+
+	l1map((uintptr)kmapaddr(base), base, limit,
+		PTEWRITE | PTEPXN | PTEUXN | PTESH(SHARE_INNER));
+}
+
+void
+meminit(void)
+{
+	char *p;
+
+	conf.mem[0].base = PGROUND((uintptr)end - KZERO);
+	conf.mem[0].limit = GiB + 128 * MiB;
+	if(p = getconf("*maxmem"))
+		conf.mem[0].limit = strtoull(p, 0, 0);
+
+	kmapram(conf.mem[0].base, conf.mem[0].limit);
+
+	conf.mem[0].npage = (conf.mem[0].limit - conf.mem[0].base)/BY2PG;
+}
+
+uintptr
+mmukmap(uintptr va, uintptr pa, usize size)
+{
+	uintptr attr, off;
+
+	if(va == 0)
+		return 0;
+
+	off = pa & BY2PG-1;
+
+	attr = va & PTEMA(7);
+	attr |= PTEWRITE | PTEUXN | PTEPXN | PTESH(SHARE_OUTER);
+
+	va &= -BY2PG;
+	pa &= -BY2PG;
+
+	l1map(va, pa, pa + off + size, attr);
+	flushtlb();
+
+	return va + off;
+}
+
+void*
+vmap(uvlong pa, vlong size)
+{
+	static uintptr base = VMAP;
+	uvlong pe = pa + size;
+	uintptr va;
+
+	va = base;
+	base += PGROUND(pe) - (pa & -BY2PG);
+	
+	return (void*)mmukmap(va | PTEDEVICE, pa, size);
+}
+
+void
+vunmap(void *, vlong)
+{
+}
+
+static uintptr*
+mmuwalk(uintptr va, int level)
+{
+	uintptr *table, pte;
+	Page *pg;
+	int i, x;
+
+	x = PTLX(va, PTLEVELS-1);
+	table = m->mmutop;
+	for(i = PTLEVELS-2; i >= level; i--){
+		pte = table[x];
+		if(pte & PTEVALID) {
+			if(pte & (0xFFFFULL<<48))
+				iprint("strange pte %#p va %#p\n", pte, va);
+			pte &= ~(0xFFFFULL<<48 | BY2PG-1);
+		} else {
+			pg = up->mmufree;
+			if(pg == nil)
+				return nil;
+			up->mmufree = pg->next;
+			pg->va = va & -PGLSZ(i+1);
+			if((pg->next = up->mmuhead[i+1]) == nil)
+				up->mmutail[i+1] = pg;
+			up->mmuhead[i+1] = pg;
+			pte = pg->pa;
+			memset(kmapaddr(pte), 0, BY2PG);
+			coherence();
+			table[x] = pte | PTEVALID | PTETABLE;
+		}
+		table = kmapaddr(pte);
+		x = PTLX(va, (uintptr)i);
+	}
+	return &table[x];
+}
+
+static Proc *asidlist[256];
+
+static int
+allocasid(Proc *p)
+{
+	static Lock lk;
+	Proc *x;
+	int a;
+
+	lock(&lk);
+	a = p->asid;
+	if(a < 0)
+		a = -a;
+	if(a == 0)
+		a = p->pid;
+	for(;; a++){
+		a %= nelem(asidlist);
+		if(a == 0)
+			continue;	// reserved
+		x = asidlist[a];
+		if(x == p || x == nil || (x->asid < 0 && x->mach == nil))
+			break;
+	}
+	p->asid = a;
+	asidlist[a] = p;
+	unlock(&lk);
+
+	return x != p;
+}
+
+static void
+freeasid(Proc *p)
+{
+	int a;
+
+	a = p->asid;
+	if(a < 0)
+		a = -a;
+	if(a > 0 && asidlist[a] == p)
+		asidlist[a] = nil;
+	p->asid = 0;
+}
+
+void
+putasid(Proc *p)
+{
+	/*
+	 * Prevent the following scenario:
+	 *	pX sleeps on cpuA, leaving its page tables in mmutop
+	 *	pX wakes up on cpuB, and exits, freeing its page tables
+	 *  pY on cpuB allocates a freed page table page and overwrites with data
+	 *  cpuA takes an interrupt, and is now running with bad page tables
+	 * In theory this shouldn't hurt because only user address space tables
+	 * are affected, and mmuswitch will clear mmutop before a user process is
+	 * dispatched.  But empirically it correlates with weird problems, eg
+	 * resetting of the core clock at 0x4000001C which confuses local timers.
+	 */
+	if(conf.nmach > 1)
+		mmuswitch(nil);
+
+	if(p->asid > 0)
+		p->asid = -p->asid;
+}
+
+void
+putmmu(uintptr va, uintptr pa, Page *pg)
+{
+	uintptr *pte, old;
+	int s;
+
+	s = splhi();
+	while((pte = mmuwalk(va, 0)) == nil){
+		spllo();
+		up->mmufree = newpage(0, nil, 0);
+		splhi();
+	}
+	old = *pte;
+	*pte = 0;
+	if((old & PTEVALID) != 0)
+		flushasidvall((uvlong)up->asid<<48 | va>>12);
+	else
+		flushasidva((uvlong)up->asid<<48 | va>>12);
+	*pte = pa | PTEPAGE | PTEUSER | PTEPXN | PTENG | PTEAF |
+		(((pa & PTEMA(7)) == PTECACHED)? PTESH(SHARE_INNER): PTESH(SHARE_OUTER));
+	if(needtxtflush(pg)){
+		cachedwbinvse(kmap(pg), BY2PG);
+		cacheiinvse((void*)va, BY2PG);
+		donetxtflush(pg);
+	}
+	splx(s);
+}
+
+static void
+mmufree(Proc *p)
+{
+	int i;
+
+	freeasid(p);
+
+	for(i=1; i<PTLEVELS; i++){
+		if(p->mmuhead[i] == nil)
+			break;
+		p->mmutail[i]->next = p->mmufree;
+		p->mmufree = p->mmuhead[i];
+		p->mmuhead[i] = p->mmutail[i] = nil;
+	}
+}
+
+void
+mmuswitch(Proc *p)
+{
+	uintptr va;
+	Page *t;
+
+	for(va = UZERO; va < USTKTOP; va += PGLSZ(PTLEVELS-1))
+		m->mmutop[PTLX(va, PTLEVELS-1)] = 0;
+
+	if(p == nil){
+		setttbr(PADDR(m->mmutop));
+		return;
+	}
+
+	if(p->newtlb){
+		mmufree(p);
+		p->newtlb = 0;
+	}
+
+	if(allocasid(p))
+		flushasid((uvlong)p->asid<<48);
+
+	setttbr((uvlong)p->asid<<48 | PADDR(m->mmutop));
+
+	for(t = p->mmuhead[PTLEVELS-1]; t != nil; t = t->next){
+		va = t->va;
+		m->mmutop[PTLX(va, PTLEVELS-1)] = t->pa | PTEVALID | PTETABLE;
+	}
+}
+
+void
+mmurelease(Proc *p)
+{
+	mmuswitch(nil);
+	mmufree(p);
+	freepages(p->mmufree, nil, 0);
+	p->mmufree = nil;
+}
+
+void
+flushmmu(void)
+{
+	int x;
+
+	x = splhi();
+	up->newtlb = 1;
+	mmuswitch(up);
+	splx(x);
+}
+
+void
+checkmmu(uintptr, uintptr)
+{
+}
--- /dev/null
+++ b/sys/src/9/arm64/pciqemu.c
@@ -1,0 +1,175 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "io.h"
+#include "../port/pci.h"
+
+#define MMIOBASE 0x10000000
+#define ECAMBASE 0x3F000000
+#define ECAMSIZE 0x01000000
+
+typedef struct Intvec Intvec;
+
+struct Intvec {
+	Pcidev *p;
+	void (*f)(Ureg*, void*);
+	void *a;
+};
+
+static uchar *ecam;
+static Intvec vec[32];
+
+static void*
+cfgaddr(int tbdf, int rno)
+{
+	return ecam + (BUSBNO(tbdf)<<20 | BUSDNO(tbdf)<<15 | BUSFNO(tbdf)<<12) + rno;
+}
+
+int
+pcicfgrw32(int tbdf, int rno, int data, int read)
+{
+	u32int *p;
+
+	if((p = cfgaddr(tbdf, rno & ~3)) != nil){
+		if(read)
+			data = *p;
+		else
+			*p = data;
+	} else {
+		data = -1;
+	}
+	return data;
+}
+
+int
+pcicfgrw16(int tbdf, int rno, int data, int read)
+{
+	u16int *p;
+
+	if((p = cfgaddr(tbdf, rno & ~1)) != nil){
+		if(read)
+			data = *p;
+		else
+			*p = data;
+	} else {
+		data = -1;
+	}
+	return data;
+}
+
+int
+pcicfgrw8(int tbdf, int rno, int data, int read)
+{
+	u8int *p;
+
+	if((p = cfgaddr(tbdf, rno)) != nil){
+		if(read)
+			data = *p;
+		else
+			*p = data;
+	} else {
+		data = -1;
+	}
+	return data;
+}
+
+static void
+pciinterrupt(Ureg *ureg, void *)
+{
+	int i;
+	Intvec *v;
+
+	for(i = 0; i < nelem(vec); i++){
+		v = &vec[i];
+		if(v->f)
+			v->f(ureg, v->a);
+	}
+}
+
+static void
+pciintrinit(void)
+{
+	intrenable(IRQpci1, pciinterrupt, nil, BUSUNKNOWN, "pci");
+	intrenable(IRQpci2, pciinterrupt, nil, BUSUNKNOWN, "pci");
+	intrenable(IRQpci3, pciinterrupt, nil, BUSUNKNOWN, "pci");
+	intrenable(IRQpci4, pciinterrupt, nil, BUSUNKNOWN, "pci");
+}
+
+void
+pciintrenable(int tbdf, void (*f)(Ureg*, void*), void *a)
+{
+	Pcidev *p;
+	int i;
+	Intvec *v;
+
+	if((p = pcimatchtbdf(tbdf)) == nil){
+		print("pciintrenable: %T: unknown device\n", tbdf);
+		return;
+	}
+
+	for(i = 0; i < nelem(vec); i++){
+		v = &vec[i];
+		if(v->f == nil){
+			v->p = p;
+			v->f = f;
+			v->a = a;
+			return;
+		}
+	}
+}
+
+void
+pciintrdisable(int tbdf, void (*f)(Ureg*, void*), void *a)
+{
+	Pcidev *p;
+	int i;
+	Intvec *v;
+
+	if((p = pcimatchtbdf(tbdf)) == nil){
+		print("pciintrdisable: %T: unknown device\n", tbdf);
+		return;
+	}
+
+	for(i = 0; i < nelem(vec); i++){
+		v = &vec[i];
+		if(v->p == p && v->f == f && v->a == a)
+			v->f = nil;
+	}
+}
+
+static void
+pcicfginit(void)
+{
+	char *p;
+	Pcidev *pciroot;
+	ulong ioa;
+	uvlong base;
+
+	fmtinstall('T', tbdffmt);
+
+	pcimaxdno = 32;
+	if(p = getconf("*pcimaxdno"))
+		pcimaxdno = strtoul(p, 0, 0);
+
+	pciscan(0, &pciroot, nil);
+	if(pciroot == nil)
+		return;
+
+	pciintrinit();
+
+	ioa = 0;
+	base = MMIOBASE;
+	pcibusmap(pciroot, &base, &ioa, 1);
+
+	if(getconf("*pcihinv"))
+		pcihinv(pciroot);
+}
+
+void
+pciqemulink(void)
+{
+	ecam = vmap(ECAMBASE, ECAMSIZE);
+	pcicfginit();
+}
--- /dev/null
+++ b/sys/src/9/arm64/qemu
@@ -1,0 +1,50 @@
+dev
+	root
+	cons
+	swap
+	env
+	pipe
+	proc
+	mnt
+	srv
+	shr
+	dup
+	tls
+	cap
+	fs
+	ether	netif
+	bridge	log
+	ip	arp chandial ip ipv6 ipaux iproute netlog nullmedium pktmedium inferno
+	uart
+	rtc
+	pci	pci
+	sd
+
+link
+	ethervirtio10	pci
+	ethersink
+	ethermedium
+	loopbackmedium
+	netdevmedium
+	pciqemu		pci
+
+ip
+	tcp
+	udp
+	il
+	ipifc
+	icmp
+	icmp6
+	igmp
+	ipmux
+misc
+	gic
+	uartqemu
+	sdvirtio10	pci sdscsi
+port
+	int cpuserver = 0;
+bootdir
+	/$objtype/bin/paqfs
+	/$objtype/bin/auth/factotum
+	bootfs.paq
+	boot
--- /dev/null
+++ b/sys/src/9/arm64/rebootcode.s
@@ -1,0 +1,48 @@
+#include "mem.h"
+#include "sysreg.h"
+
+#undef	SYSREG
+#define	SYSREG(op0,op1,Cn,Cm,op2)	SPR(((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5))
+
+TEXT _start(SB), 1, $-4
+	MOV	$setSB(SB), R28
+
+	MOV	R0, R27
+
+	MOV	code+8(FP), R1
+	MOVWU	size+16(FP), R2
+	BIC	$3, R2
+	ADD	R1, R2, R3
+
+_copy:
+	MOVW	(R1)4!, R4
+	MOVW	R4, (R0)4!
+	CMP	R1, R3
+	BNE	_copy
+
+	BL	cachedwbinv(SB)
+	BL	l2cacheuwbinv(SB)
+
+	ISB	$SY
+	MRS	SCTLR_EL1, R0
+	BIC	$(1<<0 | 1<<2 | 1<<12), R0
+	ISB	$SY
+	MSR	R0, SCTLR_EL1
+	ISB	$SY
+
+	DSB	$NSHST
+	TLBI	R0, 0,8,7,0	/* VMALLE1 */
+	DSB	$NSH
+	ISB	$SY
+
+	BL	cachedwbinv(SB)
+	BL	cacheiinv(SB)
+
+	MOVWU	$0, R0
+	MOVWU	$0, R1
+	MOVWU	$0, R2
+	MOVWU	$0, R3
+
+	MOV	R27, LR
+
+	RETURN
--- /dev/null
+++ b/sys/src/9/arm64/sysreg.c
@@ -1,0 +1,58 @@
+/*
+ * ARMv8 system registers
+ * mainly to cope with arm hard-wiring register numbers into instructions.
+ *
+ * these routines must be callable from KZERO.
+ *
+ * on a multiprocessor, process switching to another cpu is assumed
+ * to be inhibited by the caller as these registers are local to the cpu.
+ */
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+
+static void*
+mkinstr(ulong wd)
+{
+	static ulong ib[256], *ep[MAXMACH+1];
+	static Lock lk;
+	ulong *ip, *ie;
+
+	ie = ep[m->machno];
+	for(ip = ib; ip < ie; ip += 2)
+		if(*ip == wd)
+			return ip;
+
+	ilock(&lk);
+	ie = ep[MAXMACH];
+	for(; ip < ie; ip += 2)
+		if(*ip == wd)
+			goto Found;
+	if(ip >= &ib[nelem(ib)])
+		panic("mkinstr: out of instrucuction buffer");
+	ip[0] = wd;
+	ip[1] = 0xd65f03c0;	// RETURN
+	ep[MAXMACH] = ie = ip + 2;
+	cachedwbinvse(ip, 2*sizeof(*ip));
+Found:
+	iunlock(&lk);
+	cacheiinv();
+	ep[m->machno] = ie;
+	return ip;
+}
+
+uvlong
+sysrd(ulong spr)
+{
+	uvlong (*fp)(void) = mkinstr(0xd5380000UL | spr);
+	return fp();
+}
+
+void
+syswr(ulong spr, uvlong val)
+{
+	void (*fp)(uvlong) = mkinstr(0xd5180000UL | spr);
+	fp(val);
+}
--- /dev/null
+++ b/sys/src/9/arm64/sysreg.h
@@ -1,0 +1,90 @@
+#define MIDR_EL1			SYSREG(3,0,0,0,0)
+#define MPIDR_EL1			SYSREG(3,0,0,0,5)
+#define ID_AA64AFR0_EL1			SYSREG(3,0,0,5,4)
+#define ID_AA64AFR1_EL1			SYSREG(3,0,0,5,5)
+#define ID_AA64DFR0_EL1			SYSREG(3,0,0,5,0)
+#define ID_AA64DFR1_EL1			SYSREG(3,0,0,5,1)
+#define ID_AA64ISAR0_EL1		SYSREG(3,0,0,6,0)
+#define ID_AA64ISAR1_EL1		SYSREG(3,0,0,6,1)
+#define ID_AA64MMFR0_EL1		SYSREG(3,0,0,7,0)
+#define ID_AA64MMFR1_EL1		SYSREG(3,0,0,7,1)
+#define ID_AA64PFR0_EL1			SYSREG(3,0,0,4,0)
+#define ID_AA64PFR1_EL1			SYSREG(3,0,0,4,1)
+#define SCTLR_EL1			SYSREG(3,0,1,0,0)
+#define CPACR_EL1			SYSREG(3,0,1,0,2)
+#define MAIR_EL1			SYSREG(3,0,10,2,0)
+#define TCR_EL1				SYSREG(3,0,2,0,2)
+#define TTBR0_EL1			SYSREG(3,0,2,0,0)
+#define TTBR1_EL1			SYSREG(3,0,2,0,1)
+#define ESR_EL1				SYSREG(3,0,5,2,0)
+#define FAR_EL1				SYSREG(3,0,6,0,0)
+#define VBAR_EL1			SYSREG(3,0,12,0,0)
+#define VTTBR_EL2			SYSREG(3,4,2,1,0)
+#define SP_EL0				SYSREG(3,0,4,1,0)
+#define SP_EL1				SYSREG(3,4,4,1,0)
+#define SP_EL2				SYSREG(3,6,4,1,0)
+#define SCTLR_EL2			SYSREG(3,4,1,0,0)
+#define HCR_EL2				SYSREG(3,4,1,1,0)
+#define MDCR_EL2			SYSREG(3,4,1,1,1)
+#define PMCR_EL0			SYSREG(3,3,9,12,0)
+#define PMCNTENSET			SYSREG(3,3,9,12,1)
+#define PMCCNTR_EL0			SYSREG(3,3,9,13,0)
+#define PMUSERENR_EL0			SYSREG(3,3,9,14,0)
+
+#define CNTPCT_EL0			SYSREG(3,3,14,0,1)
+#define CNTVCT_EL0			SYSREG(3,3,14,0,2)
+#define CNTKCTL_EL1			SYSREG(3,0,14,1,0)
+#define	CNTFRQ_EL0			SYSREG(3,3,14,0,0)
+#define CNTV_TVAL_EL0			SYSREG(3,3,14,3,0)
+#define CNTV_CTL_EL0			SYSREG(3,3,14,3,1)
+#define CNTV_CVAL_EL0			SYSREG(3,3,14,3,2)
+#define CNTVOFF_EL2			SYSREG(3,4,14,0,3)
+
+#define TPIDR_EL0			SYSREG(3,3,13,0,2)
+#define TPIDR_EL1			SYSREG(3,0,13,0,4)
+
+#define CCSIDR_EL1			SYSREG(3,1,0,0,0)
+#define CSSELR_EL1			SYSREG(3,2,0,0,0)
+
+#define ACTLR_EL2			SYSREG(3,4,1,0,1)
+#define CPUACTLR_EL1			SYSREG(3,1,15,2,0)
+#define CPUECTLR_EL1			SYSREG(3,1,15,2,1)
+#define CBAR_EL1			SYSREG(3,1,15,3,0)
+
+#define	ICC_AP0R_EL1(m)			SYSREG(3,0,12,8,4|(m))
+#define	ICC_AP1R_EL1(m)			SYSREG(3,0,12,9,0|(m))
+#define	ICC_ASGI1R_EL1			SYSREG(3,0,12,11,6)
+#define	ICC_BPR0_EL1			SYSREG(3,0,12,8,3)
+#define	ICC_BPR1_EL1			SYSREG(3,0,12,12,3)
+#define	ICC_CTLR_EL1			SYSREG(3,0,12,12,4)
+#define	ICC_DIR_EL1			SYSREG(3,0,12,11,1)
+#define	ICC_EOIR0_EL1			SYSREG(3,0,12,8,1)
+#define	ICC_EOIR1_EL1			SYSREG(3,0,12,12,1)
+#define	ICC_HPPIR0_EL1			SYSREG(3,0,12,8,2)
+#define	ICC_HPPIR1_EL1			SYSREG(3,0,12,12,2)
+#define ICC_IAR0_EL1			SYSREG(3,0,12,8,0)
+#define	ICC_IAR1_EL1			SYSREG(3,0,12,12,0)
+#define	ICC_IGRPEN0_EL1			SYSREG(3,0,12,12,6)
+#define	ICC_IGRPEN1_EL1			SYSREG(3,0,12,12,7)
+#define	ICC_NMIAR1_EL1			SYSREG(3,0,12,9,5)
+#define	ICC_PMR_EL1			SYSREG(3,0,4,6,0)
+#define	ICC_RPR_EL1			SYSREG(3,0,12,11,3)
+#define	ICC_SGI0R_EL1			SYSREG(3,0,12,11,7)
+#define	ICC_SGI1R_EL1			SYSREG(3,0,12,11,5)
+#define ICC_SRE_EL1			SYSREG(3,0,12,12,5)
+
+/* l.s redefines this for the assembler */
+#define SYSREG(op0,op1,Cn,Cm,op2)	((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5)
+
+#define	OSHLD	(0<<2 | 1)
+#define OSHST	(0<<2 | 2)
+#define	OSH	(0<<2 | 3)
+#define NSHLD	(1<<2 | 1)
+#define NSHST	(1<<2 | 2)
+#define NSH	(1<<2 | 3)
+#define ISHLD	(2<<2 | 1)
+#define ISHST	(2<<2 | 2)
+#define ISH	(2<<2 | 3)
+#define LD	(3<<2 | 1)
+#define ST	(3<<2 | 2)
+#define SY	(3<<2 | 3)
--- /dev/null
+++ b/sys/src/9/arm64/trap.c
@@ -1,0 +1,688 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "../port/error.h"
+#include "../port/systab.h"
+
+#include <tos.h>
+#include "ureg.h"
+#include "sysreg.h"
+
+int	(*buserror)(Ureg*);
+
+/* SPSR bits user can modify */
+#define USPSRMASK	(0xFULL<<28)
+
+static void
+setupvector(u32int *v, void (*t)(void), void (*f)(void))
+{
+	int i;
+
+	for(i = 0; i < 0x80/4; i++){
+		v[i] = ((u32int*)t)[i];
+		if(v[i] == 0x14000000){
+			v[i] |= ((u32int*)f - &v[i]) & 0x3ffffff;
+			return;
+		}
+	}
+	panic("bug in vector code");
+}
+
+void
+trapinit(void)
+{
+	extern void vsys(void);
+	extern void vtrap(void);
+	extern void virq(void);
+	extern void vfiq(void);
+	extern void vserr(void);
+
+	extern void vsys0(void);
+	extern void vtrap0(void);
+	extern void vtrap1(void);
+
+	static u32int *v;
+
+	intrcpushutdown();
+	if(v == nil){
+		/* disable everything */
+		intrsoff();
+
+		v = mallocalign(0x80*4*4, 1<<11, 0, 0);
+		if(v == nil)
+			panic("no memory for vector table");
+
+		setupvector(&v[0x000/4], vtrap,	vtrap0);
+		setupvector(&v[0x080/4], virq,	vtrap0);
+		setupvector(&v[0x100/4], vfiq,	vtrap0);
+		setupvector(&v[0x180/4], vserr,	vtrap0);
+
+		setupvector(&v[0x200/4], vtrap,	vtrap1);
+		setupvector(&v[0x280/4], virq,	vtrap1);
+		setupvector(&v[0x300/4], vfiq,	vtrap1);
+		setupvector(&v[0x380/4], vserr,	vtrap1);
+
+		setupvector(&v[0x400/4], vsys,	vsys0);
+		setupvector(&v[0x480/4], virq,	vtrap0);
+		setupvector(&v[0x500/4], vfiq,	vtrap0);
+		setupvector(&v[0x580/4], vserr, vtrap0);
+
+		setupvector(&v[0x600/4], vtrap,	vtrap0);
+		setupvector(&v[0x680/4], virq,	vtrap0);
+		setupvector(&v[0x700/4], vfiq,	vtrap0);
+		setupvector(&v[0x780/4], vserr,	vtrap0);
+
+		cacheduwbse(v, 0x80*4*4);
+	}
+	cacheiinvse(v, 0x80*4*4);
+	syswr(VBAR_EL1, (uintptr)v);
+	splx(0x3<<6);	// unmask serr and debug
+}
+
+static char *traps[64] = {
+	[0x00]	"sys: trap: unknown",
+	[0x01]	"sys: trap: WFI or WFE instruction execution",
+	[0x0E]	"sys: trap: illegal execution state",
+	[0x18]	"sys: trap: illegal MSR/MRS access",
+	[0x22]	"sys: trap: misaligned pc",
+	[0x26]	"sys: trap: stack pointer misaligned",
+	[0x30]	"sys: breakpoint",
+	[0x32]	"sys: software step",
+	[0x34]	"sys: watchpoint",
+	[0x3C]	"sys: breakpoint",
+};
+
+void
+trap(Ureg *ureg)
+{
+	FPsave *f = nil;
+	u32int type, intr;
+	int user;
+
+	intr = ureg->type >> 32;
+	if(intr == 2){
+		fiq(ureg);
+		return;
+	}
+	splflo();
+	user = kenter(ureg);
+	type = (u32int)ureg->type >> 26;
+	switch(type){
+	case 0x20:	// instruction abort from lower level
+	case 0x21:	// instruction abort from same level
+	case 0x24:	// data abort from lower level
+	case 0x25:	// data abort from same level
+		f = fpukenter(ureg);
+		faultarm64(ureg);
+		break;
+	case 0x07:	// SIMD/FP
+	case 0x2C:	// FPU exception (A64 only)
+		mathtrap(ureg);
+		break;
+	case 0x00:	// unknown
+		if(intr == 1){
+			f = fpukenter(ureg);
+			if(!irq(ureg))
+				preempted();
+			else if(up != nil && up->delaysched)
+				sched();
+			break;
+		}
+		if(intr == 3){
+	case 0x2F:	// SError interrupt
+			f = fpukenter(ureg);
+			if(buserror != nil && (*buserror)(ureg))
+				break;
+			dumpregs(ureg);
+			panic("SError interrupt");
+			break;
+		}
+		/* wet floor */
+	case 0x01:	// WFI or WFE instruction execution
+	case 0x03:	// MCR or MRC access to CP15 (A32 only)
+	case 0x04:	// MCRR or MRC access to CP15 (A32 only)
+	case 0x05:	// MCR or MRC access to CP14 (A32 only)
+	case 0x06:	// LDC or STD access to CP14 (A32 only)
+	case 0x08:	// MCR or MRC to CP10 (A32 only)
+	case 0x0C:	// MRC access to CP14 (A32 only)
+	case 0x0E:	// Illegal Execution State
+	case 0x11:	// SVC instruction execution (A32 only)
+	case 0x12:	// HVC instruction execution (A32 only)
+	case 0x13:	// SMC instruction execution (A32 only)
+	case 0x15:	// SVC instruction execution (A64 only)
+	case 0x16:	// HVC instruction execution (A64 only)
+	case 0x17:	// SMC instruction execution (A64 only)
+	case 0x18:	// MSR/MRS (A64)
+	case 0x22:	// misaligned pc
+	case 0x26:	// stack pointer misaligned
+	case 0x28:	// FPU exception (A32 only)
+	case 0x30:	// breakpoint from lower level
+	case 0x31:	// breakpoint from same level
+	case 0x32:	// software step from lower level
+	case 0x33:	// software step from same level
+	case 0x34:	// watchpoint execution from lower level
+	case 0x35:	// watchpoint exception from same level
+	case 0x38:	// breapoint (A32 only)
+	case 0x3A:	// vector catch exception (A32 only)
+	case 0x3C:	// BRK instruction (A64 only)
+	default:
+		f = fpukenter(ureg);
+		if(!userureg(ureg)){
+			dumpregs(ureg);
+			panic("unhandled trap");
+		}
+		if(traps[type] == nil) type = 0;	// unknown
+		postnote(up, 1, traps[type], NDebug);
+		break;
+	}
+
+	splhi();
+	if(user){
+		if(up->procctl || up->nnote)
+			notify(ureg);
+		kexit(ureg);
+	}
+	if(type != 0x07 && type != 0x2C)
+		fpukexit(ureg, f);
+}
+
+void
+syscall(Ureg *ureg)
+{
+	vlong startns, stopns;
+	uintptr sp, ret;
+	ulong scallnr;
+	int i, s;
+	char *e;
+
+	if(!kenter(ureg))
+		panic("syscall from  kernel");
+	fpukenter(ureg);
+	
+	m->syscall++;
+	up->insyscall = 1;
+	up->pc = ureg->pc;
+	
+	sp = ureg->sp;
+	up->scallnr = scallnr = ureg->r0;
+	spllo();
+	
+	up->nerrlab = 0;
+	startns = 0;
+	ret = -1;
+	if(!waserror()){
+		if(sp < USTKTOP - BY2PG || sp > USTKTOP - sizeof(Sargs) - BY2WD){
+			validaddr(sp, sizeof(Sargs)+BY2WD, 0);
+			evenaddr(sp);
+		}
+		up->s = *((Sargs*) (sp + BY2WD));
+
+		if(up->procctl == Proc_tracesyscall){
+			syscallfmt(scallnr, ureg->pc, (va_list) up->s.args);
+			s = splhi();
+			up->procctl = Proc_stopme;
+			procctl();
+			splx(s);
+			startns = todget(nil);
+		}
+		
+		if(scallnr >= nsyscall || systab[scallnr] == nil){
+			pprint("bad sys call number %lud pc %#p", scallnr, ureg->pc);
+			postnote(up, 1, "sys: bad sys call", NDebug);
+			error(Ebadarg);
+		}
+		up->psstate = sysctab[scallnr];
+		ret = systab[scallnr]((va_list)up->s.args);
+		poperror();
+	}else{
+		e = up->syserrstr;
+		up->syserrstr = up->errstr;
+		up->errstr = e;
+	}
+	if(up->nerrlab){
+		print("bad errstack [%lud]: %d extra\n", scallnr, up->nerrlab);
+		for(i = 0; i < NERR; i++)
+			print("sp=%#p pc=%#p\n", up->errlab[i].sp, up->errlab[i].pc);
+		panic("error stack");
+	}
+	ureg->r0 = ret;
+	if(up->procctl == Proc_tracesyscall){
+		stopns = todget(nil);
+		sysretfmt(scallnr, (va_list) up->s.args, ret, startns, stopns);
+		s = splhi();
+		up->procctl = Proc_stopme;
+		procctl();
+		splx(s);
+	}
+	up->insyscall = 0;
+	up->psstate = 0;
+
+	if(scallnr == NOTED){
+		noted(ureg, *((ulong*) up->s.args));
+		/*
+		 * normally, syscall() returns to forkret()
+		 * not restoring general registers when going
+		 * to userspace. to completely restore the
+		 * interrupted context, we have to return thru
+		 * noteret(). we override return pc to jump to
+		 * to it when returning form syscall()
+		 */
+		returnto(noteret);
+
+		splhi();
+		up->fpstate &= ~FPillegal;
+	}
+	else
+		splhi();
+
+	if(scallnr != RFORK && (up->procctl || up->nnote))
+		notify(ureg);
+
+	if(up->delaysched){
+		sched();
+		splhi();
+	}
+
+	kexit(ureg);
+	fpukexit(ureg, nil);
+}
+
+int
+notify(Ureg *ureg)
+{
+	uintptr sp;
+	char *msg;
+
+	if(up->procctl)
+		procctl();
+	if(up->nnote == 0)
+		return 0;
+
+	spllo();
+	qlock(&up->debug);
+	msg = popnote(ureg);
+	if(msg == nil){
+		qunlock(&up->debug);
+		splhi();
+		return 0;
+	}
+
+	sp = ureg->sp;
+	sp -= 256;	/* debugging: preserve context causing problem */
+	sp -= sizeof(Ureg);
+	sp = STACKALIGN(sp);
+
+	if(!okaddr((uintptr)up->notify, 1, 0)
+	|| !okaddr(sp-ERRMAX-4*BY2WD, sizeof(Ureg)+ERRMAX+4*BY2WD, 1)
+	|| ((uintptr) up->notify & 3) != 0
+	|| (sp & 7) != 0){
+		qunlock(&up->debug);
+		pprint("suicide: bad address in notify: handler=%#p sp=%#p\n",
+			up->notify, sp);
+		pexit("Suicide", 0);
+	}
+
+	memmove((Ureg*)sp, ureg, sizeof(Ureg));
+	*(Ureg**)(sp-BY2WD) = up->ureg;	/* word under Ureg is old up->ureg */
+	up->ureg = (void*)sp;
+	sp -= BY2WD+ERRMAX;
+	memmove((char*)sp, msg, ERRMAX);
+	sp -= 3*BY2WD;
+	*(uintptr*)(sp+2*BY2WD) = sp+3*BY2WD;
+	*(uintptr*)(sp+1*BY2WD) = (uintptr)up->ureg;
+	ureg->r0 = (uintptr) up->ureg;
+	ureg->sp = sp;
+	ureg->pc = (uintptr) up->notify;
+	ureg->link = 0;
+	qunlock(&up->debug);
+
+	splhi();
+	fpuprocsave(up);
+	up->fpstate |= FPillegal;
+	return 1;
+}
+
+void
+noted(Ureg *ureg, ulong arg0)
+{
+	Ureg *nureg;
+	uintptr oureg, sp;
+
+	qlock(&up->debug);
+	if(arg0 != NRSTR && !up->notified){
+		qunlock(&up->debug);
+		pprint("call to noted() when not notified\n");
+		pexit("Suicide", 0);
+	}
+	up->notified = 0;
+	
+	nureg = up->ureg;
+	
+	oureg = (uintptr) nureg;
+	if(!okaddr(oureg - BY2WD, BY2WD + sizeof(Ureg), 0) || (oureg & 7) != 0){
+		qunlock(&up->debug);
+		pprint("bad ureg in noted or call to noted when not notified\n");
+		pexit("Suicide", 0);
+	}
+
+	nureg->psr = (nureg->psr & USPSRMASK) | (ureg->psr & ~USPSRMASK);
+	memmove(ureg, nureg, sizeof(Ureg));
+	
+	switch(arg0){
+	case NCONT: case NRSTR:
+		if(!okaddr(nureg->pc, BY2WD, 0) || !okaddr(nureg->sp, BY2WD, 0) ||
+				(nureg->pc & 3) != 0 || (nureg->sp & 7) != 0){
+			qunlock(&up->debug);
+			pprint("suicide: trap in noted\n");
+			pexit("Suicide", 0);
+		}
+		up->ureg = (Ureg *) (*(uintptr*) (oureg - BY2WD));
+		qunlock(&up->debug);
+		break;
+	
+	case NSAVE:
+		if(!okaddr(nureg->pc, BY2WD, 0) || !okaddr(nureg->sp, BY2WD, 0) ||
+				(nureg->pc & 3) != 0 || (nureg->sp & 7) != 0){
+			qunlock(&up->debug);
+			pprint("suicide: trap in noted\n");
+			pexit("Suicide", 0);
+		}
+		qunlock(&up->debug);
+		sp = oureg - 4 * BY2WD - ERRMAX;
+		ureg->sp = sp;
+		ureg->r0 = (uintptr) oureg;
+		((uintptr *) sp)[1] = oureg;
+		((uintptr *) sp)[0] = 0;
+		break;
+	
+	default:
+		up->lastnote->flag = NDebug;
+	
+	case NDFLT:
+		qunlock(&up->debug);
+		if(up->lastnote->flag == NDebug)
+			pprint("suicide: %s\n", up->lastnote->msg);
+		pexit(up->lastnote->msg, up->lastnote->flag != NDebug);
+	}
+}
+
+static void
+faultnote(Ureg *ureg, char *access, uintptr addr)
+{
+	extern void checkpages(void);
+	char buf[ERRMAX];
+
+	if(!userureg(ureg)){
+		dumpregs(ureg);
+		panic("fault: %s addr=%#p", access, addr);
+	}
+	checkpages();
+	snprint(buf, sizeof(buf), "sys: trap: fault %s addr=%#p", access, addr);
+	postnote(up, 1, buf, NDebug);
+}
+
+void
+faultarm64(Ureg *ureg)
+{
+	int user, read;
+	uintptr addr;
+
+	user = userureg(ureg);
+	if(user)
+		up->insyscall = 1;
+	else {
+		extern void _peekinst(void);
+
+		if(ureg->pc == (uintptr)_peekinst){
+			ureg->pc = ureg->link;
+			return;
+		}
+		if(waserror()){
+			if(up->nerrlab == 0){
+				pprint("suicide: sys: %s\n", up->errstr);
+				pexit(up->errstr, 1);
+			}
+			nexterror();
+		}
+	}
+
+	addr = getfar();
+	read = (ureg->type & (1<<6)) == 0;
+
+	switch((u32int)ureg->type & 0x3F){
+	case  4: case  5: case  6: case  7:	// Tanslation fault.
+	case  8: case  9: case 10: case 11:	// Access flag fault.
+	case 12: case 13: case 14: case 15:	// Permission fault.
+	case 48:				// tlb conflict fault.
+		if(fault(addr, ureg->pc, read) == 0)
+			break;
+
+		/* wet floor */
+	case  0: case  1: case  2: case  3:	// Address size fault.
+	case 16: 				// synchronous external abort.
+	case 24: 				// synchronous parity error on a memory access.
+	case 20: case 21: case 22: case 23:	// synchronous external abort on a table walk.
+	case 28: case 29: case 30: case 31:	// synchronous parity error on table walk.
+	case 33:				// alignment fault.
+	case 52:				// implementation defined, lockdown abort.
+	case 53:				// implementation defined, unsuppoted exclusive.
+	case 61:				// first level domain fault
+	case 62:				// second level domain fault
+	default:
+		faultnote(ureg, read? "read": "write", addr);
+	}
+
+	if(user)
+		up->insyscall = 0;
+	else
+		poperror();
+}
+
+int
+userureg(Ureg* ureg)
+{
+	return (ureg->psr & 15) == 0;
+}
+
+uintptr
+userpc(void)
+{
+	Ureg *ur = up->dbgreg;
+	return ur->pc;
+}
+
+uintptr
+dbgpc(Proc *)
+{
+	Ureg *ur = up->dbgreg;
+	if(ur == nil)
+		return 0;
+	return ur->pc;
+}
+
+void
+procfork(Proc *p)
+{
+	fpuprocfork(p);
+	p->tpidr = up->tpidr;
+}
+
+void
+procsetup(Proc *p)
+{
+	fpuprocsetup(p);
+	p->tpidr = 0;
+	syswr(TPIDR_EL0, p->tpidr);
+}
+
+void
+procsave(Proc *p)
+{
+	fpuprocsave(p);
+	if(p->kp == 0)
+		p->tpidr = sysrd(TPIDR_EL0);
+	putasid(p);	// release asid
+}
+
+void
+procrestore(Proc *p)
+{
+	fpuprocrestore(p);
+	if(p->kp == 0)
+		syswr(TPIDR_EL0, p->tpidr);
+}
+
+void
+kprocchild(Proc *p, void (*entry)(void))
+{
+	p->sched.pc = (uintptr) entry;
+	p->sched.sp = (uintptr) p - 16;
+	*(void**)p->sched.sp = kprocchild;	/* fake */
+}
+
+void
+forkchild(Proc *p, Ureg *ureg)
+{
+	Ureg *cureg;
+
+	p->sched.pc = (uintptr) forkret;
+	p->sched.sp = (uintptr) p - TRAPFRAMESIZE;
+
+	cureg = (Ureg*) (p->sched.sp + 16);
+	memmove(cureg, ureg, sizeof(Ureg));
+	cureg->r0 = 0;
+}
+
+uintptr
+execregs(uintptr entry, ulong ssize, ulong nargs)
+{
+	uintptr *sp;
+	Ureg *ureg;
+
+	sp = (uintptr*)(USTKTOP - ssize);
+	*--sp = nargs;
+
+	ureg = up->dbgreg;
+	ureg->sp = (uintptr)sp;
+	ureg->pc = entry;
+	ureg->link = 0;
+	return USTKTOP-sizeof(Tos);
+}
+
+void
+evenaddr(uintptr addr)
+{
+	if(addr & 3){
+		postnote(up, 1, "sys: odd address", NDebug);
+		error(Ebadarg);
+	}
+}
+
+void
+callwithureg(void (*f) (Ureg *))
+{
+	Ureg u;
+	
+	u.pc = getcallerpc(&f);
+	u.sp = (uintptr) &f;
+	f(&u);
+}
+
+void
+setkernur(Ureg *ureg, Proc *p)
+{
+	ureg->pc = p->sched.pc;
+	ureg->sp = p->sched.sp;
+	ureg->link = (uintptr)sched;
+}
+
+void
+setupwatchpts(Proc*, Watchpt*, int)
+{
+}
+
+void
+setregisters(Ureg* ureg, char* pureg, char* uva, int n)
+{
+	ulong v;
+
+	v = ureg->psr;
+	memmove(pureg, uva, n);
+	ureg->psr = (ureg->psr & USPSRMASK) | (v & ~USPSRMASK);
+}
+
+static void
+dumpstackwithureg(Ureg *ureg)
+{
+	uintptr v, estack, sp;
+	char *s;
+	int i;
+
+	if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){
+		iprint("dumpstack disabled\n");
+		return;
+	}
+	iprint("ktrace /kernel/path %#p %#p %#p # pc, sp, link\n",
+		ureg->pc, ureg->sp, ureg->link);
+	delay(2000);
+
+	sp = ureg->sp;
+	if(sp < KZERO || (sp & 7) != 0)
+		sp = (uintptr)&ureg;
+
+	estack = (uintptr)m+MACHSIZE;
+	if(up != nil && sp <= (uintptr)up)
+		estack = (uintptr)up;
+
+	if(sp > estack){
+		if(up != nil)
+			iprint("&up %#p sp %#p\n", up, sp);
+		else
+			iprint("&m %#p sp %#p\n", m, sp);
+		return;
+	}
+
+	i = 0;
+	for(; sp < estack; sp += sizeof(uintptr)){
+		v = *(uintptr*)sp;
+		if(KTZERO < v && v < (uintptr)etext && (v & 3) == 0){
+			iprint("%#8.8lux=%#8.8lux ", (ulong)sp, (ulong)v);
+			i++;
+		}
+		if(i == 4){
+			i = 0;
+			iprint("\n");
+		}
+	}
+	if(i)
+		iprint("\n");
+}
+
+void
+dumpstack(void)
+{
+	callwithureg(dumpstackwithureg);
+}
+
+void
+dumpregs(Ureg *ureg)
+{
+	u64int *r;
+	int i, x;
+
+	x = splhi();
+	if(up != nil)
+		iprint("cpu%d: dumpregs ureg %#p process %lud: %s\n", m->machno, ureg,
+			up->pid, up->text);
+	else
+		iprint("cpu%d: dumpregs ureg %#p\n", m->machno, ureg);
+	r = &ureg->r0;
+	for(i = 0; i < 30; i += 3)
+		iprint("R%d %.16llux  R%d %.16llux  R%d %.16llux\n", i, r[i], i+1, r[i+1], i+2, r[i+2]);
+	iprint("PC %#p  SP %#p  LR %#p  PSR %llux  TYPE %llux\n",
+		ureg->pc, ureg->sp, ureg->link,
+		ureg->psr, ureg->type);
+	splx(x);
+}
--- /dev/null
+++ b/sys/src/9/arm64/uartqemu.c
@@ -1,0 +1,366 @@
+/*
+ * PL011 UART from Miller's BCM2835 driver
+ */
+
+#include "u.h"
+#include "../port/lib.h"
+#include "../port/error.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "io.h"
+
+enum {
+	DR	=	0x00>>2,
+	RSRECR	=	0x04>>2,
+	FR	=	0x18>>2,
+		TXFE	= 1<<7,
+		RXFF	= 1<<6,
+		TXFF	= 1<<5,
+		RXFE	= 1<<4,
+		BUSY	= 1<<3,
+
+	ILPR	=	0x20>>2,
+	IBRD	=	0x24>>2,
+	FBRD	=	0x28>>2,
+	LCRH	=	0x2c>>2,
+		WLENM	= 3<<5,
+		WLEN8	= 3<<5,
+		WLEN7	= 2<<5,
+		WLEN6	= 1<<5,
+		WLEN5	= 0<<5,
+		FEN	= 1<<4,	/* fifo enable */
+		STP2	= 1<<3,	/* 2 stop bits */
+		EPS	= 1<<2,	/* even parity select */
+		PEN	= 1<<1,	/* parity enabled */
+		BRK	= 1<<0,	/* send break */
+
+	CR	=	0x30>>2,
+		CTSEN	= 1<<15,
+		RTSEN	= 1<<14,
+		RTS	= 1<<11,
+		RXE	= 1<<9,
+		TXE	= 1<<8,
+		LBE	= 1<<7,
+		UARTEN	= 1<<0,
+		
+	IFLS	=	0x34>>2,
+	IMSC	=	0x38>>2,
+		TXIM	= 1<<5,
+		RXIM	= 1<<4,
+
+	RIS	=	0x3c>>2,
+	MIS	=	0x40>>2,
+	ICR	=	0x44>>2,
+	DMACR	=	0x48>>2,
+	ITCR	=	0x80>>2,
+	ITIP	=	0x84>>2,
+	ITOP	=	0x88>>2,
+	TDR	=	0x8c>>2,
+};
+
+extern PhysUart qemuphysuart;
+
+static Uart qemuuart = {
+	.regs	= (u32int*)(VIRTIO+0x1000000ULL),
+	.name	= "uart0",
+	.freq	= 24*Mhz,
+	.baud	= 115200,
+	.phys	= &qemuphysuart,
+};
+
+static Uart*
+pnp(void)
+{
+	return &qemuuart;
+}
+
+static void
+interrupt(Ureg*, void *arg)
+{
+	Uart *uart = arg;
+	u32int *reg = (u32int*)uart->regs;
+
+	coherence();
+	while((reg[FR] & RXFE) == 0)
+		uartrecv(uart, reg[DR] & 0xFF);
+	if((reg[FR] & TXFF) == 0)
+		uartkick(uart);
+	reg[ICR] = 1<<5 | 1<<6 | 1<<7 | 1<<8 | 1<<9 | 1<<10;
+	coherence();
+}
+
+static void
+disable(Uart *uart)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	/* disable interrupt */
+	reg[IMSC] = 0;
+	coherence();
+
+	/* clear interrupt */
+	reg[ICR] = 1<<5 | 1<<6 | 1<<7 | 1<<8 | 1<<9 | 1<<10;
+	coherence();
+
+	/* wait for last transmission to complete */
+	while((reg[FR] & BUSY) != 0)
+		delay(1);
+
+	/* disable uart */
+	reg[CR] = 0;
+	coherence();
+
+	/* flush rx fifo */
+	reg[LCRH] &= ~FEN;
+	coherence();
+}
+
+static void
+uartoff(Uart *uart)
+{
+	u32int *reg = (u32int*)uart->regs;
+	u32int im;
+
+	im = reg[IMSC];
+	disable(uart);
+	reg[IMSC] = im;
+}
+
+static void
+uarton(Uart *uart)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	/* enable fifo */
+	reg[LCRH] |= FEN;
+	coherence();
+
+	/* enable uart */
+	reg[CR] = UARTEN | RXE | TXE;
+	coherence();
+}
+
+static void
+enable(Uart *uart, int ie)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	disable(uart);
+	if(ie){
+		intrenable(IRQuart, interrupt, uart, BUSUNKNOWN, uart->name);
+		reg[IMSC] = TXIM|RXIM;
+	}
+	uarton(uart);
+}
+
+static void
+linectl(Uart *uart, u32int set, u32int clr)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	if(uart->enabled)
+		uartoff(uart);
+
+	reg[LCRH] = set | (reg[LCRH] & ~clr);
+
+	if(uart->enabled)
+		uarton(uart);
+}
+
+static void
+kick(Uart *uart)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	if(uart->blocked)
+		return;
+	coherence();
+	while((reg[FR] & TXFF) == 0){
+		if(uart->op >= uart->oe && uartstageoutput(uart) == 0)
+			break;
+		reg[DR] = *(uart->op++);
+	}
+	coherence();
+}
+
+static void
+dobreak(Uart *uart, int ms)
+{
+	linectl(uart, BRK, 0);
+	delay(ms);
+	linectl(uart, 0, BRK);
+}
+
+static int
+baud(Uart *uart, int n)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	if(uart->freq <= 0 || n <= 0)
+		return -1;
+
+	if(uart->enabled)
+		uartoff(uart);
+
+	reg[IBRD] = (uart->freq >> 4) / n;
+	reg[FBRD] = (uart->freq >> 4) % n;
+
+	if(uart->enabled)
+		uarton(uart);
+
+	uart->baud = n;
+	return 0;
+}
+
+static int
+bits(Uart *uart, int n)
+{
+	switch(n){
+	case 8:
+		linectl(uart, WLEN8, WLENM);
+		break;
+	case 7:
+		linectl(uart, WLEN7, WLENM);
+		break;
+	case 6:
+		linectl(uart, WLEN6, WLENM);
+		break;
+	case 5:
+		linectl(uart, WLEN5, WLENM);
+		break;
+	default:
+		return -1;
+	}
+	uart->bits = n;
+	return 0;
+}
+
+static int
+stop(Uart *uart, int n)
+{
+	switch(n){
+	case 1:
+		linectl(uart, 0, STP2);
+		break;
+	case 2:
+		linectl(uart, STP2, 0);
+		break;
+	default:
+		return -1;
+	}
+	uart->stop = n;
+	return 0;
+}
+
+static int
+parity(Uart *uart, int n)
+{
+	switch(n){
+	case 'n':
+		linectl(uart, 0, PEN);
+		break;
+	case 'e':
+		linectl(uart, EPS|PEN, 0);
+		break;
+	case 'o':
+		linectl(uart, PEN, EPS);
+		break;
+	default:
+		return -1;
+	}
+	uart->parity = n;
+	return 0;
+}
+
+static void
+modemctl(Uart *uart, int on)
+{
+	uart->modem = on;
+}
+
+static void
+rts(Uart*, int)
+{
+}
+
+static long
+status(Uart *uart, void *buf, long n, long offset)
+{
+	char *p;
+
+	p = malloc(READSTR);
+	if(p == nil)
+		error(Enomem);
+	snprint(p, READSTR,
+		"b%d\n"
+		"dev(%d) type(%d) framing(%d) overruns(%d) "
+		"berr(%d) serr(%d)\n",
+
+		uart->baud,
+		uart->dev,
+		uart->type,
+		uart->ferr,
+		uart->oerr,
+		uart->berr,
+		uart->serr
+	);
+	n = readstr(offset, buf, n, p);
+	free(p);
+
+	return n;
+}
+
+static void
+donothing(Uart*, int)
+{
+}
+
+static void
+putc(Uart *uart, int c)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	while((reg[FR] & TXFF) != 0)
+		;
+	reg[DR] = c & 0xFF;
+}
+
+static int
+getc(Uart *uart)
+{
+	u32int *reg = (u32int*)uart->regs;
+
+	while((reg[FR] & RXFE) != 0)
+		;
+	return reg[DR] & 0xFF;
+}
+
+void
+uartconsinit(void)
+{
+	consuart = &qemuuart;
+	consuart->console = 1;
+	uartctl(consuart, "l8 pn s1");
+	uartputs(kmesg.buf, kmesg.n);
+}
+
+PhysUart qemuphysuart = {
+	.name		= "qemu",
+	.pnp		= pnp,
+	.enable		= enable,
+	.disable	= disable,
+	.kick		= kick,
+	.dobreak	= dobreak,
+	.baud		= baud,
+	.bits		= bits,
+	.stop		= stop,
+	.parity		= parity,
+	.modemctl	= donothing,
+	.rts		= rts,
+	.dtr		= donothing,
+	.status		= status,
+	.fifo		= donothing,
+	.getc		= getc,
+	.putc		= putc,
+};
--- a/sys/src/9/mkfile
+++ b/sys/src/9/mkfile
@@ -1,4 +1,5 @@
 ARCH=\
+	arm64\
 	bcm\
 	bcm64\
 	cycv\
--- a/sys/src/boot/mkfile
+++ b/sys/src/boot/mkfile
@@ -3,6 +3,7 @@
 	bitsy\
 	efi\
 	pc\
+	qemu\
 	reform\
 	zynq\
 
--- /dev/null
+++ b/sys/src/boot/qemu/boot.txt
@@ -1,0 +1,3 @@
+mw.b 0x40010000 0x0 0x10000
+load ${devtype} ${devnum}:${bootpart} 0x40010000 ${prefix}plan9.ini
+load ${devtype} ${devnum}:${bootpart} 0x40100000 ${prefix}9qemu.u && bootm 0x40100000
--- /dev/null
+++ b/sys/src/boot/qemu/mkfile
@@ -1,0 +1,9 @@
+FILES=boot.scr
+
+all:V:	$FILES
+
+clean:V:
+	rm -f $FILES
+
+boot.scr:	boot.txt
+	aux/txt2uimage -o $target $prereq