/* $Id: fault.c,v 1.2 2005/06/16 13:46:21 cgoh Exp $
 *
 * fault.c
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) Equator Technologies
 */
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/types.h>
#include <linux/smp_lock.h>

#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
#include <asm/current.h>
#include <asm/hardirq.h>
#include <asm/system.h>
#include <asm/mmu_context.h>
#include <asm/cache.h>

#include <asm/assert.h>
#include <asm/ptrace.h>

#include <libdev/etiTLB.h>
#include <libdev/etiRegisters.h>
#include <libdev/etiContext.h>
#include "context.h"

#include <kernelLib/etiDriverOsal.h>

static long nrofIExceptions, nrofDExceptions, nrofExceptions, nrofDoPageFaults;

/*  ------------------------------  Data Breakpoints API ------------------------------------- */

int        dataBrkGlobal;  // specified if the breakpoint is local to the current task or global

/* A data breakpoint is triggerred when (address & brk_mask) == ( brk_address  & brk_mask ) */

void setDataBreakpoint( void * brk_address , unsigned long brk_mask , int access , int global )
{
//       excSetDataBreakpoint(brk_address , brk_mask , access );
//       dataBrkGlobal  = global;        
}


void disableDataBreakpoint( void  )
{
//       dataBrkGlobal = 0;
//       excDisableDataBreakpoint();
}



/* ------------------------------------------------------------------------- */

#if 0
    static u32 checksumContext(ctxExceptionContext_t ctx)
    {
        Register pse  = ctx->pse;
        Register cpse0= ctx->cl[0].cpse;
        Register cpse1= ctx->cl[1].cpse;
        u32      type = ctx->header.u.s.type;
        u32      X    = 0;
        int      i;

        ctx->pse       = 0;
        ctx->cl[0].cpse= 0;
        ctx->cl[1].cpse= 0;
        ctx->header.u.s.type= 0;

        for ( i = 0; i < sizeof(*ctx); ++i ) {
	    X += ((u8*)ctx)[i];
        }

        ctx->pse       = pse;
        ctx->cl[0].cpse= cpse0;
        ctx->cl[1].cpse= cpse1;
        ctx->header.u.s.type= type;

        return X;
    }

    #define CHECK_START(ctx, X)	u32 X = checksumContext(ctx)
    #define CHECK_CONTEXT(ctx, X)	\
	    if ( X != checksumContext(ctx) ) {			\
	        printk("CTX mismatch! PC= %x\n", ctx->epcr0);	\
	    }
#else
    #define CHECK_START(a, b)
    #define CHECK_CONTEXT(a, b)
#endif

/* ------------------------------------------------------------------------------------------- */


static 
int do_page_fault(struct pt_regs *regs, 
		   unsigned long address,
		   unsigned long writeaccess,
                   unsigned long iscode
                  ) 
{

	struct task_struct*    tsk  = current;
	struct mm_struct*      mm   = tsk->mm;
	struct vm_area_struct* vma;
	unsigned long          epcr0= ((ctxExceptionContext_t) regs)->epcr0;
	unsigned long          fixup;
	int                    ret;
#if 1	// 28APR2008 cysong
	unsigned long *reg_sp;
	int i;
#endif	// 28APR2008 cysong

        nrofDoPageFaults++;

	/*
	 * If we're in an interrupt or have no user
	 * context, we must not take the fault..
	 */
	if (in_interrupt() || mm == &init_mm)
		goto no_context;

        
       	down(&mm->mmap_sem);
	vma = find_vma(mm, address);
        
	if (!vma)                            goto bad_area;
	if (vma->vm_start <= address)        goto good_area;
	if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area;
	if (expand_stack(vma, address))      goto bad_area;

 good_area:
	if (writeaccess) {
		if (!(vma->vm_flags & VM_WRITE))
			goto bad_area;
	} else {
		if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
			goto bad_area;
	}
        
	ret = handle_mm_fault(tsk, vma, address, writeaccess);
        if (!ret)
            goto do_sigbus;
        if (ret < 0) 
            goto out_of_memory;

	up(&mm->mmap_sem);
	return 1;
    


/*
 * Something tried to access memory that isn't in our memory map..
 * Fix it, but check if it's kernel or user first..
 */
bad_area:
	up(&mm->mmap_sem);
 
	printk( "\n\n\n\nIllegal %s access to 0x%x from PC 0x%x; psw=%x\n",
	      		writeaccess ? "write" : "read", address, epcr0,
			((ctxExceptionContext_t) regs)->header.u.s.psw
                 );
        printk("TID= %d (%s); ASID= %d\n", current->pid, tsk->comm, 
               current->mm->context.asid);
        printk("TLBS:\n"); tlbPrintITLB( ); tlbPrintDTLB( );
        ctxPrintContext(regs);

	if (user_mode(regs)) {
#if 1	// 28APR2008 cysong
		reg_sp = ((unsigned long *)&((ctxExceptionContext_t) regs)->cl[0])[17];
		printk( "-------------------------------------------------------\n");
		printk( "[SIGSEGV] pid:%d, pc:%08x, sp:%08x\n", tsk->pid, epcr0, reg_sp);
		printk( "-------------------------------------------------------\n");
		for (i=0; i<128; i++, reg_sp++)
			printk( "SR(%08x): %08x\n", reg_sp, *reg_sp);
		printk( "-------------------------------------------------------\n");
#endif	// 28APR2008 cysong
    		force_sig(SIGSEGV, tsk);
                        
		return 0;
	}
          
no_context:
  	/* Are we prepared to handle this kernel fault?  */
	fixup = search_exception_table(epcr0);
	if (fixup) {
		ctxExceptionContext_t ctx= regs;

		printk("%s: Exception at [0x%lx]\n", tsk->comm, epcr0);
		ctx->epcr0= fixup;
		ctx->epcr1= ctx->epcr2= 0;
		ctx->cl[0].mstat0= ctx->cl[0].mstat1= ctx->cl[0].mstat2 = 0;
		ctx->cl[1].mstat0= ctx->cl[1].mstat1= ctx->cl[1].mstat2 = 0;

		return 0;
	}

	/*
	 * Oops. The kernel tried to access some bad page. We'll have to
	 * terminate things with extreme prejudice.
	 */
	printk(KERN_ALERT "Unable to handle kernel paging request at virtual address %08lx\n",
	       address);
        
	printk("Oops\n");
        ctxPrintContext(regs);
	do_exit(SIGKILL);

 do_sigbus:
        force_sig(SIGBUS, current);
	goto bad_area;

 out_of_memory:
        printk("VM: killing process %s (out of memory)\n", tsk->comm);
        do_exit(SIGKILL);
        goto no_context;
}

    
/* ------------------------------------------------------------------------------------------- */

   

static void handle_cesr_illegal( u32 cesr, ctxExceptionContext_t ctx, int cl )
{
    u32 address = ctx->cl[cl].mvadr0, pc = ctx->epcr0;
    u32 psw     = ctx->header.u.s.psw;
    u32 signal;
#if 1	// 16may2009 cysong
	unsigned long *reg_sp;
	int i;
#endif	// 16may2009 cysong
    
    if ( cesr & bspregDALN ) {
        printk( "\nData alignment error cl%d:0x%x at PC 0x%x psw=%x\n", cl, address, pc, psw );
	printk( "mstat = { %x, %x, %x }\n", ctx->cl[cl].mstat0, ctx->cl[cl].mstat1, ctx->cl[cl].mstat2 );
	printk( "mvadr = { %x, %x, %x }\n", ctx->cl[cl].mvadr0, ctx->cl[cl].mvadr1, ctx->cl[cl].mvadr2 );
	printk( "pc    = { %x, %x, %x }\n", ctx->epcr0, ctx->epcr1, ctx->epcr2 );
        
        signal= SIGBUS;
    }

    if ( cesr & bspregPLV ) {
	int  index;

        printk( "\nPrivilege violation (cl%d) at PC 0x%x psw=%x\n", cl, pc, psw );
	index = tlbQueryITLB(pc);
	if ( index != tlbNUM_ITLB_ENTRIES )
	{
	    u32*   mem;
            u32    offset = pc & ~PAGE_MASK;
	    tlbEntry entry = tlbReadITLB(index);

#if 0	// 16may2009 cysong
	    mem = phys_to_virt(tlbPA(entry)) + offset;
	    printk( "0x%08x: %08x %08x %08x %08x\n", pc - 16, mem[-4], mem[-3], mem[-2], mem[-1] );
	    printk( "0x%08x: %08x %08x %08x %08x\n", pc, mem[0], mem[1], mem[2], mem[3] );
	    printk( "0x%08x: %08x %08x %08x %08x\n", pc + 16, mem[4], mem[5], mem[6], mem[7] );
#endif	// 16may2009 cysong
	}
        signal= SIGILL;
    }

    if ( cesr & bspregILLO ) {
	int  index;

        printk("\n Illegal operation (cl%d) at PC 0x%x psw=%x\n", cl, pc, psw );
	index = tlbQueryITLB(pc);
	if ( index != tlbNUM_ITLB_ENTRIES )
	{
	    u32*   mem;
            u32    offset = pc & ~PAGE_MASK;
	    tlbEntry entry = tlbReadITLB(index);

#if 0	// 16may2009 cysong
	    mem = (u32 *)(phys_to_virt(tlbPA(entry)) + offset);
	    printk( "0x%08x: %08x %08x %08x %08x\n", mem, mem[0], mem[1], mem[2], mem[3] );
#endif	// 16may2009 cysong
	}
        signal= SIGILL;
    }

    if ( cesr & bspregIEXZ ) {
        printk( "\nDivide by zero (cl%d) at PC 0x%x psw=%x\n", cl, pc, psw );
        signal= SIGFPE;
    }
    
    printk("TID= %d; ASID= %d; name=%s\n", 
           current->pid, current->mm->context.asid, current->comm);
    printk("TLBS:\n"); tlbPrintITLB( ); tlbPrintDTLB( );
    ctxPrintContext(ctx);
#if 1	// 16may2009 cysong
	reg_sp = ((unsigned long *)&((ctxExceptionContext_t) ctx)->cl[0])[17];
	printk( "-------------------------------------------------------\n");
	printk( "[SIGILL] pid:%d, pc:%08x, sp:%08x\n", current->pid, pc, reg_sp);
	printk( "-------------------------------------------------------\n");
	for (i=0; i<128; i++, reg_sp++)
		printk( "SR(%08x): %08x\n", reg_sp, *reg_sp);
	printk( "-------------------------------------------------------\n");
#endif	// 16may2009 cysong
    
    force_sig(signal, current);
}


static void handle_esr_illegal( u32 esr, ctxExceptionContext_t ctx )
{
    u32* sp = (u32*) ctx->cl[0].r16;
    int  i;
#if 1	// 16may2009 cysong
	unsigned long *reg_sp;
#endif	// 16may2009 cysong

    printk("Illegal PC:\n");
    printk("TID= %d; ASID= %d; name=%s\n", 
           current->pid, current->mm->context.asid, current->comm);
    printk("TLBS:\n"); tlbPrintITLB( ); tlbPrintDTLB( );
    printk("sp=%p, ret=%p\n", ctx->cl[0].r16, ctx->cl[0].r63);

    for ( i = 64; i > -8; i-- )
    {
        if ( ((u32) &sp[i] & 15) == 0 )
           printk( "\n%p:", &sp[i] );
        printk( " %08x", sp[i] );
    }
    printk("\n");
    {
	u32* pc    = (u32*) ctx->epcr0;
	int  index = tlbQueryITLB(pc);

	if ( index != tlbNUM_ITLB_ENTRIES )
	{
	    u32*     mem;
	    tlbEntry entry = tlbReadITLB(index);

#if 0	// 16may2009 cysong
	    mem = (u32*) ((u32) phys_to_virt(tlbPA(entry)) & ~255);
	    pc  = (u32*) ((u32) pc & ~255);

	    for ( i = 0; i < 64; i += 4 ) {
		printk( "%08x: %08x %08x %08x %08x\n", &pc[i], mem[i], mem[i+1], mem[i+2], mem[i+3] );
	    }
#endif	// 16may2009 cysong
	}
    }
    ctxPrintContext(ctx);
#if 1	// 16may2009 cysong
	reg_sp = ((unsigned long *)&((ctxExceptionContext_t) ctx)->cl[0])[17];
	printk( "-------------------------------------------------------\n");
	printk( "[SIGILL] pid:%d, pc:%08x, sp:%08x\n", current->pid, ctx->epcr0, reg_sp);
	printk( "-------------------------------------------------------\n");
	for (i=0; i<128; i++, reg_sp++)
		printk( "SR(%08x): %08x\n", reg_sp, *reg_sp);
	printk( "-------------------------------------------------------\n");
#endif	// 16may2009 cysong
    
    force_sig(SIGILL, current);
}

    
/* -------------------------------------------------------------------------- */

/* 
 * return_address_present: atomically check if all possible addresses in
 * the BSP's instruction fetch pipeline are mapped in the TLB. This is a
 * precondition for properly returning from the current exception: if any
 * of these addresses are not mapped, then the BSP will not generate an
 * exception again, but will dive into an inconsistent cpu state.
 * This check must be atomic, because otherwise an intervening interrupt
 * might kick the carefully constructed ITLB state out again. 
 * Therefore, it is called by the exception postprocessor (traps.c) with all 
 * exceptions disabled. In case we need to enable these exceptions, 
 * Failure is returned, so that a retry is attempted.
 * Because of the pagelocking on involved instruction pages (see below),
 * this cannot indefinitely fail.
 */

    /*
     * The core prefetches new instructions, starting at offset 0xf0 in the
     * current instruction block.
     * Thus, if we handle an instruction address at offsets 0xf0 and up, we
     * have to make sure that both the current page and the next page are in
     * the ITLB; otherwise, the rfe will result in a hang.
     */
    #define STRADDLING_THRESHOLD 0x10

    static Bool isVirtualInstruction( u32 address )
    {
        return address < PGOFFS_KERNELSPACE_START
             || etiINRANGE( address, PGOFFS_VMALLOC_START, PGOFFS_VMALLOC_LEN );
    }

    static void obtainReturnAddress( u32 address, u32 *addresses, u32 *nrAddresses )
    {
        if (isVirtualInstruction(address)) {
            addresses[ (*nrAddresses)++ ]= address;

            if ( (address & (PAGE_SIZE-1)) >= PAGE_SIZE - STRADDLING_THRESHOLD ) {
                addresses[ (*nrAddresses)++ ]= PAGE_ALIGN(address);
            }
        }
    }

    static int handle_instruction_address( u32 address, ctxExceptionContext_t ctx )
    {
        u32 result, exceptions;

        result= tlbQueryITLB(address);
        if (result < tlbNUM_ITLB_ENTRIES) { return result; }

        result= TLB_update_itlb(address);
        if (result < tlbNUM_ITLB_ENTRIES) { return result; }

        bspregEnableExceptions  (exceptions);
        do_page_fault(ctx, address, 0,1);
        bspregRestoreExceptions (exceptions);

        return tlbNUM_ITLB_ENTRIES;
    }        

    static void lock_page( u32 address, int increment ) 
    {
        u32 map_nr= MAP_NR(address);

        atomic_add( increment, &(mem_map[map_nr].count) );
    }

Bool return_address_present( ctxExceptionContext_t ctx )
{
    u32  i;
    u32  addresses[6];
    u32  indices  [6];
    u32  nrAddresses=0;
    Bool result;

   /*
    * Gather all addresses that are required for
    * a safe return from exception:
    */
    obtainReturnAddress( ctx->epcr0, addresses, &nrAddresses );

    if (ctx->epcr1 & 0x1) { obtainReturnAddress( ctx->epcr1, addresses, &nrAddresses ); }
    if (ctx->epcr2 & 0x1) { obtainReturnAddress( ctx->epcr2, addresses, &nrAddresses ); }


   /*
    * Handle possible quick exits...
    */
    if (nrAddresses == 0) { 
        return True; 
    }

    if (nrAddresses == 1) { 
        result= handle_instruction_address(addresses[0],ctx);
        return result;
    }

    /* 
     * Phase 1: get all involved pages into memory, 
     * while locking them to prevent them from being 
     * swapped out again while fetching the following
     * pages. Entries will be set in the TLB as a side-
     * effect of handle_instruction_address, but these
     * are ignored in this phase; they will be structurally
     * set for all obtained return address pages in phase 3:
     */
    i=0;
    while (i<nrAddresses) {
        u32 index  = handle_instruction_address(addresses[i],ctx);

        if (index == tlbNUM_ITLB_ENTRIES) { 
           /*
            * memory access error; abort here, since
            * the current process will die anyway because
            * of handling the kill signal when returning
            * from this function. Note that handle_instruction_address,
            * will never return if the memory access error happened
            * in a kernel mode context: that situation will result
            * in a kernel oops, causing the kernel to abort.
            */ 
            break; 
        }

        lock_page(addresses[i], 1);
        i++;
    }

    result= (i == nrAddresses);

    /* Phase 2: Now we have them all in memory, unlock the pages */
    while (i-- > 0) {
        lock_page(addresses[i], -1);
    }

    if (result) {
        /* 
         * Phase 3: Fetch all entries simultaneously into the ITLB, 
         * while locking the used TLB entries. Now all involved pages
         * are in memory, this is an atomic operation.
         * KERNEL 2.4 alert!
         */
        for (i=0; i<nrAddresses; i++) { 
            indices[i]= handle_instruction_address(addresses[i],ctx);
            TLB_lock_itlb_entry( indices[i] );
        }
        /* Phase 4: Now we have them all, unlock the entries */
        for (i=0; i<nrAddresses; i++) { 
            TLB_unlock_itlb_entry( indices[i] );
        }
    }

    return result;
}


static inline void handle_data_address( u32 mstat, u32 address, ctxExceptionContext_t ctx )
{
    u32 memtype    = mstat >> 28; /* JVDS MAGIC CONSTANT ALERT! */
    u32 iswrite    = (memtype&2);
    u32 isread     = (memtype&1);
    u32 isuncached = (memtype&4);
    u32 isweird0   = (isread | iswrite) == 3;
    u32 isweird1   = (isread | iswrite) == 0;
    u32 isvalid    = (memtype&8) && !isweird0 && !isweird1;
    u32 usermode   = user_mode(ctx);

    if (isvalid) {
        if (TLB_update_dtlb(address, iswrite, usermode) == tlbNUM_DTLB_ENTRIES) {
            if (do_page_fault(ctx, address, iswrite,0))
                TLB_update_dtlb(address, iswrite, usermode);
        }
    }
}


/* ------------------------------------------------------------------------- */

static inline void raise_breakpoint(ctxExceptionContext_t ctx) {
    CHECK_START(ctx, XXX);
    nrofExceptions++;

    CHECK_STACK("G");
    force_sig(SIGTRAP, current);
    CHECK_STACK("g");

    CHECK_CONTEXT(ctx, XXX);
}

void ALL_BREAKPOINT_EXCEPTIONS_handler(ctxExceptionContext_t ctx, UInt esr) 
{
    raise_breakpoint(ctx);
}

void CL_BREAKPOINT_EXCEPTIONS_handler(ctxExceptionContext_t ctx, UInt esr,
                                      UInt cluster) 
{
    raise_breakpoint(ctx);
}

void ILPC_handler(ctxExceptionContext_t ctx, UInt esr) 
{
    struct task_struct *tsk = current;

    CHECK_START(ctx, XXX);
    nrofExceptions++;

    CHECK_STACK("G");
    if (ctx->epcr0 == SIGRETURN_TRAP_ADDRESS && user_mode(ctx) ) { 
       /* This situation indicates a return from a signal handler */
       do_sigreturn(ctx);
    } else if ((tsk->tss.debug_flags & TSS_DEBUG_SAVED_PC_VALID) &&
               (ctx->epcr0 == (current->tss.saved_pc & ~0xff))) {
        /* This is a single step */
        ctx->epcr0 = tsk->tss.saved_pc;
        tsk->tss.debug_flags &= ~TSS_DEBUG_SAVED_PC_VALID;
        force_sig(SIGTRAP, current);
    } else {
        handle_esr_illegal( esr, ctx );
    }
    CHECK_STACK("g");

    CHECK_CONTEXT(ctx, XXX);
}
        

void ALL_ILLEGAL_EXCEPTIONS_handler(ctxExceptionContext_t ctx, UInt cesr, UInt cluster) 
{
    CHECK_START(ctx, XXX);
    nrofExceptions++;

    CHECK_STACK("G");
    handle_cesr_illegal( cesr, ctx, cluster );
    CHECK_STACK("g");

    CHECK_CONTEXT(ctx, XXX);
}


void ALL_ITLB_EXCEPTIONS_handler(ctxExceptionContext_t ctx, UInt esr) 
{
    CHECK_START(ctx, XXX);

    nrofIExceptions++;

    CHECK_STACK("I");
/*  All work performed in exception postprocessing, calling return_address_present */
    CHECK_STACK("i");

    CHECK_CONTEXT(ctx, XXX);
}


void ALL_DTLB_EXCEPTIONS_handler(ctxExceptionContext_t ctx, UInt cesr, UInt cluster) 
{
    CHECK_START(ctx, XXX);

    nrofDExceptions++;

    CHECK_STACK("D");
    handle_data_address( ctx->cl[cluster].mstat0, ctx->cl[cluster].mvadr0, ctx );
    CHECK_STACK("d");

    CHECK_CONTEXT(ctx, XXX);
}


/* -------------------------------------------------------------------------- */


#include <linux/proc_fs.h>
#include <stdarg.h>


static int read_tlb_procmem(char *buf, char **start, off_t offset, int len, int unused)
{
    len = sprintf (buf, "exceptions (itlb/dtlb/other): %u/%u/%u\n"
	                "page faults:                  %u\n",
			nrofIExceptions, nrofDExceptions, nrofExceptions,
			nrofDoPageFaults);   

    return len;
}


static struct proc_dir_entry proc_entry = {
        0,
        7, "tlbinfo",
        S_IFREG | S_IRUGO,
        1,0,0,
        0,
        NULL,
        &read_tlb_procmem,
    };

static UInt proc_tlb_register()
{
    proc_register( &proc_root, &proc_entry);
    return 0;
}


/*
 * Add it to etiDriverOsal's autoinit list:
 */
drosDefineDriverInit(proc_tlb_register)


