diff --git a/target/loongarch/cpu-csr.h b/target/loongarch/cpu-csr.h index 9097fddee1..6898947498 100644 --- a/target/loongarch/cpu-csr.h +++ b/target/loongarch/cpu-csr.h @@ -70,6 +70,8 @@ FIELD(TLBENTRY, PLV, 2, 2) FIELD(TLBENTRY, MAT, 4, 2) FIELD(TLBENTRY, G, 6, 1) FIELD(TLBENTRY, HUGE, 6, 1) +FIELD(TLBENTRY, P, 7, 1) +FIELD(TLBENTRY, W, 8, 1) FIELD(TLBENTRY, HGLOBAL, 12, 1) FIELD(TLBENTRY, LEVEL, 13, 2) FIELD(TLBENTRY_32, PPN, 8, 24) @@ -105,6 +107,8 @@ FIELD(CSR_PWCH, DIR3_BASE, 0, 6) FIELD(CSR_PWCH, DIR3_WIDTH, 6, 6) FIELD(CSR_PWCH, DIR4_BASE, 12, 6) FIELD(CSR_PWCH, DIR4_WIDTH, 18, 6) +FIELD(CSR_PWCH, HPTW_EN, 24, 1) +FIELD(CSR_PWCH, RESERVE, 25, 7) #define LOONGARCH_CSR_STLBPS 0x1e /* Stlb page size */ FIELD(CSR_STLBPS, PS, 0, 5) diff --git a/target/loongarch/cpu-mmu.h b/target/loongarch/cpu-mmu.h index dbc69c7c0f..2259de9d36 100644 --- a/target/loongarch/cpu-mmu.h +++ b/target/loongarch/cpu-mmu.h @@ -25,14 +25,76 @@ typedef struct MMUContext { hwaddr physical; int ps; /* page size shift */ int prot; + int tlb_index; + int mmu_index; + uint64_t pte_buddy[2]; } MMUContext; +static inline bool cpu_has_ptw(CPULoongArchState *env) +{ + return !!FIELD_EX64(env->CSR_PWCH, CSR_PWCH, HPTW_EN); +} + +static inline bool pte_present(CPULoongArchState *env, uint64_t entry) +{ + uint8_t present; + + if (cpu_has_ptw(env)) { + present = FIELD_EX64(entry, TLBENTRY, P); + } else { + present = FIELD_EX64(entry, TLBENTRY, V); + } + + return !!present; +} + +static inline bool pte_write(CPULoongArchState *env, uint64_t entry) +{ + uint8_t writable; + + if (cpu_has_ptw(env)) { + writable = FIELD_EX64(entry, TLBENTRY, W); + } else { + writable = FIELD_EX64(entry, TLBENTRY, D); + } + + return !!writable; +} + +/* + * The folloing functions should be called with PTW enable checked + * With hardware PTW enabled + * Bit D will be set by hardware with write access + * Bit A will be set by hardware with read/intruction fetch access + */ +static inline uint64_t pte_mkaccess(uint64_t entry) +{ + return FIELD_DP64(entry, TLBENTRY, V, 1); +} + +static inline uint64_t pte_mkdirty(uint64_t entry) +{ + return FIELD_DP64(entry, TLBENTRY, D, 1); +} + +static inline bool pte_access(uint64_t entry) +{ + return !!FIELD_EX64(entry, TLBENTRY, V); +} + +static inline bool pte_dirty(uint64_t entry) +{ + return !!FIELD_EX64(entry, TLBENTRY, D); +} + bool check_ps(CPULoongArchState *ent, uint8_t ps); TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, MMUAccessType access_type, int mmu_idx); TLBRet get_physical_address(CPULoongArchState *env, MMUContext *context, MMUAccessType access_type, int mmu_idx, int is_debug); +TLBRet loongarch_ptw(CPULoongArchState *env, MMUContext *context, + int access_type, int mmu_idx, int debug); void get_dir_base_width(CPULoongArchState *env, uint64_t *dir_base, uint64_t *dir_width, unsigned int level); hwaddr loongarch_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr); diff --git a/target/loongarch/cpu.c b/target/loongarch/cpu.c index 86490e0f72..e80a92fb2e 100644 --- a/target/loongarch/cpu.c +++ b/target/loongarch/cpu.c @@ -224,6 +224,25 @@ static void loongarch_set_msgint(Object *obj, bool value, Error **errp) cpu->env.cpucfg[1] = FIELD_DP32(cpu->env.cpucfg[1], CPUCFG1, MSG_INT, value); } +static bool loongarch_get_ptw(Object *obj, Error **errp) +{ + return LOONGARCH_CPU(obj)->ptw != ON_OFF_AUTO_OFF; +} + +static void loongarch_set_ptw(Object *obj, bool value, Error **errp) +{ + LoongArchCPU *cpu = LOONGARCH_CPU(obj); + + cpu->ptw = value ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF; + + if (kvm_enabled()) { + /* PTW feature is only support in TCG mode now */ + return; + } + + cpu->env.cpucfg[2] = FIELD_DP32(cpu->env.cpucfg[2], CPUCFG2, HPTW, value); +} + static void loongarch_cpu_post_init(Object *obj) { LoongArchCPU *cpu = LOONGARCH_CPU(obj); @@ -238,7 +257,10 @@ static void loongarch_cpu_post_init(Object *obj) loongarch_set_lasx); object_property_add_bool(obj, "msgint", loongarch_get_msgint, loongarch_set_msgint); + object_property_add_bool(obj, "ptw", loongarch_get_ptw, + loongarch_set_ptw); /* lbt is enabled only in kvm mode, not supported in tcg mode */ + if (kvm_enabled()) { kvm_loongarch_cpu_post_init(cpu); } @@ -346,6 +368,7 @@ static void loongarch_la464_initfn(Object *obj) env->CSR_PRCFG3 = FIELD_DP64(env->CSR_PRCFG3, CSR_PRCFG3, STLB_SETS, 8); cpu->msgint = ON_OFF_AUTO_OFF; + cpu->ptw = ON_OFF_AUTO_OFF; loongarch_la464_init_csr(obj); loongarch_cpu_post_init(obj); } @@ -377,6 +400,7 @@ static void loongarch_la132_initfn(Object *obj) data = FIELD_DP32(data, CPUCFG1, CRC, 1); env->cpucfg[1] = data; cpu->msgint = ON_OFF_AUTO_OFF; + cpu->ptw = ON_OFF_AUTO_OFF; } static void loongarch_max_initfn(Object *obj) @@ -388,6 +412,8 @@ static void loongarch_max_initfn(Object *obj) if (tcg_enabled()) { cpu->env.cpucfg[1] = FIELD_DP32(cpu->env.cpucfg[1], CPUCFG1, MSG_INT, 1); cpu->msgint = ON_OFF_AUTO_AUTO; + cpu->env.cpucfg[2] = FIELD_DP32(cpu->env.cpucfg[2], CPUCFG2, HPTW, 1); + cpu->ptw = ON_OFF_AUTO_AUTO; } } diff --git a/target/loongarch/cpu.h b/target/loongarch/cpu.h index b8e3b46c3a..b1d6799222 100644 --- a/target/loongarch/cpu.h +++ b/target/loongarch/cpu.h @@ -137,6 +137,7 @@ FIELD(CPUCFG2, LBT_MIPS, 20, 1) FIELD(CPUCFG2, LBT_ALL, 18, 3) FIELD(CPUCFG2, LSPW, 21, 1) FIELD(CPUCFG2, LAM, 22, 1) +FIELD(CPUCFG2, HPTW, 24, 1) /* cpucfg[3] bits */ FIELD(CPUCFG3, CCDMA, 0, 1) @@ -402,6 +403,7 @@ struct ArchCPU { uint32_t phy_id; OnOffAuto lbt; OnOffAuto pmu; + OnOffAuto ptw; OnOffAuto lsx; OnOffAuto lasx; OnOffAuto msgint; diff --git a/target/loongarch/cpu_helper.c b/target/loongarch/cpu_helper.c index 867e7c8867..a6eba4f416 100644 --- a/target/loongarch/cpu_helper.c +++ b/target/loongarch/cpu_helper.c @@ -49,12 +49,13 @@ TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, { uint64_t plv = mmu_idx; uint64_t tlb_entry, tlb_ppn; - uint8_t tlb_ps, tlb_v, tlb_d, tlb_plv, tlb_nx, tlb_nr, tlb_rplv; + uint8_t tlb_ps, tlb_plv, tlb_nx, tlb_nr, tlb_rplv; + bool tlb_v, tlb_d; tlb_entry = context->pte; tlb_ps = context->ps; - tlb_v = FIELD_EX64(tlb_entry, TLBENTRY, V); - tlb_d = FIELD_EX64(tlb_entry, TLBENTRY, D); + tlb_v = pte_present(env, tlb_entry); + tlb_d = pte_write(env, tlb_entry); tlb_plv = FIELD_EX64(tlb_entry, TLBENTRY, PLV); if (is_la64(env)) { tlb_ppn = FIELD_EX64(tlb_entry, TLBENTRY_64, PPN); @@ -96,6 +97,7 @@ TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, context->physical = (tlb_ppn << R_TLBENTRY_64_PPN_SHIFT) | (context->addr & MAKE_64BIT_MASK(0, tlb_ps)); context->prot = PAGE_READ; + context->mmu_index = tlb_plv; if (tlb_d) { context->prot |= PAGE_WRITE; } @@ -105,16 +107,52 @@ TLBRet loongarch_check_pte(CPULoongArchState *env, MMUContext *context, return TLBRET_MATCH; } -static TLBRet loongarch_page_table_walker(CPULoongArchState *env, - MMUContext *context, - int access_type, int mmu_idx) +static MemTxResult loongarch_cmpxchg_phys(CPUState *cs, hwaddr phys, + uint64_t old, uint64_t new) +{ + hwaddr addr1, l = 8; + MemoryRegion *mr; + uint8_t *ram_ptr; + uint64_t old1; + MemTxResult ret; + + rcu_read_lock(); + mr = address_space_translate(cs->as, phys, &addr1, &l, + false, MEMTXATTRS_UNSPECIFIED); + if (!memory_region_is_ram(mr)) { + /* + * Misconfigured PTE in ROM (AD bits are not preset) or + * PTE is in IO space and can't be updated atomically. + */ + rcu_read_unlock(); + return MEMTX_ACCESS_ERROR; + } + + ram_ptr = qemu_map_ram_ptr(mr->ram_block, addr1); + old1 = qatomic_cmpxchg((uint64_t *)ram_ptr, cpu_to_le64(old), + cpu_to_le64(new)); + old1 = le64_to_cpu(old1); + if (old1 == old) { + ret = MEMTX_OK; + } else { + ret = MEMTX_DECODE_ERROR; + } + rcu_read_unlock(); + + return ret; +} + +TLBRet loongarch_ptw(CPULoongArchState *env, MMUContext *context, + int access_type, int mmu_idx, int debug) { CPUState *cs = env_cpu(env); - target_ulong index, phys; + target_ulong index = 0, phys = 0; uint64_t dir_base, dir_width; - uint64_t base; + uint64_t base, pte; int level; vaddr address; + TLBRet ret; + MemTxResult ret1; address = context->addr; if ((address >> 63) & 0x1) { @@ -124,7 +162,7 @@ static TLBRet loongarch_page_table_walker(CPULoongArchState *env, } base &= TARGET_PHYS_MASK; - for (level = 4; level > 0; level--) { + for (level = 4; level >= 0; level--) { get_dir_base_width(env, &dir_base, &dir_width, level); if (dir_width == 0) { @@ -134,15 +172,24 @@ static TLBRet loongarch_page_table_walker(CPULoongArchState *env, /* get next level page directory */ index = (address >> dir_base) & ((1 << dir_width) - 1); phys = base | index << 3; - base = ldq_phys(cs->as, phys) & TARGET_PHYS_MASK; - if (FIELD_EX64(base, TLBENTRY, HUGE)) { - /* base is a huge pte */ - break; + base = ldq_phys(cs->as, phys); + if (level) { + if (FIELD_EX64(base, TLBENTRY, HUGE)) { + /* base is a huge pte */ + index = 0; + dir_base -= 1; + break; + } else { + /* Discard high bits with page directory table */ + base &= TARGET_PHYS_MASK; + } } } +restart: /* pte */ - if (FIELD_EX64(base, TLBENTRY, HUGE)) { + pte = base; + if (level > 0) { /* Huge Page. base is pte */ base = FIELD_DP64(base, TLBENTRY, LEVEL, 0); base = FIELD_DP64(base, TLBENTRY, HUGE, 0); @@ -150,17 +197,70 @@ static TLBRet loongarch_page_table_walker(CPULoongArchState *env, base = FIELD_DP64(base, TLBENTRY, HGLOBAL, 0); base = FIELD_DP64(base, TLBENTRY, G, 1); } - } else { - /* Normal Page. base points to pte */ - get_dir_base_width(env, &dir_base, &dir_width, 0); - index = (address >> dir_base) & ((1 << dir_width) - 1); - phys = base | index << 3; - base = ldq_phys(cs->as, phys); + + context->pte_buddy[index] = base; + context->pte_buddy[1 - index] = base + BIT_ULL(dir_base); + base += (BIT_ULL(dir_base) & address); + } else if (cpu_has_ptw(env)) { + index &= 1; + context->pte_buddy[index] = base; + context->pte_buddy[1 - index] = ldq_phys(cs->as, + phys + 8 * (1 - 2 * index)); } context->ps = dir_base; context->pte = base; - return loongarch_check_pte(env, context, access_type, mmu_idx); + ret = loongarch_check_pte(env, context, access_type, mmu_idx); + if (debug) { + return ret; + } + + /* + * Update bit A/D with hardware PTW supported + * + * Need atomic compchxg operation with pte update, other vCPUs may + * update pte at the same time. + */ + if (ret == TLBRET_MATCH && cpu_has_ptw(env)) { + if (access_type == MMU_DATA_STORE && pte_dirty(base)) { + return ret; + } + + if (access_type != MMU_DATA_STORE && pte_access(base)) { + return ret; + } + + base = pte_mkaccess(pte); + if (access_type == MMU_DATA_STORE) { + base = pte_mkdirty(base); + } + ret1 = loongarch_cmpxchg_phys(cs, phys, pte, base); + /* PTE updated by other CPU, reload PTE entry */ + if (ret1 == MEMTX_DECODE_ERROR) { + base = ldq_phys(cs->as, phys); + goto restart; + } + + base = context->pte_buddy[index]; + base = pte_mkaccess(base); + if (access_type == MMU_DATA_STORE) { + base = pte_mkdirty(base); + } + context->pte_buddy[index] = base; + + /* Bit A/D need be updated with both Even/Odd page with huge pte */ + if (level > 0) { + index = 1 - index; + base = context->pte_buddy[index]; + base = pte_mkaccess(base); + if (access_type == MMU_DATA_STORE) { + base = pte_mkdirty(base); + } + context->pte_buddy[index] = base; + } + } + + return ret; } static TLBRet loongarch_map_address(CPULoongArchState *env, @@ -183,7 +283,7 @@ static TLBRet loongarch_map_address(CPULoongArchState *env, * legal mapping, even if the mapping is not yet in TLB. return 0 if * there is a valid map, else none zero. */ - return loongarch_page_table_walker(env, context, access_type, mmu_idx); + return loongarch_ptw(env, context, access_type, mmu_idx, is_debug); } return TLBRET_NOMATCH; @@ -217,6 +317,7 @@ TLBRet get_physical_address(CPULoongArchState *env, MMUContext *context, if (da & !pg) { context->physical = address & TARGET_PHYS_MASK; context->prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + context->mmu_index = MMU_DA_IDX; return TLBRET_MATCH; } @@ -236,6 +337,7 @@ TLBRet get_physical_address(CPULoongArchState *env, MMUContext *context, if ((plv & env->CSR_DMW[i]) && (base_c == base_v)) { context->physical = dmw_va2pa(env, address, env->CSR_DMW[i]); context->prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + context->mmu_index = MMU_DA_IDX; return TLBRET_MATCH; } } diff --git a/target/loongarch/tcg/csr_helper.c b/target/loongarch/tcg/csr_helper.c index 5ebe15f993..c1a8ba3089 100644 --- a/target/loongarch/tcg/csr_helper.c +++ b/target/loongarch/tcg/csr_helper.c @@ -163,3 +163,18 @@ target_ulong helper_csrwr_pwcl(CPULoongArchState *env, target_ulong val) env->CSR_PWCL = val; return old_v; } + +target_ulong helper_csrwr_pwch(CPULoongArchState *env, target_ulong val) +{ + uint8_t has_ptw; + int64_t old_v = env->CSR_PWCH; + + val = FIELD_DP64(val, CSR_PWCH, RESERVE, 0); + has_ptw = FIELD_EX32(env->cpucfg[2], CPUCFG2, HPTW); + if (!has_ptw) { + val = FIELD_DP64(val, CSR_PWCH, HPTW_EN, 0); + } + + env->CSR_PWCH = val; + return old_v; + } diff --git a/target/loongarch/tcg/helper.h b/target/loongarch/tcg/helper.h index 7e508c5a7b..8a6c62f116 100644 --- a/target/loongarch/tcg/helper.h +++ b/target/loongarch/tcg/helper.h @@ -107,6 +107,7 @@ DEF_HELPER_2(csrwr_asid, i64, env, tl) DEF_HELPER_2(csrwr_tcfg, i64, env, tl) DEF_HELPER_2(csrwr_ticlr, i64, env, tl) DEF_HELPER_2(csrwr_pwcl, i64, env, tl) +DEF_HELPER_2(csrwr_pwch, i64, env, tl) DEF_HELPER_2(iocsrrd_b, i64, env, tl) DEF_HELPER_2(iocsrrd_h, i64, env, tl) DEF_HELPER_2(iocsrrd_w, i64, env, tl) diff --git a/target/loongarch/tcg/insn_trans/trans_privileged.c.inc b/target/loongarch/tcg/insn_trans/trans_privileged.c.inc index 64e53a4460..2094d182ac 100644 --- a/target/loongarch/tcg/insn_trans/trans_privileged.c.inc +++ b/target/loongarch/tcg/insn_trans/trans_privileged.c.inc @@ -79,6 +79,7 @@ void loongarch_csr_translate_init(void) SET_CSR_FUNC(ASID, NULL, gen_helper_csrwr_asid); SET_CSR_FUNC(PGD, gen_helper_csrrd_pgd, NULL); SET_CSR_FUNC(PWCL, NULL, gen_helper_csrwr_pwcl); + SET_CSR_FUNC(PWCH, NULL, gen_helper_csrwr_pwch); SET_CSR_FUNC(CPUID, gen_helper_csrrd_cpuid, NULL); SET_CSR_FUNC(TCFG, NULL, gen_helper_csrwr_tcfg); SET_CSR_FUNC(TVAL, gen_helper_csrrd_tval, NULL); diff --git a/target/loongarch/tcg/tlb_helper.c b/target/loongarch/tcg/tlb_helper.c index e119f78d92..01e0a27f0b 100644 --- a/target/loongarch/tcg/tlb_helper.c +++ b/target/loongarch/tcg/tlb_helper.c @@ -114,9 +114,8 @@ static void invalidate_tlb_entry(CPULoongArchState *env, int index) uint8_t tlb_ps; LoongArchTLB *tlb = &env->tlb[index]; int idxmap = BIT(MMU_KERNEL_IDX) | BIT(MMU_USER_IDX); - uint8_t tlb_v0 = FIELD_EX64(tlb->tlb_entry0, TLBENTRY, V); - uint8_t tlb_v1 = FIELD_EX64(tlb->tlb_entry1, TLBENTRY, V); uint64_t tlb_vppn = FIELD_EX64(tlb->tlb_misc, TLB_MISC, VPPN); + bool tlb_v; tlb_ps = FIELD_EX64(tlb->tlb_misc, TLB_MISC, PS); pagesize = MAKE_64BIT_MASK(tlb_ps, 1); @@ -124,12 +123,14 @@ static void invalidate_tlb_entry(CPULoongArchState *env, int index) addr = (tlb_vppn << R_TLB_MISC_VPPN_SHIFT) & ~mask; addr = sextract64(addr, 0, TARGET_VIRT_ADDR_SPACE_BITS); - if (tlb_v0) { + tlb_v = pte_present(env, tlb->tlb_entry0); + if (tlb_v) { tlb_flush_range_by_mmuidx(env_cpu(env), addr, pagesize, idxmap, TARGET_LONG_BITS); } - if (tlb_v1) { + tlb_v = pte_present(env, tlb->tlb_entry1); + if (tlb_v) { tlb_flush_range_by_mmuidx(env_cpu(env), addr + pagesize, pagesize, idxmap, TARGET_LONG_BITS); } @@ -158,10 +159,10 @@ static void invalidate_tlb(CPULoongArchState *env, int index) invalidate_tlb_entry(env, index); } -static void fill_tlb_entry(CPULoongArchState *env, LoongArchTLB *tlb) +/* Prepare tlb entry information in software PTW mode */ +static void sptw_prepare_context(CPULoongArchState *env, MMUContext *context) { uint64_t lo0, lo1, csr_vppn; - uint16_t csr_asid; uint8_t csr_ps; if (FIELD_EX64(env->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) { @@ -184,6 +185,24 @@ static void fill_tlb_entry(CPULoongArchState *env, LoongArchTLB *tlb) lo1 = env->CSR_TLBELO1; } + context->ps = csr_ps; + context->addr = csr_vppn << R_TLB_MISC_VPPN_SHIFT; + context->pte_buddy[0] = lo0; + context->pte_buddy[1] = lo1; +} + +static void fill_tlb_entry(CPULoongArchState *env, LoongArchTLB *tlb, + MMUContext *context) +{ + uint64_t lo0, lo1, csr_vppn; + uint16_t csr_asid; + uint8_t csr_ps; + + csr_vppn = context->addr >> R_TLB_MISC_VPPN_SHIFT; + csr_ps = context->ps; + lo0 = context->pte_buddy[0]; + lo1 = context->pte_buddy[1]; + /* Store page size in field PS */ tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, PS, csr_ps); tlb->tlb_misc = FIELD_DP64(tlb->tlb_misc, TLB_MISC, VPPN, csr_vppn); @@ -331,25 +350,19 @@ void helper_tlbrd(CPULoongArchState *env) } } -void helper_tlbwr(CPULoongArchState *env) +static void update_tlb_index(CPULoongArchState *env, MMUContext *context, + int index) { - int index = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, INDEX); LoongArchTLB *old, new = {}; - bool skip_inv = false; - uint8_t tlb_v0, tlb_v1; + bool skip_inv = false, tlb_v0, tlb_v1; old = env->tlb + index; - if (FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, NE)) { - invalidate_tlb(env, index); - return; - } - - fill_tlb_entry(env, &new); + fill_tlb_entry(env, &new, context); /* Check whether ASID/VPPN is the same */ if (old->tlb_misc == new.tlb_misc) { /* Check whether both even/odd pages is the same or invalid */ - tlb_v0 = FIELD_EX64(old->tlb_entry0, TLBENTRY, V); - tlb_v1 = FIELD_EX64(old->tlb_entry1, TLBENTRY, V); + tlb_v0 = pte_present(env, old->tlb_entry0); + tlb_v1 = pte_present(env, old->tlb_entry1); if ((!tlb_v0 || new.tlb_entry0 == old->tlb_entry0) && (!tlb_v1 || new.tlb_entry1 == old->tlb_entry1)) { skip_inv = true; @@ -364,31 +377,35 @@ void helper_tlbwr(CPULoongArchState *env) *old = new; } -void helper_tlbfill(CPULoongArchState *env) +void helper_tlbwr(CPULoongArchState *env) { - uint64_t address, entryhi; + int index = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, INDEX); + MMUContext context; + + if (FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, NE)) { + invalidate_tlb(env, index); + return; + } + + sptw_prepare_context(env, &context); + update_tlb_index(env, &context, index); +} + +static int get_tlb_random_index(CPULoongArchState *env, vaddr addr, + int pagesize) +{ + uint64_t address; int index, set, i, stlb_idx; - uint16_t pagesize, stlb_ps; - uint16_t asid, tlb_asid; + uint16_t asid, tlb_asid, stlb_ps; LoongArchTLB *tlb; uint8_t tlb_e, tlb_g; - if (FIELD_EX64(env->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) { - entryhi = env->CSR_TLBREHI; - /* Validity of pagesize is checked in helper_ldpte() */ - pagesize = FIELD_EX64(env->CSR_TLBREHI, CSR_TLBREHI, PS); - } else { - entryhi = env->CSR_TLBEHI; - /* Validity of pagesize is checked in helper_tlbrd() */ - pagesize = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, PS); - } - /* Validity of stlb_ps is checked in helper_csrwr_stlbps() */ stlb_ps = FIELD_EX64(env->CSR_STLBPS, CSR_STLBPS, PS); asid = FIELD_EX64(env->CSR_ASID, CSR_ASID, ASID); if (pagesize == stlb_ps) { /* Only write into STLB bits [47:13] */ - address = entryhi & ~MAKE_64BIT_MASK(0, R_CSR_TLBEHI_64_VPPN_SHIFT); + address = addr & ~MAKE_64BIT_MASK(0, R_CSR_TLBEHI_64_VPPN_SHIFT); set = -1; stlb_idx = (address >> (stlb_ps + 1)) & 0xff; /* [0,255] */ for (i = 0; i < 8; ++i) { @@ -435,8 +452,29 @@ void helper_tlbfill(CPULoongArchState *env) } } + return index; +} + +void helper_tlbfill(CPULoongArchState *env) +{ + vaddr entryhi; + int index, pagesize; + MMUContext context; + + if (FIELD_EX64(env->CSR_TLBRERA, CSR_TLBRERA, ISTLBR)) { + entryhi = env->CSR_TLBREHI; + /* Validity of pagesize is checked in helper_ldpte() */ + pagesize = FIELD_EX64(env->CSR_TLBREHI, CSR_TLBREHI, PS); + } else { + entryhi = env->CSR_TLBEHI; + /* Validity of pagesize is checked in helper_tlbrd() */ + pagesize = FIELD_EX64(env->CSR_TLBIDX, CSR_TLBIDX, PS); + } + + sptw_prepare_context(env, &context); + index = get_tlb_random_index(env, entryhi, pagesize); invalidate_tlb(env, index); - fill_tlb_entry(env, env->tlb + index); + fill_tlb_entry(env, env->tlb + index, &context); } void helper_tlbclr(CPULoongArchState *env) @@ -563,6 +601,18 @@ void helper_invtlb_page_asid_or_g(CPULoongArchState *env, } } +static void ptw_update_tlb(CPULoongArchState *env, MMUContext *context) +{ + int index; + + index = context->tlb_index; + if (index < 0) { + index = get_tlb_random_index(env, context->addr, context->ps); + } + + update_tlb_index(env, context, index); +} + bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size, MMUAccessType access_type, int mmu_idx, bool probe, uintptr_t retaddr) @@ -575,7 +625,45 @@ bool loongarch_cpu_tlb_fill(CPUState *cs, vaddr address, int size, /* Data access */ context.addr = address; + context.tlb_index = -1; ret = get_physical_address(env, &context, access_type, mmu_idx, 0); + if (ret == TLBRET_MATCH && context.mmu_index != MMU_DA_IDX + && cpu_has_ptw(env)) { + bool need_update = true; + + if (access_type == MMU_DATA_STORE && pte_dirty(context.pte)) { + need_update = false; + } else if (access_type != MMU_DATA_STORE && pte_access(context.pte)) { + need_update = false; + + /* + * FIXME: should context.prot be set without PAGE_WRITE with + * pte_write(context.pte) && !pte_dirty(context.pte)?? + * + * Otherwise there will be no loongarch_cpu_tlb_fill() function call + * for MMU_DATA_STORE access_type in future since QEMU TLB with + * prot PAGE_WRITE is added already + */ + } + + if (need_update) { + /* Need update bit A/D in PTE entry, take PTW again */ + ret = TLBRET_NOMATCH; + } + } + + if (ret != TLBRET_MATCH && cpu_has_ptw(env)) { + /* Take HW PTW if TLB missed or bit P is zero */ + if (ret == TLBRET_NOMATCH || ret == TLBRET_INVALID) { + ret = loongarch_ptw(env, &context, access_type, mmu_idx, 0); + if (ret == TLBRET_MATCH) { + ptw_update_tlb(env, &context); + } + } else if (context.tlb_index >= 0) { + invalidate_tlb(env, context.tlb_index); + } + } + if (ret == TLBRET_MATCH) { physical = context.physical; prot = context.prot; @@ -715,6 +803,7 @@ static TLBRet loongarch_map_tlb_entry(CPULoongArchState *env, n = (context->addr >> tlb_ps) & 0x1;/* Odd or even */ context->pte = n ? tlb->tlb_entry1 : tlb->tlb_entry0; context->ps = tlb_ps; + context->tlb_index = index; return loongarch_check_pte(env, context, access_type, mmu_idx); }