diff --git a/exec-obsolete.h b/exec-obsolete.h index 94c23d095..4dbe4768a 100644 --- a/exec-obsolete.h +++ b/exec-obsolete.h @@ -37,7 +37,7 @@ void cpu_unregister_io_memory(int table_address); struct MemoryRegionSection; void cpu_register_physical_memory_log(struct MemoryRegionSection *section, - bool readable, bool readonly); + bool readonly); void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size); void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size); @@ -121,6 +121,9 @@ static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start, void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end, int dirty_flags); + +extern const IORangeOps memory_region_iorange_ops; + #endif #endif diff --git a/exec.c b/exec.c index b81677ade..0c93b2691 100644 --- a/exec.c +++ b/exec.c @@ -160,29 +160,21 @@ typedef struct PageDesc { #define L2_BITS 10 #define L2_SIZE (1 << L2_BITS) +#define P_L2_LEVELS \ + (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1) + /* The bits remaining after N lower levels of page tables. */ -#define P_L1_BITS_REM \ - ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS) #define V_L1_BITS_REM \ ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS) -/* Size of the L1 page table. Avoid silly small sizes. */ -#if P_L1_BITS_REM < 4 -#define P_L1_BITS (P_L1_BITS_REM + L2_BITS) -#else -#define P_L1_BITS P_L1_BITS_REM -#endif - #if V_L1_BITS_REM < 4 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS) #else #define V_L1_BITS V_L1_BITS_REM #endif -#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS) #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS) -#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS) #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS) unsigned long qemu_real_host_page_size; @@ -194,15 +186,27 @@ unsigned long qemu_host_page_mask; static void *l1_map[V_L1_SIZE]; #if !defined(CONFIG_USER_ONLY) -typedef struct PhysPageDesc { - /* offset in host memory of the page + io_index in the low bits */ - ram_addr_t phys_offset; - ram_addr_t region_offset; -} PhysPageDesc; +typedef struct PhysPageEntry PhysPageEntry; + +static MemoryRegionSection *phys_sections; +static unsigned phys_sections_nb, phys_sections_nb_alloc; +static uint16_t phys_section_unassigned; + +struct PhysPageEntry { + uint16_t is_leaf : 1; + /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */ + uint16_t ptr : 15; +}; + +/* Simple allocator for PhysPageEntry nodes */ +static PhysPageEntry (*phys_map_nodes)[L2_SIZE]; +static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc; + +#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1) /* This is a multi-level map on the physical address space. - The bottom level has pointers to PhysPageDesc. */ -static void *l1_phys_map[P_L1_SIZE]; + The bottom level has pointers to MemoryRegionSections. */ +static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 }; static void io_mem_init(void); static void memory_map_init(void); @@ -398,59 +402,112 @@ static inline PageDesc *page_find(tb_page_addr_t index) } #if !defined(CONFIG_USER_ONLY) -static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) + +static void phys_map_node_reserve(unsigned nodes) { - PhysPageDesc *pd; - void **lp; - int i; - - /* Level 1. Always allocated. */ - lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1)); - - /* Level 2..N-1. */ - for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) { - void **p = *lp; - if (p == NULL) { - if (!alloc) { - return NULL; - } - *lp = p = g_malloc0(sizeof(void *) * L2_SIZE); - } - lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1)); + if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) { + typedef PhysPageEntry Node[L2_SIZE]; + phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16); + phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc, + phys_map_nodes_nb + nodes); + phys_map_nodes = g_renew(Node, phys_map_nodes, + phys_map_nodes_nb_alloc); } - - pd = *lp; - if (pd == NULL) { - int i; - int first_index = index & ~(L2_SIZE - 1); - - if (!alloc) { - return NULL; - } - - *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE); - - for (i = 0; i < L2_SIZE; i++) { - pd[i].phys_offset = io_mem_unassigned.ram_addr; - pd[i].region_offset = (first_index + i) << TARGET_PAGE_BITS; - } - } - - return pd + (index & (L2_SIZE - 1)); } -static inline PhysPageDesc phys_page_find(target_phys_addr_t index) +static uint16_t phys_map_node_alloc(void) { - PhysPageDesc *p = phys_page_find_alloc(index, 0); + unsigned i; + uint16_t ret; - if (p) { - return *p; - } else { - return (PhysPageDesc) { - .phys_offset = io_mem_unassigned.ram_addr, - .region_offset = index << TARGET_PAGE_BITS, - }; + ret = phys_map_nodes_nb++; + assert(ret != PHYS_MAP_NODE_NIL); + assert(ret != phys_map_nodes_nb_alloc); + for (i = 0; i < L2_SIZE; ++i) { + phys_map_nodes[ret][i].is_leaf = 0; + phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL; } + return ret; +} + +static void phys_map_nodes_reset(void) +{ + phys_map_nodes_nb = 0; +} + + +static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, + target_phys_addr_t *nb, uint16_t leaf, + int level) +{ + PhysPageEntry *p; + int i; + target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS); + + if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) { + lp->ptr = phys_map_node_alloc(); + p = phys_map_nodes[lp->ptr]; + if (level == 0) { + for (i = 0; i < L2_SIZE; i++) { + p[i].is_leaf = 1; + p[i].ptr = phys_section_unassigned; + } + } + } else { + p = phys_map_nodes[lp->ptr]; + } + lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)]; + + while (*nb && lp < &p[L2_SIZE]) { + if ((*index & (step - 1)) == 0 && *nb >= step) { + lp->is_leaf = true; + lp->ptr = leaf; + *index += step; + *nb -= step; + } else { + phys_page_set_level(lp, index, nb, leaf, level - 1); + } + ++lp; + } +} + +static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb, + uint16_t leaf) +{ + /* Wildly overreserve - it doesn't matter much. */ + phys_map_node_reserve(3 * P_L2_LEVELS); + + phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1); +} + +static MemoryRegionSection phys_page_find(target_phys_addr_t index) +{ + PhysPageEntry lp = phys_map; + PhysPageEntry *p; + int i; + MemoryRegionSection section; + target_phys_addr_t delta; + uint16_t s_index = phys_section_unassigned; + + for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) { + if (lp.ptr == PHYS_MAP_NODE_NIL) { + goto not_found; + } + p = phys_map_nodes[lp.ptr]; + lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; + } + + s_index = lp.ptr; +not_found: + section = phys_sections[s_index]; + index <<= TARGET_PAGE_BITS; + assert(section.offset_within_address_space <= index + && index <= section.offset_within_address_space + section.size-1); + delta = index - section.offset_within_address_space; + section.offset_within_address_space += delta; + section.offset_within_region += delta; + section.size -= delta; + return section; } static void tlb_protect_code(ram_addr_t ram_addr); @@ -1410,14 +1467,18 @@ static void breakpoint_invalidate(CPUState *env, target_ulong pc) static void breakpoint_invalidate(CPUState *env, target_ulong pc) { target_phys_addr_t addr; - target_ulong pd; ram_addr_t ram_addr; - PhysPageDesc p; + MemoryRegionSection section; addr = cpu_get_phys_page_debug(env, pc); - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; - ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK); + section = phys_page_find(addr >> TARGET_PAGE_BITS); + if (!(memory_region_is_ram(section.mr) + || (section.mr->rom_device && section.mr->readable))) { + return; + } + ram_addr = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) & TARGET_PAGE_MASK; + ram_addr |= (pc & ~TARGET_PAGE_MASK); tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); } #endif @@ -2095,24 +2156,21 @@ static void tlb_add_large_page(CPUState *env, target_ulong vaddr, env->tlb_flush_mask = mask; } -static bool is_ram_rom(ram_addr_t pd) +static bool is_ram_rom(MemoryRegionSection *s) { - pd &= ~TARGET_PAGE_MASK; - return pd == io_mem_ram.ram_addr || pd == io_mem_rom.ram_addr; + return memory_region_is_ram(s->mr); } -static bool is_romd(ram_addr_t pd) +static bool is_romd(MemoryRegionSection *s) { - MemoryRegion *mr; + MemoryRegion *mr = s->mr; - pd &= ~TARGET_PAGE_MASK; - mr = io_mem_region[pd]; return mr->rom_device && mr->readable; } -static bool is_ram_rom_romd(ram_addr_t pd) +static bool is_ram_rom_romd(MemoryRegionSection *s) { - return is_ram_rom(pd) || is_romd(pd); + return is_ram_rom(s) || is_romd(s); } /* Add a new TLB entry. At most one entry for a given virtual address @@ -2122,8 +2180,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, target_phys_addr_t paddr, int prot, int mmu_idx, target_ulong size) { - PhysPageDesc p; - unsigned long pd; + MemoryRegionSection section; unsigned int index; target_ulong address; target_ulong code_address; @@ -2136,8 +2193,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, if (size != TARGET_PAGE_SIZE) { tlb_add_large_page(env, vaddr, size); } - p = phys_page_find(paddr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(paddr >> TARGET_PAGE_BITS); #if defined(DEBUG_TLB) printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx " prot=%x idx=%d pd=0x%08lx\n", @@ -2145,15 +2201,21 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, #endif address = vaddr; - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* IO memory case (romd handled later) */ address |= TLB_MMIO; } - addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK); - if (is_ram_rom(pd)) { + if (is_ram_rom_romd(§ion)) { + addend = (unsigned long)(memory_region_get_ram_ptr(section.mr) + + section.offset_within_region); + } else { + addend = 0; + } + if (is_ram_rom(§ion)) { /* Normal RAM. */ - iotlb = pd & TARGET_PAGE_MASK; - if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) + iotlb = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) & TARGET_PAGE_MASK; + if (!section.readonly) iotlb |= io_mem_notdirty.ram_addr; else iotlb |= io_mem_rom.ram_addr; @@ -2164,8 +2226,8 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, and avoid full address decoding in every device. We can't use the high bits of pd for this because IO_MEM_ROMD uses these as a ram address. */ - iotlb = (pd & ~TARGET_PAGE_MASK); - iotlb += p.region_offset; + iotlb = memory_region_get_ram_addr(section.mr) & ~TARGET_PAGE_MASK; + iotlb += section.offset_within_region; } code_address = address; @@ -2198,11 +2260,14 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, te->addr_code = -1; } if (prot & PAGE_WRITE) { - if ((pd & ~TARGET_PAGE_MASK) == io_mem_rom.ram_addr || is_romd(pd)) { + if ((memory_region_is_ram(section.mr) && section.readonly) + || is_romd(§ion)) { /* Write access calls the I/O callback. */ te->addr_write = address | TLB_MMIO; - } else if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr && - !cpu_physical_memory_is_dirty(pd)) { + } else if (memory_region_is_ram(section.mr) + && !cpu_physical_memory_is_dirty( + section.mr->ram_addr + + section.offset_within_region)) { te->addr_write = address | TLB_NOTDIRTY; } else { te->addr_write = address; @@ -2491,34 +2556,66 @@ static inline void tlb_set_dirty(CPUState *env, typedef struct subpage_t { MemoryRegion iomem; target_phys_addr_t base; - ram_addr_t sub_io_index[TARGET_PAGE_SIZE]; - ram_addr_t region_offset[TARGET_PAGE_SIZE]; + uint16_t sub_section[TARGET_PAGE_SIZE]; } subpage_t; static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, - ram_addr_t memory, ram_addr_t region_offset); -static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, - ram_addr_t orig_memory, - ram_addr_t region_offset); -#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \ - need_subpage) \ - do { \ - if (addr > start_addr) \ - start_addr2 = 0; \ - else { \ - start_addr2 = start_addr & ~TARGET_PAGE_MASK; \ - if (start_addr2 > 0) \ - need_subpage = 1; \ - } \ - \ - if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \ - end_addr2 = TARGET_PAGE_SIZE - 1; \ - else { \ - end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \ - if (end_addr2 < TARGET_PAGE_SIZE - 1) \ - need_subpage = 1; \ - } \ - } while (0) + uint16_t section); +static subpage_t *subpage_init(target_phys_addr_t base); +static void destroy_page_desc(uint16_t section_index) +{ + MemoryRegionSection *section = &phys_sections[section_index]; + MemoryRegion *mr = section->mr; + + if (mr->subpage) { + subpage_t *subpage = container_of(mr, subpage_t, iomem); + memory_region_destroy(&subpage->iomem); + g_free(subpage); + } +} + +static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) +{ + unsigned i; + PhysPageEntry *p; + + if (lp->ptr == PHYS_MAP_NODE_NIL) { + return; + } + + p = phys_map_nodes[lp->ptr]; + for (i = 0; i < L2_SIZE; ++i) { + if (!p[i].is_leaf) { + destroy_l2_mapping(&p[i], level - 1); + } else { + destroy_page_desc(p[i].ptr); + } + } + lp->is_leaf = 0; + lp->ptr = PHYS_MAP_NODE_NIL; +} + +static void destroy_all_mappings(void) +{ + destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1); + phys_map_nodes_reset(); +} + +static uint16_t phys_section_add(MemoryRegionSection *section) +{ + if (phys_sections_nb == phys_sections_nb_alloc) { + phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16); + phys_sections = g_renew(MemoryRegionSection, phys_sections, + phys_sections_nb_alloc); + } + phys_sections[phys_sections_nb] = *section; + return phys_sections_nb++; +} + +static void phys_sections_clear(void) +{ + phys_sections_nb = 0; +} /* register physical memory. For RAM, 'size' must be a multiple of the target page size. @@ -2528,101 +2625,78 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, start_addr and region_offset are rounded down to a page boundary before calculating this offset. This should not be a problem unless the low bits of start_addr and region_offset differ. */ -void cpu_register_physical_memory_log(MemoryRegionSection *section, - bool readable, bool readonly) +static void register_subpage(MemoryRegionSection *section) +{ + subpage_t *subpage; + target_phys_addr_t base = section->offset_within_address_space + & TARGET_PAGE_MASK; + MemoryRegionSection existing = phys_page_find(base >> TARGET_PAGE_BITS); + MemoryRegionSection subsection = { + .offset_within_address_space = base, + .size = TARGET_PAGE_SIZE, + }; + target_phys_addr_t start, end; + + assert(existing.mr->subpage || existing.mr == &io_mem_unassigned); + + if (!(existing.mr->subpage)) { + subpage = subpage_init(base); + subsection.mr = &subpage->iomem; + phys_page_set(base >> TARGET_PAGE_BITS, 1, + phys_section_add(&subsection)); + } else { + subpage = container_of(existing.mr, subpage_t, iomem); + } + start = section->offset_within_address_space & ~TARGET_PAGE_MASK; + end = start + section->size; + subpage_register(subpage, start, end, phys_section_add(section)); +} + + +static void register_multipage(MemoryRegionSection *section) { target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; - ram_addr_t phys_offset = section->mr->ram_addr; - ram_addr_t region_offset = section->offset_within_region; - target_phys_addr_t addr, end_addr; - PhysPageDesc *p; - CPUState *env; - ram_addr_t orig_size = size; - subpage_t *subpage; - - if (memory_region_is_ram(section->mr)) { - phys_offset += region_offset; - region_offset = 0; - } - - if (readonly) { - phys_offset |= io_mem_rom.ram_addr; - } + target_phys_addr_t addr; + uint16_t section_index = phys_section_add(section); assert(size); - if (phys_offset == io_mem_unassigned.ram_addr) { - region_offset = start_addr; - } - region_offset &= TARGET_PAGE_MASK; - size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK; - end_addr = start_addr + (target_phys_addr_t)size; - addr = start_addr; - do { - p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 0); - if (p && p->phys_offset != io_mem_unassigned.ram_addr) { - ram_addr_t orig_memory = p->phys_offset; - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - MemoryRegion *mr = io_mem_region[orig_memory & ~TARGET_PAGE_MASK]; + phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS, + section_index); +} - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, - need_subpage); - if (need_subpage) { - if (!(mr->subpage)) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - &p->phys_offset, orig_memory, - p->region_offset); - } else { - subpage = container_of(mr, subpage_t, iomem); - } - subpage_register(subpage, start_addr2, end_addr2, phys_offset, - region_offset); - p->region_offset = 0; - } else { - p->phys_offset = phys_offset; - p->region_offset = region_offset; - if (is_ram_rom_romd(phys_offset)) - phys_offset += TARGET_PAGE_SIZE; - } - } else { - p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); - p->phys_offset = phys_offset; - p->region_offset = region_offset; - if (is_ram_rom_romd(phys_offset)) { - phys_offset += TARGET_PAGE_SIZE; - } else { - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; +void cpu_register_physical_memory_log(MemoryRegionSection *section, + bool readonly) +{ + MemoryRegionSection now = *section, remain = *section; - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, - end_addr2, need_subpage); - - if (need_subpage) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - &p->phys_offset, - io_mem_unassigned.ram_addr, - addr & TARGET_PAGE_MASK); - subpage_register(subpage, start_addr2, end_addr2, - phys_offset, region_offset); - p->region_offset = 0; - } - } - } - region_offset += TARGET_PAGE_SIZE; - addr += TARGET_PAGE_SIZE; - } while (addr != end_addr); - - /* since each CPU stores ram addresses in its TLB cache, we must - reset the modified entries */ - /* XXX: slow ! */ - for(env = first_cpu; env != NULL; env = env->next_cpu) { - tlb_flush(env, 1); + if ((now.offset_within_address_space & ~TARGET_PAGE_MASK) + || (now.size < TARGET_PAGE_SIZE)) { + now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space) + - now.offset_within_address_space, + now.size); + register_subpage(&now); + remain.size -= now.size; + remain.offset_within_address_space += now.size; + remain.offset_within_region += now.size; + } + now = remain; + now.size &= TARGET_PAGE_MASK; + if (now.size) { + register_multipage(&now); + remain.size -= now.size; + remain.offset_within_address_space += now.size; + remain.offset_within_region += now.size; + } + now = remain; + if (now.size) { + register_subpage(&now); } } + void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size) { if (kvm_enabled()) @@ -3307,14 +3381,17 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr, { subpage_t *mmio = opaque; unsigned int idx = SUBPAGE_IDX(addr); + MemoryRegionSection *section; #if defined(DEBUG_SUBPAGE) printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__, mmio, len, addr, idx); #endif - addr += mmio->region_offset[idx]; - idx = mmio->sub_io_index[idx]; - return io_mem_read(idx, addr, len); + section = &phys_sections[mmio->sub_section[idx]]; + addr += mmio->base; + addr -= section->offset_within_address_space; + addr += section->offset_within_region; + return io_mem_read(section->mr->ram_addr, addr, len); } static void subpage_write(void *opaque, target_phys_addr_t addr, @@ -3322,15 +3399,18 @@ static void subpage_write(void *opaque, target_phys_addr_t addr, { subpage_t *mmio = opaque; unsigned int idx = SUBPAGE_IDX(addr); + MemoryRegionSection *section; #if defined(DEBUG_SUBPAGE) printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %"PRIx64"\n", __func__, mmio, len, addr, idx, value); #endif - addr += mmio->region_offset[idx]; - idx = mmio->sub_io_index[idx]; - io_mem_write(idx, addr, value, len); + section = &phys_sections[mmio->sub_section[idx]]; + addr += mmio->base; + addr -= section->offset_within_address_space; + addr += section->offset_within_region; + io_mem_write(section->mr->ram_addr, addr, value, len); } static const MemoryRegionOps subpage_ops = { @@ -3372,7 +3452,7 @@ static const MemoryRegionOps subpage_ram_ops = { }; static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, - ram_addr_t memory, ram_addr_t region_offset) + uint16_t section) { int idx, eidx; @@ -3384,24 +3464,21 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__, mmio, start, end, idx, eidx, memory); #endif - if ((memory & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) { - memory = io_mem_subpage_ram.ram_addr; + if (memory_region_is_ram(phys_sections[section].mr)) { + MemoryRegionSection new_section = phys_sections[section]; + new_section.mr = &io_mem_subpage_ram; + section = phys_section_add(&new_section); } - memory &= IO_MEM_NB_ENTRIES - 1; for (; idx <= eidx; idx++) { - mmio->sub_io_index[idx] = memory; - mmio->region_offset[idx] = region_offset; + mmio->sub_section[idx] = section; } return 0; } -static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, - ram_addr_t orig_memory, - ram_addr_t region_offset) +static subpage_t *subpage_init(target_phys_addr_t base) { subpage_t *mmio; - int subpage_memory; mmio = g_malloc0(sizeof(subpage_t)); @@ -3409,13 +3486,11 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, memory_region_init_io(&mmio->iomem, &subpage_ops, mmio, "subpage", TARGET_PAGE_SIZE); mmio->iomem.subpage = true; - subpage_memory = mmio->iomem.ram_addr; #if defined(DEBUG_SUBPAGE) printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__, mmio, base, TARGET_PAGE_SIZE, subpage_memory); #endif - *phys = subpage_memory; - subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset); + subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned); return mmio; } @@ -3467,6 +3542,18 @@ void cpu_unregister_io_memory(int io_index) io_mem_used[io_index] = 0; } +static uint16_t dummy_section(MemoryRegion *mr) +{ + MemoryRegionSection section = { + .mr = mr, + .offset_within_address_space = 0, + .offset_within_region = 0, + .size = UINT64_MAX, + }; + + return phys_section_add(§ion); +} + static void io_mem_init(void) { int i; @@ -3488,6 +3575,174 @@ static void io_mem_init(void) "watch", UINT64_MAX); } +static void core_begin(MemoryListener *listener) +{ + destroy_all_mappings(); + phys_sections_clear(); + phys_map.ptr = PHYS_MAP_NODE_NIL; + phys_section_unassigned = dummy_section(&io_mem_unassigned); +} + +static void core_commit(MemoryListener *listener) +{ + CPUState *env; + + /* since each CPU stores ram addresses in its TLB cache, we must + reset the modified entries */ + /* XXX: slow ! */ + for(env = first_cpu; env != NULL; env = env->next_cpu) { + tlb_flush(env, 1); + } +} + +static void core_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + cpu_register_physical_memory_log(section, section->readonly); +} + +static void core_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ + cpu_register_physical_memory_log(section, section->readonly); +} + +static void core_log_start(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_stop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_global_start(MemoryListener *listener) +{ + cpu_physical_memory_set_dirty_tracking(1); +} + +static void core_log_global_stop(MemoryListener *listener) +{ + cpu_physical_memory_set_dirty_tracking(0); +} + +static void core_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void core_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void io_begin(MemoryListener *listener) +{ +} + +static void io_commit(MemoryListener *listener) +{ +} + +static void io_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + iorange_init(§ion->mr->iorange, &memory_region_iorange_ops, + section->offset_within_address_space, section->size); + ioport_register(§ion->mr->iorange); +} + +static void io_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ + isa_unassign_ioport(section->offset_within_address_space, section->size); +} + +static void io_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_start(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_stop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_global_start(MemoryListener *listener) +{ +} + +static void io_log_global_stop(MemoryListener *listener) +{ +} + +static void io_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void io_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static MemoryListener core_memory_listener = { + .begin = core_begin, + .commit = core_commit, + .region_add = core_region_add, + .region_del = core_region_del, + .region_nop = core_region_nop, + .log_start = core_log_start, + .log_stop = core_log_stop, + .log_sync = core_log_sync, + .log_global_start = core_log_global_start, + .log_global_stop = core_log_global_stop, + .eventfd_add = core_eventfd_add, + .eventfd_del = core_eventfd_del, + .priority = 0, +}; + +static MemoryListener io_memory_listener = { + .begin = io_begin, + .commit = io_commit, + .region_add = io_region_add, + .region_del = io_region_del, + .region_nop = io_region_nop, + .log_start = io_log_start, + .log_stop = io_log_stop, + .log_sync = io_log_sync, + .log_global_start = io_log_global_start, + .log_global_stop = io_log_global_stop, + .eventfd_add = io_eventfd_add, + .eventfd_del = io_eventfd_del, + .priority = 0, +}; + static void memory_map_init(void) { system_memory = g_malloc(sizeof(*system_memory)); @@ -3497,6 +3752,9 @@ static void memory_map_init(void) system_io = g_malloc(sizeof(*system_io)); memory_region_init(system_io, "io", 65536); set_system_io_map(system_io); + + memory_listener_register(&core_memory_listener, system_memory); + memory_listener_register(&io_memory_listener, system_io); } MemoryRegion *get_system_memory(void) @@ -3560,22 +3818,22 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, uint8_t *ptr; uint32_t val; target_phys_addr_t page; - ram_addr_t pd; - PhysPageDesc p; + MemoryRegionSection section; while (len > 0) { page = addr & TARGET_PAGE_MASK; l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); if (is_write) { - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { + if (!memory_region_is_ram(section.mr)) { target_phys_addr_t addr1; - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr1 = (addr & ~TARGET_PAGE_MASK) + + section.offset_within_region; /* XXX: could force cpu_single_env to NULL to avoid potential bugs */ if (l >= 4 && ((addr1 & 3) == 0)) { @@ -3594,9 +3852,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, io_mem_write(io_index, addr1, val, 1); l = 1; } - } else { + } else if (!section.readonly) { ram_addr_t addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) + | (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); @@ -3610,11 +3870,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, qemu_put_ram_ptr(ptr); } } else { - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { target_phys_addr_t addr1; /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr1 = (addr & ~TARGET_PAGE_MASK) + + section.offset_within_region; if (l >= 4 && ((addr1 & 3) == 0)) { /* 32 bit read access */ val = io_mem_read(io_index, addr1, 4); @@ -3633,7 +3895,8 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, } } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr(section.mr->ram_addr + + section.offset_within_region); memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l); qemu_put_ram_ptr(ptr); } @@ -3651,22 +3914,22 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr, int l; uint8_t *ptr; target_phys_addr_t page; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; while (len > 0) { page = addr & TARGET_PAGE_MASK; l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* do nothing */ } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); /* ROM/RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); @@ -3739,8 +4002,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, target_phys_addr_t todo = 0; int l; target_phys_addr_t page; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; ram_addr_t raddr = RAM_ADDR_MAX; ram_addr_t rlen; void *ret; @@ -3750,10 +4012,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { + if (!(memory_region_is_ram(section.mr) && !section.readonly)) { if (todo || bounce.buffer) { break; } @@ -3768,7 +4029,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, return bounce.buffer; } if (!todo) { - raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + raddr = memory_region_get_ram_addr(section.mr) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); } len -= l; @@ -3827,16 +4090,15 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint32_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; val = io_mem_read(io_index, addr, 4); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { @@ -3849,7 +4111,9 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: @@ -3888,16 +4152,15 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint64_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; /* XXX This is broken when device endian != cpu endian. Fix and add "endian" variable check */ @@ -3910,8 +4173,10 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: val = ldq_le_p(ptr); @@ -3957,16 +4222,15 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint64_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; val = io_mem_read(io_index, addr, 2); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { @@ -3979,8 +4243,10 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: val = lduw_le_p(ptr); @@ -4018,18 +4284,23 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val) { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; io_mem_write(io_index, addr, val, 4); } else { - unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + unsigned long addr1 = (memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); ptr = qemu_get_ram_ptr(addr1); stl_p(ptr, val); @@ -4049,15 +4320,18 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #ifdef TARGET_WORDS_BIGENDIAN io_mem_write(io_index, addr, val >> 32, 4); io_mem_write(io_index, addr + 4, (uint32_t)val, 4); @@ -4066,8 +4340,10 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) io_mem_write(io_index, addr + 4, val >> 32, 4); #endif } else { - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); stq_p(ptr, val); } } @@ -4078,15 +4354,18 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap32(val); @@ -4099,7 +4378,9 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, io_mem_write(io_index, addr, val, 4); } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); switch (endian) { @@ -4151,15 +4432,18 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap16(val); @@ -4172,7 +4456,8 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, io_mem_write(io_index, addr, val, 2); } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK) + + section.offset_within_region + (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); switch (endian) { @@ -4389,7 +4674,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr) } pd = env1->tlb_table[mmu_idx][page_index].addr_code & ~TARGET_PAGE_MASK; if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr - && !is_romd(pd)) { + && !io_mem_region[pd]->rom_device) { #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC) cpu_unassigned_access(env1, addr, 0, 1, 0, 4); #else diff --git a/hw/vhost.c b/hw/vhost.c index 5ece659f4..8d3ba5b60 100644 --- a/hw/vhost.c +++ b/hw/vhost.c @@ -436,6 +436,14 @@ static bool vhost_section(MemoryRegionSection *section) && memory_region_is_ram(section->mr); } +static void vhost_begin(MemoryListener *listener) +{ +} + +static void vhost_commit(MemoryListener *listener) +{ +} + static void vhost_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -476,6 +484,11 @@ static void vhost_region_del(MemoryListener *listener, } } +static void vhost_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static int vhost_virtqueue_set_addr(struct vhost_dev *dev, struct vhost_virtqueue *vq, unsigned idx, bool enable_log) @@ -720,6 +733,18 @@ static void vhost_virtqueue_cleanup(struct vhost_dev *dev, 0, virtio_queue_get_desc_size(vdev, idx)); } +static void vhost_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void vhost_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) { uint64_t features; @@ -744,13 +769,19 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) hdev->features = features; hdev->memory_listener = (MemoryListener) { + .begin = vhost_begin, + .commit = vhost_commit, .region_add = vhost_region_add, .region_del = vhost_region_del, + .region_nop = vhost_region_nop, .log_start = vhost_log_start, .log_stop = vhost_log_stop, .log_sync = vhost_log_sync, .log_global_start = vhost_log_global_start, .log_global_stop = vhost_log_global_stop, + .eventfd_add = vhost_eventfd_add, + .eventfd_del = vhost_eventfd_del, + .priority = 10 }; hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions)); hdev->n_mem_sections = 0; @@ -759,7 +790,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) hdev->log_size = 0; hdev->log_enabled = false; hdev->started = false; - memory_listener_register(&hdev->memory_listener); + memory_listener_register(&hdev->memory_listener, NULL); hdev->force = force; return 0; fail: diff --git a/ioport.c b/ioport.c index 36fa3a477..8a474d349 100644 --- a/ioport.c +++ b/ioport.c @@ -328,6 +328,7 @@ void portio_list_init(PortioList *piolist, piolist->ports = callbacks; piolist->nr = 0; piolist->regions = g_new0(MemoryRegion *, n); + piolist->aliases = g_new0(MemoryRegion *, n); piolist->address_space = NULL; piolist->opaque = opaque; piolist->name = name; @@ -336,6 +337,7 @@ void portio_list_init(PortioList *piolist, void portio_list_destroy(PortioList *piolist) { g_free(piolist->regions); + g_free(piolist->aliases); } static void portio_list_add_1(PortioList *piolist, @@ -345,7 +347,7 @@ static void portio_list_add_1(PortioList *piolist, { MemoryRegionPortio *pio; MemoryRegionOps *ops; - MemoryRegion *region; + MemoryRegion *region, *alias; unsigned i; /* Copy the sub-list and null-terminate it. */ @@ -362,12 +364,20 @@ static void portio_list_add_1(PortioList *piolist, ops->old_portio = pio; region = g_new(MemoryRegion, 1); + alias = g_new(MemoryRegion, 1); + /* + * Use an alias so that the callback is called with an absolute address, + * rather than an offset relative to to start + off_low. + */ memory_region_init_io(region, ops, piolist->opaque, piolist->name, - off_high - off_low); - memory_region_set_offset(region, start + off_low); + UINT64_MAX); + memory_region_init_alias(alias, piolist->name, + region, start + off_low, off_high - off_low); memory_region_add_subregion(piolist->address_space, - start + off_low, region); - piolist->regions[piolist->nr++] = region; + start + off_low, alias); + piolist->regions[piolist->nr] = region; + piolist->aliases[piolist->nr] = alias; + ++piolist->nr; } void portio_list_add(PortioList *piolist, @@ -409,15 +419,19 @@ void portio_list_add(PortioList *piolist, void portio_list_del(PortioList *piolist) { - MemoryRegion *mr; + MemoryRegion *mr, *alias; unsigned i; for (i = 0; i < piolist->nr; ++i) { mr = piolist->regions[i]; - memory_region_del_subregion(piolist->address_space, mr); + alias = piolist->aliases[i]; + memory_region_del_subregion(piolist->address_space, alias); + memory_region_destroy(alias); memory_region_destroy(mr); g_free((MemoryRegionOps *)mr->ops); g_free(mr); + g_free(alias); piolist->regions[i] = NULL; + piolist->aliases[i] = NULL; } } diff --git a/ioport.h b/ioport.h index ae3e9da0b..ab29c89fb 100644 --- a/ioport.h +++ b/ioport.h @@ -60,6 +60,7 @@ typedef struct PortioList { struct MemoryRegion *address_space; unsigned nr; struct MemoryRegion **regions; + struct MemoryRegion **aliases; void *opaque; const char *name; } PortioList; diff --git a/kvm-all.c b/kvm-all.c index e2cbc0302..711975195 100644 --- a/kvm-all.c +++ b/kvm-all.c @@ -28,6 +28,7 @@ #include "kvm.h" #include "bswap.h" #include "memory.h" +#include "exec-memory.h" /* This check must be after config-host.h is included */ #ifdef CONFIG_EVENTFD @@ -674,6 +675,14 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add) } } +static void kvm_begin(MemoryListener *listener) +{ +} + +static void kvm_commit(MemoryListener *listener) +{ +} + static void kvm_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -686,6 +695,11 @@ static void kvm_region_del(MemoryListener *listener, kvm_set_phys_mem(section, false); } +static void kvm_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void kvm_log_sync(MemoryListener *listener, MemoryRegionSection *section) { @@ -713,14 +727,95 @@ static void kvm_log_global_stop(struct MemoryListener *listener) assert(r >= 0); } +static void kvm_mem_ioeventfd_add(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + assert(match_data && section->size == 4); + + r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space, + data, true); + if (r < 0) { + abort(); + } +} + +static void kvm_mem_ioeventfd_del(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space, + data, false); + if (r < 0) { + abort(); + } +} + +static void kvm_io_ioeventfd_add(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + assert(match_data && section->size == 2); + + r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space, + data, true); + if (r < 0) { + abort(); + } +} + +static void kvm_io_ioeventfd_del(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) + +{ + int r; + + r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space, + data, false); + if (r < 0) { + abort(); + } +} + +static void kvm_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + if (section->address_space == get_system_memory()) { + kvm_mem_ioeventfd_add(section, match_data, data, fd); + } else { + kvm_io_ioeventfd_add(section, match_data, data, fd); + } +} + +static void kvm_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + if (section->address_space == get_system_memory()) { + kvm_mem_ioeventfd_del(section, match_data, data, fd); + } else { + kvm_io_ioeventfd_del(section, match_data, data, fd); + } +} + static MemoryListener kvm_memory_listener = { + .begin = kvm_begin, + .commit = kvm_commit, .region_add = kvm_region_add, .region_del = kvm_region_del, + .region_nop = kvm_region_nop, .log_start = kvm_log_start, .log_stop = kvm_log_stop, .log_sync = kvm_log_sync, .log_global_start = kvm_log_global_start, .log_global_stop = kvm_log_global_stop, + .eventfd_add = kvm_eventfd_add, + .eventfd_del = kvm_eventfd_del, + .priority = 10, }; static void kvm_handle_interrupt(CPUState *env, int mask) @@ -965,7 +1060,7 @@ int kvm_init(void) } kvm_state = s; - memory_listener_register(&kvm_memory_listener); + memory_listener_register(&kvm_memory_listener, NULL); s->many_ioeventfds = kvm_check_many_ioeventfds(); diff --git a/memory.c b/memory.c index 22816e20f..6565e2e69 100644 --- a/memory.c +++ b/memory.c @@ -27,8 +27,8 @@ unsigned memory_region_transaction_depth = 0; static bool memory_region_update_pending = false; static bool global_dirty_log = false; -static QLIST_HEAD(, MemoryListener) memory_listeners - = QLIST_HEAD_INITIALIZER(memory_listeners); +static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners + = QTAILQ_HEAD_INITIALIZER(memory_listeners); typedef struct AddrRange AddrRange; @@ -82,6 +82,71 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) return addrrange_make(start, int128_sub(end, start)); } +enum ListenerDirection { Forward, Reverse }; + +static bool memory_listener_match(MemoryListener *listener, + MemoryRegionSection *section) +{ + return !listener->address_space_filter + || listener->address_space_filter == section->address_space; +} + +#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + if (memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + if (memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ + MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \ + .mr = (fr)->mr, \ + .address_space = (as)->root, \ + .offset_within_region = (fr)->offset_in_region, \ + .size = int128_get64((fr)->addr.size), \ + .offset_within_address_space = int128_get64((fr)->addr.start), \ + .readonly = (fr)->readonly, \ + })) + struct CoalescedMemoryRange { AddrRange addr; QTAILQ_ENTRY(CoalescedMemoryRange) link; @@ -158,22 +223,12 @@ typedef struct AddressSpaceOps AddressSpaceOps; /* A system address space - I/O, memory, etc. */ struct AddressSpace { - const AddressSpaceOps *ops; MemoryRegion *root; FlatView current_map; int ioeventfd_nb; MemoryRegionIoeventfd *ioeventfds; }; -struct AddressSpaceOps { - void (*range_add)(AddressSpace *as, FlatRange *fr); - void (*range_del)(AddressSpace *as, FlatRange *fr); - void (*log_start)(AddressSpace *as, FlatRange *fr); - void (*log_stop)(AddressSpace *as, FlatRange *fr); - void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd); - void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd); -}; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -305,74 +360,7 @@ static void access_with_adjusted_size(target_phys_addr_t addr, } } -static void as_memory_range_add(AddressSpace *as, FlatRange *fr) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - }; - - cpu_register_physical_memory_log(§ion, fr->readable, fr->readonly); -} - -static void as_memory_range_del(AddressSpace *as, FlatRange *fr) -{ - MemoryRegionSection section = { - .mr = &io_mem_unassigned, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = int128_get64(fr->addr.start), - .size = int128_get64(fr->addr.size), - }; - - cpu_register_physical_memory_log(§ion, true, false); -} - -static void as_memory_log_start(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_log_stop(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 4); - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - -static const AddressSpaceOps address_space_ops_memory = { - .range_add = as_memory_range_add, - .range_del = as_memory_range_del, - .log_start = as_memory_log_start, - .log_stop = as_memory_log_stop, - .ioeventfd_add = as_memory_ioeventfd_add, - .ioeventfd_del = as_memory_ioeventfd_del, -}; - -static AddressSpace address_space_memory = { - .ops = &address_space_ops_memory, -}; +static AddressSpace address_space_memory; static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset, unsigned width, bool write) @@ -401,17 +389,17 @@ static void memory_region_iorange_read(IORange *iorange, *data = ((uint64_t)1 << (width * 8)) - 1; if (mrp) { - *data = mrp->read(mr->opaque, offset + mr->offset); + *data = mrp->read(mr->opaque, offset); } else if (width == 2) { mrp = find_portio(mr, offset, 1, false); assert(mrp); - *data = mrp->read(mr->opaque, offset + mr->offset) | - (mrp->read(mr->opaque, offset + mr->offset + 1) << 8); + *data = mrp->read(mr->opaque, offset) | + (mrp->read(mr->opaque, offset + 1) << 8); } return; } *data = 0; - access_with_adjusted_size(offset + mr->offset, data, width, + access_with_adjusted_size(offset, data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -428,73 +416,27 @@ static void memory_region_iorange_write(IORange *iorange, const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true); if (mrp) { - mrp->write(mr->opaque, offset + mr->offset, data); + mrp->write(mr->opaque, offset, data); } else if (width == 2) { mrp = find_portio(mr, offset, 1, false); assert(mrp); - mrp->write(mr->opaque, offset + mr->offset, data & 0xff); - mrp->write(mr->opaque, offset + mr->offset + 1, data >> 8); + mrp->write(mr->opaque, offset, data & 0xff); + mrp->write(mr->opaque, offset + 1, data >> 8); } return; } - access_with_adjusted_size(offset + mr->offset, &data, width, + access_with_adjusted_size(offset, &data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); } -static const IORangeOps memory_region_iorange_ops = { +const IORangeOps memory_region_iorange_ops = { .read = memory_region_iorange_read, .write = memory_region_iorange_write, }; -static void as_io_range_add(AddressSpace *as, FlatRange *fr) -{ - iorange_init(&fr->mr->iorange, &memory_region_iorange_ops, - int128_get64(fr->addr.start), int128_get64(fr->addr.size)); - ioport_register(&fr->mr->iorange); -} - -static void as_io_range_del(AddressSpace *as, FlatRange *fr) -{ - isa_unassign_ioport(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); -} - -static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 2); - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - -static const AddressSpaceOps address_space_ops_io = { - .range_add = as_io_range_add, - .range_del = as_io_range_del, - .ioeventfd_add = as_io_ioeventfd_add, - .ioeventfd_del = as_io_ioeventfd_del, -}; - -static AddressSpace address_space_io = { - .ops = &address_space_ops_io, -}; +static AddressSpace address_space_io; static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) { @@ -621,6 +563,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, unsigned fds_old_nb) { unsigned iold, inew; + MemoryRegionIoeventfd *fd; + MemoryRegionSection section; /* Generate a symmetric difference of the old and new fd sets, adding * and deleting as necessary. @@ -632,13 +576,27 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, && (inew == fds_new_nb || memory_region_ioeventfd_before(fds_old[iold], fds_new[inew]))) { - as->ops->ioeventfd_del(as, &fds_old[iold]); + fd = &fds_old[iold]; + section = (MemoryRegionSection) { + .address_space = as->root, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_del, Forward, §ion, + fd->match_data, fd->data, fd->fd); ++iold; } else if (inew < fds_new_nb && (iold == fds_old_nb || memory_region_ioeventfd_before(fds_new[inew], fds_old[iold]))) { - as->ops->ioeventfd_add(as, &fds_new[inew]); + fd = &fds_new[inew]; + section = (MemoryRegionSection) { + .address_space = as->root, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_add, Reverse, §ion, + fd->match_data, fd->data, fd->fd); ++inew; } else { ++iold; @@ -678,32 +636,6 @@ static void address_space_update_ioeventfds(AddressSpace *as) as->ioeventfd_nb = ioeventfd_nb; } -typedef void ListenerCallback(MemoryListener *listener, - MemoryRegionSection *mrs); - -/* Want "void (&MemoryListener::*callback)(const MemoryRegionSection& s)" */ -static void memory_listener_update_region(FlatRange *fr, AddressSpace *as, - size_t callback_offset) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .address_space = as->root, - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - .offset_within_address_space = int128_get64(fr->addr.start), - }; - MemoryListener *listener; - - QLIST_FOREACH(listener, &memory_listeners, link) { - ListenerCallback *callback - = *(ListenerCallback **)((void *)listener + callback_offset); - callback(listener, §ion); - } -} - -#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \ - memory_listener_update_region(fr, as, offsetof(MemoryListener, callback)) - static void address_space_update_topology_pass(AddressSpace *as, FlatView old_view, FlatView new_view, @@ -736,8 +668,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In old, but (not in new, or in new but attributes changed). */ if (!adding) { - MEMORY_LISTENER_UPDATE_REGION(frold, as, region_del); - as->ops->range_del(as, frold); + MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); } ++iold; @@ -745,12 +676,11 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In both (logging may have changed) */ if (adding) { + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop); if (frold->dirty_log_mask && !frnew->dirty_log_mask) { - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_stop); - as->ops->log_stop(as, frnew); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop); } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) { - as->ops->log_start(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_start); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start); } } @@ -760,8 +690,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In new */ if (adding) { - as->ops->range_add(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, region_add); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); } ++inew; @@ -794,6 +723,8 @@ static void memory_region_update_topology(MemoryRegion *mr) return; } + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); + if (address_space_memory.root) { address_space_update_topology(&address_space_memory); } @@ -801,6 +732,8 @@ static void memory_region_update_topology(MemoryRegion *mr) address_space_update_topology(&address_space_io); } + MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); + memory_region_update_pending = false; } @@ -863,7 +796,6 @@ void memory_region_init(MemoryRegion *mr, mr->size = int128_2_64(); } mr->addr = 0; - mr->offset = 0; mr->subpage = false; mr->enabled = true; mr->terminates = false; @@ -925,7 +857,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -979,7 +911,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); @@ -1122,11 +1054,6 @@ bool memory_region_is_rom(MemoryRegion *mr) return mr->ram && mr->readonly; } -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset) -{ - mr->offset = offset; -} - void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask = 1 << client; @@ -1156,7 +1083,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { if (fr->mr == mr) { - MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, log_sync); + MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, + Forward, log_sync); } } } @@ -1474,6 +1402,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, fr->addr.start)); ret.size = int128_get64(range.size); ret.offset_within_address_space = int128_get64(range.start); + ret.readonly = fr->readonly; return ret; } @@ -1483,30 +1412,20 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) FlatRange *fr; FOR_EACH_FLAT_RANGE(fr, &as->current_map) { - MEMORY_LISTENER_UPDATE_REGION(fr, as, log_sync); + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); } } void memory_global_dirty_log_start(void) { - MemoryListener *listener; - - cpu_physical_memory_set_dirty_tracking(1); global_dirty_log = true; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_start(listener); - } + MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward); } void memory_global_dirty_log_stop(void) { - MemoryListener *listener; - global_dirty_log = false; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_stop(listener); - } - cpu_physical_memory_set_dirty_tracking(0); + MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse); } static void listener_add_address_space(MemoryListener *listener, @@ -1524,21 +1443,36 @@ static void listener_add_address_space(MemoryListener *listener, .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), .offset_within_address_space = int128_get64(fr->addr.start), + .readonly = fr->readonly, }; listener->region_add(listener, §ion); } } -void memory_listener_register(MemoryListener *listener) +void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) { - QLIST_INSERT_HEAD(&memory_listeners, listener, link); + MemoryListener *other = NULL; + + listener->address_space_filter = filter; + if (QTAILQ_EMPTY(&memory_listeners) + || listener->priority >= QTAILQ_LAST(&memory_listeners, + memory_listeners)->priority) { + QTAILQ_INSERT_TAIL(&memory_listeners, listener, link); + } else { + QTAILQ_FOREACH(other, &memory_listeners, link) { + if (listener->priority < other->priority) { + break; + } + } + QTAILQ_INSERT_BEFORE(other, listener, link); + } listener_add_address_space(listener, &address_space_memory); listener_add_address_space(listener, &address_space_io); } void memory_listener_unregister(MemoryListener *listener) { - QLIST_REMOVE(listener, link); + QTAILQ_REMOVE(&memory_listeners, listener, link); } void set_system_memory_map(MemoryRegion *mr) diff --git a/memory.h b/memory.h index 4cf8d2f27..b7bccd196 100644 --- a/memory.h +++ b/memory.h @@ -115,7 +115,6 @@ struct MemoryRegion { MemoryRegion *parent; Int128 size; target_phys_addr_t addr; - target_phys_addr_t offset; void (*destructor)(MemoryRegion *mr); ram_addr_t ram_addr; IORange iorange; @@ -161,6 +160,7 @@ typedef struct MemoryRegionSection MemoryRegionSection; * @size: the size of the section; will not exceed @mr's boundaries * @offset_within_address_space: the address of the first byte of the section * relative to the region's address space + * @readonly: writes to this section are ignored */ struct MemoryRegionSection { MemoryRegion *mr; @@ -168,6 +168,7 @@ struct MemoryRegionSection { target_phys_addr_t offset_within_region; uint64_t size; target_phys_addr_t offset_within_address_space; + bool readonly; }; typedef struct MemoryListener MemoryListener; @@ -179,14 +180,24 @@ typedef struct MemoryListener MemoryListener; * Use with memory_listener_register() and memory_listener_unregister(). */ struct MemoryListener { + void (*begin)(MemoryListener *listener); + void (*commit)(MemoryListener *listener); void (*region_add)(MemoryListener *listener, MemoryRegionSection *section); void (*region_del)(MemoryListener *listener, MemoryRegionSection *section); + void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section); void (*log_start)(MemoryListener *listener, MemoryRegionSection *section); void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section); void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); void (*log_global_start)(MemoryListener *listener); void (*log_global_stop)(MemoryListener *listener); - QLIST_ENTRY(MemoryListener) link; + void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section, + bool match_data, uint64_t data, int fd); + void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section, + bool match_data, uint64_t data, int fd); + /* Lower = earlier (during add), later (during del) */ + unsigned priority; + MemoryRegion *address_space_filter; + QTAILQ_ENTRY(MemoryListener) link; }; /** @@ -358,14 +369,6 @@ bool memory_region_is_rom(MemoryRegion *mr); */ void *memory_region_get_ram_ptr(MemoryRegion *mr); -/** - * memory_region_set_offset: Sets an offset to be added to MemoryRegionOps - * callbacks. - * - * This function is deprecated and should not be used in new code. - */ -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset); - /** * memory_region_set_log: Turn dirty logging on or off for a region. * @@ -686,8 +689,9 @@ void memory_region_transaction_commit(void); * space * * @listener: an object containing the callbacks to be called + * @filter: if non-%NULL, only regions in this address space will be observed */ -void memory_listener_register(MemoryListener *listener); +void memory_listener_register(MemoryListener *listener, MemoryRegion *filter); /** * memory_listener_unregister: undo the effect of memory_listener_register() diff --git a/xen-all.c b/xen-all.c index b0ed1ed19..6cef5062b 100644 --- a/xen-all.c +++ b/xen-all.c @@ -392,6 +392,14 @@ static void xen_set_memory(struct MemoryListener *listener, } } +static void xen_begin(MemoryListener *listener) +{ +} + +static void xen_commit(MemoryListener *listener) +{ +} + static void xen_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -404,6 +412,11 @@ static void xen_region_del(MemoryListener *listener, xen_set_memory(listener, section, false); } +static void xen_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void xen_sync_dirty_bitmap(XenIOState *state, target_phys_addr_t start_addr, ram_addr_t size) @@ -485,14 +498,32 @@ static void xen_log_global_stop(MemoryListener *listener) { } +static void xen_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void xen_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + static MemoryListener xen_memory_listener = { + .begin = xen_begin, + .commit = xen_commit, .region_add = xen_region_add, .region_del = xen_region_del, + .region_nop = xen_region_nop, .log_start = xen_log_start, .log_stop = xen_log_stop, .log_sync = xen_log_sync, .log_global_start = xen_log_global_start, .log_global_stop = xen_log_global_stop, + .eventfd_add = xen_eventfd_add, + .eventfd_del = xen_eventfd_del, + .priority = 10, }; /* VCPU Operations, MMIO, IO ring ... */ @@ -975,7 +1006,7 @@ int xen_hvm_init(void) state->memory_listener = xen_memory_listener; QLIST_INIT(&state->physmap); - memory_listener_register(&state->memory_listener); + memory_listener_register(&state->memory_listener, get_system_memory()); state->log_for_dirtybit = NULL; /* Initialize backend core & drivers */