From 52f631665f75cc852f980ec67d4e63b220affeae Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 6 Oct 2026 05:02:20 +0800 Subject: [PATCH 01/20] Fix malloc chunk reuse and libc error checks malloc() compared a freed chunk's recorded size, which includes the chunk header, against the requested size, which does not, so a request in the last header's worth of a freed chunk got a chunk too small for it and wrote into the neighboring mapping. malloc also checked the raw mmap2 result for -1, but the system call reports failure as a negative errno, so a failed mapping was used as a chunk header. Freeing the last chunk now resets the allocator tail even when it has no predecessor. fseek() and ftell() test the raw lseek result for any negative errno, and RV32 passes the high offset word and reads the 64-bit position that _llseek reports. --- lib/c.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/lib/c.c b/lib/c.c index ce6ac3fc..8f0acce2 100644 --- a/lib/c.c +++ b/lib/c.c @@ -10,9 +10,8 @@ /* Staging buffer for the printf family that writes straight to a descriptor. * - * The longest single call in the tree is ssa.c's "insn_%p [label=%s]": a - * DUMP_INSN_LEN staging buffer plus 26 bytes around it, so 537. Every byte here - * is stack in every program shecc emits, so it stays close to that. + * Keep the buffer bounded: every byte here occupies stack space in programs + * compiled by shecc. */ #define FMT_BUF_LEN 576 @@ -792,27 +791,27 @@ int fputc(int c, FILE *stream) int fseek(FILE *stream, int offset, int whence) { - int result; - /* RV32 has only _llseek, which splits the offset and returns the result * through a pointer. Every other target takes (fd, offset, whence) * directly. lib/c.h rejects an architecture that is neither. */ #if defined(__riscv) - result = __syscall(__syscall_lseek, stream, 0, offset, NULL, whence); + long long position; + return __syscall(__syscall_lseek, stream, offset < 0 ? -1 : 0, offset, + &position, whence) < 0; #else - result = __syscall(__syscall_lseek, stream, offset, whence); + return __syscall(__syscall_lseek, stream, offset, whence) < 0; #endif - return result == -1; } int ftell(FILE *stream) { /* See fseek(): only RV32 needs the split-offset _llseek form. */ #if defined(__riscv) - int result; - __syscall(__syscall_lseek, stream, 0, 0, &result, SEEK_CUR); - return result; + long long result; + if (__syscall(__syscall_lseek, stream, 0, 0, &result, SEEK_CUR) < 0) + return -1; + return (int) result; #else return __syscall(__syscall_lseek, stream, 0, SEEK_CUR); #endif @@ -843,6 +842,12 @@ int __align_up(int size) return ALIGN_UP(size, PAGESIZE); } +/* The raw mmap2 system call reports failure as -errno, -4095 to -1, not the -1 + * of libc's wrapper. Success is always page aligned and those values never are, + * so the low bits alone tell them apart on every word size. + */ +#define __mmap_failed(addr) (((int) (addr) & (PAGESIZE - 1)) != 0) + chunk_t *__alloc_head; chunk_t *__alloc_tail; chunk_t *__freelist_head; @@ -862,7 +867,7 @@ void *malloc(int size) chunk_t *tmp = (chunk_t *) __syscall(__syscall_mmap2, NULL, __align_up(sizeof(chunk_t)), prot, flags, -1, 0); - if (tmp == (void *) -1) + if (__mmap_failed(tmp)) return NULL; __alloc_head = tmp; __alloc_tail = tmp; @@ -875,7 +880,7 @@ void *malloc(int size) chunk_t *tmp = (chunk_t *) __syscall(__syscall_mmap2, NULL, __align_up(sizeof(chunk_t)), prot, flags, -1, 0); - if (tmp == (void *) -1) + if (__mmap_failed(tmp)) return NULL; __freelist_head = tmp; __freelist_head->next = NULL; @@ -893,7 +898,8 @@ void *malloc(int size) } else { for (chunk_t *fh = __freelist_head; fh->next; fh = fh->next) { int fh_size = CHUNK_GET_SIZE(fh->size); - if (fh_size >= size && (!best_fit_chunk || fh_size < best_size)) { + if (fh_size - (int) sizeof(chunk_t) >= size && + (!best_fit_chunk || fh_size < best_size)) { best_fit_chunk = fh; best_size = fh_size; } @@ -917,7 +923,7 @@ void *malloc(int size) allocated = (chunk_t *) __syscall(__syscall_mmap2, NULL, __align_up(sizeof(chunk_t) + size), prot, flags, -1, 0); - if (allocated == (void *) -1) + if (__mmap_failed(allocated)) return NULL; allocated->size = __align_up(sizeof(chunk_t) + size); } @@ -1015,8 +1021,9 @@ void free(void *ptr) if (cur->next) { chunk_t *next = cur->next; next->prev = cur->prev; - } else if (prev) { - prev->next = NULL; + } else { + if (prev) + prev->next = NULL; __alloc_tail = prev; } From f880532f22cd546ed264a6a734cd5b251972ce48 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 7 Oct 2026 00:57:20 +0800 Subject: [PATCH 02/20] Reach every RISC-V call beyond JAL range The RV32 self-hosted image is already past 1 MiB of code, and direct calls, including the startup stub's call to main, used a JAL, which reaches only 1 MiB. Any growth that moved a callee past that limit aborted the stage 1 build. Call through AUIPC and JALR, as calls into the PLT already do, and count the extra instruction in the size estimates. --- src/riscv-codegen.c | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/riscv-codegen.c b/src/riscv-codegen.c index a41a61d4..fbc1bfc0 100644 --- a/src/riscv-codegen.c +++ b/src/riscv-codegen.c @@ -143,8 +143,8 @@ void update_elf_offset(ph2_ir_t *ph2_ir) elf_offset += ph2_ir->src1_hi >= 0 ? 8 : 4; return; case OP_call: - /* A call through the PLT may be farther than a JAL reaches. */ - elf_offset += dynlink && !find_func(ph2_ir->func_name)->bbs ? 8 : 4; + /* Any call may be farther than a JAL reaches. */ + elf_offset += 8; return; case OP_jump: case OP_load_func: @@ -313,11 +313,14 @@ void cfg_flatten(void) update_elf_offset(ph2_ir); } - /* prepare 'argc' and 'argv', then proceed to 'main' function */ + /* prepare 'argc' and 'argv', then proceed to 'main' function. The stub + * precedes every function, so 'main' can lie beyond the 1 MiB a JAL + * reaches; it is called through AUIPC and JALR instead. + */ if (dynlink) - elf_offset += 44; + elf_offset += 48; else - elf_offset += 24; + elf_offset += 28; for (func = FUNC_LIST.head; func; func = func->next) { /* Skip function declarations without bodies */ @@ -589,24 +592,23 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_call: func = find_func(ph2_ir->func_name); + + /* A self-hosted image is past the 1 MiB a JAL reaches, and the PLT + * follows the code and read-only data. Call PC-relative through AUIPC + * and JALR, which reach anywhere. + */ if (func->bbs) ofs = func->bbs->elf_offset - elf_code->size; - else if (dynlink) { - /* The PLT follows the code and read-only data, so a large image - * puts it beyond the 1 MiB a JAL reaches. Call it PC-relative - * through AUIPC and JALR, which reach anywhere. - */ + else if (dynlink) ofs = (dynamic_sections.elf_plt_start + func->plt_offset) - (elf_code_start + elf_code->size); - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); - return; - } else { + else { printf("The '%s' function is not implemented\n", ph2_ir->func_name); fflush(stdout); /* see fatal() */ abort(); } - emit(__jal(__ra, ofs)); + emit(__auipc(__ra, rv_hi(ofs))); + emit(__jalr(__ra, __ra, rv_lo(ofs))); return; case OP_load_data_address: emit(__lui(rd, rv_hi(elf_data_start + ph2_ir->src0))); @@ -1283,7 +1285,9 @@ void code_generate(void) if (dynlink) { emit(__addi(__a0, __s0, 0)); emit(__addi(__a1, __s1, 0)); - emit(__jal(__ra, MAIN_BB->elf_offset - elf_code->size)); + ofs = MAIN_BB->elf_offset - elf_code->size; + emit(__auipc(__ra, rv_hi(ofs))); + emit(__jalr(__ra, __ra, rv_lo(ofs))); /* - Restore sp, s0 and s1. * - Transfer control back to __libc_start_main() using @@ -1302,7 +1306,9 @@ void code_generate(void) emit(__addi(__t0, __s0, 0)); emit(__lw(__a0, __t0, 0)); emit(__addi(__a1, __t0, 4)); - emit(__jal(__ra, MAIN_BB->elf_offset - elf_code->size)); + ofs = MAIN_BB->elf_offset - elf_code->size; + emit(__auipc(__ra, rv_hi(ofs))); + emit(__jalr(__ra, __ra, rv_lo(ofs))); /* exit with main's return value in a0 */ emit(__addi(__a7, __zero, 93)); From 27f4277c8c7e40ef9f7184f556e634ff9c4db5a9 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 6 Oct 2026 05:02:20 +0800 Subject: [PATCH 03/20] Share ELF header serialization Compute load segment layout once and route ELF32 and ELF64 program and file headers through shared serializers, and emit ELF32 section headers from initialized structs. Byte layout and offsets are unchanged. --- src/elf.c | 627 +++++++++++++++--------------------------------------- 1 file changed, 173 insertions(+), 454 deletions(-) diff --git a/src/elf.c b/src/elf.c index b6bb6855..d77d1666 100644 --- a/src/elf.c +++ b/src/elf.c @@ -15,6 +15,14 @@ #define PAGESIZE 4096 #endif +static bool elf_dynamic_sections_ready(const strbuf_t *relplt) +{ + return !dynlink || + (dynamic_sections.elf_interp && relplt && dynamic_sections.elf_plt && + dynamic_sections.elf_got && dynamic_sections.elf_dynstr && + dynamic_sections.elf_dynsym && dynamic_sections.elf_dynamic); +} + int elf_symbol_index = 0; void elf_write_str(strbuf_t *elf_array, const char *vals) @@ -76,6 +84,13 @@ void elf_write_blk(strbuf_t *elf_array, const void *blk, int sz) elf_array->size += sz; } +#if ELF_IS_64 == 0 +static void elf_write_section_header(const elf32_shdr_t *shdr) +{ + elf_write_blk(elf_section_header, shdr, sizeof(*shdr)); +} +#endif + /* The dynamic-linking tables differ only in width between the two ELF classes, * so the generator below writes them through these helpers rather than * memcpy-ing a struct: shecc has no 64-bit integer type, so an ELF64 entry @@ -221,49 +236,56 @@ strbuf_t *elf_relplt_buf(void) return dynamic_sections.elf_relplt; } +static void elf_write_file_header(int is_64, + int entry, + int phnum, + int shoff, + int shnum, + int shstrndx) +{ + int header_size = is_64 ? 64 : sizeof(elf32_hdr_t); + int program_header_size = is_64 ? 56 : sizeof(elf32_phdr_t); + + elf_write_byte(elf_header, 0x7f); + elf_write_str(elf_header, "ELF"); + elf_write_byte(elf_header, is_64 ? 2 : 1); + elf_write_byte(elf_header, 1); + elf_write_byte(elf_header, 1); + for (int i = 0; i < 9; i++) + elf_write_byte(elf_header, 0); + + elf_write_short(elf_header, 2); + elf_write_short(elf_header, ELF_MACHINE); + elf_write_int(elf_header, 1); + if (is_64) { + elf_write_quad(elf_header, entry); + elf_write_quad(elf_header, header_size); + elf_write_quad(elf_header, 0); + } else { + elf_write_int(elf_header, entry); + elf_write_int(elf_header, header_size); + elf_write_int(elf_header, shoff); + } + elf_write_int(elf_header, ELF_FLAGS); + elf_write_short(elf_header, header_size); + elf_write_short(elf_header, program_header_size); + elf_write_short(elf_header, phnum); + elf_write_short(elf_header, is_64 ? 0 : sizeof(elf32_shdr_t)); + elf_write_short(elf_header, shnum); + elf_write_short(elf_header, shstrndx); +} + void elf_generate_header(void) { /* Check for null pointers to prevent crashes */ if (!elf_code || !elf_data || !elf_symtab || !elf_strtab || !elf_header) { fatal("ELF buffers not initialized"); - return; } -#if ELF_IS_64 == 1 - /* ELF64 executable header, 64 bytes, little-endian. - * - * No section headers are emitted, so e_shoff, e_shnum and e_shstrndx are - * all zero: a loader needs only the program headers to execute the image, - * and the dynamic loader reads PT_DYNAMIC rather than the section table. - */ - int phnum64 = 2; - if (dynlink) - phnum64 = 4; /* the two PT_LOADs plus PT_INTERP and PT_DYNAMIC */ - elf_write_byte(elf_header, 0x7f); - elf_write_str(elf_header, "ELF"); - elf_write_byte(elf_header, 2); /* EI_CLASS = ELFCLASS64 */ - elf_write_byte(elf_header, 1); /* EI_DATA = ELFDATA2LSB */ - elf_write_byte(elf_header, 1); /* EI_VERSION = EV_CURRENT */ - for (int i = 0; i < 9; i++) /* EI_OSABI, EI_ABIVERSION, padding */ - elf_write_byte(elf_header, 0); - - elf_write_short(elf_header, 2); /* e_type = ET_EXEC */ - elf_write_short(elf_header, ELF_MACHINE); /* e_machine */ - elf_write_int(elf_header, 1); /* e_version = EV_CURRENT */ - elf_write_quad(elf_header, elf_code_start + elf_entry_offset); /* e_entry */ - elf_write_quad(elf_header, 64); /* e_phoff: right after header */ - elf_write_quad(elf_header, 0); /* e_shoff: no section headers */ - elf_write_int(elf_header, ELF_FLAGS); /* e_flags */ - elf_write_short(elf_header, 64); /* e_ehsize */ - elf_write_short(elf_header, 56); /* e_phentsize */ - elf_write_short(elf_header, phnum64); /* e_phnum */ - elf_write_short(elf_header, 0); /* e_shentsize */ - elf_write_short(elf_header, 0); /* e_shnum */ - elf_write_short(elf_header, 0); /* e_shstrndx */ -#else - elf32_hdr_t hdr; - int phnum, shnum, shstrndx, shoff; + int phnum = dynlink ? 4 : 2; + int shnum = 0, shstrndx = 0, shoff = 0; +#if ELF_IS_64 == 0 if (dynlink) { /* In dynamic linking mode: * - number of program headers = 4 @@ -273,7 +295,6 @@ void elf_generate_header(void) int elf_relplt_size = dynamic_sections.use_relaplt ? dynamic_sections.elf_relaplt->size : dynamic_sections.elf_relplt->size; - phnum = 4; shnum = 15; shstrndx = 14; shoff = elf_header_len + elf_code->size + elf_data->size + @@ -290,7 +311,6 @@ void elf_generate_header(void) * - number of section headers = 8 * - section header index of .shstrtab = 7 */ - phnum = 2; shnum = 8; shstrndx = 7; @@ -348,238 +368,86 @@ void elf_generate_header(void) * ---+----------------+-------------------------------------------------+ * 34 | | | */ - /* ELF file header */ - hdr.e_ident[0] = (char) 0x7F; /* ELF magic number */ - hdr.e_ident[1] = 'E'; - hdr.e_ident[2] = 'L'; - hdr.e_ident[3] = 'F'; - hdr.e_ident[4] = 1; /* 32-bit */ - hdr.e_ident[5] = 1; /* little-endian */ - hdr.e_ident[6] = 1; /* ELF header version */ - hdr.e_ident[7] = 0; /* Target OS ABI */ - hdr.e_ident[8] = 0; /* ABI version */ - hdr.e_ident[9] = 0; /* Padding */ - hdr.e_ident[10] = 0; - hdr.e_ident[11] = 0; - hdr.e_ident[12] = 0; - hdr.e_ident[13] = 0; - hdr.e_ident[14] = 0; - hdr.e_ident[15] = 0; - hdr.e_type = 2; /* Object file type */ - hdr.e_machine = ELF_MACHINE; /* Instruction Set Architecture */ - hdr.e_version = 1; /* ELF version */ - hdr.e_entry = elf_code_start; /* entry point */ - hdr.e_phoff = sizeof(elf32_hdr_t); /* program header offset */ - hdr.e_shoff = shoff; /* section header offset */ - hdr.e_flags = ELF_FLAGS; /* flags */ - hdr.e_ehsize = sizeof(elf32_hdr_t); /* header size */ - hdr.e_phentsize = sizeof(elf32_phdr_t); /* program header size */ - hdr.e_phnum = phnum; /* number of program headers */ - hdr.e_shentsize = sizeof(elf32_shdr_t); /* section header size */ - hdr.e_shnum = shnum; /* number of section headers */ - hdr.e_shstrndx = shstrndx; /* section index with names */ - elf_write_blk(elf_header, &hdr, sizeof(elf32_hdr_t)); +#endif + /* ELF64 carries no section table; ELF32 includes its generated table. */ + elf_write_file_header( + ELF_IS_64, + ELF_IS_64 ? elf_code_start + elf_entry_offset : elf_code_start, phnum, + shoff, shnum, shstrndx); +} + +static void elf_write_program_header(int type, + int flags, + int offset, + int address, + int file_size, + int memory_size, + int alignment) +{ +#if ELF_IS_64 == 1 + elf_write_int(elf_program_header, type); + elf_write_int(elf_program_header, flags); + elf_write_quad(elf_program_header, offset); + elf_write_quad(elf_program_header, address); + elf_write_quad(elf_program_header, address); + elf_write_quad(elf_program_header, file_size); + elf_write_quad(elf_program_header, memory_size); + elf_write_quad(elf_program_header, alignment); +#else + elf32_phdr_t phdr = {type, offset, address, address, + file_size, memory_size, flags, alignment}; + elf_write_blk(elf_program_header, &phdr, sizeof(phdr)); #endif } void elf_generate_program_headers(void) { const strbuf_t *elf_relplt = elf_relplt_buf(); + int ro_size; + int data_file_ofs; + int rw_vaddr; + int rw_filesz; + int rw_memsz; + int dyn_extra = 0; + if (!elf_program_header || !elf_code || !elf_data || !elf_rodata || - (dynlink && - (!dynamic_sections.elf_interp || !elf_relplt || - !dynamic_sections.elf_plt || !dynamic_sections.elf_got || - !dynamic_sections.elf_dynstr || !dynamic_sections.elf_dynsym || - !dynamic_sections.elf_dynamic))) { + !elf_dynamic_sections_ready(elf_relplt)) { fatal("ELF section buffers not initialized"); - return; } -#if ELF_IS_64 == 1 - /* Two ELF64 PT_LOAD segments, 56 bytes each. Field order differs from - * ELF32: p_flags sits immediately after p_type rather than before p_align. - */ - int ro_size = elf_header_len + elf_code->size + elf_rodata->size; - if (dynlink) - ro_size += elf_relplt->size + dynamic_sections.elf_plt->size; - - /* read-only, executable segment: headers + .text + .rodata */ - elf_write_int(elf_program_header, 1); /* p_type = PT_LOAD */ - elf_write_int(elf_program_header, 5); /* p_flags = R|X */ - elf_write_quad(elf_program_header, 0); /* p_offset */ - elf_write_quad(elf_program_header, ELF_START); /* p_vaddr */ - elf_write_quad(elf_program_header, ELF_START); /* p_paddr */ - elf_write_quad(elf_program_header, ro_size); /* p_filesz */ - elf_write_quad(elf_program_header, ro_size); /* p_memsz */ - elf_write_quad(elf_program_header, PAGESIZE); /* p_align */ - - /* read-write segment. Statically linked it holds .data (plus .bss, which - * occupies no file space) and starts at the next page boundary so that - * p_vaddr === p_offset (mod p_align), which the kernel enforces. - * Dynamically linked it begins at .interp and covers everything the loader - * needs, which elf_preprocess() has already placed a page clear of the - * read-only segment. - */ - int data_file_ofs = ALIGN_UP(ro_size, PAGESIZE); - int rw_vaddr = elf_data_start; - int rw_filesz = elf_data->size; - int rw_memsz = elf_data->size + elf_bss_size; - + ro_size = elf_header_len + elf_code->size + elf_rodata->size; if (dynlink) { - int dyn_extra = dynamic_sections.elf_interp->size + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size + - dynamic_sections.elf_dynamic->size; - data_file_ofs = ro_size; - rw_vaddr = dynamic_sections.elf_interp_start; - rw_filesz += dyn_extra; - rw_memsz += dyn_extra; + ro_size += elf_relplt->size + dynamic_sections.elf_plt->size; + dyn_extra = dynamic_sections.elf_interp->size + + dynamic_sections.elf_got->size + + dynamic_sections.elf_dynstr->size + + dynamic_sections.elf_dynsym->size + + dynamic_sections.elf_dynamic->size; } + data_file_ofs = dynlink ? ro_size : ALIGN_UP(ro_size, PAGESIZE); + rw_vaddr = dynlink ? dynamic_sections.elf_interp_start : elf_data_start; + rw_filesz = elf_data->size + dyn_extra; + rw_memsz = elf_data->size + elf_bss_size + dyn_extra; - elf_write_int(elf_program_header, 1); /* p_type = PT_LOAD */ - elf_write_int(elf_program_header, 6); /* p_flags = R|W */ - elf_write_quad(elf_program_header, data_file_ofs); /* p_offset */ - elf_write_quad(elf_program_header, rw_vaddr); - elf_write_quad(elf_program_header, rw_vaddr); - elf_write_quad(elf_program_header, rw_filesz); - elf_write_quad(elf_program_header, rw_memsz); - elf_write_quad(elf_program_header, PAGESIZE); + elf_write_program_header(1, 5, 0, ELF_START, ro_size, ro_size, PAGESIZE); + elf_write_program_header(1, 6, data_file_ofs, rw_vaddr, rw_filesz, rw_memsz, + PAGESIZE); if (dynlink) { - /* PT_INTERP: names the dynamic loader. */ - elf_write_int(elf_program_header, 3); /* p_type = PT_INTERP */ - elf_write_int(elf_program_header, 4); /* p_flags = R */ - elf_write_quad(elf_program_header, ro_size); - elf_write_quad(elf_program_header, dynamic_sections.elf_interp_start); - elf_write_quad(elf_program_header, dynamic_sections.elf_interp_start); - elf_write_quad(elf_program_header, strlen(DYN_LINKER) + 1); - elf_write_quad(elf_program_header, strlen(DYN_LINKER) + 1); - elf_write_quad(elf_program_header, 1); - - /* PT_DYNAMIC: the table the loader walks. */ + int interp_size = strlen(DYN_LINKER) + 1; int dynamic_ofs = ro_size + dynamic_sections.elf_interp->size + dynamic_sections.elf_got->size + dynamic_sections.elf_dynstr->size + dynamic_sections.elf_dynsym->size; - int dynamic_addr = elf_dynamic_start(); - elf_write_int(elf_program_header, 2); /* p_type = PT_DYNAMIC */ - elf_write_int(elf_program_header, 6); /* p_flags = R|W */ - elf_write_quad(elf_program_header, dynamic_ofs); - elf_write_quad(elf_program_header, dynamic_addr); - elf_write_quad(elf_program_header, dynamic_addr); - elf_write_quad(elf_program_header, dynamic_sections.elf_dynamic->size); - elf_write_quad(elf_program_header, dynamic_sections.elf_dynamic->size); - elf_write_quad(elf_program_header, 8); - } -#else - elf32_phdr_t phdr; - /* Explain the meaning of each field in the ELF32 program header. - * - * | Program | | - * & | Header bytes | Explanation | - * ---+----------------+-------------------------------------------------+ - * 34 | 01 00 00 00 | p_type: Segment type; 1 -> loadable. | - * | 54 00 00 00 | p_offset: Offset of segment in the file. | - * | 54 00 01 00 | p_vaddr: Virtual address of loaded segment. | - * | 54 00 01 00 | p_paddr: Only used on systems where physical | - * | | address is relevant. | - * | 48 8a 03 00 | p_filesz: Size of the segment in the file image.| - * | 48 8a 03 00 | p_memsz: Size of the segment in memory. | - * | | This value should be greater than or | - * | | equal to p_filesz. | - * | 07 00 00 00 | p_flags: Segment-wise permissions; | - * | | 0x1 -> execute, 0x2 -> write, | - * | | 0x4 -> read | - * | 04 00 00 00 | p_align: Align segment to the specified value. | - * ---+----------------+-------------------------------------------------+ - * 54 | | | - */ - /* program header - read-only segment */ - phdr.p_type = 1; /* PT_LOAD */ - phdr.p_offset = 0; /* offset of segment */ - phdr.p_vaddr = ELF_START; /* virtual address */ - phdr.p_paddr = ELF_START; /* physical address */ - phdr.p_filesz = - elf_header_len + elf_code->size + elf_rodata->size; /* size in file */ - phdr.p_memsz = - elf_header_len + elf_code->size + elf_rodata->size; /* size in memory */ - phdr.p_flags = 5; /* flags */ - phdr.p_align = PAGESIZE; /* alignment */ - if (dynlink) { - phdr.p_filesz += elf_relplt->size + dynamic_sections.elf_plt->size; - phdr.p_memsz += elf_relplt->size + dynamic_sections.elf_plt->size; - } - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); - - /* program header - readable and writable segment */ - phdr.p_type = 1; /* PT_LOAD */ - phdr.p_offset = elf_header_len + elf_code->size + - elf_rodata->size; /* offset of segment */ - if (!dynlink) - phdr.p_offset = ALIGN_UP(phdr.p_offset, PAGESIZE); - phdr.p_vaddr = elf_data_start; /* virtual address */ - phdr.p_paddr = elf_data_start; /* physical address */ - phdr.p_filesz = elf_data->size; /* size in file */ - phdr.p_memsz = elf_data->size + elf_bss_size; /* size in memory */ - phdr.p_flags = 6; /* flags */ - phdr.p_align = PAGESIZE; /* alignment */ - if (dynlink) { - phdr.p_offset += elf_relplt->size + dynamic_sections.elf_plt->size; - phdr.p_vaddr = dynamic_sections.elf_interp_start; - phdr.p_paddr = dynamic_sections.elf_interp_start; - phdr.p_filesz += dynamic_sections.elf_interp->size + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size + - dynamic_sections.elf_dynamic->size; - phdr.p_memsz += dynamic_sections.elf_interp->size + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size + - dynamic_sections.elf_dynamic->size; - } - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); - - - if (dynlink) { - /* program header - program interpreter (.interp section) */ - phdr.p_type = 3; /* PT_INTERP */ - phdr.p_offset = elf_header_len + elf_code->size + elf_rodata->size + - elf_relplt->size + - dynamic_sections.elf_plt->size; /* offset of segment */ - phdr.p_vaddr = dynamic_sections.elf_interp_start; /* virtual address */ - phdr.p_paddr = dynamic_sections.elf_interp_start; /* physical address */ - phdr.p_filesz = strlen(DYN_LINKER) + 1; /* size in file */ - phdr.p_memsz = strlen(DYN_LINKER) + 1; /* size in memory */ - phdr.p_flags = 4; /* flags */ - phdr.p_align = 1; /* alignment */ - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); - - /* program header - .dynamic section */ - phdr.p_type = 2; /* PT_DYNAMIC */ - phdr.p_offset = - elf_header_len + elf_code->size + elf_rodata->size + - elf_relplt->size + dynamic_sections.elf_plt->size + - dynamic_sections.elf_interp->size + dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; /* offset of segment */ - phdr.p_vaddr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; /* virtual address */ - phdr.p_paddr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; /* physical address */ - phdr.p_filesz = dynamic_sections.elf_dynamic->size; /* size in file */ - phdr.p_memsz = dynamic_sections.elf_dynamic->size; /* size in memory */ - phdr.p_flags = 6; /* flags */ - phdr.p_align = 4; /* alignment */ - elf_write_blk(elf_program_header, &phdr, sizeof(elf32_phdr_t)); + elf_write_program_header(3, 4, ro_size, + dynamic_sections.elf_interp_start, interp_size, + interp_size, 1); + elf_write_program_header(2, 6, dynamic_ofs, elf_dynamic_start(), + dynamic_sections.elf_dynamic->size, + dynamic_sections.elf_dynamic->size, + ELF_IS_64 ? 8 : 4); } -#endif } void elf_generate_section_headers(void) @@ -596,19 +464,23 @@ void elf_generate_section_headers(void) /* Check for null pointers to prevent crashes */ if (!elf_section_header || !elf_code || !elf_data || !elf_rodata || !elf_symtab || !elf_strtab || !elf_shstrtab || - (dynlink && - (!dynamic_sections.elf_interp || !elf_relplt || - !dynamic_sections.elf_plt || !dynamic_sections.elf_got || - !dynamic_sections.elf_dynstr || !dynamic_sections.elf_dynsym || - !dynamic_sections.elf_dynamic))) { + !elf_dynamic_sections_ready(elf_relplt)) { fatal("ELF section buffers not initialized"); - return; } /* section header table */ elf32_shdr_t shdr; int ofs = elf_header_len, sh_name = 0; +#define WRITE_FILE_SECTION(name, type, flags, addr, size, ...) \ + do { \ + elf32_shdr_t file_section = {sh_name, type, flags, addr, \ + ofs, size, __VA_ARGS__}; \ + elf_write_section_header(&file_section); \ + ofs += file_section.sh_size; \ + sh_name += sizeof(name); \ + } while (0) + /* The following table uses the text section header as an example to explain * the ELF32 section header. * @@ -636,48 +508,17 @@ void elf_generate_section_headers(void) * | | | */ /* NULL section */ - shdr.sh_name = sh_name; - shdr.sh_type = 0; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = 0; - shdr.sh_size = 0; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 0; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + elf_write_section_header(&shdr); sh_name += 1; /* .text */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 7; - shdr.sh_addr = elf_code_start; - shdr.sh_offset = ofs; - shdr.sh_size = elf_code->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_code->size; - sh_name += strlen(".text") + 1; + WRITE_FILE_SECTION(".text", 1, 7, elf_code_start, elf_code->size, 0, 0, 4, + 0); /* .rodata */ - shdr.sh_name = sh_name; /* Offset in shstrtab for ".rodata" */ - shdr.sh_type = 1; /* SHT_PROGBITS */ - shdr.sh_flags = 2; /* SHF_ALLOC only (read-only) */ - shdr.sh_addr = elf_rodata_start; - shdr.sh_offset = ofs; - shdr.sh_size = elf_rodata->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_rodata->size; - sh_name += strlen(".rodata") + 1; + WRITE_FILE_SECTION(".rodata", 1, 2, elf_rodata_start, elf_rodata->size, 0, + 0, 4, 0); if (dynlink) { /* .rel.plt or .rela.plt */ @@ -698,193 +539,79 @@ void elf_generate_section_headers(void) __ofs = dynamic_sections.elf_relplt->size; __sh_name = strlen(".rel.plt") + 1; } - shdr.sh_name = sh_name; - shdr.sh_type = sh_type; - shdr.sh_flags = 0x42; /* 0x40 | SHF_ALLOC */ - shdr.sh_addr = sh_addr; - shdr.sh_offset = ofs; - shdr.sh_size = sh_size; - shdr.sh_link = 8; /* The section header index of .dynsym. */ - shdr.sh_info = 6; /* The section header index of .got. */ - shdr.sh_addralign = 4; - shdr.sh_entsize = sh_entsize; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, sh_type, 0x42, sh_addr, ofs, + sh_size, 8, 6, 4, sh_entsize}; + elf_write_section_header(&shdr); ofs += __ofs; sh_name += __sh_name; /* .plt */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 0x6; - shdr.sh_addr = dynamic_sections.elf_plt_start; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_plt->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 4; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_plt->size; - sh_name += strlen(".plt") + 1; + WRITE_FILE_SECTION(".plt", 1, 0x6, dynamic_sections.elf_plt_start, + dynamic_sections.elf_plt->size, 0, 0, 4, 4); /* .interp */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 0x2; - shdr.sh_addr = dynamic_sections.elf_interp_start; - shdr.sh_offset = ofs; - shdr.sh_size = strlen(DYN_LINKER) + 1; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, 1, + 0x2, dynamic_sections.elf_interp_start, + ofs, strlen(DYN_LINKER) + 1, + 0, 0, + 1, 0}; + elf_write_section_header(&shdr); ofs += dynamic_sections.elf_interp->size; sh_name += strlen(".interp") + 1; /* .got */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 0x3; - shdr.sh_addr = dynamic_sections.elf_got_start; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_got->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = PTR_SIZE; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_got->size; - sh_name += strlen(".got") + 1; + WRITE_FILE_SECTION(".got", 1, 0x3, dynamic_sections.elf_got_start, + dynamic_sections.elf_got->size, 0, 0, 4, PTR_SIZE); /* .dynstr */ - shdr.sh_name = sh_name; - shdr.sh_type = 3; - shdr.sh_flags = 0x2; - shdr.sh_addr = - dynamic_sections.elf_got_start + dynamic_sections.elf_got->size; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_dynstr->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_dynstr->size; - sh_name += strlen(".dynstr") + 1; + WRITE_FILE_SECTION( + ".dynstr", 3, 0x2, + dynamic_sections.elf_got_start + dynamic_sections.elf_got->size, + dynamic_sections.elf_dynstr->size, 0, 0, 1, 0); /* .dynsym */ - shdr.sh_name = sh_name; - shdr.sh_type = 11; - shdr.sh_flags = 0x2; - shdr.sh_addr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_dynsym->size; - shdr.sh_link = 7; /* The section header index of .dynstr. */ - shdr.sh_info = 1; /* The index of the first non-local symbol. */ - shdr.sh_addralign = 4; - shdr.sh_entsize = sizeof(elf32_sym_t); - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_dynsym->size; - sh_name += strlen(".dynsym") + 1; + WRITE_FILE_SECTION( + ".dynsym", 11, 0x2, + dynamic_sections.elf_got_start + dynamic_sections.elf_got->size + + dynamic_sections.elf_dynstr->size, + dynamic_sections.elf_dynsym->size, 7, 1, 4, sizeof(elf32_sym_t)); /* .dynamic */ - shdr.sh_name = sh_name; - shdr.sh_type = 6; - shdr.sh_flags = 0x3; - shdr.sh_addr = dynamic_sections.elf_got_start + - dynamic_sections.elf_got->size + - dynamic_sections.elf_dynstr->size + - dynamic_sections.elf_dynsym->size; - shdr.sh_offset = ofs; - shdr.sh_size = dynamic_sections.elf_dynamic->size; - shdr.sh_link = 7; /* The section header index of .dynstr. */ - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += dynamic_sections.elf_dynamic->size; - sh_name += strlen(".dynamic") + 1; + WRITE_FILE_SECTION(".dynamic", 6, 0x3, + dynamic_sections.elf_got_start + + dynamic_sections.elf_got->size + + dynamic_sections.elf_dynstr->size + + dynamic_sections.elf_dynsym->size, + dynamic_sections.elf_dynamic->size, 7, 0, 4, 0); } if (!dynlink) ofs = ALIGN_UP(ofs, PAGESIZE); /* .data */ - shdr.sh_name = sh_name; - shdr.sh_type = 1; - shdr.sh_flags = 3; - shdr.sh_addr = elf_data_start; - shdr.sh_offset = ofs; - shdr.sh_size = elf_data->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_data->size; - sh_name += strlen(".data") + 1; + WRITE_FILE_SECTION(".data", 1, 3, elf_data_start, elf_data->size, 0, 0, 4, + 0); /* .bss */ - shdr.sh_name = sh_name; /* Offset in shstrtab for ".bss" */ - shdr.sh_type = 8; /* SHT_NOBITS */ - shdr.sh_flags = 3; /* SHF_ALLOC | SHF_WRITE */ - shdr.sh_addr = elf_bss_start; - shdr.sh_offset = ofs; /* File offset (not actually used for NOBITS) */ - shdr.sh_size = elf_bss_size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 4; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = (elf32_shdr_t) {sh_name, 8, 3, elf_bss_start, ofs, elf_bss_size, 0, + 0, 4, 0}; + elf_write_section_header(&shdr); sh_name += strlen(".bss") + 1; /* Note: .bss is not written to file (SHT_NOBITS) */ /* .symtab */ - shdr.sh_name = sh_name; - shdr.sh_type = 2; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = ofs; - shdr.sh_size = elf_symtab->size; - shdr.sh_link = dynlink ? 13 : 6; /* Link to .strtab */ - shdr.sh_info = elf_symbol_index; - shdr.sh_addralign = 4; - shdr.sh_entsize = 16; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_symtab->size; - sh_name += strlen(".symtab") + 1; + WRITE_FILE_SECTION(".symtab", 2, 0, 0, elf_symtab->size, dynlink ? 13 : 6, + elf_symbol_index, 4, 16); /* .strtab */ - shdr.sh_name = sh_name; - shdr.sh_type = 3; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = ofs; - shdr.sh_size = elf_strtab->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); - ofs += elf_strtab->size; - sh_name += strlen(".strtab") + 1; + WRITE_FILE_SECTION(".strtab", 3, 0, 0, elf_strtab->size, 0, 0, 1, 0); /* .shstr */ - shdr.sh_name = sh_name; - shdr.sh_type = 3; - shdr.sh_flags = 0; - shdr.sh_addr = 0; - shdr.sh_offset = ofs; - shdr.sh_size = elf_shstrtab->size; - shdr.sh_link = 0; - shdr.sh_info = 0; - shdr.sh_addralign = 1; - shdr.sh_entsize = 0; - elf_write_blk(elf_section_header, &shdr, sizeof(elf32_shdr_t)); + shdr = + (elf32_shdr_t) {sh_name, 3, 0, 0, ofs, elf_shstrtab->size, 0, 0, 1, 0}; + elf_write_section_header(&shdr); sh_name += strlen(".shstrtab") + 1; +#undef WRITE_FILE_SECTION #endif } @@ -896,7 +623,6 @@ void elf_align_to(strbuf_t *elf_array, int boundary) /* Check for null pointers to prevent crashes */ if (!elf_array) { fatal("ELF buffers not initialized for alignment"); - return; } while (elf_array->size & (boundary - 1)) @@ -1138,14 +864,8 @@ void elf_reset_dynamic_sections(void) void elf_generate_sections(void) { const strbuf_t *elf_relplt = elf_relplt_buf(); - if (!elf_shstrtab || - (dynlink && - (!dynamic_sections.elf_interp || !elf_relplt || - !dynamic_sections.elf_plt || !dynamic_sections.elf_got || - !dynamic_sections.elf_dynstr || !dynamic_sections.elf_dynsym || - !dynamic_sections.elf_dynamic))) { + if (!elf_shstrtab || !elf_dynamic_sections_ready(elf_relplt)) { fatal("ELF section buffers not initialized"); - return; } if (dynlink) @@ -1193,7 +913,6 @@ void elf_add_symbol(const char *symbol, int pc) /* Check for null pointers to prevent crashes */ if (!symbol || !elf_symtab || !elf_strtab) { fatal("Invalid parameters for elf_add_symbol"); - return; } elf_write_int(elf_symtab, elf_strtab->size); From 6084daad54739b49c4cbe342fccfe3cc7a3eb63c Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 6 Oct 2026 05:03:14 +0800 Subject: [PATCH 04/20] Add standalone VIR core with verifier Introduce VIR, the value-oriented IR the compiler will be built on: typed values with reverse uses, block parameters instead of phi instructions, explicit ordered effects for memory and calls, and fixed-prototype call signatures. A verifier checks dominance and use-def consistency, dumps are deterministic, and SCCP, DCE, local CSE, GVN and LICM run on it. Nothing in the compiler uses it yet. Keeping the core standalone lets its invariants be tested by unit tests and a structural fuzzer before a frontend depends on them. --- Makefile | 12 +- src/vir.c | 5589 ++++++++++++++++++++++++++++++++++++++++++++++ src/vir.h | 613 +++++ tests/vir-core.c | 2701 ++++++++++++++++++++++ tests/vir-fuzz.c | 153 ++ 5 files changed, 9067 insertions(+), 1 deletion(-) create mode 100644 src/vir.c create mode 100644 src/vir.h create mode 100644 tests/vir-core.c create mode 100644 tests/vir-fuzz.c diff --git a/Makefile b/Makefile index 9fbc6972..d6015316 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,8 @@ config: .PHONY: $(STYLE_GOALS) -check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2 +check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2 \ + check-vir-core check-vir-fuzz # Run the complete check -- driver and ABI suites at stages 0 and 2 -- on every # backend, as CI does. Driver cases that need 64-bit values are gated on the @@ -255,6 +256,15 @@ check-abi-stage0: $(OUT)/$(STAGE0) check-abi-stage2: $(OUT)/$(STAGE2) tests/$(ARCH)-abi.sh 2 $(DYNLINK) $(BINDING); +.PHONY: check-vir-core check-vir-fuzz +check-vir-core: tests/vir-core.c src/vir.c src/vir.h + $(CC) $(CFLAGS) -Isrc tests/vir-core.c src/vir.c -o $(OUT)/vir-core + $(OUT)/vir-core + +check-vir-fuzz: tests/vir-fuzz.c src/vir.c src/vir.h + $(CC) $(CFLAGS) -Isrc tests/vir-fuzz.c src/vir.c -o $(OUT)/vir-fuzz + $(OUT)/vir-fuzz + # Both prerequisites are order-only, and both exist because "make -j" would # otherwise let a compile start beside the thing it reads. Selecting a target # replaces src/codegen.c with "ln -sf", which unlinks before it relinks, so a diff --git a/src/vir.c b/src/vir.c new file mode 100644 index 00000000..1e7f45db --- /dev/null +++ b/src/vir.c @@ -0,0 +1,5589 @@ +#include "vir.h" + +#include +#include +#include +#include + +typedef enum { + VIR_USE_EXACT, + VIR_USE_USER, + VIR_USE_EDGE, + VIR_USE_EFFECT, + VIR_USE_RETURN, + VIR_USE_BRANCH, +} vir_use_owner_t; + +typedef struct vir_dominance vir_dominance_t; + +static const struct { + const char *name; + unsigned char width; +} vir_types[] = { +#define VIR_TYPE_ITEM(name, spelling, width) {spelling, width}, + VIR_TYPE_LIST +#undef VIR_TYPE_ITEM +}; + +static bool vir_has_block(const vir_function_t *func, + const vir_block_t *needle); +static bool vir_block_open(const vir_function_t *func, + const vir_block_t *block); +static int vir_block_index(const vir_function_t *func, + const vir_block_t *needle); +static bool vir_has_value(const vir_function_t *func, + const vir_value_t *needle); +static bool vir_has_incoming_edge(const vir_block_t *block, + const vir_edge_t *needle); +static bool vir_has_outgoing_edge(const vir_block_t *block, + const vir_edge_t *needle); +static bool vir_has_effect(const vir_block_t *block, + const vir_effect_t *needle); +static bool vir_value_dominates(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_value_t *value, + const vir_block_t *use_block, + int use_position); +static void vir_remove_use(vir_value_t *operand, vir_use_t *needle); +static void vir_remove_matching_use(vir_value_t *value, + vir_use_owner_t kind, + const void *owner, + int operand); +static bool vir_value_dominates_effect(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_value_t *value, + const vir_effect_t *effect); +static void vir_effect_append(vir_block_t *block, vir_effect_t *effect); +static void vir_repair_value_positions(vir_block_t *block); +static vir_effect_t *vir_new_effect(vir_function_t *func, + vir_block_t *block, + vir_effect_kind_t kind); + +#define vir_remove_user_use(value, user, operand) \ + vir_remove_matching_use(value, VIR_USE_USER, user, operand) +#define vir_remove_return_use(value, block) \ + vir_remove_matching_use(value, VIR_USE_RETURN, block, -1) +#define vir_remove_branch_use(value, block) \ + vir_remove_matching_use(value, VIR_USE_BRANCH, block, -1) +#define vir_remove_effect_use(value, effect, operand) \ + vir_remove_matching_use(value, VIR_USE_EFFECT, effect, operand) + +static bool vir_use_matches(const vir_use_t *use, + vir_use_owner_t kind, + const void *owner, + int operand) +{ + bool matches; + + switch (kind) { + case VIR_USE_EXACT: + matches = use == owner; + break; + case VIR_USE_USER: + matches = use->user == owner; + break; + case VIR_USE_EDGE: + matches = use->edge == owner; + break; + case VIR_USE_EFFECT: + matches = use->effect == owner; + break; + case VIR_USE_RETURN: + matches = use->return_block == owner; + break; + case VIR_USE_BRANCH: + matches = use->branch_block == owner; + break; + default: + matches = false; + break; + } + return matches && (operand < 0 || use->operand == operand); +} + +int vir_integer_type_width(vir_type_t type) +{ + return (unsigned int) type < sizeof(vir_types) / sizeof(*vir_types) + ? vir_types[type].width + : 0; +} + +vir_type_t vir_integer_type_from_size(int bytes) +{ + for (vir_type_t type = VIR_TYPE_I8; type <= VIR_TYPE_I64; type++) + if (vir_integer_type_width(type) / 8 == bytes) + return type; + return VIR_TYPE_VOID; +} + +bool vir_type_is_scalar(vir_type_t type) +{ + return vir_integer_type_width(type) || type == VIR_TYPE_PTR; +} + +bool vir_call_shape_valid(const vir_effect_t *effect, int max_args) +{ + return effect && effect->kind == VIR_EFFECT_CALL && max_args >= 0 && + (effect->result_type == VIR_TYPE_VOID || + vir_type_is_scalar(effect->result_type)) && + (effect->result ? effect->result->type == effect->result_type + : effect->result_type == VIR_TYPE_VOID) && + effect->arg_count >= 0 && effect->arg_count <= max_args && + (!effect->arg_count || (effect->args && effect->arg_uses)); +} + +bool vir_function_shape(const vir_function_t *func, vir_function_shape_t *shape) +{ + if (!func || !shape) + return false; + memset(shape, 0, sizeof(*shape)); + for (const vir_block_t *block = func->blocks; block; block = block->next) { + shape->has_effects |= block->effects != NULL; + for (const vir_edge_t *edge = block->outgoing; edge; + edge = edge->next_outgoing) { + if (edge->arg_count < 0) + return false; + if (edge->arg_count > shape->max_edge_args) + shape->max_edge_args = edge->arg_count; + } + for (const vir_effect_t *effect = block->effects; effect; + effect = effect->next) + if (effect->kind == VIR_EFFECT_CALL) { + if (effect->arg_count < 0) + return false; + shape->has_calls = true; + if (effect->arg_count > shape->max_call_args) + shape->max_call_args = effect->arg_count; + } + for (const vir_value_t *value = block->head; value; value = value->next) + if (value->opcode == VIR_OP_STACK_ADDR) { + if (value->address_slot >= INT_MAX) + return false; + if (shape->root_count <= (int) value->address_slot) + shape->root_count = value->address_slot + 1; + } else if (value->opcode == VIR_OP_GLOBAL_ADDR) { + shape->has_globals = true; + } + } + return true; +} + +static bool vir_opt_level_valid(vir_opt_level_t level) +{ + return level >= VIR_OPT_O0 && level <= VIR_OPT_O2; +} + +static int vir_type_width(const vir_function_t *func, vir_type_t type) +{ + int width = vir_integer_type_width(type); + + if (width) + return width; + return type == VIR_TYPE_PTR ? func->pointer_bits : 0; +} + +static unsigned long long vir_width_mask(int width) +{ + return width == 64 ? ~0ULL : ((1ULL << width) - 1ULL); +} + +static unsigned long long vir_fold_unary(vir_opcode_t opcode, + int source_width, + unsigned long long value) +{ + unsigned long long mask = vir_width_mask(source_width); + + if (opcode == VIR_OP_NEG) + return (0ULL - value) & mask; + if (opcode == VIR_OP_BITNOT) + return ~value & mask; + if (opcode != VIR_OP_SEXT) + return value; + value &= mask; + if (source_width < 64 && (value & (1ULL << (source_width - 1)))) + value |= ~mask; + return value; +} + +static unsigned long long vir_fold_integer_cast(vir_opcode_t opcode, + int source_width, + int destination_width, + unsigned long long value) +{ + if (opcode == VIR_OP_SEXT) + return vir_fold_unary(opcode, source_width, value) & + vir_width_mask(destination_width); + return value & vir_width_mask(opcode == VIR_OP_TRUNC ? destination_width + : source_width); +} + +static bool vir_fold_signed_div(int width, + unsigned long long left, + unsigned long long right, + bool remainder, + unsigned long long *result) +{ + unsigned long long mask = vir_width_mask(width); + unsigned long long sign = 1ULL << (width - 1); + unsigned long long left_bits = left & mask; + unsigned long long right_bits = right & mask; + bool left_negative = (left_bits & sign) != 0; + bool right_negative = (right_bits & sign) != 0; + unsigned long long left_magnitude; + unsigned long long right_magnitude; + unsigned long long magnitude; + bool result_negative; + + if (!right_bits || (left_bits == sign && right_bits == mask)) + return false; + left_magnitude = left_negative ? (0ULL - left_bits) & mask : left_bits; + right_magnitude = right_negative ? (0ULL - right_bits) & mask : right_bits; + magnitude = remainder ? left_magnitude % right_magnitude + : left_magnitude / right_magnitude; + result_negative = + remainder ? left_negative : left_negative != right_negative; + *result = result_negative ? 0ULL - magnitude : magnitude; + *result &= mask; + return true; +} + +static bool vir_fold_integer_shift(int width, + vir_opcode_t opcode, + unsigned long long value, + unsigned long long count, + unsigned long long *result) +{ + if (count >= (unsigned int) width) + return false; + *result = opcode == VIR_OP_SHL ? value << count : value >> count; + if (opcode == VIR_OP_ASHR && count && (value & (1ULL << (width - 1)))) { + unsigned long long mask = vir_width_mask(width); + + *result |= mask ^ (mask >> count); + } + return true; +} + +static bool vir_fold_binary(const vir_function_t *func, + vir_opcode_t opcode, + vir_type_t result_type, + vir_type_t operand_type, + unsigned long long left, + unsigned long long right, + unsigned long long *result) +{ + int width = vir_type_width(func, result_type); + int operand_width = vir_type_width(func, operand_type); + unsigned long long mask; + + if (!width || !operand_width) + return false; + mask = vir_width_mask(operand_width); + + switch (opcode) { + case VIR_OP_ADD: + *result = left + right; + break; + case VIR_OP_SUB: + *result = left - right; + break; + case VIR_OP_MUL: + *result = left * right; + break; + case VIR_OP_SDIV: + case VIR_OP_SREM: + if (!vir_fold_signed_div(operand_width, left, right, + opcode == VIR_OP_SREM, result)) + return false; + break; + case VIR_OP_UDIV: + case VIR_OP_UREM: + if (!right) + return false; + *result = opcode == VIR_OP_UDIV ? left / right : left % right; + break; + case VIR_OP_SHL: + case VIR_OP_ASHR: + case VIR_OP_LSHR: + if (!vir_fold_integer_shift(operand_width, opcode, left, right, result)) + return false; + break; + case VIR_OP_BITAND: + *result = left & right; + break; + case VIR_OP_BITOR: + *result = left | right; + break; + case VIR_OP_BITXOR: + *result = left ^ right; + break; + case VIR_OP_EQ: + *result = left == right; + break; + case VIR_OP_SLT: { + unsigned long long sign = 1ULL << (operand_width - 1); + + *result = ((left & mask) ^ sign) < ((right & mask) ^ sign); + break; + } + case VIR_OP_ULT: + *result = left < right; + break; + case VIR_OP_PTRADD: + *result = left + right; + break; + default: + return false; + } + *result &= vir_width_mask(width); + return true; +} + +bool vir_fold_i32_binary(vir_opcode_t opcode, int left, int right, int *result) +{ + unsigned long long folded; + + if (!vir_fold_binary(NULL, opcode, VIR_TYPE_I32, VIR_TYPE_I32, + (unsigned int) left, (unsigned int) right, &folded)) + return false; + *result = (int) (unsigned int) folded; + return true; +} + +static unsigned long long vir_integer_mask(vir_type_t type) +{ + int width = vir_integer_type_width(type); + return vir_width_mask(width); +} + +static bool vir_valid_alignment(unsigned int alignment) +{ + return alignment && !(alignment & (alignment - 1)); +} + +static const char *vir_type_name(vir_type_t type) +{ + return (unsigned int) type < sizeof(vir_types) / sizeof(*vir_types) + ? vir_types[type].name + : "void"; +} + +static const char *vir_opcode_name(vir_opcode_t opcode) +{ + switch (opcode) { +#define VIR_OPCODE_ITEM(name, spelling) \ + case VIR_OP_##name: \ + return spelling; + VIR_OPCODE_LIST +#undef VIR_OPCODE_ITEM + default: + return "invalid"; + } +} + +#define VIR_OPCODE_ITEM(name, spelling) case VIR_OP_##name: +#define DEFINE_VIR_OPCODE_PREDICATE(name, opcodes) \ + static bool name(vir_opcode_t opcode) \ + { \ + switch (opcode) { \ + opcodes return true; \ + default: \ + return false; \ + } \ + } +DEFINE_VIR_OPCODE_PREDICATE(vir_is_pure_unary_opcode, + VIR_PURE_UNARY_OPCODE_LIST) +DEFINE_VIR_OPCODE_PREDICATE(vir_is_pure_binary_opcode, + VIR_PURE_BINARY_OPCODE_LIST) +#undef DEFINE_VIR_OPCODE_PREDICATE +#undef VIR_OPCODE_ITEM + +static void *vir_arena_alloc(vir_arena_t *arena, int size) +{ + const int align = sizeof(void *); + vir_arena_block_t *block; + int capacity; + + if (size <= 0 || size > INT_MAX - align + 1 || arena->fail_after == 0) + return NULL; + if (arena->fail_after > 0) + arena->fail_after--; + size = (size + align - 1) & ~(align - 1); + block = arena->head; + if (!block || size > block->capacity - block->used) { + capacity = arena->block_size; + if (capacity < size) + capacity = size; + if (arena->capacity > INT_MAX - capacity) + return NULL; + block = malloc(sizeof(*block) + capacity - 1); + if (!block) + return NULL; + block->next = arena->head; + block->capacity = capacity; + block->used = 0; + arena->head = block; + arena->capacity += capacity; + if (arena->capacity > arena->peak_capacity) + arena->peak_capacity = arena->capacity; + } + + void *ptr = block->data + block->used; + block->used += size; + memset(ptr, 0, size); + return ptr; +} + +void vir_function_init(vir_function_t *func, int arena_block_size) +{ + memset(func, 0, sizeof(*func)); + func->arena.block_size = arena_block_size > 0 ? arena_block_size : 4096; + func->arena.fail_after = -1; +} + +int vir_function_set_pointer_bits(vir_function_t *func, int pointer_bits) +{ + if (!func || func->blocks || (pointer_bits != 32 && pointer_bits != 64)) + return 0; + func->pointer_bits = pointer_bits; + return 1; +} + +void vir_function_release(vir_function_t *func) +{ + vir_arena_block_t *block = func->arena.head; + while (block) { + vir_arena_block_t *next = block->next; + free(block); + block = next; + } + memset(func, 0, sizeof(*func)); +} + +static void vir_collect_value_stats(const vir_value_t *value, + vir_stats_t *stats) +{ + stats->values++; + for (const vir_use_t *use = value->uses; use; use = use->next) + stats->uses++; +} + +void vir_collect_stats(const vir_function_t *func, vir_stats_t *stats) +{ + const vir_block_t *block; + memset(stats, 0, sizeof(*stats)); + stats->arena_capacity = func->arena.capacity; + stats->arena_peak_capacity = func->arena.peak_capacity; + for (block = func->blocks; block; block = block->next) { + const vir_value_t *value; + const vir_edge_t *edge; + const vir_effect_t *effect; + stats->blocks++; + for (value = block->params; value; value = value->param_next) { + stats->params++; + vir_collect_value_stats(value, stats); + } + for (value = block->head; value; value = value->next) { + vir_collect_value_stats(value, stats); + } + for (edge = block->outgoing; edge; edge = edge->next_outgoing) + stats->edges++; + for (effect = block->effects; effect; effect = effect->next) { + stats->effects++; + stats->volatile_effects += + effect->kind == VIR_EFFECT_VOLATILE_LOAD || + effect->kind == VIR_EFFECT_VOLATILE_STORE; + } + } +} + +vir_block_t *vir_block_create(vir_function_t *func) +{ + vir_block_t *block = vir_arena_alloc(&func->arena, sizeof(*block)); + if (!block) + return NULL; + block->id = func->next_block_id++; + if (func->last_block) + func->last_block->next = block; + else + func->blocks = block; + func->last_block = block; + return block; +} + +static vir_value_t *vir_value_alloc(vir_function_t *func, + vir_block_t *block, + vir_type_t type) +{ + vir_value_t *value; + + if (!vir_has_block(func, block) || !vir_type_width(func, type)) + return NULL; + value = vir_arena_alloc(&func->arena, sizeof(*value)); + if (!value) + return NULL; + value->id = func->next_value_id++; + value->type = type; + value->block = block; + return value; +} + +static vir_value_t *vir_new_value(vir_function_t *func, + vir_block_t *block, + vir_opcode_t opcode, + vir_type_t type, + int nr_ops) +{ + vir_value_t *value = vir_value_alloc(func, block, type); + + if (!value) + return NULL; + value->opcode = opcode; + value->nr_ops = nr_ops; + value->position = block->tail ? block->tail->position + 1 : 0; + value->order = block->next_order++; + if (block->tail) + block->tail->next = value; + else + block->head = value; + block->tail = value; + return value; +} + +static vir_value_t *vir_const(vir_function_t *func, + vir_block_t *block, + vir_type_t type, + unsigned long long constant) +{ + vir_value_t *value = vir_new_value(func, block, VIR_OP_CONST, type, 0); + if (value) + value->constant = constant & vir_width_mask(vir_type_width(func, type)); + return value; +} + +vir_value_t *vir_const_i32(vir_function_t *func, + vir_block_t *block, + int constant) +{ + return vir_const(func, block, VIR_TYPE_I32, + (unsigned long long) (unsigned int) constant); +} + +vir_value_t *vir_const_i1(vir_function_t *func, + vir_block_t *block, + int constant) +{ + return vir_const(func, block, VIR_TYPE_I1, !!constant); +} + +vir_value_t *vir_const_int(vir_function_t *func, + vir_block_t *block, + vir_type_t type, + unsigned long long bits) +{ + return vir_const(func, block, type, bits); +} + +vir_value_t *vir_const_ptr(vir_function_t *func, + vir_block_t *block, + unsigned long long bits) +{ + return vir_const(func, block, VIR_TYPE_PTR, bits); +} + +static bool vir_address_root_conflicts(const vir_function_t *func, + vir_address_kind_t kind, + unsigned int slot, + const char *name, + unsigned int size, + unsigned int alignment) +{ + const vir_block_t *block; + + for (block = func->blocks; block; block = block->next) { + const vir_value_t *value; + for (value = block->head; value; value = value->next) { + bool same_root = kind == VIR_ADDRESS_STACK + ? value->opcode == VIR_OP_STACK_ADDR && + value->address_slot == slot + : value->opcode == VIR_OP_GLOBAL_ADDR && + value->address_name && + !strcmp(value->address_name, name); + + if (same_root && (value->address_size != size || + value->address_alignment != alignment)) + return 1; + } + } + return 0; +} + +static vir_value_t *new_addr(vir_function_t *func, + vir_block_t *block, + vir_address_kind_t kind, + unsigned int slot, + const char *name, + unsigned int size, + unsigned int alignment) +{ + char *copy = NULL; + vir_value_t *value; + + if (!vir_has_block(func, block) || !vir_type_width(func, VIR_TYPE_PTR) || + !size || !vir_valid_alignment(alignment) || + (kind == VIR_ADDRESS_GLOBAL && (!name || !*name)) || + vir_address_root_conflicts(func, kind, slot, name, size, alignment)) + return NULL; + if (kind == VIR_ADDRESS_GLOBAL) { + size_t length = strlen(name) + 1; + + if (length > (size_t) 0x7fffffff || + !(copy = vir_arena_alloc(&func->arena, (int) length))) + return NULL; + memcpy(copy, name, length); + } + value = vir_new_value( + func, block, + kind == VIR_ADDRESS_STACK ? VIR_OP_STACK_ADDR : VIR_OP_GLOBAL_ADDR, + VIR_TYPE_PTR, 0); + if (!value) + return NULL; + value->address_kind = kind; + value->address_slot = slot; + value->address_size = size; + value->address_alignment = alignment; + value->address_name = copy; + return value; +} + +vir_value_t *vir_stack_addr(vir_function_t *func, + vir_block_t *block, + unsigned int slot, + unsigned int size, + unsigned int alignment) +{ + return new_addr(func, block, VIR_ADDRESS_STACK, slot, NULL, size, + alignment); +} + +vir_value_t *vir_global_addr(vir_function_t *func, + vir_block_t *block, + const char *name, + unsigned int size, + unsigned int alignment) +{ + return new_addr(func, block, VIR_ADDRESS_GLOBAL, 0, name, size, alignment); +} + +vir_value_t *vir_function_addr(vir_function_t *func, + vir_block_t *block, + const char *name) +{ + vir_value_t *value; + char *copy; + size_t length; + + if (!vir_has_block(func, block) || !name || !*name) + return NULL; + length = strlen(name) + 1; + if (length > INT_MAX || + !(copy = vir_arena_alloc(&func->arena, (int) length))) + return NULL; + memcpy(copy, name, length); + value = vir_new_value(func, block, VIR_OP_FUNC_ADDR, VIR_TYPE_PTR, 0); + if (value) + value->address_name = copy; + return value; +} + +vir_value_t *vir_rodata_addr(vir_function_t *func, + vir_block_t *block, + unsigned int offset) +{ + vir_value_t *value; + + if (!vir_has_block(func, block) || !vir_type_width(func, VIR_TYPE_PTR)) + return NULL; + value = vir_new_value(func, block, VIR_OP_RODATA_ADDR, VIR_TYPE_PTR, 0); + if (value) + value->constant = offset; + return value; +} + +static void vir_block_append_param(vir_block_t *block, vir_value_t *param) +{ + param->is_block_param = 1; + param->opcode = VIR_OP_PARAM; + param->block = block; + if (block->last_param) + block->last_param->param_next = param; + else + block->params = param; + block->last_param = param; + block->param_count++; +} + +vir_value_t *vir_block_add_param(vir_function_t *func, + vir_block_t *block, + vir_type_t type) +{ + vir_value_t *value = vir_value_alloc(func, block, type); + + if (!value) + return NULL; + + /* Its own opcode, so no "opcode == VIR_OP_CONST" test can mistake a block + * parameter, whose constant field is zero, for the constant 0. + */ + vir_block_append_param(block, value); + return value; +} + +static bool vir_values_owned(const vir_function_t *func, + vir_value_t *const *values, + int count) +{ + if (!func || count < 0 || (count && !values)) + return false; + for (int i = 0; i < count; i++) + if (!values[i] || !vir_has_value(func, values[i])) + return false; + return true; +} + +static bool vir_edge_args_match(const vir_function_t *func, + vir_block_t *to, + vir_value_t **args, + int arg_count, + bool check_ownership) +{ + vir_value_t *param; + int i = 0; + + if (!to || !vir_has_block(func, to) || arg_count < 0 || + arg_count != to->param_count || (arg_count && !args) || + (check_ownership && !vir_values_owned(func, args, arg_count))) + return 0; + param = to->params; + for (i = 0; i < arg_count; i++) { + if (!args[i] || !param || args[i]->type != param->type) + return 0; + param = param->param_next; + } + return 1; +} + +static vir_edge_t *vir_edge_alloc(vir_function_t *func, int arg_count) +{ + vir_edge_t *edge = vir_arena_alloc(&func->arena, sizeof(*edge)); + + if (!edge) + return NULL; + if (arg_count) { + edge->args = + vir_arena_alloc(&func->arena, sizeof(*edge->args) * arg_count); + edge->arg_uses = + vir_arena_alloc(&func->arena, sizeof(*edge->arg_uses) * arg_count); + if (!edge->args || !edge->arg_uses) + return NULL; + } + edge->arg_count = arg_count; + return edge; +} + +static vir_edge_t *vir_edge_prepare(vir_function_t *func, + vir_block_t *from, + vir_block_t *to, + vir_value_t **args, + int arg_count) +{ + vir_edge_t *edge; + + edge = vir_edge_alloc(func, arg_count); + if (!edge) + return NULL; + edge->from = from; + edge->to = to; + if (arg_count) + memcpy(edge->args, args, sizeof(*args) * arg_count); + return edge; +} + +static void vir_link_use(vir_value_t *value, vir_use_t *use) +{ + use->next = value->uses; + value->uses = use; +} + +static void vir_link_effect_use(vir_effect_t *effect, + vir_value_t *value, + vir_use_t *use, + int operand) +{ + use->effect = effect; + use->operand = operand; + vir_link_use(value, use); +} + +static void vir_edge_link_uses(vir_edge_t *edge) +{ + for (int i = 0; i < edge->arg_count; i++) { + vir_use_t *use = &edge->arg_uses[i]; + use->edge = edge; + use->operand = i; + vir_link_use(edge->args[i], use); + } +} + +static void vir_edge_replace_args(vir_edge_t *edge, + vir_value_t **args, + vir_use_t *uses, + int count) +{ + int i; + + for (i = 0; i < edge->arg_count; i++) + vir_remove_use(edge->args[i], &edge->arg_uses[i]); + edge->args = args; + edge->arg_uses = uses; + edge->arg_count = count; + vir_edge_link_uses(edge); +} + +static void vir_edge_link(vir_edge_t *edge) +{ + edge->next_outgoing = edge->from->outgoing; + edge->from->outgoing = edge; + edge->next_incoming = edge->to->incoming; + edge->to->incoming = edge; + vir_edge_link_uses(edge); +} + +static void vir_edge_remove_from_list(vir_edge_t **head, + vir_edge_t *edge, + bool outgoing) +{ + vir_edge_t **link = head; + + while (*link != edge) + link = outgoing ? &(*link)->next_outgoing : &(*link)->next_incoming; + *link = outgoing ? edge->next_outgoing : edge->next_incoming; +} + +static bool vir_edge_is_linked(const vir_function_t *func, + const vir_edge_t *edge) +{ + return func && edge && vir_has_block(func, edge->from) && + vir_has_block(func, edge->to) && + vir_has_outgoing_edge(edge->from, edge) && + vir_has_incoming_edge(edge->to, edge); +} + +vir_edge_t *vir_edge_create(vir_function_t *func, + vir_block_t *from, + vir_block_t *to, + vir_value_t **args, + int arg_count) +{ + vir_edge_t *edge; + if (!vir_block_open(func, from) || + !vir_edge_args_match(func, to, args, arg_count, true)) + return NULL; + edge = vir_edge_prepare(func, from, to, args, arg_count); + if (edge) { + vir_edge_link(edge); + from->terminator = VIR_TERM_JUMP; + } + return edge; +} + +static void vir_edge_unlink(vir_edge_t *edge) +{ + int i; + + vir_edge_remove_from_list(&edge->from->outgoing, edge, true); + vir_edge_remove_from_list(&edge->to->incoming, edge, false); + for (i = 0; i < edge->arg_count; i++) + vir_remove_use(edge->args[i], &edge->arg_uses[i]); +} + +int vir_edge_delete(vir_function_t *func, vir_edge_t *edge) +{ + vir_block_t *from; + vir_edge_t *survivor = NULL; + + if (!vir_edge_is_linked(func, edge)) + return 0; + from = edge->from; + if (from->terminator == VIR_TERM_BRANCH) { + if (!from->true_edge || !from->false_edge || + (edge != from->true_edge && edge != from->false_edge)) + return 0; + survivor = edge == from->true_edge ? from->false_edge : from->true_edge; + vir_remove_branch_use(from->branch_condition, from); + } else if (from->terminator != VIR_TERM_JUMP || from->outgoing != edge || + edge->next_outgoing) { + return 0; + } + vir_edge_unlink(edge); + if (!survivor) { + from->terminator = VIR_TERM_NONE; + return 1; + } + /* Deleting one arm leaves the other edge as a jump. */ + from->terminator = VIR_TERM_JUMP; + from->branch_condition = NULL; + from->true_edge = NULL; + from->false_edge = NULL; + return from->outgoing == survivor && !survivor->next_outgoing; +} + +int vir_edge_redirect(vir_function_t *func, vir_edge_t *edge, vir_block_t *to) +{ + if (!vir_edge_is_linked(func, edge) || + !vir_edge_args_match(func, to, edge->args, edge->arg_count, false)) + return 0; + if (edge->to == to) + return 1; + vir_edge_remove_from_list(&edge->to->incoming, edge, false); + edge->to = to; + edge->next_incoming = to->incoming; + to->incoming = edge; + return 1; +} + +vir_block_t *vir_edge_split(vir_function_t *func, vir_edge_t *edge) +{ + vir_block_t *split; + vir_edge_t *forward; + vir_value_t *target_param; + int i; + + if (!vir_edge_is_linked(func, edge)) + return NULL; + split = vir_arena_alloc(&func->arena, sizeof(*split)); + if (!split) + return NULL; + target_param = edge->to->params; + for (i = 0; i < edge->arg_count; i++) { + vir_value_t *param = vir_arena_alloc(&func->arena, sizeof(*param)); + if (!param || !target_param) + return NULL; + param->type = target_param->type; + vir_block_append_param(split, param); + target_param = target_param->param_next; + } + if (target_param) + return NULL; + forward = vir_edge_alloc(func, edge->arg_count); + if (!forward) + return NULL; + split->id = func->next_block_id++; + if (func->last_block) + func->last_block->next = split; + else + func->blocks = split; + func->last_block = split; + for (target_param = split->params, i = 0; target_param; + target_param = target_param->param_next, i++) { + target_param->id = func->next_value_id++; + forward->args[i] = target_param; + } + forward->from = split; + forward->to = edge->to; + vir_edge_link(forward); + split->terminator = VIR_TERM_JUMP; + if (!vir_edge_redirect(func, edge, split)) + return NULL; + return split; +} + +int vir_block_set_branch(vir_function_t *func, + vir_block_t *block, + vir_value_t *condition, + const vir_edge_args_t *true_desc, + const vir_edge_args_t *false_desc) +{ + vir_edge_t *true_edge; + vir_edge_t *false_edge; + vir_use_t *condition_use; + if (!condition || !true_desc || !false_desc || + condition->type != VIR_TYPE_I1 || !vir_block_open(func, block)) + return 0; + if (!vir_has_value(func, condition) || + !vir_edge_args_match(func, true_desc->to, true_desc->args, + true_desc->arg_count, true) || + !vir_edge_args_match(func, false_desc->to, false_desc->args, + false_desc->arg_count, true)) + return 0; + true_edge = vir_edge_prepare(func, block, true_desc->to, true_desc->args, + true_desc->arg_count); + if (!true_edge) + return 0; + false_edge = vir_edge_prepare(func, block, false_desc->to, false_desc->args, + false_desc->arg_count); + if (!false_edge) + return 0; + condition_use = vir_arena_alloc(&func->arena, sizeof(*condition_use)); + if (!condition_use) + return 0; + vir_edge_link(true_edge); + vir_edge_link(false_edge); + block->terminator = VIR_TERM_BRANCH; + block->branch_condition = condition; + block->true_edge = true_edge; + block->false_edge = false_edge; + condition_use->branch_block = block; + vir_link_use(condition, condition_use); + return 1; +} + +int vir_block_set_return(vir_function_t *func, + vir_block_t *block, + vir_value_t *value) +{ + vir_use_t *use; + if (!vir_block_open(func, block) || (value && !vir_has_value(func, value))) + return 0; + /* A void function returns no value, so the return records no use. */ + if (!value) { + block->terminator = VIR_TERM_RETURN; + block->return_value = NULL; + return 1; + } + use = vir_arena_alloc(&func->arena, sizeof(*use)); + if (!use) + return 0; + block->terminator = VIR_TERM_RETURN; + block->return_value = value; + use->return_block = block; + vir_link_use(value, use); + return 1; +} + +vir_effect_t *vir_effect_create(vir_function_t *func, + vir_block_t *block, + vir_effect_kind_t kind) +{ + vir_effect_t *effect; + if (!vir_block_open(func, block) || + (kind != VIR_EFFECT_CALL && kind != VIR_EFFECT_VOLATILE)) + return NULL; + effect = vir_new_effect(func, block, kind); + if (!effect) + return NULL; + vir_effect_append(block, effect); + return effect; +} + +static void vir_effect_append(vir_block_t *block, vir_effect_t *effect) +{ + effect->position = + block->last_effect ? block->last_effect->position + 1 : 0; + effect->order = + effect->result ? effect->result->order : block->next_order++; + if (block->last_effect) + block->last_effect->next = effect; + else + block->effects = effect; + block->last_effect = effect; +} + +static vir_effect_t *vir_new_effect(vir_function_t *func, + vir_block_t *block, + vir_effect_kind_t kind) +{ + vir_effect_t *effect = vir_arena_alloc(&func->arena, sizeof(*effect)); + + if (effect) { + effect->kind = kind; + effect->block = block; + } + return effect; +} + +static void vir_link_effect_operand(vir_effect_t *effect, + vir_value_t **slot, + vir_use_t **use_slot, + vir_value_t *value, + vir_use_t *use, + int operand) +{ + *slot = value; + *use_slot = use; + vir_link_effect_use(effect, value, use, operand); +} + +static void vir_link_user_operand(vir_value_t *user, + vir_value_t *operand, + vir_use_t *use, + int index) +{ + use->user = user; + use->operand = index; + vir_link_use(operand, use); +} + +static vir_value_t *vir_user_operation(vir_function_t *func, + vir_block_t *block, + vir_opcode_t opcode, + vir_type_t type, + vir_value_t *op0, + vir_value_t *op1) +{ + int nr_ops = op1 ? 2 : 1; + vir_use_t *uses = vir_arena_alloc(&func->arena, sizeof(*uses) * nr_ops); + vir_value_t *value; + + if (!uses || !(value = vir_new_value(func, block, opcode, type, nr_ops))) + return NULL; + value->op0 = op0; + vir_link_user_operand(value, op0, &uses[0], 0); + if (op1) { + value->op1 = op1; + vir_link_user_operand(value, op1, &uses[1], 1); + } + return value; +} + +static vir_value_t *vir_load_impl(vir_function_t *func, + vir_block_t *block, + vir_value_t *address, + vir_type_t type, + bool is_volatile) +{ + vir_effect_t *effect; + vir_use_t *use; + vir_value_t *value; + + if (!vir_block_open(func, block) || + (address && + (!vir_has_value(func, address) || address->type != VIR_TYPE_PTR)) || + !vir_type_width(func, type)) + return NULL; + effect = vir_new_effect( + func, block, is_volatile ? VIR_EFFECT_VOLATILE_LOAD : VIR_EFFECT_LOAD); + use = address ? vir_arena_alloc(&func->arena, sizeof(*use)) : NULL; + if (!effect || (address && !use)) + return NULL; + value = vir_new_value(func, block, VIR_OP_LOAD, type, 0); + if (!value) + return NULL; + value->def_effect = effect; + effect->result = value; + if (address) + vir_link_effect_operand(effect, &effect->address, &effect->address_use, + address, use, 0); + vir_effect_append(block, effect); + return value; +} + +#define DEFINE_MEMORY_LOAD(name, is_volatile) \ + vir_value_t *name(vir_function_t *func, vir_block_t *block, \ + vir_value_t *address, vir_type_t type) \ + { \ + if (!address) \ + return NULL; \ + return vir_load_impl(func, block, address, type, is_volatile); \ + } + +static bool vir_store_link_operands(vir_function_t *func, + vir_effect_t *effect, + vir_value_t *address, + vir_value_t *value) +{ + vir_use_t *uses = vir_arena_alloc(&func->arena, sizeof(*uses) * 2); + + if (!uses) + return false; + vir_link_effect_operand(effect, &effect->address, &effect->address_use, + address, &uses[0], 0); + vir_link_effect_operand(effect, &effect->stored_value, + &effect->stored_value_use, value, &uses[1], 1); + return true; +} + +static vir_effect_t *vir_store_impl(vir_function_t *func, + vir_block_t *block, + vir_value_t *address, + vir_value_t *value, + bool is_volatile) +{ + vir_effect_t *effect; + + if (!vir_block_open(func, block) || (!!address != !!value) || + (address && + (!vir_has_value(func, address) || !vir_has_value(func, value) || + address->type != VIR_TYPE_PTR)) || + (value && !vir_type_width(func, value->type))) + return NULL; + effect = vir_new_effect( + func, block, + is_volatile ? VIR_EFFECT_VOLATILE_STORE : VIR_EFFECT_STORE); + if (!effect || + (address && !vir_store_link_operands(func, effect, address, value))) + return NULL; + vir_effect_append(block, effect); + return effect; +} + +#define DEFINE_MEMORY_STORE(name, is_volatile) \ + vir_effect_t *name(vir_function_t *func, vir_block_t *block, \ + vir_value_t *address, vir_value_t *value) \ + { \ + if (!address || !value) \ + return NULL; \ + return vir_store_impl(func, block, address, value, is_volatile); \ + } + +DEFINE_MEMORY_LOAD(vir_load, false) +DEFINE_MEMORY_LOAD(vir_volatile_load, true) +DEFINE_MEMORY_STORE(vir_store, false) +DEFINE_MEMORY_STORE(vir_volatile_store, true) + +#undef DEFINE_MEMORY_LOAD +#undef DEFINE_MEMORY_STORE + +vir_value_t *vir_deferred_load(vir_function_t *func, + vir_block_t *block, + vir_type_t type, + bool is_volatile) +{ + return vir_load_impl(func, block, NULL, type, is_volatile); +} + +vir_effect_t *vir_deferred_store(vir_function_t *func, + vir_block_t *block, + bool is_volatile) +{ + return vir_store_impl(func, block, NULL, NULL, is_volatile); +} + +bool vir_deferred_load_resolve(vir_function_t *func, + vir_effect_t *effect, + vir_value_t *address) +{ + vir_use_t *use; + + if (!func || !effect || !address || !effect->block || !effect->result || + !vir_effect_is_load(effect->kind) || effect->address || + effect->address_use || !vir_has_block(func, effect->block) || + !vir_has_value(func, address) || address->type != VIR_TYPE_PTR || + effect->result->def_effect != effect || + !vir_value_dominates_effect(func, NULL, address, effect)) + return false; + use = vir_arena_alloc(&func->arena, sizeof(*use)); + if (!use) + return false; + vir_link_effect_operand(effect, &effect->address, &effect->address_use, + address, use, 0); + return true; +} + +bool vir_deferred_store_resolve(vir_function_t *func, + vir_effect_t *effect, + vir_value_t *address, + vir_value_t *value) +{ + if (!func || !effect || !address || !value || !effect->block || + !vir_effect_is_store(effect->kind) || effect->address || + effect->stored_value || effect->address_use || + effect->stored_value_use || !vir_has_block(func, effect->block) || + !vir_has_value(func, address) || !vir_has_value(func, value) || + address->type != VIR_TYPE_PTR || !vir_type_width(func, value->type) || + !vir_value_dominates_effect(func, NULL, address, effect) || + !vir_value_dominates_effect(func, NULL, value, effect)) + return false; + return vir_store_link_operands(func, effect, address, value); +} + +static vir_effect_t *vir_call_impl(vir_function_t *func, + vir_block_t *block, + const char *callee, + vir_value_t *callee_value, + vir_value_t **args, + int arg_count, + vir_type_t result_type) +{ + char *name; + size_t length = 0; + vir_effect_t *effect; + vir_use_t *callee_use = NULL; + vir_value_t *result = NULL; + int i; + + if (!vir_block_open(func, block) || (!!callee == !!callee_value) || + (callee && !*callee) || + (callee_value && (callee_value->type != VIR_TYPE_PTR || + !vir_has_value(func, callee_value))) || + arg_count > INT_MAX / (int) sizeof(*effect->args) || + arg_count > INT_MAX / (int) sizeof(*effect->arg_uses) || + !vir_values_owned(func, args, arg_count) || + (result_type != VIR_TYPE_VOID && !vir_type_width(func, result_type))) + return NULL; + name = NULL; + if (callee) { + length = strlen(callee) + 1; + if (length > (size_t) 0x7fffffff) + return NULL; + name = vir_arena_alloc(&func->arena, (int) length); + } + effect = vir_new_effect(func, block, VIR_EFFECT_CALL); + if ((callee && !name) || !effect) + return NULL; + if (callee_value) { + callee_use = vir_arena_alloc(&func->arena, sizeof(*callee_use)); + if (!callee_use) + return NULL; + } + if (arg_count) { + effect->args = + vir_arena_alloc(&func->arena, sizeof(*effect->args) * arg_count); + effect->arg_uses = vir_arena_alloc( + &func->arena, sizeof(*effect->arg_uses) * arg_count); + if (!effect->args || !effect->arg_uses) + return NULL; + } + if (result_type != VIR_TYPE_VOID) { + result = vir_new_value(func, block, VIR_OP_CALL, result_type, 0); + if (!result) + return NULL; + } + if (callee) + memcpy(name, callee, length); + effect->callee = name; + effect->callee_value = callee_value; + effect->callee_value_use = callee_use; + effect->arg_count = arg_count; + effect->result_type = result_type; + effect->result = result; + if (result) + result->def_effect = effect; + if (callee_use) + vir_link_effect_use(effect, callee_value, callee_use, -1); + for (i = 0; i < arg_count; i++) { + effect->args[i] = args[i]; + vir_link_effect_use(effect, args[i], &effect->arg_uses[i], i); + } + vir_effect_append(block, effect); + return effect; +} + +vir_effect_t *vir_call(vir_function_t *func, + vir_block_t *block, + const char *callee, + vir_value_t **args, + int arg_count, + vir_type_t result_type) +{ + return vir_call_impl(func, block, callee, NULL, args, arg_count, + result_type); +} + +vir_effect_t *vir_call_indirect(vir_function_t *func, + vir_block_t *block, + vir_value_t *callee, + vir_value_t **args, + int arg_count, + vir_type_t result_type) +{ + return vir_call_impl(func, block, NULL, callee, args, arg_count, + result_type); +} + +static bool vir_call_abi_type_valid(const vir_function_t *func, + vir_call_abi_type_t type, + bool allow_void) +{ + if (type.type == VIR_TYPE_VOID) + return allow_void && !type.is_unsigned && !type.is_bool; + if (type.type == VIR_TYPE_PTR) + return !type.is_unsigned && !type.is_bool && + vir_type_width(func, type.type) != 0; + if (type.type < VIR_TYPE_I8 || type.type > VIR_TYPE_I64) + return false; + if (type.is_bool) + return type.type == VIR_TYPE_I8 && !type.is_unsigned; + return true; +} + +static bool vir_call_abi_type_matches_value(vir_call_abi_type_t abi_type, + vir_type_t value_type) +{ + return abi_type.is_bool + ? abi_type.type == VIR_TYPE_I8 && value_type == VIR_TYPE_I1 + : abi_type.type == value_type; +} + +static bool vir_call_signature_matches(const vir_function_t *func, + const vir_effect_t *effect, + const vir_call_signature_t *signature) +{ + int i; + + if (!func || !effect || !signature || effect->kind != VIR_EFFECT_CALL || + (!effect->callee_value && (!effect->callee || !*effect->callee)) || + !vir_has_block(func, effect->block) || + !vir_has_effect(effect->block, effect) || + (effect->callee_value && !vir_has_value(func, effect->callee_value)) || + (signature->va_start_slot != (unsigned int) -1 && + !signature->is_variadic) || + signature->param_count < 0 || + signature->param_count != effect->arg_count || + (signature->is_variadic && + (signature->fixed_param_count < 0 || + signature->fixed_param_count > signature->param_count)) || + (signature->param_count && !signature->params) || + signature->param_count > INT_MAX / (int) sizeof(*signature->params) || + !vir_values_owned(func, effect->args, effect->arg_count) || + !vir_call_abi_type_valid(func, signature->result, true) || + !vir_call_abi_type_matches_value(signature->result, + effect->result_type)) + return false; + for (i = 0; i < signature->param_count; i++) + if (!vir_call_abi_type_valid(func, signature->params[i], false) || + !vir_call_abi_type_matches_value(signature->params[i], + effect->args[i]->type)) + return false; + return true; +} + +bool vir_call_set_signature(vir_function_t *func, + vir_effect_t *effect, + const vir_call_signature_t *signature) +{ + vir_call_signature_t *copy; + vir_call_abi_type_t *params = NULL; + + if (!effect || effect->signature || + !vir_call_signature_matches(func, effect, signature)) + return false; + copy = vir_arena_alloc(&func->arena, sizeof(*copy)); + if (!copy) + return false; + if (signature->param_count) { + params = vir_arena_alloc( + &func->arena, (int) (sizeof(*params) * signature->param_count)); + if (!params) + return false; + memcpy(params, signature->params, + sizeof(*params) * signature->param_count); + } + *copy = *signature; + copy->params = params; + copy->param_slots = NULL; + copy->va_start_slot = (unsigned int) -1; + effect->signature = copy; + return true; +} + +/* Operand and result type rules shared by construction and verification. */ +static bool vir_binary_types_valid(const vir_function_t *func, + vir_opcode_t opcode, + vir_type_t result, + vir_type_t left, + vir_type_t right) +{ + if (!vir_is_pure_binary_opcode(opcode)) + return false; + if (opcode == VIR_OP_PTRADD) + return result == VIR_TYPE_PTR && left == VIR_TYPE_PTR && + vir_integer_type_width(right) == func->pointer_bits; + if (vir_opcode_is_shift(opcode)) + return vir_type_is_i32_or_i64(left) && right == VIR_TYPE_I32 && + result == left; + if (vir_opcode_is_compare(opcode)) + return left == right && result == VIR_TYPE_I1 && + (opcode == VIR_OP_EQ ? vir_type_width(func, left) + : vir_type_is_i32_or_i64(left)); + return left == right && result == left && vir_type_is_i32_or_i64(result); +} + +vir_value_t *vir_binary(vir_function_t *func, + vir_block_t *block, + vir_opcode_t opcode, + vir_type_t type, + vir_value_t *op0, + vir_value_t *op1) +{ + if (func && opcode == VIR_OP_PTRADD && func->pointer_bits == 64 && op1 && + op1->type == VIR_TYPE_I32) + op1 = vir_sext(func, block, op1, VIR_TYPE_I64); + if (!vir_has_block(func, block) || !vir_has_value(func, op0) || + !vir_has_value(func, op1) || + !vir_binary_types_valid(func, opcode, type, op0->type, op1->type)) + return NULL; + return vir_user_operation(func, block, opcode, type, op0, op1); +} + +static bool vir_integer_cast_valid(vir_opcode_t opcode, + vir_type_t destination, + vir_type_t source) +{ + int to = vir_integer_type_width(destination); + int from = vir_integer_type_width(source); + + return to && from && + (opcode == VIR_OP_TRUNC + ? to < from + : (opcode == VIR_OP_ZEXT || opcode == VIR_OP_SEXT) && + to > from); +} + +/* Operand and result type rules shared by construction and verification. */ +static bool vir_unary_types_valid(const vir_function_t *func, + vir_opcode_t opcode, + vir_type_t result, + vir_type_t operand) +{ + if (opcode == VIR_OP_NEG || opcode == VIR_OP_BITNOT) + return result == operand && vir_type_is_i32_or_i64(result); + if (opcode == VIR_OP_ZEXT || opcode == VIR_OP_SEXT || + opcode == VIR_OP_TRUNC) + return vir_integer_cast_valid(opcode, result, operand); + if (opcode == VIR_OP_PTRTOINT) + return operand == VIR_TYPE_PTR && + vir_integer_type_width(result) == func->pointer_bits; + if (opcode == VIR_OP_INTTOPTR) + return result == VIR_TYPE_PTR && + vir_integer_type_width(operand) == func->pointer_bits; + return false; +} + +static vir_value_t *vir_unary(vir_function_t *func, + vir_block_t *block, + vir_opcode_t opcode, + vir_type_t type, + vir_value_t *operand) +{ + if (!vir_has_block(func, block) || !vir_has_value(func, operand) || + !vir_is_pure_unary_opcode(opcode) || + !vir_unary_types_valid(func, opcode, type, operand->type)) + return NULL; + return vir_user_operation(func, block, opcode, type, operand, NULL); +} + +static bool vir_is_constant(const vir_value_t *value) +{ + return value && !value->is_block_param && value->nr_ops == 0 && + value->opcode == VIR_OP_CONST; +} + +static bool vir_is_const_int(const vir_value_t *value, + unsigned long long constant) +{ + return vir_is_constant(value) && vir_type_is_i32_or_i64(value->type) && + value->constant == (constant & vir_integer_mask(value->type)); +} + +static bool vir_binary_operands_are_local(const vir_function_t *func, + const vir_block_t *block, + const vir_value_t *op0, + const vir_value_t *op1) +{ + return block && op0 && op1 && vir_has_block(func, block) && + vir_has_value(func, op0) && vir_has_value(func, op1); +} + +static bool vir_integer_operands_are_local(const vir_function_t *func, + const vir_block_t *block, + const vir_value_t *op0, + const vir_value_t *op1) +{ + return vir_binary_operands_are_local(func, block, op0, op1) && + op0->type == op1->type && vir_type_is_i32_or_i64(op0->type); +} + +static void vir_order_commutative(vir_value_t **op0, vir_value_t **op1) +{ + if ((*op1)->id < (*op0)->id) { + vir_value_t *tmp = *op0; + *op0 = *op1; + *op1 = tmp; + } +} + +#define DEFINE_BINARY_WRAPPER(name, helper, opcode) \ + vir_value_t *name(vir_function_t *func, vir_block_t *block, \ + vir_value_t *op0, vir_value_t *op1) \ + { \ + return helper(func, block, op0, op1, opcode); \ + } + +static vir_value_t *vir_integer_binary(vir_function_t *func, + vir_block_t *block, + vir_value_t *op0, + vir_value_t *op1, + vir_opcode_t opcode) +{ + unsigned long long result; + bool division = vir_opcode_is_division(opcode); + bool shift = vir_opcode_is_shift(opcode); + + if (!(shift ? vir_binary_operands_are_local(func, block, op0, op1) && + (vir_type_is_i32_or_i64(op0->type) && + op1->type == VIR_TYPE_I32) + : vir_integer_operands_are_local(func, block, op0, op1))) + return NULL; + if (shift && vir_is_const_int(op1, 0)) + return op0; + if (opcode == VIR_OP_SUB) { + if (op0 == op1) + return vir_const_int(func, block, op0->type, 0); + if (vir_is_const_int(op1, 0)) + return op0; + } else if (opcode == VIR_OP_MUL) { + if (vir_is_const_int(op0, 0) || vir_is_const_int(op1, 0)) + return vir_const_int(func, block, op0->type, 0); + if (vir_is_const_int(op0, 1)) + return op1; + if (vir_is_const_int(op1, 1)) + return op0; + } else if (opcode == VIR_OP_BITAND) { + if (vir_is_const_int(op0, 0) || vir_is_const_int(op1, 0)) + return vir_const_int(func, block, op0->type, 0); + if (vir_is_const_int(op0, vir_integer_mask(op0->type))) + return op1; + if (vir_is_const_int(op1, vir_integer_mask(op1->type))) + return op0; + } else if (!division && !shift) { + if (vir_is_const_int(op0, 0)) + return op1; + if (vir_is_const_int(op1, 0)) + return op0; + } + if (vir_opcode_is_commutative(opcode)) + vir_order_commutative(&op0, &op1); + if (vir_is_constant(op0) && vir_is_constant(op1)) { + if (!vir_fold_binary(func, opcode, op0->type, op0->type, op0->constant, + op1->constant, &result)) + return division || shift ? vir_user_operation(func, block, opcode, + op0->type, op0, op1) + : NULL; + return vir_const_int(func, block, op0->type, result); + } + return vir_user_operation(func, block, opcode, op0->type, op0, op1); +} + +DEFINE_BINARY_WRAPPER(vir_add, vir_integer_binary, VIR_OP_ADD) +DEFINE_BINARY_WRAPPER(vir_sub, vir_integer_binary, VIR_OP_SUB) +DEFINE_BINARY_WRAPPER(vir_mul, vir_integer_binary, VIR_OP_MUL) + +vir_value_t *vir_div(vir_function_t *func, + vir_block_t *block, + vir_value_t *op0, + vir_value_t *op1, + bool is_unsigned, + bool remainder) +{ + vir_opcode_t opcode; + + opcode = remainder ? (is_unsigned ? VIR_OP_UREM : VIR_OP_SREM) + : (is_unsigned ? VIR_OP_UDIV : VIR_OP_SDIV); + return vir_integer_binary(func, block, op0, op1, opcode); +} + +DEFINE_BINARY_WRAPPER(vir_bitand, vir_integer_binary, VIR_OP_BITAND) +DEFINE_BINARY_WRAPPER(vir_bitor, vir_integer_binary, VIR_OP_BITOR) +DEFINE_BINARY_WRAPPER(vir_bitxor, vir_integer_binary, VIR_OP_BITXOR) +DEFINE_BINARY_WRAPPER(vir_shl, vir_integer_binary, VIR_OP_SHL) +DEFINE_BINARY_WRAPPER(vir_ashr, vir_integer_binary, VIR_OP_ASHR) +DEFINE_BINARY_WRAPPER(vir_lshr, vir_integer_binary, VIR_OP_LSHR) + +static vir_value_t *vir_integer_unary(vir_function_t *func, + vir_block_t *block, + vir_value_t *operand, + vir_type_t type, + vir_opcode_t opcode) +{ + bool cast = opcode == VIR_OP_ZEXT || opcode == VIR_OP_SEXT || + opcode == VIR_OP_TRUNC; + + if (!operand || !vir_has_block(func, block) || + !vir_has_value(func, operand) || + (cast ? !vir_integer_cast_valid(opcode, type, operand->type) + : (type != operand->type || !vir_type_is_i32_or_i64(type)))) + return NULL; + if (!vir_is_constant(operand)) + return vir_unary(func, block, opcode, type, operand); + if (cast) + return vir_const_int( + func, block, type, + vir_fold_integer_cast(opcode, vir_integer_type_width(operand->type), + vir_integer_type_width(type), + operand->constant)); + return vir_const_int(func, block, type, + vir_fold_unary(opcode, vir_integer_type_width(type), + operand->constant)); +} + +#define DEFINE_INTEGER_UNARY(name, opcode) \ + vir_value_t *name(vir_function_t *func, vir_block_t *block, \ + vir_value_t *operand) \ + { \ + return vir_integer_unary(func, block, operand, \ + operand ? operand->type : VIR_TYPE_VOID, \ + opcode); \ + } + +DEFINE_INTEGER_UNARY(vir_neg, VIR_OP_NEG) +DEFINE_INTEGER_UNARY(vir_bitnot, VIR_OP_BITNOT) + +#undef DEFINE_INTEGER_UNARY + +static vir_value_t *vir_compare(vir_function_t *func, + vir_block_t *block, + vir_value_t *op0, + vir_value_t *op1, + vir_opcode_t opcode) +{ + unsigned long long bits0; + unsigned long long bits1; + + if (opcode == VIR_OP_EQ) { + if (!vir_binary_operands_are_local(func, block, op0, op1) || + op0->type != op1->type || !vir_type_width(func, op0->type)) + return NULL; + if (op0 == op1) + return vir_const_i1(func, block, 1); + if (vir_opcode_is_commutative(opcode)) + vir_order_commutative(&op0, &op1); + } else if (!vir_integer_operands_are_local(func, block, op0, op1)) { + return NULL; + } + if (vir_is_constant(op0) && vir_is_constant(op1)) { + bits0 = op0->constant; + bits1 = op1->constant; + if (!vir_fold_binary(func, opcode, VIR_TYPE_I1, op0->type, bits0, bits1, + &bits0)) + return NULL; + return vir_const_i1(func, block, bits0); + } + return vir_user_operation(func, block, opcode, VIR_TYPE_I1, op0, op1); +} + +DEFINE_BINARY_WRAPPER(vir_eq, vir_compare, VIR_OP_EQ) +DEFINE_BINARY_WRAPPER(vir_slt, vir_compare, VIR_OP_SLT) +DEFINE_BINARY_WRAPPER(vir_ult, vir_compare, VIR_OP_ULT) + +#undef DEFINE_BINARY_WRAPPER + +#define DEFINE_FIXED_CAST(name, cast, type) \ + vir_value_t *name(vir_function_t *func, vir_block_t *block, \ + vir_value_t *operand) \ + { \ + return cast(func, block, operand, type); \ + } + +DEFINE_FIXED_CAST(vir_zext_i1, vir_zext, VIR_TYPE_I32) +DEFINE_FIXED_CAST(vir_trunc_i1, vir_trunc, VIR_TYPE_I1) + +#undef DEFINE_FIXED_CAST + +#define DEFINE_INTEGER_CAST(name, opcode) \ + vir_value_t *name(vir_function_t *func, vir_block_t *block, \ + vir_value_t *operand, vir_type_t type) \ + { \ + return vir_integer_unary(func, block, operand, type, opcode); \ + } + +DEFINE_INTEGER_CAST(vir_zext, VIR_OP_ZEXT) +DEFINE_INTEGER_CAST(vir_sext, VIR_OP_SEXT) +DEFINE_INTEGER_CAST(vir_trunc, VIR_OP_TRUNC) + +#undef DEFINE_INTEGER_CAST + +static vir_value_t *vir_pointer_cast(vir_function_t *func, + vir_block_t *block, + vir_value_t *operand, + vir_type_t type, + vir_opcode_t opcode) +{ + bool to_integer = opcode == VIR_OP_PTRTOINT; + + if (!func || !operand || + (to_integer ? operand->type != VIR_TYPE_PTR : type != VIR_TYPE_PTR) || + vir_integer_type_width(to_integer ? type : operand->type) != + func->pointer_bits) + return NULL; + if (vir_is_constant(operand)) + return to_integer ? vir_const_int(func, block, type, operand->constant) + : vir_const_ptr(func, block, operand->constant); + return vir_unary(func, block, opcode, type, operand); +} + +vir_value_t *vir_ptrtoint(vir_function_t *func, + vir_block_t *block, + vir_value_t *operand, + vir_type_t type) +{ + return vir_pointer_cast(func, block, operand, type, VIR_OP_PTRTOINT); +} + +vir_value_t *vir_inttoptr(vir_function_t *func, + vir_block_t *block, + vir_value_t *operand) +{ + return vir_pointer_cast(func, block, operand, VIR_TYPE_PTR, + VIR_OP_INTTOPTR); +} + +vir_value_t *vir_ptradd(vir_function_t *func, + vir_block_t *block, + vir_value_t *base, + vir_value_t *byte_offset) +{ + if (func && func->pointer_bits == 64 && byte_offset && + byte_offset->type == VIR_TYPE_I32) + byte_offset = vir_sext(func, block, byte_offset, VIR_TYPE_I64); + if (!base || !byte_offset || !vir_has_block(func, block) || + !vir_has_value(func, base) || !vir_has_value(func, byte_offset) || + base->type != VIR_TYPE_PTR || + !vir_integer_type_width(byte_offset->type) || + vir_integer_type_width(byte_offset->type) != func->pointer_bits) + return NULL; + if (vir_is_constant(byte_offset) && byte_offset->constant == 0) + return base; + if (vir_is_constant(base) && vir_is_constant(byte_offset)) + return vir_const_ptr(func, block, + base->constant + byte_offset->constant); + return vir_user_operation(func, block, VIR_OP_PTRADD, VIR_TYPE_PTR, base, + byte_offset); +} + +static void vir_remove_use(vir_value_t *operand, vir_use_t *needle) +{ + vir_remove_matching_use(operand, VIR_USE_EXACT, needle, -1); +} + +static void vir_remove_matching_use(vir_value_t *value, + vir_use_owner_t kind, + const void *owner, + int operand) +{ + vir_use_t **link = &value->uses; + + while (*link && !vir_use_matches(*link, kind, owner, operand)) + link = &(*link)->next; + if (*link) + *link = (*link)->next; +} + +static void vir_remove_effect_uses(vir_effect_t *effect) +{ + if (effect->address_use) + vir_remove_effect_use(effect->address, effect, 0); + if (effect->stored_value_use) + vir_remove_effect_use(effect->stored_value, effect, 1); + if (effect->callee_value_use) + vir_remove_effect_use(effect->callee_value, effect, -1); + effect->address_use = effect->stored_value_use = effect->callee_value_use = + NULL; + for (int operand = 0; operand < effect->arg_count; operand++) + vir_remove_effect_use(effect->args[operand], effect, operand); +} + +bool vir_effect_remove(vir_function_t *func, vir_effect_t *effect) +{ + vir_effect_t **link; + vir_effect_t *previous = NULL; + + if (!func || !effect || + (effect->kind != VIR_EFFECT_STORE && + (effect->kind != VIR_EFFECT_LOAD || !effect->result || + effect->result->uses)) || + !vir_has_block(func, effect->block)) + return false; + link = &effect->block->effects; + while (*link && *link != effect) { + previous = *link; + link = &(*link)->next; + } + if (!*link) + return false; + if (effect->result) { + vir_value_t **value_link = &effect->block->head; + while (*value_link && *value_link != effect->result) + value_link = &(*value_link)->next; + if (!*value_link) + return false; + *value_link = effect->result->next; + effect->result->next = NULL; + effect->result->def_effect = NULL; + } + vir_remove_effect_uses(effect); + *link = effect->next; + if (effect->block->last_effect == effect) + effect->block->last_effect = previous; + effect->next = NULL; + vir_repair_value_positions(effect->block); + return true; +} + +static void vir_set_use_slot(vir_value_t **slot, + vir_value_t *value, + vir_use_t *use) +{ + vir_remove_use(*slot, use); + *slot = value; + vir_link_use(value, use); +} + +static vir_value_t **vir_use_value_slot(vir_use_t *use) +{ + if (use->user) + return use->operand == 0 ? &use->user->op0 : &use->user->op1; + if (use->edge) + return &use->edge->args[use->operand]; + if (use->effect) { + if (use->effect->kind == VIR_EFFECT_CALL) + return use->operand < 0 ? &use->effect->callee_value + : &use->effect->args[use->operand]; + return use->operand == 0 ? &use->effect->address + : &use->effect->stored_value; + } + return use->return_block ? &use->return_block->return_value + : &use->branch_block->branch_condition; +} + +vir_block_t *vir_use_block(const vir_use_t *use) +{ + return use->user ? use->user->block + : use->edge ? use->edge->from + : use->effect ? use->effect->block + : use->return_block ? use->return_block + : use->branch_block; +} + +int vir_use_order(const vir_use_t *use) +{ + return use->user ? use->user->order + : use->effect ? use->effect->order + : vir_use_block(use)->next_order; +} + +static bool vir_replacement_dominates_with_context( + const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_value_t *from, + const vir_value_t *to) +{ + const vir_use_t *use; + for (use = from->uses; use; use = use->next) { + const vir_block_t *block; + int position; + if (!use->user && !use->edge && use->effect) { + if (!vir_value_dominates_effect(func, dominance, to, use->effect)) + return 0; + continue; + } + if (!(block = vir_use_block(use))) + return 0; + position = use->user ? use->user->position + : block->tail ? block->tail->position + 1 + : 0; + if (!vir_value_dominates(func, dominance, to, block, position)) + return 0; + } + return 1; +} + +static bool vir_replacement_dominates(const vir_function_t *func, + const vir_value_t *from, + const vir_value_t *to) +{ + return vir_replacement_dominates_with_context(func, NULL, from, to); +} + +static void vir_replace_all_uses_unchecked(vir_value_t *from, vir_value_t *to) +{ + while (from->uses) { + vir_use_t *use = from->uses; + vir_set_use_slot(vir_use_value_slot(use), to, use); + } +} + +int vir_replace_all_uses(vir_function_t *func, + vir_value_t *from, + vir_value_t *to) +{ + if (!func || !from || !to || !vir_has_value(func, from) || + !vir_has_value(func, to) || from->type != to->type) + return 0; + if (from == to) + return 1; + if (!vir_replacement_dominates(func, from, to)) + return 0; + vir_replace_all_uses_unchecked(from, to); + return 1; +} + +static unsigned int vir_cse_hash(const vir_value_t *value) +{ + unsigned int hash = (unsigned int) value->opcode; + hash = hash * 33u + (unsigned int) value->type; + hash = hash * 33u + (unsigned int) value->op0->id; + return hash * 33u + (unsigned int) value->op1->id; +} + +static unsigned int vir_cse_capacity(const vir_function_t *func, + const vir_block_t *only_block) +{ + int values = 0; + unsigned int capacity = 1; + const vir_block_t *block = only_block ? only_block : func->blocks; + const vir_block_t *stop = only_block ? only_block->next : NULL; + + for (; block != stop; block = block->next) + for (const vir_value_t *value = block->head; value; value = value->next) + if (value->nr_ops == 2 && ++values > INT_MAX / 2) + return 0; + while (capacity < (unsigned int) values * 2u) { + if (capacity > (unsigned int) INT_MAX / 2u) + return 0; + capacity <<= 1; + } + return capacity; +} + +/* Look up each binary value of @block in the open-addressed @table and fold it + * into an equal leader. Local CSE clears and reuses the table per block; GVN + * keeps it across blocks and relies on vir_replace_all_uses() to refuse a + * leader that does not dominate. + */ +static int vir_cse_scan_block(vir_function_t *func, + vir_block_t *block, + vir_value_t **table, + unsigned int capacity, + vir_cse_stats_t *stats) +{ + int replacements = 0; + + for (vir_value_t *value = block->head; value; value = value->next) { + vir_value_t *leader; + unsigned int slot; + + if (value->nr_ops != 2) + continue; + if (stats) { + stats->values++; + stats->lookups++; + } + slot = vir_cse_hash(value) & (capacity - 1); + while ((leader = table[slot])) { + if (stats) + stats->probes++; + if (leader->opcode == value->opcode && + leader->type == value->type && leader->op0 == value->op0 && + leader->op1 == value->op1) + break; + slot = (slot + 1) & (capacity - 1); + } + if (!leader) { + table[slot] = value; + continue; + } + if (stats) + stats->hits++; + if (value->uses && vir_replace_all_uses(func, value, leader)) { + replacements++; + if (stats) + stats->replacements++; + } + } + return replacements; +} + +static int vir_cse_run(vir_function_t *func, + vir_opt_level_t opt_level, + vir_cse_stats_t *stats, + bool global) +{ + vir_value_t **table = NULL; + int replacements = 0; + unsigned int capacity = 0; + + if (stats) + memset(stats, 0, sizeof(*stats)); + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + if (global ? opt_level != VIR_OPT_O2 : opt_level == VIR_OPT_O0) + return 0; + if (global) + capacity = vir_cse_capacity(func, NULL); + else + for (vir_block_t *block = func->blocks; block; block = block->next) { + unsigned int block_capacity = vir_cse_capacity(func, block); + + if (block_capacity > capacity) + capacity = block_capacity; + } + if (!capacity || !(table = calloc(capacity, sizeof(*table)))) + return 0; + if (global && stats) { + stats->tables++; + stats->table_slots += (int) capacity; + } + for (vir_block_t *block = func->blocks; block; block = block->next) { + unsigned int block_capacity = + global ? capacity : vir_cse_capacity(func, block); + + if (!block_capacity) + continue; + if (!global) { + memset(table, 0, (size_t) block_capacity * sizeof(*table)); + if (stats) { + stats->tables++; + stats->table_slots += (int) block_capacity; + } + } + replacements += + vir_cse_scan_block(func, block, table, block_capacity, stats); + } + free(table); + return replacements; +} + +int vir_local_cse_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + vir_cse_stats_t *stats) +{ + return vir_cse_run(func, opt_level, stats, false); +} + +int vir_local_cse(vir_function_t *func, vir_opt_level_t opt_level) +{ + return vir_local_cse_with_stats(func, opt_level, NULL); +} + +int vir_gvn_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + vir_cse_stats_t *stats) +{ + return vir_cse_run(func, opt_level, stats, true); +} + +static bool vir_value_is_dead(const vir_value_t *value) +{ + const vir_effect_t *effect = value->def_effect; + + /* Only unused pure values and non-volatile loads may be discarded. */ + return !value->uses && ((!value->is_block_param && !effect) || + (effect && effect->kind == VIR_EFFECT_LOAD && + effect->result == value)); +} + +static void vir_repair_value_positions(vir_block_t *block) +{ + vir_effect_t *effect; + vir_value_t *value; + int position = 0; + int order = 0; + + block->tail = NULL; + for (value = block->head; value; value = value->next) { + value->position = position++; + block->tail = value; + } + block->last_effect = NULL; + for (position = 0, effect = block->effects; effect; + position++, effect = effect->next) { + effect->position = position; + block->last_effect = effect; + } + effect = block->effects; + value = block->head; + while (value || effect) { + if (!effect || (value && value->order < effect->order)) { + value->order = order++; + value = value->next; + } else if (!value || effect->order < value->order) { + effect->order = order++; + effect = effect->next; + } else { + value->order = order; + effect->order = order++; + value = value->next; + effect = effect->next; + } + } + block->next_order = order; +} + +/* Promote a nonescaping stack scalar used only by direct, nonvolatile memory + * effects in one block. + */ +static bool vir_promote_private_stack(vir_function_t *func, + vir_block_t *block, + vir_value_t *address) +{ + vir_value_t *stored = NULL; + vir_use_t *use; + vir_effect_t *effect; + vir_type_t access_type = VIR_TYPE_VOID; + + if (address->opcode != VIR_OP_STACK_ADDR || address->block != block || + address->address_kind != VIR_ADDRESS_STACK || !address->uses) + return false; + for (vir_block_t *owner = func->blocks; owner; owner = owner->next) + for (vir_value_t *value = owner->head; value; value = value->next) + if (value != address && value->opcode == VIR_OP_STACK_ADDR && + value->address_slot == address->address_slot) + return false; + for (use = address->uses; use; use = use->next) { + effect = use->effect; + if (use->user || use->edge || use->return_block || use->branch_block || + use->operand != 0 || !effect || effect->block != block || + effect->address != address || + (effect->kind != VIR_EFFECT_LOAD && + effect->kind != VIR_EFFECT_STORE)) + return false; + } + for (effect = block->effects; effect; effect = effect->next) { + vir_value_t *value; + + if (effect->address != address) + continue; + if (effect->kind == VIR_EFFECT_STORE) { + stored = effect->stored_value; + value = stored; + } else if (effect->kind == VIR_EFFECT_LOAD) { + if (!stored) + return false; + value = effect->result; + } else { + return false; + } + vir_type_t type; + + if (!value) + return false; + type = value->type; + if (!vir_type_width(func, type) || + (access_type != VIR_TYPE_VOID && access_type != type)) + return false; + access_type = type; + } + if (!stored) + return false; + stored = NULL; + for (effect = block->effects; effect; effect = effect->next) { + if (effect->address != address) + continue; + if (effect->kind == VIR_EFFECT_STORE) { + stored = effect->stored_value; + } else if (!vir_replace_all_uses(func, effect->result, stored)) { + return false; + } + } + for (vir_effect_t **link = &block->effects; *link;) { + effect = *link; + if (effect->address == address && effect->kind == VIR_EFFECT_STORE) { + vir_remove_effect_uses(effect); + *link = effect->next; + } else { + link = &effect->next; + } + } + vir_repair_value_positions(block); + return true; +} + +static void vir_promote_private_stack_values(vir_function_t *func) +{ + for (vir_block_t *block = func->blocks; block; block = block->next) { + for (vir_value_t *value = block->head; value; value = value->next) + if (value->opcode == VIR_OP_STACK_ADDR) + vir_promote_private_stack(func, block, value); + } +} + +/* An unused ordinary read cannot observe a store. Every other effect remains a + * barrier unless it overwrites exactly the same typed location. + */ +static void vir_drop_overwritten_stores(vir_block_t *block) +{ + for (vir_effect_t **link = &block->effects; *link;) { + vir_effect_t *effect = *link; + if (effect->kind != VIR_EFFECT_STORE) { + link = &effect->next; + continue; + } + vir_effect_t *next = effect->next; + while (next && next->kind == VIR_EFFECT_LOAD && next->result && + !next->result->uses) + next = next->next; + if (next && next->kind == VIR_EFFECT_STORE && + effect->address == next->address && + effect->stored_value->type == next->stored_value->type) { + vir_remove_effect_uses(effect); + *link = effect->next; + } else { + link = &effect->next; + } + } + vir_repair_value_positions(block); +} + +int vir_dce_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + int *passes, + int *values_scanned, + int *values_removed, + int *temporary_bytes, + int *temporary_peak_bytes) +{ + unsigned char *dead; + vir_value_t **work; + int capacity; + int head = 0; + int tail = 0; + int removed = 0; + int result = -1; + + if (passes) + *passes = 0; + if (values_scanned) + *values_scanned = 0; + if (values_removed) + *values_removed = 0; + if (temporary_bytes) + *temporary_bytes = 0; + if (temporary_peak_bytes) + *temporary_peak_bytes = 0; + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + if (opt_level == VIR_OPT_O0) + return 0; + for (vir_block_t *block = func->blocks; block; block = block->next) + vir_drop_overwritten_stores(block); + capacity = func->next_value_id; + if (capacity <= 0) + return 0; + if (capacity > INT_MAX / (int) (sizeof(*dead) + sizeof(*work))) + return -1; + dead = calloc((size_t) capacity, sizeof(*dead)); + work = malloc((size_t) capacity * sizeof(*work)); + if (!dead || !work) + goto cleanup; + if (temporary_bytes) + *temporary_bytes = capacity * (int) (sizeof(*dead) + sizeof(*work)); + if (temporary_peak_bytes) + *temporary_peak_bytes = + capacity * (int) (sizeof(*dead) + sizeof(*work)); + if (passes) + (*passes)++; + for (vir_block_t *block = func->blocks; block; block = block->next) + for (vir_value_t *value = block->head; value; value = value->next) { + if (values_scanned) + (*values_scanned)++; + if (vir_value_is_dead(value)) { + dead[value->id] = 1; + work[tail++] = value; + } + } + while (head < tail) { + vir_value_t *value = work[head++]; + + if (value->def_effect) { + vir_effect_t *load = value->def_effect; + vir_value_t *input = load->address; + + vir_remove_effect_uses(load); + if (vir_value_is_dead(input) && !dead[input->id]) { + dead[input->id] = 1; + work[tail++] = input; + } + continue; + } + for (int operand = 0; operand < value->nr_ops; operand++) { + vir_value_t *input = operand ? value->op1 : value->op0; + + vir_remove_user_use(input, value, operand); + if (vir_value_is_dead(input) && !dead[input->id]) { + dead[input->id] = 1; + work[tail++] = input; + } + } + } + for (vir_block_t *block = func->blocks; block; block = block->next) { + vir_value_t **link = &block->head; + vir_effect_t **effect_link = &block->effects; + + while (*link) { + vir_value_t *value = *link; + + if (dead[value->id]) { + *link = value->next; + removed++; + } else { + link = &value->next; + } + } + while (*effect_link) { + vir_effect_t *effect = *effect_link; + + if (effect->kind == VIR_EFFECT_LOAD && effect->result && + dead[effect->result->id]) { + *effect_link = effect->next; + } else { + effect_link = &effect->next; + } + } + vir_repair_value_positions(block); + } + result = removed; +cleanup: + free(dead); + free(work); + if (result >= 0 && values_removed) + *values_removed = removed; + return result; +} + +int vir_dce(vir_function_t *func, vir_opt_level_t opt_level) +{ + return vir_dce_with_stats(func, opt_level, NULL, NULL, NULL, NULL, NULL); +} + +enum { + VIR_SCCP_UNDEF, + VIR_SCCP_CONST, + VIR_SCCP_OVERDEF, +}; + +static bool vir_sccp_set(unsigned char *state, + unsigned long long *constant, + int id, + int next_state, + unsigned long long next_constant) +{ + if (state[id] == VIR_SCCP_OVERDEF) + return false; + if (next_state == VIR_SCCP_OVERDEF) { + state[id] = VIR_SCCP_OVERDEF; + return true; + } + if (state[id] == VIR_SCCP_UNDEF) { + state[id] = VIR_SCCP_CONST; + constant[id] = next_constant; + return true; + } + if (constant[id] == next_constant) + return false; + state[id] = VIR_SCCP_OVERDEF; + return true; +} + +static bool vir_sccp_fold(const vir_function_t *func, + const vir_value_t *value, + unsigned long long left, + unsigned long long right, + unsigned long long *result) +{ + if (value->nr_ops == 1) { + if (vir_is_pure_unary_opcode(value->opcode)) + *result = vir_fold_unary( + value->opcode, vir_type_width(func, value->op0->type), left); + else + return false; + *result &= vir_width_mask(vir_type_width(func, value->type)); + return true; + } + return vir_fold_binary(func, value->opcode, value->type, value->op0->type, + left, right, result); +} + +static bool vir_sccp_value(const vir_function_t *func, + const vir_value_t *value, + unsigned char *state, + unsigned long long *constant) +{ + unsigned long long result; + + if (value->is_block_param) + return false; + if (vir_is_constant(value)) + return vir_sccp_set(state, constant, value->id, VIR_SCCP_CONST, + value->constant); + if (value->def_effect || !value->nr_ops) + return vir_sccp_set(state, constant, value->id, VIR_SCCP_OVERDEF, 0); + if (state[value->op0->id] == VIR_SCCP_OVERDEF || + (value->nr_ops == 2 && state[value->op1->id] == VIR_SCCP_OVERDEF)) + return vir_sccp_set(state, constant, value->id, VIR_SCCP_OVERDEF, 0); + if (state[value->op0->id] != VIR_SCCP_CONST || + (value->nr_ops == 2 && state[value->op1->id] != VIR_SCCP_CONST)) + return false; + if (!vir_sccp_fold(func, value, constant[value->op0->id], + value->nr_ops == 2 ? constant[value->op1->id] : 0, + &result)) + return vir_sccp_set(state, constant, value->id, VIR_SCCP_OVERDEF, 0); + return vir_sccp_set(state, constant, value->id, VIR_SCCP_CONST, result); +} + +static int vir_sccp_edge_index(const vir_function_t *func, + const vir_edge_t *needle) +{ + int index = 0; + + for (const vir_block_t *block = func->blocks; block; block = block->next) + for (const vir_edge_t *edge = block->outgoing; edge; + edge = edge->next_outgoing, index++) + if (edge == needle) + return index; + return -1; +} + +typedef struct { + unsigned char *queued_blocks; + int *queue; + int head; + int tail; + int count; + int capacity; +} vir_sccp_worklist_t; + +/* Entry/edge activation can enqueue inactive blocks; uses only requeue active + * ones. + */ +static bool vir_sccp_queue_block(vir_sccp_worklist_t *worklist, + const vir_function_t *func, + const vir_block_t *block, + bool require_executable) +{ + int index = vir_block_index(func, block); + + if (index < 0 || worklist->queued_blocks[index] == 2 || + (require_executable && worklist->queued_blocks[index] != 1) || + worklist->count >= worklist->capacity) + return false; + worklist->queued_blocks[index] = 2; + worklist->queue[worklist->tail] = index; + worklist->tail = (worklist->tail + 1) % worklist->capacity; + worklist->count++; + return true; +} + +static void vir_sccp_schedule_uses(vir_sccp_worklist_t *worklist, + const vir_function_t *func, + const vir_value_t *value, + const vir_block_t *current_block, + const unsigned char *executable_edges) +{ + for (const vir_use_t *use = value->uses; use; use = use->next) { + if (use->user && use->user->block != current_block) + vir_sccp_queue_block(worklist, func, use->user->block, true); + else if (use->branch_block && use->branch_block != current_block) + vir_sccp_queue_block(worklist, func, use->branch_block, true); + else if (use->edge) { + int index = vir_sccp_edge_index(func, use->edge); + + if (index >= 0 && executable_edges[index]) + vir_sccp_queue_block(worklist, func, use->edge->to, true); + } + } +} + +static bool vir_sccp_param(const vir_value_t *param, + int index, + const unsigned char *executable_edges, + const vir_function_t *func, + unsigned char *state, + unsigned long long *constant) +{ + const vir_edge_t *edge; + int seen = 0; + int next_state = VIR_SCCP_UNDEF; + unsigned long long next_constant = 0; + + for (edge = param->block->incoming; edge; edge = edge->next_incoming) { + int edge_index = vir_sccp_edge_index(func, edge); + + if (edge_index < 0) + return false; + if (!executable_edges[edge_index]) + continue; + seen = 1; + if (state[edge->args[index]->id] == VIR_SCCP_OVERDEF) { + next_state = VIR_SCCP_OVERDEF; + break; + } + if (state[edge->args[index]->id] != VIR_SCCP_CONST) + continue; + if (next_state == VIR_SCCP_UNDEF) { + next_state = VIR_SCCP_CONST; + next_constant = constant[edge->args[index]->id]; + } else if (next_constant != constant[edge->args[index]->id]) { + next_state = VIR_SCCP_OVERDEF; + break; + } + } + + /* Entry parameters have no incoming edge and therefore model unknown + * function inputs. Treating them as undefined can make a loop header look + * constant before its backedge becomes executable. + */ + if (!seen) + return vir_sccp_set(state, constant, param->id, VIR_SCCP_OVERDEF, 0); + if (next_state == VIR_SCCP_UNDEF) + return false; + return vir_sccp_set(state, constant, param->id, next_state, next_constant); +} + +static void vir_sccp_mark_edge(vir_sccp_worklist_t *worklist, + const vir_function_t *func, + const vir_edge_t *edge, + unsigned char *executable_edges) +{ + int edge_index = vir_sccp_edge_index(func, edge); + + if (edge_index < 0) + return; + if (!executable_edges[edge_index]) { + executable_edges[edge_index] = 1; + vir_sccp_queue_block(worklist, func, edge->to, false); + } +} + +static vir_value_t *vir_sccp_constant_before(vir_function_t *func, + vir_value_t *value, + unsigned long long constant) +{ + vir_block_t *block = value->block; + vir_value_t *replacement = + vir_const_int(func, block, value->type, constant); + + if (!replacement) + return NULL; + if (block->head != replacement) { + vir_value_t *previous = block->head; + + while (previous->next != replacement) + previous = previous->next; + previous->next = NULL; + block->tail = previous; + replacement->next = block->head; + block->head = replacement; + } + + /* Insert before an effect-first block as well as before pure values; equal + * orders are reserved for an effect and its own result value. + */ + replacement->order = -1; + vir_repair_value_positions(block); + return replacement; +} + +static bool vir_sccp_replace_constant(vir_function_t *func, + vir_value_t *value, + const unsigned char *state, + const unsigned long long *constant, + int *values_replaced) +{ + vir_value_t *replacement; + + if (state[value->id] != VIR_SCCP_CONST || !value->uses || + !(replacement = + vir_sccp_constant_before(func, value, constant[value->id])) || + !vir_replace_all_uses(func, value, replacement)) + return false; + if (values_replaced) + (*values_replaced)++; + return true; +} + +static void vir_sccp_release(unsigned char *state, + unsigned long long *constant, + unsigned char *blocks, + unsigned char *edges, + int *work) +{ + free(state); + free(constant); + free(blocks); + free(edges); + free(work); +} + +int vir_sccp_with_stats_and_pruning(vir_function_t *func, + vir_opt_level_t opt_level, + int *stats, + bool prune_unreachable) +{ + unsigned char *state; + unsigned char *executable_blocks; + unsigned char *executable_edges; + unsigned long long *constant; + int *work; + vir_sccp_worklist_t worklist; + int capacity; + int block_count = 0; + int edge_count = 0; + int edge_marker_count; + int temporary; + int changes = 0; + int result = -1; + int simplified = 0; + int *passes = stats ? &stats[VIR_SCCP_STAT_PASSES] : NULL; + int *values_scanned = stats ? &stats[VIR_SCCP_STAT_VALUES_SCANNED] : NULL; + int *values_replaced = stats ? &stats[VIR_SCCP_STAT_VALUES_REPLACED] : NULL; + int *branches_simplified = + stats ? &stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] : NULL; + int *blocks_removed = stats ? &stats[VIR_SCCP_STAT_BLOCKS_REMOVED] : NULL; + int *temporary_bytes = stats ? &stats[VIR_SCCP_STAT_TEMPORARY_BYTES] : NULL; + int *temporary_peak_bytes = + stats ? &stats[VIR_SCCP_STAT_TEMPORARY_PEAK_BYTES] : NULL; + + if (stats) + memset(stats, 0, VIR_SCCP_STAT_COUNT * sizeof(*stats)); + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + + /* Without values there is no lattice to solve and no branch to fold; an + * empty void function is the usual case. + */ + if (opt_level == VIR_OPT_O0 || !func->blocks || !func->next_value_id) + return 0; + vir_promote_private_stack_values(func); + capacity = func->next_value_id; + for (vir_block_t *block = func->blocks; block; block = block->next) { + block_count++; + for (vir_edge_t *edge = block->outgoing; edge; + edge = edge->next_outgoing) + edge_count++; + } + if (capacity <= 0 || + capacity > INT_MAX / (int) (sizeof(*state) + sizeof(*constant))) + return -1; + temporary = capacity * (int) (sizeof(*state) + sizeof(*constant)); + if (block_count > (INT_MAX - temporary) / + (int) (sizeof(*executable_blocks) + sizeof(*work))) + return -1; + temporary += + block_count * (int) (sizeof(*executable_blocks) + sizeof(*work)); + edge_marker_count = edge_count ? edge_count : 1; + if (edge_marker_count > + (INT_MAX - temporary) / (int) sizeof(*executable_edges)) + return -1; + temporary += edge_marker_count * (int) sizeof(*executable_edges); + state = calloc((size_t) capacity, sizeof(*state)); + constant = calloc((size_t) capacity, sizeof(*constant)); + executable_blocks = + calloc((size_t) block_count, sizeof(*executable_blocks)); + executable_edges = + calloc((size_t) edge_marker_count, sizeof(*executable_edges)); + work = malloc(sizeof(*work) * (size_t) block_count); + if (!state || !constant || !executable_blocks || !executable_edges || !work) + goto cleanup; + worklist.queued_blocks = executable_blocks; + worklist.queue = work; + worklist.head = 0; + worklist.tail = 0; + worklist.count = 0; + worklist.capacity = block_count; + if (!vir_sccp_queue_block(&worklist, func, func->blocks, false)) + goto fail; + while (worklist.count) { + int block_index = worklist.queue[worklist.head]; + vir_block_t *block = func->blocks; + + worklist.head = (worklist.head + 1) % worklist.capacity; + worklist.count--; + for (int i = 0; i < block_index; i++) + block = block->next; + if (!block || worklist.queued_blocks[block_index] != 2) + goto fail; + worklist.queued_blocks[block_index] = 1; + if (passes) + (*passes)++; + int param_index = 0; + + for (vir_value_t *value = block->params; value; + value = value->param_next, param_index++) { + if (vir_sccp_param(value, param_index, executable_edges, func, + state, constant)) + vir_sccp_schedule_uses(&worklist, func, value, block, + executable_edges); + } + for (vir_value_t *value = block->head; value; value = value->next) { + if (values_scanned) + (*values_scanned)++; + if (vir_sccp_value(func, value, state, constant)) + vir_sccp_schedule_uses(&worklist, func, value, block, + executable_edges); + } + if (block->terminator == VIR_TERM_JUMP) + vir_sccp_mark_edge(&worklist, func, block->outgoing, + executable_edges); + else if (block->terminator == VIR_TERM_BRANCH) { + int condition_state = state[block->branch_condition->id]; + + if (condition_state == VIR_SCCP_CONST) + vir_sccp_mark_edge(&worklist, func, + constant[block->branch_condition->id] + ? block->true_edge + : block->false_edge, + executable_edges); + else if (condition_state == VIR_SCCP_OVERDEF) { + vir_sccp_mark_edge(&worklist, func, block->true_edge, + executable_edges); + vir_sccp_mark_edge(&worklist, func, block->false_edge, + executable_edges); + } + } + } + + /* Rewrite conditional edges before replacing the condition value: the + * replacement receives a fresh ID outside the temporary lattice. + */ + for (vir_block_t *block = func->blocks; block; block = block->next) + if (block->terminator == VIR_TERM_BRANCH && + state[block->branch_condition->id] == VIR_SCCP_CONST) { + vir_edge_t *dead = constant[block->branch_condition->id] + ? block->false_edge + : block->true_edge; + if (!vir_edge_delete(func, dead)) + goto fail; + changes++; + simplified++; + if (branches_simplified) + (*branches_simplified)++; + } + for (vir_block_t *block = func->blocks; block; block = block->next) + for (vir_value_t *value = block->params; value; + value = value->param_next) + changes += vir_sccp_replace_constant(func, value, state, constant, + values_replaced); + for (vir_block_t *block = func->blocks; block; block = block->next) + for (vir_value_t *value = block->head; value; value = value->next) + if (!vir_is_constant(value)) + changes += vir_sccp_replace_constant(func, value, state, + constant, values_replaced); + if (simplified && prune_unreachable) { + int before = block_count; + + if (!vir_remove_unreachable(func)) + goto fail; + int after = vir_function_block_count(func); + if (blocks_removed) + *blocks_removed = before - after; + changes += before - after; + } + result = changes; + if (temporary_bytes) + *temporary_bytes = temporary; + if (temporary_peak_bytes) + *temporary_peak_bytes = temporary; + goto cleanup; + +fail: + result = -1; +cleanup: + vir_sccp_release(state, constant, executable_blocks, executable_edges, + work); + return result; +} + +int vir_sccp(vir_function_t *func, vir_opt_level_t opt_level) +{ + return vir_sccp_with_stats(func, opt_level, NULL); +} + +int vir_sccp_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + int *stats) +{ + return vir_sccp_with_stats_and_pruning(func, opt_level, stats, true); +} + +typedef struct { + vir_edge_t *edge; + vir_value_t **args; + vir_use_t *uses; + vir_value_t *inserted; +} vir_edge_resize_t; + +typedef struct { + vir_edge_resize_t *edges; + int count; + int old_count; + int index; + bool insert; +} vir_param_edit_t; + +static void vir_param_edit_release(vir_param_edit_t *edit) +{ + free(edit->edges); + memset(edit, 0, sizeof(*edit)); +} + +static bool vir_param_edit_prepare(vir_function_t *func, + vir_block_t *block, + vir_value_t *removed, + int old_count, + int index, + bool insert, + vir_param_edit_t *edit) +{ + vir_edge_t *edge; + int new_count; + int i; + + memset(edit, 0, sizeof(*edit)); + if (!vir_has_block(func, block) || old_count < 0 || index < 0 || + (insert ? index > old_count : index >= old_count) || + (!insert && + (!removed || removed->block != block || !removed->is_block_param))) + return 0; + if (!insert) { + vir_value_t *param; + for (param = block->params; param && param != removed; + param = param->param_next) + ; + if (!param) + return 0; + } + for (edge = block->incoming; edge; edge = edge->next_incoming) + edit->count++; + edit->edges = + edit->count ? calloc((size_t) edit->count, sizeof(*edit->edges)) : NULL; + if (edit->count && !edit->edges) + return 0; + edit->old_count = old_count; + edit->index = index; + edit->insert = insert; + new_count = old_count + (insert ? 1 : -1); + for (edge = block->incoming, i = 0; edge; edge = edge->next_incoming, i++) { + edit->edges[i].edge = edge; + if (!new_count) + continue; + edit->edges[i].args = vir_arena_alloc( + &func->arena, sizeof(*edit->edges[i].args) * new_count); + edit->edges[i].uses = vir_arena_alloc( + &func->arena, sizeof(*edit->edges[i].uses) * new_count); + if (!edit->edges[i].args || !edit->edges[i].uses) + goto fail; + } + return 1; + +fail: + vir_param_edit_release(edit); + return 0; +} + +static void vir_param_edit_apply(vir_block_t *block, + vir_value_t *param, + vir_param_edit_t *edit) +{ + int new_count = edit->old_count + (edit->insert ? 1 : -1); + int i; + + for (i = 0; i < edit->count; i++) { + vir_edge_t *edge = edit->edges[i].edge; + int source, destination = 0; + + for (source = 0; source < edge->arg_count; source++) { + if (edit->insert && source == edit->index) + edit->edges[i].args[destination++] = edit->edges[i].inserted; + if (!edit->insert && source == edit->index) + continue; + edit->edges[i].args[destination++] = edge->args[source]; + } + if (edit->insert && edit->index == edge->arg_count) + edit->edges[i].args[destination++] = edit->edges[i].inserted; + vir_edge_replace_args(edge, edit->edges[i].args, edit->edges[i].uses, + new_count); + } + if (edit->insert) { + vir_block_append_param(block, param); + } else { + vir_value_t *cursor, *previous = NULL; + for (cursor = block->params; cursor && cursor != param; + cursor = cursor->param_next) + previous = cursor; + if (previous) + previous->param_next = param->param_next; + else + block->params = param->param_next; + if (block->last_param == param) + block->last_param = previous; + block->param_count--; + } +} + +static int vir_count_uses(const vir_value_t *value, + vir_use_owner_t kind, + const void *owner, + int operand) +{ + const vir_use_t *use; + int count = 0; + + for (use = value->uses; use; use = use->next) + if (vir_use_matches(use, kind, owner, operand)) + count++; + return count; +} + +#define vir_use_count(value, user, operand) \ + vir_count_uses(value, VIR_USE_USER, user, operand) +#define vir_edge_use_count(value, edge, operand) \ + vir_count_uses(value, VIR_USE_EDGE, edge, operand) +#define vir_effect_use_count(value, effect, operand) \ + vir_count_uses(value, VIR_USE_EFFECT, effect, operand) + +#define DEFINE_BLOCK_LIST_CONTAINS(name, type, head, link) \ + static bool name(const vir_block_t *block, const type *needle) \ + { \ + const type *item; \ + for (item = block->head; item; item = item->link) \ + if (item == needle) \ + return true; \ + return false; \ + } + +DEFINE_BLOCK_LIST_CONTAINS(vir_has_incoming_edge, + vir_edge_t, + incoming, + next_incoming) +DEFINE_BLOCK_LIST_CONTAINS(vir_has_outgoing_edge, + vir_edge_t, + outgoing, + next_outgoing) +DEFINE_BLOCK_LIST_CONTAINS(vir_has_effect, vir_effect_t, effects, next) + +#undef DEFINE_BLOCK_LIST_CONTAINS + +#define vir_return_use_count(value, block) \ + vir_count_uses(value, VIR_USE_RETURN, block, -1) +#define vir_branch_use_count(value, block) \ + vir_count_uses(value, VIR_USE_BRANCH, block, -1) + +static int vir_block_index(const vir_function_t *func, + const vir_block_t *needle) +{ + const vir_block_t *block; + int index = 0; + + if (!func || !needle) + return -1; + for (block = func->blocks; block; block = block->next, index++) + if (block == needle) + return index; + return -1; +} + +static bool vir_has_block(const vir_function_t *func, const vir_block_t *needle) +{ + return vir_block_index(func, needle) >= 0; +} + +static bool vir_block_open(const vir_function_t *func, const vir_block_t *block) +{ + return vir_has_block(func, block) && block->terminator == VIR_TERM_NONE; +} + +static int vir_count_values(const vir_function_t *func, + const vir_value_t *needle, + int id) +{ + const vir_block_t *block; + const vir_value_t *value; + int count = 0; + +#define VIR_COUNT_VALUE_CHAIN(head, link) \ + for (value = (head); value; value = value->link) \ + if (needle ? value == needle : value->id == id) { \ + if (needle) \ + return 1; \ + count++; \ + } + for (block = func->blocks; block; block = block->next) { + VIR_COUNT_VALUE_CHAIN(block->params, param_next); + VIR_COUNT_VALUE_CHAIN(block->head, next); + } +#undef VIR_COUNT_VALUE_CHAIN + return count; +} + +static bool vir_has_value(const vir_function_t *func, const vir_value_t *needle) +{ + if (!needle) + return false; + return vir_count_values(func, needle, 0) != 0; +} + +static int vir_value_id_count(const vir_function_t *func, int id) +{ + return vir_count_values(func, NULL, id); +} + +#define VIR_LIST_HAS_CYCLE(type, head, link) \ + do { \ + const type *slow = (head), *fast = (head); \ + while (fast && fast->link) { \ + slow = slow->link; \ + fast = fast->link->link; \ + if (slow == fast) \ + return 0; \ + } \ + } while (0) + +static bool vir_lists_are_acyclic(const vir_function_t *func) +{ + const vir_block_t *block; + VIR_LIST_HAS_CYCLE(vir_block_t, func->blocks, next); + for (block = func->blocks; block; block = block->next) { + const vir_value_t *value; + VIR_LIST_HAS_CYCLE(vir_value_t, block->params, param_next); + VIR_LIST_HAS_CYCLE(vir_value_t, block->head, next); + VIR_LIST_HAS_CYCLE(vir_edge_t, block->incoming, next_incoming); + VIR_LIST_HAS_CYCLE(vir_edge_t, block->outgoing, next_outgoing); + VIR_LIST_HAS_CYCLE(vir_effect_t, block->effects, next); + for (value = block->params; value; value = value->param_next) + VIR_LIST_HAS_CYCLE(vir_use_t, value->uses, next); + for (value = block->head; value; value = value->next) + VIR_LIST_HAS_CYCLE(vir_use_t, value->uses, next); + } + return 1; +} + +#undef VIR_LIST_HAS_CYCLE + +int vir_function_block_count(const vir_function_t *func) +{ + const vir_block_t *block; + int count = 0; + for (block = func->blocks; block; block = block->next) + count++; + return count; +} + +static bool vir_mark_reachable_blocks(const vir_function_t *func, + const vir_block_t *blocked, + unsigned char *reachable, + int *work) +{ + int head = 0; + int tail = 0; + + reachable[0] = 1; + work[tail++] = 0; + while (head < tail) { + const vir_block_t *block = func->blocks; + int index = work[head++]; + const vir_edge_t *edge; + + for (; index; index--) + block = block->next; + for (edge = block->outgoing; edge; edge = edge->next_outgoing) { + int target; + + if (blocked && edge->to == blocked) + continue; + target = vir_block_index(func, edge->to); + if (target < 0) { + if (blocked) + continue; + return false; + } + if (reachable[target]) + continue; + reachable[target] = 1; + work[tail++] = target; + } + } + return true; +} + +/* Non-verifier transformations use a bounded reachability query rather than the + * old quadratic int matrix. The verifier itself builds a compact bitset matrix + * once per function below. + */ +static bool vir_value_dominates_by_reachability(const vir_function_t *func, + const vir_value_t *value, + const vir_block_t *use_block, + int count) +{ + unsigned char *reachable; + int *work; + int use_index; + + if (vir_block_index(func, value->block) < 0 || + (use_index = vir_block_index(func, use_block)) < 0 || count <= 0 || + count > INT_MAX / (int) sizeof(*work)) + return false; + if (value->block == func->blocks) + return true; + reachable = calloc((size_t) count, sizeof(*reachable)); + work = malloc((size_t) count * sizeof(*work)); + if (!reachable || !work) { + free(reachable); + free(work); + return false; + } + + /* If the use is unreachable, no path can avoid the definition and the + * result is vacuously true. The same walk also handles that case. + */ + if (!vir_mark_reachable_blocks(func, value->block, reachable, work)) { + free(reachable); + free(work); + return false; + } + use_index = !reachable[use_index]; + free(reachable); + free(work); + return use_index; +} + +struct vir_dominance { + vir_block_t **blocks; + int *indices; + unsigned long *dom; + unsigned long *candidate; + int count; + int words; + int id_count; +}; + +static void vir_dominance_release(vir_dominance_t *dominance) +{ + free(dominance->blocks); + free(dominance->indices); + free(dominance->dom); + free(dominance->candidate); + memset(dominance, 0, sizeof(*dominance)); +} + +static int vir_dominance_index(const vir_dominance_t *dominance, + const vir_block_t *block) +{ + int index; + + if (!block || block->id < 0 || block->id >= dominance->id_count) + return -1; + index = dominance->indices[block->id]; + return index >= 0 && dominance->blocks[index] == block ? index : -1; +} + +static bool vir_dominance_block_dominates(const vir_dominance_t *dominance, + const vir_block_t *dominator, + const vir_block_t *block) +{ + int definition = vir_dominance_index(dominance, dominator); + int use = vir_dominance_index(dominance, block); + int word_bits = (int) (sizeof(unsigned long) * CHAR_BIT); + + return definition >= 0 && use >= 0 && + (dominance->dom[use * dominance->words + definition / word_bits] & + (1UL << (definition % word_bits))) != 0; +} + +static bool vir_dominance_init(const vir_function_t *func, + vir_dominance_t *dominance) +{ + const vir_block_t *block; + int count = vir_function_block_count(func); + int words; + int i; + + memset(dominance, 0, sizeof(*dominance)); + if (count <= 0 || func->next_block_id <= 0 || + func->next_block_id > INT_MAX / (int) sizeof(*dominance->indices)) + return false; + if (count > INT_MAX - (int) (sizeof(unsigned long) * CHAR_BIT) + 1) + return false; + words = (count + (int) (sizeof(unsigned long) * CHAR_BIT) - 1) / + (int) (sizeof(unsigned long) * CHAR_BIT); + if (words <= 0 || count > INT_MAX / words / (int) sizeof(*dominance->dom)) + return false; + dominance->blocks = malloc((size_t) count * sizeof(*dominance->blocks)); + dominance->indices = + malloc((size_t) func->next_block_id * sizeof(*dominance->indices)); + dominance->dom = + calloc((size_t) count * (size_t) words, sizeof(*dominance->dom)); + dominance->candidate = + malloc((size_t) words * sizeof(*dominance->candidate)); + if (!dominance->blocks || !dominance->indices || !dominance->dom || + !dominance->candidate) { + vir_dominance_release(dominance); + return false; + } + dominance->count = count; + dominance->words = words; + dominance->id_count = func->next_block_id; + for (i = 0; i < dominance->id_count; i++) + dominance->indices[i] = -1; + for (block = func->blocks, i = 0; block; block = block->next, i++) { + if (block->id < 0 || block->id >= dominance->id_count || + dominance->indices[block->id] >= 0) { + vir_dominance_release(dominance); + return false; + } + dominance->blocks[i] = (vir_block_t *) block; + dominance->indices[block->id] = i; + } + + for (i = 0; i < count * words; i++) + dominance->dom[i] = ~0UL; + for (i = 0; i < words; i++) + dominance->dom[i] = 0; + dominance->dom[0] = 1UL; + for (;;) { + bool changed = false; + for (i = 1; i < count; i++) { + const vir_edge_t *edge; + vir_block_t *current = dominance->blocks[i]; + int have_predecessor = 0; + int word; + + for (word = 0; word < words; word++) + dominance->candidate[word] = ~0UL; + for (edge = current->incoming; edge; edge = edge->next_incoming) { + int predecessor = vir_dominance_index(dominance, edge->from); + if (predecessor < 0) { + vir_dominance_release(dominance); + return false; + } + if (!have_predecessor) { + for (word = 0; word < words; word++) + dominance->candidate[word] = + dominance->dom[predecessor * words + word]; + have_predecessor = 1; + } else { + for (word = 0; word < words; word++) + dominance->candidate[word] &= + dominance->dom[predecessor * words + word]; + } + } + dominance + ->candidate[i / (int) (sizeof(unsigned long) * CHAR_BIT)] |= + 1UL << (i % (int) (sizeof(unsigned long) * CHAR_BIT)); + for (word = 0; word < words; word++) { + unsigned long *slot = &dominance->dom[i * words + word]; + if (*slot != dominance->candidate[word]) { + *slot = dominance->candidate[word]; + changed = true; + } + } + } + if (!changed) + break; + } + return true; +} + +/* Drop loop-invariant and forwarding parameters without source SSA bindings. */ +static int vir_simplify_params_with_context(vir_function_t *func, + const vir_dominance_t *dominance) +{ + int changed = 0, previous; + do { + previous = changed; + for (vir_block_t *block = func->blocks; block; block = block->next) { + int index = 0; + for (vir_value_t *param = block->params, *next; param; + param = next) { + vir_value_t *candidate = NULL; + bool trivial = true; + next = param->param_next; + for (vir_edge_t *edge = block->incoming; edge; + edge = edge->next_incoming) { + vir_value_t *value = edge->args[index]; + if (value == param) + continue; + if (candidate && candidate != value) + trivial = false; + candidate = value; + } + if (!trivial || !candidate || + !vir_replacement_dominates_with_context(func, dominance, + param, candidate)) { + index++; + continue; + } + vir_param_edit_t edit; + if (!vir_param_edit_prepare(func, block, param, + block->param_count, index, false, + &edit)) + return -1; + vir_replace_all_uses_unchecked(param, candidate); + vir_param_edit_apply(block, param, &edit); + vir_param_edit_release(&edit); + changed++; + } + } + } while (changed != previous); + return changed; +} + +int vir_simplify_params(vir_function_t *func, vir_opt_level_t opt_level) +{ + vir_dominance_t dominance; + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + if (opt_level == VIR_OPT_O0) + return 0; + if (!vir_dominance_init(func, &dominance)) + return -1; + int changed = vir_simplify_params_with_context(func, &dominance); + vir_dominance_release(&dominance); + return changed; +} + +/* An affine offset is safe to recur only while every narrower intermediate + * stays nonnegative and fits its signed type. Pointer-width arithmetic itself + * is modular, so ILP32 does not need the LP64 narrowing proof. + */ +static bool vir_sr_affine(const vir_function_t *func, + vir_value_t *value, + vir_value_t *iv, + unsigned long long limit, + unsigned long long *scale, + unsigned long long *bias, + int depth) +{ + unsigned long long multiplier = 1, add = 0; + if (value == iv) { + *scale = 1; + *bias = 0; + return true; + } + if (!value || depth == 16) + return false; + if (value->opcode == VIR_OP_SEXT || value->opcode == VIR_OP_ZEXT) { + if (!vir_sr_affine(func, value->op0, iv, limit, scale, bias, depth + 1)) + return false; + } else { + vir_value_t *operand = value->op0; + vir_value_t *constant = value->op1; + if ((value->opcode == VIR_OP_MUL || value->opcode == VIR_OP_ADD) && + vir_is_constant(operand)) { + constant = operand; + operand = value->op1; + } + if (!vir_is_constant(constant)) + return false; + if (value->opcode == VIR_OP_MUL) + multiplier = constant->constant; + else if (value->opcode == VIR_OP_SHL && constant->constant < 31) + multiplier = 1ULL << constant->constant; + else if (value->opcode == VIR_OP_ADD) + add = constant->constant; + else + return false; + if (!multiplier || multiplier > INT_MAX || add > INT_MAX || + !vir_sr_affine(func, operand, iv, limit, scale, bias, depth + 1) || + *scale > 0x7fffffffffffffffULL / multiplier || + *bias > (0x7fffffffffffffffULL - add) / multiplier) + return false; + *scale *= multiplier; + *bias = *bias * multiplier + add; + } + int bits = vir_integer_type_width(value->type); + unsigned long long maximum = + bits == 64 ? 0x7fffffffffffffffULL : (1ULL << (bits - 1)) - 1; + return bits && (func->pointer_bits == 32 || + (*bias <= maximum && *scale <= (maximum - *bias) / limit)); +} + +/* Root addresses are harmless to rematerialize even when first mentioned in the + * loop. Other pure base expressions move only with invariant operands. + */ +static vir_value_t *vir_sr_base(vir_function_t *func, + const vir_dominance_t *dominance, + vir_block_t *pre, + vir_value_t *value, + int depth) +{ + if (vir_value_dominates(func, dominance, value, pre, INT_MAX)) + return value; + if (depth == 16 || value->is_block_param || value->def_effect || + vir_opcode_may_trap(value->opcode)) + return NULL; + if (value->opcode == VIR_OP_GLOBAL_ADDR || + value->opcode == VIR_OP_STACK_ADDR) + for (vir_value_t *root = pre->head; root; root = root->next) + if (root->opcode == value->opcode && + (value->opcode == VIR_OP_STACK_ADDR + ? root->address_slot == value->address_slot + : !strcmp(root->address_name, value->address_name))) + return root; + if (value->opcode == VIR_OP_GLOBAL_ADDR) + return vir_global_addr(func, pre, value->address_name, + value->address_size, value->address_alignment); + if (value->opcode == VIR_OP_STACK_ADDR) + return vir_stack_addr(func, pre, value->address_slot, + value->address_size, value->address_alignment); + if (value->opcode == VIR_OP_CONST) + return vir_const(func, pre, value->type, value->constant); + if (!value->nr_ops) + return NULL; + vir_value_t *left = + vir_sr_base(func, dominance, pre, value->op0, depth + 1); + vir_value_t *right = value->nr_ops == 2 ? vir_sr_base(func, dominance, pre, + value->op1, depth + 1) + : NULL; + return left && (value->nr_ops == 1 || right) + ? vir_user_operation(func, pre, value->opcode, value->type, left, + right) + : NULL; +} + +typedef struct { + unsigned long long first; + unsigned long long step; +} vir_sr_offset_t; + +static int vir_sr_pointer(vir_function_t *func, + vir_block_t *header, + vir_edge_t *entry, + vir_edge_t *back, + vir_value_t *address, + vir_value_t *base, + const vir_sr_offset_t *offsets, + vir_value_t **pointer) +{ + int index = 0; + unsigned long long first = offsets->first; + for (vir_value_t *param = header->params; param; + param = param->param_next, index++) { + vir_value_t *initial = entry->args[index]; + vir_value_t *next = back->args[index]; + bool same_initial = !first ? initial == base + : initial->opcode == VIR_OP_PTRADD && + initial->op0 == base && + vir_is_const_int(initial->op1, first); + if (param->type == VIR_TYPE_PTR && same_initial && + next->opcode == VIR_OP_PTRADD && next->op0 == param && + vir_is_const_int(next->op1, offsets->step)) { + if (!vir_replace_all_uses(func, address, param)) + return -1; + *pointer = param; + return 1; + } + } + vir_param_edit_t edit; + vir_type_t width = func->pointer_bits == 64 ? VIR_TYPE_I64 : VIR_TYPE_I32; + if (!vir_param_edit_prepare(func, header, NULL, header->param_count, + header->param_count, true, &edit)) + return -1; + vir_value_t *offset = vir_const_int(func, entry->from, width, first); + vir_value_t *initial = + offset ? vir_ptradd(func, entry->from, base, offset) : NULL; + vir_value_t *step = vir_const_int(func, back->from, width, offsets->step); + vir_value_t *param = vir_value_alloc(func, header, VIR_TYPE_PTR); + vir_value_t *next = + initial && step && param + ? vir_user_operation(func, back->from, VIR_OP_PTRADD, VIR_TYPE_PTR, + param, step) + : NULL; + if (!next) { + vir_param_edit_release(&edit); + return -1; + } + for (int i = 0; i < edit.count; i++) + edit.edges[i].inserted = edit.edges[i].edge == entry ? initial : next; + vir_param_edit_apply(header, param, &edit); + vir_param_edit_release(&edit); + if (!vir_replace_all_uses(func, address, param)) + return -1; + *pointer = param; + return 1; +} + +/* Frontend boolean conversions may invert the comparison feeding a branch. */ +static vir_value_t *vir_sr_condition(vir_value_t *value, bool *positive) +{ + for (int depth = 0; value && depth < 16; depth++) { + if (value->opcode == VIR_OP_ZEXT && value->op0->type == VIR_TYPE_I1) { + value = value->op0; + } else if (value->opcode == VIR_OP_EQ && + ((vir_is_constant(value->op0) && !value->op0->constant) || + (vir_is_constant(value->op1) && !value->op1->constant))) { + value = vir_is_constant(value->op0) ? value->op1 : value->op0; + *positive = !*positive; + } else { + return value; + } + } + return NULL; +} + +/* A unique latch incremented by one under a signed i < bound test cannot wrap + * before exiting. Literal bounds additionally allow a pointer equality exit + * test and retirement of an otherwise unused counter recurrence. + */ +int vir_strength_reduce(vir_function_t *func, vir_opt_level_t opt_level) +{ + vir_dominance_t dominance; + int changed = 0; + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + if (opt_level == VIR_OPT_O0) + return 0; + if (vir_dce(func, opt_level) < 0 || !vir_dominance_init(func, &dominance)) + return -1; + if (vir_simplify_params_with_context(func, &dominance) < 0) { + vir_dominance_release(&dominance); + return -1; + } + for (vir_block_t *header = func->blocks; header; header = header->next) { + vir_edge_t *entry = NULL, *back = NULL; + bool positive = true; + vir_value_t *branch = header->branch_condition; + vir_value_t *condition = vir_sr_condition(branch, &positive); + if (header->terminator != VIR_TERM_BRANCH || !condition || + condition->opcode != VIR_OP_SLT || + condition->op0->type != VIR_TYPE_I32) + continue; + for (vir_edge_t *edge = header->incoming; edge; + edge = edge->next_incoming) { + if (vir_dominance_block_dominates(&dominance, header, edge->from)) { + if (back) { + back = NULL; + break; + } + back = edge; + } else { + if (entry) { + entry = NULL; + break; + } + entry = edge; + } + } + vir_value_t *iv = condition->op0; + vir_value_t *limit = condition->op1; + bool inclusive = vir_is_constant(iv) && limit->is_block_param; + if (inclusive) { + iv = condition->op1; + limit = condition->op0; + positive = !positive; + } + vir_edge_t *body_edge = + positive ? header->true_edge : header->false_edge; + if (!entry || !back || entry->from->terminator != VIR_TERM_JUMP || + back->from->terminator != VIR_TERM_JUMP || back->from == header || + !vir_dominance_block_dominates(&dominance, body_edge->to, + back->from) || + (!vir_is_constant(limit) && + vir_dominance_block_dominates(&dominance, header, limit->block))) + continue; + int index = 0; + for (vir_value_t *param = header->params; param && param != iv; + param = param->param_next) + index++; + if (!iv->is_block_param || iv->block != header || + index == header->param_count) + continue; + vir_value_t *initial = entry->args[index]; + vir_value_t *increment = back->args[index]; + if (!vir_is_constant(initial) || initial->constant > INT_MAX || + increment->opcode != VIR_OP_ADD || + !((increment->op0 == iv && vir_is_const_int(increment->op1, 1)) || + (increment->op1 == iv && vir_is_const_int(increment->op0, 1)))) + continue; + unsigned long long bound = vir_is_constant(limit) + ? limit->constant + (inclusive ? 1 : 0) + : INT_MAX; + if (!bound || bound > INT_MAX || initial->constant >= bound) + continue; + vir_value_t *walk = NULL, *walk_base = NULL; + unsigned long long walk_scale = 0, walk_bias = 0; + for (vir_block_t *block = func->blocks; block; block = block->next) { + if (!vir_dominance_block_dominates(&dominance, header, block) || + !vir_dominance_block_dominates(&dominance, block, back->from)) + continue; + for (vir_value_t *address = block->head, *next; address; + address = next) { + next = address->next; + unsigned long long scale, bias; + if (address->opcode != VIR_OP_PTRADD || !address->uses || + !vir_sr_affine(func, address->op1, iv, bound, &scale, &bias, + 0)) + continue; + vir_value_t *pointer; + vir_value_t *base = + vir_sr_base(func, &dominance, entry->from, address->op0, 0); + if (!base) + continue; + vir_sr_offset_t offsets = { + (initial->constant * scale + bias) & + vir_width_mask(func->pointer_bits), + scale}; + if (vir_sr_pointer(func, header, entry, back, address, base, + &offsets, &pointer) < 0) { + changed = -1; + goto done; + } + changed++; + walk = pointer; + walk_base = base; + walk_scale = scale; + walk_bias = bias; + } + } + if (!walk || !vir_is_constant(limit) || + (bound - initial->constant) > + vir_width_mask(func->pointer_bits) / walk_scale || + !branch->uses || branch->uses->next || + branch->uses->branch_block != header) + continue; + + /* The equality endpoint works even when the concrete pointer wraps: the + * bounded span cannot complete a full address-space revolution. + */ + vir_type_t width = + func->pointer_bits == 64 ? VIR_TYPE_I64 : VIR_TYPE_I32; + vir_value_t *offset = vir_const_int(func, entry->from, width, + bound * walk_scale + walk_bias); + vir_value_t *end = + offset ? vir_ptradd(func, entry->from, walk_base, offset) : NULL; + vir_value_t *equal = end ? vir_eq(func, header, walk, end) : NULL; + vir_value_t *zero = vir_const_i1(func, header, 0); + vir_value_t *test = + positive + ? (equal && zero ? vir_eq(func, header, equal, zero) : NULL) + : equal; + if (!test || !vir_replace_all_uses(func, branch, test)) { + changed = -1; + goto done; + } + if (vir_dce(func, opt_level) < 0) { + changed = -1; + goto done; + } + if (iv->uses && !iv->uses->next && iv->uses->user == increment && + increment->uses && !increment->uses->next && + increment->uses->edge == back && + increment->uses->operand == index) { + vir_param_edit_t edit; + if (!vir_param_edit_prepare(func, header, iv, header->param_count, + index, false, &edit)) { + changed = -1; + goto done; + } + vir_replace_all_uses_unchecked(iv, initial); + vir_param_edit_apply(header, iv, &edit); + vir_param_edit_release(&edit); + } + } +done: + vir_dominance_release(&dominance); + return changed; +} + +static bool vir_value_dominates_effect(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_value_t *value, + const vir_effect_t *effect) +{ + vir_value_t **pending; + unsigned char *seen; + int count; + int pending_count = 0; + int position; + bool dominates = true; + + if (!func || !value || !effect || !effect->block) + return false; + position = effect->block->tail ? effect->block->tail->position + 1 : 0; + if (value->def_effect) { + if (value->def_effect->block == effect->block) + return value->def_effect->position < effect->position; + return vir_value_dominates(func, dominance, value, effect->block, + position); + } + if (value->block == effect->block && + (value->is_block_param || value->order <= effect->order)) + return true; + if (value->is_block_param || value->nr_ops == 0) + return vir_value_dominates(func, dominance, value, effect->block, + position); + count = func->next_value_id; + if (value->id < 0 || value->id >= count || + count > INT_MAX / (int) sizeof(*pending)) + return false; + seen = calloc((size_t) count, sizeof(*seen)); + pending = malloc((size_t) count * sizeof(*pending)); + if (!seen || !pending) { + free(seen); + free(pending); + return false; + } + pending[pending_count++] = (vir_value_t *) value; + seen[value->id] = 1; + while (pending_count && dominates) { + const vir_value_t *current = pending[--pending_count]; + + if (current->def_effect) { + if (current->def_effect->block == effect->block) + dominates = current->def_effect->position < effect->position; + else + dominates = vir_value_dominates(func, dominance, current, + effect->block, position); + continue; + } + if (current->block != effect->block && + !vir_value_dominates(func, dominance, current, effect->block, + position)) { + dominates = false; + continue; + } + if (current->is_block_param || (current->block == effect->block && + current->order <= effect->order)) + continue; + if (current->nr_ops < 0 || current->nr_ops > 2) { + dominates = false; + continue; + } + for (int i = 0; i < current->nr_ops; i++) { + const vir_value_t *operand = i ? current->op1 : current->op0; + + if (!operand || operand->id < 0 || operand->id >= count) { + dominates = false; + break; + } + if (!seen[operand->id]) { + seen[operand->id] = 1; + pending[pending_count++] = (vir_value_t *) operand; + } + } + } + free(pending); + free(seen); + return dominates; +} + +static int vir_block_id_count(const vir_function_t *func, int id) +{ + const vir_block_t *block; + int count = 0; + for (block = func->blocks; block; block = block->next) + if (block->id == id) + count++; + return count; +} + +static bool vir_value_uses_are_dead(const vir_function_t *func, + const vir_value_t *value, + const unsigned char *reachable) +{ + const vir_use_t *use; + for (use = value->uses; use; use = use->next) { + int index = vir_block_index(func, vir_use_block(use)); + + if (index < 0 || reachable[index]) + return 0; + } + return 1; +} + +int vir_remove_unreachable(vir_function_t *func) +{ + vir_block_t *block; + vir_block_t **link; + unsigned char *reachable; + int *work; + int count; + int i; + + if (!func || !func->blocks) + return 0; + count = vir_function_block_count(func); + if (count > INT_MAX / (int) sizeof(*reachable) || + count > INT_MAX / (int) sizeof(*work)) + return 0; + reachable = calloc((size_t) count, sizeof(*reachable)); + work = malloc(sizeof(*work) * (size_t) count); + if (!reachable || !work) { + free(reachable); + free(work); + return 0; + } + if (!vir_mark_reachable_blocks(func, NULL, reachable, work)) { + free(reachable); + free(work); + return 0; + } + for (block = func->blocks, i = 0; block; block = block->next, i++) { + const vir_value_t *value; + if (reachable[i]) + continue; + for (value = block->params; value; value = value->param_next) + if (!vir_value_uses_are_dead(func, value, reachable)) + goto fail; + for (value = block->head; value; value = value->next) + if (!vir_value_uses_are_dead(func, value, reachable)) + goto fail; + } + for (block = func->blocks, i = 0; block; block = block->next, i++) { + vir_value_t *value; + if (reachable[i]) + continue; + while (block->outgoing) + vir_edge_unlink(block->outgoing); + if (block->terminator == VIR_TERM_RETURN && block->return_value) + vir_remove_return_use(block->return_value, block); + else if (block->terminator == VIR_TERM_BRANCH) + vir_remove_branch_use(block->branch_condition, block); + { + vir_effect_t *effect; + for (effect = block->effects; effect; effect = effect->next) + vir_remove_effect_uses(effect); + } + for (value = block->head; value; value = value->next) + for (int operand = 0; operand < value->nr_ops; operand++) + vir_remove_user_use(operand ? value->op1 : value->op0, value, + operand); + } + link = &func->blocks; + func->last_block = NULL; + for (i = 0; *link;) { + block = *link; + if (!reachable[i]) + *link = block->next; + else { + func->last_block = block; + link = &block->next; + } + i++; + } + free(reachable); + free(work); + return 1; + +fail: + free(reachable); + free(work); + return 0; +} + +static bool vir_is_empty_jump_forwarder(const vir_function_t *func, + const vir_block_t *block) +{ + const vir_edge_t *edge; + + return block != func->blocks && block->incoming && !block->params && + !block->head && !block->effects && + block->terminator == VIR_TERM_JUMP && (edge = block->outgoing) && + !edge->next_outgoing && !edge->arg_count && edge->to != block; +} + +/* Replace jumps into a return-only parameter block with direct returns. */ +static int vir_inline_return_param(vir_function_t *func, vir_block_t *block) +{ + vir_value_t *result; + vir_use_t *use; + vir_edge_t *edge; + int index = 0; + + if (block == func->blocks || block->terminator != VIR_TERM_RETURN || + block->head || block->effects || !(result = block->return_value) || + !result->is_block_param || result->block != block) + return 0; + for (vir_value_t *param = block->params; param != result; + param = param->param_next) { + if (!param) + return 0; + index++; + } + for (edge = block->incoming; edge; edge = edge->next_incoming) + if (!edge->from || edge->from->terminator != VIR_TERM_JUMP || + edge->from->outgoing != edge || edge->next_outgoing || + edge->arg_count != block->param_count || !edge->args || + index >= edge->arg_count) + return 0; + if (!block->incoming) + return 0; + for (edge = block->incoming; edge;) { + vir_edge_t *next = edge->next_incoming; + vir_block_t *from = edge->from; + vir_value_t *value = edge->args[index]; + + if (!(use = vir_arena_alloc(&func->arena, sizeof(*use)))) { + return -1; + } + if (!vir_edge_delete(func, edge)) + return -1; + from->terminator = VIR_TERM_RETURN; + from->return_value = value; + use->return_block = from; + vir_link_use(value, use); + edge = next; + } + vir_remove_return_use(result, block); + return 1; +} + +/* A branch on "c == true" is a branch on c, and one on "c == false" is a branch + * on c with its edges swapped. The frontend spells "x != 0" as (x == 0) == + * false, so this leaves one compare per test, which a backend fuses with its + * jump, and DCE drops the outer compare once the branch was its only use. + * Equality operands are ordered by id, so the constant may be either one. + */ +static bool vir_fold_branch_condition(vir_block_t *block) +{ + vir_value_t *condition = block->branch_condition; + vir_value_t *operand; + vir_value_t *constant; + vir_use_t *use; + + if (block->terminator != VIR_TERM_BRANCH || !condition || + condition->opcode != VIR_OP_EQ || !condition->op0 || !condition->op1) + return false; + operand = condition->op0; + constant = condition->op1; + if (operand->opcode == VIR_OP_CONST) { + operand = condition->op1; + constant = condition->op0; + } + if (constant->opcode != VIR_OP_CONST || operand->opcode == VIR_OP_CONST) + return false; + if (operand->type != VIR_TYPE_I1) { + /* C promotes comparison results to int before testing them again. Zero + * extension preserves the boolean values zero and one. + */ + if (operand->opcode != VIR_OP_ZEXT || !operand->op0 || + operand->op0->type != VIR_TYPE_I1 || constant->constant > 1) + return false; + operand = operand->op0; + } + for (use = condition->uses; use && use->branch_block != block; + use = use->next) + ; + if (!use) + return false; + vir_set_use_slot(&block->branch_condition, operand, use); + if (!constant->constant) { + vir_edge_t *edge = block->true_edge; + + block->true_edge = block->false_edge; + block->false_edge = edge; + } + return true; +} + +int vir_simplify_cfg_with_pruning(vir_function_t *func, + vir_opt_level_t opt_level, + bool prune_unreachable) +{ + int changes = 0; + bool changed; + + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + if (opt_level == VIR_OPT_O0) + return 0; + /* Folding a condition changes no block, so it is not counted. */ + for (vir_block_t *block = func->blocks; block; block = block->next) + while (vir_fold_branch_condition(block)) + ; + do { + vir_block_t *block; + + changed = false; + for (block = func->blocks; block; block = block->next) { + vir_edge_t *edge; + vir_block_t *target; + int inlined; + + inlined = vir_inline_return_param(func, block); + if (inlined < 0) + return -1; + if (inlined) { + changes++; + changed = true; + break; + } + + if (!vir_is_empty_jump_forwarder(func, block)) + continue; + target = block->outgoing->to; + for (edge = block->incoming; edge;) { + vir_edge_t *next = edge->next_incoming; + + if (!vir_edge_redirect(func, edge, target)) + return -1; + edge = next; + } + if (block->incoming) + return -1; + changes++; + changed = true; + break; + } + } while (changed); + if (changes && prune_unreachable && !vir_remove_unreachable(func)) + return -1; + return changes; +} + +int vir_simplify_cfg(vir_function_t *func, vir_opt_level_t opt_level) +{ + return vir_simplify_cfg_with_pruning(func, opt_level, true); +} + +/* Dominance is queried by reachability only; no analysis state is retained in + * the IR or allocated as an all-pairs matrix. + */ +static bool vir_value_dominates(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_value_t *value, + const vir_block_t *use_block, + int use_position) +{ + if (!value || !use_block || (!dominance && (!func || !func->blocks))) + return false; + if (value->block == use_block) + return value->is_block_param || value->position < use_position; + if (dominance) + return vir_dominance_block_dominates(dominance, value->block, + use_block); + return vir_value_dominates_by_reachability(func, value, use_block, + vir_function_block_count(func)); +} + +static bool vir_licm_add_temporary_bytes(size_t *bytes, size_t amount) +{ + if (amount > (size_t) INT_MAX - *bytes) + return false; + *bytes += amount; + return true; +} + +static void vir_licm_mark_loop(const vir_block_t *header, + const vir_block_t *latch, + const vir_dominance_t *dominance, + unsigned char *in_loop, + int *work, + int *value_blocks) +{ + int head = 0; + int tail = 0; + int header_index = vir_dominance_index(dominance, header); + int latch_index = vir_dominance_index(dominance, latch); + + if (header_index < 0 || latch_index < 0) + return; + if (!in_loop[header_index]) { + in_loop[header_index] = 1; + if (dominance->blocks[header_index]->head) + (*value_blocks)++; + } + if (!in_loop[latch_index]) { + in_loop[latch_index] = 1; + if (dominance->blocks[latch_index]->head) + (*value_blocks)++; + work[tail++] = latch_index; + } + while (head < tail) { + const vir_edge_t *edge; + const vir_block_t *block = dominance->blocks[work[head++]]; + + if (block == header) + continue; + for (edge = block->incoming; edge; edge = edge->next_incoming) { + int pred = vir_dominance_index(dominance, edge->from); + + if (pred < 0 || in_loop[pred]) + continue; + in_loop[pred] = 1; + if (dominance->blocks[pred]->head) + (*value_blocks)++; + work[tail++] = pred; + } + } +} + +static vir_block_t *vir_licm_preheader(const vir_block_t *header, + const vir_dominance_t *dominance, + const unsigned char *in_loop) +{ + const vir_edge_t *edge; + vir_block_t *preheader = NULL; + + for (edge = header->incoming; edge; edge = edge->next_incoming) { + int pred = vir_dominance_index(dominance, edge->from); + + if (pred < 0) + return NULL; + if (!in_loop[pred]) { + if (preheader) + return NULL; + preheader = edge->from; + } + } + return preheader; +} + +static bool vir_licm_value_is_candidate(const vir_value_t *value) +{ + return !value->is_block_param && !value->def_effect && value->nr_ops && + !vir_opcode_may_trap(value->opcode); +} + +#define VIR_LICM_EAGER_VALUE_BLOCK_LIMIT 128 + +/* Return an indexed candidate's number of loop-local operands; -1 excludes + * values that LICM cannot move. Reverse uses release dependents as operands + * move. + */ +static int vir_licm_value_loop_dependencies(vir_value_t *value, + int value_count, + const vir_dominance_t *dominance, + const unsigned char *in_loop) +{ + int operand; + int dependencies = 0; + + if (value->id < 0 || value->id >= value_count || + !vir_licm_value_is_candidate(value)) + return -1; + for (operand = 0; operand < value->nr_ops; operand++) { + vir_value_t *input = operand ? value->op1 : value->op0; + int index = vir_dominance_index(dominance, input->block); + + if (index < 0) + return -1; + if (in_loop[index]) + dependencies++; + } + return dependencies; +} + +/* Large sparse loops should go straight to the value worklist. Sample a + * bounded, evenly spaced set of value-bearing blocks before selecting the eager + * scan, so one ready root cannot make a long dependency chain look dense. At + * least half of the sampled candidates must be ready. Inspect no more than the + * fixed value limit in each selected block. + */ +static bool vir_licm_readiness_is_dense(vir_block_t **blocks, + int count, + int value_blocks, + int value_count, + const vir_dominance_t *dominance, + const unsigned char *in_loop, + int *sampled_values) +{ + int ordinal = 0; + int sample_index = 0; + int sample_ordinal = 0; + int candidates = 0; + int ready = 0; + + if (sampled_values) + *sampled_values = 0; + + for (int i = 0; i < count; i++) { + vir_value_t *value; + + if (!in_loop[i] || !blocks[i]->head) + continue; + if (ordinal == sample_ordinal) { + int inspected = 0; + + for (value = blocks[i]->head; + value && inspected < VIR_LICM_READINESS_SAMPLE_VALUE_LIMIT; + value = value->next, inspected++) { + int dependencies; + + if (sampled_values) + (*sampled_values)++; + dependencies = vir_licm_value_loop_dependencies( + value, value_count, dominance, in_loop); + if (dependencies < 0) + continue; + candidates++; + if (!dependencies) + ready++; + } + if (++sample_index == VIR_LICM_READINESS_SAMPLE_BLOCK_LIMIT) + break; + sample_ordinal = + (int) ((uint64_t) sample_index * (uint64_t) (value_blocks - 1) / + (VIR_LICM_READINESS_SAMPLE_BLOCK_LIMIT - 1)); + } + ordinal++; + } + return candidates > 0 && ready >= candidates / 2 + candidates % 2; +} + +static void vir_licm_move_value(vir_block_t *from, + vir_block_t *to, + vir_value_t *value) +{ + vir_value_t **link = &from->head; + + while (*link != value) + link = &(*link)->next; + *link = value->next; + value->next = NULL; + if (to->tail) + to->tail->next = value; + else + to->head = value; + to->tail = value; + value->block = to; +} + +static void vir_licm_release(unsigned char *in_loop, + int *work, + int *pending, + vir_value_t **ready, + vir_dominance_t *dominance) +{ + free(in_loop); + free(work); + free(pending); + free(ready); + vir_dominance_release(dominance); +} + +/* LICM borrows VIR's bitset dominance context for this pass only; the analysis + * is released on every exit and is never stored in the IR. + */ +int vir_licm_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + int *stats) +{ + vir_dominance_t dominance; + vir_block_t **blocks; + unsigned char *in_loop; + int *work; + int *pending = NULL; + vir_value_t **ready = NULL; + int count; + int value_count; + int bytes; + size_t temporary_bytes = 0; + int hoisted = 0; + int header_index; + + if (stats) + memset(stats, 0, VIR_LICM_STAT_COUNT * sizeof(*stats)); + if (!func || !vir_opt_level_valid(opt_level)) + return -1; + memset(&dominance, 0, sizeof(dominance)); + if (opt_level == VIR_OPT_O0 || !(count = vir_function_block_count(func))) + return 0; + if (count > INT_MAX / (int) sizeof(*blocks) || + count > INT_MAX / (int) sizeof(*in_loop) || + count > INT_MAX / (int) sizeof(*work)) + return -1; + if (!vir_dominance_init(func, &dominance)) + return -1; + count = dominance.count; + value_count = func->next_value_id; + blocks = dominance.blocks; + if (!vir_licm_add_temporary_bytes( + &temporary_bytes, (size_t) count * sizeof(*dominance.blocks)) || + !vir_licm_add_temporary_bytes( + &temporary_bytes, + (size_t) dominance.id_count * sizeof(*dominance.indices)) || + !vir_licm_add_temporary_bytes( + &temporary_bytes, (size_t) count * (size_t) dominance.words * + sizeof(*dominance.dom)) || + !vir_licm_add_temporary_bytes( + &temporary_bytes, + (size_t) dominance.words * sizeof(*dominance.candidate)) || + !vir_licm_add_temporary_bytes(&temporary_bytes, + (size_t) count * sizeof(*in_loop)) || + !vir_licm_add_temporary_bytes(&temporary_bytes, + (size_t) count * sizeof(*work))) { + vir_dominance_release(&dominance); + return -1; + } + bytes = (int) temporary_bytes; + in_loop = calloc(count, sizeof(*in_loop)); + work = malloc(count * sizeof(*work)); + if (!in_loop || !work) { + vir_licm_release(in_loop, work, pending, ready, &dominance); + return -1; + } + if (stats) + stats[VIR_LICM_STAT_TEMPORARY_BYTES] = bytes; + for (header_index = 0; header_index < count; header_index++) { + vir_block_t *header = blocks[header_index]; + const vir_edge_t *edge; + bool has_backedge = false; + int value_blocks = 0; + + memset(in_loop, 0, count * sizeof(*in_loop)); + for (edge = header->incoming; edge; edge = edge->next_incoming) { + int latch_index = vir_dominance_index(&dominance, edge->from); + + if (latch_index < 0 || + !vir_dominance_block_dominates(&dominance, header, edge->from)) + continue; + vir_licm_mark_loop(header, edge->from, &dominance, in_loop, work, + &value_blocks); + has_backedge = true; + } + if (!has_backedge) + continue; + vir_block_t *preheader = + vir_licm_preheader(header, &dominance, in_loop); + int ready_count = 0; + int ready_head = 0; + + if (!preheader) + continue; + if (stats) + stats[VIR_LICM_STAT_LOOPS]++; + + /* Concentrated loop values are cheap to try in source order. Hoist + * ready values immediately, avoiding the value-ID queues when this pass + * drains the candidates or finds no ready root. + */ + bool eager_scan = value_blocks <= VIR_LICM_EAGER_VALUE_BLOCK_LIMIT; + + if (!eager_scan) { + int sampled_values = 0; + + eager_scan = vir_licm_readiness_is_dense( + blocks, count, value_blocks, value_count, &dominance, in_loop, + stats ? &sampled_values : NULL); + if (stats) + stats[VIR_LICM_STAT_READINESS_SAMPLE_VALUES] += sampled_values; + if (stats && eager_scan) + stats[VIR_LICM_STAT_READINESS_EAGER_ADMISSIONS]++; + } + if (eager_scan) { + int candidate_count = 0; + int fast_hoisted = 0; + + for (int i = 0; i < count; i++) { + vir_value_t *value; + + if (!in_loop[i]) + continue; + value = blocks[i]->head; + while (value) { + vir_value_t *next = value->next; + int dependencies = vir_licm_value_loop_dependencies( + value, value_count, &dominance, in_loop); + + if (dependencies >= 0) { + candidate_count++; + if (!dependencies) { + vir_licm_move_value(blocks[i], preheader, value); + hoisted++; + fast_hoisted++; + } + } + value = next; + } + } + + if (!candidate_count || fast_hoisted == candidate_count) { + if (fast_hoisted) { + for (int i = 0; i < count; i++) + if (in_loop[i]) + vir_repair_value_positions(blocks[i]); + vir_repair_value_positions(preheader); + } + continue; + } + if (!fast_hoisted) + continue; + } + if (!pending && value_count > 0) { + if (value_count > INT_MAX / (int) sizeof(*pending) || + value_count > INT_MAX / (int) sizeof(*ready) || + !vir_licm_add_temporary_bytes( + &temporary_bytes, + (size_t) value_count * sizeof(*pending)) || + !vir_licm_add_temporary_bytes( + &temporary_bytes, (size_t) value_count * sizeof(*ready))) { + vir_licm_release(in_loop, work, pending, ready, &dominance); + return -1; + } + pending = malloc((size_t) value_count * sizeof(*pending)); + ready = malloc((size_t) value_count * sizeof(*ready)); + if (!pending || !ready) { + vir_licm_release(in_loop, work, pending, ready, &dominance); + return -1; + } + bytes = (int) temporary_bytes; + if (stats) + stats[VIR_LICM_STAT_TEMPORARY_BYTES] = bytes; + if (stats) + stats[VIR_LICM_STAT_VALUE_WORKLIST_ALLOCS]++; + } + for (int i = 0; i < count; i++) { + vir_value_t *value; + + if (!in_loop[i]) + continue; + for (value = blocks[i]->head; value; value = value->next) { + int dependencies; + + if (value->id < 0 || value->id >= value_count) + continue; + dependencies = vir_licm_value_loop_dependencies( + value, value_count, &dominance, in_loop); + pending[value->id] = dependencies; + if (dependencies == 0) + ready[ready_count++] = value; + } + } + while (ready_head < ready_count) { + vir_value_t *value = ready[ready_head++]; + vir_use_t *use; + + vir_licm_move_value(value->block, preheader, value); + hoisted++; + for (use = value->uses; use; use = use->next) { + vir_value_t *user = use->user; + int user_block; + + if (!user || user->id < 0 || user->id >= value_count) + continue; + user_block = vir_dominance_index(&dominance, user->block); + if (user_block < 0 || !in_loop[user_block]) + continue; + if (pending[user->id] <= 0) + continue; + if (--pending[user->id] == 0) + ready[ready_count++] = user; + } + } + for (int i = 0; i < count; i++) + if (in_loop[i]) + vir_repair_value_positions(blocks[i]); + vir_repair_value_positions(preheader); + } + vir_licm_release(in_loop, work, pending, ready, &dominance); + if (stats) + stats[VIR_LICM_STAT_VALUES_HOISTED] = hoisted; + return hoisted; +} + +int vir_licm(vir_function_t *func, vir_opt_level_t opt_level) +{ + return vir_licm_with_stats(func, opt_level, NULL); +} + +static bool vir_verify_reverse_uses(const vir_function_t *func, + const vir_value_t *value) +{ + const vir_use_t *use; + for (use = value->uses; use; use = use->next) { + if (!!use->user + !!use->edge + !!use->effect + !!use->return_block + + !!use->branch_block != + 1) + return 0; + if (use->user) { + if (!vir_has_value(func, use->user) || + (use->operand != 0 && use->operand != 1) || + (use->operand == 0 ? use->user->op0 : use->user->op1) != value) + return 0; + } else if (use->edge) { + if (!vir_has_block(func, use->edge->from) || + !vir_has_block(func, use->edge->to) || + !vir_has_outgoing_edge(use->edge->from, use->edge) || + use->operand < 0 || use->operand >= use->edge->arg_count || + use->edge->args[use->operand] != value) + return 0; + } else if (use->effect) { + const vir_effect_t *effect = use->effect; + const vir_value_t *effect_value; + if (!vir_has_block(func, effect->block) || + !vir_has_effect(effect->block, effect)) + return 0; + if (effect->kind == VIR_EFFECT_CALL) { + if (use->operand == -1) { + if (effect->callee_value_use != use || + effect->callee_value_use->effect != effect || + effect->callee_value_use->operand != -1) + return 0; + effect_value = effect->callee_value; + } else { + if (use->operand < 0 || use->operand >= effect->arg_count || + !effect->args || !effect->arg_uses || + effect->arg_uses[use->operand].effect != effect || + effect->arg_uses[use->operand].operand != use->operand) + return 0; + effect_value = effect->args[use->operand]; + } + } else { + if (use->operand != 0 && use->operand != 1) + return 0; + effect_value = + use->operand == 0 ? effect->address : effect->stored_value; + } + if (effect_value != value) + return 0; + } else if (use->return_block) { + if (!vir_has_block(func, use->return_block) || + use->return_block->terminator != VIR_TERM_RETURN || + use->return_block->return_value != value) + return 0; + } else if (!vir_has_block(func, use->branch_block) || + use->branch_block->terminator != VIR_TERM_BRANCH || + use->branch_block->branch_condition != value) { + return 0; + } + } + return 1; +} + +static int vir_verify_error(char **error, char *message) +{ + if (error) + *error = message; + return 0; +} + +static int vir_verify_invalid_value(char **error) +{ + return vir_verify_error(error, "invalid value/use structure"); +} + +static int vir_verify_invalid_order(char **error) +{ + return vir_verify_error(error, "invalid block order"); +} + +static bool vir_has_no_address_metadata(const vir_value_t *value) +{ + return value->address_kind == VIR_ADDRESS_NONE && !value->address_slot && + !value->address_size && !value->address_alignment && + !value->address_name; +} + +static bool vir_effect_has_call_metadata(const vir_effect_t *effect) +{ + return effect->callee || effect->callee_value || effect->signature || + effect->callee_value_use || effect->args || effect->arg_uses || + effect->arg_count || effect->result_type; +} + +static bool vir_verify_address_value(const vir_value_t *value) +{ + if (value->type != VIR_TYPE_PTR || value->constant || value->op0 || + value->op1 || value->def_effect || !value->address_size || + !vir_valid_alignment(value->address_alignment)) + return 0; + if (value->opcode == VIR_OP_STACK_ADDR) + return value->address_kind == VIR_ADDRESS_STACK && !value->address_name; + return value->opcode == VIR_OP_GLOBAL_ADDR && + value->address_kind == VIR_ADDRESS_GLOBAL && !value->address_slot && + value->address_name && *value->address_name; +} + +static bool vir_verify_effect_operand(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_effect_t *effect, + const vir_value_t *value, + const vir_use_t *use, + int operand) +{ + return value && vir_has_value(func, value) && use && + use->effect == effect && use->operand == operand && + vir_value_dominates_effect(func, dominance, value, effect) && + vir_effect_use_count(value, effect, operand) == 1; +} + +static bool vir_verify_value_operands(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_value_t *value) +{ + if (value->nr_ops < 1 || value->nr_ops > 2) + return false; + for (int i = 0; i < value->nr_ops; i++) { + const vir_value_t *operand = i ? value->op1 : value->op0; + + if (!operand || !vir_has_value(func, operand) || + !vir_type_width(func, operand->type) || + vir_use_count(operand, value, i) != 1 || + !vir_value_dominates(func, dominance, operand, value->block, + value->position)) + return false; + } + return vir_type_width(func, value->type) && + (value->nr_ops == 1 + ? vir_unary_types_valid(func, value->opcode, value->type, + value->op0->type) + : vir_binary_types_valid(func, value->opcode, value->type, + value->op0->type, value->op1->type)); +} + +static int vir_verify_block(const vir_function_t *func, + const vir_dominance_t *dominance, + const vir_block_t *block, + char **error) +{ + const vir_value_t *value; + const vir_edge_t *edge; + const vir_effect_t *effect; + int position = 0; + int terminator_position; + int param_count = 0; + if (block->id < 0 || vir_block_id_count(func, block->id) != 1) + return vir_verify_invalid_order(error); + for (value = block->params; value; value = value->param_next) { + if (value->block != block || !value->is_block_param || value->nr_ops || + !vir_type_width(func, value->type) || value->id < 0 || + !vir_has_no_address_metadata(value) || + vir_value_id_count(func, value->id) != 1) + return vir_verify_invalid_value(error); + if (!vir_verify_reverse_uses(func, value)) + return vir_verify_invalid_value(error); + param_count++; + } + if ((block->params == NULL) != (block->last_param == NULL) || + (block->last_param && block->last_param->param_next) || + block->param_count != param_count) + return vir_verify_invalid_order(error); + for (value = block->head; value; value = value->next) { + if (value->block != block || !vir_has_block(func, value->block) || + value->is_block_param || value->position != position || + value->id < 0 || value->order < 0 || + value->order >= block->next_order) + return vir_verify_invalid_value(error); + position++; + if (vir_value_id_count(func, value->id) != 1) + return vir_verify_invalid_value(error); + if (value->opcode != VIR_OP_STACK_ADDR && + value->opcode != VIR_OP_GLOBAL_ADDR && + value->opcode != VIR_OP_FUNC_ADDR && + !vir_has_no_address_metadata(value)) + return vir_verify_invalid_value(error); + if (value->nr_ops == 0) { + if (value->opcode == VIR_OP_CONST) { + if (!vir_type_width(func, value->type) || + value->constant != + (value->constant & + vir_width_mask(vir_type_width(func, value->type)))) + return vir_verify_invalid_value(error); + } else if (value->opcode == VIR_OP_FUNC_ADDR) { + if (value->type != VIR_TYPE_PTR || value->def_effect || + value->constant || value->address_kind || + value->address_slot || value->address_size || + value->address_alignment || !value->address_name || + !*value->address_name) + return vir_verify_invalid_value(error); + } else if (value->opcode == VIR_OP_RODATA_ADDR) { + if (value->type != VIR_TYPE_PTR || value->def_effect) + return vir_verify_invalid_value(error); + } else if (value->opcode == VIR_OP_STACK_ADDR || + value->opcode == VIR_OP_GLOBAL_ADDR) { + if (!vir_verify_address_value(value) || + vir_address_root_conflicts( + func, value->address_kind, value->address_slot, + value->address_name, value->address_size, + value->address_alignment)) + return vir_verify_invalid_value(error); + } else if ((value->opcode != VIR_OP_LOAD && + value->opcode != VIR_OP_CALL) || + !value->def_effect || + (value->opcode == VIR_OP_LOAD + ? value->def_effect->kind != VIR_EFFECT_LOAD && + value->def_effect->kind != + VIR_EFFECT_VOLATILE_LOAD + : value->def_effect->kind != VIR_EFFECT_CALL) || + value->def_effect->block != block || + value->def_effect->result != value || + (value->opcode == VIR_OP_LOAD && + (!value->def_effect->address || + value->def_effect->address->type != VIR_TYPE_PTR)) || + (value->opcode == VIR_OP_CALL && + value->def_effect->result_type != value->type) || + !vir_type_width(func, value->type)) { + return vir_verify_invalid_value(error); + } + } else if (!vir_verify_value_operands(func, dominance, value)) { + return vir_verify_invalid_value(error); + } + if (!vir_verify_reverse_uses(func, value)) + return vir_verify_invalid_value(error); + } + if (block->tail && block->tail->next) + return vir_verify_invalid_order(error); + { + const vir_value_t *ordered_value = block->head; + const vir_effect_t *ordered_effect = block->effects; + int order = 0; + + while (ordered_value || ordered_effect) { + bool paired = ordered_value && ordered_effect && + ordered_value->order == ordered_effect->order; + + if (paired) { + if (ordered_value->order != order || + ordered_value->def_effect != ordered_effect || + ordered_effect->result != ordered_value) + return vir_verify_invalid_order(error); + ordered_value = ordered_value->next; + ordered_effect = ordered_effect->next; + } else if (ordered_effect && + (!ordered_value || + ordered_effect->order < ordered_value->order)) { + if (ordered_effect->order != order || ordered_effect->result) + return vir_verify_invalid_order(error); + ordered_effect = ordered_effect->next; + } else { + if (ordered_value->order != order || ordered_value->def_effect) + return vir_verify_invalid_order(error); + ordered_value = ordered_value->next; + } + order++; + } + if (order != block->next_order) + return vir_verify_invalid_order(error); + } + terminator_position = position; + position = 0; + for (effect = block->effects; effect; effect = effect->next) { + if (effect->block != block || effect->position != position++ || + effect->order < 0 || effect->order >= block->next_order) + return vir_verify_invalid_value(error); + if (vir_effect_is_load(effect->kind)) { + if (!effect->address || !effect->result || effect->stored_value || + effect->stored_value_use || + vir_effect_has_call_metadata(effect) || + !vir_has_value(func, effect->result) || + effect->address->type != VIR_TYPE_PTR || + effect->result->def_effect != effect || + !vir_verify_effect_operand(func, dominance, effect, + effect->address, effect->address_use, + 0)) + return vir_verify_invalid_value(error); + } else if (vir_effect_is_store(effect->kind)) { + if (!effect->address || !effect->stored_value || effect->result || + vir_effect_has_call_metadata(effect) || + effect->address->type != VIR_TYPE_PTR || + !vir_verify_effect_operand(func, dominance, effect, + effect->address, effect->address_use, + 0) || + !vir_verify_effect_operand(func, dominance, effect, + effect->stored_value, + effect->stored_value_use, 1)) + return vir_verify_invalid_value(error); + } else if (effect->kind == VIR_EFFECT_CALL) { + int operand; + if (effect->address || effect->stored_value || + effect->address_use || effect->stored_value_use) + return vir_verify_invalid_value(error); + if (vir_effect_has_call_metadata(effect) || effect->result) { + if ((effect->callee != NULL) == + (effect->callee_value != NULL) || + (effect->callee && !*effect->callee) || + !vir_call_shape_valid(effect, INT_MAX) || + (effect->result_type != VIR_TYPE_VOID && + !vir_type_width(func, effect->result_type)) || + (effect->result && effect->result->def_effect != effect)) + return vir_verify_invalid_value(error); + if (effect->callee_value) { + if (effect->callee_value->type != VIR_TYPE_PTR || + !vir_verify_effect_operand( + func, dominance, effect, effect->callee_value, + effect->callee_value_use, -1)) + return vir_verify_error(error, "invalid call target"); + } else if (effect->callee_value_use) { + return vir_verify_invalid_value(error); + } + if (effect->signature && !vir_call_signature_matches( + func, effect, effect->signature)) + return vir_verify_error(error, + "invalid indirect call signature"); + for (operand = 0; operand < effect->arg_count; operand++) + if (!vir_verify_effect_operand( + func, dominance, effect, effect->args[operand], + &effect->arg_uses[operand], operand)) + return vir_verify_invalid_value(error); + } + } else if (effect->kind == VIR_EFFECT_VOLATILE) { + if (effect->address || effect->stored_value || effect->result || + effect->address_use || effect->stored_value_use || + vir_effect_has_call_metadata(effect)) + return vir_verify_invalid_value(error); + } else { + return vir_verify_invalid_value(error); + } + } + if ((block->effects == NULL) != (block->last_effect == NULL) || + (block->last_effect && block->last_effect->next)) + return vir_verify_invalid_order(error); + if (block->terminator == VIR_TERM_NONE) + return vir_verify_invalid_order(error); + if (block->terminator == VIR_TERM_JUMP && + (!block->outgoing || block->outgoing->next_outgoing || + block->return_value || block->branch_condition || block->true_edge || + block->false_edge)) + return vir_verify_invalid_order(error); + if (block->terminator == VIR_TERM_BRANCH) { + if (!block->branch_condition || + !vir_has_value(func, block->branch_condition) || + block->branch_condition->type != VIR_TYPE_I1 || !block->true_edge || + !block->false_edge || block->true_edge == block->false_edge || + block->true_edge->from != block || + block->false_edge->from != block || + !vir_has_outgoing_edge(block, block->true_edge) || + !vir_has_outgoing_edge(block, block->false_edge) || + !block->outgoing || !block->outgoing->next_outgoing || + block->outgoing->next_outgoing->next_outgoing || + vir_branch_use_count(block->branch_condition, block) != 1 || + !vir_value_dominates(func, dominance, block->branch_condition, + block, terminator_position)) + return vir_verify_invalid_value(error); + } + if (block->outgoing && block->terminator != VIR_TERM_JUMP && + block->terminator != VIR_TERM_BRANCH) + return vir_verify_invalid_order(error); + if (block->terminator == VIR_TERM_RETURN && + (block->outgoing || + (block->return_value && + (!vir_has_value(func, block->return_value) || + vir_return_use_count(block->return_value, block) != 1 || + !vir_value_dominates(func, dominance, block->return_value, block, + terminator_position))))) + return vir_verify_invalid_value(error); + for (edge = block->outgoing; edge; edge = edge->next_outgoing) { + int i; + vir_value_t *param; + if (edge->from != block || !vir_has_block(func, edge->to) || + !vir_has_incoming_edge(edge->to, edge) || + edge->arg_count != edge->to->param_count) + return vir_verify_invalid_order(error); + param = edge->to->params; + for (i = 0; i < edge->arg_count; i++) { + if (!edge->args[i] || !vir_has_value(func, edge->args[i]) || + !param || edge->args[i]->type != param->type || + vir_edge_use_count(edge->args[i], edge, i) != 1 || + !vir_value_dominates(func, dominance, edge->args[i], block, + terminator_position)) + return vir_verify_invalid_value(error); + param = param->param_next; + } + } + for (edge = block->incoming; edge; edge = edge->next_incoming) + if (edge->to != block || !vir_has_block(func, edge->from) || + !vir_has_outgoing_edge(edge->from, edge)) + return vir_verify_invalid_order(error); + return 1; +} + +int vir_verify(const vir_function_t *func, char **error) +{ + const vir_block_t *block; + vir_dominance_t dominance; + + if (!func || !func->blocks || !vir_lists_are_acyclic(func)) + return vir_verify_invalid_order(error); + if (!vir_dominance_init(func, &dominance)) + return vir_verify_invalid_order(error); + for (block = func->blocks; block; block = block->next) + if (!vir_verify_block(func, &dominance, block, error)) { + vir_dominance_release(&dominance); + return 0; + } + vir_dominance_release(&dominance); + if (error) + *error = NULL; + return 1; +} + +typedef struct vir_ssa_binding { + vir_block_t *block; + unsigned int variable; + vir_value_t *value; + vir_value_t *param; + struct vir_ssa_binding *next; +} vir_ssa_binding_t; + +typedef struct vir_ssa_block_state { + vir_block_t *block; + int sealed; + struct vir_ssa_block_state *next; +} vir_ssa_block_state_t; + +struct vir_ssa { + vir_function_t *func; + vir_ssa_binding_t *bindings; + vir_ssa_block_state_t *blocks; +}; + +bool vir_ssa_discard_variable(vir_ssa_t *ssa, unsigned int variable) +{ + vir_ssa_binding_t **link; + + if (!ssa) + return false; + for (vir_ssa_binding_t *binding = ssa->bindings; binding; + binding = binding->next) + if (binding->variable == variable && binding->param && + (binding->param->uses || binding->block->incoming)) + return false; + link = &ssa->bindings; + while (*link) { + vir_ssa_binding_t *binding = *link; + if (binding->variable != variable) { + link = &binding->next; + continue; + } + if (binding->param) { + int index = 0; + vir_param_edit_t edit; + for (vir_value_t *param = binding->block->params; + param && param != binding->param; param = param->param_next) + index++; + if (!vir_param_edit_prepare( + ssa->func, binding->block, binding->param, + binding->block->param_count, index, false, &edit)) + return false; + vir_param_edit_apply(binding->block, binding->param, &edit); + vir_param_edit_release(&edit); + } + *link = binding->next; + free(binding); + } + return true; +} + +static vir_ssa_binding_t *vir_ssa_find_binding(const vir_ssa_t *ssa, + const vir_block_t *block, + unsigned int variable, + const vir_value_t *param) +{ + vir_ssa_binding_t *binding; + for (binding = ssa->bindings; binding; binding = binding->next) + if (binding->block == block && + (param ? binding->param == param : binding->variable == variable)) + return binding; + return NULL; +} + +static vir_ssa_block_state_t *vir_ssa_block_state(vir_ssa_t *ssa, + vir_block_t *block, + int create) +{ + vir_ssa_block_state_t *state; + for (state = ssa->blocks; state; state = state->next) + if (state->block == block) + return state; + if (!create) + return NULL; + state = calloc(1, sizeof(*state)); + if (!state) + return NULL; + state->block = block; + state->next = ssa->blocks; + ssa->blocks = state; + return state; +} + +static bool vir_ssa_is_sealed(vir_ssa_t *ssa, vir_block_t *block) +{ + vir_ssa_block_state_t *state = vir_ssa_block_state(ssa, block, 0); + + return state && state->sealed; +} + +vir_ssa_t *vir_ssa_create(vir_function_t *func) +{ + vir_ssa_t *ssa; + if (!func) + return NULL; + ssa = calloc(1, sizeof(*ssa)); + if (ssa) + ssa->func = func; + return ssa; +} + +void vir_ssa_release(vir_ssa_t *ssa) +{ + vir_ssa_binding_t *binding; + vir_ssa_block_state_t *state; + if (!ssa) + return; + while ((binding = ssa->bindings)) { + ssa->bindings = binding->next; + free(binding); + } + while ((state = ssa->blocks)) { + ssa->blocks = state->next; + free(state); + } + free(ssa); +} + +int vir_ssa_write(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable, + vir_value_t *value) +{ + vir_ssa_binding_t *binding; + if (!ssa || !value || !vir_has_block(ssa->func, block) || + !vir_has_value(ssa->func, value)) + return 0; + if (!vir_value_dominates(ssa->func, NULL, value, block, + block->tail ? block->tail->position + 1 : 0)) + return 0; + binding = vir_ssa_find_binding(ssa, block, variable, NULL); + if (!binding) { + binding = calloc(1, sizeof(*binding)); + if (!binding) + return 0; + binding->block = block; + binding->variable = variable; + binding->next = ssa->bindings; + ssa->bindings = binding; + } + binding->value = value; + return 1; +} + +vir_value_t *vir_ssa_read(const vir_ssa_t *ssa, + const vir_block_t *block, + unsigned int variable) +{ + vir_ssa_binding_t *binding; + + if (!ssa || !vir_has_block(ssa->func, block)) + return NULL; + binding = vir_ssa_find_binding(ssa, block, variable, NULL); + return binding ? binding->value : NULL; +} + +bool vir_ssa_is_param(const vir_ssa_t *ssa, + const vir_block_t *block, + unsigned int variable) +{ + vir_ssa_binding_t *binding; + + if (!ssa || !vir_has_block(ssa->func, block)) + return 0; + binding = vir_ssa_find_binding(ssa, block, variable, NULL); + return binding && binding->param && binding->value == binding->param; +} + +static vir_value_t *vir_ssa_read_sealed_impl(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable, + unsigned char *visited) +{ + vir_ssa_binding_t *binding; + vir_value_t *value; + int index; + + if (!vir_has_block(ssa->func, block)) + return NULL; + index = vir_block_index(ssa->func, block); + if (index < 0 || visited[index]) + return NULL; + visited[index] = 1; + binding = vir_ssa_find_binding(ssa, block, variable, NULL); + if (binding) + return binding->value; + if (!vir_ssa_is_sealed(ssa, block) || !block->incoming || + block->incoming->next_incoming) + return NULL; + value = + vir_ssa_read_sealed_impl(ssa, block->incoming->from, variable, visited); + if (!value || !vir_ssa_write(ssa, block, variable, value)) + return NULL; + return value; +} + +vir_value_t *vir_ssa_read_sealed(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable) +{ + unsigned char *visited; + vir_value_t *value; + int count; + + if (!ssa || !block) + return NULL; + count = vir_function_block_count(ssa->func); + visited = calloc((size_t) count, sizeof(*visited)); + if (!visited) + return NULL; + value = vir_ssa_read_sealed_impl(ssa, block, variable, visited); + free(visited); + return value; +} + +vir_value_t *vir_ssa_predeclare(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable, + vir_type_t type) +{ + vir_ssa_binding_t *binding; + vir_ssa_block_state_t *state; + vir_value_t *value; + vir_param_edit_t edit = {0}; + vir_edge_t *edge; + int i = 0; + + if (!ssa || !vir_has_block(ssa->func, block) || + !vir_type_width(ssa->func, type) || vir_ssa_read(ssa, block, variable)) + return NULL; + state = vir_ssa_block_state(ssa, block, 1); + if (!state || state->sealed) + return NULL; + binding = calloc(1, sizeof(*binding)); + if (!binding) + return NULL; + if (!block->incoming) { + value = vir_block_add_param(ssa->func, block, type); + if (!value) + goto fail; + } else { + value = vir_arena_alloc(&ssa->func->arena, sizeof(*value)); + if (!value || + !vir_param_edit_prepare(ssa->func, block, NULL, block->param_count, + block->param_count, true, &edit)) + goto fail; + value->type = type; + for (edge = block->incoming; edge; edge = edge->next_incoming, i++) { + vir_value_t *incoming = vir_ssa_read(ssa, edge->from, variable); + if (!incoming || incoming->type != type || + edge->arg_count != block->param_count) + goto fail; + edit.edges[i].inserted = incoming; + } + value->id = ssa->func->next_value_id++; + vir_param_edit_apply(block, value, &edit); + } + binding->block = block; + binding->variable = variable; + binding->value = value; + binding->param = value; + binding->next = ssa->bindings; + ssa->bindings = binding; + vir_param_edit_release(&edit); + return value; + +fail: + vir_param_edit_release(&edit); + free(binding); + return NULL; +} + +int vir_ssa_seal(vir_ssa_t *ssa, vir_block_t *block) +{ + typedef struct { + vir_value_t *param; + vir_value_t *candidate; + vir_ssa_binding_t *binding; + vir_param_edit_t edit; + int index; + } vir_ssa_trivial_param_t; + vir_ssa_block_state_t *state; + vir_value_t *param; + vir_ssa_trivial_param_t *trivial_params; + int param_count; + int trivial_count = 0; + int index; + int i; + + if (!ssa || !vir_has_block(ssa->func, block)) + return 0; + state = vir_ssa_block_state(ssa, block, 1); + if (!state || state->sealed) + return 0; + param_count = block->param_count; + trivial_params = param_count + ? calloc((size_t) param_count, sizeof(*trivial_params)) + : NULL; + if (param_count && !trivial_params) + return 0; + for (param = block->params, index = 0; param; + param = param->param_next, index++) { + vir_value_t *candidate = NULL; + vir_edge_t *edge; + int trivial = 1; + + if (!vir_ssa_find_binding(ssa, block, 0, param)) + goto fail; + + /* A back edge that passes the parameter to itself adds no value, so a + * loop-invariant parameter is trivial too: its only other incoming + * value is the one it always holds. + */ + for (edge = block->incoming; edge; edge = edge->next_incoming) { + if (edge->arg_count != param_count) + goto fail; + if (edge->args[index] == param) + continue; + if (!candidate) + candidate = edge->args[index]; + else if (candidate != edge->args[index]) + trivial = 0; + } + if (candidate && candidate != param && trivial) { + if (!vir_replacement_dominates(ssa->func, param, candidate)) + goto fail; + trivial_params[trivial_count].param = param; + trivial_params[trivial_count].candidate = candidate; + trivial_params[trivial_count].binding = + vir_ssa_find_binding(ssa, block, 0, param); + trivial_params[trivial_count].index = index; + trivial_count++; + } + } + for (i = 0; i < trivial_count; i++) + if (!vir_param_edit_prepare( + ssa->func, block, trivial_params[i].param, param_count - i, + trivial_params[i].index - i, false, &trivial_params[i].edit)) + goto fail; + for (i = 0; i < trivial_count; i++) { + vir_replace_all_uses_unchecked(trivial_params[i].param, + trivial_params[i].candidate); + vir_param_edit_apply(block, trivial_params[i].param, + &trivial_params[i].edit); + trivial_params[i].binding->value = trivial_params[i].candidate; + trivial_params[i].binding->param = NULL; + } + for (i = 0; i < trivial_count; i++) + vir_param_edit_release(&trivial_params[i].edit); + free(trivial_params); + state->sealed = 1; + return 1; + +fail: + for (i = 0; i < trivial_count; i++) + vir_param_edit_release(&trivial_params[i].edit); + free(trivial_params); + return 0; +} + +static bool vir_ssa_edge_args(const vir_ssa_t *ssa, + const vir_block_t *from, + const vir_block_t *to, + vir_value_t ***args) +{ + int i = 0; + + if (!args) + return false; + *args = NULL; + if (!ssa || !from || !to || to->param_count < 0) + return false; + if (to->param_count && + !(*args = calloc((size_t) to->param_count, sizeof(**args)))) + return false; + for (const vir_value_t *param = to->params; param; + param = param->param_next) { + vir_ssa_binding_t *binding = vir_ssa_find_binding(ssa, to, 0, param); + + if (i >= to->param_count || !binding || + !((*args)[i] = vir_ssa_read(ssa, from, binding->variable))) + goto fail; + i++; + } + if (i == to->param_count) + return true; +fail: + free(*args); + *args = NULL; + return false; +} + +vir_edge_t *vir_ssa_jump(vir_ssa_t *ssa, vir_block_t *from, vir_block_t *to) +{ + vir_value_t **args; + int count = to ? to->param_count : 0; + vir_edge_t *edge; + if (!ssa || !from || !to || !vir_has_block(ssa->func, from) || + !vir_has_block(ssa->func, to) || vir_ssa_is_sealed(ssa, to)) + return NULL; + if (!vir_ssa_edge_args(ssa, from, to, &args)) + return NULL; + edge = vir_edge_create(ssa->func, from, to, args, count); + free(args); + return edge; +} + +int vir_ssa_branch(vir_ssa_t *ssa, + vir_block_t *from, + vir_value_t *condition, + vir_block_t *true_to, + vir_block_t *false_to) +{ + vir_value_t **true_args = NULL; + vir_value_t **false_args = NULL; + int true_count; + int false_count; + int result = 0; + if (!ssa || !from || !condition || !true_to || !false_to || + !vir_has_block(ssa->func, from) || !vir_has_block(ssa->func, true_to) || + !vir_has_block(ssa->func, false_to) || + vir_ssa_is_sealed(ssa, true_to) || vir_ssa_is_sealed(ssa, false_to)) + return 0; + true_count = true_to->param_count; + false_count = false_to->param_count; + if (!vir_ssa_edge_args(ssa, from, true_to, &true_args) || + !vir_ssa_edge_args(ssa, from, false_to, &false_args)) + goto out; + { + vir_edge_args_t t = {true_to, true_args, true_count}; + vir_edge_args_t f = {false_to, false_args, false_count}; + result = vir_block_set_branch(ssa->func, from, condition, &t, &f); + } +out: + free(true_args); + free(false_args); + return result; +} + +static void vir_print_call_abi_type(const vir_call_abi_type_t *type, FILE *out) +{ + fprintf(out, "%s%s", vir_type_name(type->type), + type->is_bool ? ".bool" + : type->is_unsigned ? ".unsigned" + : ""); +} + +static void vir_print_values(vir_value_t *const *values, int count, FILE *out) +{ + int i; + + for (i = 0; i < count; i++) + fprintf(out, "%s%%d%d", i ? ", " : "", values[i]->id); +} + +static void vir_print_call(const vir_effect_t *effect, FILE *out) +{ + if (effect->result) + fprintf(out, " %%d%d = call.%s ", effect->result->id, + vir_type_name(effect->result->type)); + else + fprintf(out, " call%s ", effect->callee_value ? "_indirect" : ""); + if (effect->callee_value) + fprintf(out, "%%d%d(", effect->callee_value->id); + else + fprintf(out, "@%s(", effect->callee); + vir_print_values(effect->args, effect->arg_count, out); + fputc(')', out); + if (effect->signature) { + fprintf(out, " sig("); + for (int i = 0; i < effect->signature->param_count; i++) { + fprintf(out, "%s", i ? ", " : ""); + vir_print_call_abi_type(&effect->signature->params[i], out); + } + fprintf(out, ")->"); + vir_print_call_abi_type(&effect->signature->result, out); + } + fputc('\n', out); +} + +static void vir_print_value(const vir_value_t *value, FILE *out) +{ + if (value->opcode == VIR_OP_CONST) + fprintf(out, " %%d%d = const.%s %llu\n", value->id, + vir_type_name(value->type), value->constant); + else if (value->opcode == VIR_OP_STACK_ADDR) + fprintf(out, " %%d%d = stackaddr %u, %u, align %u\n", value->id, + value->address_slot, value->address_size, + value->address_alignment); + else if (value->opcode == VIR_OP_GLOBAL_ADDR) + fprintf(out, " %%d%d = globaladdr @%s, %u, align %u\n", value->id, + value->address_name, value->address_size, + value->address_alignment); + else if (value->opcode == VIR_OP_FUNC_ADDR) + fprintf(out, " %%d%d = funcaddr @%s\n", value->id, + value->address_name); + else if (value->opcode == VIR_OP_RODATA_ADDR) + fprintf(out, " %%d%d = rodataaddr %llu\n", value->id, value->constant); + else if (value->opcode == VIR_OP_LOAD && value->def_effect->address) + fprintf(out, " %%d%d = %s.%s %%d%d\n", value->id, + value->def_effect->kind == VIR_EFFECT_VOLATILE_LOAD + ? "volatile.load" + : "load", + vir_type_name(value->type), value->def_effect->address->id); + else if (value->opcode == VIR_OP_CALL) + vir_print_call(value->def_effect, out); + else if (value->opcode == VIR_OP_LOAD) + fprintf(out, " %%d%d = load.%s \n", value->id, + vir_type_name(value->type)); + else if (value->nr_ops == 1) + fprintf(out, " %%d%d = %s.%s.%s %%d%d\n", value->id, + vir_opcode_name(value->opcode), vir_type_name(value->op0->type), + vir_type_name(value->type), value->op0->id); + else + fprintf(out, " %%d%d = %s %%d%d, %%d%d\n", value->id, + vir_opcode_name(value->opcode), value->op0->id, value->op1->id); +} + +static void vir_print_effect(const vir_effect_t *effect, FILE *out) +{ + if (vir_effect_is_store(effect->kind) && effect->address) + fprintf(out, " %s %%d%d, %%d%d\n", + effect->kind == VIR_EFFECT_VOLATILE_STORE ? "volatile.store" + : "store", + effect->address->id, effect->stored_value->id); + else if (effect->kind == VIR_EFFECT_CALL && !effect->result && + (effect->callee || effect->callee_value)) + vir_print_call(effect, out); + else + fprintf(out, " effect.%s\n", + effect->kind == VIR_EFFECT_STORE ? "store" + : effect->kind == VIR_EFFECT_CALL ? "call" + : effect->kind == VIR_EFFECT_VOLATILE_STORE ? "volatile.store" + : "volatile"); +} + +void vir_print(const vir_function_t *func, FILE *out) +{ + const vir_block_t *block; + for (block = func->blocks; block; block = block->next) { + const vir_value_t *value; + const vir_edge_t *edge; + const vir_effect_t *effect; + fprintf(out, "b%d", block->id); + if (block->params) { + const vir_value_t *param; + int first = 1; + fprintf(out, "("); + for (param = block->params; param; param = param->param_next) { + fprintf(out, "%s%%d%d:%s", first ? "" : ", ", param->id, + vir_type_name(param->type)); + first = 0; + } + fprintf(out, ")"); + } + fprintf(out, ":\n"); + value = block->head; + effect = block->effects; + while (value || effect) { + if (value && effect && effect->result == value) { + vir_print_value(value, out); + value = value->next; + effect = effect->next; + } else if (!value || (effect && effect->order < value->order)) { + vir_print_effect(effect, out); + effect = effect->next; + } else { + vir_print_value(value, out); + value = value->next; + } + } + if (block->terminator == VIR_TERM_BRANCH) { + fprintf(out, " branch %%d%d, b%d(", block->branch_condition->id, + block->true_edge->to->id); + vir_print_values(block->true_edge->args, + block->true_edge->arg_count, out); + fprintf(out, "), b%d(", block->false_edge->to->id); + vir_print_values(block->false_edge->args, + block->false_edge->arg_count, out); + fprintf(out, ")\n"); + } else { + for (edge = block->outgoing; edge; edge = edge->next_outgoing) { + fprintf(out, " jump b%d(", edge->to->id); + vir_print_values(edge->args, edge->arg_count, out); + fprintf(out, ")\n"); + } + } + if (block->terminator == VIR_TERM_RETURN && block->return_value) + fprintf(out, " return %%d%d\n", block->return_value->id); + else if (block->terminator == VIR_TERM_RETURN) + fprintf(out, " return\n"); + } +} diff --git a/src/vir.h b/src/vir.h new file mode 100644 index 00000000..39443765 --- /dev/null +++ b/src/vir.h @@ -0,0 +1,613 @@ +/* + * Typed value IR shared by the frontend, optimizers, and machine lowering. + */ +#pragma once + +#include +#include + +/* Keep the compact type/opcode vocabulary in one place. The enum values, + * verifier classification, and stable textual spelling are all derived from + * these X-macro lists so adding an operation cannot leave one view stale. + */ +/* clang-format off */ +/* Stage0 does not support multi-line macro continuations. */ +#define VIR_TYPE_LIST VIR_TYPE_ITEM(VOID, "void", 0) VIR_TYPE_ITEM(I1, "i1", 1) VIR_TYPE_ITEM(I8, "i8", 8) VIR_TYPE_ITEM(I16, "i16", 16) VIR_TYPE_ITEM(I32, "i32", 32) VIR_TYPE_ITEM(I64, "i64", 64) VIR_TYPE_ITEM(PTR, "ptr", 0) +#define VIR_PURE_UNARY_OPCODE_LIST VIR_OPCODE_ITEM(NEG, "neg") VIR_OPCODE_ITEM(BITNOT, "bitnot.i32") VIR_OPCODE_ITEM(ZEXT, "zext") VIR_OPCODE_ITEM(SEXT, "sext") VIR_OPCODE_ITEM(TRUNC, "trunc") VIR_OPCODE_ITEM(PTRTOINT, "ptrtoint") VIR_OPCODE_ITEM(INTTOPTR, "inttoptr") +#define VIR_PURE_BINARY_OPCODE_LIST VIR_OPCODE_ITEM(ADD, "add.i32") VIR_OPCODE_ITEM(SUB, "sub.i32") VIR_OPCODE_ITEM(MUL, "mul.i32") VIR_OPCODE_ITEM(SDIV, "sdiv.i32") VIR_OPCODE_ITEM(UDIV, "udiv.i32") VIR_OPCODE_ITEM(SREM, "srem.i32") VIR_OPCODE_ITEM(UREM, "urem.i32") VIR_OPCODE_ITEM(SHL, "shl.i32") VIR_OPCODE_ITEM(ASHR, "ashr.i32") VIR_OPCODE_ITEM(LSHR, "lshr.i32") VIR_OPCODE_ITEM(BITAND, "bitand.i32") VIR_OPCODE_ITEM(BITOR, "bitor.i32") VIR_OPCODE_ITEM(BITXOR, "bitxor.i32") VIR_OPCODE_ITEM(EQ, "eq.i32") VIR_OPCODE_ITEM(SLT, "slt.i32") VIR_OPCODE_ITEM(ULT, "ult.i32") VIR_OPCODE_ITEM(PTRADD, "ptradd") +#define VIR_OPCODE_LIST VIR_OPCODE_ITEM(CONST, "const") VIR_PURE_BINARY_OPCODE_LIST VIR_PURE_UNARY_OPCODE_LIST VIR_OPCODE_ITEM(STACK_ADDR, "stackaddr") VIR_OPCODE_ITEM(GLOBAL_ADDR, "globaladdr") VIR_OPCODE_ITEM(LOAD, "load") VIR_OPCODE_ITEM(CALL, "call") VIR_OPCODE_ITEM(PARAM, "param") VIR_OPCODE_ITEM(RODATA_ADDR, "rodataaddr") VIR_OPCODE_ITEM(FUNC_ADDR, "funcaddr") +/* clang-format on */ + +typedef enum { +#define VIR_TYPE_ITEM(name, spelling, width) VIR_TYPE_##name, + VIR_TYPE_LIST +#undef VIR_TYPE_ITEM +} vir_type_t; + +typedef enum { + VIR_OPT_O0, + VIR_OPT_O1, + VIR_OPT_O2, +} vir_opt_level_t; + +typedef enum { +#define VIR_OPCODE_ITEM(name, spelling) VIR_OP_##name, + VIR_OPCODE_LIST +#undef VIR_OPCODE_ITEM +} vir_opcode_t; + +typedef struct vir_value vir_value_t; +typedef struct vir_block vir_block_t; +typedef struct vir_edge vir_edge_t; +typedef struct vir_use vir_use_t; +typedef struct vir_effect vir_effect_t; +typedef struct vir_arena_block vir_arena_block_t; +typedef struct vir_ssa vir_ssa_t; + +typedef enum { + VIR_ADDRESS_NONE, + VIR_ADDRESS_STACK, + VIR_ADDRESS_GLOBAL, +} vir_address_kind_t; + +struct vir_use { + vir_value_t *user; + vir_edge_t *edge; + vir_effect_t *effect; + vir_block_t *return_block; + vir_block_t *branch_block; + vir_use_t *next; + int operand; +}; + +struct vir_value { + int id; + vir_opcode_t opcode; + vir_type_t type; + int nr_ops; + vir_value_t *op0; + vir_value_t *op1; + vir_use_t *uses; + vir_block_t *block; + int position; + int order; /* Shared construction order with ordered effects. */ + unsigned long long constant; + vir_value_t *next; + vir_value_t *param_next; + int is_block_param; + vir_effect_t *def_effect; + + /* Root-object metadata exists only on VIR_OP_{STACK,GLOBAL}_ADDR. The + * slot/name is stable source-independent provenance, not a backend offset. + */ + vir_address_kind_t address_kind; + unsigned int address_slot; + unsigned int address_size; + unsigned int address_alignment; + const char *address_name; +}; + +typedef enum { + VIR_TERM_NONE, + VIR_TERM_JUMP, + VIR_TERM_BRANCH, + VIR_TERM_RETURN, +} vir_terminator_t; + +typedef enum { + VIR_EFFECT_LOAD, + VIR_EFFECT_STORE, + VIR_EFFECT_CALL, + VIR_EFFECT_VOLATILE, + VIR_EFFECT_VOLATILE_LOAD, + VIR_EFFECT_VOLATILE_STORE, +} vir_effect_kind_t; + +static inline bool vir_effect_is_load(vir_effect_kind_t kind) +{ + return kind == VIR_EFFECT_LOAD || kind == VIR_EFFECT_VOLATILE_LOAD; +} + +static inline bool vir_effect_is_store(vir_effect_kind_t kind) +{ + return kind == VIR_EFFECT_STORE || kind == VIR_EFFECT_VOLATILE_STORE; +} + +/* An attached call signature describes every passed argument. Variadic calls + * retain the fixed parameter boundary for target ABI alignment. Signedness and + * bool identity are retained because VIR integer opcodes intentionally encode + * width, not C source-type rules. + */ +typedef struct { + vir_type_t type; + bool is_unsigned; + bool is_bool; +} vir_call_abi_type_t; + +typedef struct { + vir_call_abi_type_t result; + const vir_call_abi_type_t *params; + int param_count; + bool is_variadic; + int fixed_param_count; + const unsigned int *param_slots; + unsigned int + va_start_slot; /* UINT_MAX when no variadic cursor root exists. */ +} vir_call_signature_t; + +struct vir_block { + int id; + vir_value_t *head; + vir_value_t *tail; + vir_value_t *params; + vir_value_t *last_param; + int param_count; + vir_edge_t *incoming; + vir_edge_t *outgoing; + vir_terminator_t terminator; + vir_value_t *return_value; + vir_value_t *branch_condition; + vir_edge_t *true_edge; + vir_edge_t *false_edge; + vir_effect_t *effects; + vir_effect_t *last_effect; + int next_order; + vir_block_t *next; +}; + +struct vir_effect { + vir_effect_kind_t kind; + vir_block_t *block; + int position; + int order; + vir_value_t *address; + vir_value_t *stored_value; + vir_value_t *result; + vir_use_t *address_use; + vir_use_t *stored_value_use; + const char *callee; + vir_value_t *callee_value; + vir_use_t *callee_value_use; + vir_value_t **args; + vir_use_t *arg_uses; + int arg_count; + vir_type_t result_type; + const vir_call_signature_t *signature; + vir_effect_t *next; +}; + +struct vir_edge { + vir_block_t *from; + vir_block_t *to; + vir_value_t **args; + vir_use_t *arg_uses; + int arg_count; + vir_edge_t *next_outgoing; + vir_edge_t *next_incoming; +}; + +typedef struct { + vir_block_t *to; + vir_value_t **args; + int arg_count; +} vir_edge_args_t; + +struct vir_arena_block { + vir_arena_block_t *next; + int capacity; + int used; + char data[1]; +}; + +typedef struct { + vir_arena_block_t *head; + int block_size; + int capacity; + int peak_capacity; + int fail_after; /* Test-only: allocations remaining before forced failure. + */ +} vir_arena_t; + +typedef struct { + vir_arena_t arena; + vir_block_t *blocks; + vir_block_t *last_block; + int next_value_id; + int next_block_id; + /* Selected target layout, never inferred from the compiler host. */ + int pointer_bits; +} vir_function_t; + +typedef struct { + int blocks; + int values; + int params; + int uses; + int edges; + int effects; + int volatile_effects; + int arena_capacity; + int arena_peak_capacity; +} vir_stats_t; + +typedef struct { + int root_count; + int max_edge_args; + int max_call_args; + bool has_calls; + bool has_effects; + bool has_globals; +} vir_function_shape_t; + +typedef struct { + /* Successfully allocated per-block tables; an empty block has one slot. */ + int tables; + int table_slots; /* Slots in successfully allocated tables only. */ + int values; /* Binary values scanned in successfully allocated tables. */ + int lookups; /* One key lookup for every counted value. */ + int hits; /* Existing-key matches, including failed RAUW attempts. */ + int probes; /* Occupied slots inspected while resolving lookups. */ + int replacements; /* Successful RAUW operations. */ +} vir_cse_stats_t; + +void vir_function_init(vir_function_t *func, int arena_block_size); + +/* Bit width of an integer type; 0 for void and pointers. */ +int vir_integer_type_width(vir_type_t type); +vir_type_t vir_integer_type_from_size(int bytes); + +/* Returns false when an operation cannot be folded, including invalid division + * and shift operands. + */ +bool vir_fold_i32_binary(vir_opcode_t opcode, int left, int right, int *result); +bool vir_type_is_scalar(vir_type_t type); +bool vir_call_shape_valid(const vir_effect_t *effect, int max_args); +bool vir_function_shape(const vir_function_t *func, + vir_function_shape_t *shape); +static inline bool vir_type_is_i32_or_i64(vir_type_t type) +{ + return type == VIR_TYPE_I32 || type == VIR_TYPE_I64; +} +static inline bool vir_type_is_i32_or_i64_or_pointer(vir_type_t type) +{ + return vir_type_is_i32_or_i64(type) || type == VIR_TYPE_PTR; +} +static inline bool vir_type_is_pointer_or_i64(vir_type_t type) +{ + return type == VIR_TYPE_PTR || type == VIR_TYPE_I64; +} +static inline bool vir_opcode_is_commutative(vir_opcode_t opcode) +{ + return opcode == VIR_OP_ADD || opcode == VIR_OP_MUL || + opcode == VIR_OP_BITAND || opcode == VIR_OP_BITOR || + opcode == VIR_OP_BITXOR || opcode == VIR_OP_EQ; +} +static inline bool vir_opcode_is_compare(vir_opcode_t opcode) +{ + return opcode == VIR_OP_EQ || opcode == VIR_OP_SLT || opcode == VIR_OP_ULT; +} +static inline bool vir_opcode_is_shift(vir_opcode_t opcode) +{ + return opcode == VIR_OP_SHL || opcode == VIR_OP_ASHR || + opcode == VIR_OP_LSHR; +} +static inline bool vir_opcode_is_division(vir_opcode_t opcode) +{ + return opcode == VIR_OP_SDIV || opcode == VIR_OP_UDIV || + opcode == VIR_OP_SREM || opcode == VIR_OP_UREM; +} +/* Conservatively include division and remainder, which can fault at runtime. */ +static inline bool vir_opcode_may_trap(vir_opcode_t opcode) +{ + return vir_opcode_is_division(opcode); +} + +/* Must be called before creating blocks; only the currently supported target + * pointer widths are admitted. Pointer constants and ptr/int casts require this + * explicit layout. + */ +int vir_function_set_pointer_bits(vir_function_t *func, int pointer_bits); +void vir_function_release(vir_function_t *func); +void vir_collect_stats(const vir_function_t *func, vir_stats_t *stats); +vir_block_t *vir_block_create(vir_function_t *func); +vir_value_t *vir_block_add_param(vir_function_t *func, + vir_block_t *block, + vir_type_t type); +vir_edge_t *vir_edge_create(vir_function_t *func, + vir_block_t *from, + vir_block_t *to, + vir_value_t **args, + int arg_count); +/* Deleting a conditional arm atomically promotes the other arm to a jump. */ +int vir_edge_delete(vir_function_t *func, vir_edge_t *edge); +int vir_edge_redirect(vir_function_t *func, vir_edge_t *edge, vir_block_t *to); +vir_block_t *vir_edge_split(vir_function_t *func, vir_edge_t *edge); + +/* Remove unreachable blocks after validating that every dead value has only + * dead uses. The transformation leaves the function unchanged on failure. + */ +int vir_remove_unreachable(vir_function_t *func); +int vir_function_block_count(const vir_function_t *func); + +/* At O1/O2, redirect through empty parameter-free jump blocks. The regular + * helper also drops newly unreachable blocks; the explicit helper can defer + * that deletion for a caller that must run another pass on the rewritten + * topology. Effects and block arguments are intentionally out of scope for this + * first CFG simplification slice. A deferred caller must prune before lowering + * (and before any final reachability-sensitive verification). + */ +int vir_simplify_cfg(vir_function_t *func, vir_opt_level_t opt_level); +int vir_simplify_cfg_with_pruning(vir_function_t *func, + vir_opt_level_t opt_level, + bool prune_unreachable); +int vir_block_set_branch(vir_function_t *func, + vir_block_t *block, + vir_value_t *condition, + const vir_edge_args_t *true_edge, + const vir_edge_args_t *false_edge); +int vir_block_set_return(vir_function_t *func, + vir_block_t *block, + vir_value_t *value); + +/* Only payload-less CALL/VOLATILE placeholders use this constructor. Stores + * must carry their typed address/value uses through vir_store(). + */ +vir_effect_t *vir_effect_create(vir_function_t *func, + vir_block_t *block, + vir_effect_kind_t kind); +vir_value_t *vir_load(vir_function_t *func, + vir_block_t *block, + vir_value_t *address, + vir_type_t type); +vir_effect_t *vir_store(vir_function_t *func, + vir_block_t *block, + vir_value_t *address, + vir_value_t *value); +vir_value_t *vir_volatile_load(vir_function_t *func, + vir_block_t *block, + vir_value_t *address, + vir_type_t type); +vir_effect_t *vir_volatile_store(vir_function_t *func, + vir_block_t *block, + vir_value_t *address, + vir_value_t *value); + +/* Frontend construction may reserve a source-ordered load/store before its + * merged operands are available. Resolve it only after CFG construction. + */ +vir_value_t *vir_deferred_load(vir_function_t *func, + vir_block_t *block, + vir_type_t type, + bool is_volatile); +vir_effect_t *vir_deferred_store(vir_function_t *func, + vir_block_t *block, + bool is_volatile); +bool vir_deferred_load_resolve(vir_function_t *func, + vir_effect_t *effect, + vir_value_t *address); +bool vir_deferred_store_resolve(vir_function_t *func, + vir_effect_t *effect, + vir_value_t *address, + vir_value_t *value); + +/* Calls are ordered effects. A result_type of VIR_TYPE_VOID leaves result NULL. + * Indirect targets are explicit pointer-valued operands; ABI and alias + * semantics remain below VIR. + */ +vir_effect_t *vir_call(vir_function_t *func, + vir_block_t *block, + const char *callee, + vir_value_t **args, + int arg_count, + vir_type_t result_type); +vir_effect_t *vir_call_indirect(vir_function_t *func, + vir_block_t *block, + vir_value_t *callee, + vir_value_t **args, + int arg_count, + vir_type_t result_type); +bool vir_call_set_signature(vir_function_t *func, + vir_effect_t *effect, + const vir_call_signature_t *signature); +vir_value_t *vir_const_i32(vir_function_t *func, + vir_block_t *block, + int constant); +vir_value_t *vir_const_i1(vir_function_t *func, + vir_block_t *block, + int constant); +vir_value_t *vir_const_int(vir_function_t *func, + vir_block_t *block, + vir_type_t type, + unsigned long long bits); +vir_value_t *vir_const_ptr(vir_function_t *func, + vir_block_t *block, + unsigned long long bits); + +/* Materialize an addressable object root. Repeated identities must retain the + * same extent/alignment; byte offset and ABI placement remain below VIR. + */ +vir_value_t *vir_stack_addr(vir_function_t *func, + vir_block_t *block, + unsigned int slot, + unsigned int size, + unsigned int alignment); +vir_value_t *vir_global_addr(vir_function_t *func, + vir_block_t *block, + const char *name, + unsigned int size, + unsigned int alignment); + +/* The address of a string literal, @offset bytes into read-only data. It is an + * opaque pointer: no root, so nothing is assumed about what it points into. + */ +bool vir_effect_remove(vir_function_t *func, vir_effect_t *effect); +vir_value_t *vir_function_addr(vir_function_t *func, + vir_block_t *block, + const char *name); + +vir_value_t *vir_rodata_addr(vir_function_t *func, + vir_block_t *block, + unsigned int offset); +vir_value_t *vir_binary(vir_function_t *func, + vir_block_t *block, + vir_opcode_t opcode, + vir_type_t type, + vir_value_t *op0, + vir_value_t *op1); +#define VIR_DECLARE(name, ...) \ + vir_value_t *name(vir_function_t *func, vir_block_t *block, __VA_ARGS__) +VIR_DECLARE(vir_add, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_sub, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_mul, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_div, + vir_value_t *op0, + vir_value_t *op1, + bool is_unsigned, + bool remainder); +VIR_DECLARE(vir_shl, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_ashr, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_lshr, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_bitand, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_bitor, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_bitxor, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_eq, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_slt, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_ult, vir_value_t *op0, vir_value_t *op1); +VIR_DECLARE(vir_neg, vir_value_t *operand); +VIR_DECLARE(vir_bitnot, vir_value_t *operand); +VIR_DECLARE(vir_zext_i1, vir_value_t *operand); +VIR_DECLARE(vir_trunc_i1, vir_value_t *operand); +VIR_DECLARE(vir_zext, vir_value_t *operand, vir_type_t type); +VIR_DECLARE(vir_sext, vir_value_t *operand, vir_type_t type); +VIR_DECLARE(vir_trunc, vir_value_t *operand, vir_type_t type); +VIR_DECLARE(vir_ptrtoint, vir_value_t *operand, vir_type_t type); +VIR_DECLARE(vir_inttoptr, vir_value_t *operand); +#undef VIR_DECLARE +/* Add a byte offset represented at the selected target pointer width. */ +vir_value_t *vir_ptradd(vir_function_t *func, + vir_block_t *block, + vir_value_t *base, + vir_value_t *byte_offset); +int vir_replace_all_uses(vir_function_t *func, + vir_value_t *from, + vir_value_t *to); +int vir_local_cse(vir_function_t *func, vir_opt_level_t opt_level); +int vir_simplify_params(vir_function_t *func, vir_opt_level_t opt_level); +int vir_strength_reduce(vir_function_t *func, vir_opt_level_t opt_level); + +/* Optional pass-local accounting; no state is retained in VIR objects. Stats + * are zeroed before validation and before O0's no-work return. A failed table + * allocation leaves that block and its counters untouched; the pass continues + * with later blocks and returns its successful replacement count. + */ +int vir_local_cse_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + vir_cse_stats_t *stats); + +/* O2-only function-wide value numbering. The temporary table is pass state; + * dominance-checked RAUW rejects leaders unavailable at a use. + */ +int vir_gvn_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + vir_cse_stats_t *stats); + +/* O2-only LICM derives natural loops from temporary dominator state. It moves + * only pure values whose operands are already outside a loop into its existing + * single-edge preheader; effects, block parameters, and CFG are untouched. + */ +#define VIR_LICM_STAT_LOOPS 0 +#define VIR_LICM_STAT_VALUES_HOISTED 1 +#define VIR_LICM_STAT_TEMPORARY_BYTES 2 +#define VIR_LICM_STAT_VALUE_WORKLIST_ALLOCS 3 +#define VIR_LICM_STAT_READINESS_SAMPLE_VALUES 4 +#define VIR_LICM_STAT_READINESS_EAGER_ADMISSIONS 5 +#define VIR_LICM_STAT_COUNT 6 +#define VIR_LICM_READINESS_SAMPLE_BLOCK_LIMIT 8 +#define VIR_LICM_READINESS_SAMPLE_VALUE_LIMIT 64 +#define VIR_LICM_READINESS_SAMPLE_MAX_VALUES \ + (VIR_LICM_READINESS_SAMPLE_BLOCK_LIMIT * \ + VIR_LICM_READINESS_SAMPLE_VALUE_LIMIT) +int vir_licm_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + int *stats); +int vir_licm(vir_function_t *func, vir_opt_level_t opt_level); + +/* Remove unused pure values at O1/O2. Effects, their results, and block + * parameters remain intact: removing ordered effects requires a separate + * effect-semantics pass. The arena retains reclaimed object storage until the + * function completes, so this pass reduces downstream work, not peak arena + * capacity. + */ +int vir_dce(vir_function_t *func, vir_opt_level_t opt_level); +int vir_dce_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + int *passes, + int *values_scanned, + int *values_removed, + int *temporary_bytes, + int *temporary_peak_bytes); + +/* Sparse conditional constant propagation keeps its lattice, executable-CFG + * markers, and block-index work queue in function-scoped temporary arrays. It + * only reasons about immutable pure values; memory and calls are conservatively + * overdefined. CFG changes delete branches proved constant and, in the regular + * helpers, remove unreachable blocks. The explicit pruning variant can defer + * that removal; its caller must invoke vir_remove_unreachable() before lowering + * (and before any final reachability-sensitive verification). + */ +int vir_sccp(vir_function_t *func, vir_opt_level_t opt_level); + +/* Keep these as macros rather than a new enum type: the self-hosted frontend + * has a fixed type table, while callers only need integer array indices. + */ +#define VIR_SCCP_STAT_PASSES 0 +#define VIR_SCCP_STAT_VALUES_SCANNED 1 +#define VIR_SCCP_STAT_VALUES_REPLACED 2 +#define VIR_SCCP_STAT_BRANCHES_SIMPLIFIED 3 +#define VIR_SCCP_STAT_BLOCKS_REMOVED 4 +#define VIR_SCCP_STAT_TEMPORARY_BYTES 5 +#define VIR_SCCP_STAT_TEMPORARY_PEAK_BYTES 6 +#define VIR_SCCP_STAT_COUNT 7 +int vir_sccp_with_stats(vir_function_t *func, + vir_opt_level_t opt_level, + int *stats); +int vir_sccp_with_stats_and_pruning(vir_function_t *func, + vir_opt_level_t opt_level, + int *stats, + bool prune_unreachable); +int vir_verify(const vir_function_t *func, char **error); +vir_block_t *vir_use_block(const vir_use_t *use); +int vir_use_order(const vir_use_t *use); +void vir_print(const vir_function_t *func, FILE *out); +vir_ssa_t *vir_ssa_create(vir_function_t *func); +void vir_ssa_release(vir_ssa_t *ssa); +vir_value_t *vir_ssa_predeclare(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable, + vir_type_t type); + +/* Discard scratch bindings before wiring CFG edges, once provisional memory + * reads have replaced their SSA users. Live parameters are never removed. + */ +bool vir_ssa_discard_variable(vir_ssa_t *ssa, unsigned int variable); + +int vir_ssa_write(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable, + vir_value_t *value); +vir_value_t *vir_ssa_read(const vir_ssa_t *ssa, + const vir_block_t *block, + unsigned int variable); +bool vir_ssa_is_param(const vir_ssa_t *ssa, + const vir_block_t *block, + unsigned int variable); +vir_value_t *vir_ssa_read_sealed(vir_ssa_t *ssa, + vir_block_t *block, + unsigned int variable); +int vir_ssa_seal(vir_ssa_t *ssa, vir_block_t *block); +vir_edge_t *vir_ssa_jump(vir_ssa_t *ssa, vir_block_t *from, vir_block_t *to); +int vir_ssa_branch(vir_ssa_t *ssa, + vir_block_t *from, + vir_value_t *condition, + vir_block_t *true_to, + vir_block_t *false_to); diff --git a/tests/vir-core.c b/tests/vir-core.c new file mode 100644 index 00000000..9915971b --- /dev/null +++ b/tests/vir-core.c @@ -0,0 +1,2701 @@ +#include "vir.h" + +#include +#include +#include + +static void test_overwritten_stores(void) +{ + for (int barrier = 0; barrier < 7; barrier++) { + vir_function_t function; + char *error = NULL; + vir_function_init(&function, 64); + assert(vir_function_set_pointer_bits(&function, 64)); + vir_block_t *block = vir_block_create(&function); + vir_value_t *address = vir_stack_addr(&function, block, 0, 4, 4); + vir_value_t *one = vir_const_i32(&function, block, 1); + vir_value_t *two = vir_const_i32(&function, block, 2); + vir_effect_t *first = vir_store(&function, block, address, one); + vir_value_t *result = two; + if (barrier == 1) + assert(vir_volatile_load(&function, block, address, VIR_TYPE_I32)); + if (barrier == 2) + result = vir_load(&function, block, address, VIR_TYPE_I32); + if (barrier == 3) + assert( + vir_call(&function, block, "observe", NULL, 0, VIR_TYPE_VOID)); + if (barrier == 4) + assert(vir_volatile_store(&function, block, address, one)); + if (barrier == 5) + two = vir_const_int(&function, block, VIR_TYPE_I8, 2); + if (barrier == 6) + assert(vir_load(&function, block, address, VIR_TYPE_I32)); + assert(first && vir_store(&function, block, address, two)); + assert(vir_block_set_return(&function, block, result)); + assert(vir_verify(&function, &error)); + assert(vir_dce(&function, VIR_OPT_O0) == 0); + assert(block->effects == first); + assert(vir_dce(&function, VIR_OPT_O1) >= 0); + assert((block->effects == first) == (barrier != 0 && barrier != 6)); + assert(vir_verify(&function, &error)); + vir_function_release(&function); + } +} + +static void test_va_start_signature_metadata(void) +{ + vir_function_t function; + char *error; + vir_function_init(&function, 64); + vir_block_t *block = vir_block_create(&function); + vir_value_t *argument = vir_const_i32(&function, block, 7); + vir_value_t *args[] = {argument}; + vir_call_abi_type_t param = {VIR_TYPE_I32, false, false}; + unsigned int slot = 0; + vir_call_signature_t signature = { + {VIR_TYPE_I32, false, false}, ¶m, 1, false, 1, &slot, 0}; + vir_effect_t *call = + vir_call(&function, block, "variadic", args, 1, VIR_TYPE_I32); + assert(call && !vir_call_set_signature(&function, call, &signature)); + signature.is_variadic = true; + assert(vir_call_set_signature(&function, call, &signature)); + assert(call->signature->va_start_slot == UINT_MAX && + !call->signature->param_slots); + assert(vir_block_set_return(&function, block, call->result)); + assert(vir_verify(&function, &error)); + vir_function_release(&function); +} + +static void test_sccp_effect_first_param(void) +{ + vir_function_t function; + char *error; + vir_function_init(&function, 64); + assert(vir_function_set_pointer_bits(&function, 32)); + vir_block_t *entry = vir_block_create(&function); + vir_block_t *merge = vir_block_create(&function); + vir_value_t *address = vir_stack_addr(&function, entry, 0, 4, 4); + vir_value_t *constant = vir_const_i32(&function, entry, 7); + vir_value_t *parameter = + vir_block_add_param(&function, merge, VIR_TYPE_I32); + assert(vir_edge_create(&function, entry, merge, &constant, 1)); + vir_effect_t *store = vir_store(&function, merge, address, parameter); + assert(store && store->order == 0); + assert(vir_block_set_return(&function, merge, parameter)); + assert(vir_verify(&function, &error)); + assert(vir_sccp(&function, VIR_OPT_O1) > 0); + assert(vir_verify(&function, &error)); + assert(store->stored_value->opcode == VIR_OP_CONST); + assert(store->stored_value->order < store->order); + vir_function_release(&function); +} + +static void test_sccp_width_case(vir_type_t type, + vir_opcode_t opcode, + unsigned long long left_bits, + unsigned long long right_bits, + unsigned long long expected) +{ + vir_function_t func; + vir_block_t *entry; + vir_block_t *merge; + vir_value_t *left; + vir_value_t *param; + vir_value_t *right; + vir_value_t *value; + vir_type_t right_type = (opcode == VIR_OP_SHL || opcode == VIR_OP_ASHR || + opcode == VIR_OP_LSHR) && + type == VIR_TYPE_I64 + ? VIR_TYPE_I32 + : type; + vir_type_t result_type = + opcode == VIR_OP_EQ || opcode == VIR_OP_SLT || opcode == VIR_OP_ULT + ? VIR_TYPE_I1 + : type; + char *error; + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + left = vir_const_int(&func, entry, type, left_bits); + param = vir_block_add_param(&func, merge, type); + right = vir_const_int(&func, merge, right_type, right_bits); + value = vir_binary(&func, merge, opcode, result_type, param, right); + assert(entry && merge && left && param && right && value && + vir_edge_create(&func, entry, merge, &left, 1) && + vir_block_set_return(&func, merge, value) && + vir_verify(&func, &error)); + assert(vir_sccp(&func, VIR_OPT_O1) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->type == result_type && + merge->return_value->constant == expected && + vir_verify(&func, &error)); + vir_function_release(&func); +} + +static void test_sccp_unary_width_case(vir_type_t type, + vir_opcode_t opcode, + unsigned long long operand_bits, + unsigned long long expected) +{ + vir_function_t func; + vir_block_t *entry; + vir_block_t *merge; + vir_value_t *operand; + vir_value_t *param; + vir_value_t *value; + char *error; + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + operand = vir_const_int(&func, entry, type, operand_bits); + param = vir_block_add_param(&func, merge, type); + value = opcode == VIR_OP_NEG ? vir_neg(&func, merge, param) + : vir_bitnot(&func, merge, param); + assert(entry && merge && operand && param && value && + vir_edge_create(&func, entry, merge, &operand, 1) && + vir_block_set_return(&func, merge, value) && + vir_verify(&func, &error)); + assert(vir_sccp(&func, VIR_OPT_O1) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->type == type && + merge->return_value->constant == expected && + vir_verify(&func, &error)); + vir_function_release(&func); +} + +static void test_sccp_cast_width_case(vir_type_t input_type, + vir_opcode_t opcode, + vir_type_t result_type, + unsigned long long operand_bits, + unsigned long long expected) +{ + vir_function_t func; + vir_block_t *entry; + vir_block_t *merge; + vir_value_t *operand; + vir_value_t *param; + vir_value_t *value; + char *error; + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + operand = vir_const_int(&func, entry, input_type, operand_bits); + param = vir_block_add_param(&func, merge, input_type); + value = opcode == VIR_OP_ZEXT && input_type == VIR_TYPE_I1 + ? vir_zext_i1(&func, merge, param) + : opcode == VIR_OP_ZEXT ? vir_zext(&func, merge, param, result_type) + : opcode == VIR_OP_SEXT + ? vir_sext(&func, merge, param, result_type) + : vir_trunc(&func, merge, param, result_type); + assert(entry && merge && operand && param && value && + vir_edge_create(&func, entry, merge, &operand, 1) && + vir_block_set_return(&func, merge, value) && + vir_verify(&func, &error)); + assert(vir_sccp(&func, VIR_OPT_O1) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->type == result_type && + merge->return_value->constant == expected && + vir_verify(&func, &error)); + vir_function_release(&func); +} + +static void test_sccp_width_folds(void) +{ + static const struct { + vir_type_t type; + vir_opcode_t opcode; + unsigned long long left; + unsigned long long right; + unsigned long long expected; + } cases[] = { + {VIR_TYPE_I32, VIR_OP_ADD, 0x7fffffffULL, 1, 0x80000000ULL}, + {VIR_TYPE_I32, VIR_OP_SUB, 0, 1, 0xffffffffULL}, + {VIR_TYPE_I32, VIR_OP_MUL, 0x10000ULL, 0x10000ULL, 0}, + {VIR_TYPE_I32, VIR_OP_SDIV, 0xfffffff9ULL, 3, 0xfffffffeULL}, + {VIR_TYPE_I32, VIR_OP_SREM, 0xfffffff9ULL, 3, 0xffffffffULL}, + {VIR_TYPE_I32, VIR_OP_SDIV, 7, 0xfffffffdULL, 0xfffffffeULL}, + {VIR_TYPE_I32, VIR_OP_SREM, 7, 0xfffffffdULL, 1}, + {VIR_TYPE_I32, VIR_OP_SDIV, 0xfffffff9ULL, 0xfffffffdULL, 2}, + {VIR_TYPE_I32, VIR_OP_SREM, 0xfffffff9ULL, 0xfffffffdULL, + 0xffffffffULL}, + {VIR_TYPE_I32, VIR_OP_UDIV, 0xffffffffULL, 3, 0x55555555ULL}, + {VIR_TYPE_I32, VIR_OP_UREM, 0xffffffffULL, 3, 0}, + {VIR_TYPE_I32, VIR_OP_SHL, 1, 31, 0x80000000ULL}, + {VIR_TYPE_I32, VIR_OP_ASHR, 0x80000000ULL, 31, 0xffffffffULL}, + {VIR_TYPE_I32, VIR_OP_LSHR, 0x80000000ULL, 31, 1}, + {VIR_TYPE_I32, VIR_OP_SLT, 0x80000000ULL, 1, 1}, + {VIR_TYPE_I32, VIR_OP_ULT, 0x80000000ULL, 1, 0}, + {VIR_TYPE_I32, VIR_OP_BITAND, 0x8000000fULL, 0xffffffffULL, + 0x8000000fULL}, + {VIR_TYPE_I32, VIR_OP_BITOR, 0x80000000ULL, 0x0fULL, 0x8000000fULL}, + {VIR_TYPE_I32, VIR_OP_BITXOR, 0x80000000ULL, 0xffffffffULL, + 0x7fffffffULL}, + {VIR_TYPE_I32, VIR_OP_EQ, 0x80000000ULL, 0x80000000ULL, 1}, + {VIR_TYPE_I32, VIR_OP_EQ, 0x80000000ULL, 1, 0}, + {VIR_TYPE_I64, VIR_OP_ADD, 0xffffffffffffffffULL, 1, 0}, + {VIR_TYPE_I64, VIR_OP_SUB, 0, 1, 0xffffffffffffffffULL}, + {VIR_TYPE_I64, VIR_OP_MUL, 0x100000001ULL, 3, 0x300000003ULL}, + {VIR_TYPE_I64, VIR_OP_SDIV, 0xfffffffffffffff9ULL, 3, + 0xfffffffffffffffeULL}, + {VIR_TYPE_I64, VIR_OP_SREM, 0xfffffffffffffff9ULL, 3, + 0xffffffffffffffffULL}, + {VIR_TYPE_I64, VIR_OP_SDIV, 7, 0xfffffffffffffffdULL, + 0xfffffffffffffffeULL}, + {VIR_TYPE_I64, VIR_OP_SREM, 7, 0xfffffffffffffffdULL, 1}, + {VIR_TYPE_I64, VIR_OP_SDIV, 0xfffffffffffffff9ULL, + 0xfffffffffffffffdULL, 2}, + {VIR_TYPE_I64, VIR_OP_SREM, 0xfffffffffffffff9ULL, + 0xfffffffffffffffdULL, 0xffffffffffffffffULL}, + {VIR_TYPE_I64, VIR_OP_UDIV, 0xffffffffffffffffULL, 3, + 0x5555555555555555ULL}, + {VIR_TYPE_I64, VIR_OP_UREM, 0xffffffffffffffffULL, 3, 0}, + {VIR_TYPE_I64, VIR_OP_SHL, 1, 63, 0x8000000000000000ULL}, + {VIR_TYPE_I64, VIR_OP_ASHR, 0x8000000000000000ULL, 63, + 0xffffffffffffffffULL}, + {VIR_TYPE_I64, VIR_OP_LSHR, 0x8000000000000000ULL, 63, 1}, + {VIR_TYPE_I64, VIR_OP_SLT, 0x8000000000000000ULL, 1, 1}, + {VIR_TYPE_I64, VIR_OP_ULT, 0x8000000000000000ULL, 1, 0}, + {VIR_TYPE_I64, VIR_OP_BITAND, 0x800000000000000fULL, ~0ULL, + 0x800000000000000fULL}, + {VIR_TYPE_I64, VIR_OP_BITOR, 0x8000000000000000ULL, 0x0fULL, + 0x800000000000000fULL}, + {VIR_TYPE_I64, VIR_OP_BITXOR, 0x8000000000000000ULL, ~0ULL, + 0x7fffffffffffffffULL}, + {VIR_TYPE_I64, VIR_OP_EQ, 0x8000000000000000ULL, 0x8000000000000000ULL, + 1}, + {VIR_TYPE_I64, VIR_OP_EQ, 0x8000000000000000ULL, 1, 0}, + }; + + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + test_sccp_width_case(cases[i].type, cases[i].opcode, cases[i].left, + cases[i].right, cases[i].expected); + test_sccp_unary_width_case(VIR_TYPE_I32, VIR_OP_NEG, 0x80000000ULL, + 0x80000000ULL); + test_sccp_unary_width_case(VIR_TYPE_I32, VIR_OP_NEG, 0x80000001ULL, + 0x7fffffffULL); + test_sccp_unary_width_case(VIR_TYPE_I32, VIR_OP_NEG, 1, 0xffffffffULL); + test_sccp_unary_width_case(VIR_TYPE_I32, VIR_OP_BITNOT, 0x80000000ULL, + 0x7fffffffULL); + test_sccp_unary_width_case(VIR_TYPE_I64, VIR_OP_NEG, 0x8000000000000000ULL, + 0x8000000000000000ULL); + test_sccp_unary_width_case(VIR_TYPE_I64, VIR_OP_NEG, 0x8000000000000001ULL, + 0x7fffffffffffffffULL); + test_sccp_unary_width_case(VIR_TYPE_I64, VIR_OP_NEG, 1, ~0ULL); + test_sccp_unary_width_case(VIR_TYPE_I64, VIR_OP_BITNOT, + 0x8000000000000000ULL, 0x7fffffffffffffffULL); + test_sccp_cast_width_case(VIR_TYPE_I8, VIR_OP_ZEXT, VIR_TYPE_I32, 0xffULL, + 0xffULL); + test_sccp_cast_width_case(VIR_TYPE_I1, VIR_OP_ZEXT, VIR_TYPE_I32, 1, 1); + test_sccp_cast_width_case(VIR_TYPE_I1, VIR_OP_ZEXT, VIR_TYPE_I32, 0, 0); + test_sccp_cast_width_case(VIR_TYPE_I8, VIR_OP_SEXT, VIR_TYPE_I32, 0xffULL, + 0xffffffffULL); + test_sccp_cast_width_case(VIR_TYPE_I16, VIR_OP_ZEXT, VIR_TYPE_I32, + 0xffffULL, 0xffffULL); + test_sccp_cast_width_case(VIR_TYPE_I16, VIR_OP_SEXT, VIR_TYPE_I32, + 0xffffULL, 0xffffffffULL); + test_sccp_cast_width_case(VIR_TYPE_I32, VIR_OP_ZEXT, VIR_TYPE_I64, + 0x80000000ULL, 0x80000000ULL); + test_sccp_cast_width_case(VIR_TYPE_I32, VIR_OP_SEXT, VIR_TYPE_I64, + 0x80000000ULL, 0xffffffff80000000ULL); + test_sccp_cast_width_case(VIR_TYPE_I64, VIR_OP_TRUNC, VIR_TYPE_I32, + 0x1234567887654321ULL, 0x87654321ULL); +} + +static void test_sccp_keeps_undefined_signed_division(vir_type_t type, + unsigned long long min) +{ + vir_function_t func; + vir_block_t *entry; + vir_block_t *merge; + vir_value_t *left; + vir_value_t *param; + vir_value_t *minus_one; + vir_value_t *value; + char *error; + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + left = vir_const_int(&func, entry, type, min); + param = vir_block_add_param(&func, merge, type); + minus_one = vir_const_int(&func, merge, type, ~0ULL); + value = param && minus_one + ? vir_binary(&func, merge, VIR_OP_SDIV, type, param, minus_one) + : NULL; + assert(entry && merge && left && param && minus_one && value && + vir_edge_create(&func, entry, merge, &left, 1) && + vir_block_set_return(&func, merge, value) && + vir_verify(&func, &error)); + assert(vir_sccp(&func, VIR_OPT_O1) > 0 && + merge->return_value->opcode == VIR_OP_SDIV && + vir_verify(&func, &error)); + vir_function_release(&func); +} + +static void test_sccp_undefined_signed_division(void) +{ + test_sccp_keeps_undefined_signed_division(VIR_TYPE_I32, 0x80000000ULL); + test_sccp_keeps_undefined_signed_division(VIR_TYPE_I64, + 0x8000000000000000ULL); +} + +static void test_signed_division_builder_i64(void) +{ + vir_function_t func; + vir_block_t *entry; + vir_value_t *positive; + vir_value_t *negative; + vir_value_t *negative_seven; + vir_value_t *negative_three; + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + positive = vir_const_int(&func, entry, VIR_TYPE_I64, 7); + negative = vir_const_int(&func, entry, VIR_TYPE_I64, ~2ULL); + negative_seven = vir_const_int(&func, entry, VIR_TYPE_I64, ~6ULL); + negative_three = + vir_div(&func, entry, negative_seven, negative, false, false); + assert(positive && negative && negative_seven && negative_three && + negative_three->constant == 2 && + vir_div(&func, entry, negative_seven, negative, false, true) + ->constant == ~0ULL && + vir_div(&func, entry, positive, negative, false, false)->constant == + ~1ULL && + vir_div(&func, entry, positive, negative, false, true)->constant == + 1); + vir_function_release(&func); +} + +static void test_signed_division_overflow_builder(vir_type_t type, + unsigned long long min) +{ + vir_function_t func; + vir_block_t *entry; + vir_value_t *minimum; + vir_value_t *minus_one; + vir_value_t *quotient; + vir_value_t *remainder; + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + minimum = vir_const_int(&func, entry, type, min); + minus_one = vir_const_int(&func, entry, type, ~0ULL); + quotient = vir_div(&func, entry, minimum, minus_one, false, false); + remainder = vir_div(&func, entry, minimum, minus_one, false, true); + assert(entry && minimum && minus_one && quotient && remainder && + quotient->opcode == VIR_OP_SDIV && + remainder->opcode == VIR_OP_SREM && quotient->type == type && + remainder->type == type); + vir_function_release(&func); +} + +int main(void) +{ + test_overwritten_stores(); + test_va_start_signature_metadata(); + test_sccp_effect_first_param(); + vir_function_t func; + vir_block_t *entry; + vir_block_t *merge; + vir_block_t *otherwise; + vir_block_t *dead; + vir_value_t *one; + vir_value_t *two; + vir_value_t *entry_param; + vir_value_t *sum; + vir_value_t *product; + vir_value_t *param; + vir_value_t *stack_duplicate; + vir_value_t *global_duplicate; + vir_edge_t *edge; + vir_effect_t *effect; + vir_effect_t *call; + char *error; + FILE *out; + char text[256]; + int text_len; + + test_sccp_width_folds(); + test_sccp_undefined_signed_division(); + test_signed_division_builder_i64(); + test_signed_division_overflow_builder(VIR_TYPE_I32, 0x80000000ULL); + test_signed_division_overflow_builder(VIR_TYPE_I64, 0x8000000000000000ULL); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + func.arena.fail_after = 0; + assert(!vir_block_create(&func)); + assert(func.blocks == entry && func.last_block == entry); + func.arena.fail_after = -1; + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two); + product = vir_binary(&func, entry, VIR_OP_MUL, VIR_TYPE_I32, sum, two); + func.arena.fail_after = 0; + assert(!vir_const_i32(&func, entry, 3)); + assert(entry->tail == product && !product->next); + func.arena.fail_after = -1; + func.arena.fail_after = 0; + assert(!vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two)); + assert(entry->tail == product && !product->next); + func.arena.fail_after = 1; + assert(!vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two)); + assert(entry->tail == product && !product->next); + func.arena.fail_after = -1; + merge = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + func.arena.fail_after = 0; + assert(!vir_block_add_param(&func, merge, VIR_TYPE_I32)); + assert(merge->params == param && merge->last_param == param && + merge->param_count == 1 && !param->param_next); + func.arena.fail_after = -1; + assert(!vir_edge_create(&func, entry, merge, NULL, 1)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = 0; + assert(!vir_edge_create(&func, entry, merge, &product, 1)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = 1; + assert(!vir_edge_create(&func, entry, merge, &product, 1)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = 2; + assert(!vir_edge_create(&func, entry, merge, &product, 1)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = -1; + edge = vir_edge_create(&func, entry, merge, &product, 1); + assert(edge && edge->args[0] == product); + assert(!vir_verify(&func, &error)); + func.arena.fail_after = 0; + assert(!vir_block_set_return(&func, merge, param)); + assert(merge->terminator == VIR_TERM_NONE && !merge->return_value); + func.arena.fail_after = -1; + assert(vir_block_set_return(&func, merge, param)); + assert(!vir_effect_create(&func, merge, VIR_EFFECT_STORE)); + assert(entry->id == 0); + assert(one->id == 0 && two->id == 1 && sum->id == 2 && product->id == 3); + assert(vir_verify(&func, &error)); + vir_use_t *saved_next = one->uses->next; + one->uses->next = one->uses; + assert(!vir_verify(&func, &error)); + one->uses->next = saved_next; + assert(vir_verify(&func, &error)); + vir_stats_t stats; + vir_collect_stats(&func, &stats); + assert(stats.blocks == 2 && stats.values == 5 && stats.params == 1); + assert(stats.edges == 1 && stats.effects == 0 && stats.uses == 6); + assert(stats.arena_capacity >= stats.arena_peak_capacity); + vir_function_t foreign; + vir_block_t *foreign_block; + vir_block_t *foreign_target; + vir_value_t *foreign_value; + vir_edge_t *foreign_edge; + vir_function_init(&foreign, 64); + foreign_block = vir_block_create(&foreign); + foreign_target = vir_block_create(&foreign); + foreign_value = vir_const_i32(&foreign, foreign_block, 9); + foreign_edge = + vir_edge_create(&foreign, foreign_block, foreign_target, NULL, 0); + assert(foreign_edge); + assert(!vir_const_i32(&func, foreign_block, 9)); + assert(!vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, + foreign_value)); + assert(!vir_add(&func, entry, one, foreign_value)); + assert(!vir_eq(&func, entry, one, foreign_value)); + vir_ssa_t *foreign_ssa = vir_ssa_create(&func); + assert(foreign_ssa && !vir_ssa_jump(foreign_ssa, entry, foreign_target)); + vir_ssa_release(foreign_ssa); + product->op0 = foreign_value; + assert(!vir_verify(&func, &error)); + product->op0 = sum; + merge->incoming = foreign_edge; + assert(!vir_verify(&func, &error)); + merge->incoming = edge; + vir_function_release(&foreign); + merge->incoming = NULL; + assert(!vir_verify(&func, &error)); + merge->incoming = edge; + sum->position = 99; + assert(!vir_verify(&func, &error)); + assert(!strcmp(error, "invalid value/use structure")); + sum->position = 2; + vir_replace_all_uses(&func, sum, one); + assert(!sum->uses && product->op0 == one); + assert(vir_verify(&func, &error)); + vir_replace_all_uses(&func, product, one); + assert(edge->args[0] == one && param->is_block_param); + assert(vir_verify(&func, &error)); + vir_replace_all_uses(&func, param, two); + assert(merge->return_value == two); + assert(vir_verify(&func, &error)); + + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(!strcmp(text, + "b0:\n" + " %d0 = const.i32 1\n" + " %d1 = const.i32 2\n" + " %d2 = add.i32 %d0, %d1\n" + " %d3 = mul.i32 %d0, %d1\n" + " jump b1(%d0)\n" + "b1(%d4:i32):\n" + " return %d1\n")); + fclose(out); + vir_function_release(&func); + + /* Redirecting a pointer-carrying edge must retain its typed argument and + * reverse use just like the scalar transactional path. + */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + vir_block_t *pointer_left = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + one = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + param = vir_block_add_param(&func, merge, VIR_TYPE_PTR); + product = vir_block_add_param(&func, otherwise, VIR_TYPE_PTR); + assert(one && param && product && + vir_edge_create(&func, entry, pointer_left, NULL, 0) && + vir_edge_create(&func, pointer_left, merge, &one, 1) && + vir_block_set_return(&func, merge, param) && + vir_block_set_return(&func, otherwise, product)); + assert(vir_edge_redirect(&func, pointer_left->outgoing, otherwise)); + assert(pointer_left->outgoing->to == otherwise && + pointer_left->outgoing->args[0] == one && one->uses && + one->uses->edge == pointer_left->outgoing && !one->uses->next && + vir_verify(&func, &error)); + vir_block_t *pointer_split = vir_edge_split(&func, pointer_left->outgoing); + assert(pointer_split && pointer_split->param_count == 1 && + pointer_split->params->type == VIR_TYPE_PTR && + pointer_left->outgoing->to == pointer_split && + pointer_left->outgoing->args[0] == one && + pointer_split->outgoing->to == otherwise && + pointer_split->outgoing->args[0] == pointer_split->params && + one->uses && one->uses->edge == pointer_left->outgoing && + pointer_split->params->uses && + pointer_split->params->uses->edge == pointer_split->outgoing && + vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + edge = vir_edge_create(&func, entry, merge, &one, 1); + assert(edge && vir_edge_delete(&func, edge)); + assert(entry->terminator == VIR_TERM_NONE && !entry->outgoing && + !merge->incoming && !one->uses); + assert(vir_edge_create(&func, entry, merge, &one, 1)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(!vir_function_set_pointer_bits(&func, 16)); + entry = vir_block_create(&func); + assert(!vir_function_set_pointer_bits(&func, 32)); + assert(!vir_const_ptr(&func, entry, 0)); + assert(!vir_block_add_param(&func, entry, VIR_TYPE_PTR)); + assert(vir_block_set_return(&func, entry, vir_const_i32(&func, entry, 0))); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_const_ptr(&func, entry, 0x123456789ULL); + one = vir_const_i32(&func, entry, 0x76543210); + product = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + sum = vir_block_add_param(&func, entry, VIR_TYPE_I32); + assert(param && param->constant == 0x23456789 && + vir_ptrtoint(&func, entry, param, VIR_TYPE_I32)->constant == + 0x23456789 && + vir_inttoptr(&func, entry, one)->constant == 0x76543210); + assert(vir_ptradd(&func, entry, param, one)->constant == 0x99999999 && + vir_ptradd(&func, entry, param, vir_const_i32(&func, entry, 0)) == + param); + assert(vir_ptradd(&func, entry, vir_const_ptr(&func, entry, 0xffffffff), + vir_const_i32(&func, entry, 1)) + ->constant == 0); + two = vir_ptrtoint(&func, entry, product, VIR_TYPE_I32); + assert(two && two->opcode == VIR_OP_PTRTOINT && two->op0 == product); + assert(vir_inttoptr(&func, entry, sum)->opcode == VIR_OP_INTTOPTR); + assert(vir_ptradd(&func, entry, product, sum)->opcode == VIR_OP_PTRADD); + assert(!vir_ptrtoint(&func, entry, product, VIR_TYPE_I64) && + !vir_inttoptr(&func, entry, + vir_const_int(&func, entry, VIR_TYPE_I64, 1)) && + !vir_zext(&func, entry, product, VIR_TYPE_I64) && + !vir_ptradd(&func, entry, product, + vir_const_int(&func, entry, VIR_TYPE_I64, 1))); + assert(vir_block_set_return(&func, entry, param)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 64)); + entry = vir_block_create(&func); + sum = vir_const_int(&func, entry, VIR_TYPE_I64, 0x123456789abcdef0ULL); + param = vir_inttoptr(&func, entry, sum); + assert(param && param->constant == 0x123456789abcdef0ULL && + vir_ptrtoint(&func, entry, param, VIR_TYPE_I64)->constant == + 0x123456789abcdef0ULL); + assert(vir_ptradd(&func, entry, + vir_const_ptr(&func, entry, 0xffffffffffffffffULL), + vir_const_int(&func, entry, VIR_TYPE_I64, 1)) + ->constant == 0); + assert(vir_block_set_return(&func, entry, param)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + sum = vir_ptradd(&func, entry, param, one); + product = vir_ptradd(&func, entry, param, one); + assert(sum && product && vir_block_set_return(&func, entry, product)); + assert(vir_local_cse(&func, VIR_OPT_O1) == 1 && + entry->return_value == sum && !product->uses); + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(!strcmp(text, + "b0(%d0:ptr, %d1:i32):\n" + " %d2 = ptradd %d0, %d1\n" + " %d3 = ptradd %d0, %d1\n" + " return %d2\n")); + fclose(out); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_I32); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_add(&func, entry, param, one); + product = vir_add(&func, entry, param, two); + assert(sum && product && vir_block_set_return(&func, entry, product)); + int dce_passes; + int dce_values_scanned; + int dce_values_removed; + int dce_temporary_bytes; + int dce_temporary_peak_bytes; + assert( + vir_dce_with_stats(&func, VIR_OPT_O0, &dce_passes, &dce_values_scanned, + &dce_values_removed, &dce_temporary_bytes, + &dce_temporary_peak_bytes) == 0 && + dce_passes == 0 && dce_values_scanned == 0 && dce_values_removed == 0 && + dce_temporary_bytes == 0 && dce_temporary_peak_bytes == 0); + assert( + vir_dce_with_stats(&func, VIR_OPT_O1, &dce_passes, &dce_values_scanned, + &dce_values_removed, &dce_temporary_bytes, + &dce_temporary_peak_bytes) == 2 && + dce_passes == 1 && dce_values_scanned == 4 && dce_values_removed == 2 && + entry->head == two && entry->tail == product && two->position == 0 && + product->position == 1 && !sum->uses && !one->uses && + dce_temporary_bytes > 0 && + dce_temporary_bytes == dce_temporary_peak_bytes); + assert(vir_dce(&func, VIR_OPT_O2) == 0); + assert(vir_dce_with_stats(&func, (vir_opt_level_t) 99, &dce_passes, + &dce_values_scanned, &dce_values_removed, + &dce_temporary_bytes, + &dce_temporary_peak_bytes) == -1 && + dce_passes == 0 && dce_values_scanned == 0 && + dce_values_removed == 0 && dce_temporary_bytes == 0 && + dce_temporary_peak_bytes == 0); + assert( + vir_dce_with_stats(NULL, VIR_OPT_O1, &dce_passes, &dce_values_scanned, + &dce_values_removed, &dce_temporary_bytes, + &dce_temporary_peak_bytes) == -1 && + dce_passes == 0 && dce_values_scanned == 0 && dce_values_removed == 0 && + dce_temporary_bytes == 0 && dce_temporary_peak_bytes == 0); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + product = vir_add(&func, entry, one, two); + sum = vir_load(&func, entry, param, VIR_TYPE_I32); + assert(sum && vir_block_set_return(&func, entry, one)); + /* An unread ordinary load is dead along with its effect. */ + assert(product && vir_dce(&func, VIR_OPT_O1) == 3 && !entry->effects && + !entry->last_effect && !param->uses && entry->next_order == 1 && + vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + product = vir_add(&func, entry, one, two); + sum = vir_volatile_load(&func, entry, param, VIR_TYPE_I32); + assert(sum && vir_block_set_return(&func, entry, one)); + /* A volatile load is observable even when its value is unused. */ + assert(product && vir_dce(&func, VIR_OPT_O1) == 2 && sum->def_effect && + sum->order == 1 && sum->def_effect->order == 1 && + entry->next_order == 2 && vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + two = vir_block_add_param(&func, entry, VIR_TYPE_I32); + sum = vir_slt(&func, entry, one, two); + product = vir_slt(&func, entry, one, two); + vir_value_t *comparison = vir_ult(&func, entry, one, two); + param = vir_zext_i1(&func, entry, comparison); + two = vir_zext_i1(&func, entry, product); + one = vir_add(&func, entry, param, two); + assert(sum && product && comparison && param && two && one && + vir_block_set_return(&func, entry, one)); + assert(vir_local_cse(&func, VIR_OPT_O1) == 1 && two->op0 == sum && + param->op0 == comparison && !product->uses); + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(!strcmp(text, + "b0(%d0:i32, %d1:i32):\n" + " %d2 = slt.i32 %d0, %d1\n" + " %d3 = slt.i32 %d0, %d1\n" + " %d4 = ult.i32 %d0, %d1\n" + " %d5 = zext.i1.i32 %d4\n" + " %d6 = zext.i1.i32 %d2\n" + " %d7 = add.i32 %d5, %d6\n" + " return %d7\n")); + fclose(out); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + vir_edge_args_t duplicate_true = {merge, &one, 1}; + vir_edge_args_t duplicate_false = {merge, &two, 1}; + assert(vir_block_set_branch(&func, entry, sum, &duplicate_true, + &duplicate_false)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_edge_delete(&func, entry->true_edge)); + assert(entry->terminator == VIR_TERM_JUMP && entry->outgoing->to == merge && + merge->incoming == entry->outgoing && + entry->outgoing->args[0] == two && !one->uses && !sum->uses); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + vir_edge_args_t deleted_true = {merge, &one, 1}; + vir_edge_args_t kept_false = {otherwise, NULL, 0}; + assert(vir_block_set_branch(&func, entry, sum, &deleted_true, &kept_false)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_block_set_return(&func, otherwise, two)); + assert(vir_edge_delete(&func, entry->true_edge)); + assert(entry->terminator == VIR_TERM_JUMP && !entry->branch_condition && + !entry->true_edge && !entry->false_edge && + entry->outgoing->to == otherwise && !sum->uses && !one->uses && + !merge->incoming); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + param = vir_block_add_param(&func, otherwise, VIR_TYPE_I32); + vir_edge_args_t kept_true = {merge, NULL, 0}; + vir_edge_args_t deleted_false = {otherwise, &two, 1}; + assert(vir_block_set_branch(&func, entry, sum, &kept_true, &deleted_false)); + assert(vir_block_set_return(&func, merge, one)); + assert(vir_block_set_return(&func, otherwise, param)); + assert(vir_edge_delete(&func, entry->false_edge)); + assert(entry->terminator == VIR_TERM_JUMP && !entry->branch_condition && + !entry->true_edge && !entry->false_edge && + entry->outgoing->to == merge && !sum->uses && !two->uses && + !otherwise->incoming); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* Removing an unreachable block must unlink every ordered-effect use, + * including direct-call arguments and typed volatile memory operands. + */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + dead = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + param = vir_stack_addr(&func, entry, 9, 4, 4); + assert(entry && dead && one && param); + assert(vir_store(&func, dead, param, one)); + sum = vir_load(&func, dead, param, VIR_TYPE_I32); + assert(sum && vir_volatile_store(&func, dead, param, one)); + product = vir_volatile_load(&func, dead, param, VIR_TYPE_I32); + assert(product); + { + vir_value_t *args[] = {one, sum, product}; + assert(vir_call(&func, dead, "dead_call", args, 3, VIR_TYPE_VOID)); + } + assert(vir_block_set_return(&func, dead, product)); + assert(vir_block_set_return(&func, entry, one)); + assert(vir_verify(&func, &error)); + assert(vir_remove_unreachable(&func)); + assert(!param->uses && one->uses && one->uses->return_block == entry && + !one->uses->next && !sum->uses && !product->uses && + vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two); + product = vir_binary(&func, entry, VIR_OP_EQ, VIR_TYPE_I1, one, two); + func.arena.fail_after = 0; + assert(!vir_zext_i1(&func, entry, product)); + assert(entry->tail == product && !product->uses); + func.arena.fail_after = 1; + assert(!vir_zext_i1(&func, entry, product)); + assert(entry->tail == product && !product->uses); + func.arena.fail_after = 0; + assert(!vir_trunc_i1(&func, entry, sum)); + assert(entry->tail == product && !sum->uses); + func.arena.fail_after = 1; + assert(!vir_trunc_i1(&func, entry, sum)); + assert(entry->tail == product && !sum->uses); + func.arena.fail_after = -1; + param = vir_zext_i1(&func, entry, product); + vir_value_t *truncated = vir_trunc_i1(&func, entry, sum); + assert(param && param->opcode == VIR_OP_ZEXT && + param->type == VIR_TYPE_I32 && param->op0 == product); + assert(truncated && truncated->opcode == VIR_OP_TRUNC && + truncated->type == VIR_TYPE_I1 && truncated->op0 == sum); + assert(vir_zext_i1(&func, entry, vir_const_i1(&func, entry, 1))->constant == + 1); + assert( + vir_trunc_i1(&func, entry, vir_const_i32(&func, entry, 3))->constant == + 1); + assert(!vir_zext_i1(&func, entry, one) && + !vir_trunc_i1(&func, entry, product)); + assert(vir_block_set_return(&func, entry, param)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 64)); + entry = vir_block_create(&func); + one = vir_const_int(&func, entry, VIR_TYPE_I8, 0xff); + two = vir_const_int(&func, entry, VIR_TYPE_I8, 0x80); + sum = vir_const_int(&func, entry, VIR_TYPE_I64, 0x123456789abcdef0ULL); + product = vir_block_add_param(&func, entry, VIR_TYPE_I8); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + assert(one && two && sum && product && param && one->constant == 0xff && + two->constant == 0x80 && sum->constant == 0x123456789abcdef0ULL); + assert(vir_zext(&func, entry, one, VIR_TYPE_I64)->constant == 0xff); + assert(vir_sext(&func, entry, two, VIR_TYPE_I64)->constant == ~0x7fULL); + assert(vir_trunc(&func, entry, sum, VIR_TYPE_I16)->constant == 0xdef0); + assert(vir_zext(&func, entry, product, VIR_TYPE_I16)->opcode == + VIR_OP_ZEXT); + assert(vir_sext(&func, entry, product, VIR_TYPE_I16)->opcode == + VIR_OP_SEXT); + assert(vir_trunc(&func, entry, + vir_block_add_param(&func, entry, VIR_TYPE_I64), + VIR_TYPE_I8) + ->opcode == VIR_OP_TRUNC); + assert(!vir_zext(&func, entry, one, VIR_TYPE_I8) && + !vir_sext(&func, entry, param, VIR_TYPE_I64) && + !vir_trunc(&func, entry, one, VIR_TYPE_I16)); + assert(vir_block_set_return(&func, entry, one)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two); + product = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two); + assert(vir_block_set_return(&func, entry, product)); + vir_cse_stats_t cse_stats; + assert(vir_local_cse_with_stats(&func, VIR_OPT_O0, &cse_stats) == 0 && + cse_stats.tables == 0 && cse_stats.table_slots == 0 && + cse_stats.values == 0 && cse_stats.lookups == 0 && + cse_stats.hits == 0 && cse_stats.probes == 0 && + cse_stats.replacements == 0); + assert(vir_local_cse(&func, VIR_OPT_O0) == 0 && + entry->return_value == product); + assert(vir_local_cse_with_stats(&func, VIR_OPT_O1, &cse_stats) == 1 && + entry->return_value == sum && !product->uses); + assert(cse_stats.tables == 1 && cse_stats.values == 2 && + cse_stats.lookups == 2 && cse_stats.hits == 1 && + cse_stats.probes == 1 && cse_stats.replacements == 1 && + cse_stats.table_slots == 4); + assert(vir_local_cse(&func, VIR_OPT_O2) == 0 && + vir_local_cse_with_stats(&func, (vir_opt_level_t) 99, &cse_stats) == + -1 && + cse_stats.tables == 0 && cse_stats.table_slots == 0 && + cse_stats.values == 0 && cse_stats.lookups == 0 && + cse_stats.hits == 0 && cse_stats.probes == 0 && + cse_stats.replacements == 0 && + vir_local_cse_with_stats(NULL, VIR_OPT_O1, &cse_stats) == -1 && + cse_stats.tables == 0 && cse_stats.table_slots == 0 && + cse_stats.values == 0 && cse_stats.lookups == 0 && + cse_stats.hits == 0 && cse_stats.probes == 0 && + cse_stats.replacements == 0); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + dead = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + assert(vir_edge_create(&func, entry, merge, NULL, 0)); + assert(vir_block_set_return(&func, merge, one)); + sum = vir_binary(&func, dead, VIR_OP_ADD, VIR_TYPE_I32, one, one); + assert(sum && vir_block_set_return(&func, dead, sum)); + assert(vir_verify(&func, &error)); + assert(vir_remove_unreachable(&func)); + vir_collect_stats(&func, &stats); + assert(stats.blocks == 2 && stats.values == 1 && one->uses && + one->uses->return_block == merge && !one->uses->next); + vir_block_t *after_remove = vir_block_create(&func); + product = vir_const_i32(&func, after_remove, 2); + assert(after_remove->id == 3 && product->id > sum->id && + vir_block_set_return(&func, after_remove, product)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i32(&func, entry, 3); + product = vir_const_i32(&func, entry, 0); + assert(vir_add(&func, entry, one, product) == one); + assert(vir_sub(&func, entry, two, product) == two); + assert(vir_mul(&func, entry, one, sum) == sum); + assert(vir_mul(&func, entry, product, sum)->constant == 0); + assert(vir_sub(&func, entry, sum, sum)->constant == 0); + product = vir_const_i32(&func, entry, -9); + two = vir_const_i32(&func, entry, 2); + assert(vir_div(&func, entry, product, two, false, false)->constant == + UINT_MAX - 3u && + vir_div(&func, entry, product, two, false, true)->constant == + UINT_MAX && + vir_div(&func, entry, product, two, true, false)->constant == + 0x7ffffffb && + vir_div(&func, entry, product, two, true, true)->constant == 1); + sum = vir_const_i32(&func, entry, 0); + assert(vir_div(&func, entry, product, sum, false, false)->opcode == + VIR_OP_SDIV); + sum = vir_const_i32(&func, entry, 3); + param = vir_add(&func, entry, two, sum); + assert(param->opcode == VIR_OP_CONST && param->constant == 5); + param = vir_eq(&func, entry, one, one); + assert(param->type == VIR_TYPE_I1 && param->constant == 1); + param = vir_eq(&func, entry, two, one); + assert(param->opcode == VIR_OP_CONST && param->type == VIR_TYPE_I1 && + param->constant == 0); + product = vir_const_i32(&func, entry, -1); + sum = vir_const_i32(&func, entry, 0); + assert(vir_slt(&func, entry, product, sum)->constant == 1 && + vir_ult(&func, entry, product, sum)->constant == 0 && + vir_slt(&func, entry, sum, product)->constant == 0 && + vir_ult(&func, entry, sum, product)->constant == 1); + product = vir_const_i32(&func, entry, 0x80000000); + sum = vir_const_i32(&func, entry, 0x7fffffff); + assert(vir_slt(&func, entry, product, sum)->constant == 1 && + vir_ult(&func, entry, product, sum)->constant == 0); + assert(vir_bitand(&func, entry, product, sum)->constant == 0 && + vir_bitor(&func, entry, product, sum)->constant == 0xffffffffu && + vir_bitxor(&func, entry, product, sum)->constant == 0xffffffffu && + vir_bitnot(&func, entry, sum)->constant == 0x80000000u && + vir_neg(&func, entry, one)->constant == 0xffffffffu); + assert(vir_shl(&func, entry, one, two)->constant == 4 && + vir_lshr(&func, entry, product, one)->constant == 0x40000000 && + vir_ashr(&func, entry, product, one)->constant == 0xc0000000u && + vir_shl(&func, entry, sum, product)->opcode == VIR_OP_SHL); + param = vir_block_add_param(&func, entry, VIR_TYPE_I32); + assert( + vir_slt(&func, entry, param, one)->opcode == VIR_OP_SLT && + vir_ult(&func, entry, param, one)->opcode == VIR_OP_ULT && + vir_binary(&func, entry, VIR_OP_SLT, VIR_TYPE_I1, param, one)->opcode == + VIR_OP_SLT && + !vir_binary(&func, entry, VIR_OP_SLT, VIR_TYPE_I1, + vir_const_int(&func, entry, VIR_TYPE_I8, 1), + vir_const_int(&func, entry, VIR_TYPE_I8, 2))); + merge = vir_block_create(&func); + vir_value_t *merge_param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + product = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, param); + sum = vir_add(&func, entry, param, product); + assert(sum->op0->id < sum->op1->id); + assert(vir_edge_create(&func, entry, merge, &product, 1)); + assert(vir_block_set_return(&func, merge, merge_param)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + vir_ssa_t *ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_write(ssa, entry, 7, one)); + vir_block_t *later_ssa = vir_block_create(&func); + product = vir_const_i32(&func, later_ssa, 2); + assert(!vir_ssa_write(ssa, entry, 8, product)); + assert(vir_block_set_return(&func, later_ssa, product)); + param = vir_ssa_predeclare(ssa, merge, 7, VIR_TYPE_I32); + assert(param && vir_ssa_read(ssa, merge, 7) == param); + assert(vir_ssa_jump(ssa, entry, merge)); + assert(!vir_ssa_predeclare(ssa, merge, 8, VIR_TYPE_I32)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + vir_block_t *then_ssa = vir_block_create(&func); + vir_block_t *else_ssa = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_branch(ssa, entry, sum, then_ssa, else_ssa)); + assert(vir_ssa_write(ssa, then_ssa, 22, one)); + assert(vir_ssa_write(ssa, else_ssa, 22, two)); + assert(vir_ssa_jump(ssa, then_ssa, merge)); + assert(vir_ssa_jump(ssa, else_ssa, merge)); + param = vir_ssa_predeclare(ssa, merge, 22, VIR_TYPE_I32); + assert(param && param->is_block_param && merge->param_count == 1 && + merge->incoming->arg_count == 1 && + merge->incoming->next_incoming->arg_count == 1 && + merge->incoming->args[0] == two && + merge->incoming->next_incoming->args[0] == one); + assert(vir_ssa_seal(ssa, merge)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + then_ssa = vir_block_create(&func); + else_ssa = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_branch(ssa, entry, sum, then_ssa, else_ssa)); + assert(vir_ssa_write(ssa, then_ssa, 23, one)); + assert(vir_ssa_write(ssa, else_ssa, 23, two)); + assert(vir_ssa_jump(ssa, then_ssa, merge)); + assert(vir_ssa_jump(ssa, else_ssa, merge)); + for (int fail = 0; fail < 5; fail++) { + int next_value_id = func.next_value_id; + + func.arena.fail_after = fail; + assert(!vir_ssa_predeclare(ssa, merge, 23, VIR_TYPE_I32)); + assert(merge->param_count == 0 && merge->incoming->arg_count == 0 && + merge->incoming->next_incoming->arg_count == 0 && + func.next_value_id == next_value_id); + } + func.arena.fail_after = -1; + param = vir_ssa_predeclare(ssa, merge, 23, VIR_TYPE_I32); + assert(param && vir_ssa_seal(ssa, merge)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + then_ssa = vir_block_create(&func); + else_ssa = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_predeclare(ssa, merge, 21, VIR_TYPE_I32)); + assert(vir_ssa_branch(ssa, entry, sum, then_ssa, else_ssa)); + assert(vir_ssa_write(ssa, then_ssa, 21, one)); + assert(vir_ssa_write(ssa, else_ssa, 21, two)); + assert(vir_ssa_jump(ssa, then_ssa, merge)); + assert(vir_ssa_jump(ssa, else_ssa, merge)); + param = vir_ssa_read(ssa, merge, 21); + assert(param && param->is_block_param && vir_ssa_seal(ssa, merge) && + merge->param_count == 1 && merge->incoming->args[0] == two && + merge->incoming->next_incoming->args[0] == one); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + vir_block_t *cycle_middle = vir_block_create(&func); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_jump(ssa, entry, cycle_middle)); + assert(vir_ssa_jump(ssa, cycle_middle, entry)); + assert(vir_ssa_seal(ssa, entry) && vir_ssa_seal(ssa, cycle_middle)); + assert(!vir_ssa_read_sealed(ssa, entry, 16)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + vir_block_t *middle = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_write(ssa, entry, 15, one)); + assert(vir_ssa_jump(ssa, entry, middle)); + assert(vir_ssa_jump(ssa, middle, merge)); + assert(!vir_ssa_read_sealed(ssa, merge, 15)); + assert(vir_ssa_seal(ssa, middle) && vir_ssa_seal(ssa, merge)); + assert(vir_ssa_read_sealed(ssa, merge, 15) == one); + assert(vir_block_set_return(&func, merge, one)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 0); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_write(ssa, entry, 12, two)); + param = vir_ssa_predeclare(ssa, merge, 12, VIR_TYPE_I32); + assert(param && vir_ssa_jump(ssa, entry, merge)); + product = vir_add(&func, merge, param, one); + assert(product && vir_ssa_write(ssa, merge, 12, product)); + assert(vir_ssa_jump(ssa, merge, merge)); + assert(vir_ssa_seal(ssa, merge) && merge->param_count == 1 && + vir_ssa_read(ssa, merge, 12) == product); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + /* A loop-invariant value reaches the header from the entry and, on the back + * edge, as the header's own parameter. Sealing drops that parameter and + * forwards the entry value; only the changing counter stays. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 0); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_write(ssa, entry, 12, two) && + vir_ssa_write(ssa, entry, 14, one)); + param = vir_ssa_predeclare(ssa, merge, 12, VIR_TYPE_I32); + assert(param && vir_ssa_predeclare(ssa, merge, 14, VIR_TYPE_I32) && + vir_ssa_jump(ssa, entry, merge)); + product = vir_add(&func, merge, param, one); + assert(product && vir_ssa_write(ssa, merge, 12, product)); + assert(vir_ssa_jump(ssa, merge, merge)); + assert(vir_ssa_seal(ssa, merge) && merge->param_count == 1 && + vir_ssa_read(ssa, merge, 14) == one); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + sum = vir_const_i1(&func, entry, 1); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_write(ssa, entry, 10, one)); + assert(vir_ssa_write(ssa, entry, 11, one)); + assert(vir_ssa_write(ssa, entry, 13, one)); + param = vir_ssa_predeclare(ssa, merge, 10, VIR_TYPE_I32); + product = vir_ssa_predeclare(ssa, merge, 11, VIR_TYPE_I32); + two = vir_ssa_predeclare(ssa, merge, 13, VIR_TYPE_I32); + assert(param && product && two && + vir_ssa_branch(ssa, entry, sum, merge, merge)); + assert(vir_block_set_return(&func, merge, param)); + func.arena.fail_after = 4; + assert(!vir_ssa_seal(ssa, merge)); + assert(merge->param_count == 3 && entry->true_edge->arg_count == 3 && + entry->false_edge->arg_count == 3 && merge->return_value == param && + param->uses && vir_verify(&func, &error)); + func.arena.fail_after = -1; + assert(vir_ssa_seal(ssa, merge)); + assert(!vir_ssa_seal(ssa, merge) && merge->param_count == 0 && + merge->return_value == one && entry->true_edge->arg_count == 0 && + entry->false_edge->arg_count == 0); + assert(vir_ssa_read(ssa, merge, 10) == one); + assert(vir_ssa_read(ssa, merge, 11) == one); + assert(vir_ssa_read(ssa, merge, 13) == one); + vir_block_t *sealed_from = vir_block_create(&func); + two = vir_const_i32(&func, sealed_from, 2); + assert(vir_ssa_write(ssa, sealed_from, 10, two)); + assert(!vir_ssa_jump(ssa, sealed_from, merge)); + assert(vir_block_set_return(&func, sealed_from, two)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + vir_block_t *otherwise_ssa = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + sum = vir_const_i1(&func, entry, 1); + ssa = vir_ssa_create(&func); + assert(ssa && vir_ssa_write(ssa, entry, 9, one)); + param = vir_ssa_predeclare(ssa, merge, 9, VIR_TYPE_I32); + product = vir_ssa_predeclare(ssa, otherwise_ssa, 9, VIR_TYPE_I32); + assert(param && product); + assert(vir_ssa_branch(ssa, entry, sum, merge, otherwise_ssa)); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_block_set_return(&func, otherwise_ssa, product)); + assert(vir_verify(&func, &error)); + vir_ssa_release(ssa); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + vir_block_t *left = vir_block_create(&func); + vir_block_t *right = vir_block_create(&func); + merge = vir_block_create(&func); + vir_block_t *wrong = vir_block_create(&func); + vir_value_t *wrong_return; + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + assert(vir_block_add_param(&func, wrong, VIR_TYPE_I1)); + vir_edge_args_t left_desc = {left, NULL, 0}; + vir_edge_args_t right_desc = {right, NULL, 0}; + assert(vir_block_set_branch(&func, entry, sum, &left_desc, &right_desc)); + product = vir_binary(&func, left, VIR_OP_ADD, VIR_TYPE_I32, one, two); + assert(vir_block_set_return(&func, left, product)); + assert(vir_edge_create(&func, right, merge, &two, 1)); + assert(vir_block_set_return(&func, merge, param)); + wrong_return = vir_const_i1(&func, wrong, 1); + assert(vir_block_set_return(&func, wrong, wrong_return)); + assert(!vir_edge_redirect(&func, entry->true_edge, wrong)); + assert(vir_edge_redirect(&func, entry->true_edge, right)); + assert(entry->true_edge->to == right && entry->false_edge->to == right); + for (int fail = 0; fail < 5; fail++) { + func.arena.fail_after = fail; + assert(!vir_edge_split(&func, right->outgoing)); + assert(right->outgoing->to == merge && func.last_block == wrong); + assert(vir_verify(&func, &error)); + } + func.arena.fail_after = -1; + vir_block_t *split = vir_edge_split(&func, right->outgoing); + assert(split && right->outgoing->to == split && split->param_count == 1); + assert(split->outgoing->to == merge && + split->outgoing->args[0] == split->params); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i1(&func, entry, 1); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + vir_edge_args_t split_true = {merge, &one, 1}; + vir_edge_args_t split_false = {merge, &two, 1}; + assert(vir_block_set_branch(&func, entry, sum, &split_true, &split_false)); + split = vir_edge_split(&func, entry->true_edge); + assert(split && entry->true_edge->args[0] == one && + entry->false_edge->to == merge && + entry->false_edge->args[0] == two && split->outgoing->to == merge && + split->outgoing->args[0] == split->params); + assert(vir_block_set_return(&func, merge, param)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_I32); + one = vir_const_i32(&func, entry, 1); + edge = vir_edge_create(&func, entry, entry, &one, 1); + split = vir_edge_split(&func, edge); + assert(split && entry->outgoing->to == split && + split->outgoing->to == entry && + split->outgoing->args[0] == split->params); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + vir_value_t *alternate_condition; + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_binary(&func, entry, VIR_OP_EQ, VIR_TYPE_I1, one, two); + alternate_condition = + vir_binary(&func, entry, VIR_OP_EQ, VIR_TYPE_I1, two, one); + assert(!vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I1, one, two)); + vir_edge_args_t true_desc = {merge, NULL, 0}; + vir_edge_args_t false_desc = {NULL, NULL, 0}; + false_desc.to = otherwise; + func.arena.fail_after = 0; + assert(!vir_block_set_branch(&func, entry, sum, &true_desc, &false_desc)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = 1; + assert(!vir_block_set_branch(&func, entry, sum, &true_desc, &false_desc)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = 2; + assert(!vir_block_set_branch(&func, entry, sum, &true_desc, &false_desc)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + func.arena.fail_after = -1; + false_desc.to = NULL; + assert(!vir_block_set_branch(&func, entry, sum, &true_desc, &false_desc)); + assert(!entry->outgoing && entry->terminator == VIR_TERM_NONE); + assert(vir_replace_all_uses(&func, sum, sum)); + assert(!vir_replace_all_uses(&func, sum, one)); + func.arena.fail_after = 0; + assert(!vir_effect_create(&func, entry, VIR_EFFECT_CALL)); + assert(!entry->effects && !entry->last_effect); + func.arena.fail_after = -1; + assert(vir_effect_create(&func, entry, VIR_EFFECT_CALL)); + product = vir_const_i32(&func, merge, 3); + false_desc.to = otherwise; + assert(vir_block_set_branch(&func, entry, sum, &true_desc, &false_desc)); + assert(vir_block_set_return(&func, merge, one)); + assert(vir_block_set_return(&func, otherwise, two)); + assert(vir_verify(&func, &error)); + assert(vir_replace_all_uses(&func, sum, alternate_condition)); + assert(entry->branch_condition == alternate_condition); + assert(!sum->uses); + assert(vir_verify(&func, &error)); + assert(!vir_replace_all_uses(&func, two, product)); + assert(otherwise->return_value == two); + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(strstr(text, " %d2 = eq.i32 %d0, %d1\n") != NULL); + assert(strstr(text, " branch %d3, b1(), b2()\n") != NULL); + fclose(out); + vir_function_release(&func); + + /* Loads and stores carry reverse uses, but remain explicitly ordered + * effects. This first memory slice deliberately has no forwarding. + */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + assert(entry && param && one); + assert(!vir_load(&func, entry, one, VIR_TYPE_I32)); + assert(!vir_store(&func, entry, one, one)); + assert(!vir_effect_create(&func, entry, VIR_EFFECT_STORE)); + func.arena.fail_after = 0; + assert(!vir_load(&func, entry, param, VIR_TYPE_I32)); + assert(!entry->head && !entry->effects && !param->uses); + func.arena.fail_after = 1; + assert(!vir_load(&func, entry, param, VIR_TYPE_I32)); + assert(!entry->head && !entry->effects && !param->uses); + func.arena.fail_after = 2; + assert(!vir_load(&func, entry, param, VIR_TYPE_I32)); + assert(!entry->head && !entry->effects && !param->uses); + func.arena.fail_after = -1; + sum = vir_load(&func, entry, param, VIR_TYPE_I32); + assert(sum && sum->def_effect && sum->def_effect->position == 0); + func.arena.fail_after = 0; + assert(!vir_store(&func, entry, param, one)); + assert(!sum->uses && entry->last_effect == sum->def_effect); + func.arena.fail_after = 1; + assert(!vir_store(&func, entry, param, one)); + assert(!sum->uses && entry->last_effect == sum->def_effect); + func.arena.fail_after = -1; + effect = vir_store(&func, entry, param, one); + assert(effect && effect->position == 1); + assert(vir_replace_all_uses(&func, one, sum)); + assert(effect->stored_value == sum); + assert(vir_block_set_return(&func, entry, sum)); + assert(vir_verify(&func, &error)); + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(strstr(text, "load.i32") && strstr(text, "store %d0, %d2")); + fclose(out); + vir_function_release(&func); + + /* A load result cannot flow backward into an earlier ordered effect. */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + effect = vir_store(&func, entry, param, one); + two = vir_const_i32(&func, entry, 2); + assert(effect && two && vir_replace_all_uses(&func, one, two)); + assert(effect->stored_value == two); + sum = vir_load(&func, entry, param, VIR_TYPE_I32); + assert(effect && sum && !vir_replace_all_uses(&func, two, sum)); + product = vir_add(&func, entry, sum, one); + assert(product && !vir_replace_all_uses(&func, two, product)); + assert(effect->stored_value == two); + assert(vir_block_set_return(&func, entry, one)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* A deferred effect cannot resolve to a pure value that depends on a later + * ordered effect, even though the pure value itself can be hoisted. + */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + effect = vir_deferred_store(&func, entry, false); + sum = vir_load(&func, entry, param, VIR_TYPE_I32); + two = vir_const_i32(&func, entry, 2); + product = vir_add(&func, entry, sum, two); + assert(effect && sum && two && product && + !vir_deferred_store_resolve(&func, effect, param, product)); + vir_function_release(&func); + + /* Address roots retain object identity and layout, not backend placement. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + assert(!vir_stack_addr(&func, entry, 0, 4, 4)); + assert(!vir_global_addr(&func, entry, "object", 4, 4)); + assert(!entry->head && !func.next_value_id); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + assert(!vir_stack_addr(&func, entry, 0, 0, 4)); + assert(!vir_stack_addr(&func, entry, 0, 4, 3)); + assert(!vir_global_addr(&func, entry, "", 4, 4)); + param = vir_stack_addr(&func, entry, 7, 12, 4); + assert(param && param->address_kind == VIR_ADDRESS_STACK && + param->address_slot == 7 && param->address_size == 12 && + param->address_alignment == 4); + func.arena.fail_after = 0; + assert(!vir_stack_addr(&func, entry, 8, 4, 4)); + assert(entry->tail == param && !param->next); + func.arena.fail_after = -1; + stack_duplicate = vir_stack_addr(&func, entry, 7, 12, 4); + assert(stack_duplicate); + assert(!vir_stack_addr(&func, entry, 7, 8, 4)); + assert(!vir_stack_addr(&func, entry, 7, 12, 8)); + func.arena.fail_after = 0; + assert(!vir_global_addr(&func, entry, "object", 8, 8)); + assert(entry->tail == stack_duplicate && !stack_duplicate->next); + func.arena.fail_after = 1; + assert(!vir_global_addr(&func, entry, "object", 8, 8)); + assert(entry->tail == stack_duplicate && !stack_duplicate->next); + func.arena.fail_after = -1; + { + char name[] = "object"; + one = vir_global_addr(&func, entry, name, 8, 8); + name[0] = 'X'; + } + assert(one && one->address_kind == VIR_ADDRESS_GLOBAL && + !strcmp(one->address_name, "object") && one->address_size == 8 && + one->address_alignment == 8); + global_duplicate = vir_global_addr(&func, entry, "object", 8, 8); + assert(global_duplicate); + assert(!vir_global_addr(&func, entry, "object", 4, 4)); + two = vir_const_i32(&func, entry, 1); + sum = vir_ptradd(&func, entry, param, two); + assert(sum && sum->opcode == VIR_OP_PTRADD); + effect = vir_store(&func, entry, one, two); + product = vir_load(&func, entry, sum, VIR_TYPE_I32); + assert(effect && product && vir_block_set_return(&func, entry, product)); + assert(vir_verify(&func, &error)); + one->address_alignment = 3; + assert(!vir_verify(&func, &error)); + one->address_alignment = 8; + stack_duplicate->address_alignment = 8; + assert(!vir_verify(&func, &error)); + stack_duplicate->address_alignment = 4; + global_duplicate->address_size = 4; + assert(!vir_verify(&func, &error)); + global_duplicate->address_size = 8; + two->address_kind = VIR_ADDRESS_STACK; + assert(!vir_verify(&func, &error)); + two->address_kind = VIR_ADDRESS_NONE; + assert(vir_verify(&func, &error)); + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(strstr(text, "stackaddr 7, 12, align 4") && + strstr(text, "globaladdr @object, 8, align 8")); + fclose(out); + vir_function_release(&func); + + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + param = vir_stack_addr(&func, entry, 1, 4, 4); + one = vir_const_i32(&func, entry, 1); + effect = vir_volatile_store(&func, entry, param, one); + sum = vir_volatile_load(&func, entry, param, VIR_TYPE_I32); + assert(effect && sum && effect->kind == VIR_EFFECT_VOLATILE_STORE && + sum->def_effect->kind == VIR_EFFECT_VOLATILE_LOAD); + assert(vir_block_set_return(&func, entry, sum)); + assert(vir_verify(&func, &error)); + effect->callee = "bad"; + assert(!vir_verify(&func, &error)); + effect->callee = NULL; + sum->def_effect->arg_count = 1; + assert(!vir_verify(&func, &error)); + sum->def_effect->arg_count = 0; + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + effect = vir_effect_create(&func, entry, VIR_EFFECT_VOLATILE); + assert(effect && vir_block_set_return(&func, entry, one)); + assert(vir_verify(&func, &error)); + effect->address = one; + assert(!vir_verify(&func, &error)); + effect->address = NULL; + effect->callee = "bad"; + assert(!vir_verify(&func, &error)); + effect->callee = NULL; + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* A dead arm may use an entry constant, but that use cannot make its block + * executable or poison the live merge parameter. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + vir_block_t *dead_arm_success = vir_block_create(&func); + vir_block_t *dead_arm_failure = vir_block_create(&func); + one = vir_const_i1(&func, entry, 1); + two = vir_const_i32(&func, entry, 7); + sum = vir_const_i32(&func, entry, 9); + product = vir_binary(&func, otherwise, VIR_OP_ADD, VIR_TYPE_I32, sum, + vir_const_i32(&func, entry, 0)); + param = vir_block_add_param(&func, dead, VIR_TYPE_I32); + stack_duplicate = + vir_binary(&func, dead, VIR_OP_EQ, VIR_TYPE_I1, param, two); + assert(product && param && stack_duplicate && + vir_block_set_return(&func, dead_arm_success, two) && + vir_block_set_return(&func, dead_arm_failure, sum)); + { + vir_value_t *live_args[] = {two}; + vir_value_t *dead_args[] = {product}; + vir_edge_args_t enter_live = {merge, NULL, 0}; + vir_edge_args_t enter_dead = {otherwise, NULL, 0}; + vir_edge_args_t leave_true = {dead_arm_success, NULL, 0}; + vir_edge_args_t leave_false = {dead_arm_failure, NULL, 0}; + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_edge_create(&func, merge, dead, live_args, 1)); + assert(vir_edge_create(&func, otherwise, dead, dead_args, 1)); + assert( + vir_block_set_branch(&func, entry, one, &enter_live, &enter_dead)); + assert(vir_block_set_branch(&func, dead, stack_duplicate, &leave_true, + &leave_false)); + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) == 5 && + sccp_stats[VIR_SCCP_STAT_VALUES_REPLACED] == 1 && + sccp_stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] == 2 && + sccp_stats[VIR_SCCP_STAT_BLOCKS_REMOVED] == 2 && + entry->terminator == VIR_TERM_JUMP && + dead->terminator == VIR_TERM_JUMP && + dead->outgoing->to == dead_arm_success); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* A return-only function still allocates its one-byte edge-marker sentinel + * and must report that scoped allocation. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + assert(one && vir_block_set_return(&func, entry, one)); + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + int expected_temporary = + (int) (sizeof(unsigned char) + sizeof(unsigned long long) + + sizeof(unsigned char) + sizeof(int) + sizeof(unsigned char)); + + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) == 0 && + sccp_stats[VIR_SCCP_STAT_TEMPORARY_BYTES] == + expected_temporary && + sccp_stats[VIR_SCCP_STAT_TEMPORARY_PEAK_BYTES] == + expected_temporary); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* Only the true duplicate successor edge executes, so the merge parameter + * retains its true-edge constant despite the false edge's different value. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + dead = vir_block_create(&func); + vir_block_t *success = vir_block_create(&func); + vir_block_t *failure = vir_block_create(&func); + one = vir_const_i1(&func, entry, 1); + two = vir_const_i32(&func, entry, 7); + sum = vir_const_i32(&func, entry, 9); + param = vir_block_add_param(&func, dead, VIR_TYPE_I32); + product = vir_binary(&func, dead, VIR_OP_EQ, VIR_TYPE_I1, param, two); + assert(param && product && vir_block_set_return(&func, success, two) && + vir_block_set_return(&func, failure, sum)); + { + vir_value_t *left_args[] = {two}; + vir_value_t *right_args[] = {sum}; + vir_edge_args_t enter_left = {dead, left_args, 1}; + vir_edge_args_t enter_right = {dead, right_args, 1}; + vir_edge_args_t leave_true = {success, NULL, 0}; + vir_edge_args_t leave_false = {failure, NULL, 0}; + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert( + vir_block_set_branch(&func, entry, one, &enter_left, &enter_right)); + assert(vir_block_set_branch(&func, dead, product, &leave_true, + &leave_false)); + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) == 4 && + sccp_stats[VIR_SCCP_STAT_VALUES_REPLACED] == 1 && + sccp_stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] == 2 && + sccp_stats[VIR_SCCP_STAT_BLOCKS_REMOVED] == 1 && + entry->terminator == VIR_TERM_JUMP && + dead->terminator == VIR_TERM_JUMP && + dead->outgoing->to == success); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* SCCP arithmetic and shifts keep the selected value width. In particular, + * an i64 high word must not be truncated through an unsigned int temporary + * while folding a value that arrived through a block edge. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I64); + one = vir_const_int(&func, merge, VIR_TYPE_I64, 1); + sum = vir_binary(&func, merge, VIR_OP_ADD, VIR_TYPE_I64, param, one); + product = vir_const_int(&func, entry, VIR_TYPE_I64, 0x100000000ULL); + assert(param && one && sum && + vir_edge_create(&func, entry, merge, &product, 1) && + vir_block_set_return(&func, merge, sum)); + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->constant == 0x100000001ULL); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I64); + one = vir_const_int(&func, merge, VIR_TYPE_I64, 0); + product = vir_const_int(&func, entry, VIR_TYPE_I64, ~0ULL); + sum = vir_binary(&func, merge, VIR_OP_SLT, VIR_TYPE_I1, param, one); + assert(param && one && product && sum && + vir_edge_create(&func, entry, merge, &product, 1) && + vir_block_set_return(&func, merge, sum)); + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->constant == 1); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I64); + one = vir_const_int(&func, merge, VIR_TYPE_I32, 32); + product = vir_const_int(&func, entry, VIR_TYPE_I64, 1); + sum = vir_binary(&func, merge, VIR_OP_SHL, VIR_TYPE_I64, param, one); + assert(param && one && product && sum && + vir_edge_create(&func, entry, merge, &product, 1) && + vir_block_set_return(&func, merge, sum)); + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->constant == 0x100000000ULL); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I64); + one = vir_const_int(&func, merge, VIR_TYPE_I32, 32); + product = vir_const_int(&func, entry, VIR_TYPE_I64, 0x100000000ULL); + sum = vir_binary(&func, merge, VIR_OP_LSHR, VIR_TYPE_I64, param, one); + assert(param && one && product && sum && + vir_edge_create(&func, entry, merge, &product, 1) && + vir_block_set_return(&func, merge, sum)); + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->constant == 1); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I64); + one = vir_const_int(&func, merge, VIR_TYPE_I32, 32); + product = vir_const_int(&func, entry, VIR_TYPE_I64, 0x8000000000000000ULL); + sum = vir_binary(&func, merge, VIR_OP_ASHR, VIR_TYPE_I64, param, one); + assert(param && one && product && sum && + vir_edge_create(&func, entry, merge, &product, 1) && + vir_block_set_return(&func, merge, sum)); + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) > 0 && + merge->return_value->opcode == VIR_OP_CONST && + merge->return_value->constant == 0xffffffff80000000ULL); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* A loop header initially sees a constant entry edge, then a different + * executable backedge. Its parameter and condition must become overdefined + * rather than deleting the loop exit. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 10); + sum = vir_const_i32(&func, entry, 0); + product = vir_binary(&func, merge, VIR_OP_SLT, VIR_TYPE_I1, param, two); + stack_duplicate = + vir_binary(&func, otherwise, VIR_OP_ADD, VIR_TYPE_I32, param, one); + assert(param && product && stack_duplicate && + vir_block_set_return(&func, dead, param)); + { + vir_value_t *entry_args[] = {sum}; + vir_value_t *backedge_args[] = {stack_duplicate}; + vir_edge_args_t loop_body = {otherwise, NULL, 0}; + vir_edge_args_t loop_exit = {dead, NULL, 0}; + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_edge_create(&func, entry, merge, entry_args, 1)); + assert(vir_edge_create(&func, otherwise, merge, backedge_args, 1)); + assert(vir_block_set_branch(&func, merge, product, &loop_body, + &loop_exit)); + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) == 0 && + sccp_stats[VIR_SCCP_STAT_PASSES] >= 3 && + sccp_stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] == 0 && + sccp_stats[VIR_SCCP_STAT_BLOCKS_REMOVED] == 0 && + merge->terminator == VIR_TERM_BRANCH && + merge->true_edge->to == otherwise && + merge->false_edge->to == dead); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* SCCP keeps its lattice outside VIR, folds a forced non-canonical value, + * removes the false edge, and invalidates the resulting dead block. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two); + product = vir_const_i32(&func, entry, 3); + param = vir_binary(&func, entry, VIR_OP_EQ, VIR_TYPE_I1, sum, product); + assert(sum && product && param && vir_block_set_return(&func, merge, two) && + vir_block_set_return(&func, otherwise, one)); + { + vir_edge_args_t sccp_true_desc = {merge, NULL, 0}; + vir_edge_args_t sccp_false_desc = {otherwise, NULL, 0}; + int sccp_stats[VIR_SCCP_STAT_COUNT]; + + assert(vir_block_set_branch(&func, entry, param, &sccp_true_desc, + &sccp_false_desc)); + assert(vir_sccp_with_stats(&func, VIR_OPT_O0, sccp_stats) == 0 && + sccp_stats[VIR_SCCP_STAT_PASSES] == 0 && + sccp_stats[VIR_SCCP_STAT_VALUES_SCANNED] == 0 && + sccp_stats[VIR_SCCP_STAT_VALUES_REPLACED] == 0 && + sccp_stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] == 0 && + sccp_stats[VIR_SCCP_STAT_BLOCKS_REMOVED] == 0 && + sccp_stats[VIR_SCCP_STAT_TEMPORARY_BYTES] == 0 && + sccp_stats[VIR_SCCP_STAT_TEMPORARY_PEAK_BYTES] == 0); + assert(vir_sccp_with_stats(&func, VIR_OPT_O1, sccp_stats) == 3 && + sccp_stats[VIR_SCCP_STAT_PASSES] == 2 && + sccp_stats[VIR_SCCP_STAT_VALUES_SCANNED] == 5 && + sccp_stats[VIR_SCCP_STAT_VALUES_REPLACED] == 1 && + sccp_stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] == 1 && + sccp_stats[VIR_SCCP_STAT_BLOCKS_REMOVED] == 1 && + entry->terminator == VIR_TERM_JUMP && + entry->outgoing->to == merge && + entry->branch_condition == NULL && + sccp_stats[VIR_SCCP_STAT_TEMPORARY_BYTES] > 0 && + sccp_stats[VIR_SCCP_STAT_TEMPORARY_BYTES] == + sccp_stats[VIR_SCCP_STAT_TEMPORARY_PEAK_BYTES]); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* GVN keeps its table outside VIR and only replaces a duplicate whose + * leader dominates it. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + sum = vir_binary(&func, entry, VIR_OP_ADD, VIR_TYPE_I32, one, two); + product = vir_binary(&func, merge, VIR_OP_ADD, VIR_TYPE_I32, one, two); + assert(sum && product && vir_edge_create(&func, entry, merge, NULL, 0) && + vir_block_set_return(&func, merge, product)); + { + vir_cse_stats_t gvn_stats; + + assert(vir_gvn_with_stats(&func, VIR_OPT_O1, &gvn_stats) == 0 && + gvn_stats.tables == 0); + assert(vir_gvn_with_stats(&func, VIR_OPT_O2, &gvn_stats) == 1 && + gvn_stats.tables == 1 && gvn_stats.hits == 1 && + gvn_stats.replacements == 1 && merge->return_value == sum); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* The first diamond arm is a table leader but does not dominate the second + * arm, so GVN must retain both values. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + one = vir_block_add_param(&func, entry, VIR_TYPE_I1); + two = vir_const_i32(&func, entry, 2); + sum = vir_const_i32(&func, entry, 3); + product = vir_binary(&func, otherwise, VIR_OP_ADD, VIR_TYPE_I32, two, sum); + stack_duplicate = + vir_binary(&func, dead, VIR_OP_ADD, VIR_TYPE_I32, two, sum); + assert(param && one && two && sum && product && stack_duplicate && + vir_block_set_return(&func, merge, param)); + { + vir_value_t *then_args[] = {product}; + vir_value_t *else_args[] = {stack_duplicate}; + vir_edge_args_t gvn_true = {otherwise, NULL, 0}; + vir_edge_args_t gvn_false = {dead, NULL, 0}; + vir_cse_stats_t gvn_stats; + + assert(vir_edge_create(&func, otherwise, merge, then_args, 1)); + assert(vir_edge_create(&func, dead, merge, else_args, 1)); + assert(vir_block_set_branch(&func, entry, one, &gvn_true, &gvn_false)); + assert(vir_gvn_with_stats(&func, VIR_OPT_O2, &gvn_stats) == 0 && + gvn_stats.hits == 1 && gvn_stats.replacements == 0 && + product->uses && stack_duplicate->uses); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* LICM derives a natural loop only at O2 and hoists an effect-free value + * with operands defined outside the loop into its single-edge preheader. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + one = vir_block_add_param(&func, entry, VIR_TYPE_I1); + two = vir_block_add_param(&func, entry, VIR_TYPE_I32); + sum = vir_block_add_param(&func, entry, VIR_TYPE_I32); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + product = vir_binary(&func, otherwise, VIR_OP_MUL, VIR_TYPE_I32, two, sum); + stack_duplicate = vir_const_i32(&func, entry, 1); + assert(one && two && sum && param && product && stack_duplicate); + { + vir_value_t *entry_args[] = {stack_duplicate}; + vir_value_t *backedge_args[] = {product}; + vir_edge_args_t licm_true = {otherwise, NULL, 0}; + vir_edge_args_t licm_false = {dead, NULL, 0}; + int licm_stats[VIR_LICM_STAT_COUNT]; + int licm_hoisted; + + assert( + vir_edge_create(&func, entry, merge, entry_args, 1) && + vir_edge_create(&func, otherwise, merge, backedge_args, 1) && + vir_block_set_branch(&func, merge, one, &licm_true, &licm_false) && + vir_block_set_return(&func, dead, param)); + assert(vir_licm_with_stats(&func, VIR_OPT_O1, licm_stats) == 1 && + licm_stats[VIR_LICM_STAT_LOOPS] == 1 && + licm_stats[VIR_LICM_STAT_VALUES_HOISTED] == 1 && + licm_stats[VIR_LICM_STAT_TEMPORARY_BYTES] > 0 && + product->block == entry); + licm_hoisted = vir_licm_with_stats(&func, VIR_OPT_O2, licm_stats); + assert(licm_hoisted == 0 && licm_stats[VIR_LICM_STAT_LOOPS] == 1 && + licm_stats[VIR_LICM_STAT_VALUES_HOISTED] == 0 && + product->block == entry && otherwise->head == NULL); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* The dominance ID map must still reject an out-of-function block whose ID + * collides with a live block; the old LICM scan required pointer identity. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + two = vir_block_add_param(&func, entry, VIR_TYPE_I32); + product = vir_binary(&func, otherwise, VIR_OP_MUL, VIR_TYPE_I32, one, two); + assert(entry && merge && otherwise && dead && entry_param && one && two && + product); + { + vir_edge_args_t loop_true = {otherwise, NULL, 0}; + vir_edge_args_t loop_false = {dead, NULL, 0}; + vir_block_t stale_block; + + assert(vir_edge_create(&func, entry, merge, NULL, 0) && + vir_edge_create(&func, otherwise, merge, NULL, 0) && + vir_block_set_branch(&func, merge, entry_param, &loop_true, + &loop_false) && + vir_block_set_return(&func, dead, one) && + vir_verify(&func, &error)); + memset(&stale_block, 0, sizeof(stale_block)); + stale_block.id = entry->id; + two->block = &stale_block; + assert(vir_licm(&func, VIR_OPT_O2) == 0 && product->block == otherwise); + two->block = entry; + assert(vir_verify(&func, &error)); + } + vir_function_release(&func); + + /* Multiple latches belong to one natural loop. LICM must union the backedge + * regions before identifying the loop's entry preheader. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + vir_block_t *second_body = vir_block_create(&func); + vir_block_t *first_latch = vir_block_create(&func); + vir_block_t *second_latch = vir_block_create(&func); + vir_value_t *loop_value; + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1); + one = vir_block_add_param(&func, entry, VIR_TYPE_I1); + two = vir_block_add_param(&func, entry, VIR_TYPE_I32); + sum = vir_block_add_param(&func, entry, VIR_TYPE_I32); + param = vir_block_add_param(&func, merge, VIR_TYPE_I1); + loop_value = vir_block_add_param(&func, merge, VIR_TYPE_I32); + product = vir_binary(&func, otherwise, VIR_OP_MUL, VIR_TYPE_I32, two, sum); + { + vir_value_t *entry_args[] = {entry_param, two}; + vir_value_t *backedge_args[] = {param, product}; + vir_edge_args_t outer_exit = {dead, NULL, 0}; + vir_edge_args_t first_path = {otherwise, NULL, 0}; + vir_edge_args_t two_paths = {first_latch, NULL, 0}; + vir_edge_args_t second_path = {second_body, NULL, 0}; + int multi_latch_stats[VIR_LICM_STAT_COUNT]; + + assert(entry && merge && otherwise && dead && second_body && + first_latch && second_latch && entry_param && one && two && + sum && param && loop_value && product && + vir_edge_create(&func, entry, merge, entry_args, 2) && + vir_block_set_branch(&func, merge, param, &first_path, + &outer_exit) && + vir_block_set_branch(&func, otherwise, one, &two_paths, + &second_path) && + vir_edge_create(&func, second_body, second_latch, NULL, 0) && + vir_edge_create(&func, first_latch, merge, backedge_args, 2) && + vir_edge_create(&func, second_latch, merge, backedge_args, 2) && + vir_block_set_return(&func, dead, loop_value) && + vir_verify(&func, &error)); + assert(vir_licm_with_stats(&func, VIR_OPT_O2, multi_latch_stats) == 1 && + multi_latch_stats[VIR_LICM_STAT_LOOPS] == 1 && + multi_latch_stats[VIR_LICM_STAT_VALUES_HOISTED] == 1 && + multi_latch_stats[VIR_LICM_STAT_TEMPORARY_BYTES] > 0 && + product->block == entry && vir_verify(&func, &error)); + } + vir_function_release(&func); + + /* LICM's pass-local dominator bitsets keep sparse large-CFG storage below + * the former four-byte-per-block-pair matrix while retaining loop motion. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + dead = vir_block_create(&func); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + two = vir_block_add_param(&func, entry, VIR_TYPE_I32); + assert(entry && merge && dead && entry_param && one && two); + { + const int loop_body_blocks = 513; + vir_block_t *loop_blocks[loop_body_blocks]; + vir_value_t *invariants[loop_body_blocks]; + vir_edge_args_t loop_true; + vir_edge_args_t loop_false = {dead, NULL, 0}; + int licm_stats[VIR_LICM_STAT_COUNT]; + int block_count; + + for (int i = 0; i < loop_body_blocks; i++) { + loop_blocks[i] = vir_block_create(&func); + assert(loop_blocks[i]); + } + loop_true.to = loop_blocks[loop_body_blocks - 1]; + loop_true.args = NULL; + loop_true.arg_count = 0; + assert(vir_edge_create(&func, entry, merge, NULL, 0)); + invariants[loop_body_blocks - 1] = + vir_binary(&func, loop_blocks[loop_body_blocks - 1], VIR_OP_MUL, + VIR_TYPE_I32, one, two); + assert(invariants[loop_body_blocks - 1]); + for (int i = loop_body_blocks - 2; i >= 0; i--) { + invariants[i] = vir_binary(&func, loop_blocks[i], VIR_OP_ADD, + VIR_TYPE_I32, invariants[i + 1], two); + assert(invariants[i] && vir_edge_create(&func, loop_blocks[i + 1], + loop_blocks[i], NULL, 0)); + } + assert(vir_edge_create(&func, loop_blocks[0], merge, NULL, 0) && + vir_block_set_branch(&func, merge, entry_param, &loop_true, + &loop_false) && + vir_block_set_return(&func, dead, one)); + block_count = vir_function_block_count(&func); + assert(vir_verify(&func, &error)); + assert(vir_licm_with_stats(&func, VIR_OPT_O2, licm_stats) == + loop_body_blocks && + licm_stats[VIR_LICM_STAT_LOOPS] == 1 && + licm_stats[VIR_LICM_STAT_VALUES_HOISTED] == loop_body_blocks && + invariants[0]->block == entry && + invariants[loop_body_blocks - 1]->block == entry && + vir_verify(&func, &error)); + assert(licm_stats[VIR_LICM_STAT_TEMPORARY_BYTES] < + block_count * block_count * (int) sizeof(int)); + } + vir_function_release(&func); + + /* Nested-loop LICM must hoist values to the nearest valid preheader: an + * outer invariant moves to entry, while an inner-only invariant stays + * inside the outer loop and moves immediately before the inner header. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + vir_block_t *inner_header = vir_block_create(&func); + vir_block_t *inner_body = vir_block_create(&func); + vir_block_t *outer_latch = vir_block_create(&func); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1); + one = vir_block_add_param(&func, entry, VIR_TYPE_I1); + two = vir_block_add_param(&func, entry, VIR_TYPE_I32); + vir_value_t *three = vir_block_add_param(&func, entry, VIR_TYPE_I32); + vir_value_t *four = vir_block_add_param(&func, entry, VIR_TYPE_I32); + vir_value_t *outer_value = vir_block_add_param(&func, merge, VIR_TYPE_I32); + vir_value_t *inner_value = + vir_block_add_param(&func, inner_header, VIR_TYPE_I32); + vir_value_t *outer_invariant; + vir_value_t *inner_invariant; + vir_value_t *inner_next; + vir_value_t *outer_next; + int nested_licm_stats[VIR_LICM_STAT_COUNT]; + vir_edge_args_t outer_true = {otherwise, NULL, 0}; + vir_edge_args_t outer_false = {dead, NULL, 0}; + vir_edge_args_t inner_true = {inner_body, NULL, 0}; + vir_edge_args_t inner_false = {outer_latch, NULL, 0}; + + assert(entry && merge && otherwise && dead && inner_header && inner_body && + outer_latch && entry_param && one && two && three && four && + outer_value && inner_value); + outer_invariant = + vir_binary(&func, merge, VIR_OP_MUL, VIR_TYPE_I32, two, three); + inner_invariant = vir_binary(&func, inner_header, VIR_OP_ADD, VIR_TYPE_I32, + outer_value, four); + inner_next = vir_binary(&func, inner_body, VIR_OP_ADD, VIR_TYPE_I32, + inner_value, inner_invariant); + outer_next = vir_binary(&func, outer_latch, VIR_OP_ADD, VIR_TYPE_I32, + outer_value, two); + vir_value_t *initial_args[] = {two}; + vir_value_t *inner_backedge_args[] = {inner_next}; + vir_value_t *outer_backedge_args[] = {outer_next}; + assert(outer_invariant && inner_invariant && inner_next && outer_next && + vir_edge_create(&func, entry, merge, initial_args, 1) && + vir_block_set_branch(&func, merge, entry_param, &outer_true, + &outer_false) && + vir_edge_create(&func, otherwise, inner_header, initial_args, 1) && + vir_block_set_branch(&func, inner_header, one, &inner_true, + &inner_false) && + vir_edge_create(&func, inner_body, inner_header, inner_backedge_args, + 1) && + vir_edge_create(&func, outer_latch, merge, outer_backedge_args, 1) && + vir_block_set_return(&func, dead, outer_invariant) && + vir_verify(&func, &error)); + assert(vir_licm_with_stats(&func, VIR_OPT_O2, nested_licm_stats) == 2 && + nested_licm_stats[VIR_LICM_STAT_LOOPS] == 2 && + nested_licm_stats[VIR_LICM_STAT_VALUES_HOISTED] == 2 && + nested_licm_stats[VIR_LICM_STAT_TEMPORARY_BYTES] > 0 && + outer_invariant->block == entry && + inner_invariant->block == otherwise && vir_verify(&func, &error)); + vir_function_release(&func); + + /* CFG simplification redirects each predecessor through an empty jump + * block, preserves a duplicate branch target, and removes the forwarder. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + one = vir_const_i1(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + assert(one && two && vir_edge_create(&func, merge, dead, NULL, 0) && + vir_block_set_return(&func, dead, two)); + { + vir_edge_args_t cfg_true_desc = {merge, NULL, 0}; + vir_edge_args_t cfg_false_desc = {otherwise, NULL, 0}; + + assert(vir_edge_create(&func, otherwise, merge, NULL, 0)); + assert(vir_block_set_branch(&func, entry, one, &cfg_true_desc, + &cfg_false_desc)); + assert(vir_simplify_cfg(&func, VIR_OPT_O0) == 0); + assert(vir_simplify_cfg(&func, VIR_OPT_O1) > 0 && + entry->true_edge->to == dead && entry->false_edge->to == dead && + func.blocks == entry && entry->next == dead && !dead->next); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* Branches on promoted booleans preserve polarity and reverse-use links. */ + for (int width = 0; width < 2; width++) { + for (int constant = 0; constant < 3; constant++) { + vir_type_t type = width ? VIR_TYPE_I64 : VIR_TYPE_I32; + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + param = vir_block_add_param(&func, entry, VIR_TYPE_I1); + vir_value_t *promoted = vir_zext(&func, entry, param, type); + one = vir_const_int(&func, entry, type, constant); + product = vir_eq(&func, entry, promoted, one); + vir_edge_args_t yes = {merge, NULL, 0}; + vir_edge_args_t no = {otherwise, NULL, 0}; + assert(product && + vir_block_set_branch(&func, entry, product, &yes, &no)); + assert(vir_block_set_return(&func, merge, + vir_const_i32(&func, merge, 1))); + assert(vir_block_set_return(&func, otherwise, + vir_const_i32(&func, otherwise, 0))); + assert(vir_verify(&func, &error)); + assert(vir_simplify_cfg(&func, VIR_OPT_O1) >= 0); + assert(entry->branch_condition == (constant < 2 ? param : product)); + assert(entry->true_edge->to == (constant ? merge : otherwise)); + assert(vir_dce(&func, VIR_OPT_O1) >= 0); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + } + } + + /* Unreachable pruning removes rewritten forwarders and larger CFGs while + * preserving edge/use ownership for the later optimization passes. + */ + for (int total = 40; total <= 41; total++) { + vir_block_t *forwarder; + vir_block_t *target; + vir_function_init(&func, 64); + entry = vir_block_create(&func); + forwarder = vir_block_create(&func); + target = vir_block_create(&func); + two = vir_const_i32(&func, entry, 2); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I32); + assert(entry && forwarder && target && two && entry_param); + assert(vir_edge_create(&func, entry, forwarder, NULL, 0) && + vir_edge_create(&func, forwarder, target, NULL, 0) && + vir_block_set_return(&func, target, two)); + for (int extra = 3; extra < total; extra++) { + vir_block_t *unreachable = vir_block_create(&func); + vir_value_t *dead_value = + extra == 3 ? vir_neg(&func, unreachable, entry_param) : two; + assert(unreachable && dead_value && + vir_block_set_return(&func, unreachable, dead_value)); + } + assert(vir_simplify_cfg(&func, VIR_OPT_O1) > 0); + assert(vir_verify(&func, &error)); + if (total == 40) + assert(func.blocks == entry && entry->next == target && + !target->next); + assert(vir_function_block_count(&func) == 2 && func.blocks == entry && + entry->next == target && !target->next); + vir_function_release(&func); + } + + /* Keep the largest transactional-pruning case coupled to the optimizer: + * after dead blocks are removed, verification and LICM must still walk the + * surviving loop using the compact block-index map. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_I1); + one = vir_block_add_param(&func, entry, VIR_TYPE_I32); + param = vir_block_add_param(&func, merge, VIR_TYPE_I32); + sum = vir_binary(&func, otherwise, VIR_OP_MUL, VIR_TYPE_I32, one, one); + assert(entry && merge && otherwise && dead && entry_param && one && param && + sum); + { + vir_value_t *entry_args[] = {one}; + vir_value_t *back_args[] = {sum}; + vir_edge_args_t loop_true = {otherwise, NULL, 0}; + vir_edge_args_t loop_false = {dead, NULL, 0}; + + assert(vir_edge_create(&func, entry, merge, entry_args, 1) && + vir_edge_create(&func, otherwise, merge, back_args, 1) && + vir_block_set_branch(&func, merge, entry_param, &loop_true, + &loop_false) && + vir_block_set_return(&func, dead, param)); + } + for (int extra = 4; extra < 42; extra++) { + vir_block_t *unreachable = vir_block_create(&func); + vir_value_t *dead_value = vir_neg(&func, unreachable, one); + + assert(unreachable && dead_value && + vir_block_set_return(&func, unreachable, dead_value)); + } + assert(vir_verify(&func, &error)); + assert(vir_remove_unreachable(&func)); + assert(vir_verify(&func, &error)); + assert(vir_function_block_count(&func) == 4 && func.blocks == entry && + entry->next == merge && merge->next == otherwise && + otherwise->next == dead && !dead->next); + { + int licm_stats[VIR_LICM_STAT_COUNT]; + + assert(vir_licm_with_stats(&func, VIR_OPT_O2, licm_stats) == 1 && + licm_stats[VIR_LICM_STAT_LOOPS] == 1 && + licm_stats[VIR_LICM_STAT_VALUES_HOISTED] == 1 && + sum->block == entry); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* Large CFG SCCP rewrites defer unreachable deletion until after LICM. This + * keeps the optimizer's traversal on the original block topology; the final + * prune still removes the dead branch and unreachable tail. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + one = vir_const_i1(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + assert(entry && merge && otherwise && one && two); + { + vir_edge_args_t sccp_true = {merge, NULL, 0}; + vir_edge_args_t sccp_false = {otherwise, NULL, 0}; + + assert( + vir_block_set_return(&func, merge, two) && + vir_block_set_return(&func, otherwise, two) && + vir_block_set_branch(&func, entry, one, &sccp_true, &sccp_false)); + } + for (int extra = 3; extra < 42; extra++) { + vir_block_t *unreachable = vir_block_create(&func); + + assert(unreachable && vir_block_set_return(&func, unreachable, two)); + } + { + int sccp_stats[VIR_SCCP_STAT_COUNT]; + int licm_stats[VIR_LICM_STAT_COUNT]; + + assert(vir_sccp_with_stats_and_pruning(&func, VIR_OPT_O2, sccp_stats, + false) > 0 && + sccp_stats[VIR_SCCP_STAT_BRANCHES_SIMPLIFIED] == 1 && + sccp_stats[VIR_SCCP_STAT_BLOCKS_REMOVED] == 0 && + vir_function_block_count(&func) == 42); + assert(vir_licm_with_stats(&func, VIR_OPT_O2, licm_stats) >= 0 && + vir_verify(&func, &error)); + assert(vir_remove_unreachable(&func) && + vir_function_block_count(&func) == 2 && + vir_verify(&func, &error)); + } + vir_function_release(&func); + + /* Dominance verification must remain bounded for large CFGs. A value + * defined in a non-entry dominator reaches the end of a 513-block chain + * without allocating the old quadratic int matrix. + */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + assert(entry && merge && vir_edge_create(&func, entry, merge, NULL, 0)); + one = vir_const_i32(&func, merge, 1); + param = vir_stack_addr(&func, merge, 1, 4, 4); + { + vir_block_t *previous = merge; + for (int block_index = 0; block_index < 513; block_index++) { + vir_block_t *current = vir_block_create(&func); + assert(current && + vir_edge_create(&func, previous, current, NULL, 0)); + previous = current; + } + assert(param && vir_store(&func, previous, param, one)); + product = vir_load(&func, previous, param, VIR_TYPE_I32); + assert(product && vir_block_set_return(&func, previous, product)); + } + assert(vir_verify(&func, &error)); + vir_function_release(&func); + + /* A sibling path reaching the use without the definition must be rejected, + * even when the CFG is otherwise well formed. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + merge = vir_block_create(&func); + otherwise = vir_block_create(&func); + dead = vir_block_create(&func); + one = vir_const_i1(&func, entry, 1); + two = vir_const_i32(&func, merge, 2); + { + vir_edge_args_t true_edge = {merge, NULL, 0}; + vir_edge_args_t false_edge = {otherwise, NULL, 0}; + assert( + entry && merge && otherwise && dead && one && two && + vir_block_set_branch(&func, entry, one, &true_edge, &false_edge) && + vir_edge_create(&func, merge, dead, NULL, 0) && + vir_edge_create(&func, otherwise, dead, NULL, 0) && + vir_block_set_return(&func, dead, two)); + } + assert(!vir_verify(&func, &error)); + vir_function_release(&func); + + /* The entry dominator row must clear every bit above the first machine + * word. Otherwise a high-index definition is falsely reported as dominating + * a sibling path. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + otherwise = vir_block_create(&func); + merge = vir_block_create(&func); + { + const int chain_count = (int) (sizeof(unsigned long) * CHAR_BIT) + 1; + vir_block_t *first; + vir_block_t *previous; + vir_block_t *definition; + vir_value_t *condition; + vir_edge_args_t true_edge; + vir_edge_args_t false_edge; + + assert(entry && otherwise && merge); + first = vir_block_create(&func); + condition = vir_const_i1(&func, entry, 1); + assert(first && condition); + true_edge.to = first; + true_edge.args = NULL; + true_edge.arg_count = 0; + false_edge.to = otherwise; + false_edge.args = NULL; + false_edge.arg_count = 0; + assert(vir_block_set_branch(&func, entry, condition, &true_edge, + &false_edge)); + previous = first; + for (int index = 1; index < chain_count; index++) { + vir_block_t *current = vir_block_create(&func); + + assert(current && + vir_edge_create(&func, previous, current, NULL, 0)); + previous = current; + } + definition = vir_block_create(&func); + assert(definition); + two = vir_const_i32(&func, definition, 2); + assert(two && vir_edge_create(&func, previous, definition, NULL, 0) && + vir_edge_create(&func, otherwise, merge, NULL, 0) && + vir_edge_create(&func, definition, merge, NULL, 0) && + vir_block_set_return(&func, merge, two)); + } + assert(!vir_verify(&func, &error)); + vir_function_release(&func); + + /* Direct calls own variable-arity reverse uses and stay ordered effects. */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_i32(&func, entry, 1); + two = vir_const_i32(&func, entry, 2); + for (int fail = 0; fail < 5; fail++) { + vir_value_t *args[] = {one, two}; + func.arena.fail_after = fail; + assert(!vir_call(&func, entry, "sum", args, 2, VIR_TYPE_I32)); + assert(entry->tail == two && !two->next && !entry->effects); + } + func.arena.fail_after = -1; + { + vir_value_t *args[] = {one, two}; + char callee[] = "sum"; + effect = vir_call(&func, entry, "sink", args, 2, VIR_TYPE_VOID); + call = vir_call(&func, entry, callee, args, 2, VIR_TYPE_I32); + callee[0] = 'X'; + } + assert(effect && !effect->result && call && call->result); + assert(!strcmp(call->callee, "sum")); + assert(vir_replace_all_uses(&func, one, two)); + assert(effect->args[0] == two && call->args[0] == two); + assert(vir_block_set_return(&func, entry, call->result)); + assert(vir_verify(&func, &error)); + call->address = one; + assert(!vir_verify(&func, &error)); + call->address = NULL; + effect->stored_value = two; + assert(!vir_verify(&func, &error)); + effect->stored_value = NULL; + assert(vir_verify(&func, &error)); + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(strstr(text, "call @sink(%d1, %d1)") && + strstr(text, "%d2 = call.i32 @sum(%d1, %d1)")); + fclose(out); + vir_function_release(&func); + + /* Indirect calls retain the callee pointer as an ordinary effect use so + * RAUW, dominance verification, and deterministic dumps see the target. + */ + vir_function_init(&func, 64); + assert(vir_function_set_pointer_bits(&func, sizeof(void *) * CHAR_BIT)); + entry = vir_block_create(&func); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_const_i32(&func, entry, 1); + { + vir_value_t *args[] = {one}; + vir_call_abi_type_t param_type = {VIR_TYPE_I32, true, false}; + vir_call_signature_t signature = {{VIR_TYPE_I32, false, false}, + ¶m_type, + 1, + false, + 1, + NULL, + (unsigned int) -1}; + vir_call_signature_t bad_signature = signature; + vir_function_t other_func; + + call = + vir_call_indirect(&func, entry, entry_param, args, 1, VIR_TYPE_I32); + assert(call && !call->callee && call->callee_value == entry_param && + call->callee_value_use && call->arg_count == 1); + bad_signature.result.type = VIR_TYPE_I64; + assert(!vir_call_set_signature(&func, call, &bad_signature)); + vir_function_init(&other_func, 16); + assert(vir_function_set_pointer_bits(&other_func, + sizeof(void *) * CHAR_BIT)); + assert(vir_block_create(&other_func)); + assert(!vir_call_set_signature(&other_func, call, &signature)); + vir_function_release(&other_func); + assert(vir_call_set_signature(&func, call, &signature)); + assert(call->signature && call->signature->params != ¶m_type && + call->signature->params[0].is_unsigned); + param_type.is_unsigned = false; + assert(call->signature->params[0].is_unsigned); + param_type.is_unsigned = true; + assert(vir_replace_all_uses(&func, entry_param, param)); + assert(call->callee_value == param); + assert(vir_block_set_return(&func, entry, call->result)); + assert(vir_verify(&func, &error)); + { + vir_use_t fake_callee_use = *call->callee_value_use; + vir_use_t *callee_use = call->callee_value_use; + + call->callee_value_use = &fake_callee_use; + assert(!vir_verify(&func, &error)); + call->callee_value_use = callee_use; + } + call->callee_value = NULL; + assert(!vir_verify(&func, &error)); + call->callee_value = param; + assert(vir_verify(&func, &error)); + { + vir_call_signature_t *saved_signature = + (vir_call_signature_t *) call->signature; + vir_call_abi_type_t *saved_params = + (vir_call_abi_type_t *) saved_signature->params; + vir_type_t saved_type = saved_params[0].type; + + saved_params[0].type = VIR_TYPE_I64; + assert(!vir_verify(&func, &error)); + saved_params[0].type = saved_type; + assert(vir_verify(&func, &error)); + } + } + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(strstr(text, "%d3 = call.i32 %d1(%d2) sig(i32.unsigned)->i32")); + fclose(out); + vir_function_release(&func); + + /* A C _Bool parameter/result has an i8 ABI slot while its canonical VIR + * value is i1. Keep both identities explicit in the attached signature. + */ + vir_function_init(&func, 32); + assert(vir_function_set_pointer_bits(&func, sizeof(void *) * CHAR_BIT)); + entry = vir_block_create(&func); + entry_param = vir_block_add_param(&func, entry, VIR_TYPE_PTR); + one = vir_const_int(&func, entry, VIR_TYPE_I1, 1); + { + vir_value_t *args[] = {one}; + vir_call_abi_type_t param_type = {VIR_TYPE_I8, false, true}; + vir_call_signature_t signature = { + {VIR_TYPE_I8, false, true}, ¶m_type, 1, false, 1, NULL, + (unsigned int) -1}; + + call = + vir_call_indirect(&func, entry, entry_param, args, 1, VIR_TYPE_I1); + assert(call && vir_call_set_signature(&func, call, &signature)); + assert(vir_block_set_return(&func, entry, call->result)); + assert(vir_verify(&func, &error)); + } + out = tmpfile(); + assert(out); + vir_print(&func, out); + rewind(out); + text_len = fread(text, 1, sizeof(text) - 1, out); + assert(text_len > 0); + text[text_len] = '\0'; + assert(strstr(text, "sig(i8.bool)->i8.bool")); + fclose(out); + vir_function_release(&func); + + /* i64 pure values retain their width through folding, comparisons, and + * shifts. Shift counts deliberately remain i32. + */ + vir_function_init(&func, 64); + entry = vir_block_create(&func); + one = vir_const_int(&func, entry, VIR_TYPE_I64, 1); + two = vir_const_int(&func, entry, VIR_TYPE_I64, 0xffffffffULL); + product = vir_const_int(&func, entry, VIR_TYPE_I64, 0xffffffffffffffffULL); + param = vir_const_i32(&func, entry, 32); + sum = vir_add(&func, entry, product, one); + assert(sum && sum->opcode == VIR_OP_CONST && sum->type == VIR_TYPE_I64 && + sum->constant == 0); + sum = vir_sub(&func, entry, one, two); + assert(sum && sum->opcode == VIR_OP_CONST && + sum->constant == 0xffffffff00000002ULL); + sum = vir_shl(&func, entry, one, param); + assert(sum && sum->opcode == VIR_OP_CONST && + sum->constant == 0x100000000ULL); + assert(!vir_shl(&func, entry, one, one)); + param = vir_const_i32(&func, entry, 63); + sum = vir_ashr(&func, entry, product, param); + assert(sum && sum->opcode == VIR_OP_CONST && + sum->constant == 0xffffffffffffffffULL); + sum = vir_slt(&func, entry, product, one); + assert(sum && sum->opcode == VIR_OP_CONST && sum->constant == 1); + sum = vir_ult(&func, entry, product, one); + assert(sum && sum->opcode == VIR_OP_CONST && sum->constant == 0); + product = vir_block_add_param(&func, entry, VIR_TYPE_I64); + param = vir_const_i32(&func, entry, 1); + two = vir_shl(&func, entry, product, param); + assert(product && param && two && param->uses && param->uses->user == two); + { + vir_use_t *shift_count_use = param->uses; + + param->uses = NULL; + one->uses = shift_count_use; + two->op1 = one; + assert(!vir_verify(&func, &error)); + two->op1 = param; + one->uses = NULL; + param->uses = shift_count_use; + } + assert(vir_block_set_return(&func, entry, sum)); + assert(vir_verify(&func, &error)); + vir_function_release(&func); + return 0; +} diff --git a/tests/vir-fuzz.c b/tests/vir-fuzz.c new file mode 100644 index 00000000..97218c5f --- /dev/null +++ b/tests/vir-fuzz.c @@ -0,0 +1,153 @@ +#include "vir.h" + +#include + +int main(void) +{ + unsigned int state = 1; + int round; + + for (round = 0; round < 64; round++) { + vir_function_t func; + vir_block_t *entry; + vir_block_t *loop; + vir_block_t *merge; + vir_value_t *values[18]; + vir_value_t *params[2]; + vir_value_t *loop_param; + vir_value_t *arg; + vir_value_t *alternate_condition; + vir_value_t *widened; + vir_value_t *narrowed; + vir_value_t *address; + vir_value_t *loaded; + vir_effect_t *store; + vir_value_t *merge_args[2]; + vir_value_t *loop_args[1]; + char *error; + int i; + int old_operand; + vir_use_t *use; + vir_edge_t *old_edge; + vir_block_t *old_return_block; + vir_block_t *old_branch_block; + + vir_function_init(&func, 128); + assert(vir_function_set_pointer_bits(&func, 32)); + entry = vir_block_create(&func); + loop = vir_block_create(&func); + merge = vir_block_create(&func); + loop_param = vir_block_add_param(&func, loop, VIR_TYPE_I32); + params[0] = vir_block_add_param(&func, merge, VIR_TYPE_I32); + params[1] = vir_block_add_param(&func, merge, VIR_TYPE_I32); + state = state * 1103515245u + 12345u; + values[0] = vir_const_i32(&func, entry, (int) (state & 255)); + state = state * 1103515245u + 12345u; + values[1] = vir_const_i32(&func, entry, (int) (state & 255)); + for (i = 2; i < 18; i++) { + state = state * 1103515245u + 12345u; + values[i] = vir_binary( + &func, entry, state & 1 ? VIR_OP_ADD : VIR_OP_MUL, VIR_TYPE_I32, + values[state % i], values[(state >> 8) % i]); + assert(values[i]); + } + arg = vir_binary(&func, entry, VIR_OP_EQ, VIR_TYPE_I1, values[16], + values[17]); + alternate_condition = vir_binary(&func, entry, VIR_OP_EQ, VIR_TYPE_I1, + values[17], values[16]); + widened = vir_zext_i1(&func, entry, arg); + narrowed = vir_trunc_i1(&func, entry, values[17]); + assert(widened && narrowed && widened->type == VIR_TYPE_I32 && + narrowed->type == VIR_TYPE_I1); + address = vir_stack_addr(&func, entry, (unsigned int) round, 4, 4); + store = vir_store(&func, entry, address, values[0]); + loaded = vir_load(&func, entry, address, VIR_TYPE_I32); + assert(address && store && loaded); + merge_args[0] = widened; + merge_args[1] = values[16]; + loop_args[0] = values[17]; + vir_edge_args_t true_desc = {loop, loop_args, 1}; + vir_edge_args_t false_desc = {loop, loop_args, 1}; + assert( + vir_block_set_branch(&func, entry, arg, &true_desc, &false_desc)); + assert(params[0]->type == VIR_TYPE_I32 && + params[1]->type == VIR_TYPE_I32); + loop_args[0] = loop_param; + true_desc.to = loop; + true_desc.args = loop_args; + true_desc.arg_count = 1; + false_desc.to = merge; + false_desc.args = merge_args; + false_desc.arg_count = 2; + assert(vir_block_set_branch(&func, loop, arg, &true_desc, &false_desc)); + assert(vir_block_set_return(&func, merge, params[0])); + assert(vir_verify(&func, &error)); + + /* Exercise every reverse-use kind with a dominating replacement. */ + assert(vir_replace_all_uses(&func, values[0], values[1])); + assert(vir_replace_all_uses(&func, loop_param, values[16])); + assert(vir_replace_all_uses(&func, params[0], values[16])); + assert(vir_replace_all_uses(&func, arg, alternate_condition)); + assert(vir_verify(&func, &error)); + + /* Deterministic malformed-graph mutations exercise the verifier's + * reverse links, CFG shape, edge arguments, and placement checks. + */ + merge->incoming = NULL; + assert(!vir_verify(&func, &error)); + merge->incoming = loop->false_edge; + assert(vir_verify(&func, &error)); + + entry->false_edge = entry->true_edge; + assert(!vir_verify(&func, &error)); + entry->false_edge = entry->outgoing; + assert(vir_verify(&func, &error)); + + loop->false_edge->args[0] = alternate_condition; + assert(!vir_verify(&func, &error)); + loop->false_edge->args[0] = merge_args[0]; + assert(vir_verify(&func, &error)); + + use = values[1]->uses; + while (use && !use->user) + use = use->next; + assert(use); + old_operand = use->operand; + use->operand = 7; + assert(!vir_verify(&func, &error)); + use->operand = old_operand; + assert(vir_verify(&func, &error)); + + use = values[16]->uses; + while (use && !use->edge) + use = use->next; + assert(use); + old_edge = use->edge; + use->edge = NULL; + assert(!vir_verify(&func, &error)); + use->edge = old_edge; + assert(vir_verify(&func, &error)); + + use = values[16]->uses; + while (use && !use->return_block) + use = use->next; + assert(use); + old_return_block = use->return_block; + use->return_block = NULL; + assert(!vir_verify(&func, &error)); + use->return_block = old_return_block; + assert(vir_verify(&func, &error)); + + use = alternate_condition->uses; + while (use && !use->branch_block) + use = use->next; + assert(use); + old_branch_block = use->branch_block; + use->branch_block = NULL; + assert(!vir_verify(&func, &error)); + use->branch_block = old_branch_block; + assert(vir_verify(&func, &error)); + vir_function_release(&func); + } + return 0; +} From 83eee2cd4e077a79a5572a96bbcb7da38913ea65 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Tue, 6 Oct 2026 05:03:07 +0800 Subject: [PATCH 05/20] Compile all targets through native VIR Replace the legacy SSA middle end, its register allocator and its standalone SCCP pass. The parser builds VIR directly through a sealed-block frontend, VIR owns source values, typed memory effects, control flow, calls and target-independent optimization, and a shared machine lowering turns it into allocated machine IR that all four backends emit. That lowering handles ABI arguments and results, paired 32-bit registers, spill homes, coalescing, critical-edge copies, variadic saves, block layout and persistent callee-saved allocation. The driver accepts several input files as separate translation units, whose record definitions must agree in pointer constness and volatility, hash maps own their keys, and the lexer, preprocessor and parser share their repeated paths. A missing source file is reported as an error instead of aborting, an integer stored into a wider object is widened, a shift takes its type from the left operand alone, and a function-like macro named at the end of an expansion is invoked. Optimization runs by default, and --no-opt skips the optional VIR passes for debugging and for isolating an optimizer fault. Graded levels would ask a question with one useful answer: a lighter schedule generates the same code on nearly every program and compiles no faster, because the unoptimized graph leaves more values to lower. VIR regression suites, a corpus comparison with and without optimization and a host-compiler fuzzer guard the new pipeline. They run target binaries, the stage 2 compiler included, through the emulator like the rest of the suite, and the cross-target CI lanes get an hour, since their stage 2 suites run the self-hosted compiler under qemu for every case. --- .github/workflows/main.yml | 4 +- Makefile | 204 +- mk/arm.mk | 1 + mk/riscv.mk | 2 + src/arch-lower.c | 7 +- src/arm-codegen.c | 442 +- src/arm.c | 31 +- src/arm64-codegen.c | 287 +- src/arm64.c | 12 + src/defs.h | 396 +- src/globals.c | 2132 +++---- src/lexer.c | 761 +-- src/main.c | 408 +- src/opt-sccp.c | 98 - src/parser-call.c | 970 ++-- src/parser-const.c | 506 +- src/parser-decl.c | 936 ++-- src/parser-expr.c | 2580 +++------ src/parser-global.c | 817 ++- src/parser-init.c | 1281 ++--- src/parser-sizeof.c | 59 +- src/parser-stmt.c | 238 +- src/parser.c | 781 ++- src/peephole.c | 763 +-- src/preprocessor.c | 1261 ++--- src/reg-alloc.c | 4143 -------------- src/riscv-codegen.c | 822 +-- src/riscv.c | 19 +- src/ssa.c | 4913 ----------------- src/vir-frontend.c | 1160 ++++ src/vir-frontend.h | 25 + src/vir-lower.c | 1362 +++++ src/vir-lower.h | 20 + src/x64-codegen.c | 1702 +++--- src/x64.c | 143 +- tests/benchmark-time.sh | 95 + tests/driver.sh | 708 ++- tests/fuzz-gen.c | 427 ++ tests/fuzz.sh | 77 + tests/riscv-abi.sh | 52 +- tests/strength-reduce.c | 9 +- tests/vir-abstract-return-qualifiers.c | 25 + tests/vir-adjusted-array.c | 39 + tests/vir-arm-verifier.c | 43 + tests/vir-baseline-branch.c | 15 + tests/vir-baseline-pointer.c | 13 + tests/vir-baseline-workloads.txt | 6 + tests/vir-baseline.sh | 128 + tests/vir-bool-compound.c | 8 + tests/vir-call-return-internal.c | 26 + tests/vir-call-return.c | 12 + tests/vir-call-return.sh | 28 + tests/vir-callback-pointer-return-alias.c | 21 + tests/vir-callback-qualifier-compatibility.c | 44 + tests/vir-callback-return-metadata.c | 92 + tests/vir-const-abstract-return-pointer.c | 11 + .../vir-const-callback-pointer-return-alias.c | 8 + tests/vir-const-callback-return-pointee.c | 8 + tests/vir-const-callback-return-pointer.c | 8 + tests/vir-const-grouped-pointer-alias.c | 8 + tests/vir-const-hidden-return-alias.c | 14 + tests/vir-const-local-getter.c | 16 + tests/vir-corpus.sh | 109 + tests/vir-direct-arm64-argc-return.c | 4 + tests/vir-direct-arm64-argc.c | 4 + .../vir-direct-arm64-argv-char-intervening.c | 9 + tests/vir-direct-arm64-argv-char-load.c | 6 + ...vir-direct-arm64-argv-char-offset-direct.c | 7 + ...ir-direct-arm64-argv-char-offset-dynamic.c | 6 + ...irect-arm64-argv-char-offset-intervening.c | 9 + tests/vir-direct-arm64-argv-char-offset-two.c | 7 + tests/vir-direct-arm64-argv-char-offset.c | 5 + tests/vir-direct-arm64-argv-const-char.c | 5 + tests/vir-direct-arm64-argv-const-pointer.c | 5 + tests/vir-direct-arm64-argv-deref.c | 6 + ...vir-direct-arm64-argv-index-char-dynamic.c | 4 + ...direct-arm64-argv-index-char-intervening.c | 9 + tests/vir-direct-arm64-argv-index-char-load.c | 5 + ...ect-arm64-argv-index-char-offset-dynamic.c | 4 + ...arm64-argv-index-char-offset-intervening.c | 9 + ...-direct-arm64-argv-index-char-offset-one.c | 5 + ...irect-arm64-argv-index-char-offset-other.c | 5 + ...-direct-arm64-argv-index-char-offset-two.c | 5 + .../vir-direct-arm64-argv-index-char-other.c | 5 + tests/vir-direct-arm64-argv-index-compare.c | 4 + tests/vir-direct-arm64-argv-index-dynamic.c | 4 + tests/vir-direct-arm64-argv-index-equal.c | 4 + tests/vir-direct-arm64-argv-index-other.c | 4 + tests/vir-direct-arm64-argv-index-unused.c | 6 + tests/vir-direct-arm64-argv-index.c | 5 + tests/vir-direct-arm64-argv-pointer-compare.c | 4 + tests/vir-direct-arm64-argv-pointer-equal.c | 4 + tests/vir-direct-arm64-argv-signed-char.c | 5 + tests/vir-direct-arm64-argv-unsigned-char.c | 5 + tests/vir-direct-arm64-argv-volatile-char.c | 5 + .../vir-direct-arm64-argv-volatile-pointer.c | 5 + tests/vir-direct-arm64-argv-wrong-first.c | 5 + tests/vir-direct-arm64-argv-wrong-type.c | 5 + tests/vir-direct-arm64-argv.c | 5 + .../vir-direct-arm64-call-bool-arg-fallback.c | 17 + ...ir-direct-arm64-call-bool-arg-stack-wide.c | 27 + tests/vir-direct-arm64-call-bool-arg-stack.c | 17 + tests/vir-direct-arm64-call-bool-arg.c | 9 + tests/vir-direct-arm64-call-bool.c | 12 + tests/vir-direct-arm64-call-i32-argc.c | 9 + tests/vir-direct-arm64-call-i32-branch.c | 11 + tests/vir-direct-arm64-call-i32-eight.c | 9 + tests/vir-direct-arm64-call-i32-pointer.c | 11 + tests/vir-direct-arm64-call-i32-zero.c | 9 + tests/vir-direct-arm64-call-i32.c | 9 + tests/vir-direct-arm64-call-i64-eight.c | 20 + tests/vir-direct-arm64-call-i64.c | 9 + tests/vir-direct-arm64-call-narrow.c | 14 + tests/vir-direct-arm64-call-pointer.c | 12 + tests/vir-direct-arm64-call-void.c | 12 + tests/vir-direct-arm64-call.c | 29 + tests/vir-direct-arm64-cfg.c | 6 + tests/vir-direct-arm64-compare.c | 12 + tests/vir-direct-arm64-div.c | 16 + .../vir-direct-arm64-external-pointer-bound.c | 47 + ...irect-arm64-external-pointer-join-offset.c | 16 + .../vir-direct-arm64-external-pointer-join.c | 20 + ...-direct-arm64-external-pointer-loop-join.c | 21 + ...direct-arm64-external-pointer-mixed-join.c | 14 + ...t-arm64-external-pointer-mixed-loop-join.c | 16 + ...t-arm64-external-pointer-offset-fallback.c | 15 + ...-direct-arm64-external-void-pointer-wide.c | 15 + tests/vir-direct-arm64-global-pointer-deep.c | 9 + ...vir-direct-arm64-global-pointer-fallback.c | 10 + ...irect-arm64-global-pointer-narrow-signed.c | 10 + ...ect-arm64-global-pointer-narrow-unsigned.c | 10 + .../vir-direct-arm64-global-pointer-unknown.c | 8 + ...r-direct-arm64-global-pointer-wide-chain.c | 9 + tests/vir-direct-arm64-global-pointer-wide.c | 8 + tests/vir-direct-arm64-global-shift.c | 10 + tests/vir-direct-arm64-global.c | 13 + ...vir-direct-arm64-helper-address-fallback.c | 16 + ...irect-arm64-helper-branch-merge-fallback.c | 15 + ...ct-arm64-helper-branch-unsigned-fallback.c | 11 + tests/vir-direct-arm64-helper-branch.c | 11 + tests/vir-direct-arm64-helper-fallback.c | 9 + tests/vir-direct-arm64-helper-four-fallback.c | 9 + ...rect-arm64-helper-global-branch-external.c | 23 + ...rect-arm64-helper-global-branch-fallback.c | 18 + tests/vir-direct-arm64-helper-global-branch.c | 23 + ...lper-global-load-store-external-fallback.c | 18 + ...ir-direct-arm64-helper-global-load-store.c | 21 + ...vir-direct-arm64-helper-global-loop-deep.c | 28 + ...ir-direct-arm64-helper-global-loop-eight.c | 53 + ...direct-arm64-helper-global-loop-fallback.c | 68 + ...vir-direct-arm64-helper-global-loop-five.c | 38 + ...vir-direct-arm64-helper-global-loop-four.c | 33 + ...r-direct-arm64-helper-global-loop-nested.c | 23 + ...vir-direct-arm64-helper-global-loop-nine.c | 58 + ...ir-direct-arm64-helper-global-loop-seven.c | 48 + .../vir-direct-arm64-helper-global-loop-six.c | 43 + .../vir-direct-arm64-helper-global-loop-ten.c | 63 + tests/vir-direct-arm64-helper-global-loop.c | 18 + tests/vir-direct-arm64-helper-global-pair.c | 14 + ...ct-arm64-helper-global-volatile-fallback.c | 14 + .../vir-direct-arm64-helper-global-volatile.c | 12 + tests/vir-direct-arm64-helper-global.c | 12 + ...ct-arm64-helper-i64-local-mixed-fallback.c | 13 + ...r-direct-arm64-helper-i64-mixed-fallback.c | 9 + ...r-direct-arm64-helper-i64-three-fallback.c | 13 + tests/vir-direct-arm64-helper-i64-two.c | 12 + tests/vir-direct-arm64-helper-i64.c | 9 + tests/vir-direct-arm64-helper-leaf.c | 9 + tests/vir-direct-arm64-helper-loop-fallback.c | 16 + tests/vir-direct-arm64-helper-loop.c | 13 + tests/vir-direct-arm64-helper-merge.c | 9 + tests/vir-direct-arm64-helper-three.c | 9 + tests/vir-direct-arm64-helper-two.c | 9 + tests/vir-direct-arm64-i64-add.c | 9 + tests/vir-direct-arm64-i64-bitwise.c | 15 + tests/vir-direct-arm64-i64-cast.c | 13 + tests/vir-direct-arm64-i64-compare.c | 12 + tests/vir-direct-arm64-i64-const-movn.c | 6 + tests/vir-direct-arm64-i64-const-sparse.c | 6 + tests/vir-direct-arm64-i64-const-tie.c | 6 + tests/vir-direct-arm64-i64-const.c | 6 + tests/vir-direct-arm64-i64-cross.c | 13 + tests/vir-direct-arm64-i64-div.c | 10 + tests/vir-direct-arm64-i64-immediate.c | 16 + tests/vir-direct-arm64-i64-memory-high.c | 11 + tests/vir-direct-arm64-i64-memory.c | 8 + tests/vir-direct-arm64-i64-mul.c | 10 + tests/vir-direct-arm64-i64-shift.c | 15 + tests/vir-direct-arm64-i64-sub.c | 9 + tests/vir-direct-arm64-i64-unary.c | 11 + tests/vir-direct-arm64-join-pair.c | 14 + tests/vir-direct-arm64-join.c | 10 + tests/vir-direct-arm64-local-pointer-chain.c | 9 + ...vir-direct-arm64-local-pointer-collapsed.c | 16 + tests/vir-direct-arm64-local-pointer-cyclic.c | 10 + tests/vir-direct-arm64-local-pointer-deep.c | 10 + tests/vir-direct-arm64-local-pointer-level4.c | 12 + ...vir-direct-arm64-local-pointer-overdepth.c | 15 + ...ir-direct-arm64-local-pointer-reassigned.c | 14 + .../vir-direct-arm64-local-pointer-volatile.c | 10 + tests/vir-direct-arm64-local-pointer-wide.c | 9 + tests/vir-direct-arm64-logical.c | 4 + tests/vir-direct-arm64-loop.c | 15 + tests/vir-direct-arm64-narrow.c | 11 + .../vir-direct-arm64-pointer-join-duplicate.c | 12 + tests/vir-direct-arm64-pointer-join.c | 12 + tests/vir-direct-arm64-pressure.c | 18 + tests/vir-direct-arm64-ptradd-cfg.c | 11 + tests/vir-direct-arm64-ptradd-dynamic.c | 12 + tests/vir-direct-arm64-ptradd-large.c | 8 + tests/vir-direct-arm64-ptradd.c | 8 + tests/vir-direct-arm64-shared-dag.c | 22 + tests/vir-direct-arm64-store.c | 8 + tests/vir-direct-arm64-ult.c | 6 + tests/vir-direct-arm64-volatile.c | 14 + tests/vir-direct-arm64.c | 4 + tests/vir-direct-arm64.sh | 1275 +++++ tests/vir-direct-branch-initialized-root.c | 13 + tests/vir-direct-branch-volatile-root.c | 10 + tests/vir-direct-common.sh | 50 + tests/vir-direct-external-pointer-origin.c | 14 + ...vir-direct-external-typed-pointer-offset.c | 12 + tests/vir-direct-external-typed-pointer.c | 9 + tests/vir-direct-global-ptradd-dynamic.c | 10 + tests/vir-direct-global-ptradd-unsigned.c | 10 + tests/vir-direct-global-ptradd.c | 9 + tests/vir-direct-narrow-global.c | 17 + tests/vir-direct-x64-branch-local-root-call.c | 15 + tests/vir-direct-x64-call-aggregate.c | 16 + tests/vir-direct-x64-call-args.c | 9 + tests/vir-direct-x64-call-argv.c | 12 + tests/vir-direct-x64-call-bool-arg-fallback.c | 9 + tests/vir-direct-x64-call-bool-arg-stack.c | 13 + tests/vir-direct-x64-call-bool-arg.c | 9 + tests/vir-direct-x64-call-direct-args.c | 9 + ...-direct-x64-call-direct-global-reference.c | 16 + ...-direct-x64-call-direct-legacy-reference.c | 14 + tests/vir-direct-x64-call-direct-pointer.c | 9 + tests/vir-direct-x64-call-direct-root.c | 12 + ...vir-direct-x64-call-direct-stack-pointer.c | 9 + tests/vir-direct-x64-call-direct-stack.c | 9 + tests/vir-direct-x64-call-eight.c | 9 + tests/vir-direct-x64-call-i64-mixed.c | 12 + tests/vir-direct-x64-call-i64-stack.c | 15 + tests/vir-direct-x64-call-i64.c | 10 + tests/vir-direct-x64-call-narrow.c | 30 + tests/vir-direct-x64-call-pointer-high.c | 9 + tests/vir-direct-x64-call-pointer-result.c | 12 + tests/vir-direct-x64-call-pointer.c | 9 + tests/vir-direct-x64-call-seven.c | 9 + tests/vir-direct-x64-call-six.c | 9 + tests/vir-direct-x64-call-stack-pointer.c | 9 + tests/vir-direct-x64-call-void.c | 13 + tests/vir-direct-x64-call.c | 14 + tests/vir-direct-x64-casts.c | 9 + tests/vir-direct-x64-cfg.c | 6 + tests/vir-direct-x64-div-i64.c | 15 + tests/vir-direct-x64-div.c | 24 + tests/vir-direct-x64-edge-update-computed.c | 17 + tests/vir-direct-x64-global-load.c | 7 + tests/vir-direct-x64-global-pointer-call.c | 17 + tests/vir-direct-x64-global-pointer-slot.c | 11 + tests/vir-direct-x64-i64-memory-aggregate.c | 12 + tests/vir-direct-x64-i64-memory-alias.c | 12 + tests/vir-direct-x64-i64-memory.c | 8 + tests/vir-direct-x64-i64-volatile.c | 8 + tests/vir-direct-x64-join.c | 10 + tests/vir-direct-x64-load-i32.c | 6 + tests/vir-direct-x64-load.c | 6 + tests/vir-direct-x64-local-pointer-chain.c | 9 + tests/vir-direct-x64-local-pointer-deep.c | 10 + tests/vir-direct-x64-loop.c | 15 + tests/vir-direct-x64-merged-narrow-effect.c | 18 + ...rect-x64-merged-pointer-effect-void-call.c | 21 + tests/vir-direct-x64-merged-pointer-effect.c | 14 + tests/vir-direct-x64-merged-pointer-load.c | 16 + ...irect-x64-merged-volatile-pointer-effect.c | 14 + ...r-direct-x64-narrow-array-index-unsigned.c | 11 + tests/vir-direct-x64-narrow-array-index.c | 11 + tests/vir-direct-x64-narrow-array.c | 9 + tests/vir-direct-x64-narrow-memory.c | 15 + tests/vir-direct-x64-narrow-volatile-local.c | 9 + tests/vir-direct-x64-narrow-volatile.c | 12 + tests/vir-direct-x64-ops.c | 8 + tests/vir-direct-x64-param-derived-join.c | 12 + tests/vir-direct-x64-param-loop.c | 11 + tests/vir-direct-x64-param-scratch.c | 9 + tests/vir-direct-x64-param-shift.c | 9 + tests/vir-direct-x64-pointer-int-cast.c | 10 + tests/vir-direct-x64-pointer-int-return.c | 7 + tests/vir-direct-x64-pointer-join.c | 12 + tests/vir-direct-x64-pointer-store.c | 7 + tests/vir-direct-x64-pointer.c | 12 + tests/vir-direct-x64-ptradd-narrow-signed.c | 9 + tests/vir-direct-x64-ptradd-narrow.c | 8 + tests/vir-direct-x64-ptradd-unsigned.c | 6 + tests/vir-direct-x64-ptradd.c | 7 + tests/vir-direct-x64-ptrsub-const.c | 9 + tests/vir-direct-x64-ptrsub-unsigned-const.c | 10 + tests/vir-direct-x64-ptrsub-unsigned.c | 10 + tests/vir-direct-x64-ptrsub.c | 9 + tests/vir-direct-x64-r11-call.c | 20 + tests/vir-direct-x64-r11-leaf.c | 6 + tests/vir-direct-x64-selected-root-alias.c | 13 + tests/vir-direct-x64-stack-array.c | 7 + tests/vir-direct-x64-startup-globals.c | 7 + tests/vir-direct-x64-store-branch.c | 8 + tests/vir-direct-x64-store-init.c | 6 + tests/vir-direct-x64-store-read-arith.c | 7 + tests/vir-direct-x64-store-read-branch.c | 9 + tests/vir-direct-x64-store-read-pair.c | 9 + tests/vir-direct-x64-store-stale-branch.c | 9 + tests/vir-direct-x64-store-stale-read.c | 7 + tests/vir-direct-x64-store.c | 7 + tests/vir-direct-x64-truthy-join.c | 12 + tests/vir-direct-x64-truthy.c | 6 + tests/vir-direct-x64-udiv.c | 8 + tests/vir-direct-x64-unknown-pointer-alias.c | 15 + tests/vir-direct-x64-volatile-discard.c | 11 + tests/vir-direct-x64-volatile.c | 9 + tests/vir-direct-x64-wide-cast.c | 6 + tests/vir-direct-x64-wide-return.c | 4 + tests/vir-direct-x64.c | 7 + tests/vir-direct-x64.sh | 468 ++ tests/vir-file-seek.c | 22 + tests/vir-frontend-aggregate-call-result.c | 21 + tests/vir-frontend-bitwise.c | 20 + tests/vir-frontend-bool-call-arg.c | 34 + tests/vir-frontend-calls.c | 14 + tests/vir-frontend-casts.c | 33 + tests/vir-frontend-cfg-fallback.c | 11 + tests/vir-frontend-comparisons.c | 51 + tests/vir-frontend-cse.c | 12 + tests/vir-frontend-dce.c | 17 + tests/vir-frontend-deferred-bool.c | 17 + tests/vir-frontend-deferred-call-arg.c | 27 + tests/vir-frontend-deferred-order.c | 65 + tests/vir-frontend-deferred-store.c | 16 + tests/vir-frontend-do-exit-value.c | 16 + tests/vir-frontend-do-loop.c | 14 + tests/vir-frontend-fallback.c | 8 + tests/vir-frontend-for-loop.c | 13 + tests/vir-frontend-global-array.c | 22 + tests/vir-frontend-global-read.c | 6 + tests/vir-frontend-gvn.c | 16 + tests/vir-frontend-indirect-call.c | 155 + tests/vir-frontend-join-arith.c | 15 + tests/vir-frontend-join-branch.c | 20 + tests/vir-frontend-join-trivial.c | 15 + tests/vir-frontend-join-uninitialized.c | 13 + tests/vir-frontend-join.c | 15 + tests/vir-frontend-large-cfg.c | 66 + tests/vir-frontend-licm.c | 13 + tests/vir-frontend-load.c | 11 + tests/vir-frontend-local-array.c | 37 + tests/vir-frontend-loop-fallback.c | 13 + tests/vir-frontend-loop-index.c | 23 + tests/vir-frontend-merged-effects.c | 19 + tests/vir-frontend-multi-latch.c | 19 + tests/vir-frontend-narrow-call.c | 25 + tests/vir-frontend-nested-loop.c | 21 + tests/vir-frontend-param.c | 9 + tests/vir-frontend-pointer-call-result.c | 19 + tests/vir-frontend-pointer-call.c | 19 + tests/vir-frontend-pointer-equality.c | 11 + tests/vir-frontend-pointer-join.c | 12 + tests/vir-frontend-pointer-load.c | 12 + tests/vir-frontend-pointer-store.c | 13 + tests/vir-frontend-pointer-truth.c | 33 + tests/vir-frontend-policy.sh | 82 + tests/vir-frontend-riscv-variadic.c | 38 + tests/vir-frontend-roots-init.c | 12 + tests/vir-frontend-roots.c | 23 + tests/vir-frontend-sccp-widths.c | 28 + tests/vir-frontend-sccp.c | 14 + tests/vir-frontend-shadow.c | 4 + tests/vir-frontend-store.c | 12 + tests/vir-frontend-string-literal.c | 27 + tests/vir-frontend-swap.c | 22 + tests/vir-frontend-ternary-array.c | 23 + tests/vir-frontend-void-call-tail.c | 24 + tests/vir-frontend-void.c | 46 + tests/vir-frontend-volatile-mixed.c | 14 + tests/vir-frontend-volatile.c | 12 + tests/vir-frontend-workloads.txt | 37 + tests/vir-frontend.sh | 76 + tests/vir-global-scratch.c | 12 + tests/vir-grouped-pointer-alias.c | 15 + tests/vir-hidden-const-return-compatible.c | 19 + tests/vir-large-cfg.sh | 33 + tests/vir-licm-profile.c | 801 +++ tests/vir-lower-smoke.c | 108 + tests/vir-lower.c | 889 +++ tests/vir-lower.sh | 22 + tests/vir-machine-affinity.c | 43 + tests/vir-machine-live.c | 26 + tests/vir-multi-input-main.c | 49 + tests/vir-multi-input-root.c | 37 + tests/vir-multi-input.sh | 451 ++ tests/vir-native-regressions.sh | 19 + tests/vir-operand-signedness.c | 21 + tests/vir-pair-scratch.c | 18 + tests/vir-pointer-comparison.c | 11 + tests/vir-pointer-offset.c | 26 + tests/vir-pointer-offset.sh | 10 + tests/vir-qualifiers.sh | 68 + tests/vir-shift-width.c | 54 + tests/vir-ssa-baseline-workloads.txt | 6 + tests/vir-ssa-baseline.sh | 136 + tests/vir-ssa-selfhost-workloads.txt | 7 + tests/vir-stage0.c | 348 ++ tests/vir-stats.sh | 47 + tests/vir-strength.c | 38 + tests/vir-strength.sh | 30 + tests/vir-subtract-alias.c | 29 + tests/vir-typed-arithmetic.c | 58 + tests/vir-void-callback-array.c | 12 + tests/vir-volatile-arithmetic.c | 13 + tests/vir-volatile-callback-pointer-return.c | 40 + tests/vir-volatile-callback-return-pointer.c | 15 + tests/vir-volatile-callback.c | 20 + tests/vir-volatile-comma-alias.c | 13 + tests/vir-volatile-local-getter.c | 20 + tests/vir-volatile-local.c | 19 + tests/vir-volatile-member.c | 26 + tests/vir-volatile-pointer-member.c | 19 + tests/vir-volatile-storage.c | 23 + tests/x64-abi.sh | 33 +- tests/x64-startup-globals.sh | 15 + 429 files changed, 22110 insertions(+), 19415 deletions(-) delete mode 100644 src/opt-sccp.c delete mode 100644 src/reg-alloc.c delete mode 100644 src/ssa.c create mode 100644 src/vir-frontend.c create mode 100644 src/vir-frontend.h create mode 100644 src/vir-lower.c create mode 100644 src/vir-lower.h create mode 100644 tests/benchmark-time.sh create mode 100644 tests/fuzz-gen.c create mode 100755 tests/fuzz.sh create mode 100644 tests/vir-abstract-return-qualifiers.c create mode 100644 tests/vir-adjusted-array.c create mode 100644 tests/vir-arm-verifier.c create mode 100644 tests/vir-baseline-branch.c create mode 100644 tests/vir-baseline-pointer.c create mode 100644 tests/vir-baseline-workloads.txt create mode 100755 tests/vir-baseline.sh create mode 100644 tests/vir-bool-compound.c create mode 100644 tests/vir-call-return-internal.c create mode 100644 tests/vir-call-return.c create mode 100755 tests/vir-call-return.sh create mode 100644 tests/vir-callback-pointer-return-alias.c create mode 100644 tests/vir-callback-qualifier-compatibility.c create mode 100644 tests/vir-callback-return-metadata.c create mode 100644 tests/vir-const-abstract-return-pointer.c create mode 100644 tests/vir-const-callback-pointer-return-alias.c create mode 100644 tests/vir-const-callback-return-pointee.c create mode 100644 tests/vir-const-callback-return-pointer.c create mode 100644 tests/vir-const-grouped-pointer-alias.c create mode 100644 tests/vir-const-hidden-return-alias.c create mode 100644 tests/vir-const-local-getter.c create mode 100755 tests/vir-corpus.sh create mode 100644 tests/vir-direct-arm64-argc-return.c create mode 100644 tests/vir-direct-arm64-argc.c create mode 100644 tests/vir-direct-arm64-argv-char-intervening.c create mode 100644 tests/vir-direct-arm64-argv-char-load.c create mode 100644 tests/vir-direct-arm64-argv-char-offset-direct.c create mode 100644 tests/vir-direct-arm64-argv-char-offset-dynamic.c create mode 100644 tests/vir-direct-arm64-argv-char-offset-intervening.c create mode 100644 tests/vir-direct-arm64-argv-char-offset-two.c create mode 100644 tests/vir-direct-arm64-argv-char-offset.c create mode 100644 tests/vir-direct-arm64-argv-const-char.c create mode 100644 tests/vir-direct-arm64-argv-const-pointer.c create mode 100644 tests/vir-direct-arm64-argv-deref.c create mode 100644 tests/vir-direct-arm64-argv-index-char-dynamic.c create mode 100644 tests/vir-direct-arm64-argv-index-char-intervening.c create mode 100644 tests/vir-direct-arm64-argv-index-char-load.c create mode 100644 tests/vir-direct-arm64-argv-index-char-offset-dynamic.c create mode 100644 tests/vir-direct-arm64-argv-index-char-offset-intervening.c create mode 100644 tests/vir-direct-arm64-argv-index-char-offset-one.c create mode 100644 tests/vir-direct-arm64-argv-index-char-offset-other.c create mode 100644 tests/vir-direct-arm64-argv-index-char-offset-two.c create mode 100644 tests/vir-direct-arm64-argv-index-char-other.c create mode 100644 tests/vir-direct-arm64-argv-index-compare.c create mode 100644 tests/vir-direct-arm64-argv-index-dynamic.c create mode 100644 tests/vir-direct-arm64-argv-index-equal.c create mode 100644 tests/vir-direct-arm64-argv-index-other.c create mode 100644 tests/vir-direct-arm64-argv-index-unused.c create mode 100644 tests/vir-direct-arm64-argv-index.c create mode 100644 tests/vir-direct-arm64-argv-pointer-compare.c create mode 100644 tests/vir-direct-arm64-argv-pointer-equal.c create mode 100644 tests/vir-direct-arm64-argv-signed-char.c create mode 100644 tests/vir-direct-arm64-argv-unsigned-char.c create mode 100644 tests/vir-direct-arm64-argv-volatile-char.c create mode 100644 tests/vir-direct-arm64-argv-volatile-pointer.c create mode 100644 tests/vir-direct-arm64-argv-wrong-first.c create mode 100644 tests/vir-direct-arm64-argv-wrong-type.c create mode 100644 tests/vir-direct-arm64-argv.c create mode 100644 tests/vir-direct-arm64-call-bool-arg-fallback.c create mode 100644 tests/vir-direct-arm64-call-bool-arg-stack-wide.c create mode 100644 tests/vir-direct-arm64-call-bool-arg-stack.c create mode 100644 tests/vir-direct-arm64-call-bool-arg.c create mode 100644 tests/vir-direct-arm64-call-bool.c create mode 100644 tests/vir-direct-arm64-call-i32-argc.c create mode 100644 tests/vir-direct-arm64-call-i32-branch.c create mode 100644 tests/vir-direct-arm64-call-i32-eight.c create mode 100644 tests/vir-direct-arm64-call-i32-pointer.c create mode 100644 tests/vir-direct-arm64-call-i32-zero.c create mode 100644 tests/vir-direct-arm64-call-i32.c create mode 100644 tests/vir-direct-arm64-call-i64-eight.c create mode 100644 tests/vir-direct-arm64-call-i64.c create mode 100644 tests/vir-direct-arm64-call-narrow.c create mode 100644 tests/vir-direct-arm64-call-pointer.c create mode 100644 tests/vir-direct-arm64-call-void.c create mode 100644 tests/vir-direct-arm64-call.c create mode 100644 tests/vir-direct-arm64-cfg.c create mode 100644 tests/vir-direct-arm64-compare.c create mode 100644 tests/vir-direct-arm64-div.c create mode 100644 tests/vir-direct-arm64-external-pointer-bound.c create mode 100644 tests/vir-direct-arm64-external-pointer-join-offset.c create mode 100644 tests/vir-direct-arm64-external-pointer-join.c create mode 100644 tests/vir-direct-arm64-external-pointer-loop-join.c create mode 100644 tests/vir-direct-arm64-external-pointer-mixed-join.c create mode 100644 tests/vir-direct-arm64-external-pointer-mixed-loop-join.c create mode 100644 tests/vir-direct-arm64-external-pointer-offset-fallback.c create mode 100644 tests/vir-direct-arm64-external-void-pointer-wide.c create mode 100644 tests/vir-direct-arm64-global-pointer-deep.c create mode 100644 tests/vir-direct-arm64-global-pointer-fallback.c create mode 100644 tests/vir-direct-arm64-global-pointer-narrow-signed.c create mode 100644 tests/vir-direct-arm64-global-pointer-narrow-unsigned.c create mode 100644 tests/vir-direct-arm64-global-pointer-unknown.c create mode 100644 tests/vir-direct-arm64-global-pointer-wide-chain.c create mode 100644 tests/vir-direct-arm64-global-pointer-wide.c create mode 100644 tests/vir-direct-arm64-global-shift.c create mode 100644 tests/vir-direct-arm64-global.c create mode 100644 tests/vir-direct-arm64-helper-address-fallback.c create mode 100644 tests/vir-direct-arm64-helper-branch-merge-fallback.c create mode 100644 tests/vir-direct-arm64-helper-branch-unsigned-fallback.c create mode 100644 tests/vir-direct-arm64-helper-branch.c create mode 100644 tests/vir-direct-arm64-helper-fallback.c create mode 100644 tests/vir-direct-arm64-helper-four-fallback.c create mode 100644 tests/vir-direct-arm64-helper-global-branch-external.c create mode 100644 tests/vir-direct-arm64-helper-global-branch-fallback.c create mode 100644 tests/vir-direct-arm64-helper-global-branch.c create mode 100644 tests/vir-direct-arm64-helper-global-load-store-external-fallback.c create mode 100644 tests/vir-direct-arm64-helper-global-load-store.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-deep.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-eight.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-fallback.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-five.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-four.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-nested.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-nine.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-seven.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-six.c create mode 100644 tests/vir-direct-arm64-helper-global-loop-ten.c create mode 100644 tests/vir-direct-arm64-helper-global-loop.c create mode 100644 tests/vir-direct-arm64-helper-global-pair.c create mode 100644 tests/vir-direct-arm64-helper-global-volatile-fallback.c create mode 100644 tests/vir-direct-arm64-helper-global-volatile.c create mode 100644 tests/vir-direct-arm64-helper-global.c create mode 100644 tests/vir-direct-arm64-helper-i64-local-mixed-fallback.c create mode 100644 tests/vir-direct-arm64-helper-i64-mixed-fallback.c create mode 100644 tests/vir-direct-arm64-helper-i64-three-fallback.c create mode 100644 tests/vir-direct-arm64-helper-i64-two.c create mode 100644 tests/vir-direct-arm64-helper-i64.c create mode 100644 tests/vir-direct-arm64-helper-leaf.c create mode 100644 tests/vir-direct-arm64-helper-loop-fallback.c create mode 100644 tests/vir-direct-arm64-helper-loop.c create mode 100644 tests/vir-direct-arm64-helper-merge.c create mode 100644 tests/vir-direct-arm64-helper-three.c create mode 100644 tests/vir-direct-arm64-helper-two.c create mode 100644 tests/vir-direct-arm64-i64-add.c create mode 100644 tests/vir-direct-arm64-i64-bitwise.c create mode 100644 tests/vir-direct-arm64-i64-cast.c create mode 100644 tests/vir-direct-arm64-i64-compare.c create mode 100644 tests/vir-direct-arm64-i64-const-movn.c create mode 100644 tests/vir-direct-arm64-i64-const-sparse.c create mode 100644 tests/vir-direct-arm64-i64-const-tie.c create mode 100644 tests/vir-direct-arm64-i64-const.c create mode 100644 tests/vir-direct-arm64-i64-cross.c create mode 100644 tests/vir-direct-arm64-i64-div.c create mode 100644 tests/vir-direct-arm64-i64-immediate.c create mode 100644 tests/vir-direct-arm64-i64-memory-high.c create mode 100644 tests/vir-direct-arm64-i64-memory.c create mode 100644 tests/vir-direct-arm64-i64-mul.c create mode 100644 tests/vir-direct-arm64-i64-shift.c create mode 100644 tests/vir-direct-arm64-i64-sub.c create mode 100644 tests/vir-direct-arm64-i64-unary.c create mode 100644 tests/vir-direct-arm64-join-pair.c create mode 100644 tests/vir-direct-arm64-join.c create mode 100644 tests/vir-direct-arm64-local-pointer-chain.c create mode 100644 tests/vir-direct-arm64-local-pointer-collapsed.c create mode 100644 tests/vir-direct-arm64-local-pointer-cyclic.c create mode 100644 tests/vir-direct-arm64-local-pointer-deep.c create mode 100644 tests/vir-direct-arm64-local-pointer-level4.c create mode 100644 tests/vir-direct-arm64-local-pointer-overdepth.c create mode 100644 tests/vir-direct-arm64-local-pointer-reassigned.c create mode 100644 tests/vir-direct-arm64-local-pointer-volatile.c create mode 100644 tests/vir-direct-arm64-local-pointer-wide.c create mode 100644 tests/vir-direct-arm64-logical.c create mode 100644 tests/vir-direct-arm64-loop.c create mode 100644 tests/vir-direct-arm64-narrow.c create mode 100644 tests/vir-direct-arm64-pointer-join-duplicate.c create mode 100644 tests/vir-direct-arm64-pointer-join.c create mode 100644 tests/vir-direct-arm64-pressure.c create mode 100644 tests/vir-direct-arm64-ptradd-cfg.c create mode 100644 tests/vir-direct-arm64-ptradd-dynamic.c create mode 100644 tests/vir-direct-arm64-ptradd-large.c create mode 100644 tests/vir-direct-arm64-ptradd.c create mode 100644 tests/vir-direct-arm64-shared-dag.c create mode 100644 tests/vir-direct-arm64-store.c create mode 100644 tests/vir-direct-arm64-ult.c create mode 100644 tests/vir-direct-arm64-volatile.c create mode 100644 tests/vir-direct-arm64.c create mode 100755 tests/vir-direct-arm64.sh create mode 100644 tests/vir-direct-branch-initialized-root.c create mode 100644 tests/vir-direct-branch-volatile-root.c create mode 100644 tests/vir-direct-common.sh create mode 100644 tests/vir-direct-external-pointer-origin.c create mode 100644 tests/vir-direct-external-typed-pointer-offset.c create mode 100644 tests/vir-direct-external-typed-pointer.c create mode 100644 tests/vir-direct-global-ptradd-dynamic.c create mode 100644 tests/vir-direct-global-ptradd-unsigned.c create mode 100644 tests/vir-direct-global-ptradd.c create mode 100644 tests/vir-direct-narrow-global.c create mode 100644 tests/vir-direct-x64-branch-local-root-call.c create mode 100644 tests/vir-direct-x64-call-aggregate.c create mode 100644 tests/vir-direct-x64-call-args.c create mode 100644 tests/vir-direct-x64-call-argv.c create mode 100644 tests/vir-direct-x64-call-bool-arg-fallback.c create mode 100644 tests/vir-direct-x64-call-bool-arg-stack.c create mode 100644 tests/vir-direct-x64-call-bool-arg.c create mode 100644 tests/vir-direct-x64-call-direct-args.c create mode 100644 tests/vir-direct-x64-call-direct-global-reference.c create mode 100644 tests/vir-direct-x64-call-direct-legacy-reference.c create mode 100644 tests/vir-direct-x64-call-direct-pointer.c create mode 100644 tests/vir-direct-x64-call-direct-root.c create mode 100644 tests/vir-direct-x64-call-direct-stack-pointer.c create mode 100644 tests/vir-direct-x64-call-direct-stack.c create mode 100644 tests/vir-direct-x64-call-eight.c create mode 100644 tests/vir-direct-x64-call-i64-mixed.c create mode 100644 tests/vir-direct-x64-call-i64-stack.c create mode 100644 tests/vir-direct-x64-call-i64.c create mode 100644 tests/vir-direct-x64-call-narrow.c create mode 100644 tests/vir-direct-x64-call-pointer-high.c create mode 100644 tests/vir-direct-x64-call-pointer-result.c create mode 100644 tests/vir-direct-x64-call-pointer.c create mode 100644 tests/vir-direct-x64-call-seven.c create mode 100644 tests/vir-direct-x64-call-six.c create mode 100644 tests/vir-direct-x64-call-stack-pointer.c create mode 100644 tests/vir-direct-x64-call-void.c create mode 100644 tests/vir-direct-x64-call.c create mode 100644 tests/vir-direct-x64-casts.c create mode 100644 tests/vir-direct-x64-cfg.c create mode 100644 tests/vir-direct-x64-div-i64.c create mode 100644 tests/vir-direct-x64-div.c create mode 100644 tests/vir-direct-x64-edge-update-computed.c create mode 100644 tests/vir-direct-x64-global-load.c create mode 100644 tests/vir-direct-x64-global-pointer-call.c create mode 100644 tests/vir-direct-x64-global-pointer-slot.c create mode 100644 tests/vir-direct-x64-i64-memory-aggregate.c create mode 100644 tests/vir-direct-x64-i64-memory-alias.c create mode 100644 tests/vir-direct-x64-i64-memory.c create mode 100644 tests/vir-direct-x64-i64-volatile.c create mode 100644 tests/vir-direct-x64-join.c create mode 100644 tests/vir-direct-x64-load-i32.c create mode 100644 tests/vir-direct-x64-load.c create mode 100644 tests/vir-direct-x64-local-pointer-chain.c create mode 100644 tests/vir-direct-x64-local-pointer-deep.c create mode 100644 tests/vir-direct-x64-loop.c create mode 100644 tests/vir-direct-x64-merged-narrow-effect.c create mode 100644 tests/vir-direct-x64-merged-pointer-effect-void-call.c create mode 100644 tests/vir-direct-x64-merged-pointer-effect.c create mode 100644 tests/vir-direct-x64-merged-pointer-load.c create mode 100644 tests/vir-direct-x64-merged-volatile-pointer-effect.c create mode 100644 tests/vir-direct-x64-narrow-array-index-unsigned.c create mode 100644 tests/vir-direct-x64-narrow-array-index.c create mode 100644 tests/vir-direct-x64-narrow-array.c create mode 100644 tests/vir-direct-x64-narrow-memory.c create mode 100644 tests/vir-direct-x64-narrow-volatile-local.c create mode 100644 tests/vir-direct-x64-narrow-volatile.c create mode 100644 tests/vir-direct-x64-ops.c create mode 100644 tests/vir-direct-x64-param-derived-join.c create mode 100644 tests/vir-direct-x64-param-loop.c create mode 100644 tests/vir-direct-x64-param-scratch.c create mode 100644 tests/vir-direct-x64-param-shift.c create mode 100644 tests/vir-direct-x64-pointer-int-cast.c create mode 100644 tests/vir-direct-x64-pointer-int-return.c create mode 100644 tests/vir-direct-x64-pointer-join.c create mode 100644 tests/vir-direct-x64-pointer-store.c create mode 100644 tests/vir-direct-x64-pointer.c create mode 100644 tests/vir-direct-x64-ptradd-narrow-signed.c create mode 100644 tests/vir-direct-x64-ptradd-narrow.c create mode 100644 tests/vir-direct-x64-ptradd-unsigned.c create mode 100644 tests/vir-direct-x64-ptradd.c create mode 100644 tests/vir-direct-x64-ptrsub-const.c create mode 100644 tests/vir-direct-x64-ptrsub-unsigned-const.c create mode 100644 tests/vir-direct-x64-ptrsub-unsigned.c create mode 100644 tests/vir-direct-x64-ptrsub.c create mode 100644 tests/vir-direct-x64-r11-call.c create mode 100644 tests/vir-direct-x64-r11-leaf.c create mode 100644 tests/vir-direct-x64-selected-root-alias.c create mode 100644 tests/vir-direct-x64-stack-array.c create mode 100644 tests/vir-direct-x64-startup-globals.c create mode 100644 tests/vir-direct-x64-store-branch.c create mode 100644 tests/vir-direct-x64-store-init.c create mode 100644 tests/vir-direct-x64-store-read-arith.c create mode 100644 tests/vir-direct-x64-store-read-branch.c create mode 100644 tests/vir-direct-x64-store-read-pair.c create mode 100644 tests/vir-direct-x64-store-stale-branch.c create mode 100644 tests/vir-direct-x64-store-stale-read.c create mode 100644 tests/vir-direct-x64-store.c create mode 100644 tests/vir-direct-x64-truthy-join.c create mode 100644 tests/vir-direct-x64-truthy.c create mode 100644 tests/vir-direct-x64-udiv.c create mode 100644 tests/vir-direct-x64-unknown-pointer-alias.c create mode 100644 tests/vir-direct-x64-volatile-discard.c create mode 100644 tests/vir-direct-x64-volatile.c create mode 100644 tests/vir-direct-x64-wide-cast.c create mode 100644 tests/vir-direct-x64-wide-return.c create mode 100644 tests/vir-direct-x64.c create mode 100755 tests/vir-direct-x64.sh create mode 100644 tests/vir-file-seek.c create mode 100644 tests/vir-frontend-aggregate-call-result.c create mode 100644 tests/vir-frontend-bitwise.c create mode 100644 tests/vir-frontend-bool-call-arg.c create mode 100644 tests/vir-frontend-calls.c create mode 100644 tests/vir-frontend-casts.c create mode 100644 tests/vir-frontend-cfg-fallback.c create mode 100644 tests/vir-frontend-comparisons.c create mode 100644 tests/vir-frontend-cse.c create mode 100644 tests/vir-frontend-dce.c create mode 100644 tests/vir-frontend-deferred-bool.c create mode 100644 tests/vir-frontend-deferred-call-arg.c create mode 100644 tests/vir-frontend-deferred-order.c create mode 100644 tests/vir-frontend-deferred-store.c create mode 100644 tests/vir-frontend-do-exit-value.c create mode 100644 tests/vir-frontend-do-loop.c create mode 100644 tests/vir-frontend-fallback.c create mode 100644 tests/vir-frontend-for-loop.c create mode 100644 tests/vir-frontend-global-array.c create mode 100644 tests/vir-frontend-global-read.c create mode 100644 tests/vir-frontend-gvn.c create mode 100644 tests/vir-frontend-indirect-call.c create mode 100644 tests/vir-frontend-join-arith.c create mode 100644 tests/vir-frontend-join-branch.c create mode 100644 tests/vir-frontend-join-trivial.c create mode 100644 tests/vir-frontend-join-uninitialized.c create mode 100644 tests/vir-frontend-join.c create mode 100644 tests/vir-frontend-large-cfg.c create mode 100644 tests/vir-frontend-licm.c create mode 100644 tests/vir-frontend-load.c create mode 100644 tests/vir-frontend-local-array.c create mode 100644 tests/vir-frontend-loop-fallback.c create mode 100644 tests/vir-frontend-loop-index.c create mode 100644 tests/vir-frontend-merged-effects.c create mode 100644 tests/vir-frontend-multi-latch.c create mode 100644 tests/vir-frontend-narrow-call.c create mode 100644 tests/vir-frontend-nested-loop.c create mode 100644 tests/vir-frontend-param.c create mode 100644 tests/vir-frontend-pointer-call-result.c create mode 100644 tests/vir-frontend-pointer-call.c create mode 100644 tests/vir-frontend-pointer-equality.c create mode 100644 tests/vir-frontend-pointer-join.c create mode 100644 tests/vir-frontend-pointer-load.c create mode 100644 tests/vir-frontend-pointer-store.c create mode 100644 tests/vir-frontend-pointer-truth.c create mode 100644 tests/vir-frontend-policy.sh create mode 100644 tests/vir-frontend-riscv-variadic.c create mode 100644 tests/vir-frontend-roots-init.c create mode 100644 tests/vir-frontend-roots.c create mode 100644 tests/vir-frontend-sccp-widths.c create mode 100644 tests/vir-frontend-sccp.c create mode 100644 tests/vir-frontend-shadow.c create mode 100644 tests/vir-frontend-store.c create mode 100644 tests/vir-frontend-string-literal.c create mode 100644 tests/vir-frontend-swap.c create mode 100644 tests/vir-frontend-ternary-array.c create mode 100644 tests/vir-frontend-void-call-tail.c create mode 100644 tests/vir-frontend-void.c create mode 100644 tests/vir-frontend-volatile-mixed.c create mode 100644 tests/vir-frontend-volatile.c create mode 100644 tests/vir-frontend-workloads.txt create mode 100644 tests/vir-frontend.sh create mode 100644 tests/vir-global-scratch.c create mode 100644 tests/vir-grouped-pointer-alias.c create mode 100644 tests/vir-hidden-const-return-compatible.c create mode 100755 tests/vir-large-cfg.sh create mode 100644 tests/vir-licm-profile.c create mode 100644 tests/vir-lower-smoke.c create mode 100644 tests/vir-lower.c create mode 100644 tests/vir-lower.sh create mode 100644 tests/vir-machine-affinity.c create mode 100644 tests/vir-machine-live.c create mode 100644 tests/vir-multi-input-main.c create mode 100644 tests/vir-multi-input-root.c create mode 100644 tests/vir-multi-input.sh create mode 100755 tests/vir-native-regressions.sh create mode 100644 tests/vir-operand-signedness.c create mode 100644 tests/vir-pair-scratch.c create mode 100644 tests/vir-pointer-comparison.c create mode 100644 tests/vir-pointer-offset.c create mode 100644 tests/vir-pointer-offset.sh create mode 100644 tests/vir-qualifiers.sh create mode 100644 tests/vir-shift-width.c create mode 100644 tests/vir-ssa-baseline-workloads.txt create mode 100755 tests/vir-ssa-baseline.sh create mode 100644 tests/vir-ssa-selfhost-workloads.txt create mode 100644 tests/vir-stage0.c create mode 100755 tests/vir-stats.sh create mode 100644 tests/vir-strength.c create mode 100755 tests/vir-strength.sh create mode 100644 tests/vir-subtract-alias.c create mode 100644 tests/vir-typed-arithmetic.c create mode 100644 tests/vir-void-callback-array.c create mode 100644 tests/vir-volatile-arithmetic.c create mode 100644 tests/vir-volatile-callback-pointer-return.c create mode 100644 tests/vir-volatile-callback-return-pointer.c create mode 100644 tests/vir-volatile-callback.c create mode 100644 tests/vir-volatile-comma-alias.c create mode 100644 tests/vir-volatile-local-getter.c create mode 100644 tests/vir-volatile-local.c create mode 100644 tests/vir-volatile-member.c create mode 100644 tests/vir-volatile-pointer-member.c create mode 100644 tests/vir-volatile-storage.c create mode 100644 tests/x64-startup-globals.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5b22a57..787f7dbc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,9 @@ jobs: host-x86: name: ${{ matrix.architecture }}/${{ matrix.link_mode }}/${{ matrix.binding_mode }} (${{ matrix.compiler }}) runs-on: ubuntu-24.04 - timeout-minutes: 30 + # The stage 2 suites run the self-hosted compiler under qemu for every + # case, and the cross-compiled lanes need more than half an hour of it. + timeout-minutes: 60 strategy: # One target failing says nothing about the others, and finding out # which of them share the failure is the point of the matrix. diff --git a/Makefile b/Makefile index d6015316..07867824 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,18 @@ check_flag = $(shell $(CC) $(1) -S -o /dev/null -xc /dev/null 2>/dev/null; \ # targets have no use for, so skip it when nothing is being compiled. STYLE_GOALS := check-style check-newline check-comments check-format check-shell \ indent install-hooks uninstall-hooks check-hooks + +VIR_FRONTEND_FIXTURES := $(sort $(wildcard tests/vir-frontend-*.c)) +VIR_NATIVE_SHARED_FIXTURES := $(sort $(wildcard tests/vir-direct-global-*.c) \ + $(wildcard tests/vir-direct-branch-*.c) \ + $(wildcard tests/vir-direct-narrow-global.c) \ + tests/vir-direct-external-pointer-origin.c \ + tests/vir-direct-external-typed-pointer.c \ + tests/vir-direct-external-typed-pointer-offset.c) +VIR_NATIVE_X64_FIXTURES := $(sort $(wildcard tests/vir-direct-x64*.c) \ + $(VIR_NATIVE_SHARED_FIXTURES)) +VIR_NATIVE_ARM64_FIXTURES := $(sort $(wildcard tests/vir-direct-arm64*.c) \ + $(VIR_NATIVE_SHARED_FIXTURES)) ifneq ($(filter-out $(STYLE_GOALS),$(or $(MAKECMDGOALS),all)),) $(foreach flag, $(CFLAGS_TO_CHECK), $(eval CFLAGS += $(call check_flag, $(flag)))) endif @@ -172,9 +184,73 @@ config: $(Q)$(CONFIG_CHECK_CMD) .PHONY: $(STYLE_GOALS) +.PHONY: check-vir-frontend-stage0 check-vir-frontend-stage2 \ + check-vir-stage0 check-vir-stage2 \ + check-vir-frontend-policy-stage0 check-vir-frontend-policy-stage2 \ + check-vir-ssa-baseline check-vir-ssa-baseline-stage0 check-vir-ssa-baseline-stage2 \ + check-vir-large-cfg-stage0 check-vir-large-cfg-stage2 \ + check-vir-ssa-selfhost check-vir-ssa-selfhost-stage0 check-vir-ssa-selfhost-stage2 check: check-stage0 check-stage2 check-abi-stage0 check-abi-stage2 \ - check-vir-core check-vir-fuzz + check-vir-core check-vir-frontend-stage0 check-vir-frontend-stage2 check-vir-fuzz check-vir-stage0 check-vir-stage2 check-vir-arm-verifier-stage0 \ + check-vir-multi-input-stage0 check-vir-multi-input-stage2 \ + check-vir-arm-verifier-stage2 check-vir-stats-stage0 check-vir-stats-stage2 check-vir-baseline check-vir-ssa-baseline check-vir-large-cfg-stage0 check-vir-large-cfg-stage2 check-vir-ssa-selfhost \ + check-vir-corpus-stage0 check-fuzz-stage0 check-vir-frontend-policy-stage0 + +# Random programs checked against the host compiler and built by shecc. +# FUZZ_SEEDS and FUZZ_START widen or move the run; a failing program is kept +# as out/fuzz-.c. +.PHONY: check-fuzz-stage0 +check-fuzz-stage0: $(OUT)/$(STAGE0) tests/fuzz.sh tests/fuzz-gen.c + TARGET_EXEC="$(TARGET_EXEC)" CC="$(CC)" bash tests/fuzz.sh 0 + +# Optimization must preserve each corpus program's behavior under --no-opt. +# Only stage 0 is part of check: under emulation, stage 2 compiles the corpus +# twice inside qemu, which costs more than the other checks. +.PHONY: check-vir-corpus-stage0 check-vir-corpus-stage2 +check-vir-corpus-stage0: $(OUT)/$(STAGE0) tests/vir-corpus.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-corpus.sh 0 $(ARCH) +check-vir-corpus-stage2: $(OUT)/$(STAGE2) tests/vir-corpus.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-corpus.sh 2 $(ARCH) + +.PHONY: check-vir-multi-input-stage0 check-vir-multi-input-stage2 +check-vir-multi-input-stage0: $(OUT)/$(STAGE0) tests/vir-multi-input.sh tests/vir-multi-input-root.c tests/vir-multi-input-main.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-multi-input.sh 0 +check-vir-multi-input-stage2: $(OUT)/$(STAGE2) tests/vir-multi-input.sh tests/vir-multi-input-root.c tests/vir-multi-input-main.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-multi-input.sh 2 + +check-vir-large-cfg-stage0: $(OUT)/$(STAGE0) tests/vir-large-cfg.sh tests/vir-frontend-large-cfg.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-large-cfg.sh 0 + +check-vir-large-cfg-stage2: $(OUT)/$(STAGE2) tests/vir-large-cfg.sh tests/vir-frontend-large-cfg.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-large-cfg.sh 2 + +ifeq ($(ARCH),x64) +.PHONY: check-vir-native-x64-stage0 check-vir-native-x64-stage2 \ + check-x64-startup-globals-stage0 check-x64-startup-globals-stage2 +check: check-x64-startup-globals-stage0 check-x64-startup-globals-stage2 \ + check-vir-native-x64-stage0 check-vir-native-x64-stage2 +check-vir-native-x64-stage0: $(OUT)/$(STAGE0) $(VIR_NATIVE_X64_FIXTURES) tests/vir-direct-x64.sh tests/vir-direct-common.sh + bash tests/vir-direct-x64.sh 0 +check-vir-native-x64-stage2: $(OUT)/$(STAGE2) $(VIR_NATIVE_X64_FIXTURES) tests/vir-direct-x64.sh tests/vir-direct-common.sh + bash tests/vir-direct-x64.sh 2 + +.PHONY: check-x64-startup-globals-stage0 check-x64-startup-globals-stage2 +check-x64-startup-globals-stage0: $(OUT)/$(STAGE0) tests/vir-direct-x64-startup-globals.c tests/x64-startup-globals.sh + bash tests/x64-startup-globals.sh 0 +check-x64-startup-globals-stage2: $(OUT)/$(STAGE2) tests/vir-direct-x64-startup-globals.c tests/x64-startup-globals.sh + bash tests/x64-startup-globals.sh 2 +endif + +ifeq ($(ARCH),arm64) +.PHONY: check-vir-native-arm64-stage0 check-vir-native-arm64-stage2 +check: check-vir-native-arm64-stage0 check-vir-native-arm64-stage2 + +check-vir-native-arm64-stage0: $(OUT)/$(STAGE0) $(VIR_NATIVE_ARM64_FIXTURES) tests/vir-direct-arm64.sh tests/vir-direct-common.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-direct-arm64.sh 0 +check-vir-native-arm64-stage2: $(OUT)/$(STAGE2) $(VIR_NATIVE_ARM64_FIXTURES) tests/vir-direct-arm64.sh tests/vir-direct-common.sh + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-direct-arm64.sh 2 +endif # Run the complete check -- driver and ABI suites at stages 0 and 2 -- on every # backend, as CI does. Driver cases that need 64-bit values are gated on the @@ -196,6 +272,25 @@ check-all-targets: $(MAKE) check ARCH="$$target_arch"; \ done +.PHONY: vir-baseline +vir-baseline: $(OUT)/$(STAGE0) tests/vir-baseline.sh tests/benchmark-time.sh \ + tests/vir-baseline-workloads.txt + VIR_BASELINE_RUNS=$${VIR_BASELINE_RUNS:-5} tests/vir-baseline.sh + +.PHONY: vir-ssa-baseline +vir-ssa-baseline: $(OUT)/$(STAGE0) tests/vir-ssa-baseline.sh tests/benchmark-time.sh \ + tests/vir-ssa-baseline-workloads.txt tests/vir-frontend-large-cfg.c + tests/vir-ssa-baseline.sh 0 + +.PHONY: vir-licm-profile +VIR_LICM_PROFILE_SAMPLES ?= 7 +VIR_LICM_PROFILE_REPEATS ?= 5 +vir-licm-profile: $(OUT)/vir-licm-profile + $(OUT)/vir-licm-profile $(VIR_LICM_PROFILE_SAMPLES) $(VIR_LICM_PROFILE_REPEATS) + +$(OUT)/vir-licm-profile: tests/vir-licm-profile.c src/vir.c src/vir.h + $(CC) $(CFLAGS) -Isrc tests/vir-licm-profile.c src/vir.c -o $@ + # One checker per target: they share nothing, so "make -j check-style" runs them # concurrently and finishes in the time the slowest one takes. check-style: check-newline check-comments check-format check-shell @@ -244,7 +339,7 @@ check-stage2: $(OUT)/$(STAGE2) tests/driver.sh $(VECHO) " TEST STAGE 2\n" tests/driver.sh 2 $(DYNLINK) $(BINDING) -check-sanitizer: $(OUT)/$(STAGE0)-sanitizer tests/driver.sh +check-sanitizer: $(OUT)/$(STAGE0)-sanitizer tests/driver.sh check-vir-sanitizer $(VECHO) " TEST STAGE 0 (with sanitizers)\n" $(Q)cp $(OUT)/$(STAGE0)-sanitizer $(OUT)/shecc tests/driver.sh 0 $(DYNLINK) $(BINDING) @@ -256,21 +351,124 @@ check-abi-stage0: $(OUT)/$(STAGE0) check-abi-stage2: $(OUT)/$(STAGE2) tests/$(ARCH)-abi.sh 2 $(DYNLINK) $(BINDING); -.PHONY: check-vir-core check-vir-fuzz check-vir-core: tests/vir-core.c src/vir.c src/vir.h $(CC) $(CFLAGS) -Isrc tests/vir-core.c src/vir.c -o $(OUT)/vir-core $(OUT)/vir-core +check-vir-frontend-stage0: $(OUT)/$(STAGE0) tests/vir-frontend.sh tests/vir-frontend-workloads.txt $(VIR_FRONTEND_FIXTURES) + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend.sh 0 + +check-vir-frontend-stage2: $(OUT)/$(STAGE2) tests/vir-frontend.sh tests/vir-frontend-workloads.txt $(VIR_FRONTEND_FIXTURES) + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend.sh 2 + +check-vir-frontend-policy-stage0: $(OUT)/$(STAGE0) tests/vir-frontend-policy.sh tests/vir-frontend-gvn.c tests/vir-frontend-licm.c tests/vir-frontend-nested-loop.c tests/vir-frontend-multi-latch.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend-policy.sh 0 + +check-vir-frontend-policy-stage2: $(OUT)/$(STAGE2) tests/vir-frontend-policy.sh tests/vir-frontend-gvn.c tests/vir-frontend-licm.c tests/vir-frontend-nested-loop.c tests/vir-frontend-multi-latch.c + TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-frontend-policy.sh 2 + check-vir-fuzz: tests/vir-fuzz.c src/vir.c src/vir.h $(CC) $(CFLAGS) -Isrc tests/vir-fuzz.c src/vir.c -o $(OUT)/vir-fuzz $(OUT)/vir-fuzz +check-vir-sanitizer: tests/vir-core.c tests/vir-fuzz.c src/vir.c src/vir.h + $(CC) $(CFLAGS) $(SAN_CFLAGS) -Isrc tests/vir-core.c src/vir.c -o $(OUT)/vir-core-sanitize + ASAN_OPTIONS=detect_leaks=1 $(OUT)/vir-core-sanitize + $(CC) $(CFLAGS) $(SAN_CFLAGS) -Isrc tests/vir-fuzz.c src/vir.c -o $(OUT)/vir-fuzz-sanitize + ASAN_OPTIONS=detect_leaks=1 $(OUT)/vir-fuzz-sanitize + +.PHONY: check-vir +check-vir: check-vir-core check-vir-frontend-stage0 \ + check-vir-frontend-stage2 check-vir-fuzz check-vir-stage0 check-vir-stage2 \ + check-vir-frontend-policy-stage0 \ + check-vir-frontend-policy-stage2 \ + check-vir-arm-verifier-stage0 check-vir-arm-verifier-stage2 check-vir-sanitizer + +check-vir-stage0: $(OUT)/$(STAGE0) tests/vir-stage0.c src/vir.c src/vir.h + $(OUT)/$(STAGE0) -o $(OUT)/vir-stage0 tests/vir-stage0.c + $(TARGET_EXEC) $(OUT)/vir-stage0 + +check-vir-stage2: $(OUT)/$(STAGE2) tests/vir-stage0.c src/vir.c src/vir.h + $(TARGET_EXEC) $(OUT)/$(STAGE2) -o $(OUT)/vir-stage2 tests/vir-stage0.c + $(TARGET_EXEC) $(OUT)/vir-stage2 + +.PHONY: check-vir-native-regressions-stage0 check-vir-native-regressions-stage2 +check: check-vir-native-regressions-stage0 check-vir-native-regressions-stage2 check-vir-lower +check-vir-native-regressions-stage0: $(OUT)/$(STAGE0) + TARGET_EXEC="$(TARGET_EXEC)" tests/vir-native-regressions.sh $(OUT)/$(STAGE0) +check-vir-native-regressions-stage2: $(OUT)/$(STAGE2) + TARGET_EXEC="$(TARGET_EXEC)" tests/vir-native-regressions.sh $(TARGET_EXEC) $(OUT)/$(STAGE2) + +.PHONY: check-vir-lower +check-vir-lower: $(OUT)/$(STAGE0) tests/vir-lower.c tests/vir-lower-smoke.c tests/vir-lower.sh + CC="$(CC)" TARGET_EXEC="$(TARGET_EXEC)" bash tests/vir-lower.sh + +# This formerly exposed an Arm stage-0 verifier miscompile. Keep it in both +# bootstrap gates and on every target so CFG-heavy verifier paths cannot regress. +check-vir-arm-verifier-stage0: $(OUT)/$(STAGE0) tests/vir-arm-verifier.c src/vir.c src/vir.h + $(OUT)/$(STAGE0) -o $(OUT)/vir-arm-verifier-stage0 tests/vir-arm-verifier.c + $(TARGET_EXEC) $(OUT)/vir-arm-verifier-stage0 + +check-vir-arm-verifier-stage2: $(OUT)/$(STAGE2) tests/vir-arm-verifier.c src/vir.c src/vir.h + $(TARGET_EXEC) $(OUT)/$(STAGE2) -o $(OUT)/vir-arm-verifier-stage2 tests/vir-arm-verifier.c + $(TARGET_EXEC) $(OUT)/vir-arm-verifier-stage2 + +check-vir-stats-stage0: $(OUT)/$(STAGE0) tests/vir-stats.sh + tests/vir-stats.sh 0 + +check-vir-stats-stage2: $(OUT)/$(STAGE2) tests/vir-stats.sh + tests/vir-stats.sh 2 + +check-vir-baseline: $(OUT)/$(STAGE0) tests/vir-baseline.sh tests/benchmark-time.sh \ + tests/vir-baseline-workloads.txt + VIR_BASELINE_RUNS=1 VIR_BASELINE_NO_TIMING=1 tests/vir-baseline.sh + +check-vir-ssa-baseline-stage0: $(OUT)/$(STAGE0) tests/vir-ssa-baseline.sh tests/benchmark-time.sh \ + tests/vir-ssa-baseline-workloads.txt tests/vir-frontend-large-cfg.c + VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-baseline-workloads.txt tests/vir-ssa-baseline.sh 0 + +check-vir-ssa-baseline-stage2: $(OUT)/$(STAGE2) tests/vir-ssa-baseline.sh tests/benchmark-time.sh \ + tests/vir-ssa-baseline-workloads.txt tests/vir-frontend-large-cfg.c + TARGET_EXEC="$(TARGET_EXEC)" VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-baseline-workloads.txt tests/vir-ssa-baseline.sh 2 + +VIR_SSA_BASELINE_NORMALIZE = sed -e 's/stage=[02] /stage=both /' -e 's/ output_bytes=[0-9][0-9]*//' + +# The recipe runs both stages itself to compare them, so depending on the +# per-stage targets would run every workload twice. +check-vir-ssa-baseline: $(OUT)/$(STAGE0) $(OUT)/$(STAGE2) tests/vir-ssa-baseline.sh \ + tests/benchmark-time.sh tests/vir-ssa-baseline-workloads.txt tests/vir-frontend-large-cfg.c + @set -e; stage0=$$(mktemp); stage2=$$(mktemp); trap 'rm -f "$$stage0" "$$stage2" "$$stage0.normalized" "$$stage2.normalized"' EXIT; \ + VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-baseline-workloads.txt tests/vir-ssa-baseline.sh 0 > "$$stage0"; \ + TARGET_EXEC="$(TARGET_EXEC)" VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-baseline-workloads.txt tests/vir-ssa-baseline.sh 2 > "$$stage2"; \ + $(VIR_SSA_BASELINE_NORMALIZE) "$$stage0" > "$$stage0.normalized"; \ + $(VIR_SSA_BASELINE_NORMALIZE) "$$stage2" > "$$stage2.normalized"; \ + cmp -s "$$stage0.normalized" "$$stage2.normalized" + +check-vir-ssa-selfhost-stage0: $(OUT)/$(STAGE0) tests/vir-ssa-baseline.sh tests/benchmark-time.sh \ + tests/vir-ssa-selfhost-workloads.txt + VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-selfhost-workloads.txt tests/vir-ssa-baseline.sh 0 + +check-vir-ssa-selfhost-stage2: $(OUT)/$(STAGE2) tests/vir-ssa-baseline.sh tests/benchmark-time.sh \ + tests/vir-ssa-selfhost-workloads.txt + TARGET_EXEC="$(TARGET_EXEC)" VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-selfhost-workloads.txt tests/vir-ssa-baseline.sh 2 + +check-vir-ssa-selfhost: $(OUT)/$(STAGE0) $(OUT)/$(STAGE2) tests/vir-ssa-baseline.sh \ + tests/benchmark-time.sh tests/vir-ssa-selfhost-workloads.txt + @set -e; stage0=$$(mktemp); stage2=$$(mktemp); trap 'rm -f "$$stage0" "$$stage2" "$$stage0.normalized" "$$stage2.normalized"' EXIT; \ + VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-selfhost-workloads.txt tests/vir-ssa-baseline.sh 0 > "$$stage0"; \ + TARGET_EXEC="$(TARGET_EXEC)" VIR_SSA_BASELINE_NO_TIMING=1 VIR_SSA_BASELINE_RUNS=2 VIR_SSA_BASELINE_MANIFEST=tests/vir-ssa-selfhost-workloads.txt tests/vir-ssa-baseline.sh 2 > "$$stage2"; \ + $(VIR_SSA_BASELINE_NORMALIZE) "$$stage0" > "$$stage0.normalized"; \ + $(VIR_SSA_BASELINE_NORMALIZE) "$$stage2" > "$$stage2.normalized"; \ + cmp -s "$$stage0.normalized" "$$stage2.normalized" + # Both prerequisites are order-only, and both exist because "make -j" would # otherwise let a compile start beside the thing it reads. Selecting a target # replaces src/codegen.c with "ln -sf", which unlinks before it relinks, so a # compile racing "config" can find nothing there. And src/main.c includes # out/libc.inc, which is generated: listing it only on the stage0 link left # the two free to run in either order on a tree that has never been built. +$(OBJS) $(SAN_OBJS): $(OUT)/libc.inc + $(OUT)/%.o: %.c | config $(OUT)/libc.inc $(VECHO) " CC\t$@\n" $(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF $@.d $< diff --git a/mk/arm.mk b/mk/arm.mk index f1f33711..8ffa1b90 100644 --- a/mk/arm.mk +++ b/mk/arm.mk @@ -20,6 +20,7 @@ ARCH_DEFS = \ \#define RESERVED_GOT_NUM 3\n$\ \#define R_ARCH_JUMP_SLOT 0x16\n$\ \#define MAX_ARGS_IN_REG 4\n$\ + \#define CALLEE_SAVED_REGS 4 /* r4-r7: AAPCS, and __syscall restores them */\n$\ " # If the running machine has the "fastfetch" tool installed, the build diff --git a/mk/riscv.mk b/mk/riscv.mk index 53402b5b..98a49b31 100644 --- a/mk/riscv.mk +++ b/mk/riscv.mk @@ -18,6 +18,8 @@ ARCH_DEFS = \ \#define RESERVED_GOT_NUM 2\n$\ \#define R_ARCH_JUMP_SLOT 0x5\n$\ \#define MAX_ARGS_IN_REG 8\n$\ + \#define REG_CNT 11 /* a0-a7, then s3-s5 */\n$\ + \#define CALLEE_SAVED_REGS 3 /* s3-s5 */\n$\ " TOOLCHAIN_CANDIDATES = riscv32-unknown-linux-gnu- diff --git a/src/arch-lower.c b/src/arch-lower.c index d8ee19be..5ea25ba1 100644 --- a/src/arch-lower.c +++ b/src/arch-lower.c @@ -19,11 +19,8 @@ void arch_lower(void) { #if ELF_MACHINE == ELF_MACHINE_ARM32 - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - + FOR_EACH_FUNCTION_BODY(func) + { for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; insn = insn->next) { diff --git a/src/arm-codegen.c b/src/arm-codegen.c index 359c4ab5..42e8ed1f 100644 --- a/src/arm-codegen.c +++ b/src/arm-codegen.c @@ -11,6 +11,13 @@ #include "defs.h" #include "globals.c" +static bool arm_count_only; +static int arm_counted_words; + +void emit_ph2_ir(ph2_ir_t *ph2_ir); + +static int arm_count_ph2_ir_words(ph2_ir_t *ph2_ir); + /* Whether a load or store materializes its frame or data offset with * MOVW/MOVT/ADD because no immediate form reaches it. LDR, LDRB, STR and STRB * take 12 bits, LDRSB, LDRH, LDRSH and STRH eight, and a register pair also @@ -31,253 +38,21 @@ bool arm_offset_needs_movw(const ph2_ir_t *ph2_ir) void update_elf_offset(ph2_ir_t *ph2_ir) { - func_t *func; - switch (ph2_ir->op) { - case OP_load_constant: - /* ARMv7 uses 12 bits to encode immediate value, but the higher 4 bits - * are for rotation. See A5.2.4 "Modified immediate constants in ARM - * instructions" in ARMv7-A manual. - */ - if (ph2_ir->src0 < 0) - elf_offset += 12; - else if (ph2_ir->src0 > 255) - elf_offset += 8; - else - elf_offset += 4; - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src1 < 0) - elf_offset += 12; - else if (ph2_ir->src1 > 255) - elf_offset += 8; - else - elf_offset += 4; - } - return; - case OP_address_of: - case OP_global_address_of: - /* ARMv7 uses 12 bits to encode immediate value, but the higher 4 bits - * are for rotation. See A5.2.4 "Modified immediate constants in ARM - * instructions" in ARMv7-A manual. - */ - if (ph2_ir->src0 > 255) - elf_offset += 12; - else if (ph2_ir->src0 >= 0) - elf_offset += 4; - else - abort(); - return; - case OP_assign: - if (ph2_ir->dest != ph2_ir->src0) - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) - elf_offset += 4; - /* Exchanging the two registers of a pair takes a third move. */ - if (ph2_ir->dest_hi >= 0 && ph2_ir->dest == ph2_ir->src0_hi && - ph2_ir->dest_hi == ph2_ir->src0) - elf_offset += 4; - return; - case OP_load: - case OP_global_load: - /* LDR/LDRB use a 12-bit offset, but LDRSB and LDRH/LDRSH use eight - * bits. A larger signed-byte or halfword offset is materialized with - * MOVW/MOVT/ADD before the load, exactly as an offset beyond the - * general load range is. A wide pair also loads its high word four - * bytes further up. - */ - if (arm_offset_needs_movw(ph2_ir)) - if (ph2_ir->dest_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->src0 >= 0) - if (ph2_ir->dest_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - else - abort(); - return; - case OP_store: - case OP_global_store: - /* ARMv7 straight uses 12 bits to encode the offset of store instruction - * (no rotation), but STRH only eight. A wide pair also stores its high - * word four bytes further up. - */ - if (arm_offset_needs_movw(ph2_ir)) - if (ph2_ir->src0_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->src1 >= 0) - if (ph2_ir->src0_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - else - abort(); - return; - case OP_read: - elf_offset += ph2_ir->dest_hi >= 0 ? 8 : 4; - return; - case OP_write: - elf_offset += ph2_ir->src1_hi >= 0 ? 8 : 4; - return; - case OP_indirect: - /* blx, then under dynamic linking the r12 reload that OP_call adds. */ - elf_offset += dynlink ? 16 : 4; - return; - case OP_jump: - case OP_load_func: - case OP_lshift: - case OP_rshift: - elf_offset += ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 ? 48 : 4; - return; - case OP_mul: - elf_offset += - ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 - ? 20 - : 4; - return; - case OP_negate: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_add: - case OP_sub: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 4; - return; - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 4; - return; - case OP_bit_not: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_call: - func = find_func(ph2_ir->func_name); - if (func->bbs) - elf_offset += 4; - else if (dynlink) { - /* When calling external functions in dynamic linking mode, the - * following instructions are required: - * - movw + movt: set r8 to 'elf_data_start' - * - ldr: load a word from the address 'elf_data_start' into r12. - * (restore the global stack pointer.) - * - * Therefore, the total offset is 16 bytes (4 instructions). - */ - elf_offset += 16; - } else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } - return; - case OP_div: - case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - /* The stack-backed restoring loop below has a fixed 64-round body. - * It keeps all pair state out of operand registers: an SSA result - * may coalesce with either dying input pair. - */ - if (ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned) - elf_offset += 284; - else - elf_offset += ph2_ir->op == OP_div ? 424 : 416; - return; - } - if (hard_mul_div) { - if (ph2_ir->op == OP_div) - elf_offset += 4; - else - elf_offset += 12; - return; - } - /* div/mod emulation's offset */ - elf_offset += 124; - return; - case OP_load_data_address: - case OP_load_rodata_address: - elf_offset += 8; - return; - case OP_address_of_func: - elf_offset += 12; - return; - case OP_log_not: - elf_offset += ph2_ir->src0_hi >= 0 ? 16 : 12; - return; - case OP_gt: - case OP_lt: - case OP_geq: - case OP_leq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 16 : 12; - return; - case OP_eq: - case OP_neq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 16 : 12; - return; - case OP_branch: - if (ph2_ir->is_branch_detached) - elf_offset += 12; - else - elf_offset += 8; - if (ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_return: - elf_offset += 24; - if (ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_trunc: - /* Unsigned byte/half truncation uses a logical shift pair; signed - * narrowing uses the single-instruction SXTB/SXTH forms. - */ - if (ph2_ir->is_unsigned && (ph2_ir->src1 == 1 || ph2_ir->src1 == 2)) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_sign_ext: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi < 0) { - int source_size = (ph2_ir->src1 >> 16) & 0xFFFF; - elf_offset += - (ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer) && - (source_size == 1 || source_size == 2) - ? 12 - : 8; - } else if (ph2_ir->src0_is_unsigned) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_cast: - elf_offset += - ph2_ir->dest_hi >= 0 && - (ph2_ir->src0_hi < 0 || ph2_ir->dest_hi != ph2_ir->src0_hi) - ? 8 - : 4; - return; - default: - fatal("Unknown opcode"); - } + elf_offset += arm_count_ph2_ir_words(ph2_ir) * 4; +} + +static ph2_frame_layout_t arm_frame_layout(func_t *func, ph2_ir_t *define) +{ + /* Eight-byte alignment follows the nine saved words and one padding word. + */ + return (ph2_frame_layout_t) {ALIGN_UP(func->stack_size, MIN_ALIGNMENT) + 40, + arm_count_ph2_ir_words(define) * 4, 0, false}; } void cfg_flatten(void) { func_t *func; + ph2_ir_prepare(false); if (dynlink) elf_offset = 132; /* __libc_start_main call, main_wrapper, and the @@ -291,10 +66,7 @@ void cfg_flatten(void) GLOBAL_FUNC->bbs->elf_offset = elf_offset; - for (ph2_ir_t *ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) { - update_elf_offset(ph2_ir); - } + ph2_ir_visit_globals(update_elf_offset); /* prepare 'argc' and 'argv', then proceed to 'main' function */ if (dynlink) @@ -302,75 +74,50 @@ void cfg_flatten(void) else elf_offset += 32; /* 6 insns for main call + 2 for exit */ - for (func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - /* reserve stack */ - ph2_ir_t *flatten_ir = add_ph2_ir(OP_define); - flatten_ir->src0 = func->stack_size; - flatten_ir->func_name = intern_string(func->return_def.var_name); - - /* The actual offset of the top of the local stack is the sum of: - * - 36 bytes (pushing registers r4-r11 and lr onto the stack) - * - 4 bytes - * (to ensure 8-byte alignment after pushing the 9 registers) - * - ALIGN_UP(func->stack_size, 8) - * - * Note that func->stack_size does not include the 36 + 4 bytes, so an - * additional 40 bytes should be added to ALIGN_UP(func->stack_size, 8). - */ - int stack_top_ofs = ALIGN_UP(func->stack_size, MIN_ALIGNMENT) + 40; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->elf_offset = elf_offset; - - if (bb == func->bbs) { - /* retrieve the global stack pointer and save ra, sp */ - elf_offset += 28; - } + ph2_ir_flatten_functions(arm_frame_layout, update_elf_offset); +} - for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; - insn = insn->next) { - /* For the instructions whose ofs_based_on_stack_top is set, - * recalculate the operand's offset by adding stack_top_ofs. - */ - if (insn->ofs_based_on_stack_top) { - switch (insn->op) { - case OP_load: - case OP_address_of: - insn->src0 = insn->src0 + stack_top_ofs; - break; - case OP_store: - insn->src1 = insn->src1 + stack_top_ofs; - break; - default: - /* Ignore opcodes with the ofs_based_on_stack_top flag - * set since only the three opcodes above needs to - * access a variable's address. - */ - break; - } - } - flatten_ir = add_existed_ph2_ir(insn); +void emit(int code) +{ + if (arm_count_only) + ++arm_counted_words; + else + elf_write_int(elf_code, code); +} - if (insn->op == OP_return) { - /* restore sp */ - flatten_ir->src1 = bb->belong_to->stack_size; - } +static int arm_count_ph2_ir_words(ph2_ir_t *ph2_ir) +{ + bool saved_count_only = arm_count_only; + int saved_counted_words = arm_counted_words; + char *saved_fatal_function_context = fatal_function_context; - /* Branch detachment is determined in the arch-lowering stage */ + arm_count_only = true; + arm_counted_words = 0; + emit_ph2_ir(ph2_ir); + int words = arm_counted_words; - update_elf_offset(flatten_ir); - } - } - } + arm_count_only = saved_count_only; + arm_counted_words = saved_counted_words; + fatal_function_context = saved_fatal_function_context; + return words; } -void emit(int code) +static int arm_bitwise_insn(opcode_t op, + arm_reg dest, + arm_reg left, + arm_reg right) { - elf_write_int(elf_code, code); + switch (op) { + case OP_bit_and: + return __and_r(__AL, dest, left, right); + case OP_bit_or: + return __or_r(__AL, dest, left, right); + case OP_bit_xor: + return __eor_r(__AL, dest, left, right); + default: + fatal("unsupported ARM bitwise operation"); + } + return 0; } void emit_ph2_ir(ph2_ir_t *ph2_ir) @@ -458,8 +205,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) * emitting one here would push every later address out by four bytes * and leave the data segment's p_offset and p_vaddr incongruent. */ - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - rd == ph2_ir->src0_hi) { + if (PH2_PAIR_DEST_SRC0(ph2_ir) && rd == ph2_ir->src0_hi) { /* A pair moved one register over, as a slot round trip collapsed * into a move can leave it: the low destination is the high source, * so move the high word first, through r8 when the two registers @@ -477,8 +223,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) } if (rd != rn) emit(__mov_r(__AL, rd, rn)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) + if (PH2_PAIR_DEST_SRC0(ph2_ir) && ph2_ir->dest_hi != ph2_ir->src0_hi) emit(__mov_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi)); return; case OP_load: @@ -565,19 +310,11 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_read: if (ph2_ir->dest_hi >= 0) { - /* A pair read through a pointer. When the low destination is the - * address register itself, fetch the high word first so the address - * survives until both words are loaded. - */ if (ph2_ir->src1 != 8) fatal("unsupported Arm pair load width"); - if (rd == rn) { - emit(__lw(__AL, ph2_ir->dest_hi, rn, 4)); - emit(__lw(__AL, rd, rn, 0)); - } else { - emit(__lw(__AL, rd, rn, 0)); - emit(__lw(__AL, ph2_ir->dest_hi, rn, 4)); - } +#define ARM_WORD_LOAD(dst, base, offset) emit(__lw(__AL, dst, base, offset)) + EMIT_PAIR_LOAD(ARM_WORD_LOAD, rd, ph2_ir->dest_hi, rn); +#undef ARM_WORD_LOAD return; } if (ph2_ir->src1 == 1) @@ -675,16 +412,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__movt(__AL, rd, ph2_ir->src0 + elf_rodata_start)); return; case OP_address_of_func: - func = find_func(ph2_ir->func_name); - if (func->bbs) - ofs = elf_code_start + func->bbs->elf_offset; - else if (dynlink) - ofs = dynamic_sections.elf_plt_start + func->plt_offset; - else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } + ofs = function_entry_address(ph2_ir->func_name); emit(__movw(__AL, __r8, ofs)); emit(__movt(__AL, __r8, ofs)); emit(__sw(__AL, __r8, rn, 0)); @@ -730,8 +458,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__bx(__AL, __lr)); return; case OP_add: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) { emit(__adds_r(__AL, rd, rn, rm)); emit(__adc_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, ph2_ir->src1_hi)); @@ -739,8 +466,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__add_r(__AL, rd, rn, rm)); return; case OP_sub: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) { emit(__subs_r(__AL, rd, rn, rm)); emit(__sbc_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, ph2_ir->src1_hi)); @@ -748,8 +474,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sub_r(__AL, rd, rn, rm)); return; case OP_mul: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) { /* Sum the cross terms before UMULL writes the result pair, so the * product stays right even if that pair shares a register with an * operand. The allocator does not hand out such a pair today, and @@ -765,8 +490,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_div: case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) { bool is_unsigned = ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned; @@ -984,7 +708,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__rsb_i(__NE, rd, 0, rd)); return; case OP_lshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (PH2_PAIR_DEST_SRC0(ph2_ir)) { emit(__cmp_i(__AL, rm, 32)); emit(__b(__CS, 32)); emit(__mov_r(__AL, __r8, rn)); @@ -1002,7 +726,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sll(__AL, rd, rn, rm)); return; case OP_rshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (PH2_PAIR_DEST_SRC0(ph2_ir)) { int shift_kind = ph2_ir->src0_is_unsigned ? logic_rs : arith_rs; emit(__cmp_i(__AL, rm, 32)); @@ -1076,7 +800,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) rd, 1)); return; case OP_negate: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (PH2_PAIR_DEST_SRC0(ph2_ir)) { emit(__rsbs_i(__AL, rd, 0, rn)); emit(__rsc_i(__AL, ph2_ir->dest_hi, 0, ph2_ir->src0_hi)); } else @@ -1084,29 +808,16 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_bit_not: emit(__mvn_r(__AL, rd, rn)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) + if (PH2_PAIR_DEST_SRC0(ph2_ir)) emit(__mvn_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi)); return; case OP_bit_and: - emit(__and_r(__AL, rd, rn, rm)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__and_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); - return; case OP_bit_or: - emit(__or_r(__AL, rd, rn, rm)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__or_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); - return; case OP_bit_xor: - emit(__eor_r(__AL, rd, rn, rm)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__eor_r(__AL, ph2_ir->dest_hi, ph2_ir->src0_hi, - ph2_ir->src1_hi)); + emit(arm_bitwise_insn(ph2_ir->op, rd, rn, rm)); + if (PH2_PAIR_BINARY(ph2_ir)) + emit(arm_bitwise_insn(ph2_ir->op, ph2_ir->dest_hi, ph2_ir->src0_hi, + ph2_ir->src1_hi)); return; case OP_log_not: if (ph2_ir->src0_hi >= 0) { @@ -1340,10 +1051,7 @@ void code_generate(void) emit(__bx(__AL, __lr)); } - ph2_ir_t *ph2_ir; - for (ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) - emit_ph2_ir(ph2_ir); + ph2_ir_visit_globals(emit_ph2_ir); /* prepare 'argc' and 'argv', then proceed to 'main' function */ if (MAIN_BB) { @@ -1382,7 +1090,7 @@ void code_generate(void) } for (int i = 0; i < ph2_ir_idx; i++) { - ph2_ir = PH2_IR_FLATTEN[i]; + ph2_ir_t *ph2_ir = PH2_IR_FLATTEN[i]; emit_ph2_ir(ph2_ir); } } diff --git a/src/arm.c b/src/arm.c index ee101317..08949bd4 100644 --- a/src/arm.c +++ b/src/arm.c @@ -218,16 +218,16 @@ int __srl(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) rm + (1 << 4) + (1 << 5) + (rs << 8)); } -int __srl_amt(arm_cond_t cond, - int s, - shift_type shift, - arm_reg rd, - arm_reg rm, - int amt) -{ - return arm_encode(cond, s + (arm_mov << 1) + (0 << 5), 0, rd, - rm + (0 << 4) + (shift << 5) + (amt << 7)); -} +#define DEFINE_SHIFT_IMMEDIATE(name) \ + int name(arm_cond_t cond, int s, shift_type shift, arm_reg rd, arm_reg rm, \ + int amt) \ + { \ + return arm_encode(cond, s + (arm_mov << 1) + (0 << 5), 0, rd, \ + rm + (0 << 4) + (shift << 5) + (amt << 7)); \ + } +DEFINE_SHIFT_IMMEDIATE(__srl_amt) +DEFINE_SHIFT_IMMEDIATE(__sll_amt) +#undef DEFINE_SHIFT_IMMEDIATE int __sll(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) { @@ -235,17 +235,6 @@ int __sll(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) rm + (1 << 4) + (0 << 5) + (rs << 8)); } -int __sll_amt(arm_cond_t cond, - int s, - shift_type shift, - arm_reg rd, - arm_reg rm, - int amt) -{ - return arm_encode(cond, s + (arm_mov << 1) + (0 << 5), 0, rd, - rm + (0 << 4) + (shift << 5) + (amt << 7)); -} - int __sra(arm_cond_t cond, arm_reg rd, arm_reg rm, arm_reg rs) { /* Arithmetic right shift with register Bit 4 = 1 (register-specified shift) diff --git a/src/arm64-codegen.c b/src/arm64-codegen.c index dcbbe435..60499fbc 100644 --- a/src/arm64-codegen.c +++ b/src/arm64-codegen.c @@ -17,6 +17,56 @@ */ #define A64_STACK_ALIGN 16 #define A64_SAVE_BYTES 48 +void emit(int insn); +void emit_ph2_ir(ph2_ir_t *ir); +void a64_mov_imm(int d, int v); +void a64_mov_sp(int d); +void a64_mem(int access, int size, int rt, int rn, int ofs); +void a64_extend(int d, int n, int size, bool is_unsigned); +void a64_bl_addr(int target); + +/* Offset planning counts the same instruction stream used for emission. */ +static bool a64_count_only; +static int a64_counted_words; + +static void arm64_halfword_counts(unsigned long long value, + int *zero_words, + int *one_words) +{ + *zero_words = 0; + *one_words = 0; + for (int shift = 0; shift < 64; shift += 16) { + unsigned int word = (unsigned int) (value >> shift) & 0xffffU; + *zero_words += word != 0; + *one_words += word != 0xffffU; + } +} + +static void emit_arm64_wide_const(int reg, unsigned long long value) +{ + int zero_words, one_words, first = -1; + bool use_movn; + + arm64_halfword_counts(value, &zero_words, &one_words); + use_movn = one_words < zero_words; + for (int shift = 0; shift < 64; shift += 16) { + unsigned int word = (unsigned int) (value >> shift) & 0xffffU; + bool differs = use_movn ? word != 0xffffU : word != 0; + + if (!differs) + continue; + if (first < 0) { + first = shift / 16; + emit(use_movn ? a64_movn_insn(true, reg, (~word) & 0xffffU, first) + : a64_movz_insn(true, reg, word, first)); + } else { + emit(a64_movk_insn(true, reg, word, shift / 16)); + } + } + if (first < 0) + emit(use_movn ? a64_movn_insn(true, reg, 0, 0) + : a64_movz_insn(true, reg, 0, 0)); +} int a64_reg(int r) { @@ -24,7 +74,12 @@ int a64_reg(int r) } void emit(int insn) { - elf_write_int(elf_code, insn); + if (a64_count_only) { + if (a64_counted_words < INT_MAX) + a64_counted_words++; + } else { + elf_write_int(elf_code, insn); + } } void a64_mov(int d, int n) { @@ -44,6 +99,9 @@ void a64_mov_imm(int d, int v) * compiler. A negative C int must therefore become a sign-extended * X-register value, not 0x00000000ffffffff-style zero extension. MOVN * supplies the upper one bits while MOVK fills the second halfword. + * + * Always two instructions: callers pass segment addresses that are not + * final when code size is estimated, so the length cannot depend on v. */ if (v < 0) emit(a64_movn_insn(true, d, ~v, 0)); @@ -52,30 +110,23 @@ void a64_mov_imm(int d, int v) emit(a64_movk_insn(true, d, v >> 16, 1)); } -/* A 32-bit unsigned constant has zeroes above bit 31 when it later widens to an - * X-register scalar. The signed helper intentionally uses MOVN for a negative - * int, so keep this spelling separate. - */ -void a64_mov_imm_unsigned(int d, int v) -{ - unsigned int value = v; - - emit(a64_movz_insn(true, d, value, 0)); - emit(a64_movk_insn(true, d, value >> 16, 1)); -} - -/* Phase-2 constants retain their upper word in src1. Materialise all four - * halfwords for an eight-byte scalar rather than silently discarding it in the - * ordinary int helper above. +/* A phase-2 constant is final before sizes are estimated, so unlike an address + * it takes only the halfwords it needs; update_elf_offset() counts this very + * emission. An eight-byte constant keeps its upper word in src1, and a smaller + * one widens to the X register by its C signedness. */ -void a64_mov_imm_wide(int d, int lo, int hi) +static void a64_load_constant(int d, const ph2_ir_t *p) { - unsigned int low = lo, high = hi; + unsigned long long value; - emit(a64_movz_insn(true, d, low, 0)); - emit(a64_movk_insn(true, d, low >> 16, 1)); - emit(a64_movk_insn(true, d, high, 2)); - emit(a64_movk_insn(true, d, high >> 16, 3)); + if (p->size_bytes == 8) + value = (unsigned int) p->src0 | + (unsigned long long) (unsigned int) p->src1 << 32; + else if (p->is_unsigned) + value = (unsigned int) p->src0; + else + value = (unsigned long long) (long long) p->src0; + emit_arm64_wide_const(d, value); } /* Rd = Rn / Rm, unsigned when either operand is. */ @@ -120,19 +171,6 @@ void a64_sxtw(int d, int n) emit(a64_sext_insn(d, n, 32)); } -/* Xd = Xn +/- sign_extend(Wm). The extended-register forms fold the widening of - * an int index into the address arithmetic that consumes it, so pointer - * arithmetic costs the same one instruction as any other add. - */ -void a64_add_sxtw(int d, int n, int m) -{ - emit(a64_add_ext_insn(d, n, m, A64_EXT_SXTW)); -} -void a64_sub_sxtw(int d, int n, int m) -{ - emit(a64_sub_ext_insn(d, n, m, A64_EXT_SXTW)); -} - /* Sign-extend the low @size bytes of Xn into Xd. Always one instruction, so * update_elf_offset()'s default estimate stays correct. */ @@ -189,6 +227,8 @@ void a64_mem(int access, int size, int rt, int rn, int ofs) */ void a64_check_disp(int disp, int bits, char *what) { + if (a64_count_only) + return; int lim = 1 << (bits - 1); if (disp < -lim || disp >= lim) fatal(what); @@ -224,17 +264,6 @@ int a64_adrp_insn(int d, int pc, int target) return a64_adrp_pages_insn(d, pages); } -/* Which operand of an address expression is the int index that must widen: 0 - * for neither, 1 for src0, 2 for src1. Exactly one source being an address is - * what makes the other an index, so the two source flags decide alone. - */ -int a64_ptr_index(ph2_ir_t *p) -{ - if (p->src0_is_pointer == p->src1_is_pointer) - return 0; - return p->src0_is_pointer ? 2 : 1; -} - /* A comparison is as wide as the values it compares. An address has to be * compared whole, an array included, since what reaches the instruction is its * decayed base. The source flags say so and is_pointer does not: it counts @@ -263,66 +292,33 @@ a64_cond_t a64_cond(opcode_t op, bool is_unsigned) } } -/* All estimates below exactly match emit_ph2_ir(). Fixed-width AArch64 makes - * this deliberately simpler and more reliable than a relocation pass. +static int arm64_count_ph2_ir_words(ph2_ir_t *ir) +{ + char *saved_fatal_function_context = fatal_function_context; + a64_count_only = true; + a64_counted_words = 0; + emit_ph2_ir(ir); + int words = a64_counted_words; + a64_count_only = false; + + fatal_function_context = saved_fatal_function_context; + return words; +} + +/* Count the actual instruction sequence so offset planning stays in step with + * the emitter as it changes. */ void update_elf_offset(ph2_ir_t *ir) { - int n = 1; - switch (ir->op) { - case OP_allocat: - n = 0; - break; - case OP_assign: - n = ir->dest == ir->src0 ? 0 : 1; - break; - case OP_load_constant: - case OP_load_data_address: - case OP_load_rodata_address: - n = ir->op == OP_load_constant && ir->size_bytes == 8 ? 4 : 2; - break; - case OP_address_of: - case OP_global_address_of: - n = 3; - break; - case OP_load: - case OP_global_load: - n = a64_mem_count(ir->size_bytes, ir->src0); - break; - case OP_store: - case OP_global_store: - n = a64_mem_count(ir->size_bytes, ir->src1); - break; - case OP_address_of_func: - n = 3; - break; - case OP_return: - n = (ir->src0 >= 0 && a64_reg(ir->src0) != 0) ? 8 : 7; - break; - case OP_branch: - n = 2; - break; - case OP_mod: - n = 2; - break; - case OP_eq: - case OP_neq: - case OP_gt: - case OP_lt: - case OP_geq: - case OP_leq: - case OP_log_not: - n = 2; - break; - default: - break; - } - elf_offset += n * 4; + if (ir->op != OP_allocat) + elf_offset += arm64_count_ph2_ir_words(ir) * 4; } void cfg_flatten(void) { func_t *f; + ph2_ir_prepare(false); + bool has_main = MAIN_BB != NULL; /* Entry sequence lengths, which the block offsets below start after. * Static: 12 instructions of setup, then the 9-instruction __syscall @@ -342,13 +338,12 @@ void cfg_flatten(void) else elf_offset = 21 * 4; GLOBAL_FUNC->bbs->elf_offset = elf_offset; - for (ph2_ir_t *p = GLOBAL_FUNC->bbs->ph2_ir_list.head; p; p = p->next) - update_elf_offset(p); + ph2_ir_visit_globals(update_elf_offset); /* code_generate() emits the call to main only when there is one, so a * translation unit without main must not be charged for it either. */ - if (MAIN_BB) { + if (has_main) { int global_frame = ALIGN_UP(GLOBAL_FUNC->stack_size, A64_STACK_ALIGN); if (dynlink) elf_offset += (10 + a64_mem_count(8, global_frame) + @@ -372,37 +367,22 @@ void cfg_flatten(void) */ int stack_top_ofs = ALIGN_UP(f->stack_size, A64_STACK_ALIGN) + A64_SAVE_BYTES; - for (basic_block_t *b = f->bbs; b; b = b->rpo_next) { - b->elf_offset = elf_offset; - - /* The entry block's offset deliberately names the prologue so calls - * enter at a valid function entry. Later block labels must however - * account for that prologue before their first IR word. - */ - if (b == f->bbs) { - elf_offset += 7 * 4; - if (dynlink && !strcmp(f->return_def.var_name, "main")) - elf_offset += 3 * 4; - } - for (ph2_ir_t *p = b->ph2_ir_list.head; p; p = p->next) { - if (p->ofs_based_on_stack_top) { - if (p->op == OP_load || p->op == OP_address_of) - p->src0 += stack_top_ofs; - else if (p->op == OP_store) - p->src1 += stack_top_ofs; - } - ph2_ir_t *q = add_existed_ph2_ir(p); - if (q->op == OP_return) - q->src1 = f->stack_size; - update_elf_offset(q); - } - } + int prologue_bytes = arm64_count_ph2_ir_words(d) * 4; + ph2_ir_flatten_function(f, stack_top_ofs, prologue_bytes, 0, false, + update_elf_offset); } } void emit_ph2_ir(ph2_ir_t *p) { int d = a64_reg(p->dest), n = a64_reg(p->src0), m = a64_reg(p->src1); + if (op_is_comparison(p->op)) { + emit(a64_cmp_reg_insn(a64_cmp_wide(p), n, m)); + emit(a64_cset_insn( + a64_cmp_wide(p), d, + a64_cond(p->op, p->src0_is_unsigned || p->src1_is_unsigned))); + return; + } switch (p->op) { case OP_define: { bool reload_global_base = dynlink && !strcmp(p->func_name, "main"); @@ -434,12 +414,7 @@ void emit_ph2_ir(ph2_ir_t *p) return; } case OP_load_constant: - if (p->size_bytes == 8) - a64_mov_imm_wide(d, p->src0, p->src1); - else if (p->is_unsigned) - a64_mov_imm_unsigned(d, p->src0); - else - a64_mov_imm(d, p->src0); + a64_load_constant(d, p); return; case OP_assign: if (d != n) @@ -472,31 +447,10 @@ void emit_ph2_ir(ph2_ir_t *p) a64_mem(A64_STORE, p->dest, m, n, 0); return; case OP_add: - /* Index expressions are int-valued, so the index has to widen before it - * joins a 64-bit address; otherwise -1 becomes +4294967295 on LP64. - * Addition is commutative, so either operand may be the index. - * - * The fall-through stays the X form for both cases a64_ptr_index() - * reports as 0: two addresses, which is pointer minus pointer and - * genuinely 64-bit, and two ints, whose upper half no reader looks at. - * See the width note at the comparisons below. - */ - if (a64_ptr_index(p) == 2) - a64_add_sxtw(d, n, m); - else if (a64_ptr_index(p) == 1) - a64_add_sxtw(d, m, n); - else - a64_add(d, n, m); + a64_add(d, n, m); return; case OP_sub: - /* Only pointer-minus-int widens: int-minus-pointer is not an address - * expression, and pointer-minus-pointer is already 64-bit on both - * sides. - */ - if (a64_ptr_index(p) == 2) - a64_sub_sxtw(d, n, m); - else - a64_sub(d, n, m); + a64_sub(d, n, m); return; case OP_mul: emit(a64_mul_insn(p->size_bytes == 8, d, n, m)); @@ -532,17 +486,6 @@ void emit_ph2_ir(ph2_ir_t *p) case OP_bit_not: emit(a64_mvn_insn(p->size_bytes == 8, d, n)); return; - case OP_eq: - case OP_neq: - case OP_gt: - case OP_lt: - case OP_geq: - case OP_leq: - emit(a64_cmp_reg_insn(a64_cmp_wide(p), n, m)); - emit(a64_cset_insn( - a64_cmp_wide(p), d, - a64_cond(p->op, p->src0_is_unsigned || p->src1_is_unsigned))); - return; /* Width follows the operand, exactly as the comparisons above do. A pointer * or a long long must be tested whole, or one whose low word happens to be @@ -589,6 +532,7 @@ void emit_ph2_ir(ph2_ir_t *p) func_t *f = find_func(p->func_name); if (!f) fatal("arm64 call to unknown function"); + if (!f->bbs) { if (!dynlink) fatal("arm64 external call requires --dynlink"); @@ -701,6 +645,8 @@ void plt_generate(void) void code_generate(void) { int global_frame = ALIGN_UP(GLOBAL_FUNC->stack_size, A64_STACK_ALIGN); + bool has_main = MAIN_BB != NULL; + int main_offset = has_main ? MAIN_BB->elf_offset : -1; /* At Linux process entry argc is at [sp] and argv begins at sp + 8; they * are loaded into x20 and x21. Those two are allocatable by the global @@ -761,13 +707,12 @@ void code_generate(void) emit(a64_svc_insn(0)); emit(a64_ret_insn()); } - for (ph2_ir_t *p = GLOBAL_FUNC->bbs->ph2_ir_list.head; p; p = p->next) - emit_ph2_ir(p); - if (MAIN_BB) { + ph2_ir_visit_globals(emit_ph2_ir); + if (has_main) { if (dynlink) { a64_mem(A64_LOAD, 8, 23, A64_SP, global_frame); a64_mem(A64_LOAD, 8, 24, A64_SP, global_frame + 8); - a64_mov_imm(0, elf_code_start + MAIN_BB->elf_offset); + a64_mov_imm(0, elf_code_start + main_offset); a64_mov(1, 23); a64_mov(2, 24); a64_mov(3, A64_ZR); @@ -779,7 +724,7 @@ void code_generate(void) } else { a64_mov(0, 23); a64_mov(1, 24); - emit(a64_bl_insn((MAIN_BB->elf_offset - elf_code->size) / 4)); + emit(a64_bl_insn((main_offset - elf_code->size) / 4)); a64_mov_imm(8, 93); emit(a64_svc_insn(0)); } diff --git a/src/arm64.c b/src/arm64.c index da61e83b..35b09bad 100644 --- a/src/arm64.c +++ b/src/arm64.c @@ -89,6 +89,18 @@ int a64_sub_imm_insn(bool sf, int rd, int rn, int imm12) return a64_sf(sf) | 0x51000000 | ((imm12 & 0xfff) << 10) | (rn << 5) | rd; } +int a64_add_sub_imm_insn(bool sub, + bool sf, + int rd, + int rn, + int imm12, + bool shift) +{ + return (sub ? a64_sub_imm_insn(sf, rd, rn, imm12) + : a64_add_imm_insn(sf, rd, rn, imm12)) | + (shift << 22); +} + /* CMP #imm is SUBS into the zero register. */ int a64_cmp_imm_insn(bool sf, int rn, int imm12) { diff --git a/src/defs.h b/src/defs.h index a3be7731..9054bf41 100644 --- a/src/defs.h +++ b/src/defs.h @@ -75,12 +75,11 @@ * this does not inflate scalar type storage. */ #define MAX_FIELDS 128 -#define MAX_TYPES 256 +/* The self-hosting compiler has outgrown the original 256-entry table. */ +#define MAX_TYPES 512 #define MAX_LABELS 256 /* Elements captured from an implicitly sized array initializer. */ #define MAX_IMPLICIT_ARRAY 256 -/* A self-compile emits ~101k ph2_ir; one pointer per slot in PH2_IR_FLATTEN. */ -#define MAX_IR_INSTR 262144 #define MAX_BB_PRED 128 #define MAX_BB_DOM_SUCC 64 #define MAX_CODE 262144 @@ -99,7 +98,6 @@ #define MAX_RELAPLT 1024 #define MAX_PLT 1024 #define MAX_GOTPLT 1024 -#define MAX_CONSTANTS 1024 #define MAX_NESTING 128 /* How many instructions an if may speculate when flattened into a select, and @@ -128,15 +126,13 @@ */ #define DEFAULT_ARENA_SIZE 262144 /* 256 KiB - standard default */ #define SMALL_ARENA_SIZE 65536 /* 64 KiB - for small allocations */ -#define LARGE_ARENA_SIZE 524288 /* 512 KiB - for instruction arena */ +#define LARGE_ARENA_SIZE 524288 /* 512 KiB - for token storage */ #define DEFAULT_FUNCS_SIZE 64 #define DEFAULT_SRC_FILE_COUNT 8 /* Arena compaction bitmask flags for selective memory reclamation */ #define COMPACT_ARENA_BLOCK 0x01 /* BLOCK_ARENA - variables/blocks */ -#define COMPACT_ARENA_INSN 0x02 /* INSN_ARENA - instructions */ #define COMPACT_ARENA_BB 0x04 /* BB_ARENA - basic blocks */ -#define COMPACT_ARENA_HASHMAP 0x08 /* HASHMAP_ARENA - hash nodes */ #define COMPACT_ARENA_GENERAL 0x10 /* GENERAL_ARENA - misc allocations */ #define ELF_START 0x10000 @@ -241,7 +237,7 @@ * how many nesting levels still multiply it. */ #define LOOP_USE_WEIGHT 8 -#define MAX_WEIGHTED_LOOP_DEPTH 3 +#define MAX_WEIGHTED_LOOP_DEPTH 4 /* How many registers at the top of the allocator's file a call preserves. Only * such a register can hold a value across a call, so only these may be given to @@ -268,27 +264,49 @@ typedef struct arena_block { typedef struct { arena_block_t *head; int total_bytes; /* Track total allocation for profiling */ + int peak_bytes; /* High-water payload capacity before compaction. */ int block_size; /* Default block size for new blocks */ } arena_t; +typedef struct { + arena_block_t *head; + int offset; +} arena_mark_t; + +arena_mark_t arena_mark(arena_t *arena); +int arena_bytes_since(arena_t *arena, arena_mark_t mark); +/* Discard allocations after a live mark; return false if the mark is stale. */ +bool arena_rewind(arena_t *arena, arena_mark_t mark); + /* string-based hash map definitions */ typedef struct hashmap_node { char *key; void *val; bool occupied; + bool owns_key; } hashmap_node_t; typedef struct { int size; int cap; hashmap_node_t *table; + bool has_owned_keys; } hashmap_t; +hashmap_t *hashmap_create(int cap); +void hashmap_put(hashmap_t *map, char *key, void *val); +void hashmap_put_borrowed(hashmap_t *map, char *key, void *val); +hashmap_node_t *hashmap_get_node(hashmap_t *map, char *key); +void *hashmap_get(hashmap_t *map, char *key); +bool hashmap_contains(hashmap_t *map, char *key); +void hashmap_free(hashmap_t *map); + /* lexer tokens */ typedef enum { T_start, /* FIXME: Unused, intended for lexer state machine init */ - T_eof, /* end-of-file (EOF) */ + T_translation_unit, /* internal boundary between source input files */ + T_eof, /* end-of-file (EOF) */ T_numeric, T_floating, /* C99 floating literal; lowering is staged separately */ T_identifier, @@ -430,6 +448,9 @@ typedef struct token_stream { token_t *tail; } token_stream_t; +bool numeric_suffix_is_valid(const char *suffix); +int numeric_literal_base(const char *literal); + /* String pool for identifier deduplication */ typedef struct { hashmap_t *strings; /* Map string -> interned string */ @@ -461,8 +482,6 @@ typedef enum { /* intermediate use in front-end. No code generation */ OP_generic, - OP_phi, - OP_unwound_phi, /* work like address_of + store */ /* rd = rs2 ? rs1 : rs3 -- select without a branch. */ OP_cmov, @@ -473,7 +492,8 @@ typedef enum { OP_indirect, /* indirect call with function pointer */ OP_return, /* explicit return */ - OP_allocat, /* allocate space on stack */ + OP_va_start, /* address of the first unnamed ABI argument */ + OP_allocat, /* allocate space on stack */ OP_assign, OP_load_constant, /* load constant */ OP_load_data_address, /* lookup address of a constant in data section */ @@ -483,7 +503,6 @@ typedef enum { OP_branch, /* conditional jump */ OP_jump, /* unconditional jump */ OP_func_ret, /* returned value */ - OP_label, /* for goto label */ /* function pointer */ OP_address_of_func, /* resolve function entry */ @@ -533,39 +552,17 @@ typedef enum { OP_start } opcode_t; -/* variable definition */ - -/* Depth of the SSA renaming stack: how many definitions of one variable can be - * live along a single dominator path. - */ -#define MAX_RENAME_STACK 64 - -typedef struct { - int counter; - - /* Grown on demand: only a base variable is ever renamed, so the SSA - * versions copied from it -- the large majority of all variables -- would - * otherwise each carry an unused MAX_RENAME_STACK array. - */ - int *stack; - int stack_idx; - int stack_cap; -} rename_t; - -typedef struct ref_block ref_block_t; - -struct ref_block_list { - ref_block_t *head, *tail; -}; - -typedef struct ref_block_list ref_block_list_t; - -typedef struct insn insn_t; +#define OP_USUAL_ARITHMETIC_CASES \ + case OP_add: \ + case OP_sub: \ + case OP_mul: \ + case OP_div: \ + case OP_mod: \ + case OP_bit_and: \ + case OP_bit_or: \ + case OP_bit_xor -typedef struct use_chain_node { - insn_t *insn; - struct use_chain_node *next, *prev; -} use_chain_t; +/* variable definition */ typedef struct var var_t; typedef struct type type_t; @@ -589,8 +586,14 @@ struct var { * intern_string(). Never NULL -- an unnamed variable holds "". */ char *var_name; + + /* Original C spelling when an internal-linkage name is mangled for the + * merged object/function tables. NULL for names emitted as-is. + */ + char *source_name; int ptr_level; bool is_func; + bool is_function_designator; /* A pointer-to-callback object is not callable itself. Its compact callback * pointee signature is restored only after one dereference. @@ -643,43 +646,15 @@ struct var { bool has_initializer; /* a file-scope definition supplied an initializer */ bool is_const_qualified; /* true if variable has const qualifier */ bool is_volatile; /* declaration used the volatile qualifier */ + bool is_volatile_access; /* computed address designates volatile storage */ bool is_const_pointer; /* true for the outermost `* const` qualifier */ /* One bit per pointer level, counted from the base type. The legacy * is_const_pointer flag describes only the outermost level; this retains * qualifiers on intermediate pointers such as `int * const *`. */ unsigned int pointer_const_mask; + unsigned int pointer_volatile_mask; bool address_taken; /* true if variable address was taken (&var) */ - /* Working state for strength_reduce(): how many instructions in the - * function write the variable, whether it is written inside the loop being - * examined, and how much its value moves per iteration when it does. All - * three are recomputed per loop; nothing outside that pass reads them. - */ - int def_cnt; - int loop_stamp; - int iv_gen; - int iv_step; - - /* pin_registers()'s tally for the variable: what its namings are worth - * weighted by loop depth, the reverse-post-order number of the last block - * that named it, whether it was ever named in two, and whether a loop named - * it. Stamped per function so that no array has to hold the candidates -- - * the file has a handful of registers and a function names hundreds of - * variables, and the one worth a register is not reliably among the first - * few met. - */ - int pin_gen; - int pin_weight; - int pin_blk; - bool pin_cross; - bool pin_hot; - - /* Defined inside an arm that if_convert() flattened into a select. The - * register its variable is pinned to still holds the value flowing into the - * select, which the arms read and the select overwrites, so a value - * computed on the way there must go somewhere else. - */ - bool in_select_arm; int array_size; bool has_direct_array_declarator; bool has_unsized_array; /* `T name[]`: bound is supplied by initializer */ @@ -706,6 +681,8 @@ struct var { */ int pointee_array_element_ptr_level; int offset; /* offset from stack or frame, index 0 is reserved */ + int vir_source_index; + /* Record bit-field metadata. `offset` is the containing storage unit's byte * offset. `is_bitfield` distinguishes an ordinary member from the valid * unnamed zero-width field used as an allocation-unit barrier. @@ -716,37 +693,6 @@ struct var { int bit_storage_size; int init_val; /* for global initialization */ int init_val_hi; /* upper word of an 8-byte integer constant */ - /* Generation stamps used by compute_live_in() to test set membership in - * constant time instead of rescanning live_kill and live_in per element. - */ - int kill_gen; - int in_gen; - /* Stamp for the successor-union set merge_live_in() builds. */ - int merge_gen; - struct var *base; - int subscript; - - /* Every SSA version of this variable, grown on demand. A fixed 128-entry - * array made every var_t 1 KiB heavier -- and var_t is embedded by value in - * type_t's field table and in every function's parameter list -- while the - * append was unchecked, so a variable assigned more than 128 times in one - * function wrote past the end. - */ - struct var **subscripts; - int subscripts_idx; - int subscripts_cap; - - /* SSA renaming state, allocated on first use by var_rename(). Only a base - * variable is ever renamed, so the versions copied from it -- the large - * majority of all variables -- each carried an unused 24-byte rename_t - * inline. Field order here is load-bearing: reordering var_t breaks the - * bootstrap, so the pointer stays where the struct sat. - */ - rename_t *rename; - ref_block_list_t ref_block_list; /* blocks which kill variable */ - use_chain_t *users_head, *users_tail; - struct insn *last_assign; - int consumed; bool is_ternary_ret; bool is_logical_ret; bool is_const; /* whether a constant representaion or not */ @@ -757,22 +703,9 @@ struct var { * nothing uses the value. */ bool is_assignment_reload; - int phys_reg; /* low physical register (-1 if unassigned) */ - /* The high word of a 32-bit-target wide scalar. It remains -1 for the - * ordinary single-register representation and on LP64 targets. - */ - int phys_reg_hi; - int first_use; /* First instruction index where variable is used */ - int last_use; /* Last instruction index where variable is used */ - int use_count; /* Number of times variable is used */ bool space_is_allocated; /* whether space is allocated for this variable */ bool has_backing_storage; - /* This flag is used to indicate to the compiler that the offset of the - * variable is based on the top of the local stack. - */ - bool ofs_based_on_stack_top; - /* True when this variable was synthesized to hold a compound literal (e.g., * array or struct literal temporaries). */ @@ -857,8 +790,9 @@ typedef struct typedef_binding { struct block { var_list_t locals; - /* C tags and enumeration constants have lexical, not translation-unit, - * scope. Variables remain in locals; these lists serve parser lookups. + /* Tags and enumerators follow C lexical scope. File-scope tag/enum bindings + * live on the active translation-unit root; typedef bindings remain on the + * migration path through TYPES. Variables remain in locals. */ void *type_tags; void *constants; @@ -874,6 +808,7 @@ int read_const_sizeof_type(block_t *scope); int read_const_wstring_size(void); int read_sizeof_constant(block_t *scope); int read_const_expr_operand(block_t *scope); +int alignment_type(type_t *type); void add_block_typedef(block_t *block, char name[], type_t *type); bool find_block_typedef(block_t *block, const char *name); type_t *find_visible_type(const char *name, block_t *block); @@ -1063,6 +998,7 @@ struct type { * subsequently written at a use site. */ unsigned int pointer_const_mask; + unsigned int pointer_volatile_mask; bool is_union; /* preserves union semantics for anonymous typedef unions */ bool has_flexible_array_member; /* cannot be embedded by value in a record */ @@ -1143,44 +1079,6 @@ typedef struct type_tag { struct type_tag *next; } type_tag_t; -struct phi_operand { - var_t *var; - basic_block_t *from; - struct phi_operand *next; -}; - -typedef struct phi_operand phi_operand_t; - -struct insn { - struct insn *next, *prev; - int idx; - opcode_t opcode; - var_t *rd; - var_t *rs1; - var_t *rs2; - - /* The value OP_cmov keeps when its condition does not hold. A select needs - * three inputs and the two source fields are taken by the chosen value and - * the condition; every other opcode leaves this NULL. - */ - var_t *rs3; - int sz; - bool useful; /* Used in DCE process. Set true if instruction is useful. */ - basic_block_t *belong_to; - phi_operand_t *phi_ops; - - /* Callee or goto-label name, interned rather than copied. add_insn() - * already interned the text before copying it in, so the array was 64 of - * this struct's 136 bytes for a string the pool owns anyway, on all ~73k - * instructions a self-compile builds. NULL when the opcode names nothing. - */ - char *str; -}; - -typedef struct { - insn_t *head, *tail; -} insn_list_t; - typedef struct { ph2_ir_t *head, *tail; } ph2_ir_list_t; @@ -1192,31 +1090,10 @@ typedef struct { bb_connection_type_t type; } bb_connection_t; -struct symbol { - var_t *var; - int index; - struct symbol *next; -}; - -typedef struct symbol symbol_t; - -typedef struct { - symbol_t *head, *tail; -} symbol_list_t; - struct basic_block { - /* Members are grouped by width -- 16-byte lists, then pointers, then ints, - * then the flags -- so the struct carries no interior padding. Mixed in - * declaration order it was 320 bytes for 302 bytes of fields, on every one - * of the ~51k blocks a self-compile creates. - */ - insn_list_t insn_list; + struct vir_block *vir; ph2_ir_list_t ph2_ir_list; - var_list_t live_gen; - var_list_t live_kill; - var_list_t live_in; - var_list_t live_out; - symbol_list_t symbol_list; /* variable declaration */ + unsigned int machine_use, machine_def, machine_liveout; /* Predecessor edges, grown on demand. A fixed MAX_BB_PRED array cost two * kilobytes in every basic block -- by far the largest thing in one -- @@ -1224,50 +1101,10 @@ struct basic_block { */ bb_connection_t *prev; - /* Register file on entry to this block, captured by reg_alloc() at the end - * of the predecessor it falls out of. Non-NULL only when a file was handed - * over, and bb_export_regs() does that only for an edge that is all three - * of: the predecessor's single successor, that predecessor's rpo_next, and - * this block's single predecessor. A sole predecessor alone is NOT enough - * -- a branch target is emitted wherever the backend's linear walk puts it, - * so the registers reaching it are not the ones the branch left. - * - * Allocated only for a block that actually inherits a file: fewer than one - * block in thirteen does, so a REG_CNT array here cost 64 bytes on all ~51k - * blocks to serve 7% of them. - */ - struct var **entry_regs; - - /* Used in instruction dumping when ir_dump is enabled, and allocated only - * then: a fixed array here is 128 bytes on every one of the tens of - * thousands of blocks a self-compile creates, all of it zeroed for nothing - * in the default path. - */ - char *bb_label_name; struct basic_block *next; /* normal BB */ struct basic_block *then_; /* conditional BB */ struct basic_block *else_; - struct basic_block *idom; - struct basic_block *r_idom; struct basic_block *rpo_next; - struct basic_block *rpo_r_next; - - /* Dominance and reverse-dominance frontiers. These were fixed worst-case - * arrays sized MAX_BB_DOM_SUCC / MAX_BB_RDOM_SUCC, which cost 2560 bytes in - * every basic block while a typical block uses a handful of entries. Worse, - * the appends were unchecked and a self-compile really does push df_idx to - * 72, overrunning a 64-entry DF into the RDF that followed it. Growing them - * on demand removes both the waste and the fixed ceiling. - */ - struct basic_block **DF; - struct basic_block **RDF; - - /* Dominator-tree children, grown on demand for the same reason as prev[] - * and the frontiers: a fixed array cost half a kilobyte in every block. - */ - struct basic_block **dom_next; - struct basic_block *dom_prev; - struct basic_block *rdom_prev; func_t *belong_to; block_t *scope; @@ -1287,34 +1124,11 @@ struct basic_block { */ int ph2_base; int rpo; - int rpo_r; - - /* How many loops enclose the block. pin_registers() weights a variable's - * uses by it: a name inside a loop stands for as many reads as the loop has - * iterations, and ranking by the plain count gave a register to a variable - * named four times in straight-line code over one named once in the - * innermost loop. - */ - int loop_depth; - - /* What one naming of a variable in this block is worth to pin_registers(), - * derived from loop_depth once rather than on every operand of every - * instruction the tally walks. - */ - int loop_weight; + int elf_offset; - /* Stamp marking the block as already counted for the loop being walked, so - * that one loop raises its depth once however many ways in there are. + /* VIR block-map index plus one; zero means this parser block is unmapped. */ - int loop_mark; - int df_idx; - int rdf_idx; - int df_cap; - int rdf_cap; - int visited; - int dom_next_idx; - int dom_next_cap; - int elf_offset; + int vir_map_id; /* Whether any emitted branch or jump names this block as its target. A * block no edge jumps to is reached only by falling out of the block @@ -1322,19 +1136,8 @@ struct basic_block { * registers hold across the boundary. */ bool is_branch_target; - bool useful; /* indicate whether this BB contains useful instructions */ }; -struct ref_block { - basic_block_t *bb; - struct ref_block *next; -}; - -/* Syntactic representation of func, combines syntactic details (e.g., return - * type, parameters) with SSA-related information (e.g., basic blocks, control - * flow) to support parsing, analysis, optimization, and code generation. - */ - typedef struct { char label_name[MAX_ID_LEN]; basic_block_t *bb; @@ -1371,35 +1174,17 @@ struct func { */ bool is_inline; - /* inline_calls()'s verdict on this body and the return that ends it, - * stamped with the round that reached them: a body is examined once per - * round rather than once per call site that names it. - */ - int inline_gen; - bool inline_ok; - struct insn *inline_ret; int stack_size; + void *vir_context; - /* SSA info */ + /* Parser cursors become the allocated machine CFG after lowering. */ basic_block_t *bbs; basic_block_t *exit; - symbol_list_t global_sym_list; - int bb_cnt; int visited; - /* How many callee-saved registers this function's prologue must preserve, - * counted from RBX upward. Functions that never need them pay nothing. - */ + /* Callee-saved registers the backend must preserve. */ int saved_regs; - /* Registers holding a variable for the whole function, one bit each. The - * backend needs these: its notion of what is live out of a block comes from - * the successor's entry registers, which say nothing about a variable that - * is resident everywhere, and it would otherwise drop the code that puts a - * value into one as dead. - */ - int pinned_regs; - /* Information used for dynamic linking */ bool is_used; /* A direct aggregate-return call requires shecc's private sret ABI. */ @@ -1413,17 +1198,25 @@ typedef struct { func_t *head, *tail; } func_list_t; -typedef struct { - func_t *func; - basic_block_t *bb; - void (*preorder_cb)(func_t *, basic_block_t *); - void (*postorder_cb)(func_t *, basic_block_t *); -} bb_traversal_args_t; +/* Iterate over functions with bodies, skipping declarations. */ +#define FOR_EACH_FUNCTION_BODY(func) \ + for (func_t *func = FUNC_LIST.head; func; func = func->next) \ + if ((func)->bbs) + +/* Load a two-word value without clobbering its address register. */ +#define EMIT_PAIR_LOAD(load, rd, rd_hi, base) \ + do { \ + if ((rd) == (base)) { \ + load(rd_hi, base, 4); \ + load(rd, base, 0); \ + } else { \ + load(rd, base, 0); \ + load(rd_hi, base, 4); \ + } \ + } while (0) -typedef struct { - var_t *var; - int polluted; -} regfile_t; +#define PH2_PAIR_DEST_SRC0(ir) ((ir)->dest_hi >= 0 && (ir)->src0_hi >= 0) +#define PH2_PAIR_BINARY(ir) (PH2_PAIR_DEST_SRC0(ir) && (ir)->src1_hi >= 0) /* In the ELF specification, the following data types are defined for data * representation: @@ -1555,4 +1348,27 @@ typedef struct { int r_addend; /* Elf32_Sword */ } elf32_rela_t; +int ph2_ir_flatten_function(func_t *func, + int stack_top_ofs, + int prologue_bytes, + int saved_regs, + bool restore_saved_regs, + void (*update_offset)(ph2_ir_t *)); +ph2_ir_t *ph2_ir_flatten_insn(ph2_ir_t *insn, + int stack_top_ofs, + int stack_size, + int saved_regs, + bool restore_saved_regs); +int func_highest_used_reg(func_t *func, int first_callee_saved); +bool op_is_binary_alu(opcode_t op); +bool op_writes_dest(opcode_t op); +unsigned int ph2_ir_defs(ph2_ir_t *ir); +void ph2_compute_liveness(void); +bool op_is_integer_binary(opcode_t op); +bool op_is_comparison(opcode_t op); +bool op_is_scalar_unary(opcode_t op); +bool op_is_scalar_cast(opcode_t op); +const char *opcode_mnemonic(opcode_t op); +basic_block_t *bb_sole_pred(const basic_block_t *bb); + #define ELF32_ST_INFO(b, t) (((b) << 4) + ((t) & 0xf)) diff --git a/src/globals.c b/src/globals.c index 3ba7e0c9..19772c1f 100644 --- a/src/globals.c +++ b/src/globals.c @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include #include @@ -21,6 +22,8 @@ __noreturn void limit_error(char *msg); /* Forward declaration for string interning */ char *intern_string(char *str); +func_t *find_func(char *func_name); +__noreturn void fatal(const char *msg); /* Lexer */ token_t *cur_token; @@ -32,12 +35,54 @@ strbuf_t *LIBC_SRC; hashmap_t *SRC_FILE_MAP; hashmap_t *FUNC_MAP; -hashmap_t *CONSTANTS_MAP; /* Types */ type_t *TYPES; int types_idx = 0; +int builtin_types_idx; +static unsigned int internal_linkage_name_idx; + +static const char *file_scope_var_name(const var_t *var) +{ + return var->source_name ? var->source_name : var->var_name; +} + +static void assign_internal_linkage_name(var_t *var, const char *source_name) +{ + char name[48]; + + var->source_name = intern_string((char *) source_name); + snprintf(name, sizeof(name), "__shecc_internal_%u", + internal_linkage_name_idx++); + var->var_name = intern_string(name); +} + +static const char *function_source_name(const func_t *func) +{ + return file_scope_var_name(&func->return_def); +} + +typedef enum { + ORDINARY_NONE = 0, + ORDINARY_CONSTANT = 1, + ORDINARY_VARIABLE = 2, + ORDINARY_TYPEDEF = 4, + ORDINARY_PARAMETER = 8, + ORDINARY_ANY = 15 +} ordinary_kind_t; + +typedef struct tu_ordinary_binding { + ordinary_kind_t kind; + void *value; +} tu_ordinary_binding_t; + +static void register_tu_ordinary_name(const char *name, + ordinary_kind_t kind, + void *value); +static ordinary_kind_t lookup_tu_ordinary_name(const char *name, + int kinds, + void **value); type_t *TY_void; type_t *TY_char; @@ -58,10 +103,6 @@ type_t *TY_long_double; /* Arenas */ -arena_t *INSN_ARENA; - -/* HASHMAP_ARENA is responsible for hashmap_node_t allocation */ -arena_t *HASHMAP_ARENA; /* BLOCK_ARENA is responsible for block_t / var_t allocation */ arena_t *BLOCK_ARENA; @@ -74,23 +115,23 @@ arena_t *BB_ARENA; */ arena_t *TOKEN_ARENA; -/* GENERAL_ARENA is responsible for functions, symbols, constants, aliases, - * macros, and traversal args +/* GENERAL_ARENA is responsible for functions, symbols, constants, aliases, and + * macros */ arena_t *GENERAL_ARENA; -int bb_label_idx = 0; - ph2_ir_t **PH2_IR_FLATTEN; int ph2_ir_idx = 0; +int ph2_ir_capacity; func_list_t FUNC_LIST; func_t *GLOBAL_FUNC; block_t *GLOBAL_BLOCK; +block_t *CURRENT_TU_SCOPE; basic_block_t *MAIN_BB; int elf_offset = 0; +int translation_unit_count; -regfile_t REGS[REG_CNT]; hashmap_t *INCLUSION_MAP; @@ -114,19 +155,160 @@ int elf_bss_size; int elf_entry_offset = 0; dynamic_sections_t dynamic_sections; +/* C99 integer suffix order: [u][l|ll][u]. */ +bool numeric_suffix_is_valid(const char *suffix) +{ + int pos = 0; + + if ((suffix[pos] | 32) == 'u') + pos++; + if ((suffix[pos] | 32) == 'l') { + pos++; + if ((suffix[pos] | 32) == 'l') + pos++; + } + if ((suffix[pos] | 32) == 'u') + pos++; + return suffix[pos] == '\0'; +} + +/* The radix implied by an integer literal's leading characters. */ +int numeric_literal_base(const char *literal) +{ + if (literal[0] != '0') + return 10; + if ((literal[1] | 32) == 'x') + return 16; + if ((literal[1] | 32) == 'b') + return 2; + return 8; +} + +opcode_t operator_for_token(token_kind_t token) +{ + switch (token) { + case T_plus: + return OP_add; + case T_minus: + return OP_sub; + case T_bit_not: + return OP_bit_not; + case T_log_not: + return OP_log_not; + case T_asterisk: + return OP_mul; + case T_divide: + return OP_div; + case T_mod: + return OP_mod; + case T_lshift: + return OP_lshift; + case T_rshift: + return OP_rshift; + case T_log_and: + return OP_log_and; + case T_log_or: + return OP_log_or; + case T_eq: + return OP_eq; + case T_noteq: + return OP_neq; + case T_lt: + return OP_lt; + case T_le: + return OP_leq; + case T_gt: + return OP_gt; + case T_ge: + return OP_geq; + case T_ampersand: + return OP_bit_and; + case T_bit_or: + return OP_bit_or; + case T_bit_xor: + return OP_bit_xor; + case T_question: + return OP_ternary; + default: + return OP_generic; + } +} + +/* Shared by the parser and preprocessor's expression evaluator. */ +int get_operator_prio(opcode_t op) +{ + switch (op) { + case OP_ternary: + return 3; + case OP_log_or: + return 4; + case OP_log_and: + return 5; + case OP_bit_or: + return 6; + case OP_bit_xor: + return 7; + case OP_bit_and: + return 8; + case OP_eq: + case OP_neq: + return 9; + case OP_lt: + case OP_leq: + case OP_gt: + case OP_geq: + return 10; + case OP_lshift: + case OP_rshift: + return 11; + case OP_add: + case OP_sub: + return 12; + case OP_mul: + case OP_div: + case OP_mod: + return 13; + default: + return 0; + } +} + /* Command line compilation flags */ bool dynlink = false; bool imm_binding = false; bool libc = true; +bool optimize = true; bool expand_only = false; bool dump_ir = false; +bool dump_vir = false; bool dump_dot = false; bool hard_mul_div = false; bool warn_string_literals = false; bool strict_c99 = false; +bool dump_stats = false; char *include_dirs[MAX_INCLUDE_DIRS]; int include_dirs_idx = 0; +/* Allocation counters are a stable pre-VIR baseline even when later passes + * delete or rewrite the objects they counted. + */ +int stats_func_count = 0; +int stats_var_count = 0; +int stats_field_slot_count = 0; +int stats_bb_count = 0; +int stats_ph2_ir_count = 0; +static int stats_tu_index_lookups = 0; +static int stats_tu_index_registrations = 0; +static hashmap_t *TU_ORDINARY_MAP; + +/* Payload capacity currently retained by all live arenas, and its true + * allocation-time high water. This deliberately excludes arena headers, malloc + * metadata, and non-arena allocations. + */ +int stats_live_arena_capacity = 0; +int stats_peak_arena_capacity = 0; +int stats_retired_token_arena_peak = 0; + /* Create a new arena block with given capacity. * @capacity: The capacity of the arena block. Must be positive. * @@ -181,6 +363,10 @@ arena_t *arena_init(int initial_capacity) } arena->head = arena_block_create(initial_capacity); arena->total_bytes = initial_capacity; + arena->peak_bytes = initial_capacity; + stats_live_arena_capacity += initial_capacity; + if (stats_live_arena_capacity > stats_peak_arena_capacity) + stats_peak_arena_capacity = stats_live_arena_capacity; /* Use the initial capacity as the default block size for future growth. */ arena->block_size = initial_capacity; return arena; @@ -218,6 +404,11 @@ void *arena_alloc(arena_t *arena, int size) new_block->next = arena->head; arena->head = new_block; arena->total_bytes += new_capacity; + stats_live_arena_capacity += new_capacity; + if (stats_live_arena_capacity > stats_peak_arena_capacity) + stats_peak_arena_capacity = stats_live_arena_capacity; + if (arena->total_bytes > arena->peak_bytes) + arena->peak_bytes = arena->total_bytes; } void *ptr = arena->head->memory + arena->head->offset; @@ -225,6 +416,59 @@ void *arena_alloc(arena_t *arena, int size) return ptr; } +/* Mark consumed arena payload without charging future allocations. The mark is + * valid while its block remains in the arena's newest-to-oldest chain. + */ +arena_mark_t arena_mark(arena_t *arena) +{ + arena_mark_t mark; + + mark.head = arena ? arena->head : NULL; + mark.offset = mark.head ? mark.head->offset : 0; + return mark; +} + +/* Return payload allocated since a mark, including new blocks and growth of its + * original head block. + * + * Return -1 if the marked block is no longer present. + */ +int arena_bytes_since(arena_t *arena, arena_mark_t mark) +{ + int bytes = 0; + arena_block_t *block = arena ? arena->head : NULL; + + while (block && block != mark.head) { + bytes += block->offset; + block = block->next; + } + if (block != mark.head) + return -1; + return bytes + (mark.head ? mark.head->offset - mark.offset : 0); +} + +bool arena_rewind(arena_t *arena, arena_mark_t mark) +{ + if (!arena || !mark.head) + return false; + + arena_block_t *block = arena->head; + while (block && block != mark.head) + block = block->next; + if (!block || mark.offset > mark.head->offset) + return false; + + while (arena->head != mark.head) { + block = arena->head; + arena->head = block->next; + arena->total_bytes -= block->capacity; + stats_live_arena_capacity -= block->capacity; + arena_block_free(block); + } + arena->head->offset = mark.offset; + return true; +} + /* arena_alloc() plus explicit zero‑initialization. * @arena: The arena to allocate memory from. Must not be NULL. * @n: Number of elements. @@ -330,12 +574,16 @@ char *arena_strdup(arena_t *arena, const char *str) /* Typed allocators for consistent memory management */ func_t *arena_alloc_func(void) { - return arena_calloc(GENERAL_ARENA, 1, sizeof(func_t)); + func_t *func = arena_calloc(GENERAL_ARENA, 1, sizeof(func_t)); + + stats_func_count++; + return func; } -symbol_t *arena_alloc_symbol(void) +basic_block_t *arena_alloc_bb(void) { - return arena_calloc(GENERAL_ARENA, 1, sizeof(symbol_t)); + stats_bb_count++; + return arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); } constant_t *arena_alloc_constant(void) @@ -347,14 +595,11 @@ constant_t *arena_alloc_constant(void) return c; } -bb_traversal_args_t *arena_alloc_traversal_args(void) -{ - /* Keep using calloc for safety */ - return arena_calloc(GENERAL_ARENA, 1, sizeof(bb_traversal_args_t)); -} - void arena_free(arena_t *arena) { + if (!arena) + return; + arena_block_t *block = arena->head; arena_block_t *next; @@ -364,6 +609,7 @@ void arena_free(arena_t *arena) block = next; } + stats_live_arena_capacity -= arena->total_bytes; free(arena); } @@ -420,6 +666,7 @@ hashmap_t *hashmap_create(int cap) map->size = 0; map->cap = round_up_pow2(cap); + map->has_owned_keys = false; map->table = calloc(map->cap, sizeof(hashmap_node_t)); if (!map->table) { @@ -432,7 +679,7 @@ hashmap_t *hashmap_create(int cap) } -void hashmap_rehash(hashmap_t *map) +static void hashmap_rehash(hashmap_t *map) { if (!map) return; @@ -472,36 +719,32 @@ void hashmap_rehash(hashmap_t *map) map->table[index].key = key; map->table[index].val = val; map->table[index].occupied = true; + map->table[index].owns_key = old_table[i].owns_key; map->size++; } } free(old_table); } -/* Put a key-value pair into given hashmap. If key already contains a value, - * then replace it with new value, the old value will be freed. - * @map: The hashmap to be put into. Must not be NULL. - * @key: The key string. May be NULL. - * @val: The value pointer. May be NULL. This value's lifetime is held by - * hashmap. +/* Find an existing key or the empty slot for a new one, growing the table + * before its load factor exceeds 50%. */ -void hashmap_put(hashmap_t *map, char *key, void *val) +static hashmap_node_t *hashmap_prepare_slot(hashmap_t *map, char *key) { + int index; + int start; + if (!map) - return; + return NULL; - /* Check if size of map exceeds load factor 50% (or 1/2 of capacity) */ if ((map->cap >> 1) <= map->size) hashmap_rehash(map); - int index = hashmap_hash_index(map->cap, key); - int start = index; - + index = hashmap_hash_index(map->cap, key); + start = index; while (map->table[index].occupied) { - if (!strcmp(map->table[index].key, key)) { - map->table[index].val = val; - return; - } + if (!strcmp(map->table[index].key, key)) + return &map->table[index]; index = (index + 1) & (map->cap - 1); if (index == start) { @@ -511,15 +754,64 @@ void hashmap_put(hashmap_t *map, char *key, void *val) } } - map->table[index].key = arena_strdup(HASHMAP_ARENA, key); - map->table[index].val = val; - map->table[index].occupied = true; - map->size++; + return &map->table[index]; +} + +/* Put an owned copy of @key into the map. Replacing an existing value keeps its + * original key and ownership. + * @map: The hashmap to be put into. May be NULL. + * @key: Non-NULL key string. + * @val: The value pointer. May be NULL. The map does not own the value. + */ +static void hashmap_put_impl(hashmap_t *map, + char *key, + void *val, + bool copy_key) +{ + hashmap_node_t *node = hashmap_prepare_slot(map, key); + + if (!node) + return; + + if (!node->occupied) { + if (copy_key) { + size_t key_size = strlen(key) + 1; + char *stored_key = malloc(key_size); + + if (!stored_key) { + printf("Failed to allocate hashmap key\n"); + fflush(stdout); /* see fatal() */ + abort(); + } + memcpy(stored_key, key, key_size); + key = stored_key; + } + node->key = key; + node->occupied = true; + node->owns_key = copy_key; + map->has_owned_keys |= copy_key; + map->size++; + } + + node->val = val; +} + +void hashmap_put(hashmap_t *map, char *key, void *val) +{ + hashmap_put_impl(map, key, val, true); +} + +/* The caller must provide a non-NULL key and keep it alive for the map's + * lifetime. + */ +void hashmap_put_borrowed(hashmap_t *map, char *key, void *val) +{ + hashmap_put_impl(map, key, val, false); } /* Get key-value pair node from hashmap from given key. - * @map: The hashmap to be looked up. Must no be NULL. - * @key: The key string. May be NULL. + * @map: The hashmap to be looked up. May be NULL. + * @key: Non-NULL key string. * * Return: The look up result, if the key-value pair entry exists, then returns * address of itself, NULL otherwise. @@ -545,8 +837,8 @@ hashmap_node_t *hashmap_get_node(hashmap_t *map, char *key) } /* Get value from hashmap from given key. - * @map: The hashmap to be looked up. Must no be NULL. - * @key: The key string. May be NULL. + * @map: The hashmap to be looked up. May be NULL. + * @key: Non-NULL key string. * * Return: The look up result, if the key-value pair entry exists, then returns * its value's address, NULL otherwise. @@ -558,8 +850,8 @@ void *hashmap_get(hashmap_t *map, char *key) } /* Check if the key-value pair entry exists from given key. - * @map: The hashmap to be looked up. Must no be NULL. - * @key: The key string. May be NULL. + * @map: The hashmap to be looked up. May be NULL. + * @key: Non-NULL key string. * * Return: The look up result, if the key-value pair entry exists, then returns * true, false otherwise. @@ -569,18 +861,104 @@ bool hashmap_contains(hashmap_t *map, char *key) return hashmap_get_node(map, key); } -/* Free the hashmap, this also frees key-value pair entry's value. - * @map: The hashmap to be looked up. Must no be NULL. +/* Free the map table, any owned key copies, and the header. Mapped values have + * separate lifetimes and are not freed here. + * @map: The hashmap to be freed. May be NULL. */ void hashmap_free(hashmap_t *map) { if (!map) return; + if (map->has_owned_keys) + for (int i = 0; i < map->cap; i++) + if (map->table[i].occupied && map->table[i].owns_key) + free(map->table[i].key); free(map->table); free(map); } +void reset_tu_ordinary_index(void) +{ + hashmap_free(TU_ORDINARY_MAP); + TU_ORDINARY_MAP = hashmap_create(32); + if (!TU_ORDINARY_MAP) + fatal("Failed to allocate translation-unit ordinary-name index"); +} + +static void register_tu_ordinary_name(const char *name, + ordinary_kind_t kind, + void *value) +{ + tu_ordinary_binding_t *binding; + char *stable_name; + + if (!TU_ORDINARY_MAP) + fatal("Translation-unit ordinary-name index is not initialized"); + stable_name = intern_string((char *) name); + binding = hashmap_get(TU_ORDINARY_MAP, stable_name); + if (binding) + return; + binding = arena_alloc(BLOCK_ARENA, sizeof(*binding)); + binding->kind = kind; + binding->value = value; + hashmap_put_borrowed(TU_ORDINARY_MAP, stable_name, binding); + if (dump_stats) + stats_tu_index_registrations++; +} + +static ordinary_kind_t lookup_tu_ordinary_name(const char *name, + int kinds, + void **value) +{ + tu_ordinary_binding_t *binding; + + if (dump_stats) + stats_tu_index_lookups++; + binding = + TU_ORDINARY_MAP ? hashmap_get(TU_ORDINARY_MAP, (char *) name) : NULL; + if (!binding || !(kinds & binding->kind)) + return ORDINARY_NONE; + if (value) + *value = binding->value; + return binding->kind; +} + +static type_t *resolve_type_alias(type_t *alias) +{ + type_t *base = alias->base_struct; + + if (alias->base_type != TYPE_typedef || alias->size || alias->ptr_level || + alias->is_direct_function_type) + return alias; + + /* A zero-sized alias can be a forward declaration of a record, but a + * function type with a void return is also zero-sized and is not an alias + * to its return type. + */ + if (!base || (!alias->is_const_qualified && !alias->is_volatile_qualified)) + return base; + if (!base->size) + return alias; + + /* Keep qualifiers on the alias while filling in the completed tag's layout. + */ + alias->size = base->size; + alias->alignment = base->alignment; + alias->fields = base->fields; + alias->num_fields = base->num_fields; + alias->is_union = base->is_union; + alias->has_flexible_array_member = base->has_flexible_array_member; + return alias; +} + +/* Reject mismatched prefixes before strcmp. */ +static bool identifier_matches(const char *name, const char *query) +{ + return name[0] == query[0] && (!query[0] || name[1] == query[1]) && + !strcmp(name, query); +} + /* Find the type by the given name. * @type_name: The name to be searched. * @flag: @@ -592,95 +970,188 @@ void hashmap_free(hashmap_t *map) */ type_t *find_type(const char *type_name, int flag) { - char head = type_name[0]; - for (int i = 0; i < types_idx; i++) { - if (TYPES[i].type_name[0] != head) + type_t *type = &TYPES[i]; + + if (!identifier_matches(type->type_name, type_name)) continue; - if (TYPES[i].base_type == TYPE_struct || - TYPES[i].base_type == TYPE_union) { - if (flag == 1) - continue; - if (!strcmp(TYPES[i].type_name, type_name)) - return &TYPES[i]; - } else { - if (flag == 2) - continue; - if (!strcmp(TYPES[i].type_name, type_name)) { - /* If it is a forwardly declared alias of a structure, return - * the base structure type. A function type alias with a void - * return is no such alias despite its zero size. - */ - type_t *alias = &TYPES[i]; - type_t *base = alias->base_struct; - - if (alias->base_type != TYPE_typedef || alias->size || - alias->ptr_level || alias->is_direct_function_type) - return alias; - - /* The base would drop the qualifiers of `typedef const struct S - * cs_t;`, so a qualified alias keeps its own descriptor and - * takes the layout once the tag has been completed. - */ - if (!base || (!alias->is_const_qualified && - !alias->is_volatile_qualified)) - return base; - if (!base->size) - return alias; - alias->size = base->size; - alias->alignment = base->alignment; - alias->fields = base->fields; - alias->num_fields = base->num_fields; - alias->is_union = base->is_union; - alias->has_flexible_array_member = - base->has_flexible_array_member; - return alias; + bool is_tag = + type->base_type == TYPE_struct || type->base_type == TYPE_union; + + if ((is_tag && flag == 1) || (!is_tag && flag == 2)) + continue; + return is_tag ? type : resolve_type_alias(type); + } + return NULL; +} + +/* Builtin types are initialized once before translation units are parsed. User + * typedefs share TYPES for their descriptors, but must be reached only through + * their translation-unit/lexical bindings. + */ +static type_t *find_builtin_type(const char *name) +{ + for (int i = 0; i < builtin_types_idx; i++) { + type_t *type = &TYPES[i]; + + if (type->base_type != TYPE_struct && type->base_type != TYPE_union && + identifier_matches(type->type_name, name)) + return type; + } + return NULL; +} + +bool ph2_ir_function_included(const func_t *func, bool include_declarations) +{ + return func && (func->bbs || include_declarations); +} + +void ph2_ir_prepare(bool include_declarations) +{ + int count = 0; + + if (ph2_ir_idx != 0 || PH2_IR_FLATTEN) + fatal("phase-2 IR prepared more than once"); + + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + if (!ph2_ir_function_included(func, include_declarations)) + continue; + if (count >= (INT_MAX - ((int) sizeof(void *) - 1)) / + (int) sizeof(*PH2_IR_FLATTEN)) + limit_error("too many phase-2 IR instructions"); + count++; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { + if (count >= (INT_MAX - ((int) sizeof(void *) - 1)) / + (int) sizeof(*PH2_IR_FLATTEN)) + limit_error("too many phase-2 IR instructions"); + count++; } } } - return NULL; + + ph2_ir_capacity = count; + if (count) + PH2_IR_FLATTEN = arena_alloc( + GENERAL_ARENA, (int) ((size_t) count * sizeof(*PH2_IR_FLATTEN))); } ph2_ir_t *add_existed_ph2_ir(ph2_ir_t *ph2_ir) { - if (ph2_ir_idx >= MAX_IR_INSTR) - limit_error("too many phase-2 IR instructions"); + if (ph2_ir_idx >= ph2_ir_capacity) + fatal("phase-2 IR pre-count mismatch"); PH2_IR_FLATTEN[ph2_ir_idx++] = ph2_ir; return ph2_ir; } +ph2_ir_t *ph2_ir_flatten_insn(ph2_ir_t *insn, + int stack_top_ofs, + int stack_size, + int saved_regs, + bool restore_saved_regs) +{ + if (insn->ofs_based_on_stack_top) { + if (insn->op == OP_load || insn->op == OP_address_of) + insn->src0 += stack_top_ofs; + else if (insn->op == OP_store) + insn->src1 += stack_top_ofs; + } + ph2_ir_t *flat = add_existed_ph2_ir(insn); + if (insn->op == OP_return) { + flat->src1 = stack_size; + if (restore_saved_regs) + flat->src2 = saved_regs; + } + return flat; +} + +int ph2_ir_flatten_function(func_t *func, + int stack_top_ofs, + int prologue_bytes, + int saved_regs, + bool restore_saved_regs, + void (*update_offset)(ph2_ir_t *)) +{ + int count = 0; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + bb->elf_offset = elf_offset; + if (bb == func->bbs) + elf_offset += prologue_bytes; + for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; insn = insn->next) { + ph2_ir_t *flat = + ph2_ir_flatten_insn(insn, stack_top_ofs, func->stack_size, + saved_regs, restore_saved_regs); + count++; + if (update_offset) + update_offset(flat); + } + } + return count; +} + +static ph2_ir_t *ph2_ir_create(opcode_t op) +{ + ph2_ir_t *instruction = arena_calloc(BB_ARENA, 1, sizeof(ph2_ir_t)); + stats_ph2_ir_count++; + instruction->op = op; + instruction->src0_hi = instruction->src1_hi = instruction->dest_hi = -1; + instruction->size_bytes = PTR_SIZE; + return instruction; +} + +ph2_ir_t *bb_add_ph2_ir(basic_block_t *bb, opcode_t op) +{ + ph2_ir_t *instruction = ph2_ir_create(op); + if (bb->ph2_ir_list.tail) + bb->ph2_ir_list.tail->next = instruction; + else + bb->ph2_ir_list.head = instruction; + bb->ph2_ir_list.tail = instruction; + return instruction; +} + ph2_ir_t *add_ph2_ir(opcode_t op) { - ph2_ir_t *ph2_ir = arena_alloc(BB_ARENA, sizeof(ph2_ir_t)); - ph2_ir->op = op; - /* Initialize all fields explicitly */ - ph2_ir->next = NULL; - ph2_ir->is_branch_detached = 0; - ph2_ir->src0 = 0; - ph2_ir->src1 = 0; + return add_existed_ph2_ir(ph2_ir_create(op)); +} - /* Only a select names a third source, but the allocation is not zeroed and - * every field is set here by hand. - */ - ph2_ir->src2 = 0; - ph2_ir->dest = 0; - ph2_ir->func_name = NULL; - ph2_ir->next_bb = NULL; - ph2_ir->then_bb = NULL; - ph2_ir->else_bb = NULL; - ph2_ir->ofs_based_on_stack_top = false; - - /* Default to the full slot. Slots are PTR_SIZE wide, so a wide access is - * always valid; only an address-taken narrow scalar may be written behind - * the allocator's back, and reg-alloc narrows those explicitly. - */ - ph2_ir->size_bytes = PTR_SIZE; - ph2_ir->is_pointer = false; - ph2_ir->src0_is_pointer = false; - ph2_ir->src1_is_pointer = false; - ph2_ir->is_volatile = false; - return add_existed_ph2_ir(ph2_ir); +#if ELF_MACHINE == ELF_MACHINE_ARM32 || ELF_MACHINE == ELF_MACHINE_RV32 || \ + ELF_MACHINE == ELF_MACHINE_AARCH64 +static void ph2_ir_visit_globals(void (*visit)(ph2_ir_t *)) +{ + for (ph2_ir_t *ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ir; ir = ir->next) + visit(ir); +} +#endif + +#if ELF_MACHINE == ELF_MACHINE_ARM32 || ELF_MACHINE == ELF_MACHINE_RV32 +typedef struct { + int stack_top; + int prologue_bytes; + int saved_regs; + bool restore_saved_regs; +} ph2_frame_layout_t; + +static void ph2_ir_flatten_functions( + ph2_frame_layout_t (*frame_layout)(func_t *, ph2_ir_t *), + void (*update_offset)(ph2_ir_t *)) +{ + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + ph2_ir_t *define; + ph2_frame_layout_t frame; + + if (!func->bbs) + continue; + define = add_ph2_ir(OP_define); + define->src0 = func->stack_size; + define->func_name = intern_string(func->return_def.var_name); + frame = frame_layout(func, define); + ph2_ir_flatten_function(func, frame.stack_top, frame.prologue_bytes, + frame.saved_regs, frame.restore_saved_regs, + update_offset); + } } +#endif block_t *add_block(block_t *parent, func_t *func) { @@ -729,7 +1200,7 @@ char *intern_string(char *str) interned = arena_alloc(GENERAL_ARENA, len); strcpy(interned, str); - hashmap_put(string_pool->strings, interned, interned); + hashmap_put_borrowed(string_pool->strings, interned, interned); return interned; } @@ -804,6 +1275,126 @@ bool hex_escape_exceeds_byte(const char *text) return false; } +/* Decode one escape beginning at text[*input]. Narrow strings mask hex runs to + * a byte; wide strings retain their value as one int unit. + */ +static bool decode_escape_value(const char *text, + int *input, + bool wide, + unsigned int *value, + bool *is_ucn) +{ + int i = *input; + + *is_ucn = false; + switch (text[i]) { + case 'a': + *value = '\a'; + i++; + break; + case 'b': + *value = '\b'; + i++; + break; + case 'f': + *value = '\f'; + i++; + break; + case 'n': + *value = '\n'; + i++; + break; + case 'r': + *value = '\r'; + i++; + break; + case 't': + *value = '\t'; + i++; + break; + case 'v': + *value = '\v'; + i++; + break; + case '\\': + *value = '\\'; + i++; + break; + case '\'': + *value = '\''; + i++; + break; + case '"': + *value = '"'; + i++; + break; + case '?': + *value = '?'; + i++; + break; + case 'e': + if (strict_c99) + return false; + *value = 27; + i++; + break; + case 'x': + i++; + if (!isxdigit((unsigned char) text[i])) + return false; + *value = 0; + while (isxdigit((unsigned char) text[i])) { + int digit = hex_digit_value(text[i]); + + if (wide) { + if (*value > 0x07ffffffU) + return false; + *value = (*value << 4) + digit; + } else + *value = ((*value << 4) + digit) & 0xff; + i++; + } + if (wide && *value > 0x7fffffffU) + return false; + break; + case 'u': + case 'U': { + int digits = text[i] == 'u' ? 4 : 8; + + *value = 0; + i++; + for (int n = 0; n < digits; n++) { + if (!isxdigit((unsigned char) text[i]) || + (wide && *value > 0x07ffffffU)) + return false; + *value = (*value << 4) + hex_digit_value(text[i++]); + } + + /* C99 excludes surrogates, out-of-range values, and basic-source + * characters other than $, @, and ` from universal escapes. + */ + if ((wide && *value > 0x7fffffffU) || *value > 0x10ffff || + (*value >= 0xd800 && *value <= 0xdfff) || + (*value < 0xa0 && *value != '$' && *value != '@' && *value != '`')) + return false; + *is_ucn = true; + break; + } + default: + if (!text[i]) + return false; + if (text[i] >= '0' && text[i] <= '7') { + *value = 0; + for (int n = 0; n < 3 && text[i] >= '0' && text[i] <= '7'; n++) + *value = *value * 8 + (text[i++] - '0'); + } else + *value = (unsigned char) text[i++]; + break; + } + *input = i; + return true; +} + int unescape_string(const char *input, char *output, int output_size) { if (!input || !output || output_size == 0) @@ -813,160 +1404,31 @@ int unescape_string(const char *input, char *output, int output_size) while (input[i] != '\0' && j < output_size - 1) { if (input[i] != '\\') { - /* Regular characters */ output[j++] = input[i++]; continue; } - i++; - switch (input[i]) { - case 'a': - output[j++] = '\a'; - i++; - break; - case 'b': - output[j++] = '\b'; - i++; - break; - case 'f': - output[j++] = '\f'; - i++; - break; - case 'e': - if (strict_c99) { - output[j] = '\0'; - return -1; - } - output[j++] = 27; - i++; - break; - case 'n': - output[j++] = '\n'; - i++; - break; - case 'r': - output[j++] = '\r'; - i++; - break; - case 't': - output[j++] = '\t'; - i++; - break; - case 'v': - output[j++] = '\v'; - i++; - break; - case '\\': - output[j++] = '\\'; - i++; - break; - case '\'': - output[j++] = '\''; - i++; - break; - case '"': - output[j++] = '"'; - i++; - break; - case '?': - output[j++] = '\?'; - i++; - break; - case 'x': { - /* C99 hexadecimal escapes consume the complete run of hex digits, - * unlike octal escapes which are limited to three. - */ - i++; /* Skips 'x' */ - - if (!isxdigit(input[i])) { - /* Terminate before bailing: callers read output[0], and an - * unterminated buffer left them reading stack garbage. - */ - output[j] = '\0'; - return -1; - } - - /* Only the low byte reaches the output, so keep only that much - * rather than shift a long digit run out of range. A narrow literal - * whose value does not fit has already been diagnosed by - * hex_escape_exceeds_byte(); a wide one takes its value from - * decode_wstring_units() instead. - */ - unsigned int value = 0; - while (isxdigit(input[i])) { - value = ((value << 4) + hex_digit_value(input[i])) & 0xff; - i++; - } + unsigned int value; + bool is_ucn; - output[j++] = (char) value; - break; + if (!decode_escape_value(input, &i, false, &value, &is_ucn)) { + output[j] = '\0'; + return -1; } - case 'u': - case 'U': { - int digits = input[i] == 'u' ? 4 : 8; - unsigned int value = 0; - - i++; - for (int digit = 0; digit < digits; digit++) { - if (!isxdigit(input[i])) { - output[j] = '\0'; - return -1; - } - value = (value << 4) + hex_digit_value(input[i++]); - } - - /* C99 6.4.3 excludes surrogate code points, values above the - * Unicode range, and basic-source characters other than $, @, and - * `. The latter must be spelt directly in source. - */ - if (value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) || - (value < 0xa0 && value != '$' && value != '@' && - value != '`')) { - output[j] = '\0'; - return -1; - } + if (is_ucn) { j = append_utf8(output, j, output_size, value); if (j < 0) { output[0] = '\0'; return -1; } - break; - } - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': { - /* Octal escape sequence: \ooo (up to 3 digits) */ - int value = 0; - int digit_count = 0; - - while (input[i] >= '0' && input[i] <= '7' && digit_count < 3) { - value = value * 8 + (input[i] - '0'); - i++; - digit_count++; - } - + } else output[j++] = (char) value; - break; - } - default: - /* Unknown escape sequence - treat as literal character */ - output[j++] = input[i++]; - break; - } } output[j] = '\0'; - - /* Check if we ran out of output space */ if (input[i] != '\0') return -1; - return j; } @@ -1006,99 +1468,11 @@ int decode_wstring_units(const char *text, int *units, int capacity) continue; } in++; - switch (text[in]) { - case 'a': - value = '\a'; - in++; - break; - case 'b': - value = '\b'; - in++; - break; - case 'f': - value = '\f'; - in++; - break; - case 'n': - value = '\n'; - in++; - break; - case 'r': - value = '\r'; - in++; - break; - case 't': - value = '\t'; - in++; - break; - case 'v': - value = '\v'; - in++; - break; - case '\\': - value = '\\'; - in++; - break; - case '\'': - value = '\''; - in++; - break; - case '"': - value = '"'; - in++; - break; - case '?': - value = '?'; - in++; - break; - case 'e': - if (strict_c99) - return -1; - value = 27; - in++; - break; - case 'x': - in++; - if (!isxdigit(text[in])) - return -1; - value = 0; - while (isxdigit(text[in])) { - if (value > 0x07ffffffU) - return -1; - value = (value << 4) + hex_digit_value(text[in++]); - } - if (value > 0x7fffffffU) - return -1; - break; - case 'u': - case 'U': { - int digits = text[in] == 'u' ? 4 : 8; - - value = 0; - in++; - for (int i = 0; i < digits; i++) { - if (!isxdigit(text[in])) - return -1; - if (value > 0x07ffffffU) - return -1; - value = (value << 4) + hex_digit_value(text[in++]); - } - if (value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff) || - (value < 0xa0 && value != '$' && value != '@' && value != '`')) - return -1; - break; - } - default: - if (text[in] < '0' || text[in] > '7') - value = (unsigned char) text[in++]; - else { - value = 0; - for (int i = 0; i < 3 && text[in] >= '0' && text[in] <= '7'; - i++) - value = value * 8 + (text[in++] - '0'); - } - break; - } + + bool is_ucn; + + if (!decode_escape_value(text, &in, true, &value, &is_ucn)) + return -1; units[out++] = (int) value; } return out; @@ -1131,7 +1505,8 @@ bool wide_character_constant(const char *literal, int *value) */ int parse_numeric_constant(const char *buffer) { - int i = 0; + int base = numeric_literal_base(buffer); + int i = base == 16 || base == 2 ? 2 : 0; unsigned int value = 0; while (buffer[i]) { /* The lexer keeps C99 integer suffixes in the token. Their type is @@ -1139,36 +1514,18 @@ int parse_numeric_constant(const char *buffer) */ if ((buffer[i] | 32) == 'u' || (buffer[i] | 32) == 'l') break; - if (i == 1 && (buffer[i] | 32) == 'x') { /* hexadecimal */ - value = 0; - i = 2; - while (buffer[i] && (buffer[i] | 32) != 'u' && - (buffer[i] | 32) != 'l') { - char c = buffer[i++]; - value <<= 4; - if (isdigit(c)) - value += c - '0'; - c |= 32; /* convert to lower case */ - if (c >= 'a' && c <= 'f') - value += (c - 'a') + 10; - } - return value; - } - if (i == 1 && (buffer[i] | 32) == 'b') { /* binary */ - value = 0; - i = 2; - while (buffer[i] && (buffer[i] | 32) != 'u' && - (buffer[i] | 32) != 'l') { - char c = buffer[i++]; - value <<= 1; - value += (c == '1'); - } - return value; - } - if (buffer[0] == '0') /* octal */ - value = value * 8 + buffer[i++] - '0'; - else - value = value * 10 + buffer[i++] - '0'; + char c = buffer[i++]; + value *= base; + if (base == 2) + value += c == '1'; + else if (base == 16) { + c |= 32; + if (isdigit(c)) + value += c - '0'; + else if (c >= 'a' && c <= 'f') + value += c - 'a' + 10; + } else + value += c - '0'; } return value; } @@ -1191,6 +1548,7 @@ void type_ensure_fields(type_t *type) return; type->fields = arena_calloc(GENERAL_ARENA, MAX_FIELDS, sizeof(var_t)); + stats_field_slot_count += MAX_FIELDS; /* The field variables come out of a zeroed allocation, so give their * interned name pointers the empty string a reader can dereference. @@ -1235,28 +1593,9 @@ type_t *add_named_type(char *name) return type; } -void add_constant(char alias[], int value) -{ - constant_t *constant = arena_alloc_constant(); - if (!constant) { - printf("Failed to allocate constant_t\n"); - return; - } - - /* Use interned string for constant name */ - strcpy(constant->alias, intern_string(alias)); - constant->value = value; - constant->next = NULL; - hashmap_put(CONSTANTS_MAP, alias, constant); -} - -constant_t *find_constant(char alias[]) -{ - return hashmap_get(CONSTANTS_MAP, alias); -} - /* Enum names declared in a block shadow enclosing names, but must disappear - * with that block. Keep their bindings on the parser's existing block tree. + * with that block. File-scope names use the active translation-unit root of the + * same scope tree. */ void add_scoped_constant(block_t *block, char alias[], int value) { @@ -1268,49 +1607,77 @@ void add_scoped_constant(block_t *block, char alias[], int value) constant->value = value; constant->next = block->constants; block->constants = constant; + if (block == CURRENT_TU_SCOPE) + register_tu_ordinary_name(alias, ORDINARY_CONSTANT, constant); } -/* The kinds of binding an ordinary identifier can have in one scope. Objects, - * functions, typedef names and enumeration constants share C99's ordinary - * identifier name space (6.2.3p1), so each scope binds a name to at most one of - * them. The values are bits, so a lookup can ask for several kinds at once. +/* A file-scope enumeration constant has translation-unit scope, not process + * scope. Keep the parser API but store it on the active input's root. */ -typedef enum { - ORDINARY_NONE = 0, - ORDINARY_CONSTANT = 1, /* enumeration constant declared in the block */ - ORDINARY_VARIABLE = 2, /* local object, or block-scope function alias */ - ORDINARY_TYPEDEF = 4, /* typedef name declared in the block */ - ORDINARY_PARAMETER = 8, /* parameter, seen from the function body's block */ - ORDINARY_ANY = 15 -} ordinary_kind_t; +void add_constant(char alias[], int value) +{ + add_scoped_constant(CURRENT_TU_SCOPE, alias, value); +} -/* The first local of @block at or after index *@pos named @name, or NULL, with - * *@pos left just past it so that a caller can continue to the next one. - * - * A block's locals include all of its IR temporaries, and every declaration - * scans its block's locals at least twice, so the scan is quadratic in the size - * of a block and its compare is what a large one pays for. Settle the first two - * bytes before the library call: temporaries differ in the first, and names - * sharing a prefix letter, such as a run of generated declarations, usually in - * the second. A nonzero first byte that matches means neither name has ended. +/* Keep names declared at file scope in the active unit's ordinary namespace. + * Global storage is merged for code generation, but this small per-unit index + * preserves scope-sensitive collisions with no-linkage enumerators. */ -var_t *find_block_local(block_t *block, const char *name, int *pos) +void add_tu_global_var(var_t *var) { - char head = name[0]; - char second = head ? name[1] : 0; + if (lookup_tu_ordinary_name(file_scope_var_name(var), ORDINARY_VARIABLE, + NULL)) + return; + register_tu_ordinary_name(file_scope_var_name(var), ORDINARY_VARIABLE, var); +} - for (int i = *pos; i < block->locals.size; i++) { - var_t *var = block->locals.elements[i]; - const char *var_name = var->var_name; +static var_t *find_var_in_list(var_list_t *vars, const char *name, int *pos) +{ + for (int i = pos ? *pos : 0; i < vars->size; i++) { + var_t *var = vars->elements[i]; - if (var_name[0] != head || (head && var_name[1] != second)) - continue; - if (!strcmp(var_name, name)) { - *pos = i + 1; + if (identifier_matches(var->var_name, name)) { + if (pos) + *pos = i + 1; return var; } } - *pos = block->locals.size; + if (pos) + *pos = vars->size; + return NULL; +} + +/* Resume at *@pos so callers can continue after a previous match. */ +var_t *find_block_local(block_t *block, const char *name, int *pos) +{ + if (block == CURRENT_TU_SCOPE) { + void *binding = NULL; + + if ((*pos)++) + return NULL; + if (lookup_tu_ordinary_name(name, ORDINARY_VARIABLE, &binding)) + return binding; + return NULL; + } + return find_var_in_list(&block->locals, name, pos); +} + +static var_t *find_tu_declaration(const char *name) +{ + void *binding = NULL; + + if (lookup_tu_ordinary_name(name, ORDINARY_VARIABLE, &binding)) + return binding; + return NULL; +} + +static var_t *find_func_param(func_t *func, const char *name) +{ + if (!func) + return NULL; + for (int i = 0; i < func->num_params; i++) + if (identifier_matches(func->param_defs[i].var_name, name)) + return &func->param_defs[i]; return NULL; } @@ -1326,15 +1693,16 @@ ordinary_kind_t find_block_ordinary(block_t *block, int kinds, void **binding) { - char head = name[0]; void *unused; if (!binding) binding = &unused; + if (block == CURRENT_TU_SCOPE) + return lookup_tu_ordinary_name(name, kinds, binding); if (kinds & ORDINARY_CONSTANT) { for (constant_t *constant = block->constants; constant; constant = constant->next) { - if (!strcmp(constant->alias, name)) { + if (identifier_matches(constant->alias, name)) { *binding = constant; return ORDINARY_CONSTANT; } @@ -1349,27 +1717,49 @@ ordinary_kind_t find_block_ordinary(block_t *block, } if (kinds & ORDINARY_TYPEDEF) { for (typedef_binding_t *td = block->typedefs; td; td = td->next) { - if (td->name[0] == head && !strcmp(td->name, name)) { + if (identifier_matches(td->name, name)) { *binding = td; return ORDINARY_TYPEDEF; } } } if ((kinds & ORDINARY_PARAMETER) && !block->parent && block->func) { - func_t *func = block->func; + var_t *param = find_func_param(block->func, name); - for (int i = 0; i < func->num_params; i++) { - var_t *param = &func->param_defs[i]; - - if (param->var_name[0] == head && !strcmp(param->var_name, name)) { - *binding = param; - return ORDINARY_PARAMETER; - } + if (param) { + *binding = param; + return ORDINARY_PARAMETER; } } return ORDINARY_NONE; } +/* File-scope typedef aliases still use the legacy process-wide type table for + * lookup, but their declaration names also belong in the active unit's + * ordinary-identifier index for same-unit collision checks. + */ +void add_tu_typedef(type_t *type) +{ + const char *name = type->type_name; + + if (!name[0]) + return; + ordinary_kind_t kind = + find_block_ordinary(CURRENT_TU_SCOPE, name, ORDINARY_ANY, NULL); + if (kind == ORDINARY_TYPEDEF) + return; + if (kind != ORDINARY_NONE) + error_at("typedef name conflicts with an ordinary identifier", + cur_token_loc()); + + typedef_binding_t *binding = arena_alloc(BLOCK_ARENA, sizeof(*binding)); + binding->name = intern_string(type->type_name); + binding->type = type; + binding->next = CURRENT_TU_SCOPE->typedefs; + CURRENT_TU_SCOPE->typedefs = binding; + register_tu_ordinary_name(name, ORDINARY_TYPEDEF, binding); +} + /* The enumeration constant @alias as seen from @block, or NULL when none is * visible. Constants share the ordinary identifier name space, so an object, * parameter or typedef name declared in a nearer scope hides an outer constant @@ -1388,11 +1778,13 @@ constant_t *find_scoped_constant(char alias[], block_t *block) for (owner = block; owner && owner != GLOBAL_BLOCK; owner = owner->parent) if (find_block_ordinary(owner, alias, ORDINARY_CONSTANT, &constant)) break; - if (!owner || owner == GLOBAL_BLOCK) - constant = find_constant(alias); - if (!constant) - return NULL; - for (; block != owner; block = block->parent) { + if (!owner || owner == GLOBAL_BLOCK) { + /* Function scopes are not parented to the translation-unit scope. */ + owner = GLOBAL_BLOCK; + if (!lookup_tu_ordinary_name(alias, ORDINARY_CONSTANT, &constant)) + return NULL; + } + for (; block && block != owner; block = block->parent) { if (find_block_ordinary( block, alias, ORDINARY_VARIABLE | ORDINARY_TYPEDEF | ORDINARY_PARAMETER, @@ -1417,7 +1809,7 @@ void add_type_tag(block_t *block, char name[], type_t *type) type_t *find_local_type_tag(char name[], block_t *block) { for (type_tag_t *tag = block->type_tags; tag; tag = tag->next) - if (!strcmp(tag->name, name)) + if (identifier_matches(tag->name, name)) return tag->type; return NULL; } @@ -1427,6 +1819,15 @@ type_t *find_local_type_tag(char name[], block_t *block) */ #define ENUM_TAG_KIND TYPE_int +/* File-scope tags live in the active translation unit, not the merged block + * that owns global object declarations. Some parser paths still pass that + * merged block to type lookup; normalize those file-scope arguments here. + */ +static block_t *tag_scope(block_t *block) +{ + return !block || block == GLOBAL_BLOCK ? CURRENT_TU_SCOPE : block; +} + /* C99 6.7.2.3p3: every declaration of a tag names the same kind of type, and * struct, union and enum tags share one name space, so a tag found under * another keyword is an error rather than a miss. @kind is TYPE_struct, @@ -1442,19 +1843,20 @@ static type_t *check_tag_kind(type_t *type, base_type_t kind) /* The tag @name of @kind as seen from @block, or NULL when no tag of that name * is visible. Tags are registered with the block that declares them, file-scope - * ones with GLOBAL_BLOCK, which a function body's chain does not reach on its - * own, so a NULL @block sees file scope only. A tag declared in some other - * block is never found. Looking in the tag table, not the type table, also - * keeps a typedef name out of it. + * ones with CURRENT_TU_SCOPE, which a function body's chain does not reach on + * its own, so a NULL @block sees the current file scope only. A tag declared in + * another translation unit is never found. Looking in the tag table, not the + * type table, also keeps a typedef name out of it. */ static type_t *find_visible_tag(char name[], block_t *block, base_type_t kind) { type_t *type = NULL; - for (; block && !type; block = block->parent) + block = tag_scope(block); + for (; block && block != CURRENT_TU_SCOPE && !type; block = block->parent) type = find_local_type_tag(name, block); if (!type) - type = find_local_type_tag(name, GLOBAL_BLOCK); + type = find_local_type_tag(name, CURRENT_TU_SCOPE); return check_tag_kind(type, kind); } @@ -1483,10 +1885,10 @@ type_t *find_record_tag(char name[], block_t *block, base_type_t kind) */ type_t *reference_record_tag(char name[], block_t *block, base_type_t kind) { + block = tag_scope(block); type_t *type = find_record_tag(name, block, kind); - return type ? type - : declare_record_tag(name, block ? block : GLOBAL_BLOCK, kind); + return type ? type : declare_record_tag(name, block, kind); } /* The tag @name that @block itself declares, created incomplete when it has @@ -1495,6 +1897,7 @@ type_t *reference_record_tag(char name[], block_t *block, base_type_t kind) */ type_t *local_record_tag(char name[], block_t *block, base_type_t kind) { + block = tag_scope(block); type_t *type = check_tag_kind(find_local_type_tag(name, block), kind); return type ? type : declare_record_tag(name, block, kind); @@ -1537,6 +1940,7 @@ type_t *reference_enum_tag(char name[], block_t *block) /* The enum tag @name that @block itself declares, or NULL. */ type_t *local_enum_tag(char name[], block_t *block) { + block = tag_scope(block); return check_tag_kind(find_local_type_tag(name, block), ENUM_TAG_KIND); } @@ -1570,6 +1974,11 @@ void add_block_typedef(block_t *block, char name[], type_t *type) */ type_t *find_visible_type(const char *name, block_t *block) { + bool searched_tu = false; + + if (!block || block == GLOBAL_BLOCK) + block = CURRENT_TU_SCOPE; + /* Parameters are tried with the function body's outermost block, so they * too hide a file-scope typedef unless a nearer binding has hidden them. */ @@ -1578,12 +1987,24 @@ type_t *find_visible_type(const char *name, block_t *block) ordinary_kind_t kind = find_block_ordinary(block, name, ORDINARY_ANY, &binding); + if (block == CURRENT_TU_SCOPE) + searched_tu = true; + if (kind == ORDINARY_TYPEDEF) + return resolve_type_alias(((typedef_binding_t *) binding)->type); + if (kind != ORDINARY_NONE) + return NULL; + } + if (!searched_tu) { + void *binding; + ordinary_kind_t kind = + find_block_ordinary(CURRENT_TU_SCOPE, name, ORDINARY_ANY, &binding); + if (kind == ORDINARY_TYPEDEF) - return ((typedef_binding_t *) binding)->type; + return resolve_type_alias(((typedef_binding_t *) binding)->type); if (kind != ORDINARY_NONE) return NULL; } - return find_type(name, 1); + return find_builtin_type(name); } var_t *find_member(const char token[], type_t *type) @@ -1598,66 +2019,35 @@ var_t *find_member(const char token[], type_t *type) if (!type) return NULL; - char head = token[0]; - for (int i = 0; i < type->num_fields; i++) { - if (type->fields[i].var_name[0] != head) - continue; - if (!strcmp(type->fields[i].var_name, token)) + if (identifier_matches(type->fields[i].var_name, token)) return &type->fields[i]; } return NULL; } -/* Name lookup is the parser's inner loop: every identifier walks the enclosing - * scopes, then the parameter list, then the globals, and all but the one match - * is a strcmp against a name that differs immediately. A call into the C - * library's vectorized strcmp costs far more than the comparison it performs on - * such names, so each scan settles the common case -- a different first letter - * -- before making the call. Names are never empty, so reading the first byte - * of either side is always in bounds. - */ +/* Search locals from the innermost scope outward, then parameters. */ var_t *find_local_var(const char *token, block_t *block) { func_t *func = block->func; - char head = token[0]; + var_t *var; for (; block; block = block->parent) { - var_list_t *var_list = &block->locals; - for (int i = 0; i < var_list->size; i++) { - var_t *var = var_list->elements[i]; - if (var->var_name[0] != head) - continue; - if (!strcmp(var->var_name, token)) - return var; - } + if ((var = find_var_in_list(&block->locals, token, NULL))) + return var; } - if (func) { - for (int i = 0; i < func->num_params; i++) { - var_t *param = &func->param_defs[i]; - if (param->var_name[0] != head) - continue; - if (!strcmp(param->var_name, token)) - return param; - } - } - return NULL; + return find_func_param(func, token); } var_t *find_global_var(const char *token) { - var_list_t *var_list = &GLOBAL_BLOCK->locals; - char head = token[0]; + var_t *unit_var = find_tu_declaration(token); - for (int i = 0; i < var_list->size; i++) { - var_t *var = var_list->elements[i]; - if (var->var_name[0] != head) - continue; - if (!strcmp(var->var_name, token)) - return var; - } - return NULL; + if (unit_var && !unit_var->is_block_scope_function_declaration) + return find_func(unit_var->var_name) ? NULL : unit_var; + + return find_var_in_list(&GLOBAL_BLOCK->locals, token, NULL); } var_t *find_var(char *token, block_t *parent) @@ -1705,17 +2095,18 @@ int size_var(var_t *var) */ func_t *add_func(char *func_name, bool synthesize) { + char *stable_name; func_t *func = hashmap_get(FUNC_MAP, func_name); if (func) return func; + stable_name = intern_string(func_name); func = arena_alloc_func(); - hashmap_put(FUNC_MAP, func_name, func); + hashmap_put_borrowed(FUNC_MAP, stable_name, func); for (int i = 0; i < MAX_PARAMS; i++) func->param_defs[i].var_name = ""; - /* Use interned string for function name */ - func->return_def.var_name = intern_string(func_name); + func->return_def.var_name = stable_name; /* Prepare space for function arguments. * @@ -1771,6 +2162,19 @@ func_t *find_func(char *func_name) return hashmap_get(FUNC_MAP, func_name); } +int function_entry_address(char *name) +{ + func_t *func = find_func(name); + + if (func->bbs) + return elf_code_start + func->bbs->elf_offset; + if (dynlink) + return dynamic_sections.elf_plt_start + func->plt_offset; + printf("The '%s' function is not implemented\n", name); + fflush(stdout); + abort(); +} + /* Create a basic block and set the scope of variables to 'parent' block */ basic_block_t *bb_create(block_t *parent) { @@ -1778,7 +2182,7 @@ basic_block_t *bb_create(block_t *parent) * zeroing (live_gen, live_kill, live_in, live_out, DF, RDF, dom_next, etc.) * This is simpler and safer than manually initializing everything. */ - basic_block_t *bb = arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); + basic_block_t *bb = arena_alloc_bb(); /* Initialize non-zero fields */ bb->scope = parent; @@ -1790,28 +2194,9 @@ basic_block_t *bb_create(block_t *parent) */ bb->elf_offset = -1; - if (dump_ir || dump_dot) { - /* MAX_VAR_LEN spent 128 bytes on a string that is always ".label." plus - * an int. A self-compile calls bb_create() 52k times, so that was 6.4 - * MiB of arena where 1.2 MiB does. - */ - bb->bb_label_name = arena_alloc(GENERAL_ARENA, MAX_LABEL_LEN); - snprintf(bb->bb_label_name, MAX_LABEL_LEN, ".label.%d", bb_label_idx++); - } - return bb; } -/* Bumped once per compute_live_in() call; a variable belongs to the set being - * built when its stamp equals the current value. - */ -int liveness_gen; - -/* Bumped once per recompute_live_out() call, stamping the union of successor - * live_in sets as it is assembled. - */ -int live_merge_gen; - /* log2 of @v when it is a power of two, otherwise -1. */ int exact_log2(int v) { @@ -1839,67 +2224,21 @@ void *arena_grow(arena_t *arena, int limit, char *what) { - int new_cap = *cap ? *cap << 1 : first; + int new_cap; + + if (*cap < 0 || elem_sz <= 0 || first <= 0 || *cap > INT_MAX / 2) + return NULL; + new_cap = *cap ? *cap << 1 : first; + if (new_cap > INT_MAX / elem_sz) + return NULL; if (limit && new_cap > limit) limit_error(what); void *grown = arena_realloc(arena, ptr, *cap * elem_sz, new_cap * elem_sz); - *cap = new_cap; + if (grown) + *cap = new_cap; return grown; } -/* Record another SSA version of @v, growing the version array as needed. */ -void var_add_subscript(var_t *v, var_t *sub) -{ - if (v->subscripts_idx >= v->subscripts_cap) - v->subscripts = - arena_grow(BLOCK_ARENA, (char *) v->subscripts, &v->subscripts_cap, - sizeof(var_t *), 4, 0, NULL); - v->subscripts[v->subscripts_idx++] = sub; -} - -/* The first SSA version of @v, or NULL when it has none. Callers relied on the - * old array being zero-filled to get NULL here. - */ -var_t *var_subscript0(var_t *v) -{ - if (!v->subscripts_idx) - return NULL; - return v->subscripts[0]; -} - -/* Detach @v from any version array it inherited from a by-value copy. */ -void var_reset_subscripts(var_t *v) -{ - /* memcpy'd from a base, so the copy would otherwise alias the base's - * renaming state as well as its version list. - */ - v->rename = NULL; - v->subscripts = NULL; - v->subscripts_idx = 0; - v->subscripts_cap = 0; -} - -/* Append to a basic block's dominance frontier, growing the array as needed. - * The array starts unallocated, so a block that never contributes to a frontier - * costs nothing beyond the pointer. - */ -void bb_add_df(basic_block_t *bb, basic_block_t *df) -{ - if (bb->df_idx >= bb->df_cap) - bb->DF = arena_grow(BB_ARENA, (char *) bb->DF, &bb->df_cap, - sizeof(basic_block_t *), 8, 0, NULL); - bb->DF[bb->df_idx++] = df; -} - -/* The reverse-dominance-frontier counterpart of bb_add_df(). */ -void bb_add_rdf(basic_block_t *bb, basic_block_t *rdf) -{ - if (bb->rdf_idx >= bb->rdf_cap) - bb->RDF = arena_grow(BB_ARENA, (char *) bb->RDF, &bb->rdf_cap, - sizeof(basic_block_t *), 8, 0, NULL); - bb->RDF[bb->rdf_idx++] = rdf; -} - /* The pred-succ pair must have only one connection */ void bb_connect(basic_block_t *pred, basic_block_t *succ, @@ -1942,31 +2281,7 @@ void bb_connect(basic_block_t *pred, default: abort(); } -} - -/* The pred-succ pair must have only one connection */ -void bb_disconnect(basic_block_t *pred, basic_block_t *succ) -{ - for (int i = 0; i < succ->prev_idx; i++) { - if (succ->prev[i].bb == pred) { - switch (succ->prev[i].type) { - case NEXT: - pred->next = NULL; - break; - case THEN: - pred->then_ = NULL; - break; - case ELSE: - pred->else_ = NULL; - break; - default: - abort(); - } - - succ->prev[i].bb = NULL; - break; - } - } + vir_frontend_note_edge(pred, succ); } /* Count the predecessors still wired to 'bb'. bb_disconnect() leaves holes in @@ -1984,37 +2299,64 @@ int bb_pred_count(const basic_block_t *bb) return n; } -/* The symbol is an argument of function or the variable in declaration */ -void add_symbol(basic_block_t *bb, var_t *var) +/* Return the sole live predecessor, or NULL when the block has zero or many. */ +basic_block_t *bb_sole_pred(const basic_block_t *bb) { - if (!bb) - return; - symbol_t *sym; - for (sym = bb->symbol_list.head; sym; sym = sym->next) { - if (sym->var == var) - return; - } + basic_block_t *pred = NULL; - sym = arena_alloc_symbol(); - sym->var = var; + for (int i = 0; i < bb->prev_idx; i++) { + basic_block_t *candidate = bb->prev[i].bb; - if (!bb->symbol_list.head) { - sym->index = 0; - bb->symbol_list.head = sym; - bb->symbol_list.tail = sym; - } else { - sym->index = bb->symbol_list.tail->index + 1; - bb->symbol_list.tail->next = sym; - bb->symbol_list.tail = sym; + if (!candidate) + continue; + if (pred) + return NULL; + pred = candidate; } + return pred; } -/* Whether @var is a volatile object named by its declaration, as opposed to a - * temporary the parser generated. Temporaries are named ".tN". +/* Pointer-star qualifiers describe pointer storage independently of its base. */ +static int var_volatile_pointer_depth(const var_t *var) +{ + return var->ptr_level + var->type->ptr_level + var->is_func + + !!(var->array_size || var->has_unsized_array); +} +bool var_volatile_storage(const var_t *var) +{ + if (!var || !var->type) + return false; + int depth = var_volatile_pointer_depth(var); + return depth ? depth <= 32 && + (var->pointer_volatile_mask & (1U << (depth - 1))) + : var->is_volatile || var->type->is_volatile_qualified; +} +bool var_volatile_pointee(const var_t *var) +{ + if (!var || !var->type) + return false; + int depth = var_volatile_pointer_depth(var); + const func_t *signature = var->pointee_func_signature; + if (!signature && !var->is_func && var->type->func_signature && + !var->type->is_direct_function_type) + signature = var->type->func_signature; + if (signature) { + const var_t *result = &signature->return_def; + int return_depth = + var->type->func_signature && !var->type->is_direct_function_type + ? 0 + : result->ptr_level + result->type->ptr_level; + if (depth == return_depth + 1) + return var->callback_is_volatile; + } + return depth > 1 ? depth <= 33 && + (var->pointer_volatile_mask & (1U << (depth - 2))) + : var->is_volatile || var->type->is_volatile_qualified; +} bool var_is_volatile_object(const var_t *var) { - return var && var->is_volatile && var->var_name[0] != '.'; + return var_volatile_storage(var) && var->var_name[0] != '.'; } /* A volatile object a primary expression has named and no instruction has read @@ -2036,26 +2378,6 @@ void add_insn(block_t *block, bb->scope = block; - insn_t *n = arena_alloc(INSN_ARENA, sizeof(insn_t)); - n->next = NULL; - n->prev = NULL; - n->opcode = op; - n->rd = rd; - n->rs1 = rs1; - n->rs2 = rs2; - - /* Only a select names a third source. The allocation is not zeroed and - * every field is set here by hand, so this one has to be too. - */ - n->rs3 = NULL; - n->sz = sz; - n->useful = false; - n->belong_to = bb; - n->phi_ops = NULL; - n->idx = 0; - - n->str = str ? intern_string(str) : NULL; - /* An instruction reading the object discard_operand() watches has given it * the read it is owed. */ @@ -2067,6 +2389,8 @@ void add_insn(block_t *block, * optimization */ if ((op == OP_address_of || op == OP_global_address_of) && rs1) { + if (rd) + rd->is_volatile_access = var_volatile_storage(rs1); rs1->address_taken = true; rs1->is_const = false; /* disable constant optimization */ } @@ -2076,16 +2400,81 @@ void add_insn(block_t *block, * its slot as though its address had escaped, so that each access by name * reaches memory rather than a register copy. */ - if (op == OP_allocat && rd && rd->is_volatile && !rd->is_global) + if (op == OP_allocat && var_volatile_storage(rd) && !rd->is_global) rd->address_taken = true; - if (!bb->insn_list.head) - bb->insn_list.head = n; - else - bb->insn_list.tail->next = n; + vir_frontend_note_insn(bb, op, rd, rs1, rs2, sz, + str ? intern_string(str) : NULL); +} + +void dump_stats_phase(const char *phase) +{ + if (!dump_stats) + return; + + /* Keep each call within the compiler's current fixed-argument ABI limit. + * The line grouping is also intentionally stable for benchmark parsers. + */ + fprintf(stderr, "stats phase=%s funcs=%d vars=%d bbs=%d\n", phase, + stats_func_count, stats_var_count, stats_bb_count); + if (!strcmp(phase, "parse")) { + fprintf(stderr, "stats phase=parse translation_units=%d\n", + translation_unit_count); + fprintf(stderr, + "stats phase=parse tu_index_lookups=%d " + "tu_index_registrations=%d\n", + stats_tu_index_lookups, stats_tu_index_registrations); + } + fprintf(stderr, + "stats phase=%s field_slots=%d param_slots=%d var_slots=%d\n", + phase, stats_field_slot_count, stats_func_count * MAX_PARAMS, + stats_var_count + stats_field_slot_count + + stats_func_count * MAX_PARAMS); + vir_stats_t totals = {0}; + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + const vir_function_t *graph = vir_frontend_function(func); + if (!graph) + continue; + vir_stats_t stats; + vir_collect_stats(graph, &stats); + totals.blocks += stats.blocks; + totals.values += stats.values; + totals.params += stats.params; + totals.effects += stats.effects; + totals.arena_capacity += stats.arena_capacity; + } + fprintf(stderr, + "stats phase=%s vir_blocks=%d vir_values=%d vir_params=%d\n", phase, + totals.blocks, totals.values, totals.params); + fprintf(stderr, "stats phase=%s vir_effects=%d vir_arena=%d ph2=%d\n", + phase, totals.effects, totals.arena_capacity, stats_ph2_ir_count); + fprintf(stderr, "stats phase=%s arena_capacity block=%d bb=%d\n", phase, + BLOCK_ARENA ? BLOCK_ARENA->total_bytes : 0, + BB_ARENA ? BB_ARENA->total_bytes : 0); + fprintf(stderr, "stats phase=%s arena_capacity token=%d general=%d\n", + phase, TOKEN_ARENA ? TOKEN_ARENA->total_bytes : 0, + GENERAL_ARENA ? GENERAL_ARENA->total_bytes : 0); + fprintf(stderr, "stats phase=%s arena_capacity_peak block=%d bb=%d\n", + phase, BLOCK_ARENA ? BLOCK_ARENA->peak_bytes : 0, + BB_ARENA ? BB_ARENA->peak_bytes : 0); + fprintf( + stderr, "stats phase=%s arena_capacity_peak token=%d general=%d\n", + phase, + TOKEN_ARENA ? TOKEN_ARENA->peak_bytes : stats_retired_token_arena_peak, + GENERAL_ARENA ? GENERAL_ARENA->peak_bytes : 0); + fprintf(stderr, "stats phase=%s arena_total_capacity=%d\n", phase, + stats_live_arena_capacity); + fprintf(stderr, "stats phase=%s arena_total_capacity_peak=%d\n", phase, + stats_peak_arena_capacity); +} + +void dump_stats_layout(void) +{ + if (!dump_stats) + return; - n->prev = bb->insn_list.tail; - bb->insn_list.tail = n; + fprintf(stderr, "stats layout var=%d bb=%d ph2=%d\n", (int) sizeof(var_t), + (int) sizeof(basic_block_t), (int) sizeof(ph2_ir_t)); } strbuf_t *strbuf_create(int init_capacity) @@ -2178,23 +2567,22 @@ void strbuf_free(strbuf_t *src) */ void global_init(void) { + internal_linkage_name_idx = 0; FUNC_LIST.head = NULL; FUNC_LIST.tail = NULL; - memset(REGS, 0, sizeof(regfile_t) * REG_CNT); /* Initialize arenas first so we can use them for allocation */ BLOCK_ARENA = arena_init(DEFAULT_ARENA_SIZE); /* Variables/blocks */ - INSN_ARENA = arena_init(LARGE_ARENA_SIZE); /* Instructions - high usage */ - BB_ARENA = arena_init(SMALL_ARENA_SIZE); /* Basic blocks - low usage */ - HASHMAP_ARENA = arena_init(DEFAULT_ARENA_SIZE); /* Hash nodes */ + BB_ARENA = arena_init(SMALL_ARENA_SIZE); /* Basic blocks - low usage */ TOKEN_ARENA = arena_init(LARGE_ARENA_SIZE); GENERAL_ARENA = - arena_init(DEFAULT_ARENA_SIZE); /* For TYPES and PH2_IR_FLATTEN */ + arena_init(DEFAULT_ARENA_SIZE); /* For TYPES and flattened IR */ /* Use arena allocation for better memory management */ TYPES = arena_calloc(GENERAL_ARENA, MAX_TYPES, sizeof(type_t)); - PH2_IR_FLATTEN = - arena_alloc(GENERAL_ARENA, MAX_IR_INSTR * sizeof(ph2_ir_t *)); + PH2_IR_FLATTEN = NULL; + ph2_ir_capacity = 0; + ph2_ir_idx = 0; /* Initialize string pool for identifier deduplication */ string_pool = arena_alloc(GENERAL_ARENA, sizeof(string_pool_t)); @@ -2208,7 +2596,6 @@ void global_init(void) TOKEN_CACHE = hashmap_create(DEFAULT_SRC_FILE_COUNT); SRC_FILE_MAP = hashmap_create(DEFAULT_SRC_FILE_COUNT); FUNC_MAP = hashmap_create(DEFAULT_FUNCS_SIZE); - CONSTANTS_MAP = hashmap_create(MAX_CONSTANTS); LIBC_SRC = strbuf_create(4096); elf_code = strbuf_create(MAX_CODE); @@ -2291,6 +2678,7 @@ int arena_free_trailing_blocks(arena_t *arena) arena_block_t *next = block->next; freed += block->capacity; arena->total_bytes -= block->capacity; + stats_live_arena_capacity -= block->capacity; arena_block_free(block); block = next; } @@ -2306,14 +2694,22 @@ int arena_free_trailing_blocks(arena_t *arena) * reach the parser through intern_string(), which copies into GENERAL_ARENA, * and every parser entry point copies a token's text into a local buffer before * storing it. So once parse() returns, all 17 MiB of it is garbage that would - * otherwise stay resident through the memory peak in reg_alloc(). + * otherwise stay resident through the memory peak in machine lowering. * * The source buffers in SRC_FILE_MAP exist only to quote a line in a parse * error, so they go at the same time. */ void release_token_arena(void) { + /* The lexer maps borrow pointers into TOKEN_ARENA, so destroy them before + * releasing their values. Parsing is complete and they have no later + * readers. global_release() may call this cleanup again safely. + */ + lexer_cleanup(); + if (TOKEN_ARENA) { + if (TOKEN_ARENA->peak_bytes > stats_retired_token_arena_peak) + stats_retired_token_arena_peak = TOKEN_ARENA->peak_bytes; arena_free(TOKEN_ARENA); TOKEN_ARENA = NULL; } @@ -2337,73 +2733,42 @@ void release_token_arena(void) } } -/* Compact all arenas to reduce memory usage after compilation phases. This - * safely frees only trailing empty blocks without invalidating pointers. - * - * Return: Total bytes freed across all arenas. - */ -int compact_all_arenas(void) -{ - int total_saved = 0; - - /* Free trailing blocks from each arena */ - total_saved += arena_free_trailing_blocks(BLOCK_ARENA); - total_saved += arena_free_trailing_blocks(INSN_ARENA); - total_saved += arena_free_trailing_blocks(BB_ARENA); - total_saved += arena_free_trailing_blocks(HASHMAP_ARENA); - total_saved += arena_free_trailing_blocks(GENERAL_ARENA); - - return total_saved; -} - -/* Compact specific arenas based on compilation phase. Different phases have - * different memory usage patterns. - * - * @phase_mask: Bitmask using COMPACT_ARENA_* defines - * to indicate which arenas to compact. - * - * Return: Total bytes freed. - */ -int compact_arenas_selective(int phase_mask) +static int compact_arenas(int phase_mask) { int total_saved = 0; if (phase_mask & COMPACT_ARENA_BLOCK) total_saved += arena_free_trailing_blocks(BLOCK_ARENA); - - if (phase_mask & COMPACT_ARENA_INSN) - total_saved += arena_free_trailing_blocks(INSN_ARENA); - if (phase_mask & COMPACT_ARENA_BB) total_saved += arena_free_trailing_blocks(BB_ARENA); - - if (phase_mask & COMPACT_ARENA_HASHMAP) - total_saved += arena_free_trailing_blocks(HASHMAP_ARENA); - if (phase_mask & COMPACT_ARENA_GENERAL) total_saved += arena_free_trailing_blocks(GENERAL_ARENA); return total_saved; } +/* Compact selected arenas for their compilation phase. */ +int compact_arenas_selective(int phase_mask) +{ + return compact_arenas(phase_mask); +} + void global_release(void) { - /* Cleanup lexer hashmaps */ - lexer_cleanup(); + /* Release token and source storage before the remaining compiler state. */ + release_token_arena(); /* Free string interning hashmaps */ if (string_pool && string_pool->strings) hashmap_free(string_pool->strings); if (string_literal_pool && string_literal_pool->literals) hashmap_free(string_literal_pool->literals); + hashmap_free(TU_ORDINARY_MAP); + TU_ORDINARY_MAP = NULL; arena_free(BLOCK_ARENA); - arena_free(INSN_ARENA); arena_free(BB_ARENA); - arena_free(HASHMAP_ARENA); - if (TOKEN_ARENA) - arena_free(TOKEN_ARENA); - arena_free(GENERAL_ARENA); /* free TYPES and PH2_IR_FLATTEN */ + arena_free(GENERAL_ARENA); /* free TYPES */ /* Every value in TOKEN_CACHE is one heap-allocated token_stream_t: the * tokens it spans come from TOKEN_ARENA, which is already gone, but the @@ -2417,31 +2782,34 @@ void global_release(void) } } hashmap_free(TOKEN_CACHE); - hashmap_free(SRC_FILE_MAP); hashmap_free(FUNC_MAP); hashmap_free(INCLUSION_MAP); - hashmap_free(CONSTANTS_MAP); - - strbuf_free(LIBC_SRC); - strbuf_free(elf_code); - strbuf_free(elf_data); - strbuf_free(elf_rodata); - strbuf_free(elf_header); - strbuf_free(elf_program_header); - strbuf_free(elf_symtab); - strbuf_free(elf_strtab); - strbuf_free(elf_shstrtab); - strbuf_free(elf_section_header); - strbuf_free(dynamic_sections.elf_interp); - strbuf_free(dynamic_sections.elf_dynamic); - strbuf_free(dynamic_sections.elf_dynsym); - strbuf_free(dynamic_sections.elf_dynstr); - if (dynamic_sections.use_relaplt) - strbuf_free(dynamic_sections.elf_relaplt); - else - strbuf_free(dynamic_sections.elf_relplt); - strbuf_free(dynamic_sections.elf_plt); - strbuf_free(dynamic_sections.elf_got); + + strbuf_t *buffers[] = { + LIBC_SRC, + elf_code, + elf_data, + elf_rodata, + elf_header, + elf_program_header, + elf_symtab, + elf_strtab, + elf_shstrtab, + elf_section_header, + dynamic_sections.elf_interp, + dynamic_sections.elf_dynamic, + dynamic_sections.elf_dynsym, + dynamic_sections.elf_dynstr, + dynamic_sections.use_relaplt ? dynamic_sections.elf_relaplt + : dynamic_sections.elf_relplt, + dynamic_sections.elf_plt, + dynamic_sections.elf_got, + }; + for (size_t i = 0; i < sizeof(buffers) / sizeof(*buffers); i++) + strbuf_free(buffers[i]); + PH2_IR_FLATTEN = NULL; + ph2_ir_idx = 0; + ph2_ir_capacity = 0; } /* The function whose body the back half of the pipeline is working on, or NULL @@ -2580,300 +2948,14 @@ __noreturn void error_at(char *msg, source_location_t *loc) exit(1); } -void print_indent(int indent) -{ - for (int i = 0; i < indent; i++) - printf("\t"); -} - -void dump_bb_insn(const func_t *func, - const basic_block_t *bb, - bool *at_func_start) -{ - if (!bb) - return; - const var_t *rd, *rs1, *rs2; - - if (bb != func->bbs && bb->insn_list.head) { - if (!at_func_start[0]) - printf("%s:\n", bb->bb_label_name); - else - at_func_start[0] = false; - } - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - rd = insn->rd; - rs1 = insn->rs1; - rs2 = insn->rs2; - - switch (insn->opcode) { - case OP_unwound_phi: - /* Ignored */ - continue; - case OP_allocat: - print_indent(1); - printf("allocat %s", rd->type->type_name); - - for (int i = 0; i < rd->ptr_level; i++) - printf("*"); - - printf(" %%%s", rd->var_name); - - if (rd->array_size > 0) - printf("[%d]", rd->array_size); - - break; - case OP_load_constant: - print_indent(1); - printf("const %%%s, %d", rd->var_name, rd->init_val); - break; - case OP_load_data_address: - print_indent(1); - /* offset from .data section */ - printf("%%%s = .data (%d)", rd->var_name, rd->init_val); - break; - case OP_load_rodata_address: - print_indent(1); - /* offset from .rodata section */ - printf("%%%s = .rodata (%d)", rd->var_name, rd->init_val); - break; - case OP_address_of: - print_indent(1); - printf("%%%s = &(%%%s)", rd->var_name, rs1->var_name); - break; - case OP_assign: - print_indent(1); - printf("%%%s = %%%s", rd->var_name, rs1->var_name); - break; - case OP_branch: - print_indent(1); - printf("br %%%s, %s, %s", rs1->var_name, bb->then_->bb_label_name, - bb->else_->bb_label_name); - break; - case OP_jump: - print_indent(1); - printf("jmp %s", bb->next->bb_label_name); - break; - case OP_label: - print_indent(0); - printf("%s:", insn->str); - break; - case OP_push: - print_indent(1); - printf("push %%%s", rs1->var_name); - break; - case OP_call: - print_indent(1); - printf("call @%s", insn->str); - break; - case OP_func_ret: - print_indent(1); - printf("retval %%%s", rd->var_name); - break; - case OP_return: - print_indent(1); - if (rs1) - printf("ret %%%s", rs1->var_name); - else - printf("ret"); - break; - case OP_read: - print_indent(1); - printf("%%%s = (%%%s), %d", rd->var_name, rs1->var_name, insn->sz); - break; - case OP_write: - print_indent(1); - if (rs1->is_func) - printf("(%%%s) = @%s", rs1->var_name, rs2->var_name); - else - printf("(%%%s) = %%%s, %d", rs1->var_name, rs2->var_name, - insn->sz); - break; - case OP_indirect: - print_indent(1); - printf("indirect call @(%%%s)", rs1->var_name); - break; - case OP_negate: - print_indent(1); - printf("neg %%%s, %%%s", rd->var_name, rs1->var_name); - break; - case OP_add: - print_indent(1); - printf("%%%s = add %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_sub: - print_indent(1); - printf("%%%s = sub %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_mul: - print_indent(1); - printf("%%%s = mul %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_div: - print_indent(1); - printf("%%%s = div %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_mod: - print_indent(1); - printf("%%%s = mod %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_eq: - print_indent(1); - printf("%%%s = eq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_neq: - print_indent(1); - printf("%%%s = neq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_gt: - print_indent(1); - printf("%%%s = gt %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_lt: - print_indent(1); - printf("%%%s = lt %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_geq: - print_indent(1); - printf("%%%s = geq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_leq: - print_indent(1); - printf("%%%s = leq %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_bit_and: - print_indent(1); - printf("%%%s = and %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_bit_or: - print_indent(1); - printf("%%%s = or %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_bit_not: - print_indent(1); - printf("%%%s = not %%%s", rd->var_name, rs1->var_name); - break; - case OP_bit_xor: - print_indent(1); - printf("%%%s = xor %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_log_and: - print_indent(1); - printf("%%%s = and %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_log_or: - print_indent(1); - printf("%%%s = or %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_log_not: - print_indent(1); - printf("%%%s = not %%%s", rd->var_name, rs1->var_name); - break; - case OP_rshift: - print_indent(1); - printf("%%%s = rshift %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_lshift: - print_indent(1); - printf("%%%s = lshift %%%s, %%%s", rd->var_name, rs1->var_name, - rs2->var_name); - break; - case OP_trunc: - print_indent(1); - printf("%%%s = trunc %%%s, %d", rd->var_name, rs1->var_name, - insn->sz); - break; - case OP_sign_ext: - print_indent(1); - printf("%%%s = sign_ext %%%s, %d", rd->var_name, rs1->var_name, - insn->sz); - break; - case OP_cast: - print_indent(1); - printf("%%%s = cast %%%s", rd->var_name, rs1->var_name); - break; - default: - printf("", insn->opcode); - break; - } - - printf("\n"); - } -} - -void dump_bb_insn_by_dom(func_t *func, basic_block_t *bb, bool *at_func_start) +int abi_arg_next(int cursor, const var_t *var, bool variadic) { - dump_bb_insn(func, bb, at_func_start); - for (int i = 0; bb && i < bb->dom_next_idx; i++) - dump_bb_insn_by_dom(func, bb->dom_next[i], at_func_start); -} - -void dump_insn(void) -{ - printf("====\n"); - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - bool at_func_start = true; - - printf("def %s", func->return_def.type->type_name); - - for (int i = 0; i < func->return_def.ptr_level; i++) - printf("*"); - printf(" @%s(", func->return_def.var_name); - - for (int i = 0; i < func->num_params; i++) { - if (i != 0) - printf(", "); - printf("%s", func->param_defs[i].type->type_name); - - for (int k = 0; k < func->param_defs[i].ptr_level; k++) - printf("*"); - printf(" %%%s", func->param_defs[i].var_name); - } - printf(") {\n"); - - dump_bb_insn_by_dom(func, func->bbs, &at_func_start); - - /* Handle implicit return */ - for (int i = 0; func->exit && i < func->exit->prev_idx; i++) { - const basic_block_t *bb = func->exit->prev[i].bb; - if (!bb) - continue; - - if (func->return_def.type != TY_void) - continue; - - if (bb->insn_list.tail) - if (bb->insn_list.tail->opcode == OP_return) - continue; - - print_indent(1); - printf("ret\n"); - } - - printf("}\n"); - } - - printf("====\n"); + bool pair = PTR_SIZE == 4 && var && var->type && !var->ptr_level && + !var->array_size && !var->is_func && + var->type->base_type != TYPE_struct && + var->type->base_type != TYPE_union && var->type->size == 8; + if (pair && (ELF_MACHINE != ELF_MACHINE_RV32 || variadic || + cursor >= MAX_ARGS_IN_REG)) + cursor = ALIGN_UP(cursor, 2); + return cursor + (pair ? 2 : 1); } diff --git a/src/lexer.c b/src/lexer.c index 3a1fc434..1796e383 100644 --- a/src/lexer.c +++ b/src/lexer.c @@ -10,11 +10,6 @@ #include "defs.h" #include "globals.c" -/* Hash table constants */ -#define NUM_DIRECTIVES 12 -#define NUM_KEYWORDS 32 - -/* Token mapping structure for elegant initialization */ typedef struct { char *name; token_kind_t token; @@ -28,18 +23,25 @@ hashmap_t *KEYWORD_MAP = NULL; token_kind_t *directive_tokens_storage = NULL; token_kind_t *keyword_tokens_storage = NULL; +static void init_token_map(hashmap_t **map, + token_kind_t **storage, + const token_mapping_t *mappings, + int count, + int capacity) +{ + *map = hashmap_create(capacity); + *storage = arena_alloc(TOKEN_ARENA, count * sizeof(**storage)); + for (int i = 0; i < count; i++) { + (*storage)[i] = mappings[i].token; + hashmap_put_borrowed(*map, mappings[i].name, &(*storage)[i]); + } +} + void lex_init_directives(void) { if (DIRECTIVE_MAP) return; - DIRECTIVE_MAP = hashmap_create(16); /* Small capacity for directives */ - - /* Initialization using struct compound literals for elegance */ - directive_tokens_storage = - arena_alloc(GENERAL_ARENA, NUM_DIRECTIVES * sizeof(token_kind_t)); - - /* Use array compound literal for directive mappings */ token_mapping_t directives[] = { {"#define", T_cppd_define}, {"#elif", T_cppd_elif}, {"#else", T_cppd_else}, {"#endif", T_cppd_endif}, @@ -48,13 +50,8 @@ void lex_init_directives(void) {"#include", T_cppd_include}, {"#pragma", T_cppd_pragma}, {"#undef", T_cppd_undef}, {"#line", T_cppd_line}, }; - - /* hashmap insertion */ - for (int i = 0; i < NUM_DIRECTIVES; i++) { - directive_tokens_storage[i] = directives[i].token; - hashmap_put(DIRECTIVE_MAP, directives[i].name, - &directive_tokens_storage[i]); - } + init_token_map(&DIRECTIVE_MAP, &directive_tokens_storage, directives, + sizeof(directives) / sizeof(*directives), 16); } void lex_init_keywords(void) @@ -62,13 +59,6 @@ void lex_init_keywords(void) if (KEYWORD_MAP) return; - KEYWORD_MAP = hashmap_create(32); /* Capacity for keywords */ - - /* Initialization using struct compound literals for elegance */ - keyword_tokens_storage = - arena_alloc(GENERAL_ARENA, NUM_KEYWORDS * sizeof(token_kind_t)); - - /* Use array compound literal for keyword mappings */ token_mapping_t keywords[] = { {"if", T_if}, {"while", T_while}, @@ -103,12 +93,8 @@ void lex_init_keywords(void) {"_Complex", T_complex}, {"_Imaginary", T_imaginary}, }; - - /* hashmap insertion */ - for (int i = 0; i < NUM_KEYWORDS; i++) { - keyword_tokens_storage[i] = keywords[i].token; - hashmap_put(KEYWORD_MAP, keywords[i].name, &keyword_tokens_storage[i]); - } + init_token_map(&KEYWORD_MAP, &keyword_tokens_storage, keywords, + sizeof(keywords) / sizeof(*keywords), 32); } /* Hash table lookup for preprocessor directives */ @@ -125,8 +111,11 @@ token_kind_t lookup_directive(char *token) } /* Hash table lookup for C keywords */ -token_kind_t lookup_keyword(char *token) +token_kind_t lookup_keyword(char *token, int length) { + if (length < 2 || length > 10) + return T_identifier; + if (!KEYWORD_MAP) lex_init_keywords(); @@ -151,9 +140,8 @@ void lexer_cleanup(void) KEYWORD_MAP = NULL; } - /* Token storage arrays are allocated from GENERAL_ARENA and will be - * automatically freed when the arena is freed in global_release(). No need - * to explicitly free them here. + /* Token storage arrays share TOKEN_ARENA's parse lifetime and are released + * by release_token_arena(). */ directive_tokens_storage = NULL; keyword_tokens_storage = NULL; @@ -320,15 +308,14 @@ int file_read_all(FILE *f, char *dst, int len) } #endif -strbuf_t *read_file(const char *filename) +/* @filename's contents, or NULL when it cannot be opened. */ +strbuf_t *try_read_file(const char *filename) { FILE *f = fopen(filename, "rb"); strbuf_t *src; - if (!f) { - printf("filename: %s\n", filename); - fatal("source file cannot be found."); - } + if (!f) + return NULL; fseek(f, 0, SEEK_END); int len = ftell(f); @@ -342,17 +329,34 @@ strbuf_t *read_file(const char *filename) return src; } -strbuf_t *get_file_buf(char *filename) +/* @filename's contents, read once and kept for every later request; NULL when + * it cannot be opened, so that a caller with a better place to report the + * mistake can. + */ +strbuf_t *try_get_file_buf(char *filename) { strbuf_t *buf; - if (!hashmap_contains(SRC_FILE_MAP, filename)) { - buf = read_file(filename); - hashmap_put(SRC_FILE_MAP, filename, buf); - } else { - buf = hashmap_get(SRC_FILE_MAP, filename); - } + if (hashmap_contains(SRC_FILE_MAP, filename)) + return hashmap_get(SRC_FILE_MAP, filename); + buf = try_read_file(filename); + if (buf) + hashmap_put_borrowed(SRC_FILE_MAP, filename, buf); + return buf; +} +strbuf_t *get_file_buf(char *filename) +{ + strbuf_t *buf = try_get_file_buf(filename); + + /* A file that cannot be opened is a mistake in how the compiler was + * invoked, not a broken invariant, so it exits rather than abort()ing. + */ + if (!buf) { + printf("[Error]: cannot open source file '%s'\n", filename); + fflush(NULL); + exit(1); + } return buf; } @@ -371,6 +375,16 @@ token_t *new_token(token_kind_t kind, const source_location_t *loc, int len) return token; } +static token_t *new_token_at_column(source_location_t *loc, + token_kind_t kind, + int len) +{ + token_t *token = new_token(kind, loc, len); + + loc->column += len; + return token; +} + /* A '#' opens a directive only when nothing but white space precedes it on its * logical line. The physical column cannot tell: a backslash-newline puts a '#' * at column 1 of the next physical line while phase 2 has already joined it to @@ -440,9 +454,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) paste_len += source_char_width(buf, buf->size); read_char(buf); } - token = new_token(T_hashhash, loc, paste_len); - loc->column += paste_len; - return token; + return new_token_at_column(loc, T_hashhash, paste_len); } if (!lex_at_line_start) { @@ -450,9 +462,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) for (int i = 0; i < hash_chars; i++) read_char(buf); - token = new_token(T_hash, loc, hash_len); - loc->column += hash_len; - return token; + return new_token_at_column(loc, T_hash, hash_len); } int sz = 0, source_len = hash_len; @@ -511,9 +521,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) if (directive_kind == T_identifier) directive_kind = T_cppd_unknown; - token = new_token(directive_kind, loc, source_len); - loc->column += source_len; - return token; + return new_token_at_column(loc, directive_kind, source_len); } /* Leave a UCN to lex_word(); an ordinary backslash remains available for @@ -521,9 +529,7 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) */ if (ch == '\\' && peek_char(buf, 1) != 'u' && peek_char(buf, 1) != 'U') { read_char(buf); - token = new_token(T_backslash, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_backslash, 1); } if (ch == '\n') { @@ -577,14 +583,10 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) if (ch == '=') { ch = read_char(buf); - token = new_token(T_divideeq, loc, 2); - loc->column += 2; - return token; + return new_token_at_column(loc, T_divideeq, 2); } - token = new_token(T_divide, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_divide, 1); } if (ch == ' ') { @@ -594,23 +596,17 @@ token_t *lex_layout(strbuf_t *buf, source_location_t *loc, char ch) while (read_char(buf) == ' ') sz++; - token = new_token(T_whitespace, loc, sz); - loc->column += sz; - return token; + return new_token_at_column(loc, T_whitespace, sz); } if (ch == '\t') { read_char(buf); - token = new_token(T_tab, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_tab, 1); } if (ch == '\0') { read_char(buf); - token = new_token(T_eof, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_eof, 1); } return NULL; @@ -930,118 +926,70 @@ token_t *lex_literal(strbuf_t *buf, source_location_t *loc, char ch) */ token_t *lex_punct(strbuf_t *buf, source_location_t *loc, char ch) { - token_t *token; - - if (ch == '(') { - ch = read_char(buf); - token = new_token(T_open_bracket, loc, 1); - loc->column++; - return token; - } - - if (ch == ')') { - ch = read_char(buf); - token = new_token(T_close_bracket, loc, 1); - loc->column++; - return token; - } - - if (ch == '{') { - ch = read_char(buf); - token = new_token(T_open_curly, loc, 1); - loc->column++; - return token; - } - if (ch == '<' && peek_char(buf, 1) == '%') { read_char(buf); read_char(buf); - token = new_token(T_open_curly, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '}') { - ch = read_char(buf); - token = new_token(T_close_curly, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_open_curly, 2); } if (ch == '%' && peek_char(buf, 1) == '>') { read_char(buf); read_char(buf); - token = new_token(T_close_curly, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '[') { - ch = read_char(buf); - token = new_token(T_open_square, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_close_curly, 2); } if (ch == '<' && peek_char(buf, 1) == ':') { read_char(buf); read_char(buf); - token = new_token(T_open_square, loc, 2); - loc->column += 2; - return token; - } - - if (ch == ']') { - ch = read_char(buf); - token = new_token(T_close_square, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_open_square, 2); } if (ch == ':' && peek_char(buf, 1) == '>') { read_char(buf); read_char(buf); - token = new_token(T_close_square, loc, 2); - loc->column += 2; - return token; - } - - if (ch == ',') { - ch = read_char(buf); - token = new_token(T_comma, loc, 1); - loc->column++; - return token; - } - - if (ch == '~') { - ch = read_char(buf); - token = new_token(T_bit_not, loc, 1); - loc->column++; - return token; - } - - if (ch == ';') { - read_char(buf); - token = new_token(T_semicolon, loc, 1); - loc->column++; - return token; - } - - if (ch == '?') { - read_char(buf); - token = new_token(T_question, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_close_square, 2); } - if (ch == ':') { - read_char(buf); - token = new_token(T_colon, loc, 1); - loc->column++; - return token; + token_kind_t kind; + switch (ch) { + case '(': + kind = T_open_bracket; + break; + case ')': + kind = T_close_bracket; + break; + case '{': + kind = T_open_curly; + break; + case '}': + kind = T_close_curly; + break; + case '[': + kind = T_open_square; + break; + case ']': + kind = T_close_square; + break; + case ',': + kind = T_comma; + break; + case '~': + kind = T_bit_not; + break; + case ';': + kind = T_semicolon; + break; + case '?': + kind = T_question; + break; + case ':': + kind = T_colon; + break; + default: + return NULL; } - - return NULL; + read_char(buf); + return new_token_at_column(loc, kind, 1); } /* Operators, each of which may or may not continue into a longer one. @@ -1049,254 +997,114 @@ token_t *lex_punct(strbuf_t *buf, source_location_t *loc, char ch) * Returns NULL when 'ch' is none of its business, so that lex_token() can offer * the character to the next reader in line. */ -token_t *lex_operator(strbuf_t *buf, source_location_t *loc, char ch) +static token_t *lex_short_operator(strbuf_t *buf, + source_location_t *loc, + char pair_char, + token_kind_t pair_kind, + char alternate_char, + token_kind_t alternate_kind, + token_kind_t assign_kind, + token_kind_t single_kind) { - token_t *token; - - if (ch == '^') { - ch = read_char(buf); - - if (ch == '=') { - ch = read_char(buf); - token = new_token(T_xoreq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_bit_xor, loc, 1); - loc->column++; - return token; - } - - if (ch == '*') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_asteriskeq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_asterisk, loc, 1); - loc->column++; - return token; - } - - if (ch == '&') { - ch = read_char(buf); - - if (ch == '&') { - read_char(buf); - token = new_token(T_log_and, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_andeq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_ampersand, loc, 1); - loc->column++; - return token; - } - - if (ch == '|') { - ch = read_char(buf); + int length = 1; + token_kind_t kind = single_kind; - if (ch == '|') { - read_char(buf); - token = new_token(T_log_or, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_oreq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_bit_or, loc, 1); - loc->column++; - return token; - } - - if (ch == '<') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_le, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '<') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_lshifteq, loc, 3); - loc->column += 3; - return token; - } - - token = new_token(T_lshift, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_lt, loc, 1); - loc->column++; - return token; - } - - if (ch == '%') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_modeq, loc, 2); - loc->column += 2; - return token; - } + char ch = read_char(buf); + if (pair_char && ch == pair_char) { + kind = pair_kind; + length++; + } else if (alternate_char && ch == alternate_char) { + kind = alternate_kind; + length++; + } else if (ch == '=' && assign_kind != T_start) { + kind = assign_kind; + length++; + } + if (length == 2) + read_char(buf); + return new_token_at_column(loc, kind, length); +} - token = new_token(T_mod, loc, 1); - loc->column++; - return token; - } +static token_t *lex_angle_operator(strbuf_t *buf, + source_location_t *loc, + char first, + token_kind_t single, + token_kind_t equal, + token_kind_t shift, + token_kind_t shift_equal) +{ + char ch = read_char(buf); + int length = 1; + token_kind_t kind = single; - if (ch == '>') { + if (ch == '=') { + kind = equal; + length = 2; + read_char(buf); + } else if (ch == first) { ch = read_char(buf); - + kind = shift; + length = 2; if (ch == '=') { + kind = shift_equal; + length++; read_char(buf); - token = new_token(T_ge, loc, 2); - loc->column += 2; - return token; } - - if (ch == '>') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_rshifteq, loc, 3); - loc->column += 3; - return token; - } - - token = new_token(T_rshift, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_gt, loc, 1); - loc->column++; - return token; } + return new_token_at_column(loc, kind, length); +} - if (ch == '!') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_noteq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_log_not, loc, 1); - loc->column++; - return token; - } +token_t *lex_operator(strbuf_t *buf, source_location_t *loc, char ch) +{ + if (ch == '^') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, T_xoreq, + T_bit_xor); + if (ch == '*') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, + T_asteriskeq, T_asterisk); + if (ch == '&') + return lex_short_operator(buf, loc, '&', T_log_and, 0, T_start, T_andeq, + T_ampersand); + if (ch == '|') + return lex_short_operator(buf, loc, '|', T_log_or, 0, T_start, T_oreq, + T_bit_or); + + if (ch == '<') + return lex_angle_operator(buf, loc, '<', T_lt, T_le, T_lshift, + T_lshifteq); + + if (ch == '%') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, T_modeq, + T_mod); + + if (ch == '>') + return lex_angle_operator(buf, loc, '>', T_gt, T_ge, T_rshift, + T_rshifteq); + + if (ch == '!') + return lex_short_operator(buf, loc, 0, T_start, 0, T_start, T_noteq, + T_log_not); if (ch == '.') { ch = read_char(buf); if (ch == '.' && peek_char(buf, 1) == '.') { buf->size += 2; - token = new_token(T_elipsis, loc, 3); - loc->column += 3; - return token; + return new_token_at_column(loc, T_elipsis, 3); } - token = new_token(T_dot, loc, 1); - loc->column++; - return token; - } - - if (ch == '-') { - ch = read_char(buf); - - if (ch == '>') { - read_char(buf); - token = new_token(T_arrow, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '-') { - read_char(buf); - token = new_token(T_decrement, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_minuseq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_minus, loc, 1); - loc->column++; - return token; + return new_token_at_column(loc, T_dot, 1); } - if (ch == '+') { - ch = read_char(buf); - - if (ch == '+') { - read_char(buf); - token = new_token(T_increment, loc, 2); - loc->column += 2; - return token; - } - - if (ch == '=') { - read_char(buf); - token = new_token(T_pluseq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_plus, loc, 1); - loc->column++; - return token; - } - - if (ch == '=') { - ch = read_char(buf); - - if (ch == '=') { - read_char(buf); - token = new_token(T_eq, loc, 2); - loc->column += 2; - return token; - } - - token = new_token(T_assign, loc, 1); - loc->column++; - return token; - } + if (ch == '-') + return lex_short_operator(buf, loc, '-', T_decrement, '>', T_arrow, + T_minuseq, T_minus); + if (ch == '+') + return lex_short_operator(buf, loc, '+', T_increment, 0, T_start, + T_pluseq, T_plus); + if (ch == '=') + return lex_short_operator(buf, loc, '=', T_eq, 0, T_start, T_start, + T_assign); return NULL; } @@ -1318,7 +1126,6 @@ static int lex_ucn_identifier(char *out, int out_size, strbuf_t *buf) { int digits = peek_char(buf, 1) == 'u' ? 4 : 8; unsigned int value = 0; - int out_len; for (int i = 0; i < digits; i++) { int digit = hex_digit_value(peek_char(buf, i + 2)); @@ -1337,33 +1144,7 @@ static int lex_ucn_identifier(char *out, int out_size, strbuf_t *buf) value > 0x10ffff || (value >= 0xd800 && value <= 0xdfff)) return -1; - if (value <= 0x7f) - out_len = 1; - else if (value <= 0x7ff) - out_len = 2; - else if (value <= 0xffff) - out_len = 3; - else - out_len = 4; - if (out_len >= out_size) - return -1; - - if (out_len == 1) { - out[0] = value; - } else if (out_len == 2) { - out[0] = 0xc0 | (value >> 6); - out[1] = 0x80 | (value & 0x3f); - } else if (out_len == 3) { - out[0] = 0xe0 | (value >> 12); - out[1] = 0x80 | ((value >> 6) & 0x3f); - out[2] = 0x80 | (value & 0x3f); - } else { - out[0] = 0xf0 | (value >> 18); - out[1] = 0x80 | ((value >> 12) & 0x3f); - out[2] = 0x80 | ((value >> 6) & 0x3f); - out[3] = 0x80 | (value & 0x3f); - } - return out_len; + return append_utf8(out, 0, out_size, value); } /* Identifiers, and the keywords spelled like them. @@ -1413,113 +1194,7 @@ token_t *lex_word(strbuf_t *buf, source_location_t *loc, char ch) } while (isalnum(ch) || ch == '_' || lex_ucn_starts(buf)); token_buffer[sz] = 0; - /* Fast path for common keywords - avoid hashmap lookup */ - token_kind_t kind = T_identifier; - - /* Check most common keywords inline based on token length and first - * character. - */ - switch (sz) { - case 2: /* 2-letter keywords: if, do */ - if (token_buffer[0] == 'i' && token_buffer[1] == 'f') - kind = T_if; - else if (token_buffer[0] == 'd' && token_buffer[1] == 'o') - kind = T_do; - break; - - case 3: /* 3-letter keywords: for */ - if (token_buffer[0] == 'f' && token_buffer[1] == 'o' && - token_buffer[2] == 'r') - kind = T_for; - break; - - case 4: /* 4-letter keywords: else, enum, case */ - if (token_buffer[0] == 'e') { - if (!memcmp(token_buffer, "else", 4)) - kind = T_else; - else if (!memcmp(token_buffer, "enum", 4)) - kind = T_enum; - } else if (!memcmp(token_buffer, "case", 4)) - kind = T_case; - else if (!memcmp(token_buffer, "goto", 4)) - kind = T_goto; - break; - - case 5: /* 5-letter keywords: while, break, union, const, float */ - if (token_buffer[0] == 'w' && !memcmp(token_buffer, "while", 5)) - kind = T_while; - else if (token_buffer[0] == 'b' && - !memcmp(token_buffer, "break", 5)) - kind = T_break; - else if (token_buffer[0] == 'u' && - !memcmp(token_buffer, "union", 5)) - kind = T_union; - else if (token_buffer[0] == 'c' && - !memcmp(token_buffer, "const", 5)) - kind = T_const; - else if (token_buffer[0] == 'f' && - !memcmp(token_buffer, "float", 5)) - kind = T_float; - break; - - case 6: /* 6-letter keywords: return, struct, switch, sizeof, static, - extern, inline - */ - if (token_buffer[0] == 'r' && !memcmp(token_buffer, "return", 6)) - kind = T_return; - else if (token_buffer[0] == 'e' && - !memcmp(token_buffer, "extern", 6)) - kind = T_extern; - else if (token_buffer[0] == 'i' && - !memcmp(token_buffer, "inline", 6)) - kind = T_inline; - else if (token_buffer[0] == 'd' && - !memcmp(token_buffer, "double", 6)) - kind = T_double; - else if (token_buffer[0] == 's') { - if (!memcmp(token_buffer, "struct", 6)) - kind = T_struct; - else if (!memcmp(token_buffer, "switch", 6)) - kind = T_switch; - else if (!memcmp(token_buffer, "sizeof", 6)) - kind = T_sizeof; - else if (!memcmp(token_buffer, "static", 6)) - kind = T_static; - } - break; - - case 7: /* 7-letter keywords: typedef, default */ - if (!memcmp(token_buffer, "typedef", 7)) - kind = T_typedef; - else if (!memcmp(token_buffer, "default", 7)) - kind = T_default; - break; - - case 8: /* 8-letter keywords: continue, register, restrict, volatile */ - if (!memcmp(token_buffer, "continue", 8)) - kind = T_continue; - else if (!memcmp(token_buffer, "register", 8)) - kind = T_register; - else if (!memcmp(token_buffer, "restrict", 8)) - kind = T_restrict; - else if (!memcmp(token_buffer, "volatile", 8)) - kind = T_volatile; - break; - - default: - /* Keywords longer than 8 chars or identifiers - use hashmap */ - break; - } - - /* Fall back to the hashmap for anything the switch does not name. No - * keyword is shorter than two characters or longer than eight, so a - * name outside that range cannot be one and needs no lookup -- which is - * most of the identifiers in a real program. - */ - if (kind == T_identifier && !strcmp(token_buffer, "_Imaginary")) - kind = T_imaginary; - else if (kind == T_identifier && sz >= 2 && sz <= 8) - kind = lookup_keyword(token_buffer); + token_kind_t kind = lookup_keyword(token_buffer, sz); token = new_token(kind, loc, source_len); token->literal = intern_string(token_buffer); @@ -1688,6 +1363,18 @@ static char *normalize_filename(const char *filename) return intern_string(path); } +static token_stream_t *cache_token_stream(char *filename, + token_t *head, + token_t *tail) +{ + token_stream_t *stream = malloc(sizeof(*stream)); + + stream->head = head; + stream->tail = tail; + hashmap_put_borrowed(TOKEN_CACHE, filename, stream); + return stream; +} + token_stream_t *gen_file_token_stream(char *filename) { token_t head; @@ -1734,11 +1421,7 @@ token_stream_t *gen_file_token_stream(char *filename) error_at("Internal error, expected eof at the end of file", &cur->location); - tks = malloc(sizeof(token_stream_t)); - tks->head = head.next; - tks->tail = cur; - hashmap_put(TOKEN_CACHE, filename, tks); - return tks; + return cache_token_stream(filename, head.next, cur); } token_stream_t *gen_libc_token_stream(void) @@ -1756,12 +1439,12 @@ token_stream_t *gen_libc_token_stream(void) return tks; if (!hashmap_contains(SRC_FILE_MAP, filename)) - hashmap_put(SRC_FILE_MAP, filename, LIBC_SRC); + hashmap_put_borrowed(SRC_FILE_MAP, filename, LIBC_SRC); /* This buffer was built by appending, so its capacity is whatever the * doubling left and runs past the text into memory that was never written * -- while the scan below, like the one over a file, stops at capacity. - * Terminate it the way read_file() leaves a file: the text, a NUL, and + * Terminate it the way try_read_file() leaves a file: the text, a NUL, and * capacity naming one past the text. Without this the lexer reads * uninitialised bytes, and what it finds there depends on the allocator, * which is enough to make the compiler emit different code from one build @@ -1796,11 +1479,7 @@ token_stream_t *gen_libc_token_stream(void) error_at("Internal error, expected eof at the end of file", &cur->location); - tks = malloc(sizeof(token_stream_t)); - tks->head = head.next; - tks->tail = cur; - hashmap_put(TOKEN_CACHE, filename, tks); - return tks; + return cache_token_stream(filename, head.next, cur); } /* Fetches current token's location. */ diff --git a/src/main.c b/src/main.c index 24fc2fe0..35862fe8 100644 --- a/src/main.c +++ b/src/main.c @@ -15,6 +15,7 @@ /* The inclusion must follow the fixed order, otherwise it fails to build. */ #include "defs.h" +#include "vir-frontend.h" /* Initialize global objects */ #include "globals.c" @@ -31,11 +32,12 @@ /* C language syntactic analyzer */ #include "parser.c" -/* architecture-independent middle-end */ -#include "ssa.c" +/* Native value IR construction and optimization. */ +#include "vir-frontend.c" +#include "vir.c" /* Register allocator */ -#include "reg-alloc.c" +#include "vir-lower.c" /* Peephole optimization */ #include "peephole.c" @@ -99,30 +101,236 @@ char *dot_output_name(char *input) return output; } + +static void check_native_pass(const char *name, int result) +{ + if (result < 0) { + fprintf(stderr, "VIR %s failed in %s\n", name, + fatal_function_context ? fatal_function_context : "globals"); + fatal("VIR optimization failed"); + } +} + +static void optimize_native_graph(vir_function_t *graph) +{ + vir_opt_level_t level = optimize ? VIR_OPT_O2 : VIR_OPT_O0; + + if (!vir_remove_unreachable(graph)) + fatal("Invalid VIR reachability"); + check_native_pass("sccp", vir_sccp(graph, level)); + check_native_pass("cfg", vir_simplify_cfg(graph, level)); + check_native_pass("cse", vir_local_cse(graph, level)); + check_native_pass("gvn", vir_gvn_with_stats(graph, level, NULL)); + check_native_pass("licm", vir_licm(graph, level)); + check_native_pass("strength", vir_strength_reduce(graph, level)); + check_native_pass("cse", vir_local_cse(graph, level)); + check_native_pass("dce", vir_dce(graph, level)); + char *error = NULL; + if (!vir_verify(graph, &error)) { + fprintf(stderr, "Invalid optimized VIR in %s: %s\n", + fatal_function_context ? fatal_function_context : "globals", + error ? error : "verification failed"); + free(error); + fatal("Invalid optimized VIR"); + } +} + +static void mark_native_references(const vir_function_t *graph) +{ + for (const vir_block_t *block = graph->blocks; block; block = block->next) { + for (const vir_effect_t *effect = block->effects; effect; + effect = effect->next) + if (effect->kind == VIR_EFFECT_CALL && effect->callee) { + func_t *target = find_func((char *) effect->callee); + if (target) + target->is_used = true; + } + for (const vir_value_t *value = block->head; value; value = value->next) + if (value->opcode == VIR_OP_FUNC_ADDR) { + func_t *target = find_func((char *) value->address_name); + if (target) + target->is_used = true; + } + } +} + +static void lower_native_function(func_t *func, bool global) +{ + vir_function_t *graph = (vir_function_t *) vir_frontend_function(func); + vir_machine_function_t machine = {0}; + int global_bytes = GLOBAL_FUNC->stack_size; + if (!vir_lower_machine(graph, vir_frontend_signature(func), + vir_frontend_global_offset, NULL, &machine)) + fatal("VIR machine lowering failed"); + func->bbs = machine.entry; + if (!global) { + func->stack_size = machine.stack_size; + if (!strcmp(func->return_def.var_name, "main")) + MAIN_BB = machine.entry; + } else { + int scratch_base = + machine.stack_size ? (global_bytes + 7) & -8 : global_bytes; + if (machine.block_count != 1) + fatal("Unexpected control flow in global initializer"); + ph2_ir_t *previous = NULL; + for (ph2_ir_t *ir = machine.entry->ph2_ir_list.head; ir; + ir = ir->next) { + if (ir->op == OP_load) { + ir->op = OP_global_load; + ir->src0 += scratch_base; + } else if (ir->op == OP_store) { + ir->op = OP_global_store; + ir->src1 += scratch_base; + } else if (ir->op == OP_address_of) { + ir->op = OP_global_address_of; + ir->src0 += scratch_base; + } else if (ir->op == OP_return) { + if (previous) + previous->next = NULL; + else + machine.entry->ph2_ir_list.head = NULL; + machine.entry->ph2_ir_list.tail = previous; + break; + } + previous = ir; + } + func->stack_size = scratch_base + machine.stack_size; + } + for (basic_block_t *block = machine.entry; block; block = block->rpo_next) + block->belong_to = func; +} + +static void compile_native_vir(const char *output) +{ + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + func->is_used = false; + func->visited = 0; + } + vir_frontend_layout_globals(); + vir_function_t *global = + (vir_function_t *) vir_frontend_function(GLOBAL_FUNC); + if (global) { + optimize_native_graph(global); + mark_native_references(global); + } + func_t *main_func = find_func("main"); + if (main_func) + main_func->is_used = true; + else + for (func_t *func = FUNC_LIST.head; func; func = func->next) + func->is_used = vir_frontend_function(func) != NULL; + bool changed; + do { + changed = false; + for (func_t *func = FUNC_LIST.head; func; func = func->next) { + vir_function_t *graph = + (vir_function_t *) vir_frontend_function(func); + if (!func->is_used || !graph || func->visited) + continue; + func->visited = true; + fatal_function_context = func->return_def.var_name; + optimize_native_graph(graph); + mark_native_references(graph); + changed = true; + } + } while (changed); + FILE *dot = dump_dot ? fopen(output, "w") : NULL; + if (dump_dot && !dot) + fatal("Cannot open DOT output"); + if (dot) + fprintf(dot, "digraph VIR {\n"); + int function_index = 0; + for (func_t *func = FUNC_LIST.head; func; + func = func->next, function_index++) { + vir_function_t *graph = (vir_function_t *) vir_frontend_function(func); + if (func->is_used && !graph) { + char message[MAX_LINE_LEN]; + if (func->is_static || !dynlink) { + snprintf(message, MAX_LINE_LEN, "undefined %sfunction '%s'", + func->is_static ? "static " : "", + function_source_name(func)); + error_at(message, NULL); + } + if (func->returns_aggregate) { + snprintf(message, MAX_LINE_LEN, + "aggregate-return function '%s' has its address taken " + "but is not defined", + func->return_def.var_name); + error_at(message, NULL); + } + } + if (!func->is_used || !graph) { + func->bbs = NULL; + continue; + } + if (dump_ir || dump_vir) { + FILE *stream = dump_ir ? stdout : stderr; + fprintf(stream, "function %s\n", func->return_def.var_name); + vir_print(graph, stream); + } + if (dot) { + for (vir_block_t *block = graph->blocks; block; + block = block->next) { + fprintf(dot, "f%d_b%d [label=\"%s block %d\"];\n", + function_index, block->id, func->return_def.var_name, + block->id); + for (vir_edge_t *edge = block->outgoing; edge; + edge = edge->next_outgoing) + fprintf(dot, "f%d_b%d -> f%d_b%d;\n", function_index, + block->id, function_index, edge->to->id); + } + } else + lower_native_function(func, false); + } + if (dot) { + fprintf(dot, "}\n"); + fclose(dot); + } else if (global) + lower_native_function(GLOBAL_FUNC, true); + dump_stats_phase("vir"); +} + int main(int argc, char *argv[]) { + static const struct { + const char *name; + bool *setting; + bool value; + } flags[] = { + {"--dump-ir", &dump_ir, true}, + {"--stats", &dump_stats, true}, + {"--dump-vir", &dump_vir, true}, + {"--warn-string-literals", &warn_string_literals, true}, + {"--std=c99", &strict_c99, true}, + {"--dot", &dump_dot, true}, + {"+m", &hard_mul_div, true}, + {"--no-libc", &libc, false}, + {"--no-opt", &optimize, false}, + {"--dynlink", &dynlink, true}, + {"-E", &expand_only, true}, + }; char *out = NULL; - char *in = NULL; + char **inputs; + int input_count = 0; token_stream_t *libc_token_stream = NULL, *token_stream; - token_t *tk; + token_t *input_tokens = NULL, *input_tail = NULL; bool has_binding_option = false; + inputs = malloc((size_t) argc * sizeof(*inputs)); + if (!inputs) + fatal("Unable to allocate input file list"); + for (int i = 1; i < argc; i++) { - if (!strcmp(argv[i], "--dump-ir")) - dump_ir = true; - else if (!strcmp(argv[i], "--warn-string-literals")) - warn_string_literals = true; - else if (!strcmp(argv[i], "--std=c99")) - strict_c99 = true; - else if (!strcmp(argv[i], "--dot")) - dump_dot = true; - else if (!strcmp(argv[i], "+m")) - hard_mul_div = true; - else if (!strcmp(argv[i], "--no-libc")) - libc = false; - else if (!strcmp(argv[i], "--dynlink")) - dynlink = true; - else if (!strcmp(argv[i], "-z")) { + size_t flag = 0; + + while (flag < sizeof(flags) / sizeof(*flags) && + strcmp(argv[i], flags[flag].name)) + flag++; + if (flag < sizeof(flags) / sizeof(*flags)) { + *flags[flag].setting = flags[flag].value; + continue; + } + if (!strcmp(argv[i], "-z")) { if (i + 1 >= argc) usage_error("-z requires \"lazy\" or \"now\""); @@ -134,9 +342,7 @@ int main(int argc, char *argv[]) else usage_error("-z requires \"lazy\" or \"now\""); i++; - } else if (!strcmp(argv[i], "-E")) - expand_only = true; - else if (!strcmp(argv[i], "-I")) { + } else if (!strcmp(argv[i], "-I")) { if (i + 1 >= argc) usage_error("-I requires an include directory"); if (include_dirs_idx == MAX_INCLUDE_DIRS) @@ -155,15 +361,16 @@ int main(int argc, char *argv[]) } else if (argv[i][0] == '-') { usage_error("Unidentified option"); } else - in = argv[i]; + inputs[input_count++] = argv[i]; } - if (!in) { + if (!input_count) { printf( - "Usage: shecc [-I directory] [-o output] [+m] [--dot] " - "[--dump-ir] " + "Usage: shecc [-I directory] [-o output] [+m] [--dot] [--dump-ir] " + "[--dump-vir] [--stats] " + "[--no-opt] " "[--warn-string-literals] [--std=c99] [--no-libc] " - "[--dynlink] [-z ] [-E] \n"); + "[--dynlink] [-z ] [-E] [input.c ...]\n"); usage_error("Missing source file"); } @@ -177,18 +384,25 @@ int main(int argc, char *argv[]) if (dump_dot && dump_ir) usage_error("--dot cannot be combined with --dump-ir"); - if (dump_dot && !out) - out = dot_output_name(in); + if (dump_dot && !out) { + if (input_count != 1) + usage_error("--dot requires -o with multiple input files"); + out = dot_output_name(inputs[0]); + } /* The graph is written by truncating its output, so naming the input * destroys the source. That happens both when -o names it outright and when * an input already ending in .dot derives its own name. */ - if (dump_dot && !strcmp(out, in)) - usage_error("--dot would overwrite the input; name another output"); + if (dump_dot) + for (int i = 0; i < input_count; i++) + if (!strcmp(out, inputs[i])) + usage_error( + "--dot would overwrite an input; name another output"); /* initialize global objects */ global_init(); + dump_stats_layout(); /* include libc */ if (libc) { @@ -198,118 +412,80 @@ int main(int argc, char *argv[]) libc_token_stream = gen_libc_token_stream(); } - token_stream = gen_file_token_stream(in); - - /* concat libc's and input file's token stream */ - if (libc_token_stream) { - libc_token_stream->tail->next = token_stream->head; - token_stream = libc_token_stream; + /* Preprocess each input independently so macro definitions and #pragma once + * state do not leak between translation units. The existing implicit libc + * prelude remains attached to the first input only. + */ + for (int i = 0; i < input_count; i++) { + token_t *unit_tokens; + + token_stream = gen_file_token_stream(inputs[i]); + if (i == 0 && libc_token_stream) { + libc_token_stream->tail->next = token_stream->head; + token_stream = libc_token_stream; + } + unit_tokens = preprocess(token_stream->head); + if (!unit_tokens) + fatal("Preprocessor produced no input tokens"); + if (input_tail) { + /* An EOF ends preprocessing for one input, but not compilation of + * the next input. Reuse it as an explicit parser-visible boundary + * that also preserves the line separator emitted by -E. + */ + input_tail->kind = T_translation_unit; + input_tail->next = unit_tokens; + } else { + input_tokens = unit_tokens; + } + input_tail = unit_tokens; + while (input_tail->kind != T_eof && input_tail->next) + input_tail = input_tail->next; + if (input_tail->kind != T_eof) + fatal("Preprocessor output has no end-of-file token"); } - tk = preprocess(token_stream->head); - if (expand_only) { - emit_preprocessed_token(tk); + emit_preprocessed_token(input_tokens); + free(inputs); exit(0); } - /* load and parse source code into IR */ - parse(pp_strip_layout(tk)); + /* load and parse source code into IR; each input is one unit */ + translation_unit_count = input_count; + parse(pp_strip_layout(input_tokens)); + free(inputs); + dump_stats_phase("parse"); /* Tokens, macros and the source buffers they point into are dead once * parsing finishes; releasing them here keeps 17 MiB out of the peak. */ release_token_arena(); - /* Compact arenas after parsing to free temporary parse structures */ - compact_all_arenas(); - - ssa_build(); - - /* Like lacc's -dot target, visualize the SSA CFG directly rather than - * lowering it into an executable. Prune first: every compile prepends the - * whole of lib/c.c, and the functions nothing reaches are noise in the - * picture -- two thirds of the graph on tests/fib.c. - */ + /* Finalize graphs after all declarations and initializers are known. */ + vir_frontend_finish_globals(); + compile_native_vir(out); if (dump_dot) { - prune_unused_funcs(); - dump_cfg(out); + vir_frontend_release(); global_release(); exit(0); } - unwind_phi(); - - /* Copy small helpers into their callers before anything else looks at them, - * so the optimizer sees one body rather than a call boundary. - */ - inline_calls(); - - /* dump first phase IR */ - if (dump_ir) - dump_insn(); - - /* Nothing beyond this point should spend time on functions no call can - * reach; on a small input, almost all of the embedded libc is that. - */ - prune_unused_funcs(); - - /* SSA-based optimization */ - optimize(); - - /* Flatten unpredictable ifs into branchless selects. - * - * After the optimizer rather than inside ssa_build(): a select reads a - * third operand, and the passes in optimize() walk instructions two sources - * at a time. One of them would rewrite a copy feeding that third operand - * and leave the select reading a value nothing defines. - */ - if_convert(); - - /* Send short-circuit arms straight to their destination. */ - thread_const_branches(); - - /* Both passes above rewired the CFG the dominator tree was built from, and - * everything below asks that tree which edges close a loop. - */ - rebuild_dom(); - - /* Walk arrays with a pointer rather than recomputing addresses. */ - strength_reduce(); - - /* Compact arenas after SSA optimization to free temporary SSA structures */ - compact_all_arenas(); - - /* Give each register-pair operation operands of its width. */ - widen_pair_operands(); - - /* SSA-based liveness analyses */ - liveness_analysis(); - - /* Compact after liveness analysis - mainly traversal args in GENERAL_ARENA - */ - compact_arenas_selective(COMPACT_ARENA_GENERAL); - - /* allocate register from IR */ - reg_alloc(); - - /* Compact after register allocation - mainly INSN and BB arenas */ - compact_arenas_selective(COMPACT_ARENA_INSN | COMPACT_ARENA_BB); - + ph2_compute_liveness(); peephole(); /* Apply arch-specific IR tweaks before final codegen */ arch_lower(); + ph2_compute_liveness(); /* flatten CFG to linear instruction */ cfg_flatten(); + dump_stats_phase("lower"); + vir_frontend_release(); /* Compact after CFG flattening - BB and GENERAL no longer needed */ compact_arenas_selective(COMPACT_ARENA_BB | COMPACT_ARENA_GENERAL); - /* dump second phase IR */ - if (dump_ir) - dump_ph2_ir(); + /* ELF preprocess: * 1. generate all sections except for .text section. diff --git a/src/opt-sccp.c b/src/opt-sccp.c deleted file mode 100644 index d5ce04e6..00000000 --- a/src/opt-sccp.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * shecc - Self-Hosting and Educational C Compiler. - * - * shecc is freely redistributable under the BSD 2 clause license. See the file - * "LICENSE" for information on usage and redistribution of this file. - */ - -/* Constant cast optimization pass. - * - * Folds the 'load_constant' + 'trunc' pattern into a single constant load. - * General constant propagation and folding live in const_folding() in ssa.c, - * which runs over the same IR immediately afterwards. - */ - -/* Narrow a constant to 'size' bytes, keeping its sign. A signed narrow type - * sign-extends when widened, so masking alone would make "char c = -1" compare - * as 255. A size the caller does not narrow is returned unchanged. - */ -int sign_extend_const(int value, int size) -{ - if (size == 1) { - value = value & 0xFF; - if (value & 0x80) - value = value | ~0xFF; - } else if (size == 2) { - value = value & 0xFFFF; - if (value & 0x8000) - value = value | ~0xFFFF; - } - return value; -} - -/* Targeted constant truncation peephole optimization */ -bool optimize_constant_casts(func_t *func) -{ - if (!func || !func->bbs) - return false; - - bool changed = false; - - /* Simple peephole optimization: const + trunc pattern */ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn && insn->next; - insn = insn->next) { - insn_t *next_insn = insn->next; - - /* Look for pattern: const %.tX, VALUE followed by %.tY = trunc - * %.tX, SIZE - */ - if (insn->opcode == OP_load_constant && - next_insn->opcode == OP_trunc && insn->rd && next_insn->rs1 && - insn->rd == next_insn->rs1 && next_insn->sz > 0 && - !next_insn->rd->is_global) { - int value = insn->rd->init_val; - int result = value; - - if (next_insn->sz != 1 && next_insn->sz != 2 && - next_insn->sz != 4) - continue; /* not a width we narrow to */ - result = sign_extend_const(value, next_insn->sz); - - /* An unsigned narrow type zero-extends instead: folding - * "unsigned short x = 0xfffe" to -2 made (int) x compare as -2. - */ - bool is_unsigned = next_insn->rd->type && - next_insn->rd->type->is_unsigned && - !next_insn->rd->ptr_level; - if (is_unsigned && next_insn->sz < 4) - result = value & ((1 << (next_insn->sz * 8)) - 1); - - /* Optimize: Replace both instructions with single const */ - insn->rd = next_insn->rd; /* Update dest to final target */ - insn->rd->is_const = true; - insn->rd->init_val = result; - - /* The high word belongs to the narrowed value too. A 64-bit - * target loads an argument constant at full width, and a zero - * high word made "f(0xe231fffab478ULL)" pass its int parameter - * as 0xfffab478 rather than -347016. - */ - insn->rd->init_val_hi = result < 0 && !is_unsigned ? -1 : 0; - - /* Remove the truncation instruction by converting it to - * NOP-like - */ - next_insn->opcode = OP_load_constant; - next_insn->rd->is_const = true; - next_insn->rd->init_val = result; - next_insn->rs1 = NULL; - next_insn->sz = 0; - - changed = true; - } - } - } - - return changed; -} diff --git a/src/parser-call.c b/src/parser-call.c index 8d759fac..cad6b75d 100644 --- a/src/parser-call.c +++ b/src/parser-call.c @@ -26,10 +26,7 @@ void read_func_parameters_with_sret(func_t *func, lex_expect(T_open_bracket); while (!lex_accept(T_close_bracket)) { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); param = opstack_pop(); param = materialize_function_designator(parent, bb, param); @@ -69,13 +66,11 @@ void read_func_parameters_with_sret(func_t *func, if (!is_record_object(param)) error_at("Record argument required", cur_token_loc()); - var_t *copy = require_typed_var(parent, target->type); - copy->var_name = gen_name(); + var_t *copy = name_var(require_typed_var(parent, target->type)); add_insn(parent, *bb, OP_allocat, copy, NULL, NULL, 0, NULL); emit_record_copy(parent, bb, copy, param); - param = require_ref_var(parent, target->type, 0); - param->var_name = gen_name(); + param = name_var(require_ref_var(parent, target->type, 0)); add_insn(parent, *bb, OP_address_of, param, copy, NULL, 0, NULL); } else if (!has_effective_pointer(target) && !target->array_size) { @@ -96,12 +91,9 @@ void read_func_parameters_with_sret(func_t *func, if (!is_record_object(param)) error_at("Record argument required", cur_token_loc()); var_t *copy = require_typed_var(parent, param->type); - - copy->var_name = gen_name(); add_insn(parent, *bb, OP_allocat, copy, NULL, NULL, 0, NULL); emit_record_copy(parent, bb, copy, param); - param = require_ref_var(parent, param->type, 0); - param->var_name = gen_name(); + param = name_var(require_ref_var(parent, param->type, 0)); add_insn(parent, *bb, OP_address_of, param, copy, NULL, 0, NULL); } else if (!is_pointer_like_value(param) && @@ -203,6 +195,11 @@ void copy_call_result_array_shape(var_t *result, const var_t *return_def) : return_def->type->array_element_ptr_level); result->is_const_qualified = return_def->is_const_qualified || return_def->type->is_const_qualified; + result->pointer_const_mask = return_def->pointer_const_mask; + result->is_volatile = + return_def->is_volatile || return_def->type->is_volatile_qualified; + result->pointer_volatile_mask = return_def->pointer_volatile_mask | + return_def->type->pointer_volatile_mask; } /* The element type of array typedef @type, or @type itself when it is none. A @@ -233,17 +230,17 @@ static var_t *array_subscript_remainder(block_t *parent, for (int i = 0; i < consumed; i++) fixed_array_shape_drop_outer(&shape); if (unevaluated_expression_depth) { - result = require_typed_ptr_var(parent, type, element_ptr_level); + result = + name_var(require_typed_ptr_var(parent, type, element_ptr_level)); fixed_array_shape_to_var(result, &shape); - result->var_name = gen_name(); return result; } fixed_array_shape_drop_outer(&shape); - result = require_typed_ptr_var(parent, type, element_ptr_level + 1); + result = + name_var(require_typed_ptr_var(parent, type, element_ptr_level + 1)); fixed_array_shape_to_pointee_var(result, &shape); if (shape.rank) result->pointee_array_element_ptr_level = element_ptr_level; - result->var_name = gen_name(); add_insn(parent, *bb, OP_assign, result, address, NULL, 0, NULL); return result; } @@ -282,7 +279,6 @@ void lower_call_result_array_postfix(var_t **value, opstack_pop(); address = base; do { - var_t *index; int stride = element_ptr_level ? PTR_SIZE : base->type->size; if (depth >= dimensions) @@ -294,37 +290,14 @@ void lower_call_result_array_postfix(var_t **value, stride = fixed_array_shape_stride(&shape, depth - 1, stride); lex_expect(T_open_square); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (stride != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, base->type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = + read_subscript_address(parent, bb, address, base->type, 1, stride); depth++; } while (lex_peek(T_open_square, NULL)); if (depth == dimensions) { - var_t *element = require_typed_var(parent, element_type); - opcode_t compound_op = OP_generic; - bool assignment; - bool record; - - element->ptr_level = element_ptr_level; + var_t *element = name_var( + require_typed_ptr_var(parent, element_type, element_ptr_level)); /* A call returning a pointer to an array of callbacks carries their * prototype on the result rather than on its scalar base type. @@ -334,101 +307,8 @@ void lower_call_result_array_postfix(var_t **value, : base->func_signature; element->is_const_qualified = base->is_const_qualified || base->type->is_const_qualified; - element->var_name = gen_name(); - - /* A record element is no register value: it stays at its address for a - * member selection or a record copy to read, as push_object_at() leaves - * one. - */ - record = is_record_object(element) && !element->func_signature && - !is_incomplete_record_object(element); - if (record) - element->defers_record_copy = true; - else - add_insn(parent, *bb, OP_read, element, address, NULL, - element->ptr_level ? PTR_SIZE : base->type->size, NULL); - element->is_compound_literal_reference = true; - element->compound_literal_address = address; - - assignment = - lex_accept(T_assign) || accept_compound_assign_op(&compound_op); - if (assignment) { - var_t *assigned; - - if (element->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - assigned = opstack_pop(); - if (record) { - if (compound_op != OP_generic || !is_record_object(assigned)) - error_at("Invalid record assignment", cur_token_loc()); - emit_record_copy_to_address(parent, bb, address, assigned); - *value = assigned; - opstack_push(*value); - return; - } - if (compound_op != OP_generic) { - if (is_pointer_operation(compound_op, element, assigned)) { - handle_pointer_arithmetic(parent, bb, compound_op, element, - assigned); - assigned = opstack_pop(); - } else { - var_t *current = - integer_promote_operand(parent, bb, element); - - assigned = integer_promote_operand(parent, bb, assigned); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &assigned); - var_t *combined = require_var(parent); - - combined->var_name = gen_name(); - combined->type = integer_binary_result_type( - compound_op, current, assigned); - add_insn(parent, *bb, compound_op, combined, current, - assigned, 0, NULL); - assigned = combined; - } - } - assigned = resize_var(parent, bb, assigned, element); - add_insn(parent, *bb, OP_write, NULL, address, assigned, - element->ptr_level ? PTR_SIZE : base->type->size, NULL); - *value = assigned; - opstack_push(*value); - return; - } - - if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { - opcode_t op = lex_accept(T_increment) ? OP_add : OP_sub; - var_t *one; - var_t *updated; - - if (record) - error_at( - "Increment or decrement requires a scalar modifiable " - "lvalue", - cur_token_loc()); - if (element->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, element, one)) { - handle_pointer_arithmetic(parent, bb, op, element, one); - updated = opstack_pop(); - } else { - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, element, one); - add_insn(parent, *bb, op, updated, element, one, 0, NULL); - } - updated = resize_var(parent, bb, updated, element); - add_insn(parent, *bb, OP_write, NULL, address, updated, - element->ptr_level ? PTR_SIZE : base->type->size, NULL); - } + push_object_at(parent, bb, element, address, + element->ptr_level ? PTR_SIZE : base->type->size); *value = element; } else { /* The first subscript selected the pointed-to array itself. */ @@ -437,16 +317,7 @@ void lower_call_result_array_postfix(var_t **value, address); (*value)->is_const_qualified = base->is_const_qualified || base->type->is_const_qualified; - } - opstack_push(*value); - var_t *callable = *value; - if (callable->func_signature && lex_peek(T_open_bracket, NULL)) { - func_t *signature = callable->func_signature; - - var_t *result = - emit_indirect_call_result(callable, signature, true, parent, bb); - *value = result; - lower_call_result_array_postfix(value, parent, bb); + opstack_push(*value); } } @@ -457,7 +328,7 @@ void lower_call_result_prefix_update(var_t **value, { if (op == OP_generic) return; - *value = lower_reference_update(opstack_pop(), op, parent, bb); + *value = lower_reference_update(opstack_pop(), op, parent, bb, true); opstack_push(*value); } @@ -466,46 +337,11 @@ void lower_call_result_prefix_update(var_t **value, */ void read_builtin_offsetof(block_t *parent, basic_block_t **bb) { - char name[MAX_ID_LEN]; - type_t *record; - var_t *field; var_t *result; - int offset = 0; - bool has_nested_member; - - lex_expect(T_identifier); - lex_expect(T_open_bracket); - base_type_t record_kind = accept_record_keyword(); - if (record_kind) { - lex_ident(T_identifier, name); - record = find_record_tag(name, parent, record_kind); - } else { - lex_ident(T_identifier, name); - record = find_type(name, true); - } - if (!is_record_type(record)) - error_at("offsetof requires a struct or union type", cur_token_loc()); - lex_expect(T_comma); - do { - lex_ident(T_identifier, name); - field = find_member(name, record); - if (!field) - error_at("Unknown record member", cur_token_loc()); - offset += field->offset; - offset += read_offsetof_array_subscripts(field, parent); - record = field->type; - has_nested_member = lex_accept(T_dot); - if (has_nested_member && (field->ptr_level || !is_record_type(record))) - error_at("Nested offsetof member requires a record", - cur_token_loc()); - } while (has_nested_member); - lex_expect(T_close_bracket); - result = require_typed_var(parent, find_type("size_t", true)); - result->var_name = gen_name(); - result->init_val = offset; + result = load_constant(parent, *bb, read_offsetof_designator(parent), + find_builtin_type("size_t")); result->is_const = true; opstack_push(result); - add_insn(parent, *bb, OP_load_constant, result, NULL, NULL, 0, NULL); } /* Parse a C99 type name for the va_arg intrinsic. A type name has no @@ -551,9 +387,7 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) ; while (lex_accept(T_asterisk)) { result->ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } /* An array typedef is not itself an object type suitable for va_arg, but @@ -598,9 +432,7 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) while (lex_accept(T_asterisk)) { nested_ptr++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } if (!nested_ptr) error_at("va_arg type name needs an abstract pointer declarator", @@ -611,8 +443,7 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) while (lex_peek(T_open_square, NULL) || lex_peek(T_open_bracket, NULL)) { if (lex_accept(T_open_square)) { - int dims = 0; - int total = 0; + fixed_array_shape_t pointee_shape = {0}; if (type->base_type == TYPE_void) error_at( @@ -631,35 +462,15 @@ void read_builtin_va_arg_type(block_t *parent, va_arg_type_t *result) * used by normal declarations so the first subscript advances a * whole row (or plane), not one scalar element. */ - do { - int bound; - - if (dims >= 4) - error_at( - "Array type names support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) - error_at("pointer-to-array type needs a bound", - cur_token_loc()); - bound = read_const_expr(parent); - if (bound <= 0) - error_at("pointer-to-array bound must be positive", - cur_token_loc()); - lex_expect(T_close_square); - if (!dims) - total = bound; - else { - total *= bound; - if (dims == 1) - result->pointee_array_dim2 = bound; - else if (dims == 2) - result->pointee_array_dim3 = bound; - else - result->pointee_array_dim4 = bound; - } - dims++; - } while (lex_accept(T_open_square)); - result->pointee_array_size = total; + read_array_shape_bounds( + parent, &pointee_shape, + "Array type names support at most four dimensions", + "pointer-to-array type needs a bound", + "pointer-to-array bound must be positive", true, true); + fixed_array_shape_apply( + &pointee_shape, &result->pointee_array_size, + &result->pointee_array_dim2, &result->pointee_array_dim3, + &result->pointee_array_dim4); result->pointee_element_size = type->size; result->pointee_element_ptr_level = return_ptr_level + type->ptr_level; @@ -727,9 +538,7 @@ void emit_record_copy_from_address(block_t *parent, var_t *dest, var_t *src_addr) { - var_t *dest_addr = require_ref_var(parent, dest->type, 0); - - dest_addr->var_name = gen_name(); + var_t *dest_addr = name_var(require_ref_var(parent, dest->type, 0)); add_insn(parent, *bb, OP_address_of, dest_addr, dest, NULL, 0, NULL); emit_record_copy_between(parent, bb, dest_addr, src_addr, size_var(dest)); } @@ -786,14 +595,16 @@ void push_object_at(block_t *parent, } /* Apply ++ or -- (@op is OP_add or OP_sub) to @object, a value that records the - * address it was loaded from, and store the result there. + * address it was loaded from, and store the result there. Postfix callers can + * skip the bit-field reload when they only need the old value. * * Returns the value the object then holds. */ var_t *lower_reference_update(var_t *object, opcode_t op, block_t *parent, - basic_block_t **bb) + basic_block_t **bb, + bool need_updated_value) { var_t *one; var_t *updated; @@ -809,16 +620,12 @@ var_t *lower_reference_update(var_t *object, if (effective_pointer_depth(object) ? object->is_const_pointer : object->is_const_qualified) error_at("assignment of read-only location", cur_token_loc()); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); + one = load_constant(parent, *bb, 1, TY_int); if (is_pointer_operation(op, object, one)) { handle_pointer_arithmetic(parent, bb, op, object, one); updated = opstack_pop(); } else { - updated = require_var(parent); - updated->var_name = gen_name(); + updated = require_named_var(parent); updated->type = integer_binary_result_type(op, object, one); add_insn(parent, *bb, op, updated, object, one, 0, NULL); } @@ -826,10 +633,12 @@ var_t *lower_reference_update(var_t *object, if (object->compound_literal_bitfield) { write_bitfield_value(parent, bb, object->compound_literal_address, updated, object->compound_literal_bitfield); - updated = - read_bitfield_value(parent, bb, object->compound_literal_address, - object->compound_literal_bitfield); - updated->is_bitfield = false; + if (need_updated_value) { + updated = read_bitfield_value(parent, bb, + object->compound_literal_address, + object->compound_literal_bitfield); + updated->is_bitfield = false; + } } else add_insn(parent, *bb, OP_write, NULL, object->compound_literal_address, updated, @@ -840,6 +649,36 @@ var_t *lower_reference_update(var_t *object, return updated; } +void read_builtin_va_start(block_t *parent, basic_block_t **bb) +{ + lex_expect(T_identifier); + lex_expect(T_open_bracket); + read_assignment_or_expression(parent, bb); + var_t *destination = opstack_pop(); + if (!has_effective_pointer(destination)) + error_at("va_start first argument must address a va_list", + cur_token_loc()); + lex_expect(T_comma); + char name[MAX_VAR_LEN]; + int parentheses = 0; + while (lex_accept(T_open_bracket)) + parentheses++; + lex_peek(T_identifier, name); + lex_expect(T_identifier); + while (parentheses--) + lex_expect(T_close_bracket); + func_t *func = parent->func; + if (!func || !func->va_args || !func->num_params || + find_var(name, parent) != &func->param_defs[func->num_params - 1]) + error_at("va_start requires the last named variadic parameter", + cur_token_loc()); + lex_expect(T_close_bracket); + var_t *value = name_var(require_typed_ptr_var(parent, TY_int, 1)); + add_insn(parent, *bb, OP_va_start, value, NULL, NULL, 0, NULL); + add_insn(parent, *bb, OP_write, NULL, destination, value, PTR_SIZE, NULL); + opstack_push(name_var(require_typed_var(parent, TY_void))); +} + void read_builtin_va_arg(block_t *parent, basic_block_t **bb) { va_arg_type_t requested; @@ -850,10 +689,7 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) lex_expect(T_identifier); lex_expect(T_open_bracket); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); ap_addr = opstack_pop(); if (!has_effective_pointer(ap_addr)) error_at("va_arg first argument must be va_list lvalue", @@ -870,11 +706,9 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) if (requested.is_array || requested.is_function) error_at("va_arg type must be an object type", cur_token_loc()); - old = require_typed_ptr_var(parent, TY_int, 1); - old->var_name = gen_name(); + old = name_var(require_typed_ptr_var(parent, TY_int, 1)); add_insn(parent, *bb, OP_read, old, ap_addr, NULL, PTR_SIZE, NULL); - step = require_typed_var(parent, TY_int); - step->var_name = gen_name(); + step = name_var(require_typed_var(parent, TY_int)); /* A long long on a 32-bit target takes two ABI words starting at an even * one, both for AAPCS32 and for a variadic argument on RV32, and the callee @@ -883,23 +717,23 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) */ if (PTR_SIZE < 8 && !requested.ptr_level && !is_record_type(requested.type) && requested.type->size == 8) { - var_t *mask = require_typed_var(parent, TY_int); - var_t *aligned = require_typed_ptr_var(parent, TY_int, 1); + var_t *mask = name_var(require_typed_var(parent, TY_int)); + var_t *word = name_var(require_typed_var(parent, TY_uint)); + var_t *aligned = name_var(require_typed_var(parent, TY_uint)); + add_insn(parent, *bb, OP_cast, word, old, NULL, PTR_SIZE, NULL); step->init_val = 7; step->is_const = true; add_insn(parent, *bb, OP_load_constant, step, NULL, NULL, 0, NULL); - aligned->var_name = gen_name(); - add_insn(parent, *bb, OP_add, aligned, old, step, 0, NULL); - mask->var_name = gen_name(); + add_insn(parent, *bb, OP_add, aligned, word, step, 0, NULL); mask->init_val = -8; mask->is_const = true; add_insn(parent, *bb, OP_load_constant, mask, NULL, NULL, 0, NULL); - old = require_typed_ptr_var(parent, TY_int, 1); - old->var_name = gen_name(); - add_insn(parent, *bb, OP_bit_and, old, aligned, mask, 0, NULL); - step = require_typed_var(parent, TY_int); - step->var_name = gen_name(); + word = name_var(require_typed_var(parent, TY_uint)); + add_insn(parent, *bb, OP_bit_and, word, aligned, mask, 0, NULL); + old = name_var(require_typed_ptr_var(parent, TY_int, 1)); + add_insn(parent, *bb, OP_cast, old, word, NULL, PTR_SIZE, NULL); + step = name_var(require_typed_var(parent, TY_int)); } /* This direct IR add is byte-addressed; unlike parsed pointer arithmetic it @@ -912,26 +746,20 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) : PTR_SIZE; step->is_const = true; add_insn(parent, *bb, OP_load_constant, step, NULL, NULL, 0, NULL); - next = require_typed_ptr_var(parent, TY_int, 1); - next->var_name = gen_name(); + next = name_var(require_typed_ptr_var(parent, TY_int, 1)); add_insn(parent, *bb, OP_add, next, old, step, 0, NULL); add_insn(parent, *bb, OP_write, NULL, ap_addr, next, PTR_SIZE, NULL); if (is_record_type(requested.type) && !requested.ptr_level) { - var_t *source = require_ref_var(parent, requested.type, 0); - var_t *result = require_typed_var(parent, requested.type); - - source->var_name = gen_name(); + var_t *source = name_var(require_ref_var(parent, requested.type, 0)); + var_t *result = name_var(require_typed_var(parent, requested.type)); add_insn(parent, *bb, OP_read, source, old, NULL, PTR_SIZE, NULL); - result->var_name = gen_name(); add_insn(parent, *bb, OP_allocat, result, NULL, NULL, 0, NULL); emit_record_copy_from_address(parent, bb, result, source); opstack_push(result); } else { - var_t *result = - require_typed_ptr_var(parent, requested.type, requested.ptr_level); - - result->var_name = gen_name(); + var_t *result = name_var( + require_typed_ptr_var(parent, requested.type, requested.ptr_level)); add_insn(parent, *bb, OP_read, result, old, NULL, requested.ptr_level ? PTR_SIZE : requested.type->size, NULL); result->func_signature = requested.func_signature; @@ -961,9 +789,7 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) * pointer's width where an int was stored. */ type_t *element_type = - requested.pointee_element_ptr_level - ? pointee_type_from_pointer_typedef(requested.type) - : requested.type; + pointee_type_from_pointer_typedef(requested.type); while (lex_accept(T_open_square)) { var_t *index; @@ -991,24 +817,9 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) else for (int i = depth; i < dimension_count; i++) multiplier *= dims[i]; - if (multiplier != 1) { - var_t *scale = require_typed_var(parent, TY_int); - scale->var_name = gen_name(); - scale->init_val = multiplier; - scale->is_const = true; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, - 0, NULL); - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, - NULL); - index = scaled; - } - address = require_typed_ptr_var( - parent, element_type, - requested.pointee_element_ptr_level + 1); - address->var_name = gen_name(); - add_insn(parent, *bb, OP_add, address, base, index, 0, NULL); + address = lower_array_element_address( + parent, *bb, base, index, element_type, + requested.pointee_element_ptr_level + 1, multiplier, true); depth++; if (depth == dimension_count + 1 && !requested.pointee_element_ptr_level && @@ -1017,18 +828,16 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) * read one scalar of the record's size, which drops bytes * and is a width no narrow backend can load. */ - var_t *value = require_typed_var(parent, requested.type); - - value->var_name = gen_name(); + var_t *value = + name_var(require_typed_var(parent, requested.type)); add_insn(parent, *bb, OP_allocat, value, NULL, NULL, 0, NULL); emit_record_copy_from_address(parent, bb, value, address); base = value; } else if (depth == dimension_count + 1) { - var_t *value = require_typed_ptr_var( + var_t *value = name_var(require_typed_ptr_var( parent, element_type, - requested.pointee_element_ptr_level); - value->var_name = gen_name(); + requested.pointee_element_ptr_level)); add_insn(parent, *bb, OP_read, value, address, NULL, element_size, NULL); base = value; @@ -1039,8 +848,8 @@ void read_builtin_va_arg(block_t *parent, basic_block_t **bb) } if (requested.func_signature && lex_peek(T_open_bracket, NULL)) { - result = emit_indirect_call_result(result, requested.func_signature, - true, parent, bb); + emit_indirect_call_result(result, requested.func_signature, true, + parent, bb); } } } @@ -1073,12 +882,10 @@ var_t *load_function_pointer_object(block_t *parent, for (int i = 0; i < parent->func->num_params; i++) { var_t *param = &parent->func->param_defs[i]; - if (object == param || object->base == param) { + if (object == param) { func_t *param_signature = get_func_signature(object); - var_t *value = require_typed_ptr_var( - parent, param_signature->return_def.type, 1); - - value->var_name = gen_name(); + var_t *value = name_var(require_typed_ptr_var( + parent, param_signature->return_def.type, 1)); value->func_signature = param_signature; add_insn(parent, *bb, OP_assign, value, object, NULL, 0, NULL); return value; @@ -1088,12 +895,9 @@ var_t *load_function_pointer_object(block_t *parent, var_t *address = require_ref_var(parent, object->type, object->ptr_level); func_t *signature = get_func_signature(object); - var_t *target = require_typed_ptr_var( - parent, signature ? signature->return_def.type : object->type, 1); - - address->var_name = gen_name(); + var_t *target = name_var(require_typed_ptr_var( + parent, signature ? signature->return_def.type : object->type, 1)); add_insn(parent, *bb, OP_address_of, address, object, NULL, 0, NULL); - target->var_name = gen_name(); target->func_signature = object->func_signature; add_insn(parent, *bb, OP_read, target, address, NULL, PTR_SIZE, NULL); return target; @@ -1125,9 +929,8 @@ var_t *materialize_function_designator(block_t *parent, if (!func) return value; - object = require_typed_ptr_var(parent, func->return_def.type, - func->return_def.ptr_level); - object->var_name = gen_name(); + object = name_var(require_typed_ptr_var(parent, func->return_def.type, + func->return_def.ptr_level)); object->is_func = true; object->func_signature = func; add_insn(parent, *bb, OP_allocat, object, NULL, NULL, 0, NULL); @@ -1152,6 +955,13 @@ void read_indirect_call_with_sret(var_t *callee, UNUSED(callee); + /* The evaluated temporary is the actual OP_indirect operand. Preserve the + * prototype on that value too, since later SSA lowering sees the operand + * rather than the source declaration that supplied the type. + */ + if (target && signature) + target->func_signature = signature; + read_func_parameters_with_sret(signature, sret, parent, bb); add_insn(parent, *bb, OP_indirect, NULL, target, NULL, 0, NULL); @@ -1172,14 +982,14 @@ var_t *prepare_aggregate_call_result(block_t *parent, func_t *signature, var_t **sret) { - var_t *result = require_typed_var(parent, signature->return_def.type); + var_t *result = + name_var(require_typed_var(parent, signature->return_def.type)); var_t *destination; - result->var_name = gen_name(); add_insn(parent, *bb, OP_allocat, result, NULL, NULL, 0, NULL); - destination = require_ref_var(parent, signature->return_def.type, 0); - destination->var_name = gen_name(); + destination = + name_var(require_ref_var(parent, signature->return_def.type, 0)); add_insn(parent, *bb, OP_address_of, destination, result, NULL, 0, NULL); *sret = destination; return result; @@ -1205,21 +1015,42 @@ static var_t *callback_slot_call_result(block_t *parent, if (slot && !return_def->type->array_size && !return_def->array_size) { result = require_typed_ptr_var(parent, return_def->type, return_def->ptr_level); - result->var_name = gen_name(); result->pointee_func_signature = slot; return result; } if (!callback || return_def->type->is_direct_function_type || return_def->type->array_size || return_def->ptr_level < 1) return NULL; - result = require_typed_ptr_var( + result = name_var(require_typed_ptr_var( parent, callback->return_def.type, - callback->return_def.ptr_level + return_def->ptr_level); - result->var_name = gen_name(); + callback->return_def.ptr_level + return_def->ptr_level)); result->pointee_func_signature = callback; return result; } +static var_t *emit_call_result_value(block_t *parent, + basic_block_t **bb, + const var_t *return_def, + bool direct_call) +{ + var_t *result = callback_slot_call_result(parent, return_def); + + if (!result) { + func_t *returned_signature = return_def->type->func_signature; + bool returns_function = direct_call && returned_signature; + + result = name_var(require_typed_ptr_var( + parent, + returns_function ? returned_signature->return_def.type + : return_def->type, + returns_function ? 1 : return_def->ptr_level)); + result->func_signature = returned_signature; + copy_call_result_array_shape(result, return_def); + } + add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); + return result; +} + /* Keep the ABI-only aggregate destination coupled to call emission. Call sites * that merely discard an aggregate result still must provide it. */ @@ -1228,7 +1059,6 @@ var_t *emit_direct_call_result(func_t *func, block_t *parent, basic_block_t **bb) { - func_t *returned_signature = func->return_def.type->func_signature; var_t *result = NULL; if (func->returns_aggregate) { @@ -1239,20 +1069,8 @@ var_t *emit_direct_call_result(func_t *func, } else { read_func_call(func, parent, bb); if (want_value) - result = callback_slot_call_result(parent, &func->return_def); - if (result) { - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } else if (want_value) { - result = require_typed_ptr_var( - parent, - returned_signature ? returned_signature->return_def.type - : func->return_def.type, - returned_signature ? 1 : func->return_def.ptr_level); - result->var_name = gen_name(); - result->func_signature = returned_signature; - copy_call_result_array_shape(result, &func->return_def); - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } + result = + emit_call_result_value(parent, bb, &func->return_def, true); } if (want_value) opstack_push(result); @@ -1282,17 +1100,8 @@ var_t *emit_indirect_call_result(var_t *callee, } else { read_indirect_call(callee, parent, bb); if (want_value && signature) - result = callback_slot_call_result(parent, &signature->return_def); - if (result) { - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } else if (want_value && signature) { - result = require_typed_ptr_var(parent, signature->return_def.type, - signature->return_def.ptr_level); - result->var_name = gen_name(); - result->func_signature = signature->return_def.type->func_signature; - copy_call_result_array_shape(result, &signature->return_def); - add_insn(parent, *bb, OP_func_ret, result, NULL, NULL, 0, NULL); - } + result = emit_call_result_value(parent, bb, &signature->return_def, + false); } if (want_value) opstack_push(result); @@ -1301,12 +1110,8 @@ var_t *emit_indirect_call_result(var_t *callee, var_t *bitfield_constant(block_t *parent, basic_block_t **bb, unsigned value) { - var_t *result = require_var(parent); - result->var_name = gen_name(); - result->init_val = (int) value; + var_t *result = load_constant(parent, *bb, (int) value, TY_uint); result->is_const = true; - result->type = TY_uint; - add_insn(parent, *bb, OP_load_constant, result, NULL, NULL, 0, NULL); return result; } @@ -1317,8 +1122,7 @@ var_t *bitfield_binary(block_t *parent, var_t *right, type_t *type) { - var_t *result = require_var(parent); - result->var_name = gen_name(); + var_t *result = require_named_var(parent); result->type = type; add_insn(parent, *bb, op, result, left, right, 0, NULL); return result; @@ -1329,13 +1133,14 @@ unsigned bitfield_mask(const var_t *field) return field->bit_width == 32 ? ~0U : (1U << field->bit_width) - 1; } -var_t *read_bitfield_value(block_t *parent, - basic_block_t **bb, - var_t *address, - const var_t *field) +static var_t *read_bitfield_value_impl(block_t *parent, + basic_block_t **bb, + var_t *address, + const var_t *field, + bool assignment_reload) { - var_t *value = require_var(parent); - value->var_name = gen_name(); + var_t *value = require_named_var(parent); + value->is_assignment_reload = assignment_reload; value->type = TY_uint; add_insn(parent, *bb, OP_read, value, address, NULL, field->bit_storage_size, NULL); @@ -1381,6 +1186,14 @@ var_t *read_bitfield_value(block_t *parent, return value; } +var_t *read_bitfield_value(block_t *parent, + basic_block_t **bb, + var_t *address, + const var_t *field) +{ + return read_bitfield_value_impl(parent, bb, address, field, false); +} + /* The value of an assignment to a bit-field: the field read back from the unit * just written. C11 6.5.16p3 permits that read without requiring it, so mark it * removable when nothing uses the value; otherwise "s.flag = 1;" on a volatile @@ -1391,13 +1204,7 @@ var_t *reload_assigned_bitfield(block_t *parent, var_t *address, const var_t *field) { - insn_t *before = (*bb)->insn_list.tail; - basic_block_t *start = *bb; - var_t *value = read_bitfield_value(parent, bb, address, field); - - /* read_bitfield_value() emits the read first, into the given block. */ - insn_t *read = before ? before->next : start->insn_list.head; - read->rd->is_assignment_reload = true; + var_t *value = read_bitfield_value_impl(parent, bb, address, field, true); value->is_bitfield = false; return value; } @@ -1410,8 +1217,7 @@ void write_bitfield_value(block_t *parent, { if (is_bool_type(field->type)) value = normalize_bool(parent, bb, value); - var_t *old = require_var(parent); - old->var_name = gen_name(); + var_t *old = require_named_var(parent); old->type = TY_uint; add_insn(parent, *bb, OP_read, old, address, NULL, field->bit_storage_size, NULL); @@ -1423,8 +1229,7 @@ void write_bitfield_value(block_t *parent, parent, bb, OP_lshift, value, bitfield_constant(parent, bb, field->bit_offset), TY_uint); unsigned mask = bitfield_mask(field) << field->bit_offset; - var_t *clear = require_var(parent); - clear->var_name = gen_name(); + var_t *clear = require_named_var(parent); clear->type = TY_uint; add_insn(parent, *bb, OP_bit_not, clear, bitfield_constant(parent, bb, mask), NULL, 0, NULL); @@ -1468,9 +1273,8 @@ static void copy_pointee_array_shape(var_t *destination, const var_t *source) { fixed_array_shape_t shape = fixed_array_shape_from_pointee_var(source); - fixed_array_shape_to_pointee_var(destination, &shape); - destination->pointee_array_element_ptr_level = - source->pointee_array_element_ptr_level; + set_pointee_array_shape(destination, &shape, + source->pointee_array_element_ptr_level); } /* A dereference of a pointer to a pointer to an array, `*pp` for an int @@ -1525,19 +1329,13 @@ static void take_expression_address(block_t *parent, basic_block_t **bb) if (!effective_pointer_depth(operand) && !operand->array_size) error_at("Cannot dereference from non-pointer typed variable", cur_token_loc()); - result = require_typed_ptr_var( - parent, operand->type, operand->ptr_level + !!operand->array_size); - result->var_name = gen_name(); + result = name_var(require_typed_ptr_var( + parent, operand->type, operand->ptr_level + !!operand->array_size)); result->is_const_qualified = operand->is_const_qualified; result->pointer_const_mask = operand->pointer_const_mask; result->func_signature = operand->func_signature; result->pointee_func_signature = operand->pointee_func_signature; - result->pointee_array_size = operand->pointee_array_size; - result->pointee_array_dim2 = operand->pointee_array_dim2; - result->pointee_array_dim3 = operand->pointee_array_dim3; - result->pointee_array_dim4 = operand->pointee_array_dim4; - result->pointee_array_element_ptr_level = - operand->pointee_array_element_ptr_level; + copy_pointee_array_shape(result, operand); add_insn(parent, *bb, OP_assign, result, operand, NULL, 0, NULL); opstack_push(result); return; @@ -1564,9 +1362,8 @@ static void take_expression_address(block_t *parent, basic_block_t **bb) } else { error_at("Address-of requires an lvalue operand", cur_token_loc()); } - result = - require_typed_ptr_var(parent, operand->type, operand->ptr_level + 1); - result->var_name = gen_name(); + result = name_var( + require_typed_ptr_var(parent, operand->type, operand->ptr_level + 1)); result->is_const_qualified = operand->is_const_qualified; result->is_string_literal = operand->is_string_literal; if (operand->func_signature) @@ -1599,9 +1396,8 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) error_at("__func__ is only defined inside a function", next_token_loc()); lex_expect(T_identifier); - vd = require_typed_ptr_var(parent, TY_char, 1); - vd->var_name = gen_name(); - vd->init_val = write_symbol(parent->func->return_def.var_name); + vd = name_var(require_typed_ptr_var(parent, TY_char, 1)); + vd->init_val = write_symbol(function_source_name(parent->func)); vd->is_string_literal = true; vd->is_func_name_array_address = true; opstack_push(vd); @@ -1623,9 +1419,7 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) "function", next_token_loc()); lex_expect(T_identifier); - vd = require_func_symbol_var(parent); - vd->is_func = true; - vd->var_name = intern_string(token); + vd = function_designator_value(parent, addressed_func); opstack_push(vd); return; } @@ -1650,13 +1444,12 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) fixed_array_shape_drop_outer(&shape); if (shape.rank) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = lvalue.decl->type; vd->ptr_level = lvalue.decl->ptr_level; vd->is_const_qualified = lvalue.decl->is_const_qualified; - copy_pointee_array_shape(vd, lvalue.decl); - fixed_array_shape_to_pointee_var(vd, &shape); + set_pointee_array_shape( + vd, &shape, lvalue.decl->pointee_array_element_ptr_level); add_insn(parent, *bb, OP_assign, vd, rs1, NULL, 0, NULL); opstack_push(vd); return; @@ -1665,8 +1458,7 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) if (!lvalue.is_reference) { rs1 = opstack_pop(); - vd = require_ref_var(parent, lvalue.type, lvalue.ptr_level); - vd->var_name = gen_name(); + vd = name_var(require_ref_var(parent, lvalue.type, lvalue.ptr_level)); /* Address-of moves a pointer object's qualification one level inward: * `&p`, for `int * const p`, is `int * const *`, not `const int **`. @@ -1717,8 +1509,7 @@ void handle_address_of_operator(block_t *parent, basic_block_t **bb) for (int i = 0; i < lvalue.subscript_depth; i++) fixed_array_shape_drop_outer(&shape); - fixed_array_shape_to_pointee_var(vd, &shape); - vd->pointee_array_element_ptr_level = lvalue.decl->ptr_level; + set_pointee_array_shape(vd, &shape, lvalue.decl->ptr_level); } add_insn(parent, *bb, OP_assign, vd, rs1, NULL, 0, NULL); } else { @@ -1774,8 +1565,7 @@ static bool lower_pointee_array_dereference(var_t *source, */ if (!element_type->ptr_level) element_type = array_typedef_element_type(element_type); - row = require_var(parent); - row->type = element_type; + row = name_var(require_typed_var(parent, element_type)); fixed_array_shape_t shape = fixed_array_shape_from_pointee_var(source); fixed_array_shape_to_var(row, &shape); @@ -1798,7 +1588,6 @@ static bool lower_pointee_array_dereference(var_t *source, row->is_const_pointer = row->pointer_const_mask & (1U << (row->ptr_level - 1)); } - row->var_name = gen_name(); add_insn(parent, *bb, OP_assign, row, source, NULL, 0, NULL); opstack_push(row); return true; @@ -1815,8 +1604,8 @@ static int pointee_array_row_stride(const var_t *var) return var->pointee_array_size * element_type->size; } -/* Byte extent remaining after direct-array subscript depth. This is shared by - * ordinary postfix indexing and the bounded grouped pointer-to-array path. +/* Byte extent remaining after an array subscript depth, shared by indexing and + * initializer layout. */ static int fixed_array_shape_stride(const fixed_array_shape_t *shape, int subscript_depth, @@ -1906,17 +1695,15 @@ static var_t *decay_dereferenced_array(block_t *parent, basic_block_t **bb, var_t *array) { - var_t *pointer = - require_typed_ptr_var(parent, array->type, array->ptr_level + 1); + var_t *pointer = name_var( + require_typed_ptr_var(parent, array->type, array->ptr_level + 1)); if (array->array_dim2) { fixed_array_shape_t shape = fixed_array_shape_from_var(array); fixed_array_shape_drop_outer(&shape); - fixed_array_shape_to_pointee_var(pointer, &shape); - pointer->pointee_array_element_ptr_level = array->ptr_level; + set_pointee_array_shape(pointer, &shape, array->ptr_level); } - pointer->var_name = gen_name(); pointer->is_const_qualified = array->is_const_qualified; add_insn(parent, *bb, OP_assign, pointer, array, NULL, 0, NULL); return pointer; @@ -1931,6 +1718,23 @@ static bool dereferenced_call_follows(void) cur_token->next->next->kind == T_open_bracket; } +static void name_dereferenced_value(var_t *value, const var_t *source) +{ + value->var_name = gen_name(); + value->is_const_qualified = source->is_const_qualified; + value->is_volatile = source->is_volatile; + value->callback_is_volatile = source->callback_is_volatile; + int depth = source->ptr_level + source->type->ptr_level; + value->pointer_volatile_mask = + depth > 0 && depth <= 32 + ? source->pointer_volatile_mask & ~(1U << (depth - 1)) + : source->pointer_volatile_mask; + value->pointer_const_mask = dereferenced_pointer_const_mask(source); + value->is_const_pointer = + value->ptr_level > 0 && value->ptr_level <= 32 && + (value->pointer_const_mask & (1U << (value->ptr_level - 1))); +} + /* Dereference the pointer value @rs1 and push the object it designates. */ void push_dereference(block_t *parent, basic_block_t **bb, var_t *rs1) { @@ -1944,8 +1748,7 @@ void push_dereference(block_t *parent, basic_block_t **bb, var_t *rs1) * element. Dereferencing it restores the array, which immediately * decays back to char * in this expression context. */ - vd = require_typed_ptr_var(parent, TY_char, 1); - vd->var_name = gen_name(); + vd = name_var(require_typed_ptr_var(parent, TY_char, 1)); vd->is_string_literal = true; opstack_push(vd); add_insn(parent, *bb, OP_assign, vd, rs1, NULL, 0, NULL); @@ -1981,13 +1784,8 @@ void push_dereference(block_t *parent, basic_block_t **bb, var_t *rs1) ? PTR_SIZE : pointer_typedef_pointee_size( deref_type, pointee_type_from_pointer_typedef(deref_type)); - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; + name_dereferenced_value(vd, rs1); vd->is_string_literal = rs1->is_string_literal; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); /* Pointer arithmetic can produce a pointer to a callback typedef. The * actual dereference consumes that object-pointer level and yields the @@ -2047,12 +1845,9 @@ static bool push_dereferenced_function(block_t *parent, /* An array of function pointers decays to the address of its first * element, from which `*fps` reads the pointer, as fps[0] does. */ - var_t *address = require_typed_ptr_var(parent, rs1->type, 1); - var_t *target = require_typed_ptr_var(parent, rs1->type, 1); - - address->var_name = gen_name(); + var_t *address = name_var(require_typed_ptr_var(parent, rs1->type, 1)); + var_t *target = name_var(require_typed_ptr_var(parent, rs1->type, 1)); add_insn(parent, *bb, OP_assign, address, rs1, NULL, 0, NULL); - target->var_name = gen_name(); target->func_signature = rs1->func_signature; add_insn(parent, *bb, OP_read, target, address, NULL, PTR_SIZE, NULL); opstack_push(target); @@ -2065,11 +1860,63 @@ static bool push_dereferenced_function(block_t *parent, return false; } -void handle_single_dereference(block_t *parent, basic_block_t **bb) +static bool restore_dereferenced_callback(var_t *value, + var_t *source, + type_t *deref_type, + int *size); +static bool push_dereferenced_object(block_t *parent, + basic_block_t **bb, + var_t *source); + +static void handle_simple_dereferences(block_t *parent, + basic_block_t **bb, + int count) { - var_t *vd, *rs1; - int sz; + char token[MAX_VAR_LEN]; + lvalue_t lvalue; + var_t *var; + var_t *value; + + if (!lex_peek(T_identifier, token)) + error_at("Expected an identifier", next_token_loc()); + var = find_var(token, parent); + + /* A raw function name is a designator, not an object lvalue. Keep the + * single-star symbol case on its direct path; the multi-star parser + * historically requires an object declaration here. + */ + func_t *func = count == 1 && !var ? find_visible_func(token, parent) : NULL; + if (func) { + lex_expect(T_identifier); + value = function_designator_value(parent, func); + opstack_push(value); + return; + } + + read_lvalue(&lvalue, var, parent, bb, true, OP_generic); + + for (int i = 0; i < count; i++) { + value = opstack_pop(); + if (i == 0 && !decays_when_dereferenced(value) && + push_dereferenced_function(parent, bb, &lvalue, var, value)) + break; + if (decays_when_dereferenced(value)) { + push_dereference(parent, bb, value); + } else if (lower_pointee_array_dereference(value, parent, bb)) { + continue; + } else if (count == 1 && value->func_signature && + !effective_pointer_depth(value)) { + value->ptr_level = 1; + opstack_push(value); + break; + } else if (push_dereferenced_object(parent, bb, value)) { + break; + } + } +} +void handle_single_dereference(block_t *parent, basic_block_t **bb) +{ if (lex_peek(T_open_bracket, NULL) || lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL) || lex_peek(T_ampersand, NULL) || dereferenced_call_follows()) { @@ -2082,120 +1929,55 @@ void handle_single_dereference(block_t *parent, basic_block_t **bb) read_expr_operand(parent, bb); push_dereference(parent, bb, opstack_pop()); } else { - /* Handle simple identifier dereference: *var */ - char token[MAX_VAR_LEN]; - lvalue_t lvalue; - - if (!lex_peek(T_identifier, token)) - error_at("Expected an identifier", next_token_loc()); - - /* Builtins are expression operands, not objects in the local symbol - * table. Keep the identifier lvalue path below for `*pointer` - * assignments, but let a pointer-valued builtin such as `*va_arg(...)` - * use the same rvalue dereference lowering as `*(expression)`. - */ - if (!strcmp(token, "__builtin_va_arg")) { - type_t *deref_type; - int deref_ptr; - - read_expr_operand(parent, bb); - rs1 = opstack_pop(); - deref_type = rs1->type ? rs1->type : TY_int; - deref_ptr = rs1->ptr_level + deref_type->ptr_level - 1; - vd = require_deref_var(parent, deref_type, rs1->ptr_level); - sz = deref_ptr > 0 ? PTR_SIZE : deref_type->size; - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - push_object_at(parent, bb, vd, rs1, sz); - return; - } - var_t *var = find_var(token, parent); - - /* A raw function name is a function designator, not an object lvalue. - * Dereferencing it is a no-op in C99 (`*f` is another designator), so - * route it through the same symbol representation used by ordinary - * direct calls instead of asking read_lvalue() to find object storage. - */ - if (!var && find_visible_func(token, parent)) { - lex_expect(T_identifier); - rs1 = require_func_symbol_var(parent); - rs1->var_name = intern_string(token); - rs1->is_func = true; - opstack_push(rs1); - return; - } - read_lvalue(&lvalue, var, parent, bb, true, OP_generic); - - rs1 = opstack_pop(); - if (decays_when_dereferenced(rs1)) { - push_dereference(parent, bb, rs1); - return; - } - if (push_dereferenced_function(parent, bb, &lvalue, var, rs1)) - return; - if (lower_pointee_array_dereference(rs1, parent, bb)) - return; - - /* A member function-pointer typedef was already loaded by - * read_lvalue(). Unary `*` turns that pointer into a function - * designator; it does not read from the code address. - */ - if (rs1->func_signature && !effective_pointer_depth(rs1)) { - rs1->ptr_level = 1; - opstack_push(rs1); - return; - } + handle_simple_dereferences(parent, bb, 1); + } +} - /* `read_lvalue()` may have resolved a member expression. Derive the - * final indirection from its evaluated value rather than from the - * initial record object, so `*record.pointer` dereferences the member - * pointer instead of attempting to dereference the record itself. - */ - type_t *deref_type = rs1->type ? rs1->type : TY_int; - int deref_ptr = rs1->ptr_level + deref_type->ptr_level - 1; - - vd = require_deref_var(parent, deref_type, rs1->ptr_level); - sz = deref_ptr > 0 ? PTR_SIZE : deref_type->size; - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - - /* `*slot` is the function-pointer value when slot is a pointer to a - * callback typedef. The outer declarator's signature was deliberately - * cleared to prevent the invalid `slot(...)` form, so restore the - * element signature only after the real dereference has occurred. - */ - if (rs1->pointee_func_signature - ? callback_slot_depth(rs1) == 1 - : ((rs1->func_signature || - (deref_type && deref_type->func_signature)) && - effective_pointer_depth(rs1) == 1)) { - vd->func_signature = - rs1->pointee_func_signature - ? rs1->pointee_func_signature - : (rs1->func_signature ? rs1->func_signature - : deref_type->func_signature); - vd->ptr_level = 1; - sz = PTR_SIZE; - } - keep_callback_slot(vd, rs1); - keep_pointee_array_shape(vd, rs1); - push_object_at(parent, bb, vd, rs1, sz); +static bool restore_dereferenced_callback(var_t *value, + var_t *source, + type_t *deref_type, + int *size) +{ + if (source->pointee_func_signature + ? callback_slot_depth(source) == 1 + : ((source->func_signature || + (deref_type && deref_type->func_signature)) && + effective_pointer_depth(source) == 1)) { + value->func_signature = + source->pointee_func_signature + ? source->pointee_func_signature + : (source->func_signature ? source->func_signature + : deref_type->func_signature); + value->ptr_level = 1; + *size = PTR_SIZE; + return true; } + return false; } -void handle_multiple_dereference(block_t *parent, basic_block_t **bb) +static bool push_dereferenced_object(block_t *parent, + basic_block_t **bb, + var_t *source) { - var_t *vd, *rs1; - int sz; + type_t *type = source->type ? source->type : TY_int; + int pointer_level = source->ptr_level + type->ptr_level - 1; + int size = pointer_level > 0 + ? PTR_SIZE + : pointer_typedef_pointee_size( + type, pointee_type_from_pointer_typedef(type)); + var_t *value = require_deref_var(parent, type, source->ptr_level); + bool is_callback; + + name_dereferenced_value(value, source); + is_callback = restore_dereferenced_callback(value, source, type, &size); + keep_callback_slot(value, source); + keep_pointee_array_shape(value, source); + push_object_at(parent, bb, value, source, size); + return is_callback; +} +void handle_multiple_dereference(block_t *parent, basic_block_t **bb) +{ /* Handle consecutive asterisks for multiple dereference: **pp, ***ppp, and * the parenthesized ***(expr) form. */ @@ -2213,79 +1995,7 @@ void handle_multiple_dereference(block_t *parent, basic_block_t **bb) for (int i = 0; i < deref_count; i++) push_dereference(parent, bb, opstack_pop()); } else { - /* Handle **pp, ***ppp case with simple identifier */ - char token[MAX_VAR_LEN]; - lvalue_t lvalue; - - if (!lex_peek(T_identifier, token)) - error_at("Expected an identifier", next_token_loc()); - var_t *var = find_var(token, parent); - read_lvalue(&lvalue, var, parent, bb, true, OP_generic); - - /* Apply dereferences one by one. Once they reach a function, the - * remaining ones are no-ops: `**fp` designates the function `*fp` does. - */ - bool designator = false; - for (int i = 0; i < deref_count; i++) { - rs1 = opstack_pop(); - if (designator) { - opstack_push(rs1); - continue; - } - if (i == 0 && !decays_when_dereferenced(rs1) && - push_dereferenced_function(parent, bb, &lvalue, var, rs1)) { - designator = true; - continue; - } - if (decays_when_dereferenced(rs1)) { - push_dereference(parent, bb, rs1); - continue; - } - if (lower_pointee_array_dereference(rs1, parent, bb)) - continue; - - /* A member lvalue has already been read into rs1. Each unary - * asterisk must consume that evaluated pointer, not re-derive its - * type from the initial record identifier. - */ - type_t *deref_type = rs1->type ? rs1->type : TY_int; - int deref_ptr = rs1->ptr_level + deref_type->ptr_level - 1; - - vd = require_deref_var(parent, deref_type, rs1->ptr_level); - sz = deref_ptr > 0 - ? PTR_SIZE - : pointer_typedef_pointee_size( - deref_type, - pointee_type_from_pointer_typedef(deref_type)); - vd->var_name = gen_name(); - vd->is_const_qualified = rs1->is_const_qualified; - vd->pointer_const_mask = dereferenced_pointer_const_mask(rs1); - vd->is_const_pointer = - vd->ptr_level > 0 && vd->ptr_level <= 32 && - (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - - /* Only the dereference of a pointer to a callback, the last read of - * `**slots` or `**fpp`, reaches the callback object. Earlier reads - * still produce a pointer-to-callback and must not be callable. - */ - if (rs1->pointee_func_signature - ? callback_slot_depth(rs1) == 1 - : ((rs1->func_signature || - (deref_type && deref_type->func_signature)) && - effective_pointer_depth(rs1) == 1)) { - vd->func_signature = - rs1->pointee_func_signature - ? rs1->pointee_func_signature - : (rs1->func_signature ? rs1->func_signature - : deref_type->func_signature); - vd->ptr_level = 1; - sz = PTR_SIZE; - designator = true; - } - keep_callback_slot(vd, rs1); - keep_pointee_array_shape(vd, rs1); - push_object_at(parent, bb, vd, rs1, sz); - } + handle_simple_dereferences(parent, bb, deref_count); } } @@ -2324,30 +2034,9 @@ void lower_member_postfix(var_t *address, ? PTR_SIZE : field->type->size; int stride = fixed_array_shape_stride(&shape, depth, element_size); - var_t *index; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (stride != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, - NULL); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, field->type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address(parent, bb, address, field->type, + 1, stride); depth++; if (depth == shape.rank && is_record_type(field->type) && !field->ptr_level) @@ -2374,10 +2063,8 @@ void lower_member_postfix(var_t *address, if (depth < shape.rank || effective_pointer_depth(field) != 1 || field->is_func || !is_record_type(pointee)) error_at("Invalid record member access", cur_token_loc()); - var_t *pointer = - require_typed_ptr_var(parent, field->type, field->ptr_level); - - pointer->var_name = gen_name(); + var_t *pointer = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level)); add_insn(parent, *bb, OP_read, pointer, address, NULL, PTR_SIZE, NULL); address = pointer; @@ -2387,9 +2074,8 @@ void lower_member_postfix(var_t *address, field->is_const_qualified || field->type->is_const_qualified; } else { lex_expect(T_dot); - if (field) - record_const = record_const || field->is_const_qualified || - field->type->is_const_qualified; + record_const = record_const || field->is_const_qualified || + field->type->is_const_qualified; } if (!record_type) error_at("Member access requires a record", cur_token_loc()); @@ -2412,18 +2098,16 @@ void lower_member_postfix(var_t *address, /* sizeof observes the array itself, before any decay. */ for (int i = 0; i < depth; i++) fixed_array_shape_drop_outer(&shape); - result = require_typed_ptr_var(parent, field->type, field->ptr_level); + result = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level)); fixed_array_shape_to_var(result, &shape); - result->var_name = gen_name(); opstack_push(result); } else if (decayed) { for (int i = 0; i <= depth; i++) fixed_array_shape_drop_outer(&shape); - result = - require_typed_ptr_var(parent, field->type, field->ptr_level + 1); - fixed_array_shape_to_pointee_var(result, &shape); - result->pointee_array_element_ptr_level = field->ptr_level; - result->var_name = gen_name(); + result = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level + 1)); + set_pointee_array_shape(result, &shape, field->ptr_level); add_insn(parent, *bb, OP_assign, result, address, NULL, 0, NULL); opstack_push(result); } else if (is_bitfield(field)) { @@ -2432,8 +2116,8 @@ void lower_member_postfix(var_t *address, if (is_lvalue) mark_value_reference(result, address, field); } else { - result = require_typed_ptr_var(parent, field->type, field->ptr_level); - result->var_name = gen_name(); + result = name_var( + require_typed_ptr_var(parent, field->type, field->ptr_level)); result->func_signature = field->func_signature; /* A function-pointer member is loaded as the pointer value a call diff --git a/src/parser-const.c b/src/parser-const.c index 25bafd69..0f9e6806 100644 --- a/src/parser-const.c +++ b/src/parser-const.c @@ -29,7 +29,7 @@ static int read_global_sizeof_expression(block_t *scope) int read_const_wstring_size(void) { int values[MAX_STRING_LEN]; - type_t *wide_type = find_type("wchar_t", true); + type_t *wide_type = find_builtin_type("wchar_t"); return (read_wstring_units(values, MAX_STRING_LEN) + 1) * wide_type->size; } @@ -53,120 +53,38 @@ int read_sizeof_constant(block_t *scope) int eval_expression_imm(opcode_t op, int op1, int op2) { - /* return immediate result */ - int tmp = op2; int res = 0; - switch (op) { - case OP_add: - if (checking_enum_constant && ((op2 > 0 && op1 > INT_MAX - op2) || - (op2 < 0 && op1 < INT_MIN - op2))) + + if (checking_enum_constant) { + if (op == OP_add && ((op2 > 0 && op1 > INT_MAX - op2) || + (op2 < 0 && op1 < INT_MIN - op2))) error_at("Enumerator value exceeds int range", cur_token_loc()); - res = op1 + op2; - break; - case OP_sub: - if (checking_enum_constant && ((op2 < 0 && op1 > INT_MAX + op2) || - (op2 > 0 && op1 < INT_MIN + op2))) + if (op == OP_sub && ((op2 < 0 && op1 > INT_MAX + op2) || + (op2 > 0 && op1 < INT_MIN + op2))) error_at("Enumerator value exceeds int range", cur_token_loc()); - res = op1 - op2; - break; - case OP_mul: - if (checking_enum_constant && op1 && op2 && + if (op == OP_mul && op1 && op2 && ((op1 > 0 && op2 > 0 && op1 > INT_MAX / op2) || (op1 > 0 && op2 < 0 && op2 < INT_MIN / op1) || (op1 < 0 && op2 > 0 && op1 < INT_MIN / op2) || (op1 < 0 && op2 < 0 && op1 < INT_MAX / op2))) error_at("Enumerator value exceeds int range", cur_token_loc()); - res = op1 * op2; - break; - case OP_div: + if (op == OP_lshift && + (op2 < 0 || op2 >= 32 || op1 < 0 || op1 > (INT_MAX >> op2))) + error_at("Enumerator value exceeds int range", cur_token_loc()); + } + if (op == OP_div || op == OP_mod) { if (!op2) - error_at("Division by zero in constant expression", + error_at(op == OP_div ? "Division by zero in constant expression" + : "Modulo by zero in constant expression", cur_token_loc()); - - /* INT_MIN / -1 has no representable result; on x86 it raises SIGFPE - * rather than producing one. - */ if (op1 == INT_MIN && op2 == -1) error_at("Overflow in constant expression", cur_token_loc()); - res = op1 / op2; - break; - case OP_mod: - if (!op2) - error_at("Modulo by zero in constant expression", cur_token_loc()); - if (op1 == INT_MIN && op2 == -1) - error_at("Overflow in constant expression", cur_token_loc()); - /* Use bitwise AND for modulo optimization when divisor is power of 2 */ - if (tmp == INT_MIN) { - res = op1 % op2; - break; - } - tmp = tmp < 0 ? -tmp : tmp; - tmp &= (tmp - 1); - if (tmp != 0) { - res = op1 % op2; - break; - } - op2 = op2 < 0 ? -op2 : op2; - res = op1 & (op2 - 1); - if (op1 < 0 && res != 0) - res -= op2; - break; - case OP_lshift: - if (checking_enum_constant && - (op2 < 0 || op2 >= 32 || op1 < 0 || op1 > (INT_MAX >> op2))) - error_at("Enumerator value exceeds int range", cur_token_loc()); - - /* A count outside the int width has no defined result, and the host - * compiler must not be asked to produce one. Shift the unsigned bit - * pattern so a set sign bit is not host undefined behavior either. - */ - if (op2 < 0 || op2 >= 32) - error_at("Shift count out of range in constant expression", - cur_token_loc()); - res = (int) ((unsigned int) op1 << op2); - break; - case OP_rshift: - if (op2 < 0 || op2 >= 32) - error_at("Shift count out of range in constant expression", - cur_token_loc()); - res = op1 >> op2; - break; - case OP_log_and: - res = op1 && op2; - break; - case OP_log_or: - res = op1 || op2; - break; - case OP_eq: - res = op1 == op2; - break; - case OP_neq: - res = op1 != op2; - break; - case OP_lt: - res = op1 < op2; - break; - case OP_gt: - res = op1 > op2; - break; - case OP_leq: - res = op1 <= op2; - break; - case OP_geq: - res = op1 >= op2; - break; - case OP_bit_and: - res = op1 & op2; - break; - case OP_bit_or: - res = op1 | op2; - break; - case OP_bit_xor: - res = op1 ^ op2; - break; - default: - error_at("The requested operation is not supported.", cur_token_loc()); } + if ((op == OP_lshift || op == OP_rshift) && (op2 < 0 || op2 >= 32)) + error_at("Shift count out of range in constant expression", + cur_token_loc()); + if (!vir_frontend_fold_i32(op, op1, op2, &res)) + error_at("The requested operation is not supported.", cur_token_loc()); return res; } @@ -258,22 +176,14 @@ static bool initializer_needs_wide_reader(token_t *token, block_t *scope, bool casts) { - int bracket_depth = 0; + int depth = 0; for (; token; token = token->next) { if ((token->kind == T_numeric && - (numeric_literal_needs_wide_path(token->literal) || - numeric_literal_needs_typed_global_path(token->literal))) || + numeric_literal_needs_global_reader(token->literal)) || (casts && global_integer_cast_starts_at(token, scope))) return true; - if (token->kind == T_open_bracket) - bracket_depth++; - else if (token->kind == T_close_bracket) { - if (bracket_depth == 0) - return false; - bracket_depth--; - } else if (bracket_depth == 0 && - (token->kind == T_semicolon || token->kind == T_comma)) + if (!initializer_scan_continue(token, &depth, false)) return false; } return false; @@ -296,21 +206,13 @@ bool typed_global_literal_appears_before_initializer_end(token_t *token) */ bool string_element_appears_before_initializer_end(token_t *token) { - int bracket_depth = 0; + int depth = 0; for (; token; token = token->next) { if ((token->kind == T_string || token->kind == T_wstring) && token->next && token->next->kind == T_open_square) return true; - if (token->kind == T_open_bracket) - bracket_depth++; - else if (token->kind == T_close_bracket) { - if (bracket_depth == 0) - return false; - bracket_depth--; - } else if (bracket_depth == 0 && - (token->kind == T_semicolon || token->kind == T_comma || - token->kind == T_close_curly)) + if (!initializer_scan_continue(token, &depth, true)) return false; } return false; @@ -564,15 +466,15 @@ bool emit_wide_global_word_arithmetic(block_t *parent, } if (op == OP_div || op == OP_mod) { - unsigned int rem_lo = 0, rem_hi = 0, quo_lo = 0, quo_hi = 0; - /* The usual arithmetic conversions choose the signedness: a signed long * long divided by an unsigned int stays signed long long. */ type_t *common = integer_common_type(left, right); bool is_unsigned = common->is_unsigned; - bool neg_left = !is_unsigned && (hi >> 31); - bool neg_right = !is_unsigned && (rhs_hi >> 31); + pp_integer_t dividend = { + .lo = lo, .hi = hi, .is_unsigned = is_unsigned}; + pp_integer_t divisor = { + .lo = rhs_lo, .hi = rhs_hi, .is_unsigned = is_unsigned}; if (rhs_lo == 0 && rhs_hi == 0) error_at("division by zero in global constant expression", @@ -586,47 +488,9 @@ bool emit_wide_global_word_arithmetic(block_t *parent, (common->size <= TY_int->size ? lo == 0x80000000U : hi == 0x80000000U && lo == 0)) error_at("Overflow in constant expression", cur_token_loc()); - if (neg_left) { - lo = ~lo + 1; - hi = ~hi + (lo == 0); - } - if (neg_right) { - rhs_lo = ~rhs_lo + 1; - rhs_hi = ~rhs_hi + (rhs_lo == 0); - } - for (int i = 0; i < 64; i++) { - unsigned int incoming = hi >> 31; - - hi = (hi << 1) | (lo >> 31); - lo <<= 1; - rem_hi = (rem_hi << 1) | (rem_lo >> 31); - rem_lo = (rem_lo << 1) | incoming; - quo_hi = (quo_hi << 1) | (quo_lo >> 31); - quo_lo <<= 1; - if (rem_hi > rhs_hi || (rem_hi == rhs_hi && rem_lo >= rhs_lo)) { - unsigned int borrow = rem_lo < rhs_lo; - - rem_lo -= rhs_lo; - rem_hi = rem_hi - rhs_hi - borrow; - quo_lo |= 1; - } - } - if (op == OP_div) { - out_lo = quo_lo; - out_hi = quo_hi; - if (neg_left != neg_right) { - out_lo = ~out_lo + 1; - out_hi = ~out_hi + (out_lo == 0); - } - } else { - out_lo = rem_lo; - out_hi = rem_hi; - if (neg_left) { - out_lo = ~out_lo + 1; - out_hi = ~out_hi + (out_lo == 0); - } - } - store_wide_global_word_result(parent, bb, result, out_lo, out_hi); + pp_divmod(÷nd, &divisor, op == OP_mod); + store_wide_global_word_result(parent, bb, result, dividend.lo, + dividend.hi); return true; } @@ -727,8 +591,7 @@ static var_t *promote_wide_global_operand(block_t *parent, if (value->ptr_level || wide_global_address_operand(value) || value->type->size >= TY_int->size) return value; - promoted = require_typed_var(parent, TY_int); - promoted->var_name = gen_name(); + promoted = name_var(require_typed_var(parent, TY_int)); promoted->init_val = value->init_val; promoted->init_val_hi = wide_global_narrow_high(value->init_val, false); promoted->is_const = true; @@ -774,8 +637,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, zero->type = value->type; zero->init_val = 0; add_insn(parent, bb, OP_load_constant, zero, NULL, NULL, 0, NULL); - result = require_var(parent); - result->var_name = gen_name(); + result = require_named_var(parent); result->type = value->type; emit_wide_global_word_arithmetic(parent, bb, result, OP_sub, zero, value); @@ -786,8 +648,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, value = promote_wide_global_operand( parent, bb, read_wide_global_literal_primary(parent, bb, scope)); - result = require_var(parent); - result->var_name = gen_name(); + result = require_named_var(parent); result->type = value->type; emit_wide_global_word_arithmetic(parent, bb, result, OP_bit_not, value, NULL); @@ -797,8 +658,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, var_t *result; value = read_wide_global_literal_primary(parent, bb, scope); - result = require_typed_var(parent, TY_int); - result->var_name = gen_name(); + result = name_var(require_typed_var(parent, TY_int)); result->init_val = !wide_global_operand_is_true(value); result->init_val_hi = 0; result->is_const = true; @@ -825,8 +685,8 @@ var_t *read_wide_global_literal_primary(block_t *parent, return address; } if (lex_accept(T_sizeof)) { - value = require_typed_var(parent, find_type("size_t", true)); - value->var_name = gen_name(); + value = + name_var(require_typed_var(parent, find_builtin_type("size_t"))); value->init_val = read_sizeof_constant(scope); value->init_val_hi = 0; value->is_const = true; @@ -859,8 +719,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, if (operand->type->size <= TY_int->size) hi = wide_global_narrow_high(operand->init_val, operand->type->is_unsigned); - value = require_typed_var(parent, type); - value->var_name = gen_name(); + value = name_var(require_typed_var(parent, type)); fold_integer_constant_cast(value, (unsigned int) operand->init_val, hi); value->is_const = true; add_insn(parent, bb, OP_load_constant, value, NULL, NULL, 0, NULL); @@ -878,8 +737,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, error_at("Typed global initializer needs a constant operand", next_token_loc()); lex_expect(T_identifier); - value = require_typed_var(parent, TY_int); - value->var_name = gen_name(); + value = name_var(require_typed_var(parent, TY_int)); /* Every operand this reader returns carries its high word, the * extension of an int-sized value, so no consumer rebuilds it. @@ -900,8 +758,7 @@ var_t *read_wide_global_literal_primary(block_t *parent, error_at("Wide string literal element is not a constant", next_token_loc()); read_string_literal_element(scope, &element); - value = require_typed_var(parent, TY_int); - value->var_name = gen_name(); + value = name_var(require_typed_var(parent, TY_int)); value->init_val = element; value->init_val_hi = wide_global_narrow_high(element, false); value->is_const = true; @@ -934,6 +791,42 @@ var_t *read_wide_global_literal_primary(block_t *parent, * subexpressions, without sending a high word through the legacy int-only * constant evaluator. */ +static void reduce_wide_global_top_op(block_t *parent, + basic_block_t *bb, + opcode_t *op_stack, + bool *protected_rhs, + var_t **val_stack, + int *op_count, + int *value_count) +{ + var_t *right = val_stack[--*value_count]; + var_t *left = val_stack[--*value_count]; + int top = --*op_count; + var_t *result = require_var(parent); + + if (protected_rhs[top]) + wide_global_unevaluated_depth--; + result->var_name = gen_name(); + result->type = integer_binary_result_type(op_stack[top], left, right); + if (!emit_wide_global_word_arithmetic(parent, bb, result, op_stack[top], + left, right)) + add_insn(parent, bb, op_stack[top], result, left, right, 0, NULL); + val_stack[(*value_count)++] = result; +} + +static void reduce_wide_global_pending_ops(block_t *parent, + basic_block_t *bb, + opcode_t *op_stack, + bool *protected_rhs, + var_t **val_stack, + int *op_count, + int *value_count) +{ + while (*op_count > 0) + reduce_wide_global_top_op(parent, bb, op_stack, protected_rhs, + val_stack, op_count, value_count); +} + var_t *read_wide_global_literal_expression(block_t *parent, basic_block_t *bb, block_t *scope) @@ -959,24 +852,9 @@ var_t *read_wide_global_literal_expression(block_t *parent, * operator first so the condition is the complete expression, not * merely the primary immediately before `?`. */ - while (op_stack_index > 0) { - var_t *right = val_stack[--val_stack_index]; - var_t *left = val_stack[--val_stack_index]; - var_t *result = require_var(parent); - - op_stack_index--; - if (protected_rhs[op_stack_index]) - wide_global_unevaluated_depth--; - result->var_name = gen_name(); - result->type = integer_binary_result_type( - op_stack[op_stack_index], left, right); - if (!emit_wide_global_word_arithmetic(parent, bb, result, - op_stack[op_stack_index], - left, right)) - add_insn(parent, bb, op_stack[op_stack_index], result, left, - right, 0, NULL); - val_stack[val_stack_index++] = result; - } + reduce_wide_global_pending_ops(parent, bb, op_stack, protected_rhs, + val_stack, &op_stack_index, + &val_stack_index); condition = val_stack[--val_stack_index]; condition_true = wide_global_operand_is_true(condition); @@ -1016,23 +894,10 @@ var_t *read_wide_global_literal_expression(block_t *parent, } while (op_stack_index > 0 && get_operator_prio(op_stack[op_stack_index - 1]) >= - get_operator_prio(op)) { - var_t *right = val_stack[--val_stack_index]; - var_t *left = val_stack[--val_stack_index]; - var_t *result = require_var(parent); - - op_stack_index--; - if (protected_rhs[op_stack_index]) - wide_global_unevaluated_depth--; - result->var_name = gen_name(); - result->type = integer_binary_result_type(op_stack[op_stack_index], - left, right); - if (!emit_wide_global_word_arithmetic( - parent, bb, result, op_stack[op_stack_index], left, right)) - add_insn(parent, bb, op_stack[op_stack_index], result, left, - right, 0, NULL); - val_stack[val_stack_index++] = result; - } + get_operator_prio(op)) + reduce_wide_global_top_op(parent, bb, op_stack, protected_rhs, + val_stack, &op_stack_index, + &val_stack_index); if (op_stack_index >= MAX_OPERATOR_STACK_SIZE || val_stack_index >= MAX_OPERATOR_STACK_SIZE) fatal("Wide global initializer is too complex"); @@ -1053,23 +918,9 @@ var_t *read_wide_global_literal_expression(block_t *parent, parent, bb, read_wide_global_literal_primary(parent, bb, scope)); op = get_operator(); } - while (op_stack_index > 0) { - var_t *right = val_stack[--val_stack_index]; - var_t *left = val_stack[--val_stack_index]; - var_t *result = require_var(parent); - - op_stack_index--; - if (protected_rhs[op_stack_index]) - wide_global_unevaluated_depth--; - result->var_name = gen_name(); - result->type = - integer_binary_result_type(op_stack[op_stack_index], left, right); - if (!emit_wide_global_word_arithmetic( - parent, bb, result, op_stack[op_stack_index], left, right)) - add_insn(parent, bb, op_stack[op_stack_index], result, left, right, - 0, NULL); - val_stack[val_stack_index++] = result; - } + reduce_wide_global_pending_ops(parent, bb, op_stack, protected_rhs, + val_stack, &op_stack_index, + &val_stack_index); return val_stack[0]; } @@ -1171,7 +1022,8 @@ bool read_global_assignment_var(var_t *var) if (cast_signature) { func_t *saved_signature = global_function_cast_signature; - global_function_cast_signature = cast_signature; + global_function_cast_signature = + saved_signature ? saved_signature : cast_signature; read_global_assignment_var(var); global_function_cast_signature = saved_signature; return true; @@ -1218,20 +1070,17 @@ bool read_global_assignment_var(var_t *var) address_dereference_identifier = consume_global_function_address_dereference(); - if (!find_visible_func(address_dereference_identifier->literal, - scope)) + func_t *addressed = find_visible_func( + address_dereference_identifier->literal, scope); + if (!addressed) error_at("Function address requires a visible declaration", cur_token_loc()); if (!var->is_func && !var->ptr_level && !(var->type && var->type->ptr_level)) error_at("Function address requires a pointer initializer", cur_token_loc()); - addr = require_ref_var(parent, var->type, var->ptr_level); - symbol = require_func_symbol_var(parent); - addr->var_name = gen_name(); - symbol->is_func = true; - symbol->var_name = - intern_string(address_dereference_identifier->literal); + addr = name_var(require_ref_var(parent, var->type, var->ptr_level)); + symbol = function_designator_value(parent, addressed); diagnose_global_function_conversion(symbol, var); add_insn(parent, bb, OP_address_of, addr, var, NULL, 0, NULL); add_insn(parent, bb, OP_write, NULL, addr, symbol, PTR_SIZE, NULL); @@ -1252,11 +1101,9 @@ bool read_global_assignment_var(var_t *var) cur_token_loc()); var_t *addr = require_ref_var(parent, var->type, var->ptr_level); - var_t *symbol = require_func_symbol_var(parent); + var_t *symbol = function_designator_value(parent, func); addr->var_name = gen_name(); - symbol->is_func = true; - symbol->var_name = intern_string(token); diagnose_global_function_conversion(symbol, var); lex_expect(T_identifier); if (grouped_function_designator) @@ -1275,10 +1122,8 @@ bool read_global_assignment_var(var_t *var) if (object && object->is_global && (explicit_address || object->array_size)) { fixed_array_shape_t shape = fixed_array_shape_from_var(object); - var_t *object_addr = - require_ref_var(parent, object->type, object->ptr_level); - - object_addr->var_name = gen_name(); + var_t *object_addr = name_var( + require_ref_var(parent, object->type, object->ptr_level)); object_addr->is_global_address = true; /* Taking the address of a callback object creates a slot. @@ -1305,8 +1150,8 @@ bool read_global_assignment_var(var_t *var) ? PTR_SIZE : object->type->size; var_t *byte_offset = require_var(parent); - var_t *offset_addr = require_ref_var(parent, object->type, - object->ptr_level); + var_t *offset_addr = name_var(require_ref_var( + parent, object->type, object->ptr_level)); stride = global_pointer_cast_stride ? global_pointer_cast_stride @@ -1315,7 +1160,6 @@ bool read_global_assignment_var(var_t *var) byte_offset->init_val = index * stride; add_insn(parent, bb, OP_load_constant, byte_offset, NULL, NULL, 0, NULL); - offset_addr->var_name = gen_name(); offset_addr->is_global_address = true; offset_addr->pointee_func_signature = object_addr->pointee_func_signature; @@ -1346,15 +1190,16 @@ bool read_global_assignment_var(var_t *var) * address. */ type_t *literal_type; + var_t return_decl; var_t *literal; var_t *literal_addr; int literal_ptr_level = 0; func_t *callback = NULL; lex_expect(T_open_bracket); - literal_type = read_type_name_specifiers(GLOBAL_BLOCK); - while (lex_accept(T_asterisk)) - literal_ptr_level++; + read_type_name_decl(GLOBAL_BLOCK, &return_decl); + literal_type = return_decl.type; + literal_ptr_level = return_decl.ptr_level; /* `&(int (*)(int)){f}` and `&(callback_t){f}` address a callback * object; the pointer they initialize is a callback slot. @@ -1362,8 +1207,8 @@ bool read_global_assignment_var(var_t *var) if (literal_type && abstract_function_pointer_follows()) { int callback_level; - callback = read_abstract_function_pointer( - literal_type, literal_ptr_level, &callback_level); + callback = read_abstract_function_pointer(&return_decl, + &callback_level); if (callback_level != 1) error_at("Incompatible compound literal address", cur_token_loc()); @@ -1383,8 +1228,8 @@ bool read_global_assignment_var(var_t *var) !compatible_function_signature(slot, callback)) error_at("Incompatible compound literal address", cur_token_loc()); - literal = require_typed_var(GLOBAL_BLOCK, literal_type); - literal->var_name = gen_name(); + literal = + name_var(require_typed_var(GLOBAL_BLOCK, literal_type)); literal->is_global = true; literal->ptr_level = literal_type->func_signature ? 0 : literal_ptr_level; @@ -1399,9 +1244,8 @@ bool read_global_assignment_var(var_t *var) read_global_assignment_var(literal); lex_accept(T_comma); lex_expect(T_close_curly); - literal_addr = - require_ref_var(parent, literal->type, literal->ptr_level); - literal_addr->var_name = gen_name(); + literal_addr = name_var( + require_ref_var(parent, literal->type, literal->ptr_level)); literal_addr->is_global_address = true; literal_addr->pointee_func_signature = callback; add_insn(parent, bb, OP_address_of, literal_addr, literal, NULL, @@ -1417,8 +1261,7 @@ bool read_global_assignment_var(var_t *var) (var->type == TY_void && !literal_ptr_level))) error_at("Incompatible compound literal address", cur_token_loc()); - literal = require_typed_var(GLOBAL_BLOCK, literal_type); - literal->var_name = gen_name(); + literal = name_var(require_typed_var(GLOBAL_BLOCK, literal_type)); literal->is_global = true; literal->ptr_level = literal_ptr_level; add_insn(GLOBAL_BLOCK, bb, OP_allocat, literal, NULL, NULL, 0, @@ -1434,9 +1277,8 @@ bool read_global_assignment_var(var_t *var) lex_accept(T_comma); lex_expect(T_close_curly); } - literal_addr = - require_ref_var(parent, literal->type, literal->ptr_level); - literal_addr->var_name = gen_name(); + literal_addr = name_var( + require_ref_var(parent, literal->type, literal->ptr_level)); literal_addr->is_global_address = true; add_insn(parent, bb, OP_address_of, literal_addr, literal, NULL, 0, NULL); @@ -1483,127 +1325,15 @@ bool read_global_assignment_var(var_t *var) return true; } - opcode_t op_stack[MAX_OPERATOR_STACK_SIZE]; - opcode_t op, next_op; - int val_stack[MAX_OPERATOR_STACK_SIZE]; - int op_stack_index = 0, val_stack_index = 0; - int operand1, operand2; - operand1 = read_const_expr_operand(scope); - op = get_operator(); - /* only one value after assignment */ - if (op == OP_generic) { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = operand1; - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - - rs1 = vd; - emit_global_scalar_assignment(parent, bb, var, rs1); - return true; - } - if (op == OP_ternary) { - lex_expect(T_question); - eval_ternary_imm(operand1, var); - return true; - } - operand2 = read_const_expr_operand(scope); - next_op = get_operator(); - if (next_op == OP_generic) { - /* only two operands, apply and return */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = eval_expression_imm(op, operand1, operand2); - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - - rs1 = vd; - reject_global_integer_pointer(var, rs1); - add_insn(parent, bb, OP_assign, var, rs1, NULL, 0, NULL); - return true; - } - - /* using stack if operands more than two */ - op_stack[op_stack_index++] = op; - op = next_op; - val_stack[val_stack_index++] = operand1; - val_stack[val_stack_index++] = operand2; - - while (op != OP_generic && op != OP_ternary) { - if (op_stack_index > 0) { - /* we have a continuation, use stack */ - int same_op = 0; - do { - opcode_t stack_op = op_stack[op_stack_index - 1]; - if (get_operator_prio(stack_op) >= get_operator_prio(op)) { - operand1 = val_stack[val_stack_index - 2]; - operand2 = val_stack[val_stack_index - 1]; - val_stack_index -= 2; - - /* apply stack operator and push result back */ - val_stack[val_stack_index++] = - eval_expression_imm(stack_op, operand1, operand2); - - /* pop op stack */ - op_stack_index--; - } else { - same_op = 1; - } - /* continue util next operation is higher prio */ - } while (op_stack_index > 0 && same_op == 0); - } - /* push next operand on stack */ - if (val_stack_index >= MAX_OPERATOR_STACK_SIZE || - op_stack_index >= MAX_OPERATOR_STACK_SIZE) - fatal("Constant expression too complex"); - val_stack[val_stack_index++] = read_const_expr_operand(scope); - /* push operator on stack */ - op_stack[op_stack_index++] = op; - op = get_operator(); - } - /* unwind stack and apply operations */ - while (op_stack_index > 0) { - opcode_t stack_op = op_stack[op_stack_index - 1]; - - /* pop stack and apply operators */ - operand1 = val_stack[val_stack_index - 2]; - operand2 = val_stack[val_stack_index - 1]; - val_stack_index -= 2; - - /* apply stack operator and push value back on stack */ - val_stack[val_stack_index++] = - eval_expression_imm(stack_op, operand1, operand2); - - if (op_stack_index == 1) { - if (op == OP_ternary) { - lex_expect(T_question); - eval_ternary_imm(val_stack[0], var); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = val_stack[0]; - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); - - rs1 = vd; - emit_global_scalar_assignment(parent, bb, var, rs1); - } - return true; - } + bool has_ternary; + int value = read_const_expr_until_ternary(scope, &has_ternary); - /* pop op stack */ - op_stack_index--; - } - if (op == OP_ternary) { + if (has_ternary) { lex_expect(T_question); - eval_ternary_imm(val_stack[0], var); + eval_ternary_imm(value, var); } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = val_stack[0]; - add_insn(parent, GLOBAL_FUNC->bbs, OP_load_constant, vd, NULL, NULL, - 0, NULL); - - rs1 = vd; - emit_global_scalar_assignment(parent, GLOBAL_FUNC->bbs, var, rs1); + rs1 = load_constant(parent, bb, value, TY_int); + emit_global_scalar_assignment(parent, bb, var, rs1); } return true; } diff --git a/src/parser-decl.c b/src/parser-decl.c index 268d9663..55745748 100644 --- a/src/parser-decl.c +++ b/src/parser-decl.c @@ -85,9 +85,7 @@ sizeof_derivation_t read_sizeof_abstract_declarator(block_t *scope, while (lex_accept(T_asterisk)) { derivation = SIZEOF_DERIVED_POINTER; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } /* A parenthesis opens an inner declarator only where one can begin; @@ -277,7 +275,9 @@ type_t *read_scalar_type_specifiers(bool *is_const, /* Consume the specifier list of a type name used only for its metadata, and * return the type it names, or NULL when none of it names a type. */ -type_t *read_type_name_specifiers(block_t *scope) +static type_t *read_qualified_type_name_specifiers(block_t *scope, + bool *const_out, + bool *volatile_out) { char token[MAX_ID_LEN]; bool is_const = false; @@ -285,8 +285,11 @@ type_t *read_type_name_specifiers(block_t *scope) type_t *type = read_scalar_type_specifiers(&is_const, &is_volatile, NULL); base_type_t record_kind; - if (type) + if (type) { + *const_out |= is_const; + *volatile_out |= is_volatile; return type; + } record_kind = accept_record_keyword(); if (record_kind) { lex_ident(T_identifier, token); @@ -300,12 +303,79 @@ type_t *read_type_name_specifiers(block_t *scope) lex_expect(T_identifier); } /* A qualifier may follow the record, enum or typedef name as well. */ - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + while (lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || + lex_peek(T_restrict, NULL)) { + if (lex_accept(T_const)) + is_const = true; + else if (lex_accept(T_volatile)) + is_volatile = true; + else + lex_expect(T_restrict); + } + *const_out |= is_const; + *volatile_out |= is_volatile; return type; } +type_t *read_type_name_specifiers(block_t *scope) +{ + bool is_const = false, is_volatile = false; + return read_qualified_type_name_specifiers(scope, &is_const, &is_volatile); +} + +/* Retain qualifiers on the named return type without changing its typedef. */ +static type_t *qualify_type_name(type_t *type, bool is_const, bool is_volatile) +{ + if (!type || !type->ptr_level || type->func_signature || + type->pointee_func_signature || (!is_const && !is_volatile)) + return type; + type_t *qualified = arena_alloc(GENERAL_ARENA, sizeof(*qualified)); + memcpy(qualified, type, sizeof(*qualified)); + if (type->ptr_level <= 32) { + if (is_const) + qualified->pointer_const_mask |= 1U << (type->ptr_level - 1); + if (is_volatile) + qualified->pointer_volatile_mask |= 1U << (type->ptr_level - 1); + } + return qualified; +} + +void init_type_name_decl(var_t *decl, + type_t *type, + bool is_const, + bool is_volatile) +{ + memset(decl, 0, sizeof(*decl)); + decl->type = qualify_type_name(type, is_const, is_volatile); + if (!type || !type->ptr_level) { + decl->is_const_qualified = is_const; + decl->is_volatile = is_volatile; + } +} + +void read_type_name_decl(block_t *scope, var_t *decl) +{ + bool is_const = false, is_volatile = false; + type_t *type = + read_qualified_type_name_specifiers(scope, &is_const, &is_volatile); + init_type_name_decl(decl, type, is_const, is_volatile); + while (lex_accept(T_asterisk)) { + decl->ptr_level++; + while (lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || + lex_peek(T_restrict, NULL)) { + if (lex_accept(T_const)) { + if (decl->ptr_level <= 32) + decl->pointer_const_mask |= 1U << (decl->ptr_level - 1); + } else if (lex_accept(T_volatile)) { + int depth = decl->ptr_level + (type ? type->ptr_level : 0); + if (depth <= 32) + decl->pointer_volatile_mask |= 1U << (depth - 1); + } else + lex_expect(T_restrict); + } + } +} + /* Integer constant expressions may contain sizeof(type-name). This parser only * needs type metadata, so keep it separate from expression lowering and avoid * emitting the otherwise unevaluated sizeof IR into an array bound or @@ -364,6 +434,44 @@ static int read_offsetof_array_subscripts(var_t *field, block_t *scope) } } +static int read_offsetof_designator(block_t *scope) +{ + char name[MAX_ID_LEN]; + type_t *record; + var_t *field; + int offset = 0; + bool has_nested_member; + + lex_expect(T_identifier); + lex_expect(T_open_bracket); + base_type_t record_kind = accept_record_keyword(); + if (record_kind) { + lex_ident(T_identifier, name); + record = find_record_tag(name, scope, record_kind); + } else { + lex_ident(T_identifier, name); + record = find_visible_type(name, scope); + } + if (!is_record_type(record)) + error_at("offsetof requires a struct or union type", cur_token_loc()); + lex_expect(T_comma); + do { + lex_ident(T_identifier, name); + field = find_member(name, record); + if (!field) + error_at("Unknown record member", cur_token_loc()); + offset += field->offset; + offset += read_offsetof_array_subscripts(field, scope); + record = field->type; + has_nested_member = lex_accept(T_dot); + if (has_nested_member && (field->ptr_level || !is_record_type(record))) + error_at("Nested offsetof member requires a record", + cur_token_loc()); + } while (has_nested_member); + lex_expect(T_close_bracket); + return offset; +} + /* Whether the next operand is a string literal, with any adjacent ones joined * to it, followed by a subscript, as in `"ab"[1]`. */ @@ -446,45 +554,8 @@ int read_const_expr_operand(block_t *scope) : parse_character_constant(buffer); } if (lex_peek(T_identifier, buffer)) { - if (!strcmp(buffer, "__builtin_offsetof")) { - char type_name[MAX_ID_LEN]; - char member_name[MAX_ID_LEN]; - type_t *record; - var_t *field; - int offset = 0; - bool has_nested_member; - - lex_expect(T_identifier); - lex_expect(T_open_bracket); - base_type_t record_kind = accept_record_keyword(); - if (record_kind) { - lex_ident(T_identifier, type_name); - record = find_record_tag(type_name, scope, record_kind); - } else { - lex_ident(T_identifier, type_name); - record = find_type(type_name, true); - } - if (!is_record_type(record)) - error_at("offsetof requires a struct or union type", - cur_token_loc()); - lex_expect(T_comma); - do { - lex_ident(T_identifier, member_name); - field = find_member(member_name, record); - if (!field) - error_at("Unknown record member", cur_token_loc()); - offset += field->offset; - offset += read_offsetof_array_subscripts(field, scope); - record = field->type; - has_nested_member = lex_accept(T_dot); - if (has_nested_member && - (field->ptr_level || !is_record_type(record))) - error_at("Nested offsetof member requires a record", - cur_token_loc()); - } while (has_nested_member); - lex_expect(T_close_bracket); - return offset; - } + if (!strcmp(buffer, "__builtin_offsetof")) + return read_offsetof_designator(scope); lex_expect(T_identifier); constant_t *con = find_scoped_constant(buffer, scope); if (con) @@ -497,17 +568,23 @@ int read_const_expr_operand(block_t *scope) bool constant_expression_needs_typed_value(token_t *token); -int read_const_expr(block_t *scope) +static int read_const_expr_impl(block_t *scope, + bool stop_at_ternary, + bool *has_ternary) { opcode_t op_stack[MAX_CONST_EXPR_OPS]; int val_stack[MAX_CONST_EXPR_OPS]; int op_n = 0, val_n = 0; + if (stop_at_ternary) + *has_ternary = false; + /* An int cannot hold a literal such as 4294967296 or give 3000000000U its * unsigned rank. Fold an expression holding one in the typed two-word * evaluator, and accept its value only if the int result keeps it. */ - if (constant_expression_needs_typed_value(cur_token->next)) { + if (!stop_at_ternary && + constant_expression_needs_typed_value(cur_token->next)) { pp_integer_t typed_value; block_t *saved_scope = pp_integer_constant_scope; unsigned int extension; @@ -544,6 +621,10 @@ int read_const_expr(block_t *scope) val_stack[val_n - 1] = eval_expression_imm( op_stack[op_n], val_stack[val_n - 1], val_stack[val_n]); } + if (stop_at_ternary) { + *has_ternary = true; + return val_stack[0]; + } lex_expect(T_question); bool saved_checking = checking_enum_constant; checking_enum_constant = saved_checking && val_stack[0]; @@ -565,6 +646,8 @@ int read_const_expr(block_t *scope) val_stack[val_n - 1] = eval_expression_imm( op_stack[op_n], val_stack[val_n - 1], val_stack[val_n]); } + if (stop_at_ternary && op_n >= MAX_OPERATOR_STACK_SIZE - 1) + fatal("Constant expression too complex"); if (op_n >= MAX_CONST_EXPR_OPS - 1) error_at("Constant expression nests too deeply", next_token_loc()); op_stack[op_n++] = op; @@ -580,6 +663,18 @@ int read_const_expr(block_t *scope) return val_stack[0]; } +int read_const_expr(block_t *scope) +{ + return read_const_expr_impl(scope, false, NULL); +} + +static int read_const_expr_until_ternary(block_t *scope, bool *has_ternary) +{ + /* read_global_assignment_var() already routed typed and wide expressions. + */ + return read_const_expr_impl(scope, true, has_ternary); +} + /* Whether the declarator ahead is a parenthesized name, `int (x)` or `int * (a)[3]`, which C99 6.7.5p6 declares exactly as the name alone. In a parameter * a typedef name in parentheses is instead an abstract function declarator. @@ -595,31 +690,41 @@ static bool grouped_declarator_name_follows(block_t *scope, bool is_param) !(is_param && find_visible_type(name->literal, scope)); } +static token_t *function_pointer_declarator_suffix(token_t *open, + int *star_count) +{ + token_t *token = open && open->kind == T_open_bracket ? open->next : NULL; + + if (star_count) + *star_count = 0; + if (!token || token->kind != T_asterisk) + return NULL; + while (token && (token->kind == T_asterisk || token->kind == T_const || + token->kind == T_volatile || token->kind == T_restrict)) { + if (star_count && token->kind == T_asterisk) + (*star_count)++; + token = token->next; + } + return token; +} + /* Whether the abstract declarator of a function pointer type name, such as * `(*)(int)` in `(int (*)(int))` or `(**)(void)`, starts at the next token. */ bool abstract_function_pointer_follows(void) { - token_t *token = cur_token->next; + token_t *token = function_pointer_declarator_suffix(cur_token->next, NULL); - if (!token || token->kind != T_open_bracket || !(token = token->next) || - token->kind != T_asterisk) - return false; - while (token && (token->kind == T_asterisk || token->kind == T_const || - token->kind == T_volatile || token->kind == T_restrict)) - token = token->next; return token && token->kind == T_close_bracket && token->next && token->next->kind == T_open_bracket; } -/* Read that abstract declarator for a type name whose specifiers and leading - * stars named @type and @ptr_level, the return type. +/* Read an abstract callback declarator retaining its complete return type. * * Return the prototype and set @pointer_level to the stars inside the * parentheses. */ -func_t *read_abstract_function_pointer(type_t *type, - int ptr_level, +func_t *read_abstract_function_pointer(const var_t *return_decl, int *pointer_level) { func_t *func = arena_alloc_func(); @@ -628,15 +733,12 @@ func_t *read_abstract_function_pointer(type_t *type, lex_expect(T_open_bracket); while (lex_accept(T_asterisk)) { (*pointer_level)++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } lex_expect(T_close_bracket); - func->return_def.type = type; - func->return_def.ptr_level = ptr_level; - func->returns_aggregate = - is_record_type(type) && !ptr_level && !type->ptr_level; + memcpy(&func->return_def, return_decl, sizeof(func->return_def)); + func->returns_aggregate = is_record_type(return_decl->type) && + !has_effective_pointer(return_decl); read_parameter_list_decl(func, true); return func; } @@ -661,6 +763,16 @@ static bool abstract_function_parameter_follows(block_t *scope) first->kind == T_volatile; } +/* Array suffixes on a callback object do not belong to its return type. */ +static void set_function_return(var_t *result, const var_t *declarator) +{ + memcpy(result, declarator, sizeof(*result)); + result->array_size = 0; + result->array_dim2 = result->array_dim3 = result->array_dim4 = 0; + result->has_direct_array_declarator = false; + result->has_unsized_array = false; +} + /* Read the parameter list of a parameter declared with a function type, which * C99 6.7.5.3p8 adjusts to a pointer to that function, into @vd with the * representation of `int (*fn)(int)`. @@ -669,7 +781,7 @@ static void read_adjusted_function_parameter(var_t *vd) { func_t *func = arena_alloc_func(); - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); func->returns_aggregate = is_record_type(func->return_def.type) && !has_effective_pointer(&func->return_def); read_parameter_list_decl(func, true); @@ -678,25 +790,63 @@ static void read_adjusted_function_parameter(var_t *vd) vd->parenthesized_function_pointer_level = 1; } -void read_inner_var_decl(var_t *vd, - bool anon, - bool is_param, - bool is_record_member); +static void read_array_parameter_qualifiers(var_t *vd, + bool is_param, + int dim, + bool *static_bound, + bool *const_bound) +{ + while (lex_peek(T_static, NULL) || lex_peek(T_const, NULL) || + lex_peek(T_volatile, NULL) || lex_peek(T_restrict, NULL)) { + if (!is_param || dim) + error_at("array bracket qualifiers require a parameter", + cur_token_loc()); + if (lex_accept(T_static)) { + if (*static_bound) + error_at("duplicate static array parameter qualifier", + cur_token_loc()); + *static_bound = true; + } else if (lex_accept(T_const)) { + *const_bound = true; + } else if (lex_accept(T_volatile)) { + int depth = vd->type->ptr_level + vd->ptr_level; + if (depth < 32) + vd->pointer_volatile_mask |= 1U << depth; + } else { + lex_expect(T_restrict); + } + } +} -/* The closing parenthesis matching the opening one at @open. */ -static token_t *matching_close_bracket(token_t *open) +static int read_array_parameter_bound(var_t *vd, + bool is_param, + int dim, + bool *const_bound) { - int depth = 0; + bool static_bound = false; - for (; open; open = open->next) { - if (open->kind == T_open_bracket) - depth++; - else if (open->kind == T_close_bracket && !--depth) - return open; + read_array_parameter_qualifiers(vd, is_param, dim, &static_bound, + const_bound); + if (lex_peek(T_close_square, NULL)) { + if (static_bound) + error_at("static array parameter needs a bound", cur_token_loc()); + return 0; } - return NULL; + int bound = read_const_expr(vd->scope); + + if (static_bound && bound <= 0) + error_at("static array parameter needs a positive bound", + cur_token_loc()); + if (bound <= 0) + error_at("Array size must be positive", cur_token_loc()); + return bound; } +void read_inner_var_decl(var_t *vd, + bool anon, + bool is_param, + bool is_record_member); + /* Whether the declarator ahead declares a function returning a function * pointer, `(*get(void))(int)`, or a pointer to one, `(*(*pg)(void))(int)`: a * star inside a parenthesis followed by a parameter list, and another parameter @@ -705,20 +855,14 @@ static token_t *matching_close_bracket(token_t *open) static bool function_pointer_return_follows(void) { token_t *open = cur_token->next; - token_t *token; + token_t *token = function_pointer_declarator_suffix(open, NULL); token_t *close; - if (!open || open->kind != T_open_bracket || !(token = open->next) || - token->kind != T_asterisk) - return false; - while (token && (token->kind == T_asterisk || token->kind == T_const || - token->kind == T_volatile || token->kind == T_restrict)) - token = token->next; if (!token || !(token->kind == T_open_bracket || (token->kind == T_identifier && token->next && token->next->kind == T_open_bracket))) return false; - close = matching_close_bracket(open); + close = skip_balanced_delimiter(open, T_open_bracket, T_close_bracket); return close && close->next && close->next->kind == T_open_bracket; } @@ -734,14 +878,15 @@ static void read_function_pointer_return_declarator(var_t *vd, bool is_record_member) { token_t *start = cur_token; - token_t *close = matching_close_bracket(cur_token->next); + token_t *close = skip_balanced_delimiter(cur_token->next, T_open_bracket, + T_close_bracket); token_t *end; func_t *callback = arena_alloc_func(); type_t *callback_type = add_type(); int stars = 0; cur_token = close; - memcpy(&callback->return_def, vd, sizeof(var_t)); + set_function_return(&callback->return_def, vd); callback->return_def.var_name = NULL; callback->returns_aggregate = is_record_type(vd->type) && !has_effective_pointer(&callback->return_def); @@ -755,6 +900,7 @@ static void read_function_pointer_return_declarator(var_t *vd, callback_type->type_name[0] = '\0'; callback_type->ptr_level = 0; callback_type->pointer_const_mask = 0; + callback_type->pointer_volatile_mask = 0; callback_type->size = PTR_SIZE; callback_type->alignment = PTR_SIZE; callback_type->func_signature = callback; @@ -764,9 +910,7 @@ static void read_function_pointer_return_declarator(var_t *vd, lex_expect(T_open_bracket); while (lex_accept(T_asterisk)) { stars++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } /* `(**get(void))(int)` returns a pointer to such a callback, and each @@ -775,6 +919,7 @@ static void read_function_pointer_return_declarator(var_t *vd, vd->type = callback_type; vd->ptr_level = stars - 1; vd->pointer_const_mask = 0; + vd->pointer_volatile_mask = 0; vd->is_const_pointer = false; read_inner_var_decl(vd, anon, is_param, is_record_member); @@ -783,7 +928,7 @@ static void read_function_pointer_return_declarator(var_t *vd, func_t *func = arena_alloc_func(); type_t *function_type = add_type(); - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); read_parameter_list_decl(func, true); function_type->base_type = TYPE_typedef; function_type->size = PTR_SIZE; @@ -825,6 +970,13 @@ void read_inner_var_decl(var_t *vd, vd->ptr_level = 0; } + if (lex_peek(T_asterisk, NULL) && vd->type && vd->type->func_signature && + !vd->type->is_direct_function_type && !vd->type->ptr_level && + !vd->ptr_level) { + vd->callback_is_volatile = vd->pointer_volatile_mask & 1U; + vd->pointer_volatile_mask &= ~1U; + } + /* `typedef int (*const cfn_t)(int)` makes a cfn_t object read-only, but * `cfn_t *p` only points to such a callback: its qualifier stays on the * type rather than on p's own pointer level. @@ -849,9 +1001,11 @@ void read_inner_var_decl(var_t *vd, vd->is_const_pointer = true; if (vd->ptr_level <= 32) vd->pointer_const_mask |= 1U << (vd->ptr_level - 1); - } else if (lex_accept(T_volatile)) - vd->is_volatile = true; - else if (lex_accept(T_restrict)) + } else if (lex_accept(T_volatile)) { + int depth = vd->type->ptr_level + vd->ptr_level; + if (depth <= 32) + vd->pointer_volatile_mask |= 1U << (depth - 1); + } else if (lex_accept(T_restrict)) ; /* restrict is an aliasing contract, not storage state. */ else break; @@ -929,7 +1083,10 @@ void read_inner_var_decl(var_t *vd, else vd->parenthesized_function_pointer_inner_qualified = true; - vd->is_volatile = true; + int depth = + vd->type->ptr_level + vd->ptr_level + nested_ptr_level; + if (depth <= 32) + vd->pointer_volatile_mask |= 1U << (depth - 1); } else if (lex_accept(T_restrict)) { vd->parenthesized_function_pointer_restrict = true; if (nested_ptr_level >= 2) @@ -955,34 +1112,15 @@ void read_inner_var_decl(var_t *vd, * allocation can share their existing paths. */ for (int dim = 0; lex_accept(T_open_square); dim++) { - bool parameter_static_bound = false; bool parameter_const_bound = false; inner_array = true; if (dim >= 4) error_at("Array declarators support at most four dimensions", cur_token_loc()); - while (lex_peek(T_static, NULL) || lex_peek(T_const, NULL) || - lex_peek(T_volatile, NULL) || lex_peek(T_restrict, NULL)) { - if (!is_param || dim) - error_at("array bracket qualifiers require a parameter", - cur_token_loc()); - if (lex_accept(T_static)) { - if (parameter_static_bound) - error_at("duplicate static array parameter qualifier", - cur_token_loc()); - parameter_static_bound = true; - } else if (lex_accept(T_const)) - parameter_const_bound = true; - else if (lex_accept(T_volatile)) - vd->is_volatile = true; - else - lex_expect(T_restrict); - } - if (lex_peek(T_close_square, NULL)) { - if (parameter_static_bound) - error_at("static array parameter needs a bound", - cur_token_loc()); + int bound = read_array_parameter_bound(vd, is_param, dim, + ¶meter_const_bound); + if (!bound) { if (dim) error_at( "Only the outer function-pointer array bound may " @@ -990,24 +1128,9 @@ void read_inner_var_decl(var_t *vd, cur_token_loc()); vd->has_unsized_array = true; } else { - int bound = read_const_expr(vd->scope); - - if (parameter_static_bound && bound <= 0) - error_at("static array parameter needs a positive bound", - cur_token_loc()); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (dim == 0) - vd->array_size = bound; - else { - if (dim == 1) - vd->array_dim2 = bound; - else if (dim == 2) - vd->array_dim3 = bound; - else if (dim == 3) - vd->array_dim4 = bound; - vd->array_size *= bound; - } + set_array_dimension(&vd->array_size, &vd->array_dim2, + &vd->array_dim3, &vd->array_dim4, dim, + bound, false); } if (parameter_const_bound && dim == 0) { vd->is_const_pointer = true; @@ -1022,7 +1145,7 @@ void read_inner_var_decl(var_t *vd, * from the pointer object's own storage extent. */ if (lex_peek(T_open_square, NULL)) { - int dims = 0; + fixed_array_shape_t pointee_shape = {0}; /* The nested stars make this a pointer-to-array. Preserve whether * its array element was plain void before adding them; a typedef or @@ -1043,33 +1166,16 @@ void read_inner_var_decl(var_t *vd, vd->is_const_pointer = vd->ptr_level <= 32 && (vd->pointer_const_mask & (1U << (vd->ptr_level - 1))); - while (lex_accept(T_open_square)) { - int bound; - - if (dims >= 4) - error_at( - "Array declarators support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) - error_at("Pointer-to-array needs a bound", cur_token_loc()); - bound = read_const_expr(vd->scope); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (dims == 0) - vd->pointee_array_size = bound; - else { - if (dims == 1) - vd->pointee_array_dim2 = bound; - else if (dims == 2) - vd->pointee_array_dim3 = bound; - else - vd->pointee_array_dim4 = bound; - vd->pointee_array_size *= bound; - } - lex_expect(T_close_square); - dims++; - } - if (dims && void_array_element) + read_array_shape_bounds( + vd->scope, &pointee_shape, + "Array declarators support at most four dimensions", + "Pointer-to-array needs a bound", "Array size must be positive", + true, false); + fixed_array_shape_apply(&pointee_shape, &vd->pointee_array_size, + &vd->pointee_array_dim2, + &vd->pointee_array_dim3, + &vd->pointee_array_dim4); + if (pointee_shape.rank && void_array_element) error_at("void type cannot be an array element", cur_token_loc()); return; @@ -1129,7 +1235,7 @@ void read_inner_var_decl(var_t *vd, * declarator. Copy it into the syntax-only signature before the * function-pointer marker is set on vd. */ - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); func->returns_aggregate = is_record_type(func->return_def.type) && !has_effective_pointer(&func->return_def); read_parameter_list_decl(func, true); @@ -1141,6 +1247,8 @@ void read_inner_var_decl(var_t *vd, vd->func_signature = func; vd->is_func = true; vd->parenthesized_function_pointer_level = nested_ptr_level; + vd->callback_is_const = callback_const; + vd->callback_is_volatile = callback_volatile; if (nested_ptr_level >= 2) { /* `int (**slot)(int)` is an ordinary pointer object whose pointee * is the compact one-pointer callback representation. Keep that @@ -1157,8 +1265,6 @@ void read_inner_var_decl(var_t *vd, if (callback_restrict) error_at("restrict requires a pointer to an object type", cur_token_loc()); - vd->callback_is_const = callback_const; - vd->callback_is_volatile = callback_volatile; if (callback_volatile && !vd->parenthesized_function_pointer_outer_volatile) vd->is_volatile = false; @@ -1175,6 +1281,10 @@ void read_inner_var_decl(var_t *vd, vd->pointer_const_mask = (vd->pointer_const_mask & return_mask) | ((slot_mask >> 1) << return_depth); + slot_mask = vd->pointer_volatile_mask >> return_depth; + vd->pointer_volatile_mask = + (vd->pointer_volatile_mask & return_mask) | + ((slot_mask >> 1) << return_depth); } vd->ptr_level += nested_ptr_level - 1; vd->is_const_pointer = @@ -1262,36 +1372,14 @@ void read_inner_var_decl(var_t *vd, for (int dim = 0; lex_accept(T_open_square); dim++) { vd->has_direct_array_declarator = true; - bool parameter_static_bound = false; bool parameter_const_bound = false; if (dim >= 4) error_at("Array declarators support at most four dimensions", cur_token_loc()); - while (lex_peek(T_static, NULL) || lex_peek(T_const, NULL) || - lex_peek(T_volatile, NULL) || lex_peek(T_restrict, NULL)) { - if (!is_param || dim) - error_at("array bracket qualifiers require a parameter", - cur_token_loc()); - if (lex_accept(T_static)) { - if (parameter_static_bound || dim) - error_at( - "static is only valid in the outermost array " - "parameter bound", - cur_token_loc()); - parameter_static_bound = true; - } else if (lex_accept(T_const)) - parameter_const_bound = true; - else if (lex_accept(T_volatile)) - vd->is_volatile = true; - else - lex_expect(T_restrict); - } - if (lex_peek(T_close_square, NULL)) { - if (parameter_static_bound) - error_at("static array parameter needs a bound", - cur_token_loc()); - + int next_dim = read_array_parameter_bound(vd, is_param, dim, + ¶meter_const_bound); + if (!next_dim) { /* An omitted leading size is only a pointer when nothing * follows it: "int a[]" is "int *", but "int a[][4]" points at * rows of four and is indexed exactly like "int a[3][4]". @@ -1304,26 +1392,9 @@ void read_inner_var_decl(var_t *vd, else vd->ptr_level++; } else { - int next_dim = read_const_expr(vd->scope); - - if (parameter_static_bound && next_dim <= 0) - error_at("static array parameter needs a positive bound", - cur_token_loc()); - - if (dim == 0) { - vd->array_size = next_dim; - } else { - if (dim == 1) - vd->array_dim2 = next_dim; - else if (dim == 2) - vd->array_dim3 = next_dim; - else if (dim == 3) - vd->array_dim4 = next_dim; - if (vd->array_size > 0) - vd->array_size *= next_dim; - else - vd->array_size = next_dim; - } + set_array_dimension(&vd->array_size, &vd->array_dim2, + &vd->array_dim3, &vd->array_dim4, dim, + next_dim, true); } if (parameter_const_bound && dim == 0) { vd->is_const_pointer = true; @@ -1408,7 +1479,7 @@ void read_inner_var_decl(var_t *vd, bool saved_block_typedef_declarator = parsing_block_typedef_declarator; - memcpy(&func->return_def, vd, sizeof(var_t)); + set_function_return(&func->return_def, vd); func->returns_aggregate = is_record_type(func->return_def.type) && !has_effective_pointer(&func->return_def); @@ -1620,7 +1691,7 @@ type_t *read_record_body(block_t *parent, int max_size = 0; bitfield_layout_t bits = {0}; bool has_flexible_array_member = false; - block_t *scope = parent ? parent : GLOBAL_BLOCK; + block_t *scope = parent ? parent : CURRENT_TU_SCOPE; if (has_tag) { type = local_record_tag(token, scope, kind); @@ -1764,6 +1835,7 @@ void read_full_var_decl(var_t *vd, vd->pointee_func_signature = type->pointee_func_signature; vd->is_func = vd->func_signature != NULL; vd->is_const_qualified = type->is_const_qualified; + vd->pointer_volatile_mask = type->pointer_volatile_mask; if (type->func_signature && !type->is_direct_function_type && (type->pointer_const_mask & 1U)) { vd->is_const_pointer = true; @@ -1777,9 +1849,8 @@ void read_full_var_decl(var_t *vd, if (type->pointee_array_size) { fixed_array_shape_t shape = fixed_array_shape_from_pointee_type(type); - fixed_array_shape_to_pointee_var(vd, &shape); - vd->pointee_array_element_ptr_level = - type->pointee_array_element_ptr_level; + set_pointee_array_shape(vd, &shape, + type->pointee_array_element_ptr_level); } if (type->ptr_level && type->ptr_level <= 32) vd->is_const_pointer = @@ -1814,8 +1885,16 @@ void read_full_var_decl(var_t *vd, break; } vd->is_inline = is_inline; - vd->is_volatile = - declaration_volatile || is_volatile || type->is_volatile_qualified; + vd->is_volatile = type->is_volatile_qualified; + if (declaration_volatile || is_volatile) { + int depth = type->ptr_level + (type->func_signature && + !type->is_direct_function_type && + !type->ptr_level); + if (depth && depth <= 32) + vd->pointer_volatile_mask |= 1U << (depth - 1); + else + vd->is_volatile = true; + } if (is_const || declaration_const) { if (type->ptr_level) { vd->is_const_pointer = true; @@ -1825,6 +1904,9 @@ void read_full_var_decl(var_t *vd, vd->is_const_qualified = true; } + vd->type = qualify_type_name(type, is_const || declaration_const, + is_volatile || declaration_volatile); + read_inner_var_decl(vd, anon, is_param, is_record_member); if (!parsing_sizeof_function_signature && vd->func_signature && @@ -1842,7 +1924,19 @@ void read_full_var_decl(var_t *vd, /* starting next_token, need to check the type */ void read_partial_var_decl(var_t *vd, var_t *template) { - UNUSED(template); + int depth = vd->type->ptr_level + (vd->type->func_signature && + !vd->type->is_direct_function_type && + !vd->type->ptr_level); + vd->pointer_volatile_mask = vd->type->pointer_volatile_mask; + if (depth && depth <= 32) { + if (template) + vd->pointer_volatile_mask |= + template->pointer_volatile_mask & + (depth == 32 ? ~0U : (1U << depth) - 1); + else if (vd->is_volatile) + vd->pointer_volatile_mask |= 1U << (depth - 1); + vd->is_volatile = vd->type->is_volatile_qualified; + } read_inner_var_decl(vd, false, false, false); } @@ -1985,8 +2079,7 @@ void read_literal_param(block_t *parent, basic_block_t *bb) int length = read_concatenated_string(combined); const int index = write_string_symbol(combined, length); - var_t *vd = require_typed_ptr_var(parent, TY_char, true); - vd->var_name = gen_name(); + var_t *vd = name_var(require_typed_ptr_var(parent, TY_char, true)); vd->init_val = index; vd->is_const_qualified = true; vd->is_string_literal = true; @@ -2002,183 +2095,28 @@ void read_literal_param(block_t *parent, basic_block_t *bb) */ void parse_string_array_init(var_t *var, block_t *parent, basic_block_t **bb) { - char combined[MAX_STRING_LEN]; - int len; - int count; - - len = read_concatenated_string(combined) + 1; - if (var->has_unsized_array) { - var->array_size = len; - var->has_unsized_array = false; - } else if (len - 1 > var->array_size) - error_at("String initializer is too long for character array", - cur_token_loc()); - - /* The terminating null is dropped when the array has room only for the - * characters (C99 6.7.8p14), so never store past the array. - */ - if (len > var->array_size) - len = var->array_size; - - /* Elements past the string are zero. Static storage already starts out - * zeroed, but an automatic array is reinitialized on every entry to its - * declaration and must clear whatever the slot held before. - */ - count = parent == GLOBAL_BLOCK ? len : var->array_size; - for (int i = 0; i < count; i++) { - var_t *value = require_var(parent); - var_t *addr; - - value->var_name = gen_name(); - value->init_val = i < len ? (unsigned char) combined[i] : 0; - value->is_const = true; - add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - addr = compute_element_address(parent, bb, var, i, 1); - add_insn(parent, *bb, OP_write, NULL, addr, value, 1, NULL); - } + parse_string_array_init_to(var, parent, bb, var, var, false); } void parse_wstring_array_init(var_t *var, block_t *parent, basic_block_t **bb) { - int values[MAX_STRING_LEN]; - int length; - int units; - - length = read_wstring_units(values, MAX_STRING_LEN); - - /* As for a char array, the terminating null may be dropped (C99 6.7.8p15). - */ - units = length + 1; - if (var->has_unsized_array) { - var->array_size = units; - var->has_unsized_array = false; - } else if (length > var->array_size) - error_at("Wide string initializer is too long for array", - cur_token_loc()); - - for (int i = 0; i < var->array_size; i++) { - var_t *value = require_typed_var(parent, var->type); - var_t *addr; - - value->var_name = gen_name(); - value->init_val = i < length ? values[i] : 0; - value->is_const = true; - add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - addr = compute_element_address(parent, bb, var, i, var->type->size); - add_insn(parent, *bb, OP_write, NULL, addr, value, var->type->size, - NULL); - } + parse_string_array_init_to(var, parent, bb, var, var, true); } -bool numeric_has_unsigned_suffix(const char *token) -{ - for (int i = 0; token[i]; i++) - if ((token[i] | 32) == 'u') - return true; - return false; -} +typedef struct { + int unsigned_count; + int long_count; +} numeric_suffix_info_t; -/* C99 permits U, L, LL, UL, ULL, LU, and LLU (case-insensitively). Keep this - * separate from type selection: malformed suffixes must not become a valid wide - * literal merely because their letters happen to be counted. - */ -bool numeric_suffix_is_valid(const char *suffix) +static numeric_suffix_info_t numeric_suffix_info(const char *token) { - int pos = 0; - - if ((suffix[pos] | 32) == 'u') - pos++; - if ((suffix[pos] | 32) == 'l') { - pos++; - if ((suffix[pos] | 32) == 'l') - pos++; - } - if ((suffix[pos] | 32) == 'u') - pos++; - return suffix[pos] == '\0'; -} - -bool numeric_has_long_long_suffix(const char *token) -{ - int long_suffix_count = 0; - - for (int i = 0; token[i]; i++) - if ((token[i] | 32) == 'l') - long_suffix_count++; - return long_suffix_count == 2; -} - -int numeric_long_suffix_count(const char *token) -{ - int count = 0; + numeric_suffix_info_t info = {0}; - for (int i = 0; token[i]; i++) - if ((token[i] | 32) == 'l') - count++; - return count; -} - -/* The file-scope constant evaluator is still word-sized. Decide from the token - * spelling whether it must use the two-word literal path before that evaluator - * consumes and narrows it. - */ -bool numeric_literal_needs_wide_path(const char *token) -{ - const char *digits = token; - int count = 0; - bool is_unsigned = numeric_has_unsigned_suffix(token); - - if (numeric_has_long_long_suffix(token)) - return true; - if (digits[0] == '0' && (digits[1] | 32) == 'x') { - digits += 2; - while (isxdigit(digits[count])) - count++; - return count > 8; - } - if (digits[0] == '0' && (digits[1] | 32) == 'b') { - digits += 2; - while (digits[count] == '0' || digits[count] == '1') - count++; - return count > 32; + for (int i = 0; token[i]; i++) { + info.unsigned_count += (token[i] | 32) == 'u'; + info.long_count += (token[i] | 32) == 'l'; } - if (digits[0] == '0') { - while (digits[count] >= '0' && digits[count] <= '7') - count++; - return count > 12 || - (count == 12 && strncmp(digits, "037777777777", count) > 0); - } - while (isdigit(digits[count])) - count++; - if (count > 10) - return true; - if (count < 10) - return false; - return strncmp(digits, is_unsigned ? "4294967295" : "2147483647", count) > - 0; -} - -/* Some bootstrap stages still materialize the exact decimal 2^31 token as an - * int bit pattern before the global initializer path sees it. This path was - * selected from the original token spelling, so restore the required wide - * candidate type before phase-2 chooses its constant-load width. - */ -void force_wide_global_literal_type(var_t *value, const char *token) -{ - /* The typed global path also handles a one-word unsigned literal such as - * 0xffffffff: it must retain TY_uint for shifts, comparisons, and division, - * but it is not a long-long candidate. Promoting it here made every 32-bit - * target reject a valid C99 unsigned-int initializer merely because that - * path was selected. - */ - if (!numeric_literal_needs_wide_path(token)) - return; - if (value->type->size >= 8) - return; - value->type = numeric_has_unsigned_suffix(token) || - (unsigned int) value->init_val_hi > 0x7fffffffU - ? TY_ulong_long - : TY_long_long; + return info; } /* Accumulate a 64-bit token in four 16-bit limbs. Each intermediate stays small @@ -2209,51 +2147,97 @@ bool numeric_mul_add_wide(unsigned int *hi, return true; } -/* The global-initializer fast path historically carries only an int value. - * Route literals whose C99 candidate is unsigned through the typed path even - * when they fit in one word: otherwise 0xffffffff is folded as -1 before a - * right shift, comparison, or division sees its unsigned rank. - */ -bool numeric_literal_needs_typed_global_path(const char *token) +typedef struct { + int base; + int digits; + const char *digit_text; + bool is_unsigned; + bool decimal; + bool needs_wide; + bool needs_typed_global; +} numeric_literal_info_t; + +static numeric_literal_info_t numeric_literal_info(const char *token, + bool include_typed) { + numeric_suffix_info_t suffix = numeric_suffix_info(token); + numeric_literal_info_t info = {10, 0, token, false, true, false, false}; unsigned int hi = 0; unsigned int lo = 0; - int i = 0; - int base = 10; - bool is_decimal = true; - if (numeric_has_unsigned_suffix(token)) - return true; - if (token[0] == '0') { - if ((token[1] | 32) == 'x') { - i = 2; - base = 16; - is_decimal = false; - while (isxdigit(token[i])) { - char digit = token[i++]; - - if (isdigit(digit)) - digit -= '0'; - else - digit = (digit | 32) - 'a' + 10; - numeric_mul_add_wide(&hi, &lo, base, digit); - } - } else if ((token[1] | 32) == 'b') { - i = 2; - base = 2; - is_decimal = false; - while (token[i] == '0' || token[i] == '1') - numeric_mul_add_wide(&hi, &lo, base, token[i++] - '0'); - } else { - base = 8; - is_decimal = false; - while (token[i] >= '0' && token[i] <= '7') - numeric_mul_add_wide(&hi, &lo, base, token[i++] - '0'); - } + info.is_unsigned = suffix.unsigned_count != 0; + if (suffix.long_count == 2) { + info.needs_wide = true; + info.needs_typed_global = include_typed && info.is_unsigned; + return info; } - if (is_decimal) - return false; - return hi != 0 || lo > 0x7fffffffU; + info.base = numeric_literal_base(token); + info.decimal = info.base == 10; + if (info.base == 16 || info.base == 2) + info.digit_text += 2; + int max_typed_digits = info.base == 16 ? 8 : info.base == 2 ? 32 : 12; + while (info.digit_text[info.digits]) { + char c = info.digit_text[info.digits] | 32; + unsigned int digit = isdigit(c) ? c - '0' : c - 'a' + 10; + + if ((!isdigit(c) && !isxdigit(c)) || digit >= (unsigned int) info.base) + break; + if (include_typed && !info.decimal && !info.is_unsigned && + info.digits < max_typed_digits) + numeric_mul_add_wide(&hi, &lo, info.base, digit); + info.digits++; + } + if (info.base == 16) + info.needs_wide = info.digits > 8; + else if (info.base == 2) + info.needs_wide = info.digits > 32; + else if (info.base == 8) + info.needs_wide = info.digits > 12 || + (info.digits == 12 && + strncmp(token, "037777777777", info.digits) > 0); + else + info.needs_wide = + info.digits > 10 || + (info.digits == 10 && + strncmp(token, info.is_unsigned ? "4294967295" : "2147483647", + info.digits) > 0); + info.needs_typed_global = + include_typed && + (info.is_unsigned || (!info.decimal && (hi || lo > 0x7fffffffU))); + return info; +} + +/* Keep token routing in one classifier: the word-sized evaluator must preserve + * unsigned candidates and route wide values before narrowing them. + */ +bool numeric_literal_needs_wide_path(const char *token) +{ + return numeric_literal_info(token, false).needs_wide; +} + +bool numeric_literal_needs_global_reader(const char *token) +{ + numeric_literal_info_t info = numeric_literal_info(token, true); + + return info.needs_wide || info.needs_typed_global; +} + +/* Some bootstrap stages materialize decimal 2^31 as an int bit pattern. Restore + * its wide candidate type before phase 2 chooses its constant-load width. + */ +void force_wide_global_literal_type(var_t *value, const char *token) +{ + numeric_literal_info_t literal = numeric_literal_info(token, false); + + /* The global-reader path also handles unsigned int values such as + * 0xffffffff, which this helper must not promote to long long. + */ + if (!literal.needs_wide || value->type->size >= 8) + return; + value->type = + literal.is_unsigned || (unsigned int) value->init_val_hi > 0x7fffffffU + ? TY_ulong_long + : TY_long_long; } void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) @@ -2268,11 +2252,13 @@ void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) bool has_unsigned_suffix; int long_suffix_count; bool is_long_long; + numeric_suffix_info_t suffix; lex_ident_n(T_numeric, token, MAX_TOKEN_LEN); - has_unsigned_suffix = numeric_has_unsigned_suffix(token); - long_suffix_count = numeric_long_suffix_count(token); - is_long_long = long_suffix_count >= 2; + suffix = numeric_suffix_info(token); + has_unsigned_suffix = suffix.unsigned_count != 0; + long_suffix_count = suffix.long_count; + is_long_long = suffix.long_count >= 2; if (token[0] == '-') { is_neg = !is_neg; @@ -2357,8 +2343,7 @@ void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) !has_unsigned_suffix && !long_suffix_count && !value_hi && value == 0x80000000U; - var_t *vd = require_var(parent); - vd->var_name = gen_name(); + var_t *vd = require_named_var(parent); if (narrow_int_min) { vd->type = TY_int; } else if (is_long_long || value_hi || @@ -2395,17 +2380,13 @@ void read_numeric_param(block_t *parent, basic_block_t *bb, bool is_neg) void read_char_param(block_t *parent, basic_block_t *bb) { - char literal[MAX_TOKEN_LEN], unescaped[MAX_TOKEN_LEN]; + char literal[MAX_TOKEN_LEN]; lex_ident(T_char, literal); - unescape_string(literal, unescaped, MAX_TOKEN_LEN); - - var_t *vd = require_typed_var(parent, TY_int); - vd->var_name = gen_name(); - vd->init_val = parse_character_constant(literal); + int value = parse_character_constant(literal); + var_t *vd = load_constant(parent, bb, value, TY_int); vd->is_const = true; opstack_push(vd); - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); } /* The current execution wide-character representation is int. Decode the same @@ -2414,17 +2395,13 @@ void read_char_param(block_t *parent, basic_block_t *bb) */ void read_wchar_param(block_t *parent, basic_block_t *bb) { - char literal[MAX_TOKEN_LEN], unescaped[MAX_TOKEN_LEN]; + char literal[MAX_TOKEN_LEN]; lex_ident(T_wchar, literal); - unescape_string(literal, unescaped, MAX_TOKEN_LEN); - - var_t *vd = require_typed_var(parent, TY_int); - vd->var_name = gen_name(); - vd->init_val = parse_wide_character_constant(literal); + int value = parse_wide_character_constant(literal); + var_t *vd = load_constant(parent, bb, value, TY_int); vd->is_const = true; opstack_push(vd); - add_insn(parent, bb, OP_load_constant, vd, NULL, NULL, 0, NULL); } void read_wstring_param(block_t *parent, basic_block_t *bb) @@ -2432,8 +2409,8 @@ void read_wstring_param(block_t *parent, basic_block_t *bb) int values[MAX_STRING_LEN]; int length = read_wstring_units(values, MAX_STRING_LEN); - var_t *vd = require_typed_ptr_var(parent, find_type("wchar_t", true), 1); - vd->var_name = gen_name(); + var_t *vd = name_var( + require_typed_ptr_var(parent, find_builtin_type("wchar_t"), 1)); vd->init_val = write_wide_symbol(values, length); vd->is_const_qualified = true; vd->is_string_literal = true; @@ -2452,18 +2429,10 @@ bool string_address_offset_starts_here(void) int bracket_depth = 0; for (token_t *token = cur_token->next; token; token = token->next) { - if (token->kind == T_open_bracket) - bracket_depth++; - else if (token->kind == T_close_bracket) { - if (bracket_depth == 0) - return false; - bracket_depth--; - } else if (bracket_depth == 0 && - (token->kind == T_semicolon || token->kind == T_comma || - token->kind == T_close_curly)) + if (!initializer_scan_continue(token, &bracket_depth, true)) return false; - else if (bracket_depth == 0 && - (token->kind == T_string || token->kind == T_wstring)) { + if (bracket_depth == 0 && + (token->kind == T_string || token->kind == T_wstring)) { token_t *after = token; while (after->next && after->next->kind == token->kind) @@ -2485,12 +2454,28 @@ bool string_address_offset_starts_here(void) * literal's address advanced by the integer constants added to or subtracted * from it, in element units. */ +static var_t *string_literal_address(block_t *parent, + basic_block_t **bb, + var_t *literal, + int index) +{ + var_t *address = compute_element_address(parent, bb, literal, index, + literal->type->size); + + if (address != literal) { + address->type = literal->type; + address->ptr_level = literal->ptr_level; + address->is_const_qualified = true; + address->is_string_literal = true; + } + return address; +} + var_t *read_string_address_offset(block_t *parent, basic_block_t *bb, block_t *scope) { var_t *literal; - var_t *address; int index = 0; if (!lex_peek(T_string, NULL) && !lex_peek(T_wstring, NULL)) { @@ -2516,15 +2501,7 @@ var_t *read_string_address_offset(block_t *parent, literal = opstack_pop(); if (lex_peek(T_plus, NULL) || lex_peek(T_minus, NULL)) index += read_global_address_offset(scope, parent, bb); - address = compute_element_address(parent, &bb, literal, index, - literal->type->size); - if (address != literal) { - address->type = literal->type; - address->ptr_level = literal->ptr_level; - address->is_const_qualified = true; - address->is_string_literal = true; - } - return address; + return string_literal_address(parent, &bb, literal, index); } /* The address constant `&"ab"[1]` in a static initializer, with the '&' already @@ -2536,7 +2513,6 @@ var_t *read_string_literal_element_address(block_t *parent, { token_t *literal_token = cur_token; var_t *literal; - var_t *address; int index; /* Check the index against the literal, then read the literal again to emit @@ -2552,13 +2528,5 @@ var_t *read_string_literal_element_address(block_t *parent, lex_expect(T_open_square); read_const_expr(scope); lex_expect(T_close_square); - address = compute_element_address(parent, &bb, literal, index, - literal->type->size); - if (address != literal) { - address->type = literal->type; - address->ptr_level = literal->ptr_level; - address->is_const_qualified = true; - address->is_string_literal = true; - } - return address; + return string_literal_address(parent, &bb, literal, index); } diff --git a/src/parser-expr.c b/src/parser-expr.c index 508ec68d..e78b797a 100644 --- a/src/parser-expr.c +++ b/src/parser-expr.c @@ -12,279 +12,74 @@ * definition that precedes it there and cannot be compiled on its own. */ -/* The grouped row forms below accept only `(*name)` where name points to a - * scalar fixed array; any other operand, such as `(*pp)[1]` on an int **, - * belongs to ordinary expression parsing. - */ -static bool names_scalar_row_pointer(token_t *token, block_t *parent) -{ - var_t *var = find_var(token->literal, parent); - - return var && var->pointee_array_size && - !var->pointee_array_element_ptr_level && - effective_pointer_depth(var) == 1; -} - -/* Keep grouped row-element postfix support deliberately exact until general - * grouped lvalues have an address representation. This scanner never consumes - * tokens, so every non-match remains owned by ordinary expression parsing. - */ -static bool grouped_scalar_pointee_row_postfix_starts(block_t *parent) -{ - token_t *token = cur_token ? cur_token->next : NULL; - int suffixes = 0; - - if (!token || token->kind != T_open_bracket || !(token = token->next) || - token->kind != T_asterisk || !(token = token->next) || - token->kind != T_identifier || - !names_scalar_row_pointer(token, parent) || !(token = token->next) || - token->kind != T_close_bracket || !(token = token->next)) - return false; - - while (suffixes < 4 && token && token->kind == T_open_square) { - int depth = 0; - - for (; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || - token->kind == T_close_curly) { - if (!--depth) { - token = token->next; - break; - } - } - } - if (!token) - return false; - suffixes++; - } - - return suffixes && - (token->kind == T_increment || token->kind == T_decrement); -} +static var_t *combine_assignment_value(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *current, + var_t *value); +static var_t *combine_addressed_assignment(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *address, + const lvalue_t *target, + var_t *value); +static var_t *store_addressed_assignment(block_t *parent, + basic_block_t **bb, + var_t *address, + const lvalue_t *target, + var_t *value, + bool use_resize_to); +static bool accept_assignment_op(opcode_t *op); -static void handle_grouped_scalar_pointee_row_postfix(block_t *parent, - basic_block_t **bb) +static opcode_t read_increment_operator(void) { - char name[MAX_VAR_LEN]; - var_t *source, *row, *index, *address, *old, *one, *updated; - fixed_array_shape_t shape; - int dimensions; - int depth = 0; - opcode_t op; - - lex_expect(T_open_bracket); - lex_expect(T_asterisk); - lex_ident(T_identifier, name); - source = find_var(name, parent); - if (!source) - error_at("Undeclared identifier", cur_token_loc()); - if (!lower_pointee_array_dereference(source, parent, bb)) - error_at("Grouped pointer-to-array update requires a scalar fixed row", - cur_token_loc()); - row = opstack_pop(); - if (row->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - shape = fixed_array_shape_from_var(row); - dimensions = shape.rank; - if (dimensions >= 3 && - (is_record_type(row->type) || row->type->is_floating)) - error_at( - "Grouped rank-three/four postfix update requires an integer scalar", - cur_token_loc()); - if (dimensions > 4) - error_at( - "Grouped pointer-to-array postfix update supports at most four " - "dimensions", - cur_token_loc()); - lex_expect(T_close_bracket); - address = row; - while (lex_accept(T_open_square)) { - int stride; - - if (depth >= dimensions) - error_at("Too many grouped array subscripts", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - stride = fixed_array_shape_stride(&shape, depth, row->type->size); - if (stride != 1) { - one = require_var(parent); - one->var_name = gen_name(); - one->init_val = stride; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, updated, index, one, 0, NULL); - index = updated; - } - updated = require_typed_ptr_var(parent, row->type, 1); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_add, updated, address, index, 0, NULL); - address = updated; - depth++; - } - if (depth != dimensions) - error_at("Grouped pointer-to-array postfix update needs all subscripts", - cur_token_loc()); - old = require_typed_var(parent, row->type); - old->var_name = gen_name(); - add_insn(parent, *bb, OP_read, old, address, NULL, row->type->size, NULL); if (lex_accept(T_increment)) - op = OP_add; - else { - lex_expect(T_decrement); - op = OP_sub; - } - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, old, one); - add_insn(parent, *bb, op, updated, old, one, 0, NULL); - updated = resize_to(parent, bb, updated, row->type, 0); - add_insn(parent, *bb, OP_write, NULL, address, updated, row->type->size, - NULL); - opstack_push(old); + return OP_add; + lex_expect(T_decrement); + return OP_sub; } -/* Keep prefix support equally narrow: the generic prefix path has no general - * grouped-lvalue address representation. This scanner is non-mutating so a - * non-match remains entirely owned by that generic path. - */ -static bool grouped_scalar_pointee_row_prefix_starts(block_t *parent) +static var_t *scale_pointer_index(block_t *parent, + basic_block_t *bb, + var_t *index, + int stride, + bool rematerialize_scale) { - token_t *token = cur_token ? cur_token->next : NULL; - int suffixes = 0; - - if (!token || (token->kind != T_increment && token->kind != T_decrement) || - !(token = token->next) || token->kind != T_open_bracket || - !(token = token->next) || token->kind != T_asterisk || - !(token = token->next) || token->kind != T_identifier || - !names_scalar_row_pointer(token, parent) || !(token = token->next) || - token->kind != T_close_bracket || !(token = token->next)) - return false; + long long scaled; + var_t *scale; + var_t *result; - while (suffixes < 4 && token && token->kind == T_open_square) { - int depth = 0; - - for (; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || - token->kind == T_close_curly) { - if (!--depth) { - token = token->next; - break; - } - } - } - if (!token) - return false; - suffixes++; - } - return suffixes; + if (stride == 1) + return index; + scaled = (long long) index->init_val * stride; + if (index->is_const && !index->is_global && index->type && + index->type->size <= TY_int->size && scaled >= INT_MIN && + scaled <= INT_MAX) + return load_constant(parent, bb, (int) scaled, + integer_promoted_type(index)); + scale = load_constant(parent, bb, stride, TY_int); + scale->is_const = rematerialize_scale; + result = name_var(require_typed_var(parent, integer_promoted_type(index))); + add_insn(parent, bb, OP_mul, result, index, scale, 0, NULL); + return result; } -static void handle_grouped_scalar_pointee_row_prefix(block_t *parent, - basic_block_t **bb) +static var_t *lower_array_element_address(block_t *parent, + basic_block_t *bb, + var_t *base, + var_t *index, + type_t *element_type, + int pointer_level, + int stride, + bool rematerialize_scale) { - char name[MAX_VAR_LEN]; - var_t *source, *row, *index, *address, *current, *one, *updated; - fixed_array_shape_t shape; - int dimensions; - int depth = 0; - opcode_t op; - - if (lex_accept(T_increment)) - op = OP_add; - else { - lex_expect(T_decrement); - op = OP_sub; - } - lex_expect(T_open_bracket); - lex_expect(T_asterisk); - lex_ident(T_identifier, name); - source = find_var(name, parent); - if (!source) - error_at("Undeclared identifier", cur_token_loc()); - if (!lower_pointee_array_dereference(source, parent, bb)) - error_at("Grouped pointer-to-array update requires a scalar fixed row", - cur_token_loc()); - row = opstack_pop(); - if (row->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - shape = fixed_array_shape_from_var(row); - dimensions = shape.rank; - if (dimensions >= 3 && - (is_record_type(row->type) || row->type->is_floating)) - error_at( - "Grouped rank-three/four prefix update requires an integer scalar", - cur_token_loc()); - if (dimensions > 4) - error_at( - "Grouped pointer-to-array prefix update supports at most four " - "dimensions", - cur_token_loc()); - lex_expect(T_close_bracket); - address = row; - while (lex_accept(T_open_square)) { - int stride; + var_t *address; - if (depth >= dimensions) - error_at("Too many grouped array subscripts", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - stride = fixed_array_shape_stride(&shape, depth, row->type->size); - if (stride != 1) { - one = require_var(parent); - one->var_name = gen_name(); - one->init_val = stride; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, updated, index, one, 0, NULL); - index = updated; - } - updated = require_typed_ptr_var(parent, row->type, 1); - updated->var_name = gen_name(); - add_insn(parent, *bb, OP_add, updated, address, index, 0, NULL); - address = updated; - depth++; - } - if (depth != dimensions) - error_at("Grouped pointer-to-array prefix update needs all subscripts", - cur_token_loc()); - current = require_typed_var(parent, row->type); - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, row->type->size, - NULL); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, current, one); - add_insn(parent, *bb, op, updated, current, one, 0, NULL); - updated = resize_to(parent, bb, updated, row->type, 0); - add_insn(parent, *bb, OP_write, NULL, address, updated, row->type->size, - NULL); - opstack_push(updated); + index = scale_pointer_index(parent, bb, index, stride, rematerialize_scale); + address = + name_var(require_typed_ptr_var(parent, element_type, pointer_level)); + add_insn(parent, bb, OP_add, address, base, index, 0, NULL); + return address; } /* The type name read after "(" by read_parenthesized_operand(), for the cast or @@ -317,6 +112,92 @@ typedef struct { int func_pointer_level; } paren_type_name_t; +/* The explicit array declarators in local and file-scope compound literals + * share one grammar. The caller maps this shape to expression or storage + * metadata after parsing. + */ +static bool read_compound_array_shape(block_t *scope, + int pointer_level, + bool file_scope, + paren_type_name_t *shape) +{ + if (lex_accept(T_open_bracket)) { + if (pointer_level || !lex_peek(T_asterisk, NULL)) + error_at("Array compound literal needs a pointer declarator", + cur_token_loc()); + do { + lex_expect(T_asterisk); + shape->array_element_ptr_level++; + skip_type_qualifiers(); + } while (lex_peek(T_asterisk, NULL)); + lex_expect(T_open_square); + if (!lex_peek(T_close_square, NULL)) { + shape->array_size = read_const_expr(scope); + if (shape->array_size <= 0) + error_at("Array compound literal needs a positive bound", + cur_token_loc()); + } else { + shape->array_outer_unsized = true; + } + lex_expect(T_close_square); + lex_expect(T_close_bracket); + fixed_array_shape_t pointee_shape = {0}; + + read_array_shape_bounds( + scope, &pointee_shape, + "Array declarators support at most four dimensions", NULL, + "Array size must be positive", false, false); + fixed_array_shape_apply(&pointee_shape, &shape->pointee_array_size, + &shape->pointee_array_dim2, + &shape->pointee_array_dim3, + &shape->pointee_array_dim4); + + if (!file_scope && !pointee_shape.rank) + error_at("Array compound literal needs a row bound", + cur_token_loc()); + shape->array_dims = 1; + shape->parenthesized_array = true; + return true; + } + + while (lex_accept(T_open_square)) { + int bound = 0; + + if (shape->array_dims >= 4) + error_at( + file_scope + ? "Array compound literal supports at most four dimensions" + : "Array declarators support at most four dimensions", + cur_token_loc()); + if (file_scope && lex_peek(T_close_square, NULL)) { + if (shape->array_dims) + error_at("Only the outer array bound may be omitted", + cur_token_loc()); + shape->array_outer_unsized = true; + lex_expect(T_close_square); + shape->array_dims++; + continue; + } + if (file_scope || lex_peek(T_numeric, NULL)) { + bound = read_const_expr(scope); + if (bound <= 0) + error_at("Array compound literal needs a positive bound", + file_scope ? cur_token_loc() : next_token_loc()); + } + if (!bound && shape->array_dims) + error_at("Only the outer array bound may be omitted", + cur_token_loc()); + set_array_dimension(&shape->array_size, &shape->array_dim2, + &shape->array_dim3, &shape->array_dim4, + shape->array_dims, bound, !file_scope); + if (!shape->array_dims && !bound) + shape->array_outer_unsized = true; + shape->array_dims++; + lex_expect(T_close_square); + } + return shape->array_dims != 0; +} + /* A subscript applied to the value on top of the operand stack, which has no * declaration for read_lvalue() to follow. */ @@ -332,6 +213,7 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) int subscript_depth = 0; int array_dims = 1 + !!base->array_dim2 + !!base->array_dim3 + !!base->array_dim4; + fixed_array_shape_t shape = fixed_array_shape_from_var(base); func_t *element_signature = NULL; /* A row of callbacks, `(*rows)[i]` for a pointer to `fn_t[2]`, holds @@ -355,51 +237,17 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) */ address = base; do { - int stride = element_size; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (subscript_depth == 0 && base->array_dim2) { - stride *= base->array_dim2; - if (base->array_dim3) - stride *= base->array_dim3; - if (base->array_dim4) - stride *= base->array_dim4; - } else if (subscript_depth == 1 && base->array_dim3) { - stride *= base->array_dim3; - if (base->array_dim4) - stride *= base->array_dim4; - } else if (subscript_depth == 2 && base->array_dim4) { - stride *= base->array_dim4; - } - if (stride != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, - NULL); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = - require_typed_ptr_var(parent, base->type, base->ptr_level + 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address( + parent, bb, address, base->type, base->ptr_level + 1, + fixed_array_shape_stride(&shape, subscript_depth, + element_size)); subscript_depth++; } while (subscript_depth < array_dims && lex_accept(T_open_square)); if (subscript_depth < array_dims && !element_signature) { - opstack_push(array_subscript_remainder( - parent, bb, base->type, base->ptr_level, - fixed_array_shape_from_var(base), subscript_depth, address)); + opstack_push(array_subscript_remainder(parent, bb, base->type, + base->ptr_level, shape, + subscript_depth, address)); } else if (subscript_depth < array_dims) { opstack_push(address); } else { @@ -415,8 +263,8 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) element_ptr_level += element_type->array_element_ptr_level; element_type = element_type->array_element_type; } - vd = require_typed_ptr_var(parent, element_type, element_ptr_level); - vd->var_name = gen_name(); + vd = name_var( + require_typed_ptr_var(parent, element_type, element_ptr_level)); vd->is_const_qualified = base->is_const_qualified; vd->pointer_const_mask = base->pointer_const_mask; vd->is_const_pointer = base->is_const_pointer; @@ -436,10 +284,7 @@ static void lower_value_subscript(block_t *parent, basic_block_t **bb) /* E1[E2] is (*((E1)+(E2))): pointer arithmetic scales the index by the * element size, and the dereference yields the element object. */ - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); index = opstack_pop(); lex_expect(T_close_square); if (is_pointer_like_value(base) == is_pointer_like_value(index) || @@ -505,8 +350,7 @@ static void lower_value_member(block_t *parent, basic_block_t **bb) * as a call or assignment result, is not. */ is_lvalue = base->is_compound_literal || is_named_object(base, parent); - address = require_ref_var(parent, base->type, 0); - address->var_name = gen_name(); + address = name_var(require_ref_var(parent, base->type, 0)); address->is_const_qualified = base->is_const_qualified; add_insn(parent, *bb, OP_address_of, address, base, NULL, 0, NULL); } @@ -561,18 +405,13 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { var_t *object = opstack_pop(); - opcode_t op = OP_sub; - - if (lex_accept(T_increment)) - op = OP_add; - else - lex_expect(T_decrement); + opcode_t op = read_increment_operator(); if (!object->is_compound_literal_reference && is_named_object(object, parent)) { /* A grouped named object, `(count)++`, updates the variable. */ - var_t *old = - require_typed_ptr_var(parent, object->type, object->ptr_level); - var_t *one = require_typed_var(parent, TY_int); + var_t *old = name_var( + require_typed_ptr_var(parent, object->type, object->ptr_level)); + var_t *one; var_t *updated; if (object->array_size) @@ -584,27 +423,16 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) "Increment or decrement requires a scalar modifiable " "lvalue", cur_token_loc()); - old->var_name = gen_name(); add_insn(parent, *bb, OP_assign, old, object, NULL, 0, NULL); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, object, one)) { - handle_pointer_arithmetic(parent, bb, op, object, one); - updated = opstack_pop(); - } else { - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, object, one); - add_insn(parent, *bb, op, updated, object, one, 0, NULL); - } + one = load_constant(parent, *bb, 1, TY_int); + updated = combine_assignment_value(parent, bb, op, object, one); updated = resize_var(parent, bb, updated, object); mark_var_mutated(object); add_insn(parent, *bb, OP_assign, object, updated, NULL, 0, NULL); opstack_push(old); return; } - lower_reference_update(object, op, parent, bb); + lower_reference_update(object, op, parent, bb, false); /* The expression has the old value and is not itself an lvalue. */ object->is_compound_literal_reference = false; @@ -612,34 +440,53 @@ void lower_postfix_operators(block_t *parent, basic_block_t **bb) } } -/* A parenthesized expression, possibly a comma expression, and the postfix - * operators applied to its value. +/* Parse a full comma expression. Grouped operands clear compound-literal + * reference state on each right-hand value because the comma result is an + * ordinary expression value. */ -static void read_grouped_operand(block_t *parent, basic_block_t **bb) +static void read_comma_expression(block_t *parent, + basic_block_t **bb, + bool clear_reference) { - /* Regular parenthesized expression */ - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); while (lex_accept(T_comma)) { - /* A comma inside an explicit parenthesized expression is the C99 - * sequencing operator, not an argument or initializer delimiter. - * Discard the completed left value after its IR is emitted; the final - * expression supplies the result, which is not an lvalue. - */ discard_operand(parent, *bb); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - operand_stack[operand_stack_idx - 1]->is_compound_literal_reference = - false; + read_assignment_or_expression(parent, bb); + if (clear_reference) + operand_stack[operand_stack_idx - 1] + ->is_compound_literal_reference = false; } +} + +/* A parenthesized expression, possibly a comma expression, and the postfix + * operators applied to its value. + */ +static void read_grouped_operand(block_t *parent, basic_block_t **bb) +{ + read_comma_expression(parent, bb, true); lex_expect(T_close_bracket); lower_postfix_operators(parent, bb); } +static void lower_compound_literal_assignment(block_t *parent, + basic_block_t **bb, + var_t *address, + bool is_const, + const lvalue_t *target) +{ + opcode_t op = OP_generic; + var_t *value; + + if (!accept_assignment_op(&op)) + return; + if (is_const) + error_at("assignment of read-only location", cur_token_loc()); + read_assignment_or_expression(parent, bb); + value = combine_addressed_assignment(parent, bb, op, address, target, + opstack_pop()); + store_addressed_assignment(parent, bb, address, target, value, false); +} + /* Subscripts, and the updates or stores that follow them, applied directly to * an array compound literal. */ @@ -653,44 +500,12 @@ static void lower_array_literal_subscripts(block_t *parent, var_t *element; int element_size = compound_var->type->size; int subscript_depth = 0; + fixed_array_shape_t shape = fixed_array_shape_from_var(compound_var); while (lex_accept(T_open_square)) { - var_t *index; - int stride = element_size; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (subscript_depth == 0 && compound_var->array_dim2) { - stride *= compound_var->array_dim2; - if (compound_var->array_dim3) - stride *= compound_var->array_dim3; - if (compound_var->array_dim4) - stride *= compound_var->array_dim4; - } else if (subscript_depth == 1 && compound_var->array_dim3) { - stride *= compound_var->array_dim3; - if (compound_var->array_dim4) - stride *= compound_var->array_dim4; - } else if (subscript_depth == 2 && compound_var->array_dim4) { - stride *= compound_var->array_dim4; - } - if (stride != 1) { - var_t *scale = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, compound_var->type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address( + parent, bb, address, compound_var->type, 1, + fixed_array_shape_stride(&shape, subscript_depth, element_size)); subscript_depth++; } @@ -699,88 +514,26 @@ static void lower_array_literal_subscripts(block_t *parent, * pointer value, not a scalar read, and keeps the bounds the subscripts * left for sizeof and for its own later subscripts. */ - opstack_push(array_subscript_remainder( - parent, bb, compound_var->type, compound_var->ptr_level, - fixed_array_shape_from_var(compound_var), subscript_depth, - address)); + opstack_push(array_subscript_remainder(parent, bb, compound_var->type, + compound_var->ptr_level, shape, + subscript_depth, address)); } else { - opcode_t compound_op = OP_generic; - if (lex_accept(T_assign) || accept_compound_assign_op(&compound_op)) { - if (compound_var->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - var_t *value; - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - value = opstack_pop(); - if (compound_op != OP_generic) { - var_t *current = require_typed_var(parent, compound_var->type); - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, - element_size, NULL); - /* A pointer object steps by its element size. */ - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, - value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, - 0, NULL); - value = combined; - } - } - value = convert_stored_value(parent, bb, value, compound_var->type, - compound_var->ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, element_size, - NULL); - } + lower_compound_literal_assignment( + parent, bb, address, compound_var->is_const_qualified, + &(lvalue_t) {.size = element_size, + .value_ptr_level = compound_var->ptr_level, + .type = compound_var->type}); + element = name_var(require_typed_var(parent, compound_var->type)); + add_insn(parent, *bb, OP_read, element, address, NULL, element_size, + NULL); + element->is_const_qualified = compound_var->is_const_qualified; + mark_value_reference(element, address, NULL); if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { - opcode_t op = OP_sub; - var_t *one; - var_t *updated; - - if (lex_accept(T_increment)) - op = OP_add; - else - lex_expect(T_decrement); - - if (compound_var->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - element = require_typed_var(parent, compound_var->type); - element->var_name = gen_name(); - add_insn(parent, *bb, OP_read, element, address, NULL, element_size, - NULL); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, element, one); - add_insn(parent, *bb, op, updated, element, one, 0, NULL); - updated = resize_var(parent, bb, updated, element); - add_insn(parent, *bb, OP_write, NULL, address, updated, - element_size, NULL); - opstack_push(element); - } else { - element = require_typed_var(parent, compound_var->type); - element->var_name = gen_name(); - add_insn(parent, *bb, OP_read, element, address, NULL, element_size, - NULL); - element->is_compound_literal_reference = true; - element->is_const_qualified = compound_var->is_const_qualified; - element->compound_literal_address = address; - opstack_push(element); + lower_reference_update(element, read_increment_operator(), parent, + bb, false); + element->is_compound_literal_reference = false; } + opstack_push(element); } } @@ -794,17 +547,15 @@ static void lower_record_literal_members(block_t *parent, { char field_name[MAX_ID_LEN]; var_t *base = opstack_pop(); - var_t *base_addr = require_ref_var(parent, base->type, 0); + var_t *base_addr = name_var(require_ref_var(parent, base->type, 0)); var_t *field; var_t *address; var_t *element; - opcode_t compound_op = OP_generic; type_t *value_type; int value_ptr_level; int value_size; type_t *record_type = tn->type; - base_addr->var_name = gen_name(); add_insn(parent, *bb, OP_address_of, base_addr, base, NULL, 0, NULL); address = base_addr; while (1) { @@ -836,135 +587,44 @@ static void lower_record_literal_members(block_t *parent, * remaining dimensions, just as ordinary array postfix parsing does. */ int subscript_depth = 0; + fixed_array_shape_t shape = fixed_array_shape_from_var(field); while (field->array_size && lex_accept(T_open_square)) { - var_t *index; - int stride = value_size; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (subscript_depth == 0 && field->array_dim2 > 0) { - stride *= field->array_dim2; - if (field->array_dim3 > 0) - stride *= field->array_dim3; - if (field->array_dim4 > 0) - stride *= field->array_dim4; - } else if (subscript_depth == 1 && field->array_dim3 > 0) { - stride *= field->array_dim3; - if (field->array_dim4 > 0) - stride *= field->array_dim4; - } else if (subscript_depth == 2 && field->array_dim4 > 0) { - stride *= field->array_dim4; - } - if (stride != 1) { - var_t *scale = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = stride; - add_insn(parent, *bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - var_t *indexed = require_typed_ptr_var(parent, value_type, 1); - indexed->var_name = gen_name(); - add_insn(parent, *bb, OP_add, indexed, address, index, 0, NULL); - address = indexed; + address = read_subscript_address( + parent, bb, address, value_type, 1, + fixed_array_shape_stride(&shape, subscript_depth, value_size)); value_ptr_level = 0; subscript_depth++; } - if (lex_accept(T_assign) || accept_compound_assign_op(&compound_op)) { - if (compound_object->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - var_t *value; - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - value = opstack_pop(); - if (compound_op != OP_generic) { - var_t *current; - if (is_bitfield(field)) - current = read_bitfield_value(parent, bb, address, field); - else { - current = require_typed_var(parent, value_type); - current->ptr_level = value_ptr_level; - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, - value_size, NULL); - } - /* A pointer object steps by its element size. */ - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, - value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, 0, - NULL); - value = combined; - } - } - if (is_bitfield(field)) - write_bitfield_value(parent, bb, address, value, field); - else { - value = convert_stored_value(parent, bb, value, value_type, - value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, value_size, - NULL); - } - } + lower_compound_literal_assignment( + parent, bb, address, compound_object->is_const_qualified, + &(lvalue_t) {.size = value_size, + .value_ptr_level = value_ptr_level, + .type = value_type, + .decl = field}); if (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) { - opcode_t op = lex_accept(T_increment) ? OP_add : OP_sub; + opcode_t op = read_increment_operator(); var_t *old; - var_t *one; - var_t *updated; - if (compound_object->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); if (is_bitfield(field)) old = read_bitfield_value(parent, bb, address, field); else { - old = require_typed_var(parent, value_type); - old->ptr_level = value_ptr_level; - old->var_name = gen_name(); + old = name_var( + require_typed_ptr_var(parent, value_type, value_ptr_level)); add_insn(parent, *bb, OP_read, old, address, NULL, value_size, NULL); } - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(op, old, one); - add_insn(parent, *bb, op, updated, old, one, 0, NULL); - updated = resize_var(parent, bb, updated, old); - if (is_bitfield(field)) - write_bitfield_value(parent, bb, address, updated, field); - else - add_insn(parent, *bb, OP_write, NULL, address, updated, value_size, - NULL); + old->is_const_qualified = compound_object->is_const_qualified; + mark_value_reference(old, address, is_bitfield(field) ? field : NULL); + lower_reference_update(old, op, parent, bb, false); element = old; } else if (is_bitfield(field)) { element = read_bitfield_value(parent, bb, address, field); element->is_bitfield = false; } else { - element = require_typed_var(parent, value_type); - element->ptr_level = value_ptr_level; - element->var_name = gen_name(); + element = name_var( + require_typed_ptr_var(parent, value_type, value_ptr_level)); add_insn(parent, *bb, OP_read, element, address, NULL, value_size, NULL); } @@ -982,11 +642,9 @@ static void lower_record_literal_members(block_t *parent, if (field->array_dim4) field_dims++; if (!field->array_size || subscript_depth >= field_dims) { - element->is_compound_literal_reference = true; element->is_const_qualified = compound_object->is_const_qualified; - element->compound_literal_address = address; - if (is_bitfield(field)) - element->compound_literal_bitfield = field; + mark_value_reference(element, address, + is_bitfield(field) ? field : NULL); } opstack_push(element); } @@ -1002,8 +660,7 @@ static void read_compound_literal_operand(block_t *parent, var_t *compound_object = NULL; /* Create variable for compound literal result */ - var_t *compound_var = require_typed_var(parent, tn->type); - compound_var->var_name = gen_name(); + var_t *compound_var = name_var(require_typed_var(parent, tn->type)); compound_var->is_compound_literal = true; compound_var->is_const_qualified = tn->const_qualified; compound_var->is_const_pointer = tn->const_pointer; @@ -1022,14 +679,10 @@ static void read_compound_literal_operand(block_t *parent, tn->ptr_level = 0; /* Reset for normal processing */ bool consumed_close_brace = false; - /* parse_array_init() consumes its own opening brace. The older - * one-dimensional literal helper expects it already consumed. - */ - if (!is_array_literal || - (tn->array_dims <= 1 && !tn->array_element_ptr_level)) + /* Array initialization owns its opening brace. */ + if (!is_array_literal) lex_expect(T_open_curly); - if (!is_array_literal || - (tn->array_dims <= 1 && !tn->array_element_ptr_level)) + if (!is_array_literal) reject_empty_initializer_in_strict_c99(); /* Check if this is a pointer compound literal */ if (is_array_literal) { @@ -1044,10 +697,7 @@ static void read_compound_literal_operand(block_t *parent, compound_var->pointee_array_dim3 = tn->pointee_array_dim3; compound_var->pointee_array_dim4 = tn->pointee_array_dim4; add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, NULL); - if (tn->array_dims > 1 || tn->array_element_ptr_level) - parse_array_init(compound_var, parent, bb); - else - parse_array_compound_literal(compound_var, parent, bb); + parse_array_init(compound_var, parent, bb); if (compound_var->array_size == 0) { compound_var->init_val = 0; @@ -1077,8 +727,7 @@ static void read_compound_literal_operand(block_t *parent, compound_var->pointee_func_signature = tn->func_signature; } if (lex_peek(T_close_curly, NULL)) { - initializer = require_var(parent); - initializer->var_name = gen_name(); + initializer = require_named_var(parent); initializer->is_const = true; add_insn(parent, *bb, OP_load_constant, initializer, NULL, NULL, 0, NULL); @@ -1121,9 +770,8 @@ static void read_compound_literal_operand(block_t *parent, false); } else { /* Empty pointer compound literal: (int*){} */ - initializer = require_typed_var(parent, tn->type); - initializer->ptr_level = tn->ptr_level; - initializer->var_name = gen_name(); + initializer = name_var( + require_typed_ptr_var(parent, tn->type, tn->ptr_level)); initializer->init_val = 0; add_insn(parent, *bb, OP_load_constant, initializer, NULL, NULL, 0, NULL); @@ -1147,8 +795,8 @@ static void read_compound_literal_operand(block_t *parent, struct_type = struct_type->base_struct; add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, NULL); - var_t *compound_addr = require_ref_var(parent, compound_var->type, 0); - compound_addr->var_name = gen_name(); + var_t *compound_addr = + name_var(require_ref_var(parent, compound_var->type, 0)); add_insn(parent, *bb, OP_address_of, compound_addr, compound_var, NULL, 0, NULL); parse_struct_field_init(parent, bb, struct_type, compound_addr); @@ -1166,10 +814,7 @@ static void read_compound_literal_operand(block_t *parent, compound_var->array_size = 0; add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, NULL); - var_t *zero = require_typed_var(parent, tn->type); - zero->var_name = gen_name(); - zero->init_val = 0; - add_insn(parent, *bb, OP_load_constant, zero, NULL, NULL, 0, NULL); + var_t *zero = load_constant(parent, *bb, 0, tn->type); emit_object_assignment(parent, bb, compound_var, zero); opstack_push(compound_var); compound_object = compound_var; @@ -1185,94 +830,15 @@ static void read_compound_literal_operand(block_t *parent, error_at("Too many elements in scalar compound literal", cur_token_loc()); - /* Retained for the parser's array-literal extension; scalar - * spellings above have already rejected a second initializer. - */ - if (lex_peek(T_comma, NULL)) { - /* Array compound literal: (int[]){1, 2, 3} */ - var_t *first_element = opstack_pop(); - - /* Store elements temporarily */ - var_t *elements[256]; - elements[0] = first_element; - int element_count = 1; - - /* Parse remaining elements */ - while (lex_accept(T_comma)) { - if (lex_peek(T_close_curly, NULL)) - break; /* Trailing comma */ - - read_expr(parent, bb); - read_ternary_operation(parent, bb); - if (element_count < 256) { - elements[element_count] = opstack_pop(); - } else { - opstack_pop(); /* Discard if too many */ - } - element_count++; - } - - /* Set array metadata */ - compound_var->array_size = element_count; - compound_var->init_val = first_element->init_val; - - /* Allocate space for the array on stack */ - add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, - NULL); - - /* Initialize each element */ - for (int i = 0; i < element_count && i < 256; i++) { - if (!elements[i]) - continue; - - /* Store element at offset i * sizeof(element) */ - var_t *elem_offset = require_var(parent); - elem_offset->init_val = i * tn->type->size; - elem_offset->var_name = gen_name(); - add_insn(parent, *bb, OP_load_constant, elem_offset, NULL, - NULL, 0, NULL); - - /* Calculate address of element */ - var_t *elem_addr = require_var(parent); - elem_addr->ptr_level = 1; - elem_addr->var_name = gen_name(); - add_insn(parent, *bb, OP_add, elem_addr, compound_var, - elem_offset, 0, NULL); - - /* Store the element value */ - add_insn(parent, *bb, OP_write, NULL, elem_addr, - elements[i], tn->type->size, NULL); - } - - /* Store first element value for array-to-scalar */ - compound_var->init_val = first_element->init_val; - - /* Create result that provides first element access. This - * enables array compound literals in scalar contexts: int x = - * (int[]){1,2,3}; // x gets 1 int y = 5 + (int[]){10}; // adds - * 5 + 10 - */ - var_t *result_var = require_var(parent); - result_var->var_name = gen_name(); - result_var->type = compound_var->type; - result_var->ptr_level = 0; - result_var->array_size = 0; - - /* Read first element from the array */ - add_insn(parent, *bb, OP_read, result_var, compound_var, NULL, - compound_var->type->size, NULL); - opstack_push(result_var); - } else { - /* Single value: (int){42} - scalar compound literal */ - var_t *initializer = opstack_pop(); - diagnose_integer_to_pointer_conversion(initializer, - compound_var, false); - add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, - NULL); - emit_object_assignment(parent, bb, compound_var, initializer); - compound_object = compound_var; - opstack_push(compound_var); - } + /* Single value: (int){42} - scalar compound literal */ + var_t *initializer = opstack_pop(); + diagnose_integer_to_pointer_conversion(initializer, compound_var, + false); + add_insn(parent, *bb, OP_allocat, compound_var, NULL, NULL, 0, + NULL); + emit_object_assignment(parent, bb, compound_var, initializer); + compound_object = compound_var; + opstack_push(compound_var); } } @@ -1308,7 +874,7 @@ static void read_compound_literal_operand(block_t *parent, */ if (compound_object && !is_record_type(tn->type) && (lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL))) { - opcode_t op = lex_accept(T_increment) ? OP_add : OP_sub; + opcode_t op = read_increment_operator(); var_t *old; var_t *one; var_t *updated; @@ -1317,62 +883,28 @@ static void read_compound_literal_operand(block_t *parent, compound_object->is_const_pointer) error_at("assignment of read-only location", cur_token_loc()); opstack_pop(); - old = require_typed_var(parent, compound_object->type); - old->ptr_level = compound_object->ptr_level; - old->var_name = gen_name(); + old = name_var(require_typed_ptr_var(parent, compound_object->type, + compound_object->ptr_level)); add_insn(parent, *bb, OP_assign, old, compound_object, NULL, 0, NULL); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, compound_object, one)) { - handle_pointer_arithmetic(parent, bb, op, compound_object, one); - updated = opstack_pop(); - } else { - updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = - integer_binary_result_type(op, compound_object, one); - add_insn(parent, *bb, op, updated, compound_object, one, 0, NULL); - } + one = load_constant(parent, *bb, 1, TY_int); + updated = + combine_assignment_value(parent, bb, op, compound_object, one); updated = resize_var(parent, bb, updated, compound_object); add_insn(parent, *bb, OP_assign, compound_object, updated, NULL, 0, NULL); opstack_push(old); } opcode_t scalar_compound_op = OP_generic; - if (compound_object && (lex_accept(T_assign) || - accept_compound_assign_op(&scalar_compound_op))) { + if (compound_object && accept_assignment_op(&scalar_compound_op)) { if (compound_object->is_const_qualified || compound_object->is_const_pointer) error_at("assignment of read-only location", cur_token_loc()); opstack_pop(); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); var_t *value = opstack_pop(); - if (scalar_compound_op != OP_generic) { - var_t *current = compound_object; - - if (is_pointer_operation(scalar_compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, scalar_compound_op, - current, value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands( - parent, bb, scalar_compound_op, ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = integer_binary_result_type(scalar_compound_op, - current, value); - add_insn(parent, *bb, scalar_compound_op, combined, current, - value, 0, NULL); - value = combined; - } - } + if (scalar_compound_op != OP_generic) + value = combine_assignment_value(parent, bb, scalar_compound_op, + compound_object, value); emit_object_assignment(parent, bb, compound_object, value); opstack_push(compound_object); } @@ -1385,6 +917,19 @@ static void read_compound_literal_operand(block_t *parent, * source payload: `(unsigned char)256`, `(short)65536`, and a 64-to-32 cast all * become the integer constant zero and may therefore serve as null pointers. */ +static int sign_extend_narrow_int(unsigned int value, int size) +{ + unsigned int bits, mask, sign; + + if (size != 1 && size != 2) + return (int) value; + bits = size * 8; + mask = (1U << bits) - 1; + sign = 1U << (bits - 1); + value &= mask; + return value & sign ? (int) value - (int) (sign << 1) : (int) value; +} + void fold_integer_constant_cast(var_t *result, unsigned int lo, unsigned int hi) { if (result->type->is_bool) { @@ -1398,9 +943,9 @@ void fold_integer_constant_cast(var_t *result, unsigned int lo, unsigned int hi) unsigned int bits = result->type->size * 8; unsigned int mask = (1U << bits) - 1; - lo &= mask; - if (!result->type->is_unsigned && (lo & (1U << (bits - 1)))) - lo |= ~mask; + lo = result->type->is_unsigned ? lo & mask + : (unsigned int) sign_extend_narrow_int( + lo, result->type->size); result->init_val = (int) lo; result->init_val_hi = result->init_val < 0 ? -1 : 0; } else if (result->type->size == TY_int->size) { @@ -1443,8 +988,8 @@ static void read_cast_operand(block_t *parent, expr_var = materialize_function_designator(parent, bb, expr_var); /* Create variable for cast result */ - var_t *cast_var = require_typed_ptr_var(parent, tn->type, tn->ptr_level); - cast_var->var_name = gen_name(); + var_t *cast_var = + name_var(require_typed_ptr_var(parent, tn->type, tn->ptr_level)); cast_var->is_const_qualified = tn->const_qualified; cast_var->is_const_pointer = tn->const_pointer; cast_var->pointer_const_mask = tn->pointer_const_mask; @@ -1627,6 +1172,9 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) /* Check for pointer types: int*, char*, etc. */ int ptr_level = 0; + var_t return_decl; + init_type_name_decl(&return_decl, type, has_const_type, + tn.volatile_qualified); while (lex_accept(T_asterisk)) { ptr_level++; while (lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || @@ -1635,9 +1183,14 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) tn.const_pointer = true; if (ptr_level <= 32) tn.pointer_const_mask |= 1U << (ptr_level - 1); - } else if (lex_accept(T_volatile)) + if (ptr_level <= 32) + return_decl.pointer_const_mask |= 1U << (ptr_level - 1); + } else if (lex_accept(T_volatile)) { + int depth = ptr_level + type->ptr_level; + if (depth <= 32) + return_decl.pointer_volatile_mask |= 1U << (depth - 1); tn.volatile_qualified = true; - else + } else lex_expect(T_restrict); } } @@ -1645,115 +1198,13 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) bool is_array = false; if (abstract_function_pointer_follows()) { + return_decl.ptr_level = ptr_level; tn.func_signature = read_abstract_function_pointer( - type, ptr_level, &tn.func_pointer_level); + &return_decl, &tn.func_pointer_level); ptr_level = 0; } - /* A parenthesized abstract declarator, such as `(int (*[])[2]){...}`, - * is an array whose elements are pointers to rows. Keep the outer array - * and pointee bounds separate, matching named pointer-to-array - * declarations. - */ - if (lex_accept(T_open_bracket)) { - int pointee_dims = 0; - - if (ptr_level || !lex_peek(T_asterisk, NULL)) - error_at("Array compound literal needs a pointer declarator", - cur_token_loc()); - do { - lex_expect(T_asterisk); - tn.array_element_ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } while (lex_peek(T_asterisk, NULL)); - lex_expect(T_open_square); - if (!lex_peek(T_close_square, NULL)) { - tn.array_size = read_const_expr(parent); - if (tn.array_size <= 0) - error_at("Array compound literal needs a positive bound", - cur_token_loc()); - } else { - tn.array_outer_unsized = true; - } - lex_expect(T_close_square); - lex_expect(T_close_bracket); - while (lex_accept(T_open_square)) { - int bound = read_const_expr(parent); - - if (pointee_dims >= 4) - error_at( - "Array declarators support at most four " - "dimensions", - cur_token_loc()); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (pointee_dims == 0) - tn.pointee_array_size = bound; - else { - if (pointee_dims == 1) - tn.pointee_array_dim2 = bound; - else if (pointee_dims == 2) - tn.pointee_array_dim3 = bound; - else - tn.pointee_array_dim4 = bound; - tn.pointee_array_size *= bound; - } - lex_expect(T_close_square); - pointee_dims++; - } - if (!pointee_dims) - error_at("Array compound literal needs a row bound", - cur_token_loc()); - is_array = true; - tn.array_dims = 1; - tn.parenthesized_array = true; - } - - /* Parse every array bound in a compound-literal type name. `var_t` - * records the complete flattened element count plus up to three - * trailing dimensions, the same representation ordinary declarators use - * for `int a[2][3]`. - */ - while (!tn.parenthesized_array && lex_accept(T_open_square)) { - int bound = 0; - - is_array = true; - if (tn.array_dims >= 4) - error_at("Array declarators support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_numeric, NULL)) { - /* The constant expression reader keeps a literal too wide for - * an int from wrapping into a small bound. - */ - bound = read_const_expr(parent); - if (bound <= 0) - error_at("Array compound literal needs a positive bound", - next_token_loc()); - } - if (!bound && tn.array_dims) - error_at("Only the outer array bound may be omitted", - cur_token_loc()); - if (!tn.array_dims) - tn.array_size = bound; - else { - if (tn.array_size) - tn.array_size *= bound; - else - tn.array_size = bound; - if (tn.array_dims == 1) - tn.array_dim2 = bound; - else if (tn.array_dims == 2) - tn.array_dim3 = bound; - else - tn.array_dim4 = bound; - } - if (!tn.array_dims && !bound) - tn.array_outer_unsized = true; - tn.array_dims++; - lex_expect(T_close_square); - } + is_array = read_compound_array_shape(parent, ptr_level, false, &tn); if (!is_array && type->array_size) { is_array = true; tn.array_size = type->array_size; @@ -1805,35 +1256,42 @@ static void read_parenthesized_operand(block_t *parent, basic_block_t **bb) } } -/* Replace the operand on top of the stack with its negation, the unary minus of - * C99 6.5.3.3, folding a constant. +/* Lower an integer unary operation, folding constants with the operand's type. */ -static void negate_operand(block_t *parent, basic_block_t **bb) +static void read_integer_unary_operand(block_t *parent, + basic_block_t **bb, + opcode_t op) { var_t *rs1 = opstack_pop(); var_t *vd; reject_record_operand(rs1); if (is_pointer_like_value(rs1) || rs1->is_func) - error_at("unary minus requires an arithmetic operand", cur_token_loc()); + error_at(op == OP_negate + ? "unary minus requires an arithmetic operand" + : "bitwise complement requires an integer operand", + cur_token_loc()); rs1 = integer_promote_operand(parent, bb, rs1); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = rs1->type; - opstack_push(vd); if (rs1->is_const && !rs1->ptr_level && !rs1->is_global) { vd->is_const = true; - vd->init_val = -rs1->init_val; - vd->init_val_hi = ~rs1->init_val_hi + (vd->init_val == 0); + vd->init_val = op == OP_negate ? -rs1->init_val : ~rs1->init_val; + vd->init_val_hi = op == OP_negate + ? ~rs1->init_val_hi + (vd->init_val == 0) + : ~rs1->init_val_hi; - /* The high word of an int result follows its type, as for a cast: "-9U" - * is 0xfffffff7 with a zero high word. + /* Preserve the promoted type: -9U has a zero high word, and + * ~(4294967294U) is 1 rather than a sign-extended 64-bit value. */ fold_integer_constant_cast(vd, vd->init_val, vd->init_val_hi); + opstack_push(vd); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - } else - add_insn(parent, *bb, OP_negate, vd, rs1, NULL, 0, NULL); + } else { + opstack_push(vd); + add_insn(parent, *bb, op, vd, rs1, NULL, 0, NULL); + } } static void read_expr_operand_body(block_t *parent, basic_block_t **bb) @@ -1841,17 +1299,6 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) var_t *vd, *rs1; bool is_neg = false; - if (grouped_scalar_pointee_row_postfix_starts(parent)) { - handle_grouped_scalar_pointee_row_postfix(parent, bb); - return; - } - - if ((lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL)) && - grouped_scalar_pointee_row_prefix_starts(parent)) { - handle_grouped_scalar_pointee_row_prefix(parent, bb); - return; - } - bool prefix_increment = lex_peek(T_increment, NULL); bool prefix_decrement = lex_peek(T_decrement, NULL); if ((prefix_increment || prefix_decrement) && cur_token->next->next && @@ -1873,7 +1320,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) read_expr_operand(parent, bb); object = opstack_pop(); if (object->is_compound_literal_reference) { - opstack_push(lower_reference_update(object, op, parent, bb)); + opstack_push(lower_reference_update(object, op, parent, bb, true)); return; } if (object->array_size && is_named_object(object, parent)) @@ -1887,19 +1334,8 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) cur_token_loc()); if (object->is_const_qualified) error_at("assignment of read-only location", cur_token_loc()); - one = require_typed_var(parent, TY_int); - one->var_name = gen_name(); - one->init_val = 1; - add_insn(parent, *bb, OP_load_constant, one, NULL, NULL, 0, NULL); - if (is_pointer_operation(op, object, one)) { - handle_pointer_arithmetic(parent, bb, op, object, one); - vd = opstack_pop(); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(op, object, one); - add_insn(parent, *bb, op, vd, object, one, 0, NULL); - } + one = load_constant(parent, *bb, 1, TY_int); + vd = combine_assignment_value(parent, bb, op, object, one); vd = resize_var(parent, bb, vd, object); mark_var_mutated(object); add_insn(parent, *bb, OP_assign, object, vd, NULL, 0, NULL); @@ -1933,7 +1369,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) (cur_token->next->next && cur_token->next->next->kind == T_open_square)) { read_expr_operand(parent, bb); - negate_operand(parent, bb); + read_integer_unary_operand(parent, bb, OP_negate); return; } is_neg = true; @@ -1978,16 +1414,14 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) /* Constant folding for logical NOT */ if (rs1 && rs1->is_const && !rs1->ptr_level && !rs1->is_global) { - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = TY_int; vd->is_const = true; vd->init_val = !(rs1->init_val || rs1->init_val_hi); opstack_push(vd); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); } else { - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); /* C99 6.5.3.3: logical negation always yields int. Preserving the * operand's type made !pointer inherit the pointed-to record size, @@ -2001,35 +1435,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) } } else if (lex_accept(T_bit_not)) { read_expr_operand(parent, bb); - - rs1 = opstack_pop(); - reject_record_operand(rs1); - if (is_pointer_like_value(rs1) || rs1->is_func) - error_at("bitwise complement requires an integer operand", - cur_token_loc()); - rs1 = integer_promote_operand(parent, bb, rs1); - - /* Constant folding for bitwise NOT */ - if (rs1 && rs1->is_const && !rs1->ptr_level && !rs1->is_global) { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = rs1->type; - vd->is_const = true; - vd->init_val = ~rs1->init_val; - vd->init_val_hi = ~rs1->init_val_hi; - - /* "(unsigned long long) ~4294967294U" is 1, not 0xffffffff00000001. - */ - fold_integer_constant_cast(vd, vd->init_val, vd->init_val_hi); - opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); - } else { - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = rs1->type; - opstack_push(vd); - add_insn(parent, *bb, OP_bit_not, vd, rs1, NULL, 0, NULL); - } + read_integer_unary_operand(parent, bb, OP_bit_not); } else if (lex_accept(T_ampersand)) { handle_address_of_operator(parent, bb); } else if (lex_accept(T_asterisk)) { @@ -2069,6 +1475,8 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) if (!strcmp(token, "__builtin_offsetof")) { read_builtin_offsetof(parent, bb); + } else if (!strcmp(token, "__builtin_va_start")) { + read_builtin_va_start(parent, bb); } else if (!strcmp(token, "__builtin_va_arg")) { read_builtin_va_arg(parent, bb); } else if (!strcmp(token, "__func__")) { @@ -2076,27 +1484,16 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) error_at("__func__ is only defined inside a function", next_token_loc()); lex_expect(T_identifier); - vd = require_typed_ptr_var(parent, TY_char, true); - vd->var_name = gen_name(); - vd->init_val = write_symbol(parent->func->return_def.var_name); + vd = name_var(require_typed_ptr_var(parent, TY_char, true)); + vd->init_val = write_symbol(function_source_name(parent->func)); vd->is_string_literal = true; opstack_push(vd); add_insn(parent, *bb, OP_load_rodata_address, vd, NULL, NULL, 0, NULL); if (lex_accept(T_open_square)) { - var_t *base = opstack_pop(); - var_t *index; - var_t *address; - - read_expr(parent, bb); - read_ternary_operation(parent, bb); - index = opstack_pop(); - lex_expect(T_close_square); - address = require_typed_ptr_var(parent, TY_char, true); - address->var_name = gen_name(); - add_insn(parent, *bb, OP_add, address, base, index, 0, NULL); - vd = require_typed_var(parent, TY_char); - vd->var_name = gen_name(); + var_t *address = read_subscript_address( + parent, bb, opstack_pop(), TY_char, true, 1); + vd = name_var(require_typed_var(parent, TY_char)); opstack_push(vd); add_insn(parent, *bb, OP_read, vd, address, NULL, 1, NULL); } @@ -2104,10 +1501,9 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) /* An enumeration constant is an integer constant, so one that is * zero is a null pointer constant (C99 6.3.2.3p3). */ - vd = require_var(parent); + vd = name_var(require_var(parent)); vd->init_val = con->value; vd->is_const = true; - vd->var_name = gen_name(); opstack_push(vd); lex_expect(T_identifier); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); @@ -2193,9 +1589,7 @@ static void read_expr_operand_body(block_t *parent, basic_block_t **bb) lower_call_result_prefix_update(&vd, prefix_op, parent, bb); } else { /* indirective function pointer assignment */ - vd = require_func_symbol_var(parent); - vd->is_func = true; - vd->var_name = intern_string(token); + vd = function_designator_value(parent, func); opstack_push(vd); } } else if (lex_accept(T_open_curly)) { @@ -2233,6 +1627,46 @@ bool is_logical(opcode_t op) return op == OP_log_and || op == OP_log_or; } +static opcode_t compound_assignment_opcode(token_kind_t kind) +{ + switch (kind) { + case T_pluseq: + return OP_add; + case T_minuseq: + return OP_sub; + case T_asteriskeq: + return OP_mul; + case T_divideeq: + return OP_div; + case T_modeq: + return OP_mod; + case T_lshifteq: + return OP_lshift; + case T_rshifteq: + return OP_rshift; + case T_xoreq: + return OP_bit_xor; + case T_oreq: + return OP_bit_or; + case T_andeq: + return OP_bit_and; + default: + return OP_generic; + } +} + +static bool assignment_operator(token_kind_t kind) +{ + return kind == T_assign || compound_assignment_opcode(kind) != OP_generic; +} + +static bool assignment_operator_follows(void) +{ + token_t *token = cur_token->next; + + return token && assignment_operator(token->kind); +} + /* Consume a compound-assignment operator ("+=", "-=", ...) and report the * arithmetic it applies. * @@ -2241,40 +1675,30 @@ bool is_logical(opcode_t op) */ bool accept_compound_assign_op(opcode_t *op) { - if (lex_accept(T_pluseq)) - op[0] = OP_add; - else if (lex_accept(T_minuseq)) - op[0] = OP_sub; - else if (lex_accept(T_asteriskeq)) - op[0] = OP_mul; - else if (lex_accept(T_divideeq)) - op[0] = OP_div; - else if (lex_accept(T_modeq)) - op[0] = OP_mod; - else if (lex_accept(T_lshifteq)) - op[0] = OP_lshift; - else if (lex_accept(T_rshifteq)) - op[0] = OP_rshift; - else if (lex_accept(T_xoreq)) - op[0] = OP_bit_xor; - else if (lex_accept(T_oreq)) - op[0] = OP_bit_or; - else if (lex_accept(T_andeq)) - op[0] = OP_bit_and; - else + token_t *token = cur_token->next; + opcode_t opcode; + + if (!token || + (opcode = compound_assignment_opcode(token->kind)) == OP_generic || + !lex_accept(token->kind)) return false; + op[0] = opcode; return true; } +static bool accept_assignment_op(opcode_t *op) +{ + if (lex_accept(T_assign)) { + op[0] = OP_generic; + return true; + } + return accept_compound_assign_op(op); +} + bool lvalue_write_follows(opcode_t prefix_op) { - return prefix_op != OP_generic || lex_peek(T_assign, NULL) || - lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL) || - lex_peek(T_pluseq, NULL) || lex_peek(T_minuseq, NULL) || - lex_peek(T_asteriskeq, NULL) || lex_peek(T_divideeq, NULL) || - lex_peek(T_modeq, NULL) || lex_peek(T_lshifteq, NULL) || - lex_peek(T_rshifteq, NULL) || lex_peek(T_xoreq, NULL) || - lex_peek(T_oreq, NULL) || lex_peek(T_andeq, NULL); + return prefix_op != OP_generic || assignment_operator_follows() || + lex_peek(T_increment, NULL) || lex_peek(T_decrement, NULL); } int get_pointer_element_size(var_t *ptr_var) @@ -2338,25 +1762,22 @@ int get_pointer_element_size(var_t *ptr_var) * void pointer (void **) is different: its elements are pointer objects and * therefore have a known size. */ +bool is_direct_void_pointer_type(const type_t *type, int ptr_level) +{ + return type && type->base_type == TYPE_void && + (ptr_level == 1 || (ptr_level == 0 && type->ptr_level == 1)); +} + bool is_direct_void_pointer(const var_t *var) { - if (!var || !var->type || var->type->base_type != TYPE_void) + if (!var || var->array_size || var->has_unsized_array) return false; /* An array of void pointers decays to void ** before arithmetic. Its * elements are complete pointer objects, even though the declared base type * and direct declarator depth otherwise resemble void *. */ - if (var->array_size || var->has_unsized_array) - return false; - return var->ptr_level == 1 || - (var->ptr_level == 0 && var->type->ptr_level == 1); -} - -bool is_direct_void_pointer_type(const type_t *type, int ptr_level) -{ - return type && type->base_type == TYPE_void && - (ptr_level == 1 || (ptr_level == 0 && type->ptr_level == 1)); + return is_direct_void_pointer_type(var->type, var->ptr_level); } /* Describe in @decayed the pointer to its first row that the multidimensional @@ -2393,13 +1814,22 @@ static var_t *decay_function_pointer_array(block_t *parent, if (!var || !var->is_func || !var->func_signature || var->array_dim2 || !(var->array_size || var->has_unsized_array)) return var; - pointer = require_typed_ptr_var(parent, var->type, 1); - pointer->var_name = gen_name(); + pointer = name_var(require_typed_ptr_var(parent, var->type, 1)); pointer->pointee_func_signature = var->func_signature; add_insn(parent, *bb, OP_assign, pointer, var, NULL, 0, NULL); return pointer; } +static int pointer_arithmetic_stride(var_t *pointer) +{ + if (is_pointee_array_pointer(pointer) && !is_array_declarator(pointer)) + return pointee_array_row_stride(pointer); + if (is_array_declarator(pointer) && pointer->array_dim2 && + !has_effective_pointer(pointer) && !pointer->is_func) + return fixed_array_decay_stride(pointer); + return get_pointer_element_size(pointer); +} + /* Helper function to handle pointer arithmetic (add/sub with scaling) */ void handle_pointer_arithmetic(block_t *parent, basic_block_t **bb, @@ -2490,39 +1920,29 @@ void handle_pointer_arithmetic(block_t *parent, bool right_row = is_pointee_array_pointer(orig_rs2) && !is_array_declarator(orig_rs2); - if ((left_row || right_row) && - !pointee_array_shapes_compatible(orig_rs1, orig_rs2)) - error_at( - "Pointer subtraction requires compatible pointed-to types", - cur_token_loc()); - if (!compatible_decl_type(left_pointee, right_pointee) || + if (((left_row || right_row) && + !pointee_array_shapes_compatible(orig_rs1, orig_rs2)) || + !compatible_decl_type(left_pointee, right_pointee) || left_depth != right_depth) error_at( "Pointer subtraction requires compatible pointed-to types", cur_token_loc()); - element_size = PTR_SIZE; /* Default */ - - element_size = left_row ? pointee_array_row_stride(orig_rs1) - : get_pointer_element_size(orig_rs1); + element_size = pointer_arithmetic_stride(orig_rs1); /* Perform subtraction first */ - var_t *diff = require_var(parent); - diff->var_name = gen_name(); + var_t *diff = require_named_var(parent); add_insn(parent, *bb, OP_sub, diff, rs1, rs2, 0, NULL); /* Then divide by element size if needed */ if (element_size > 1) { - var_t *size_const = require_var(parent); - size_const->var_name = gen_name(); - size_const->init_val = element_size; - add_insn(parent, *bb, OP_load_constant, size_const, NULL, NULL, - 0, NULL); - - var_t *result = require_var(parent); - result->var_name = gen_name(); - add_insn(parent, *bb, OP_div, result, diff, size_const, 0, - NULL); + int shift = exact_log2(element_size); + var_t *size_const = load_constant( + parent, *bb, shift >= 0 ? shift : element_size, TY_int); + + var_t *result = require_named_var(parent); + add_insn(parent, *bb, shift >= 0 ? OP_rshift : OP_div, result, + diff, size_const, 0, NULL); opstack_push(result); } else { opstack_push(diff); @@ -2534,53 +1954,38 @@ void handle_pointer_arithmetic(block_t *parent, if (is_pointer_like_value(rs1)) { ptr_var = rs1; int_var = rs2; - element_size = - is_pointee_array_pointer(rs1) && !is_array_declarator(rs1) - ? pointee_array_row_stride(rs1) - : (is_array_declarator(rs1) && rs1->array_dim2 && - !has_effective_pointer(rs1) && !rs1->is_func - ? fixed_array_decay_stride(rs1) - : get_pointer_element_size(rs1)); + element_size = pointer_arithmetic_stride(rs1); } else if (is_pointer_like_value(rs2)) { /* Only for addition (p + n == n + p) */ if (op == OP_add) { ptr_var = rs2; int_var = rs1; - element_size = - is_pointee_array_pointer(rs2) && !is_array_declarator(rs2) - ? pointee_array_row_stride(rs2) - : (is_array_declarator(rs2) && rs2->array_dim2 && - !has_effective_pointer(rs2) && !rs2->is_func - ? fixed_array_decay_stride(rs2) - : get_pointer_element_size(rs2)); + element_size = pointer_arithmetic_stride(rs2); /* Swap operands so pointer is rs1 */ rs1 = ptr_var; rs2 = int_var; } } + if (ptr_var && int_var->is_const && !int_var->is_global && + !int_var->init_val && !int_var->init_val_hi && !ptr_var->array_size && + !ptr_var->has_unsized_array) { + opstack_push(ptr_var); + return; + } + /* If we need to scale the integer operand */ if (ptr_var && element_size > 1) { - /* Create multiplication by element size */ - var_t *size_const = require_var(parent); - size_const->var_name = gen_name(); - size_const->init_val = element_size; - add_insn(parent, *bb, OP_load_constant, size_const, NULL, NULL, 0, - NULL); - - var_t *scaled = require_var(parent); - scaled->var_name = gen_name(); - add_insn(parent, *bb, OP_mul, scaled, int_var, size_const, 0, NULL); - - /* Use scaled value as rs2 */ - rs2 = scaled; + rs2 = scale_pointer_index(parent, *bb, int_var, element_size, true); } /* Perform the operation */ - var_t *vd = require_var(parent); + var_t *vd = name_var(require_var(parent)); /* Preserve pointer type metadata on results of pointer arithmetic */ if (ptr_var) { vd->type = ptr_var->type; + vd->is_volatile_access = + ptr_var->is_volatile_access || var_volatile_pointee(ptr_var); /* An array operand decays to a pointer before arithmetic. Retain that * extra level on the result so `slots + 0` for an array of pointer @@ -2610,7 +2015,6 @@ void handle_pointer_arithmetic(block_t *parent, fixed_array_shape_to_pointee_var(vd, &shape); } } - vd->var_name = gen_name(); opstack_push(vd); add_insn(parent, *bb, op, vd, rs1, rs2, 0, NULL); } @@ -2753,6 +2157,36 @@ void normalize_integer_binary_operands(block_t *parent, right[0] = resize_to(parent, bb, right[0], common, 0); } +static var_t *emit_binary_result(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *left, + var_t *right, + type_t *result_type) +{ + var_t *result = require_named_var(parent); + result->type = result_type; + add_insn(parent, *bb, op, result, left, right, 0, NULL); + return result; +} + +static var_t *combine_assignment_value(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *current, + var_t *value) +{ + if (is_pointer_operation(op, current, value)) { + handle_pointer_arithmetic(parent, bb, op, current, value); + return opstack_pop(); + } + current = integer_promote_operand(parent, bb, current); + value = integer_promote_operand(parent, bb, value); + normalize_integer_binary_operands(parent, bb, op, ¤t, &value); + return emit_binary_result(parent, bb, op, current, value, + integer_binary_result_type(op, current, value)); +} + bool emit_wide_global_word_arithmetic(block_t *parent, basic_block_t *bb, var_t *result, @@ -2764,8 +2198,9 @@ bool emit_wide_global_word_arithmetic(block_t *parent, * alone. Fold those in two-word form before the usual arithmetic conversions * emit IR that hides the constants, so an unsigned expression such as sizeof(x) * - sizeof(x) stays an integer constant expression and, when zero, a null - * pointer constant. Division by zero and a shift outside the operand width are - * left to the target, as the int folder leaves them. + * pointer constant. Division by zero, signed minimum/-1 overflow, and shifts + * outside the operand width are left to the target, as the int folder leaves + * them. */ static bool fold_wide_constant_binary(block_t *parent, basic_block_t **bb, @@ -2784,43 +2219,45 @@ static bool fold_wide_constant_binary(block_t *parent, rs1->type->size <= TY_int->size && rs2->type->size <= TY_int->size) return false; - switch (op) { - case OP_div: - case OP_mod: - if (!rs2->init_val && - (rs2->type->size <= TY_int->size || !rs2->init_val_hi)) - return false; - break; - case OP_lshift: - case OP_rshift: - if ((rs2->type->size > TY_int->size && rs2->init_val_hi) || - rs2->init_val < 0 || rs2->init_val >= rs1->type->size * 8) - return false; - break; - case OP_add: - case OP_sub: - case OP_mul: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - break; - default: + if (!op_is_binary_alu(op)) + return false; + if ((op == OP_div || op == OP_mod) && !rs2->init_val && + (rs2->type->size <= TY_int->size || !rs2->init_val_hi)) + return false; + if ((op == OP_lshift || op == OP_rshift) && + ((rs2->type->size > TY_int->size && rs2->init_val_hi) || + rs2->init_val < 0 || rs2->init_val >= rs1->type->size * 8)) + return false; + if ((op == OP_div || op == OP_mod) && !unsigned_int_operand(rs1) && + !unsigned_int_operand(rs2) && rs1->type->size > TY_int->size && + rs1->init_val == 0 && rs1->init_val_hi == INT_MIN && + rs2->init_val == -1 && + (rs2->type->size <= TY_int->size || rs2->init_val_hi == -1)) return false; - } - vd = require_typed_var(parent, integer_binary_result_type(op, rs1, rs2)); - vd->var_name = gen_name(); + vd = name_var( + require_typed_var(parent, integer_binary_result_type(op, rs1, rs2))); emit_wide_global_word_arithmetic(parent, *bb, vd, op, rs1, rs2); opstack_push(vd); return true; } +static void pop_binary_operands(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t **left, + var_t **right) +{ + *right = opstack_pop(); + *left = opstack_pop(); + reject_record_operand(*left); + reject_record_operand(*right); + if (op == OP_eq || op == OP_neq) { + *left = materialize_function_designator(parent, bb, *left); + *right = materialize_function_designator(parent, bb, *right); + } +} + void read_expr_body(block_t *parent, basic_block_t **bb) { var_t *vd, *rs1, *rs2; @@ -2865,40 +2302,15 @@ void read_expr_body(block_t *parent, basic_block_t **bb) do { opcode_t top_op = oper_stack[oper_stack_idx - 1]; if (get_operator_prio(top_op) >= get_operator_prio(op)) { - rs2 = opstack_pop(); - rs1 = opstack_pop(); - reject_record_operand(rs1); - reject_record_operand(rs2); + pop_binary_operands(parent, bb, top_op, &rs1, &rs2); /* As in the final reduction below, equality compares a * function designator as its pointer. Reduced here, before * a following operator such as ||, a raw symbol compared * as garbage. */ - if (top_op == OP_eq || top_op == OP_neq) { - rs1 = materialize_function_designator(parent, bb, rs1); - rs2 = materialize_function_designator(parent, bb, rs2); - } - - /* Handle pointer arithmetic for addition and subtraction */ - if (is_pointer_operation(top_op, rs1, rs2)) { - /* handle_pointer_arithmetic handles both pointer - * differences and regular pointer arithmetic internally - */ - handle_pointer_arithmetic(parent, bb, top_op, rs1, rs2); - oper_stack_idx--; - continue; - } - - rs1 = integer_promote_operand(parent, bb, rs1); - rs2 = integer_promote_operand(parent, bb, rs2); - normalize_integer_binary_operands(parent, bb, top_op, &rs1, - &rs2); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(top_op, rs1, rs2); + vd = combine_assignment_value(parent, bb, top_op, rs1, rs2); opstack_push(vd); - add_insn(parent, *bb, top_op, vd, rs1, rs2, 0, NULL); oper_stack_idx--; } else @@ -3008,23 +2420,15 @@ void read_expr_body(block_t *parent, basic_block_t **bb) while (oper_stack_idx > 0) { opcode_t top_op = oper_stack[--oper_stack_idx]; - rs2 = opstack_pop(); - rs1 = opstack_pop(); - reject_record_operand(rs1); - reject_record_operand(rs2); + pop_binary_operands(parent, bb, top_op, &rs1, &rs2); /* Equality compares function pointers after the C99 function-to- * pointer conversion. A raw symbol has no SSA value and otherwise looks * like zero to the backend, making `function == 0` spuriously true. * Other arithmetic operations retain their function-pointer constraint - * diagnostics below. + * diagnostics below. An array of function pointers decays to an object + * pointer. */ - if (top_op == OP_eq || top_op == OP_neq) { - rs1 = materialize_function_designator(parent, bb, rs1); - rs2 = materialize_function_designator(parent, bb, rs2); - } - - /* An array of function pointers decays to an object pointer. */ if ((top_op == OP_lt || top_op == OP_leq || top_op == OP_gt || top_op == OP_geq) && ((rs1 && !is_array_declarator(rs1) && @@ -3098,88 +2502,13 @@ void read_expr_body(block_t *parent, basic_block_t **bb) !unsigned_int_operand(rs1) && !unsigned_int_operand(rs2) && rs1->type->size <= TY_int->size && rs2->type->size <= TY_int->size) { - /* Both operands are compile-time constants */ - int result = 0; - bool folded = true; - - switch (top_op) { - case OP_add: - result = rs1->init_val + rs2->init_val; - break; - case OP_sub: - result = rs1->init_val - rs2->init_val; - break; - case OP_mul: - result = rs1->init_val * rs2->init_val; - break; - case OP_div: - if (rs2->init_val != 0) - result = rs1->init_val / rs2->init_val; - else - folded = false; /* Division by zero */ - break; - case OP_mod: - if (rs2->init_val != 0) - result = rs1->init_val % rs2->init_val; - else - folded = false; /* Modulo by zero */ - break; - case OP_bit_and: - result = rs1->init_val & rs2->init_val; - break; - case OP_bit_or: - result = rs1->init_val | rs2->init_val; - break; - case OP_bit_xor: - result = rs1->init_val ^ rs2->init_val; - break; - - /* A shift count outside the promoted int width has no defined - * result, and the compiler must not compute one in the host either: - * leave such a shift to the target. Otherwise shift the unsigned - * bit pattern, so a bit reaching the sign is not host undefined - * behavior. - */ - case OP_lshift: - if (rs2->init_val < 0 || rs2->init_val >= 32) - folded = false; - else - result = - (int) ((unsigned int) rs1->init_val << rs2->init_val); - break; - case OP_rshift: - if (rs2->init_val < 0 || rs2->init_val >= 32) - folded = false; - else - result = rs1->init_val >> rs2->init_val; - break; - case OP_eq: - result = rs1->init_val == rs2->init_val; - break; - case OP_neq: - result = rs1->init_val != rs2->init_val; - break; - case OP_lt: - result = rs1->init_val < rs2->init_val; - break; - case OP_leq: - result = rs1->init_val <= rs2->init_val; - break; - case OP_gt: - result = rs1->init_val > rs2->init_val; - break; - case OP_geq: - result = rs1->init_val >= rs2->init_val; - break; - default: - folded = false; - break; - } + int result; + bool folded = vir_frontend_fold_i32(top_op, rs1->init_val, + rs2->init_val, &result); if (folded) { /* Create constant result */ - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->type = result_type; vd->is_const = true; vd->init_val = result; @@ -3195,19 +2524,14 @@ void read_expr_body(block_t *parent, basic_block_t **bb) NULL); } else { /* Normal operation - folding failed or not supported */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = result_type; + vd = emit_binary_result(parent, bb, top_op, rs1, rs2, + result_type); opstack_push(vd); - add_insn(parent, *bb, top_op, vd, rs1, rs2, 0, NULL); } } else { /* Normal operation */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = result_type; + vd = emit_binary_result(parent, bb, top_op, rs1, rs2, result_type); opstack_push(vd); - add_insn(parent, *bb, top_op, vd, rs1, rs2, 0, NULL); } } while (has_prev_log_op) { @@ -3308,6 +2632,88 @@ static int lvalue_step_size(const lvalue_t *lvalue, var_t *var) return 1; } +static void set_lvalue_pointer_pointee(lvalue_t *lvalue) +{ + type_t *pointee = pointee_type_from_pointer_typedef(lvalue->type); + + lvalue->size = pointer_typedef_pointee_size(lvalue->type, pointee); + lvalue->type = pointee; +} + +static var_t *read_addressed_value(block_t *parent, + basic_block_t **bb, + var_t *address, + const lvalue_t *lvalue, + bool assignment_result, + bool removable) +{ + var_t *value; + + if (is_bitfield(lvalue->decl)) + return assignment_result + ? reload_assigned_bitfield(parent, bb, address, lvalue->decl) + : read_bitfield_value(parent, bb, address, lvalue->decl); + value = name_var( + require_typed_ptr_var(parent, lvalue->type, lvalue->value_ptr_level)); + value->is_assignment_reload = removable && assignment_result; + add_insn(parent, *bb, OP_read, value, address, NULL, lvalue->size, NULL); + return value; +} + +static var_t *combine_addressed_assignment(block_t *parent, + basic_block_t **bb, + opcode_t op, + var_t *address, + const lvalue_t *target, + var_t *value) +{ + return op == OP_generic ? value + : combine_assignment_value( + parent, bb, op, + read_addressed_value(parent, bb, address, + target, false, false), + value); +} + +static var_t *store_addressed_assignment(block_t *parent, + basic_block_t **bb, + var_t *address, + const lvalue_t *target, + var_t *value, + bool use_resize_to) +{ + if (is_bitfield(target->decl)) { + write_bitfield_value(parent, bb, address, value, target->decl); + return value; + } + if (use_resize_to) { + if (!value->is_func) + value = resize_to(parent, bb, value, target->type, + target->value_ptr_level); + } else + value = convert_stored_value(parent, bb, value, target->type, + target->value_ptr_level); + add_insn(parent, *bb, OP_write, NULL, address, value, target->size, NULL); + return value; +} + +static var_t *read_lvalue_update_value(block_t *parent, + basic_block_t **bb, + const lvalue_t *lvalue) +{ + var_t *address; + var_t *value; + + if (!lvalue->is_reference) + return operand_stack[operand_stack_idx - 1]; + address = opstack_pop(); + lvalue_t target = *lvalue; + target.type = pointee_type_from_pointer_typedef(lvalue->type); + value = read_addressed_value(parent, bb, address, &target, false, false); + opstack_push(value); + return address; +} + /* What follows a completed lvalue: an update or read of the object it names. * read_lvalue() finishes with this. */ @@ -3331,12 +2737,16 @@ static void lower_lvalue_tail(lvalue_t *lvalue, */ if (lvalue->is_reference) { rs1 = operand_stack[operand_stack_idx - 1]; - t = require_var(parent); - t->var_name = gen_name(); - t->type = pointer_row_element_type - ? pointer_row_element_type - : pointee_type_from_pointer_typedef(lvalue->type); + t = require_named_var(parent); + t->type = pointee_type_from_pointer_typedef( + pointer_row_element_type ? pointer_row_element_type : lvalue->type); t->ptr_level = lvalue->value_ptr_level; + t->is_volatile = var->is_volatile; + int depth = t->ptr_level + t->type->ptr_level; + t->pointer_volatile_mask = + depth > 0 && depth <= 32 + ? var->pointer_volatile_mask & ~(1U << (depth - 1)) + : 0; if (pointer_row_element_type) t->is_const_qualified = lvalue->type->is_const_qualified; @@ -3364,6 +2774,8 @@ static void lower_lvalue_tail(lvalue_t *lvalue, * after consuming an extra object-pointer level. */ t->func_signature = lvalue->type->func_signature; + if (t->func_signature && !t->ptr_level) + t->ptr_level = 1; /* `fpp[i]` of an `int (**fpp)(int)` loads a callable function pointer, * and so do `fps[i]` of an `int (*fps[N])(int)` and a member `s.fp`. @@ -3419,25 +2831,19 @@ static void lower_lvalue_tail(lvalue_t *lvalue, if ((prefix_op == OP_add || prefix_op == OP_sub) && is_direct_void_pointer_type(lvalue->type, lvalue->ptr_level)) error_at("Pointer arithmetic on void* is invalid", cur_token_loc()); - vd = require_var(parent); - vd->var_name = gen_name(); - - vd->init_val = lvalue_step_size(lvalue, var); - if (lvalue_is_wide_integer(lvalue)) - vd->type = lvalue->type; + vd = load_constant( + parent, *bb, lvalue_step_size(lvalue, var), + lvalue_is_wide_integer(lvalue) ? lvalue->type : TY_int); opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); rs2 = opstack_pop(); if (lvalue->is_reference) rs1 = opstack_pop(); else rs1 = operand_stack[operand_stack_idx - 1]; - vd = require_var(parent); - vd->var_name = gen_name(); - if (lvalue_is_wide_integer(lvalue)) - vd->type = lvalue->type; - add_insn(parent, *bb, prefix_op, vd, rs1, rs2, 0, NULL); + vd = emit_binary_result( + parent, bb, prefix_op, rs1, rs2, + lvalue_is_wide_integer(lvalue) ? lvalue->type : TY_int); if (lvalue->is_reference) { rs1 = vd; @@ -3455,9 +2861,6 @@ static void lower_lvalue_tail(lvalue_t *lvalue, rs1 = convert_stored_value(parent, bb, rs1, t->type, t->ptr_level); - /* The column of arguments of the new insn of 'OP_write' is - * different from 'ph1_ir' - */ add_insn(parent, *bb, OP_write, NULL, vd, rs1, lvalue->size, NULL); } @@ -3483,17 +2886,11 @@ static void lower_lvalue_tail(lvalue_t *lvalue, /* Consume the '--' as well: testing for '++' alone left it in the * stream, so `bits.field--` failed to parse. */ - opcode_t postfix_op = OP_sub; - if (lex_accept(T_increment)) - postfix_op = OP_add; - else - lex_expect(T_decrement); + opcode_t postfix_op = read_increment_operator(); var_t *old = opstack_pop(); var_t *address = opstack_pop(); var_t *one = bitfield_constant(parent, bb, 1); - var_t *updated = require_var(parent); - - updated->var_name = gen_name(); + var_t *updated = require_named_var(parent); updated->type = integer_binary_result_type(postfix_op, old, one); add_insn(parent, *bb, postfix_op, updated, old, one, 0, NULL); write_bitfield_value(parent, bb, address, updated, lvalue->decl); @@ -3508,13 +2905,9 @@ static void lower_lvalue_tail(lvalue_t *lvalue, * operator, and before a call (C99 6.5.2.2p10). Deferring it to the end * of the statement also ran updates in a skipped operand. */ - vd = require_var(parent); - vd->var_name = gen_name(); - - vd->init_val = lvalue_step_size(lvalue, var); - if (lvalue_is_wide_integer(lvalue)) - vd->type = lvalue->type; - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); + vd = load_constant( + parent, *bb, lvalue_step_size(lvalue, var), + lvalue_is_wide_integer(lvalue) ? lvalue->type : TY_int); rs2 = vd; /* Consume whichever operator is actually there. Testing only for '++' @@ -3522,11 +2915,7 @@ static void lower_lvalue_tail(lvalue_t *lvalue, * decrement parsed only where the leftover token happened to be * harmless -- "i--;" as a statement worked, "a = i--" did not. */ - opcode_t postfix_op = OP_sub; - if (lex_accept(T_increment)) - postfix_op = OP_add; - else - lex_expect(T_decrement); + opcode_t postfix_op = read_increment_operator(); /* The expression has the value the object held before the update. A * reference was already loaded into a temporary; a variable is itself @@ -3537,8 +2926,7 @@ static void lower_lvalue_tail(lvalue_t *lvalue, rs1 = opstack_pop(); } else { rs1 = operand_stack[operand_stack_idx - 1]; - t = require_var(parent); - t->var_name = gen_name(); + t = require_named_var(parent); t->type = rs1->type; t->ptr_level = rs1->ptr_level; t->func_signature = rs1->func_signature; @@ -3548,8 +2936,7 @@ static void lower_lvalue_tail(lvalue_t *lvalue, copy_pointee_array_shape(t, rs1); add_insn(parent, *bb, OP_assign, t, rs1, NULL, 0, NULL); } - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); if (!lvalue->is_reference && is_pointee_array_pointer(var)) { vd->type = lvalue->type; vd->ptr_level = var->ptr_level; @@ -3617,7 +3004,7 @@ static bool is_function_parameter(const var_t *var, block_t *parent) for (int i = 0; i < parent->func->num_params; i++) { var_t *param = &parent->func->param_defs[i]; - if (var == param || var->base == param) + if (var == param) return true; } return false; @@ -3659,6 +3046,8 @@ void read_lvalue(lvalue_t *lvalue, * qualification; the object a pointer member points to does not. */ bool record_is_const = false; + bool record_is_volatile = false; + bool object_is_volatile = var_volatile_storage(var); /* Callers pass a find_var() result, which is NULL for a name that was never * declared. @@ -3702,7 +3091,7 @@ void read_lvalue(lvalue_t *lvalue, lvalue->pointer_const_mask = var->pointer_const_mask; opstack_push(var); - if (var->is_volatile) + if (var_is_volatile_object(var)) unread_volatile_object = var; if (lex_peek(T_open_square, NULL) || lex_peek(T_arrow, NULL) || @@ -3713,6 +3102,9 @@ void read_lvalue(lvalue_t *lvalue, lex_peek(T_dot, NULL)) { array_row = NULL; if (lex_accept(T_open_square)) { + object_is_volatile = + var_volatile_pointee(var) || + (record_is_volatile && is_array_declarator(var)); int indexed_ptr_level; fixed_array_shape_t row_shape; bool indexes_fixed_array_pointer_slot; @@ -3730,8 +3122,9 @@ void read_lvalue(lvalue_t *lvalue, (lvalue->value_ptr_level || pending_fixed_array_pointer_slot) && is_member) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + vd->type = pointee_type_from_pointer_typedef(lvalue->type); + vd->ptr_level = lvalue->value_ptr_level; if (pending_scalar_row_pointer_slot) { vd->type = var->type; vd->ptr_level = 1; @@ -3765,12 +3158,7 @@ void read_lvalue(lvalue_t *lvalue, lvalue->size = PTR_SIZE; } else if (lvalue->type->ptr_level && !loaded_scalar_row && !lvalue->type->pointee_array_size) { - type_t *pointee = - pointee_type_from_pointer_typedef(lvalue->type); - - lvalue->size = - pointer_typedef_pointee_size(lvalue->type, pointee); - lvalue->type = pointee; + set_lvalue_pointer_pointee(lvalue); } else { lvalue->size = lvalue->type->size; } @@ -3849,7 +3237,7 @@ void read_lvalue(lvalue_t *lvalue, pointer_row_element_type = var->type->pointee_array_element_type ? var->type->pointee_array_element_type - : var->type; + : array_typedef_element_type(var->type); } else if (!subscript_depth && is_array_declarator(var) && var->type->array_element_ptr_level > 0) { pointer_row_element_type = var->type->array_element_type @@ -3870,17 +3258,12 @@ void read_lvalue(lvalue_t *lvalue, * pointer points to */ if (lvalue->type->ptr_level > 0) { - type_t *pointee = - pointee_type_from_pointer_typedef(lvalue->type); - /* void pointers retain the existing byte-stride extension; * every other typedef pointer advances by its actual * pointee, including a tagged record reached through a * typedef alias. */ - lvalue->size = - pointer_typedef_pointee_size(lvalue->type, pointee); - lvalue->type = pointee; + set_lvalue_pointer_pointee(lvalue); } else { lvalue->size = lvalue->type->size; } @@ -3904,17 +3287,15 @@ void read_lvalue(lvalue_t *lvalue, if (subscript_depth == 0 && var->array_dim2 > 0) { multiplier = fixed_array_subscript_stride(var, subscript_depth, lvalue->size); - } else if (subscript_depth == 0 && var->pointee_array_size > 0 && - indexes_pointee_row) { - multiplier = fixed_pointee_array_subscript_stride( - var, subscript_depth, lvalue->size); - } else if (indexes_loaded_scalar_pointee_row) { - multiplier = fixed_pointee_array_subscript_stride( - loaded_scalar_row, 0, lvalue->size); } else if (loaded_scalar_row) { + int row_depth = + indexes_loaded_scalar_pointee_row ? 0 : subscript_depth - 1; + multiplier = fixed_pointee_array_subscript_stride( - loaded_scalar_row, subscript_depth - 1, lvalue->size); - } else if (subscript_depth >= 1 && var->pointee_array_size > 0) { + loaded_scalar_row, row_depth, lvalue->size); + } else if (var->pointee_array_size > 0 && + ((subscript_depth == 0 && indexes_pointee_row) || + subscript_depth >= 1)) { multiplier = fixed_pointee_array_subscript_stride( var, subscript_depth, lvalue->size); } else if ((subscript_depth == 1 && var->array_dim3 > 0) || @@ -3924,24 +3305,25 @@ void read_lvalue(lvalue_t *lvalue, } if (multiplier != 1) { - vd = require_var(parent); - vd->init_val = multiplier; - vd->var_name = gen_name(); + vd = load_constant(parent, *bb, multiplier, TY_int); opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); rs2 = opstack_pop(); rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + + /* The scaled offset retains the promoted index type. In + * particular, an unsigned int index must remain unsigned when + * the later pointer addition widens it to pointer width. + */ + vd->type = integer_promoted_type(rs1); opstack_push(vd); add_insn(parent, *bb, OP_mul, vd, rs1, rs2, 0, NULL); } rs2 = opstack_pop(); rs1 = opstack_pop(); - vd = require_var(parent); + vd = name_var(require_var(parent)); /* A subscript expression computes the address of its selected * element. Preserve that pointer provenance: unary '&' leaves an @@ -3956,7 +3338,7 @@ void read_lvalue(lvalue_t *lvalue, * `int **`, not `int *`. */ vd->ptr_level = lvalue->ptr_level + 1; - vd->var_name = gen_name(); + vd->is_volatile_access = object_is_volatile; opstack_push(vd); add_insn(parent, *bb, OP_add, vd, rs1, rs2, 0, NULL); @@ -3975,8 +3357,7 @@ void read_lvalue(lvalue_t *lvalue, for (int i = 0; i < subscript_depth + 2; i++) fixed_array_shape_drop_outer(&row_shape); if (row_shape.rank) { - fixed_array_shape_to_pointee_var(vd, &row_shape); - vd->pointee_array_element_ptr_level = var->ptr_level; + set_pointee_array_shape(vd, &row_shape, var->ptr_level); } vd->is_const_qualified = var->is_const_qualified; lvalue->value_ptr_level = indexed_ptr_level; @@ -3999,9 +3380,9 @@ void read_lvalue(lvalue_t *lvalue, vd->ptr_level = row_source->pointee_array_element_ptr_level + 1; if (row_shape.rank) { - fixed_array_shape_to_pointee_var(vd, &row_shape); - vd->pointee_array_element_ptr_level = - row_source->pointee_array_element_ptr_level; + set_pointee_array_shape( + vd, &row_shape, + row_source->pointee_array_element_ptr_level); } vd->is_const_qualified = var->is_const_qualified; lvalue->value_ptr_level = indexed_ptr_level; @@ -4069,14 +3450,16 @@ void read_lvalue(lvalue_t *lvalue, * pointer's declaration however the pointer itself is. */ record_is_const = var->is_const_qualified; + record_is_volatile = var_volatile_pointee(var); /* resolve where the pointer points at from the calculated * address in a structure. */ if (is_member) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + vd->type = pointee_type_from_pointer_typedef(lvalue->type); + vd->ptr_level = lvalue->value_ptr_level; opstack_push(vd); add_insn(parent, *bb, OP_read, vd, rs1, NULL, PTR_SIZE, NULL); @@ -4084,11 +3467,11 @@ void read_lvalue(lvalue_t *lvalue, } else { lex_expect(T_dot); record_is_const = lvalue->is_const_qualified; + record_is_volatile = object_is_volatile; if (!is_address_got) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); opstack_push(vd); add_insn(parent, *bb, OP_address_of, vd, rs1, NULL, 0, NULL); @@ -4105,6 +3488,8 @@ void read_lvalue(lvalue_t *lvalue, error_at("Unknown struct or union member", next_token_loc()); lvalue->type = var->type; lvalue->decl = var; + object_is_volatile = + record_is_volatile || var_volatile_storage(var); /* The member, not the pointer row element it was selected from, now * types the value that is read. @@ -4133,16 +3518,15 @@ void read_lvalue(lvalue_t *lvalue, lvalue->is_reference = !is_array_declarator(var); /* move pointer to offset of structure */ - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); vd->init_val = var->offset; opstack_push(vd); add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, NULL); rs2 = opstack_pop(); rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); + vd->is_volatile_access = object_is_volatile; if (is_array_declarator(var)) { /* A one-dimensional array member designates its first element's @@ -4316,14 +3700,10 @@ void finalize_logical(opcode_t op, * Otherwise, create a false branch and assign a false value for logical-or * operation. */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = op == OP_log_and; - add_insn(parent, op == OP_log_and ? then_next : else_bb, OP_load_constant, - vd, NULL, NULL, 0, NULL); - - log_op_res = require_var(parent); - log_op_res->var_name = gen_name(); + vd = load_constant(parent, op == OP_log_and ? then_next : else_bb, + op == OP_log_and, TY_int); + + log_op_res = require_named_var(parent); add_insn(parent, op == OP_log_and ? then_next : else_bb, OP_assign, log_op_res, vd, NULL, 0, NULL); @@ -4337,11 +3717,8 @@ void finalize_logical(opcode_t op, * If handing a logical-and operation, assign a false value. else, assign a * true value for a logical-or operation. */ - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = op != OP_log_and; - add_insn(parent, op == OP_log_and ? else_bb : then, OP_load_constant, vd, - NULL, NULL, 0, NULL); + vd = load_constant(parent, op == OP_log_and ? else_bb : then, + op != OP_log_and, TY_int); add_insn(parent, op == OP_log_and ? else_bb : then, OP_assign, log_op_res, vd, NULL, 0, NULL); @@ -4358,18 +3735,7 @@ void finalize_logical(opcode_t op, */ void read_control_expression(block_t *parent, basic_block_t **bb) { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - - while (lex_accept(T_comma)) { - discard_operand(parent, *bb); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - } + read_comma_expression(parent, bb, false); /* A function designator in a controlling expression decays to its code * pointer before OP_branch tests it. Raw symbols intentionally have no @@ -4394,126 +3760,6 @@ basic_block_t *read_full_expression_statement(block_t *parent, return bb; } -/* This is deliberately narrower than expression assignment recognition: a - * grouped pointer-to-row store has no general lvalue representation yet. - */ -static bool grouped_scalar_pointee_row_store_starts(block_t *parent) -{ - token_t *token = cur_token ? cur_token->next : NULL; - int depth = 0; - - if (!token || token->kind != T_open_bracket || !(token = token->next) || - token->kind != T_asterisk || !(token = token->next) || - token->kind != T_identifier || - !names_scalar_row_pointer(token, parent) || !(token = token->next) || - token->kind != T_close_bracket || !(token = token->next) || - token->kind != T_open_square) - return false; - for (; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || - token->kind == T_close_curly) { - if (!--depth) - break; - } - } - if (!token || !(token = token->next) || - (token->kind != T_assign && token->kind != T_pluseq && - token->kind != T_minuseq)) - return false; - for (depth = 0, token = token->next; token; token = token->next) { - if (token->kind == T_open_square || token->kind == T_open_bracket || - token->kind == T_open_curly) - depth++; - else if (token->kind == T_close_square || - token->kind == T_close_bracket || token->kind == T_close_curly) - depth--; - else if (!depth && token->kind == T_comma) - return false; - else if (!depth && token->kind == T_semicolon) - return true; - } - return false; -} - -static basic_block_t *handle_grouped_scalar_pointee_row_store(block_t *parent, - basic_block_t *bb) -{ - char name[MAX_VAR_LEN]; - var_t *source, *row, *index, *address, *value; - - lex_expect(T_open_bracket); - lex_expect(T_asterisk); - lex_ident(T_identifier, name); - source = find_var(name, parent); - if (!source) - error_at("Undeclared identifier", cur_token_loc()); - if (!lower_pointee_array_dereference(source, parent, &bb)) - error_at("Grouped pointer-to-array store requires a scalar fixed row", - cur_token_loc()); - row = opstack_pop(); - if (row->is_const_qualified) - error_at("assignment of read-only location", cur_token_loc()); - lex_expect(T_close_bracket); - lex_expect(T_open_square); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - index = opstack_pop(); - lex_expect(T_close_square); - if (row->type->size != 1) { - var_t *scale = require_var(parent); - var_t *scaled = require_var(parent); - scale->var_name = gen_name(); - scale->init_val = row->type->size; - add_insn(parent, bb, OP_load_constant, scale, NULL, NULL, 0, NULL); - scaled->var_name = gen_name(); - add_insn(parent, bb, OP_mul, scaled, index, scale, 0, NULL); - index = scaled; - } - address = require_typed_ptr_var(parent, row->type, 1); - address->var_name = gen_name(); - add_insn(parent, bb, OP_add, address, row, index, 0, NULL); - opcode_t compound_op = OP_generic; - if (lex_accept(T_pluseq)) - compound_op = OP_add; - else if (lex_accept(T_minuseq)) - compound_op = OP_sub; - else - lex_expect(T_assign); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - value = opstack_pop(); - if (compound_op != OP_generic) { - if (value->is_func || is_pointer_like_value(value) || - is_record_type(value->type)) - error_at("Invalid compound assignment operand", cur_token_loc()); - var_t *current = require_typed_var(parent, row->type); - current->var_name = gen_name(); - add_insn(parent, bb, OP_read, current, address, NULL, row->type->size, - NULL); - current = integer_promote_operand(parent, &bb, current); - value = integer_promote_operand(parent, &bb, value); - normalize_integer_binary_operands(parent, &bb, compound_op, ¤t, - &value); - var_t *updated = require_var(parent); - updated->var_name = gen_name(); - updated->type = integer_binary_result_type(compound_op, current, value); - add_insn(parent, bb, compound_op, updated, current, value, 0, NULL); - value = resize_to(parent, &bb, updated, row->type, 0); - } - value = convert_stored_value(parent, &bb, value, row->type, 0); - add_insn(parent, bb, OP_write, NULL, address, value, row->type->size, NULL); - lex_expect(T_semicolon); - return bb; -} - /* The middle operand of ?: is an expression, rather than merely an * assignment-expression, so top-level commas belong to the selected true * branch. Keep this small sequencing layer here until all full-expression entry @@ -4521,18 +3767,7 @@ static basic_block_t *handle_grouped_scalar_pointee_row_store(block_t *parent, */ void read_conditional_true_expression(block_t *parent, basic_block_t **bb) { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - - while (lex_accept(T_comma)) { - discard_operand(parent, *bb); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } - } + read_comma_expression(parent, bb, false); } /* An integer constant expression with value zero is the sole scalar that can @@ -4574,14 +3809,10 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) var_t *true_val = opstack_pop(); - /* false branch */ - if (!read_assignment_expression(parent, &else_)) - read_expr(parent, &else_); - - /* The third operand has conditional-expression grammar, making nested - * conditionals right-associative: a ? b : c ? d : e. + /* false branch The third operand has conditional-expression grammar, making + * nested conditionals right-associative: a ? b : c ? d : e. */ - read_ternary_operation(parent, &else_); + read_assignment_or_expression(parent, &else_); bb_connect(*bb, else_entry, ELSE); var_t *false_val = opstack_pop(); @@ -4638,15 +3869,13 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) */ true_addr = record_value_address(parent, then_, true_val); false_addr = record_value_address(parent, else_, false_val); - selected = require_ref_var(parent, true_val->type, 0); - selected->var_name = gen_name(); + selected = name_var(require_ref_var(parent, true_val->type, 0)); add_insn(parent, then_, OP_assign, selected, true_addr, NULL, 0, NULL); add_insn(parent, else_, OP_assign, selected, false_addr, NULL, 0, NULL); bb_connect(then_, end_ternary, NEXT); bb_connect(else_, end_ternary, NEXT); - vd = require_typed_var(parent, true_val->type); - vd->var_name = gen_name(); + vd = name_var(require_typed_var(parent, true_val->type)); add_insn(parent, end_ternary, OP_allocat, vd, NULL, NULL, 0, NULL); emit_record_copy_from_address(parent, &end_ternary, vd, selected); opstack_push(vd); @@ -4654,9 +3883,11 @@ void read_ternary_operation(block_t *parent, basic_block_t **bb) return; } - vd = require_var(parent); - vd->var_name = gen_name(); - if (true_pointee_signature || false_pointee_signature) { + vd = require_named_var(parent); + if (true_val->type == TY_void && !true_val->ptr_level && + false_val->type == TY_void && !false_val->ptr_level) { + vd->type = TY_void; + } else if (true_pointee_signature || false_pointee_signature) { func_t *signature = true_pointee_signature ? true_pointee_signature : false_pointee_signature; var_t *pointer_value = true_pointee_signature ? true_val : false_val; @@ -4757,13 +3988,11 @@ bool read_body_assignment(var_t *var, int one = 0; opcode_t op = OP_generic; lvalue_t lvalue; - int size = 0; /* has memory address that we want to set */ read_lvalue(&lvalue, var, parent, bb, false, OP_generic); while (lvalue_paren_depth-- > 0) lex_expect(T_close_bracket); - size = lvalue.size; if (lvalue.is_array && lvalue_write_follows(prefix_op)) error_at("assignment to expression with array type", @@ -4788,8 +4017,7 @@ bool read_body_assignment(var_t *var, */ if (lvalue.is_reference) { rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); opstack_push(vd); add_insn(parent, *bb, OP_read, vd, rs1, NULL, PTR_SIZE, NULL); } @@ -4824,46 +4052,19 @@ bool read_body_assignment(var_t *var, * of stack as the one of operands and the destination. */ if (one == 1) { - if (lvalue.is_reference) { - t = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = pointee_type_from_pointer_typedef(lvalue.type); - vd->ptr_level = lvalue.value_ptr_level; - opstack_push(vd); - if (is_bitfield(lvalue.decl)) { - opstack_pop(); - vd = read_bitfield_value(parent, bb, t, lvalue.decl); - opstack_push(vd); - } else { - add_insn(parent, *bb, OP_read, vd, t, NULL, lvalue.size, - NULL); - } - } else - t = operand_stack[operand_stack_idx - 1]; + t = read_lvalue_update_value(parent, bb, &lvalue); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->init_val = increment_size; - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); + vd = load_constant(parent, *bb, increment_size, TY_int); rs2 = vd; rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); add_insn(parent, *bb, op, vd, rs1, rs2, 0, NULL); - if (lvalue.is_reference) { - if (is_bitfield(lvalue.decl)) - write_bitfield_value(parent, bb, t, vd, lvalue.decl); - else { - vd = convert_stored_value(parent, bb, vd, lvalue.type, - lvalue.value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, t, vd, size, - NULL); - } - } else { + if (lvalue.is_reference) + store_addressed_assignment(parent, bb, t, &lvalue, vd, + false); + else { vd = resize_var(parent, bb, vd, t); mark_var_mutated(t); add_insn(parent, *bb, OP_assign, t, vd, NULL, 0, NULL); @@ -4871,48 +4072,18 @@ bool read_body_assignment(var_t *var, if (assignment_result) { if (!lvalue.is_reference) { assignment_result[0] = vd; - } else if (is_bitfield(lvalue.decl)) { - var_t *stored = reload_assigned_bitfield(parent, bb, t, - lvalue.decl); - assignment_result[0] = stored; } else { - var_t *stored = require_typed_var(parent, lvalue.type); - stored->ptr_level = lvalue.value_ptr_level; - stored->var_name = gen_name(); - stored->is_assignment_reload = true; - add_insn(parent, *bb, OP_read, stored, t, NULL, - lvalue.size, NULL); - assignment_result[0] = stored; + assignment_result[0] = read_addressed_value( + parent, bb, t, &lvalue, true, true); } } } else { - if (lvalue.is_reference) { - t = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = pointee_type_from_pointer_typedef(lvalue.type); - vd->ptr_level = lvalue.value_ptr_level; - opstack_push(vd); - if (is_bitfield(lvalue.decl)) { - opstack_pop(); - vd = read_bitfield_value(parent, bb, t, lvalue.decl); - opstack_push(vd); - } else { - add_insn(parent, *bb, OP_read, vd, t, NULL, lvalue.size, - NULL); - } - } else - t = operand_stack[operand_stack_idx - 1]; - - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); + t = read_lvalue_update_value(parent, bb, &lvalue); - /* read_expr stops before `?`; compound assignment has the - * same conditional-expression RHS grammar as ordinary - * assignment. - */ - read_ternary_operation(parent, bb); - } + /* Compound assignment has the same conditional-expression RHS + * grammar as ordinary assignment. + */ + read_assignment_or_expression(parent, bb); var_t *rhs_val = opstack_pop(); reject_record_operand(rhs_val); @@ -4929,23 +4100,10 @@ bool read_body_assignment(var_t *var, * add -56 to a wider left operand. */ rhs_val = integer_promote_operand(parent, bb, rhs_val); - opstack_push(rhs_val); - vd = require_var(parent); - vd->init_val = increment_size; - vd->var_name = gen_name(); - opstack_push(vd); - add_insn(parent, *bb, OP_load_constant, vd, NULL, NULL, 0, - NULL); - - rs2 = opstack_pop(); - rs1 = opstack_pop(); - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(OP_mul, rs1, rs2); - opstack_push(vd); - add_insn(parent, *bb, OP_mul, vd, rs1, rs2, 0, NULL); - - rs2 = opstack_pop(); + if (increment_size != 1) + rhs_val = scale_pointer_index(parent, *bb, rhs_val, + increment_size, true); + rs2 = rhs_val; rs1 = opstack_pop(); /* Compound assignment performs the same integer promotions and @@ -4961,27 +4119,14 @@ bool read_body_assignment(var_t *var, normalize_integer_binary_operands(parent, bb, op, &rs1, &rs2); } - vd = require_var(parent); - vd->var_name = gen_name(); - vd->type = integer_binary_result_type(op, rs1, rs2); - add_insn(parent, *bb, op, vd, rs1, rs2, 0, NULL); + vd = emit_binary_result( + parent, bb, op, rs1, rs2, + integer_binary_result_type(op, rs1, rs2)); - if (lvalue.is_reference) { - if (is_bitfield(lvalue.decl)) - write_bitfield_value(parent, bb, t, vd, lvalue.decl); - else { - /* Convert back to the object type before the store, as - * the direct-variable path does. The store alone - * narrows the bytes, but the expression value is this - * register, and a _Bool object must receive 0 or 1 - * rather than the low byte of the sum. - */ - vd = resize_to(parent, bb, vd, lvalue.type, - lvalue.value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, t, vd, - lvalue.size, NULL); - } - } else { + if (lvalue.is_reference) + vd = store_addressed_assignment(parent, bb, t, &lvalue, vd, + true); + else { vd = resize_var(parent, bb, vd, t); add_insn(parent, *bb, OP_assign, t, vd, NULL, 0, NULL); } @@ -4991,20 +4136,15 @@ bool read_body_assignment(var_t *var, * the arithmetic result, so its expression value must be * reloaded just like an ordinary bit-field assignment. */ - if (is_bitfield(lvalue.decl)) { - var_t *stored = reload_assigned_bitfield(parent, bb, t, - lvalue.decl); - assignment_result[0] = stored; - } else { - assignment_result[0] = vd; - } + assignment_result[0] = + is_bitfield(lvalue.decl) + ? read_addressed_value(parent, bb, t, &lvalue, true, + true) + : vd; } } } else { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); if (lvalue.is_func) { rs2 = opstack_pop(); @@ -5018,12 +4158,11 @@ bool read_body_assignment(var_t *var, * function address. */ if (rs2->is_func && find_var(rs2->var_name, parent) == rs2) { - t = require_ref_var(parent, rs2->type, rs2->ptr_level); - t->var_name = gen_name(); + t = name_var( + require_ref_var(parent, rs2->type, rs2->ptr_level)); add_insn(parent, *bb, OP_address_of, t, rs2, NULL, 0, NULL); - vd = require_var(parent); - vd->var_name = gen_name(); + vd = require_named_var(parent); add_insn(parent, *bb, OP_read, vd, t, NULL, PTR_SIZE, NULL); rs2 = vd; } @@ -5032,9 +4171,8 @@ bool read_body_assignment(var_t *var, * variable. */ if (!lvalue.is_reference) { - var_t *addr = - require_ref_var(parent, lvalue.type, lvalue.ptr_level); - addr->var_name = gen_name(); + var_t *addr = name_var( + require_ref_var(parent, lvalue.type, lvalue.ptr_level)); add_insn(parent, *bb, OP_address_of, addr, rs1, NULL, 0, NULL); rs1 = addr; @@ -5078,15 +4216,11 @@ bool read_body_assignment(var_t *var, error_at("incompatible types in assignment", cur_token_loc()); - if (is_bitfield(lvalue.decl)) - write_bitfield_value(parent, bb, rs1, rs2, lvalue.decl); - else if (record_store) + if (record_store) emit_record_copy_to_address(parent, bb, rs1, rs2); - else { - rs2 = convert_stored_value(parent, bb, rs2, lvalue.type, - lvalue.value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, rs1, rs2, size, NULL); - } + else + store_addressed_assignment(parent, bb, rs1, &lvalue, rs2, + false); if (assignment_result && record_store) { /* The stored record is a copy of the right operand. */ assignment_result[0] = rs2; @@ -5096,19 +4230,8 @@ bool read_body_assignment(var_t *var, * the source, so reload its extracted value instead of * leaking the unconverted right operand. */ - if (is_bitfield(lvalue.decl)) { - var_t *stored = reload_assigned_bitfield( - parent, bb, rs1, lvalue.decl); - assignment_result[0] = stored; - } else { - var_t *stored = require_typed_var(parent, lvalue.type); - stored->ptr_level = lvalue.value_ptr_level; - stored->var_name = gen_name(); - stored->is_assignment_reload = true; - add_insn(parent, *bb, OP_read, stored, rs1, NULL, size, - NULL); - assignment_result[0] = stored; - } + assignment_result[0] = read_addressed_value( + parent, bb, rs1, &lvalue, true, true); } } else { rs1 = opstack_pop(); @@ -5166,6 +4289,8 @@ bool assignment_expression_follows(void) int depth = 0; for (token_t *t = cur_token->next; t; t = t->next) { + if (!depth && assignment_operator(t->kind)) + return true; switch (t->kind) { case T_open_bracket: case T_open_square: @@ -5179,20 +4304,6 @@ bool assignment_expression_follows(void) return false; depth--; break; - case T_assign: - case T_pluseq: - case T_minuseq: - case T_asteriskeq: - case T_divideeq: - case T_modeq: - case T_lshifteq: - case T_rshifteq: - case T_andeq: - case T_oreq: - case T_xoreq: - if (!depth) - return true; - break; case T_question: /* `?:` has lower precedence than assignment. An assignment in an * arm belongs to that conditional expression, not to the leading @@ -5221,17 +4332,11 @@ bool assignment_expression_follows(void) bool compound_literal_starts_expression(void) { token_t *t = cur_token->next; - int depth = 0; if (!t || t->kind != T_open_bracket) return false; - for (; t; t = t->next) { - if (t->kind == T_open_bracket) - depth++; - else if (t->kind == T_close_bracket && --depth == 0) - return t->next && t->next->kind == T_open_curly; - } - return false; + t = skip_balanced_delimiter(t, T_open_bracket, T_close_bracket); + return t && t->next && t->next->kind == T_open_curly; } /* Whether the assignment ahead has a parenthesized left operand followed by @@ -5240,61 +4345,33 @@ bool compound_literal_starts_expression(void) * member chain on a dereference, `(*q).a.b = v`, stays with the dereference * path. */ +static bool postfix_lvalue_follows(const token_t *token) +{ + return token && (token->kind == T_dot || token->kind == T_arrow || + token->kind == T_open_square); +} + static bool grouped_postfix_lvalue_follows(void) { token_t *token = cur_token->next; token_t *inner; - int depth = 0; if (!token || token->kind != T_open_bracket) return false; inner = token->next; - for (; token; token = token->next) { - if (token->kind == T_open_bracket) - depth++; - else if (token->kind == T_close_bracket && !--depth) - break; - } - if (!token || !(token = token->next) || - (token->kind != T_dot && token->kind != T_arrow && - token->kind != T_open_square)) + token = skip_balanced_delimiter(token, T_open_bracket, T_close_bracket); + if (!token || !(token = token->next) || !postfix_lvalue_follows(token)) return false; if (inner && inner->kind == T_asterisk) { while (token && token->kind == T_dot && token->next && token->next->kind == T_identifier) token = token->next->next; - if (token && token->kind != T_dot && token->kind != T_arrow && - token->kind != T_open_square) + if (!postfix_lvalue_follows(token)) return false; } return true; } -/* Whether the next token assigns to what precedes it. */ -static bool assignment_operator_follows(void) -{ - token_t *token = cur_token->next; - - if (!token) - return false; - switch (token->kind) { - case T_assign: - case T_pluseq: - case T_minuseq: - case T_asteriskeq: - case T_divideeq: - case T_modeq: - case T_lshifteq: - case T_rshifteq: - case T_xoreq: - case T_oreq: - case T_andeq: - return true; - default: - return false; - } -} - /* Whether the assignment ahead has a call result as the root of its left * operand, `one()->y = v` or `rows()[0][1].x = v`. No declaration describes * such an operand, so the lvalue parser cannot follow it, but the postfix @@ -5303,23 +4380,16 @@ static bool assignment_operator_follows(void) static bool call_postfix_lvalue_follows(void) { token_t *token = cur_token->next; - int depth = 0; if (!token || token->kind != T_identifier) return false; token = token->next; if (!token || token->kind != T_open_bracket) return false; - for (; token; token = token->next) { - if (token->kind == T_open_bracket) - depth++; - else if (token->kind == T_close_bracket && !--depth) - break; - } + token = skip_balanced_delimiter(token, T_open_bracket, T_close_bracket); if (!token || !(token = token->next)) return false; - return token->kind == T_dot || token->kind == T_arrow || - token->kind == T_open_square; + return postfix_lvalue_follows(token); } /* Assign through a left operand that the expression parser lowers as a value, @@ -5349,17 +4419,16 @@ static void read_grouped_postfix_assignment(block_t *parent, basic_block_t **bb) target->is_func) error_at("Assignment requires a modifiable lvalue", cur_token_loc()); pointer_object = effective_pointer_depth(target) > 0; - if (pointer_object ? target->is_const_pointer : target->is_const_qualified) + if ((pointer_object && target->is_const_pointer) || + (!pointer_object && target->is_const_qualified) || + (target->func_signature && target->is_const_qualified)) error_at("assignment of read-only location", cur_token_loc()); address = target->compound_literal_address; size = pointer_object || target->func_signature ? PTR_SIZE : target->type->size; - if (!lex_accept(T_assign) && !accept_compound_assign_op(&compound_op)) + if (!accept_assignment_op(&compound_op)) error_at("Expected assignment operator", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); value = opstack_pop(); if (is_record_object(target)) { @@ -5379,54 +4448,19 @@ static void read_grouped_postfix_assignment(block_t *parent, basic_block_t **bb) (pointer_object && compound_op != OP_add && compound_op != OP_sub)))) error_at("Invalid assignment operand", cur_token_loc()); - if (compound_op != OP_generic) { - var_t *current; - var_t *combined; - - if (target->compound_literal_bitfield) - current = read_bitfield_value(parent, bb, address, - target->compound_literal_bitfield); - else { - current = - require_typed_ptr_var(parent, target->type, target->ptr_level); - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, size, NULL); - } - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, value); - combined = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, ¤t, - &value); - combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, 0, - NULL); - } - value = combined; - } - - if (target->compound_literal_bitfield) { - write_bitfield_value(parent, bb, address, value, - target->compound_literal_bitfield); - result = reload_assigned_bitfield(parent, bb, address, - target->compound_literal_bitfield); - opstack_push(result); - return; - } - if (!value->is_func) - value = resize_to(parent, bb, value, target->type, target->ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, size, NULL); - - /* The assignment has the value the object then holds. */ - result = require_typed_ptr_var(parent, target->type, target->ptr_level); - result->var_name = gen_name(); + lvalue_t target_lvalue = { + .size = size, + .value_ptr_level = target->ptr_level, + .type = target->type, + .decl = target->compound_literal_bitfield, + }; + value = combine_addressed_assignment(parent, bb, compound_op, address, + &target_lvalue, value); + store_addressed_assignment(parent, bb, address, &target_lvalue, value, + true); + result = + read_addressed_value(parent, bb, address, &target_lvalue, true, false); result->func_signature = target->func_signature; - add_insn(parent, *bb, OP_read, result, address, NULL, size, NULL); opstack_push(result); } @@ -5453,22 +4487,9 @@ static var_t *dereferenced_array_store_base(block_t *parent, int *stars) if (!array || !is_array_declarator(array) || array->is_func || array->func_signature || array->pointee_func_signature) return NULL; - switch (token->next->kind) { - case T_assign: - case T_pluseq: - case T_minuseq: - case T_asteriskeq: - case T_divideeq: - case T_modeq: - case T_lshifteq: - case T_rshifteq: - case T_xoreq: - case T_oreq: - case T_andeq: - return array; - default: + if (!assignment_operator(token->next->kind)) return NULL; - } + return array; } bool read_assignment_expression(block_t *parent, basic_block_t **bb) @@ -5610,75 +4631,38 @@ bool read_assignment_expression(block_t *parent, basic_block_t **bb) (address_mask & (1U << (address_depth - 2)))) || (!field && points_to_const_callback(object_address))) error_at("assignment of read-only location", cur_token_loc()); - if (!lex_accept(T_assign) && !accept_compound_assign_op(&compound_op)) + if (!accept_assignment_op(&compound_op)) error_at("Expected assignment after pointer dereference", cur_token_loc()); - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); value = opstack_pop(); - if (compound_op != OP_generic) { - var_t *current; - if (field && is_bitfield(field)) { - current = read_bitfield_value(parent, bb, address, field); - } else { - current = require_typed_var(parent, value_type); - current->ptr_level = value_ptr_level; - current->var_name = gen_name(); - add_insn(parent, *bb, OP_read, current, address, NULL, - store_size, NULL); - } - /* A pointer object steps by its element size. */ - if (is_pointer_operation(compound_op, current, value)) { - handle_pointer_arithmetic(parent, bb, compound_op, current, - value); - value = opstack_pop(); - } else { - current = integer_promote_operand(parent, bb, current); - value = integer_promote_operand(parent, bb, value); - normalize_integer_binary_operands(parent, bb, compound_op, - ¤t, &value); - var_t *combined = require_var(parent); - combined->var_name = gen_name(); - combined->type = - integer_binary_result_type(compound_op, current, value); - add_insn(parent, *bb, compound_op, combined, current, value, 0, - NULL); - value = combined; - } - } - - if (field && is_bitfield(field)) - write_bitfield_value(parent, bb, address, value, field); - else if (is_record_type(value_type) && !value_ptr_level) { + lvalue_t target = { + .size = store_size, + .value_ptr_level = value_ptr_level, + .type = value_type, + .decl = field, + }; + value = combine_addressed_assignment(parent, bb, compound_op, address, + &target, value); + + if (!is_bitfield(field) && is_record_type(value_type) && + !value_ptr_level) { if (compound_op != OP_generic || !is_record_object(value)) error_at("Invalid record assignment", cur_token_loc()); emit_record_copy_to_address(parent, bb, address, value); opstack_push(value); return true; } else { - value = convert_stored_value(parent, bb, value, value_type, - value_ptr_level); - add_insn(parent, *bb, OP_write, NULL, address, value, store_size, - NULL); + store_addressed_assignment(parent, bb, address, &target, value, + false); } /* Reload after the store: assignment expressions observe the stored * object representation, including narrow integer conversion. */ - if (field && is_bitfield(field)) { - result = reload_assigned_bitfield(parent, bb, address, field); - } else { - result = require_typed_var(parent, value_type); - result->ptr_level = value_ptr_level; - result->var_name = gen_name(); - result->is_assignment_reload = true; - add_insn(parent, *bb, OP_read, result, address, NULL, store_size, - NULL); - } + result = read_addressed_value(parent, bb, address, &target, true, true); if (is_bool_scalar(result->type, result->ptr_level)) result = normalize_bool(parent, bb, result); opstack_push(result); diff --git a/src/parser-global.c b/src/parser-global.c index 3580ed99..4b7370b5 100644 --- a/src/parser-global.c +++ b/src/parser-global.c @@ -106,6 +106,20 @@ static bool identifier_list_param_matches(const var_t *defined, return compatible_function_param_decl(defined, declared); } +static void error_named_declaration(const char *message, const var_t *var) +{ + char diagnostic[MAX_LINE_LEN]; + + snprintf(diagnostic, sizeof(diagnostic), "%s '%s'", message, + file_scope_var_name(var)); + error_at(diagnostic, next_token_loc()); +} + +static void error_conflicting_function_declaration(const var_t *var) +{ + error_named_declaration("conflicting types for function declaration", var); +} + /* A declarator's base type is already known when this runs. Keeping function * completion independent of how that type was spelled lets enum, record, and * ordinary scalar declarations share linkage and redeclaration checks. @@ -121,12 +135,30 @@ bool read_global_function_declarator(block_t *block, * redeclaration. Without this check the back end emitted colliding labels * and the resulting program could jump through object storage. */ - var_t *object = find_global_var(var->var_name); - + const char *source_name = file_scope_var_name(var); + var_t *unit_decl = find_tu_declaration(source_name); + var_t *object = + unit_decl && !find_func(unit_decl->var_name) + ? unit_decl + : (!is_static && !unit_decl ? find_global_var(source_name) : NULL); + + if (find_block_ordinary(CURRENT_TU_SCOPE, var->var_name, + ORDINARY_CONSTANT | ORDINARY_TYPEDEF, NULL)) + error_at("function declaration conflicts with an ordinary identifier", + next_token_loc()); if (object && object != var) error_at("function declaration conflicts with global object", next_token_loc()); + if (unit_decl && find_func(unit_decl->var_name) && unit_decl->is_static) { + var->source_name = unit_decl->source_name; + var->var_name = unit_decl->var_name; + } else if (is_static && !unit_decl) { + assign_internal_linkage_name(var, source_name); + } + + add_tu_global_var(var); + func_t *func = find_func(var->var_name); func_t func_tmp; bool check_decl = false; @@ -169,7 +201,6 @@ bool read_global_function_declarator(block_t *block, next_token_loc()); func->is_static = check_decl && func_tmp.is_static ? true : is_static; func->is_inline = var->is_inline; - var_reset_subscripts(&func->return_def); block->locals.size--; /* Parse this declarator independently of any earlier declaration. A later @@ -203,19 +234,16 @@ bool read_global_function_declarator(block_t *block, func->return_def.ptr_level != func_tmp.return_def.ptr_level || func->return_def.is_const_qualified != func_tmp.return_def.is_const_qualified) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); if (func->has_prototype && func_tmp.has_prototype) { if (func->num_params != func_tmp.num_params || func->va_args != func_tmp.va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) { const var_t *now = &func->param_defs[i]; const var_t *before = &func_tmp.param_defs[i]; if (!compatible_function_param_decl(now, before)) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); } } else if (func->has_prototype && !func_tmp.has_prototype && func_tmp.num_params) { @@ -223,13 +251,11 @@ bool read_global_function_declarator(block_t *block, * promoted parameter types. */ if (func->num_params != func_tmp.num_params || func->va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) if (!identifier_list_param_matches(&func_tmp.param_defs[i], &func->param_defs[i])) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); } else if (strict_c99 && func->has_prototype && !func_tmp.has_prototype) { /* A variadic prototype and parameters promoted from char, short, or @@ -238,26 +264,22 @@ bool read_global_function_declarator(block_t *block, * mode, where existing old-style sources rely on it. */ if (func->va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) if (parameter_changes_under_default_promotion( &func->param_defs[i])) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); } else if (!func->has_prototype && func->num_params && func_tmp.has_prototype) { /* An old-style definition of a function declared with a prototype * keeps its own parameter names and that prototype. */ if (func->num_params != func_tmp.num_params || func_tmp.va_args) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); for (int i = 0; i < func->num_params; i++) if (!identifier_list_param_matches(&func->param_defs[i], &func_tmp.param_defs[i])) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); func->has_prototype = true; } else if (!func->has_prototype && func_tmp.has_prototype) { /* An empty-list definition has no named parameters. It cannot @@ -267,8 +289,7 @@ bool read_global_function_declarator(block_t *block, */ if (lex_peek(T_open_curly, NULL) && (func_tmp.num_params || func_tmp.va_args)) - error_at("conflicting types for function declaration", - next_token_loc()); + error_conflicting_function_declaration(var); /* A prior prototype remains visible after a compatible `f()` * declaration and still constrains subsequent calls. @@ -296,7 +317,7 @@ bool read_global_function_declarator(block_t *block, error_at("function definition cannot take its type from a typedef", next_token_loc()); if (check_decl && func_tmp.bbs) - error_at("redefinition of function", next_token_loc()); + error_named_declaration("redefinition of function", var); if (is_incomplete_record_object(&func->return_def)) error_at("function definition cannot return incomplete record type", next_token_loc()); @@ -334,19 +355,35 @@ var_t *resolve_global_declarator(block_t *block, bool is_static, bool *is_redeclaration) { + const char *source_name = file_scope_var_name(var); + var_t *unit_decl = find_tu_declaration(source_name); var_t *previous = NULL; *is_redeclaration = false; + if (find_block_ordinary(CURRENT_TU_SCOPE, var->var_name, + ORDINARY_CONSTANT | ORDINARY_TYPEDEF, NULL)) + error_at( + "global object declaration conflicts with an ordinary identifier", + next_token_loc()); + /* The ordinary identifier namespace is shared with functions. This is * intentionally before object redeclaration handling: a function is not a * compatible tentative definition of an object, even when both happen to * have the same declared scalar type. */ - if (find_func(var->var_name)) + if ((unit_decl && find_func(unit_decl->var_name)) || + (!is_static && !unit_decl && find_func(var->var_name))) error_at("global object declaration conflicts with function", next_token_loc()); + if (unit_decl && unit_decl->is_static) { + var->source_name = unit_decl->source_name; + var->var_name = unit_decl->var_name; + } else if (is_static && !unit_decl) { + assign_internal_linkage_name(var, source_name); + } + for (int i = 0; i + 1 < block->locals.size; i++) { var_t *candidate = block->locals.elements[i]; if (!strcmp(candidate->var_name, var->var_name)) { @@ -354,8 +391,10 @@ var_t *resolve_global_declarator(block_t *block, break; } } - if (!previous) + if (!previous) { + add_tu_global_var(var); return var; + } *is_redeclaration = true; @@ -386,12 +425,13 @@ var_t *resolve_global_declarator(block_t *block, previous->pointee_array_dim4 != var->pointee_array_dim4 || previous->is_const_qualified != var->is_const_qualified || previous->is_volatile != var->is_volatile) - error_at("conflicting types for global declaration", next_token_loc()); + error_named_declaration("conflicting types for global declaration", + var); if (!previous->is_static && is_static) error_at("static declaration follows non-static declaration", next_token_loc()); if (lex_peek(T_assign, NULL) && previous->has_initializer) - error_at("redefinition of global variable", next_token_loc()); + error_named_declaration("redefinition of global variable", var); /* A later bound completes the shared object's type. */ if (previous->has_unsized_array && !var->has_unsized_array) { @@ -406,9 +446,53 @@ var_t *resolve_global_declarator(block_t *block, if (operand_stack_idx && operand_stack[operand_stack_idx - 1] == var) operand_stack[operand_stack_idx - 1] = previous; block->locals.size--; + add_tu_global_var(previous); return previous; } +static var_t *prepare_global_object_definition(block_t *block, + var_t *var, + bool is_static) +{ + bool is_redeclaration; + bool is_definition = !var->is_extern; + + if (is_definition && is_incomplete_record_object(var)) + error_at("Incomplete struct/union type cannot define an object", + cur_token_loc()); + var = resolve_global_declarator(block, var, is_static, &is_redeclaration); + if (is_definition && (!is_redeclaration || var->is_extern)) { + var->is_extern = false; + add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, var, NULL, NULL, 0, NULL); + } + return var; +} + +static var_t *read_global_declarator_start(block_t *block, + type_t *decl_type, + bool is_const, + bool is_static, + bool is_volatile, + bool is_extern, + bool is_inline, + bool full) +{ + var_t *var = + full ? require_var(block) : require_typed_var(block, decl_type); + + var->is_global = true; + var->is_static = is_static; + var->is_inline = is_inline; + var->is_const_qualified = is_const; + var->is_volatile = is_volatile; + if (full) + read_full_var_decl(var, false, false, false); + else + read_inner_var_decl(var, false, false, false); + var->is_extern = is_extern && !lex_peek(T_assign, NULL); + return var; +} + /* Read one declarator after the first in a global declaration. Each shares the * declaration's base type: "int a = 1, b, c = 3;". */ @@ -420,34 +504,19 @@ bool read_global_declarator(block_t *block, bool is_extern, bool allow_definition) { - bool is_redeclaration; - var_t *nv = require_typed_var(block, decl_type); - nv->is_global = true; - nv->is_static = is_static; - nv->is_const_qualified = is_const; - nv->is_volatile = is_volatile; - read_inner_var_decl(nv, false, false, false); - nv->is_extern = is_extern && !lex_peek(T_assign, NULL); + var_t *nv = + read_global_declarator_start(block, decl_type, is_const, is_static, + is_volatile, is_extern, false, false); if (lex_peek(T_open_bracket, NULL) || (nv->is_func && nv->type->is_direct_function_type)) return read_global_function_declarator(block, nv, is_static, allow_definition); - bool is_definition = !nv->is_extern; - if (is_definition && is_incomplete_record_object(nv)) - error_at("Incomplete struct/union type cannot define an object", - cur_token_loc()); - nv = resolve_global_declarator(block, nv, is_static, &is_redeclaration); - if (is_definition && (!is_redeclaration || nv->is_extern)) { - nv->is_extern = false; - add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, nv, NULL, NULL, 0, NULL); - } + nv = prepare_global_object_definition(block, nv, is_static); read_global_init_var(nv, block); discard_global_declarator_operand(nv); return false; } -void consume_global_compound_literal(void); - /* Lower a scalar record initializer into the global initializer block. Global * array elements already use parse_struct_field_init(); scalar records need the * same field-address writes rather than merely consuming their braces. @@ -483,7 +552,7 @@ void parse_global_compound_record_init(var_t *var, block_t *block) compound_type = record_kind ? find_record_tag(type_name, var->scope, record_kind) - : find_type(type_name, 1); + : find_visible_type(type_name, var->scope); target_type = var->type; if (target_type->base_type == TYPE_typedef && target_type->base_struct) target_type = target_type->base_struct; @@ -540,6 +609,8 @@ void parse_global_compound_array_init(var_t *var, block_t *block) type_t *element_type; var_t *array; int element_ptr_level = 0; + paren_type_name_t shape = {0}; + bool parenthesized_array; /* The element type may be qualified or spelled with several keywords, as in * `(const unsigned char[])`. @@ -549,148 +620,62 @@ void parse_global_compound_array_init(var_t *var, block_t *block) read_type_name_specifiers(var->scope ? var->scope : GLOBAL_BLOCK); while (lex_accept(T_asterisk)) { element_ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } - /* `(int (*[])[2]){...}` is an array whose elements are pointers to rows. - * The inner suffix supplies the backing array bound; the suffixes after `)` - * describe each pointer element's pointee. - */ - if (lex_accept(T_open_bracket)) { - int pointee_dims = 0; - - if (element_ptr_level || !lex_peek(T_asterisk, NULL)) - error_at("Array compound literal needs a pointer declarator", - cur_token_loc()); - do { - lex_expect(T_asterisk); - element_ptr_level++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } while (lex_peek(T_asterisk, NULL)); - lex_expect(T_open_square); - - array = require_typed_var(GLOBAL_BLOCK, element_type); - array->var_name = gen_name(); - array->is_global = true; - array->ptr_level = element_ptr_level; - if (!lex_peek(T_close_square, NULL)) { - array->array_size = read_const_expr(GLOBAL_BLOCK); - if (array->array_size <= 0) - error_at("Array compound literal needs a positive bound", - cur_token_loc()); - } else { - array->has_unsized_array = true; - } - lex_expect(T_close_square); - lex_expect(T_close_bracket); - while (lex_accept(T_open_square)) { - int bound = read_const_expr(GLOBAL_BLOCK); - - if (pointee_dims >= 4) - error_at("Array declarators support at most four dimensions", - cur_token_loc()); - if (bound <= 0) - error_at("Array size must be positive", cur_token_loc()); - if (pointee_dims == 0) - array->pointee_array_size = bound; - else { - if (pointee_dims == 1) - array->pointee_array_dim2 = bound; - else if (pointee_dims == 2) - array->pointee_array_dim3 = bound; - else - array->pointee_array_dim4 = bound; - array->pointee_array_size *= bound; - } - lex_expect(T_close_square); - pointee_dims++; - } + parenthesized_array = read_compound_array_shape( + GLOBAL_BLOCK, element_ptr_level, true, &shape) && + shape.parenthesized_array; + array = name_var(require_typed_var(GLOBAL_BLOCK, element_type)); + array->is_global = true; + array->ptr_level = + parenthesized_array ? shape.array_element_ptr_level : element_ptr_level; + array->array_size = shape.array_size; + array->array_dim2 = shape.array_dim2; + array->array_dim3 = shape.array_dim3; + array->array_dim4 = shape.array_dim4; + array->has_unsized_array = shape.array_outer_unsized; + if (parenthesized_array) { + array->pointee_array_size = shape.pointee_array_size; + array->pointee_array_dim2 = shape.pointee_array_dim2; + array->pointee_array_dim3 = shape.pointee_array_dim3; + array->pointee_array_dim4 = shape.pointee_array_dim4; lex_expect(T_close_bracket); if (!element_type || element_type != var->type || - var->ptr_level != element_ptr_level + 1 || + var->ptr_level != array->ptr_level + 1 || var->pointee_array_size != array->pointee_array_size || var->pointee_array_dim2 != array->pointee_array_dim2 || var->pointee_array_dim3 != array->pointee_array_dim3 || var->pointee_array_dim4 != array->pointee_array_dim4) error_at("Incompatible array compound literal", cur_token_loc()); - if (!lex_peek(T_open_curly, NULL)) - error_at("Array compound literal needs an initializer", - next_token_loc()); - add_insn(GLOBAL_BLOCK, GLOBAL_FUNC->bbs, OP_allocat, array, NULL, NULL, - 0, NULL); - parse_array_init(array, GLOBAL_BLOCK, &GLOBAL_FUNC->bbs); - add_insn(block, GLOBAL_FUNC->bbs, OP_assign, var, array, NULL, 0, NULL); - return; - } - array = require_typed_var(GLOBAL_BLOCK, element_type); - array->var_name = gen_name(); - array->is_global = true; - array->ptr_level = element_ptr_level; - for (int dim = 0; lex_accept(T_open_square); dim++) { - int bound; + } else { + lex_expect(T_close_bracket); - if (dim >= 4) - error_at("Array compound literal supports at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) { - if (dim) - error_at("Only the outer array bound may be inferred", - cur_token_loc()); - array->has_unsized_array = true; - lex_expect(T_close_square); - continue; - } - bound = read_const_expr(GLOBAL_BLOCK); - if (bound <= 0) - error_at("Array compound literal needs a positive bound", - cur_token_loc()); - if (!dim) - array->array_size = bound; - else { - if (dim == 1) - array->array_dim2 = bound; - else if (dim == 2) - array->array_dim3 = bound; - else - array->array_dim4 = bound; - array->array_size *= bound; + /* The backing array has an outer compound-literal bound and the typedef + * element's inner row shape. parse_array_init() keeps that shape on + * var_t rather than type_t. + */ + if (element_type && element_type->array_size) { + int trailing = fixed_array_trailing_count(element_type->array_dim2, + element_type->array_dim3, + element_type->array_dim4); + if (element_type->array_dim4) + error_at( + "Array compound literal supports at most four dimensions", + cur_token_loc()); + array->array_dim2 = element_type->array_size / trailing; + array->array_dim3 = element_type->array_dim2; + array->array_dim4 = element_type->array_dim3; + if (array->array_size) + array->array_size *= element_type->array_size; } - lex_expect(T_close_square); - } - lex_expect(T_close_bracket); - /* The backing array has an outer compound-literal bound and the typedef - * element's inner row shape. parse_array_init() keeps that shape on var_t - * rather than type_t. - */ - if (element_type && element_type->array_size) { - int trailing = 1; - - if (element_type->array_dim2) - trailing *= element_type->array_dim2; - if (element_type->array_dim3) - trailing *= element_type->array_dim3; - if (element_type->array_dim4) - trailing *= element_type->array_dim4; - if (element_type->array_dim4) - error_at("Array compound literal supports at most four dimensions", - cur_token_loc()); - array->array_dim2 = element_type->array_size / trailing; - array->array_dim3 = element_type->array_dim2; - array->array_dim4 = element_type->array_dim3; - if (array->array_size) - array->array_size *= element_type->array_size; + if (!element_type || element_type != var->type || + element_ptr_level + 1 != var->ptr_level || + var->pointee_array_size != array->array_dim2) + error_at("Incompatible array compound literal", cur_token_loc()); } - - if (!element_type || element_type != var->type || - element_ptr_level + 1 != var->ptr_level || - var->pointee_array_size != array->array_dim2) - error_at("Incompatible array compound literal", cur_token_loc()); if (!lex_peek(T_open_curly, NULL)) error_at("Array compound literal needs an initializer", next_token_loc()); @@ -701,64 +686,6 @@ void parse_global_compound_array_init(var_t *var, block_t *block) add_insn(block, GLOBAL_FUNC->bbs, OP_assign, var, array, NULL, 0, NULL); } -/* Struct and union objects accept brace initializers, unlike scalar globals. - * Keep their continuation declarators on the same path as the first one so that - * linkage, qualifiers, and declarator-specific modifiers cannot diverge. - */ -bool read_global_record_declarator(block_t *block, - type_t *decl_type, - bool is_const, - bool is_static, - bool is_volatile, - bool is_extern, - bool allow_definition) -{ - bool is_redeclaration; - var_t *var = require_typed_var(block, decl_type); - var->is_global = true; - var->is_static = is_static; - var->is_const_qualified = is_const; - var->is_volatile = is_volatile; - read_inner_var_decl(var, false, false, false); - var->is_extern = is_extern && !lex_peek(T_assign, NULL); - if (lex_peek(T_open_bracket, NULL) || - (var->is_func && var->type->is_direct_function_type)) - return read_global_function_declarator(block, var, is_static, - allow_definition); - bool is_definition = !var->is_extern; - if (is_definition && is_incomplete_record_object(var)) - error_at("Incomplete struct/union type cannot define an object", - cur_token_loc()); - var = resolve_global_declarator(block, var, is_static, &is_redeclaration); - if (is_definition && (!is_redeclaration || var->is_extern)) { - var->is_extern = false; - add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, var, NULL, NULL, 0, NULL); - } - - if (!lex_accept(T_assign)) { - discard_global_declarator_operand(var); - return false; - } - - var->has_initializer = true; - - if (lex_peek(T_open_curly, NULL) && - (var->array_size > 0 || var->has_unsized_array || var->ptr_level > 0)) { - parse_array_init(var, block, &GLOBAL_FUNC->bbs); - } else if (global_compound_literal_starts_here() && - (var->ptr_level || var->type->ptr_level)) { - parse_global_compound_array_init(var, block); - } else if (global_compound_literal_starts_here()) { - parse_global_compound_record_init(var, block); - } else if (lex_peek(T_open_curly, NULL)) { - parse_global_record_init(var, block); - } else { - read_global_assignment_var(var); - } - discard_global_declarator_operand(var); - return false; -} - /* Read the declarators of a file-scope declaration through its terminating * semicolon, or through the body of a function definition. Every declarator * shares the base type, and object and function declarators may be mixed, as in @@ -771,22 +698,14 @@ void read_global_declarator_list(block_t *block, bool is_const, bool is_static, bool is_volatile, - bool is_extern, - bool is_record) + bool is_extern) { bool first = true; read_type_qualifiers(&is_const, &is_volatile, false); do { - bool ended = - is_record - ? read_global_record_declarator(block, decl_type, is_const, - is_static, is_volatile, - is_extern, first) - : read_global_declarator(block, decl_type, is_const, is_static, - is_volatile, is_extern, first); - - if (ended) + if (read_global_declarator(block, decl_type, is_const, is_static, + is_volatile, is_extern, first)) return; first = false; } while (lex_accept(T_comma)); @@ -800,17 +719,9 @@ void read_global_decl(block_t *block, bool is_inline, bool is_volatile) { - bool is_redeclaration; - var_t *var = require_var(block); - var->is_global = true; - var->is_static = is_static; - var->is_inline = is_inline; - var->is_const_qualified = is_const; - var->is_volatile = is_volatile; - - /* new function, or variables under parent */ - read_full_var_decl(var, false, false, false); - var->is_extern = is_extern && !lex_peek(T_assign, NULL); + var_t *var = + read_global_declarator_start(block, NULL, is_const, is_static, + is_volatile, is_extern, is_inline, true); /* `unary_t f;` with a function typedef declares the function f. */ if (lex_peek(T_open_bracket, NULL) || @@ -818,20 +729,10 @@ void read_global_decl(block_t *block, if (read_global_function_declarator(block, var, is_static, true)) return; } else { - bool is_definition = !var->is_extern; if (var->is_inline) error_at("inline specifier requires a function declarator", next_token_loc()); - if (is_definition && is_incomplete_record_object(var)) - error_at("Incomplete struct/union type cannot define an object", - cur_token_loc()); - var = - resolve_global_declarator(block, var, is_static, &is_redeclaration); - if (is_definition && (!is_redeclaration || var->is_extern)) { - var->is_extern = false; - add_insn(block, GLOBAL_FUNC->bbs, OP_allocat, var, NULL, NULL, 0, - NULL); - } + var = prepare_global_object_definition(block, var, is_static); /* is a variable */ if (lex_peek(T_assign, NULL)) { @@ -856,56 +757,9 @@ void read_global_decl(block_t *block, lex_expect(T_semicolon); } -void consume_global_compound_literal(void) -{ - lex_expect(T_open_curly); - - if (!lex_peek(T_close_curly, NULL)) { - for (;;) { - /* Just consume constant values for now */ - if (lex_peek(T_numeric, NULL)) { - lex_accept(T_numeric); - } else if (lex_peek(T_minus, NULL)) { - lex_accept(T_minus); - lex_accept(T_numeric); - } else if (lex_peek(T_string, NULL)) { - lex_accept(T_string); - } else if (lex_peek(T_char, NULL) || lex_peek(T_wchar, NULL)) { - lex_next(); - } else { - error_at( - "Global struct initialization requires constant values", - next_token_loc()); - } - - if (!lex_accept(T_comma)) - break; - if (lex_peek(T_close_curly, NULL)) - break; - } - } - lex_expect(T_close_curly); -} - void initialize_struct_field(var_t *nv, var_t *v, int offset) { - nv->type = v->type; - nv->var_name = ""; - nv->ptr_level = 0; - nv->is_func = false; - nv->is_global = false; - nv->is_const_qualified = false; - nv->array_size = 0; - nv->offset = offset; - nv->is_bitfield = false; - nv->bit_width = 0; - nv->bit_offset = 0; - nv->bit_storage_size = 0; - nv->init_val = 0; - nv->base = NULL; - nv->subscript = 0; - var_reset_subscripts(nv); - nv->is_compound_literal = false; + *nv = (var_t) {.type = v->type, .var_name = "", .offset = offset}; } /* The scalar or typedef-name specifier of a file-scope typedef, and any @@ -923,7 +777,7 @@ static const type_t *read_global_typedef_base(bool *typedef_const, if (!base) { lex_ident(T_identifier, base_type); - base = find_type(base_type, true); + base = find_visible_type(base_type, CURRENT_TU_SCOPE); } if (!base) error_at("Unable to find base type", cur_token_loc()); @@ -960,9 +814,18 @@ static void read_global_typedef_declarator(block_t *block, type->has_flexible_array_member = base->has_flexible_array_member; type->ptr_level = base->ptr_level; type->pointer_const_mask = base->pointer_const_mask; + type->pointer_volatile_mask = base->pointer_volatile_mask; type->is_const_qualified = typedef_const || base->is_const_qualified; - type->is_volatile_qualified = - typedef_volatile || base->is_volatile_qualified; + type->is_volatile_qualified = base->is_volatile_qualified; + if (typedef_volatile) { + int depth = base->ptr_level + (base->func_signature && + !base->is_direct_function_type && + !base->ptr_level); + if (depth && depth <= 32) + type->pointer_volatile_mask |= 1U << (depth - 1); + else + type->is_volatile_qualified = true; + } /* `const ptr_t` qualifies the pointer that the base typedef hides, not its * pointee. @@ -1007,10 +870,8 @@ static void read_global_typedef_declarator(block_t *block, star_const_mask |= 1U << (type->ptr_level - 1); } } else if (lex_accept(T_volatile)) { - /* As for an object declarator, a volatile pointer marks the - * whole declaration volatile. - */ - type->is_volatile_qualified = true; + if (type->ptr_level <= 32) + type->pointer_volatile_mask |= 1U << (type->ptr_level - 1); } else if (lex_accept(T_restrict)) { ; } else @@ -1031,9 +892,22 @@ static void read_global_typedef_declarator(block_t *block, var_t declarator = {0}; bool saved_sizeof_signature = parsing_sizeof_function_signature; - declarator.type = (type_t *) base; + init_type_name_decl(&declarator, (type_t *) base, typedef_const, + typedef_volatile); declarator.scope = block; - declarator.ptr_level = type->ptr_level; + declarator.ptr_level = type->ptr_level - base->ptr_level; + declarator.pointer_const_mask = + base->ptr_level < 32 ? type->pointer_const_mask >> base->ptr_level + : 0; + declarator.pointer_volatile_mask = type->pointer_volatile_mask; + /* A returned compact callback keeps its own qualifiers on base. */ + if (base->func_signature && !base->is_direct_function_type && + !type->ptr_level) { + declarator.pointer_const_mask = 0; + declarator.pointer_volatile_mask = 0; + } + declarator.is_const_qualified = type->is_const_qualified; + declarator.is_volatile = type->is_volatile_qualified; /* A callback typedef carries only a function signature. Its floating * parameters do not materialize values until a call, which remains @@ -1051,14 +925,18 @@ static void read_global_typedef_declarator(block_t *block, !base->func_signature && !base->array_size) { strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); type->type_name[MAX_TYPE_LEN - 1] = '\0'; - type->ptr_level = declarator.ptr_level; + type->ptr_level = effective_pointer_depth(&declarator); type->size = PTR_SIZE; - type->pointer_const_mask |= declarator.pointer_const_mask; + if (base->ptr_level < 32) + type->pointer_const_mask |= declarator.pointer_const_mask + << base->ptr_level; + type->pointer_volatile_mask |= declarator.pointer_volatile_mask; if (declarator.is_volatile) type->is_volatile_qualified = true; /* `typedef int (**slot_t)(int)` names a callback slot. */ type->pointee_func_signature = declarator.pointee_func_signature; + add_tu_typedef(type); return; } @@ -1077,9 +955,10 @@ static void read_global_typedef_declarator(block_t *block, (!base->func_signature || (!base->is_direct_function_type && !base->ptr_level)) && !base->pointee_func_signature && - declarator.ptr_level == - declarator.pointee_array_element_ptr_level + 1 && - declarator.pointee_array_element_ptr_level <= 1) { + effective_pointer_depth(&declarator) == + declarator.pointee_array_element_ptr_level + base->ptr_level + + 1 && + declarator.pointee_array_element_ptr_level + base->ptr_level <= 1) { strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); type->type_name[MAX_TYPE_LEN - 1] = '\0'; @@ -1094,19 +973,19 @@ static void read_global_typedef_declarator(block_t *block, * `int *(*rows_t)[2]`: the declarator already counts its stars on * the row's element, which then points to the base's pointee. */ - type->ptr_level = declarator.ptr_level; + type->ptr_level = effective_pointer_depth(&declarator); type->size = PTR_SIZE; - type->pointer_const_mask |= declarator.pointer_const_mask; - type->pointee_array_size = declarator.pointee_array_size; - type->pointee_array_dim2 = declarator.pointee_array_dim2; - type->pointee_array_dim3 = declarator.pointee_array_dim3; - type->pointee_array_dim4 = declarator.pointee_array_dim4; - type->pointee_array_element_ptr_level = - declarator.pointee_array_element_ptr_level; + if (base->ptr_level < 32) + type->pointer_const_mask |= declarator.pointer_const_mask + << base->ptr_level; + type->pointer_volatile_mask |= declarator.pointer_volatile_mask; + declarator.pointee_array_element_ptr_level += base->ptr_level; + copy_pointee_array_shape_to_type(type, &declarator); type->pointee_array_element_type = base->ptr_level ? pointee_type_from_pointer_typedef((type_t *) base) : (type_t *) base; + add_tu_typedef(type); return; } if (!declarator.is_func) @@ -1122,6 +1001,7 @@ static void read_global_typedef_declarator(block_t *block, memcpy(type, declarator.type, sizeof(type_t)); strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); type->type_name[MAX_TYPE_LEN - 1] = '\0'; + add_tu_typedef(type); return; } strncpy(type->type_name, declarator.var_name, MAX_TYPE_LEN - 1); @@ -1146,9 +1026,18 @@ static void read_global_typedef_declarator(block_t *block, * signature; the typedef alias itself is the pointer-sized callback * object, not a derived pointer alias. */ + int return_depth = declarator.ptr_level + declarator.type->ptr_level; type->ptr_level = 0; - type->pointer_const_mask = declarator.pointer_const_mask; - type->is_volatile_qualified = declarator.is_volatile; + type->pointer_const_mask = + declarator.ptr_level < 32 + ? declarator.pointer_const_mask >> declarator.ptr_level + : 0; + type->pointer_volatile_mask = + return_depth < 32 ? declarator.pointer_volatile_mask >> return_depth + : 0; + type->is_const_qualified = false; + type->is_volatile_qualified = false; + add_tu_typedef(type); return; } @@ -1167,14 +1056,17 @@ static void read_global_typedef_declarator(block_t *block, func->return_def.type = (type_t *) base; func->return_def.ptr_level = type->ptr_level; func->return_def.pointer_const_mask = type->pointer_const_mask; + func->return_def.pointer_volatile_mask = type->pointer_volatile_mask; func->return_def.scope = block; type->ptr_level = 0; type->pointer_const_mask = 0; + type->pointer_volatile_mask = 0; type->size = base->size; read_parameter_list_decl(func, true); type->base_type = TYPE_typedef; type->func_signature = func; type->is_direct_function_type = true; + add_tu_typedef(type); return; } @@ -1184,20 +1076,10 @@ static void read_global_typedef_declarator(block_t *block, */ fixed_array_shape_t base_shape = fixed_array_shape_from_type(type); fixed_array_shape_t decl_shape = {0}; - while (lex_accept(T_open_square)) { - int bound; - - if (decl_shape.rank >= MAX_FIXED_ARRAY_RANK) - error_at("Array declarators support at most four dimensions", - cur_token_loc()); - if (lex_peek(T_close_square, NULL)) - error_at("Typedef array needs a positive bound", cur_token_loc()); - bound = read_const_expr(block); - if (bound <= 0) - error_at("Typedef array needs a positive bound", cur_token_loc()); - decl_shape.bounds[decl_shape.rank++] = bound; - lex_expect(T_close_square); - } + read_array_shape_bounds( + block, &decl_shape, "Array declarators support at most four dimensions", + "Typedef array needs a positive bound", + "Typedef array needs a positive bound", true, false); if (decl_shape.rank) { fixed_array_shape_t shape = fixed_array_shape_prepend(&decl_shape, &base_shape); @@ -1221,6 +1103,7 @@ static void read_global_typedef_declarator(block_t *block, ? pointee_type_from_pointer_typedef((type_t *) base) : (type_t *) base); } + add_tu_typedef(type); } /* The declarator list of a file-scope typedef whose specifier resolved to @@ -1311,7 +1194,7 @@ static void read_global_typedef(block_t *block) if (lex_peek(T_open_curly, NULL)) record = read_record_body(NULL, kind, has_tag, token); else if (has_tag) - tag = local_record_tag(token, GLOBAL_BLOCK, kind); + tag = local_record_tag(token, CURRENT_TU_SCOPE, kind); read_type_qualifiers(&typedef_const, &typedef_volatile, false); bool is_plain = global_record_typedef_declarator_is_plain(); @@ -1350,6 +1233,7 @@ static void read_global_typedef(block_t *block) /* Only the alias is qualified; the tag must stay plain. */ type->is_const_qualified = typedef_const; type->is_volatile_qualified = typedef_volatile; + add_tu_typedef(type); if (!is_plain) { read_global_typedef_declarators(block, typedef_const, @@ -1387,7 +1271,7 @@ static void read_global_typedef(block_t *block) void read_global_statement(void) { char token[MAX_ID_LEN]; - block_t *block = GLOBAL_BLOCK; /* global block */ + block_t *block = GLOBAL_BLOCK; /* merged object/function namespace */ bool is_const = false; bool is_static = false; bool is_extern = false; @@ -1461,7 +1345,7 @@ void read_global_statement(void) /* variable declaration using existing record tag? */ if (!lex_peek(T_open_curly, NULL)) { - type = local_record_tag(token, GLOBAL_BLOCK, kind); + type = local_record_tag(token, CURRENT_TU_SCOPE, kind); /* A declaration with no declarator only declares the tag. At file * scope a repeated one names the same type, so it is valid whether @@ -1471,7 +1355,7 @@ void read_global_statement(void) return; read_global_declarator_list(block, type, is_const, is_static, - is_volatile, is_extern, true); + is_volatile, is_extern); return; } @@ -1482,7 +1366,7 @@ void read_global_statement(void) */ if (!lex_accept(T_semicolon)) read_global_declarator_list(block, type, is_const, is_static, - is_volatile, is_extern, true); + is_volatile, is_extern); } else if (lex_peek(T_enum, NULL)) { bool is_definition; type_t *type = read_enum_specifier(NULL, &is_definition); @@ -1497,7 +1381,7 @@ void read_global_statement(void) next_token_loc()); if (!is_definition || !lex_accept(T_semicolon)) read_global_declarator_list(block, type, is_const, is_static, - is_volatile, is_extern, false); + is_volatile, is_extern); } else if (lex_accept(T_typedef)) { read_global_typedef(block); } else if (lex_peek(T_identifier, NULL) || lex_peek(T_signed, NULL) || @@ -1508,6 +1392,21 @@ void read_global_statement(void) error_at("Syntax error in global statement", next_token_loc()); } +static type_t *add_builtin_scalar(const char *name, + base_type_t base_type, + int size, + bool is_unsigned, + bool is_signed_char) +{ + type_t *type = add_named_type((char *) name); + + type->base_type = base_type; + type->size = size; + type->is_unsigned = is_unsigned; + type->is_signed_char = is_signed_char; + return type; +} + void parse_internal(void) { /* set starting point of global stack manually */ @@ -1517,7 +1416,7 @@ void parse_internal(void) * occupy a full target pointer, not the historic 32-bit word. */ GLOBAL_FUNC->stack_size = PTR_SIZE; - GLOBAL_FUNC->bbs = arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); + GLOBAL_FUNC->bbs = arena_alloc_bb(); GLOBAL_FUNC->bbs->belong_to = GLOBAL_FUNC; /* Prevent nullptr deref in RA */ GLOBAL_FUNC->bbs->elf_offset = -1; /* not yet emitted */ @@ -1593,9 +1492,7 @@ void parse_internal(void) TY_ushort->is_unsigned = true; /* Unlike `long`, which deliberately shares the current 32-bit int ABI, long - * long has a distinct type and an eight-byte object representation. Parser - * admission and target lowering are staged separately so 32-bit backends - * never silently truncate it. + * long has a distinct type and an eight-byte object representation. */ TY_long_long = add_named_type("long long"); TY_long_long->base_type = TYPE_long_long; @@ -1610,14 +1507,10 @@ void parse_internal(void) * builtin type table so declarations, casts, sizeof, and prototypes use the * same pointer-width representation as the rest of the compiler. */ - type_t *TY_size = add_named_type("size_t"); - TY_size->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_size->size = PTR_SIZE; - TY_size->is_unsigned = true; - - type_t *TY_ptrdiff = add_named_type("ptrdiff_t"); - TY_ptrdiff->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_ptrdiff->size = PTR_SIZE; + add_builtin_scalar("size_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, true, false); + add_builtin_scalar("ptrdiff_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, false, false); /* belongs to C99's freestanding library. Its va_list is the * compiler ABI's int-based cursor, so retain that scalar base while @@ -1631,124 +1524,47 @@ void parse_internal(void) /* The execution wide-character type is int until wide literal lowering is * implemented; declaring the C99 typedef remains useful independently. */ - type_t *TY_wchar = add_named_type("wchar_t"); - TY_wchar->base_type = TYPE_int; - TY_wchar->size = TY_int->size; - - type_t *TY_sig_atomic = add_named_type("sig_atomic_t"); - TY_sig_atomic->base_type = TYPE_int; - TY_sig_atomic->size = TY_int->size; - - type_t *TY_wint = add_named_type("wint_t"); - TY_wint->base_type = TYPE_int; - TY_wint->size = TY_int->size; - TY_wint->is_unsigned = true; + add_builtin_scalar("wchar_t", TYPE_int, TY_int->size, false, false); + add_builtin_scalar("sig_atomic_t", TYPE_int, TY_int->size, false, false); + add_builtin_scalar("wint_t", TYPE_int, TY_int->size, true, false); /* C99 aliases share the target scalar representations. Keep them * named in the builtin table so declarations, casts, sizeof, and prototypes * use the same ABI metadata as their underlying types. */ - type_t *TY_int8 = add_named_type("int8_t"); - TY_int8->base_type = TYPE_char; - TY_int8->size = 1; - TY_int8->is_signed_char = true; - type_t *TY_uint8 = add_named_type("uint8_t"); - TY_uint8->base_type = TYPE_char; - TY_uint8->size = 1; - TY_uint8->is_unsigned = true; - type_t *TY_int16 = add_named_type("int16_t"); - TY_int16->base_type = TYPE_short; - TY_int16->size = 2; - type_t *TY_uint16 = add_named_type("uint16_t"); - TY_uint16->base_type = TYPE_short; - TY_uint16->size = 2; - TY_uint16->is_unsigned = true; - type_t *TY_int32 = add_named_type("int32_t"); - TY_int32->base_type = TYPE_int; - TY_int32->size = 4; - type_t *TY_uint32 = add_named_type("uint32_t"); - TY_uint32->base_type = TYPE_int; - TY_uint32->size = 4; - TY_uint32->is_unsigned = true; - type_t *TY_int64 = add_named_type("int64_t"); - TY_int64->base_type = TYPE_long_long; - TY_int64->size = 8; - type_t *TY_uint64 = add_named_type("uint64_t"); - TY_uint64->base_type = TYPE_long_long; - TY_uint64->size = 8; - TY_uint64->is_unsigned = true; - type_t *TY_intmax = add_named_type("intmax_t"); - TY_intmax->base_type = TYPE_long_long; - TY_intmax->size = 8; - type_t *TY_uintmax = add_named_type("uintmax_t"); - TY_uintmax->base_type = TYPE_long_long; - TY_uintmax->size = 8; - TY_uintmax->is_unsigned = true; - type_t *TY_intptr = add_named_type("intptr_t"); - TY_intptr->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_intptr->size = PTR_SIZE; - type_t *TY_uintptr = add_named_type("uintptr_t"); - TY_uintptr->base_type = PTR_SIZE == 8 ? TYPE_long_long : TYPE_long; - TY_uintptr->size = PTR_SIZE; - TY_uintptr->is_unsigned = true; - - type_t *TY_int_least8 = add_named_type("int_least8_t"); - TY_int_least8->base_type = TY_int8->base_type; - TY_int_least8->size = TY_int8->size; - TY_int_least8->is_signed_char = true; - type_t *TY_uint_least8 = add_named_type("uint_least8_t"); - TY_uint_least8->base_type = TY_uint8->base_type; - TY_uint_least8->size = TY_uint8->size; - TY_uint_least8->is_unsigned = true; - type_t *TY_int_least16 = add_named_type("int_least16_t"); - TY_int_least16->base_type = TYPE_short; - TY_int_least16->size = 2; - type_t *TY_uint_least16 = add_named_type("uint_least16_t"); - TY_uint_least16->base_type = TYPE_short; - TY_uint_least16->size = 2; - TY_uint_least16->is_unsigned = true; - type_t *TY_int_least32 = add_named_type("int_least32_t"); - TY_int_least32->base_type = TYPE_int; - TY_int_least32->size = 4; - type_t *TY_uint_least32 = add_named_type("uint_least32_t"); - TY_uint_least32->base_type = TYPE_int; - TY_uint_least32->size = 4; - TY_uint_least32->is_unsigned = true; - type_t *TY_int_least64 = add_named_type("int_least64_t"); - TY_int_least64->base_type = TYPE_long_long; - TY_int_least64->size = 8; - type_t *TY_uint_least64 = add_named_type("uint_least64_t"); - TY_uint_least64->base_type = TYPE_long_long; - TY_uint_least64->size = 8; - TY_uint_least64->is_unsigned = true; - type_t *TY_int_fast8 = add_named_type("int_fast8_t"); - TY_int_fast8->base_type = TYPE_int; - TY_int_fast8->size = 4; - type_t *TY_uint_fast8 = add_named_type("uint_fast8_t"); - TY_uint_fast8->base_type = TYPE_int; - TY_uint_fast8->size = 4; - TY_uint_fast8->is_unsigned = true; - type_t *TY_int_fast16 = add_named_type("int_fast16_t"); - TY_int_fast16->base_type = TYPE_int; - TY_int_fast16->size = 4; - type_t *TY_uint_fast16 = add_named_type("uint_fast16_t"); - TY_uint_fast16->base_type = TYPE_int; - TY_uint_fast16->size = 4; - TY_uint_fast16->is_unsigned = true; - type_t *TY_int_fast32 = add_named_type("int_fast32_t"); - TY_int_fast32->base_type = TYPE_int; - TY_int_fast32->size = 4; - type_t *TY_uint_fast32 = add_named_type("uint_fast32_t"); - TY_uint_fast32->base_type = TYPE_int; - TY_uint_fast32->size = 4; - TY_uint_fast32->is_unsigned = true; - type_t *TY_int_fast64 = add_named_type("int_fast64_t"); - TY_int_fast64->base_type = TYPE_long_long; - TY_int_fast64->size = 8; - type_t *TY_uint_fast64 = add_named_type("uint_fast64_t"); - TY_uint_fast64->base_type = TYPE_long_long; - TY_uint_fast64->size = 8; - TY_uint_fast64->is_unsigned = true; + type_t *TY_int8 = add_builtin_scalar("int8_t", TYPE_char, 1, false, true); + type_t *TY_uint8 = add_builtin_scalar("uint8_t", TYPE_char, 1, true, false); + add_builtin_scalar("int16_t", TYPE_short, 2, false, false); + add_builtin_scalar("uint16_t", TYPE_short, 2, true, false); + add_builtin_scalar("int32_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint32_t", TYPE_int, 4, true, false); + add_builtin_scalar("int64_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uint64_t", TYPE_long_long, 8, true, false); + add_builtin_scalar("intmax_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uintmax_t", TYPE_long_long, 8, true, false); + add_builtin_scalar("intptr_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, false, false); + add_builtin_scalar("uintptr_t", PTR_SIZE == 8 ? TYPE_long_long : TYPE_long, + PTR_SIZE, true, false); + + add_builtin_scalar("int_least8_t", TY_int8->base_type, TY_int8->size, false, + true); + add_builtin_scalar("uint_least8_t", TY_uint8->base_type, TY_uint8->size, + true, false); + add_builtin_scalar("int_least16_t", TYPE_short, 2, false, false); + add_builtin_scalar("uint_least16_t", TYPE_short, 2, true, false); + add_builtin_scalar("int_least32_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_least32_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_least64_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uint_least64_t", TYPE_long_long, 8, true, false); + add_builtin_scalar("int_fast8_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_fast8_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_fast16_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_fast16_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_fast32_t", TYPE_int, 4, false, false); + add_builtin_scalar("uint_fast32_t", TYPE_int, 4, true, false); + add_builtin_scalar("int_fast64_t", TYPE_long_long, 8, false, false); + add_builtin_scalar("uint_fast64_t", TYPE_long_long, 8, true, false); /* builtin type _Bool was introduced in C99 specification, it is more * well-known as macro type bool, which is defined in (in @@ -1759,35 +1575,27 @@ void parse_internal(void) TY_bool->size = 1; TY_bool->is_bool = true; + builtin_types_idx = types_idx; + GLOBAL_BLOCK = add_block(NULL, NULL); /* global block */ - elf_add_symbol("", 0); /* undef symbol */ + CURRENT_TU_SCOPE = add_block(NULL, NULL); + reset_tu_ordinary_index(); + elf_add_symbol("", 0); /* undef symbol */ if (dynlink) { - /* In dynamic mode, __syscall won't be implemented. - * - * Simply declare a 'syscall' function as follows if the program needs - * to use 'syscall': - * - * int syscall(int number, ...); - * - * shecc will treat it as an external function, and the compiled program - * will eventually use the implementation provided by the external C - * library. - * - * If shecc supports the 'long' data type in the future, it would be - * better to declare syscall using its original prototype: - * - * long syscall(long number, ...); + /* Dynamic callers declare the libc entry with a machine-word ABI: + * intptr_t syscall(intptr_t number, ...); intptr_t preserves + * pointer-valued results on every target. */ } else { /* Linux syscall */ func_t *func = add_func("__syscall", true); - func->return_def.type = TY_int; + func->return_def.type = PTR_SIZE == 8 ? TY_long_long : TY_int; func->num_params = 0; func->va_args = 1; func->bbs = NULL; /* Otherwise, allocate a basic block to implement in static mode. */ - func->bbs = arena_calloc(BB_ARENA, 1, sizeof(basic_block_t)); + func->bbs = arena_alloc_bb(); func->bbs->elf_offset = -1; /* not yet emitted */ } @@ -1801,6 +1609,11 @@ void parse_internal(void) /* lexer initialization */ do { + if (lex_accept(T_translation_unit)) { + CURRENT_TU_SCOPE = add_block(NULL, NULL); + reset_tu_ordinary_index(); + continue; + } read_global_statement(); } while (!lex_accept(T_eof)); diff --git a/src/parser-init.c b/src/parser-init.c index a7333cef..49193278 100644 --- a/src/parser-init.c +++ b/src/parser-init.c @@ -41,49 +41,54 @@ var_t *read_string_address_offset(block_t *parent, basic_block_t *bb, block_t *scope); +static void parse_array_init_at(var_t *var, + block_t *parent, + basic_block_t **bb, + var_t *base_addr, + bool sparse_zero); + +static bool initializer_separator(bool comma_consumed) +{ + return comma_consumed || lex_accept(T_comma); +} + /* Forward declaration for ternary handling used by initializers */ void read_ternary_operation(block_t *parent, basic_block_t **bb); /* Parse array initializer to determine size for implicit arrays and optionally * emit initialization code. */ -var_t *compute_element_address(block_t *parent, - basic_block_t **bb, - var_t *base_addr, - int index, - int elem_size) +static var_t *compute_byte_offset_address(block_t *parent, + basic_block_t **bb, + var_t *base_addr, + int byte_offset) { - if (index == 0) + if (byte_offset == 0) return base_addr; - var_t *offset = require_var(parent); - offset->var_name = gen_name(); - offset->init_val = index * elem_size; - add_insn(parent, *bb, OP_load_constant, offset, NULL, NULL, 0, NULL); + var_t *offset = load_constant(parent, *bb, byte_offset, TY_int); - var_t *addr = require_var(parent); - addr->var_name = gen_name(); + var_t *addr = require_named_var(parent); add_insn(parent, *bb, OP_add, addr, base_addr, offset, 0, NULL); return addr; } +var_t *compute_element_address(block_t *parent, + basic_block_t **bb, + var_t *base_addr, + int index, + int elem_size) +{ + return compute_byte_offset_address(parent, bb, base_addr, + index * elem_size); +} + var_t *compute_field_address(block_t *parent, basic_block_t **bb, var_t *struct_addr, const var_t *field) { - if (field->offset == 0) - return struct_addr; - - var_t *offset = require_var(parent); - offset->var_name = gen_name(); - offset->init_val = field->offset; - add_insn(parent, *bb, OP_load_constant, offset, NULL, NULL, 0, NULL); - - var_t *addr = require_var(parent); - addr->var_name = gen_name(); - add_insn(parent, *bb, OP_add, addr, struct_addr, offset, 0, NULL); - return addr; + return compute_byte_offset_address(parent, bb, struct_addr, field->offset); } /* A record assignment is a value copy, not the scalar OP_assign used for @@ -100,6 +105,27 @@ bool is_record_object(const var_t *var) var->type->num_fields > 0)); } +/* Choose the widest indirect access allowed by the remaining bytes and + * alignment. Widths descend by halves, matching backend scalar stores. + */ +static int byte_access_width(int remaining, int alignment, int offset) +{ + for (int width = 4; width > 1; width /= 2) + if (remaining >= width && alignment >= width && !(alignment % width) && + !(offset % width)) + return width; + return 1; +} + +static int initializer_alignment(const var_t *var) +{ + if (!var) + return 1; + return var->is_func || effective_pointer_depth(var) + ? PTR_SIZE + : alignment_type(var->type); +} + /* Copy size bytes between two known record addresses in 4-, 2- and 1-byte * slices, the widest indirect accesses every backend encodes. A record is an * object, not an integer value, so one OP_write of its whole size would leave @@ -112,18 +138,12 @@ void emit_record_copy_between(block_t *parent, int size) { for (int offset = 0; offset < size;) { - int width = 1; - if (size - offset >= 4) - width = 4; - else if (size - offset >= 2) - width = 2; + int width = byte_access_width(size - offset, PTR_SIZE, offset); var_t *src_part = compute_element_address(parent, bb, src_addr, offset, 1); var_t *dest_part = compute_element_address(parent, bb, dest_addr, offset, 1); - var_t *value = require_var(parent); - - value->var_name = gen_name(); + var_t *value = require_named_var(parent); add_insn(parent, *bb, OP_read, value, src_part, NULL, width, NULL); add_insn(parent, *bb, OP_write, NULL, dest_part, value, width, NULL); offset += width; @@ -140,8 +160,7 @@ var_t *record_value_address(block_t *parent, basic_block_t *bb, var_t *record) if (record->defers_record_copy) return record->compound_literal_address; - address = require_ref_var(parent, record->type, 0); - address->var_name = gen_name(); + address = name_var(require_ref_var(parent, record->type, 0)); add_insn(parent, bb, OP_address_of, address, record, NULL, 0, NULL); return address; } @@ -188,7 +207,6 @@ void emit_object_assignment(block_t *parent, * as `int (*fn)(int) = target;`. */ var_t *dest_addr = require_ref_var(parent, dest->type, dest->ptr_level); - dest_addr->var_name = gen_name(); add_insn(parent, *bb, OP_address_of, dest_addr, dest, NULL, 0, NULL); add_insn(parent, *bb, OP_write, NULL, dest_addr, src, PTR_SIZE, NULL); } else { @@ -198,6 +216,7 @@ void emit_object_assignment(block_t *parent, } type_t *read_type_name_specifiers(block_t *scope); +void read_type_name_decl(block_t *scope, var_t *decl); int read_const_expr_operand(block_t *scope); int read_global_address_offset(block_t *scope, block_t *parent, @@ -217,8 +236,7 @@ int global_pointer_cast_stride = 0; func_t *global_function_cast_signature = NULL; bool abstract_function_pointer_follows(void); -func_t *read_abstract_function_pointer(type_t *type, - int ptr_level, +func_t *read_abstract_function_pointer(const var_t *return_decl, int *pointer_level); /* If a cast to a function pointer type, `(int (*)(int))` or `(callback_t)`, @@ -232,22 +250,17 @@ func_t *read_global_function_pointer_cast(block_t *scope) type_t *type; int stars = 0; int pointer_level = 0; + var_t return_decl; if (!lex_accept(T_open_bracket)) return NULL; - type = read_type_name_specifiers(scope); + read_type_name_decl(scope, &return_decl); + type = return_decl.type; + stars = return_decl.ptr_level; if (type) { - while (lex_accept(T_const) || lex_accept(T_volatile)) - ; - while (lex_accept(T_asterisk)) { - stars++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } if (abstract_function_pointer_follows()) { signature = - read_abstract_function_pointer(type, stars, &pointer_level); + read_abstract_function_pointer(&return_decl, &pointer_level); if (pointer_level != 1) signature = NULL; } else if (type->func_signature && !type->is_direct_function_type && @@ -264,8 +277,6 @@ func_t *read_global_function_pointer_cast(block_t *scope) return signature; } -static token_t *matching_close_bracket(token_t *open); - /* Say whether cur_token->next opens a cast to an object pointer type in a * static initializer. C99 6.6 lets an address constant and an integer constant * be converted by such a cast. @@ -292,18 +303,14 @@ bool global_pointer_cast_starts_here(block_t *scope) /* A pointer to function pointers, `(int (**)(void))`, points to * pointer objects: two or more stars, then a parameter list. */ - int stars = 0; - - for (token = token->next; - token && - (token->kind == T_asterisk || token->kind == T_const || - token->kind == T_volatile || token->kind == T_restrict); - token = token->next) - stars += token->kind == T_asterisk; + int stars; + + token = function_pointer_declarator_suffix(token, &stars); if (stars < 2 || !token || token->kind != T_close_bracket || !(token = token->next) || token->kind != T_open_bracket) return false; - token = matching_close_bracket(token); + token = + skip_balanced_delimiter(token, T_open_bracket, T_close_bracket); if (!token || !(token = token->next)) return false; return token->kind == T_close_bracket; @@ -347,19 +354,16 @@ bool global_pointer_cast_starts_here(block_t *scope) */ int read_global_pointer_cast(block_t *scope, func_t **slot_signature) { + var_t return_decl; type_t *type; int depth = 0; int size; *slot_signature = NULL; lex_expect(T_open_bracket); - type = read_type_name_specifiers(scope); - while (lex_accept(T_asterisk)) { - depth++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; - } + read_type_name_decl(scope, &return_decl); + type = return_decl.type; + depth = return_decl.ptr_level; if (!type) error_at("Unknown type in pointer cast", cur_token_loc()); @@ -369,7 +373,7 @@ int read_global_pointer_cast(block_t *scope, func_t **slot_signature) if (abstract_function_pointer_follows()) { int pointer_level; func_t *signature = - read_abstract_function_pointer(type, depth, &pointer_level); + read_abstract_function_pointer(&return_decl, &pointer_level); lex_expect(T_close_bracket); if (pointer_level == 2) @@ -549,6 +553,21 @@ var_t *read_global_address_designator(block_t *scope, return object_addr; } +static var_t *read_global_object_address(block_t *scope, + block_t *parent, + basic_block_t **bb, + var_t *object, + bool decays) +{ + var_t *address = + name_var(require_ref_var(parent, object->type, object->ptr_level)); + + address->is_global_address = true; + add_insn(parent, *bb, OP_address_of, address, object, NULL, 0, NULL); + return read_global_address_designator(scope, parent, bb, &object, address, + decays); +} + /* The block that names in an initializer lowered into @parent resolve in. A * block-scope static is lowered through GLOBAL_BLOCK, yet its designators and * constants belong to the lexical scope of its declaration. @@ -619,12 +638,12 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) if (address_dereference) { address_dereference_identifier = consume_global_function_address_dereference(); - if (!find_visible_func(address_dereference_identifier->literal, scope)) + func_t *addressed = + find_visible_func(address_dereference_identifier->literal, scope); + if (!addressed) error_at("Function address requires a visible declaration", cur_token_loc()); - val = require_func_symbol_var(parent); - val->var_name = intern_string(address_dereference_identifier->literal); - val->is_func = true; + val = function_designator_value(parent, addressed); return val; } if (global_pointer_cast_starts_here(scope)) { @@ -679,9 +698,7 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) lex_expect(T_identifier); if (grouped_function_designator) lex_expect(T_close_bracket); - val = require_func_symbol_var(parent); - val->var_name = intern_string(name); - val->is_func = true; + val = function_designator_value(parent, func); return val; } if (explicit_address) { @@ -694,14 +711,8 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) if (object && object->is_global) { lex_expect(T_identifier); - val = require_ref_var(parent, object->type, - object->ptr_level); - val->var_name = gen_name(); - val->is_global_address = true; - add_insn(parent, *bb, OP_address_of, val, object, NULL, 0, - NULL); - return read_global_address_designator(scope, parent, bb, - &object, val, false); + return read_global_object_address(scope, parent, bb, object, + false); } } else { var_t *object = find_var(name, scope); @@ -715,14 +726,8 @@ var_t *parse_global_constant_value(block_t *parent, basic_block_t **bb) if (object && object->is_global && (subscripted ? object->array_dim2 : object->array_size)) { lex_expect(T_identifier); - val = require_ref_var(parent, object->type, - object->ptr_level); - val->var_name = gen_name(); - val->is_global_address = true; - add_insn(parent, *bb, OP_address_of, val, object, NULL, 0, - NULL); - return read_global_address_designator(scope, parent, bb, - &object, val, true); + return read_global_object_address(scope, parent, bb, object, + true); } } } @@ -787,36 +792,35 @@ void parse_string_row_init(block_t *parent, var_t *target_addr, int start); -/* Store @zero, a loaded zero constant, into each of @size bytes at @addr. The - * stores are byte-wide so that a record or a wide element needs no store width - * a narrow backend cannot encode. +/* Store @zero into each byte at @addr, using @alignment to select the widest + * safe indirect store. */ void emit_zero_bytes(block_t *parent, basic_block_t **bb, var_t *addr, int size, + int alignment, var_t *zero) { - for (int offset = 0; offset < size; offset++) { - var_t *byte_addr = compute_element_address(parent, bb, addr, offset, 1); - add_insn(parent, *bb, OP_write, NULL, byte_addr, zero, 1, NULL); + for (int offset = 0; offset < size;) { + int width = byte_access_width(size - offset, alignment, offset); + var_t *part = compute_element_address(parent, bb, addr, offset, 1); + + add_insn(parent, *bb, OP_write, NULL, part, zero, width, NULL); + offset += width; } } var_t *emit_zero_constant(block_t *parent, basic_block_t **bb) { - var_t *zero = require_var(parent); - - zero->var_name = gen_name(); - zero->init_val = 0; - add_insn(parent, *bb, OP_load_constant, zero, NULL, NULL, 0, NULL); - return zero; + return load_constant(parent, *bb, 0, TY_int); } /* Store zero into every byte of elements [from, to) of the array at @base. */ void emit_zero_elements(block_t *parent, basic_block_t **bb, var_t *base, + const var_t *element, int from, int to, int elem_size) @@ -826,10 +830,20 @@ void emit_zero_elements(block_t *parent, if (from >= to) return; zero = emit_zero_constant(parent, bb); - for (int i = from; i < to; i++) - emit_zero_bytes(parent, bb, - compute_element_address(parent, bb, base, i, elem_size), - elem_size, zero); + for (int i = from; i < to; i++) { + var_t *address = + compute_element_address(parent, bb, base, i, elem_size); + + if (elem_size == 1 || + (elem_size <= 4 && element && element->type && + (element->is_func || effective_pointer_depth(element) || + !is_record_type(element->type)))) + add_insn(parent, *bb, OP_write, NULL, address, zero, elem_size, + NULL); + else + emit_zero_bytes(parent, bb, address, elem_size, + initializer_alignment(element), zero); + } } /* The width of one element of @array: a pointer for an array of pointers or of @@ -902,13 +916,78 @@ bool accept_slot_designator(block_t *scope, int bound, int *slot) return true; } +static var_t *read_array_scalar_initializer(block_t *parent, + basic_block_t **bb, + bool allow_global_constant, + block_t *global_array_scope) +{ + if (parent == GLOBAL_BLOCK && allow_global_constant) { + char name[MAX_ID_LEN]; + + if (global_array_scope && lex_peek(T_identifier, name) && + !global_tested_operand_starts_here(global_array_scope) && + !find_scoped_constant(name, global_array_scope) && + !find_visible_func(name, global_array_scope)) { + var_t *object = find_var(name, global_array_scope); + + if (!object || !object->is_global || !object->array_size) + error_at( + global_array_scope == GLOBAL_BLOCK + ? "Global array initialization requires constant values" + : "Identifier is not an integer constant", + next_token_loc()); + } + return parse_global_constant_value(parent, bb); + } + read_expr(parent, bb); + read_ternary_operation(parent, bb); + return opstack_pop(); +} + +static bool parse_array_element_init(block_t *parent, + basic_block_t **bb, + const var_t *field, + var_t *address, + int elem_size, + bool overwritten, + var_t **parsed_value) +{ + var_t *value = NULL; + bool comma_consumed = false; + + if (parsed_value) + *parsed_value = NULL; + + if (lex_peek(T_open_curly, NULL) && is_record_type(field->type)) { + type_t *record_type = resolve_record_type(field->type); + lex_expect(T_open_curly); + parse_struct_field_init(parent, bb, record_type, address); + lex_expect(T_close_curly); + } else if (unbraced_record_starts_here(field)) { + comma_consumed = parse_unbraced_record_init(parent, bb, field->type, + address, overwritten); + } else { + value = read_array_scalar_initializer(parent, bb, !parsed_value, NULL); + } + + if (value) { + if (parsed_value) + *parsed_value = value; + var_t *stored = value->is_func + ? value + : resize_to(parent, bb, value, field->type, + parsed_value ? 0 : field->ptr_level); + add_insn(parent, *bb, OP_write, NULL, address, stored, elem_size, NULL); + } + return comma_consumed; +} + /* Parse one row of a two-dimensional array. An unbraced row returns true when a * brace-elided record element consumed the comma after the row's last * initializer, so the enclosing list must not expect it again. Only a braced * row owns designators; in an unbraced one they belong to the enclosing list. - * As in the plane and hyperplane helpers below, `filled` is one past the - * highest slot written, so after designators in any order the zero fill covers - * exactly the slots nothing wrote. + * `filled` is one past the highest slot written, so after designators in any + * order the zero fill covers exactly the slots nothing wrote. */ bool parse_array_field_row_values(block_t *parent, basic_block_t **bb, @@ -937,7 +1016,6 @@ bool parse_array_field_row_values(block_t *parent, return false; } while (!lex_peek(T_close_curly, NULL)) { - var_t *value = NULL; var_t *elem_addr; if (braced) @@ -946,42 +1024,19 @@ bool parse_array_field_row_values(block_t *parent, if (count >= field->array_dim2) error_at("Too many elements in array initializer", next_token_loc()); - emit_zero_elements(parent, bb, target_addr, start + filled, + emit_zero_elements(parent, bb, target_addr, field, start + filled, start + count, elem_size); elem_addr = compute_element_address(parent, bb, target_addr, start + count, elem_size); - comma_consumed = false; - if (lex_peek(T_open_curly, NULL) && is_record_type(field->type)) { - type_t *record_type = resolve_record_type(field->type); - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, record_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(field)) { - comma_consumed = parse_unbraced_record_init( - parent, bb, field->type, elem_addr, count < filled); - } else if (parent == GLOBAL_BLOCK) { - value = parse_global_constant_value(parent, bb); - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - value = opstack_pop(); - } - - if (value) { - var_t *stored = value->is_func - ? value - : resize_to(parent, bb, value, field->type, - field->ptr_level); - add_insn(parent, *bb, OP_write, NULL, elem_addr, stored, elem_size, - NULL); - } + comma_consumed = parse_array_element_init( + parent, bb, field, elem_addr, elem_size, count < filled, NULL); count++; if (count > filled) filled = count; if (count == field->array_dim2) { - if (braced && (comma_consumed || lex_accept(T_comma)) && + if (braced && initializer_separator(comma_consumed) && !lex_peek(T_close_curly, NULL)) { if (lex_peek(T_open_square, NULL)) continue; @@ -990,7 +1045,7 @@ bool parse_array_field_row_values(block_t *parent, } break; } - if (!comma_consumed && !lex_accept(T_comma)) + if (!initializer_separator(comma_consumed)) break; comma_consumed = true; } @@ -999,58 +1054,62 @@ bool parse_array_field_row_values(block_t *parent, comma_consumed = false; } - emit_zero_elements(parent, bb, target_addr, start + filled, + emit_zero_elements(parent, bb, target_addr, field, start + filled, start + field->array_dim2, elem_size); return comma_consumed; } -void parse_array_field_row_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start) -{ - parse_array_field_row_values(parent, bb, field, target_addr, start, true); -} - -/* Parse one braced plane of a three-dimensional array. Rows reuse the - * two-dimensional helper, which also supplies C99's trailing zero fill. +/* Parse nested array dimensions recursively; the row parser owns scalar + * elements and string rows, while each outer level handles its designators and + * zero-filled gaps. */ -void parse_array_field_plane_values(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start, - bool braced) +static bool parse_nested_array_values(block_t *parent, + basic_block_t **bb, + const var_t *array, + var_t *target_addr, + int start, + bool braced) { - var_t row; - int rows = 0; + var_t child; + int count = 0; int filled = 0; - int row_width = field->array_dim3; - int elem_size = array_element_size(field); - - memcpy(&row, field, sizeof(row)); - row.array_dim2 = row_width; - row.array_dim3 = 0; + int stride; + int elem_size = array_element_size(array); + + if (!array->array_dim3) + return parse_array_field_row_values(parent, bb, array, target_addr, + start, braced); + memcpy(&child, array, sizeof(child)); + fixed_array_var_drop_outer(&child); + stride = fixed_array_inner_count(&child); if (braced) { lex_expect(T_open_curly); reject_empty_initializer_in_strict_c99(); } while (!lex_peek(T_close_curly, NULL)) { + bool child_comma; + accept_slot_designator(initializer_name_scope(parent), - field->array_dim2, &rows); - if (rows >= field->array_dim2) - error_at("Too many rows in array initializer", next_token_loc()); - emit_zero_elements(parent, bb, target_addr, start + filled * row_width, - start + rows * row_width, elem_size); - bool row_comma = parse_array_field_row_values( - parent, bb, &row, target_addr, start + rows * row_width, - lex_peek(T_open_curly, NULL)); - rows++; - if (rows > filled) - filled = rows; - if (rows == field->array_dim2) { - if (braced && (row_comma || lex_accept(T_comma)) && + array->array_dim2, &count); + if (count >= array->array_dim2) + error_at(array->array_dim4 ? "Too many planes in array initializer" + : "Too many rows in array initializer", + next_token_loc()); + emit_zero_elements(parent, bb, target_addr, array, + start + filled * stride, start + count * stride, + elem_size); + child_comma = parse_nested_array_values(parent, bb, &child, target_addr, + start + count * stride, + lex_peek(T_open_curly, NULL)); + count++; + if (count > filled) + filled = count; + if (array->array_dim4) { + if (!lex_accept(T_comma)) + break; + continue; + } else if (count == array->array_dim2) { + if (braced && initializer_separator(child_comma) && !lex_peek(T_close_curly, NULL)) { if (lex_peek(T_open_square, NULL)) continue; @@ -1058,69 +1117,27 @@ void parse_array_field_plane_values(block_t *parent, next_token_loc()); } break; - } - if (!row_comma && !lex_accept(T_comma)) + } else if (!initializer_separator(child_comma)) { break; + } } if (braced) lex_expect(T_close_curly); - - emit_zero_elements(parent, bb, target_addr, start + filled * row_width, - start + field->array_dim2 * row_width, elem_size); -} - -void parse_array_field_plane_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start) -{ - parse_array_field_plane_values(parent, bb, field, target_addr, start, true); + emit_zero_elements(parent, bb, target_addr, array, start + filled * stride, + start + array->array_dim2 * stride, elem_size); + return false; } -/* Parse one braced outer slice of a four-dimensional array. Each contained - * three-dimensional plane reuses the existing plane parser, so row bounds and - * trailing zero fill remain identical at every nesting level. - */ -void parse_array_field_hyperplane_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr, - int start) +static int parse_nested_array_initializer(block_t *parent, + basic_block_t **bb, + const var_t *array, + var_t *target_addr, + int start) { - var_t plane; - int planes = 0; - int filled = 0; - int plane_size = field->array_dim3 * field->array_dim4; - int elem_size = array_element_size(field); - - memcpy(&plane, field, sizeof(plane)); - plane.array_size = plane_size; - plane.array_dim2 = field->array_dim3; - plane.array_dim3 = field->array_dim4; - plane.array_dim4 = 0; - lex_expect(T_open_curly); - reject_empty_initializer_in_strict_c99(); - while (!lex_peek(T_close_curly, NULL)) { - accept_slot_designator(initializer_name_scope(parent), - field->array_dim2, &planes); - if (planes >= field->array_dim2) - error_at("Too many planes in array initializer", next_token_loc()); - emit_zero_elements(parent, bb, target_addr, start + filled * plane_size, - start + planes * plane_size, elem_size); - parse_array_field_plane_values(parent, bb, &plane, target_addr, - start + planes * plane_size, - lex_peek(T_open_curly, NULL)); - planes++; - if (planes > filled) - filled = planes; - if (!lex_accept(T_comma)) - break; - } - lex_expect(T_close_curly); - - emit_zero_elements(parent, bb, target_addr, start + filled * plane_size, - start + field->array_dim2 * plane_size, elem_size); + if (!array->array_dim2 || !lex_peek(T_open_curly, NULL)) + return 0; + parse_nested_array_values(parent, bb, array, target_addr, start, true); + return fixed_array_inner_count(array); } /* Read a string literal and every adjacent one after it, decoded and joined @@ -1152,51 +1169,122 @@ int read_concatenated_string(char *combined) return used; } -/* An array member can be initialized directly by a string literal just like a - * standalone character array. The member has no independent var_t storage, so - * write through its already-computed field address rather than routing this - * through parse_string_array_init(). - */ -void parse_string_field_init(block_t *parent, - basic_block_t **bb, - const var_t *field, - var_t *target_addr) +static void emit_string_array_units(block_t *parent, + basic_block_t **bb, + const var_t *array, + var_t *base, + int count, + const void *units, + int initialized, + bool wide) { - char combined[MAX_STRING_LEN]; - int len; - - /* The terminating null is dropped when only the characters fit (C99 - * 6.7.8p14). - */ - len = read_concatenated_string(combined) + 1; - if (len - 1 > field->array_size) - error_at("String initializer is too long for character array", - cur_token_loc()); + int size = array->type->size; - for (int i = 0; i < field->array_size; i++) { - var_t *value = require_var(parent); - var_t *addr = compute_element_address(parent, bb, target_addr, i, 1); + for (int i = 0; i < count; i++) { + var_t *value = + wide ? require_typed_var(parent, array->type) : require_var(parent); + var_t *address; value->var_name = gen_name(); - value->init_val = i < len ? (unsigned char) combined[i] : 0; + value->init_val = i >= initialized ? 0 + : wide ? ((const int *) units)[i] + : ((const unsigned char *) units)[i]; value->is_const = true; add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - add_insn(parent, *bb, OP_write, NULL, addr, value, 1, NULL); + address = compute_element_address(parent, bb, base, i, size); + add_insn(parent, *bb, OP_write, NULL, address, value, size, NULL); } } +static void initialize_string_array(const var_t *array, + block_t *parent, + basic_block_t **bb, + var_t *base, + const void *units, + int length, + bool wide, + var_t *infer_bound) +{ + /* Both decoders omit the terminator. Narrow arrays initialize it when it + * fits; wide arrays leave its zero-filled unit to the array storage. + */ + int inferred_size = length + 1; + int initialized = wide ? length : inferred_size; + int array_size = array->array_size; + + if (infer_bound && infer_bound->has_unsized_array) { + array_size = inferred_size; + infer_bound->array_size = array_size; + infer_bound->has_unsized_array = false; + } else if (length > array_size) + error_at(wide ? "Wide string initializer is too long for array" + : "String initializer is too long for character array", + cur_token_loc()); + + if (infer_bound && !wide && initialized > array_size) + initialized = array_size; + int count = infer_bound && !wide && parent == GLOBAL_BLOCK ? initialized + : array_size; + emit_string_array_units(parent, bb, array, base, count, units, initialized, + wide); +} + +/* Decode and lower a string initializer; target_addr may be a field address, + * while infer_bound is set only for a declared array with unknown size. + */ +static void parse_string_array_init_to(const var_t *array, + block_t *parent, + basic_block_t **bb, + var_t *target_addr, + var_t *infer_bound, + bool wide) +{ + union { + char narrow[MAX_STRING_LEN]; + int wide[MAX_STRING_LEN]; + } units; + int length = wide ? read_wstring_units(units.wide, MAX_STRING_LEN) + : read_concatenated_string(units.narrow); + + initialize_string_array( + array, parent, bb, target_addr, + wide ? (const void *) units.wide : (const void *) units.narrow, length, + wide, infer_bound); +} + +/* A member has no independent var_t storage, so use its computed address. */ +void parse_string_field_init(block_t *parent, + basic_block_t **bb, + const var_t *field, + var_t *target_addr) +{ + parse_string_array_init_to(field, parent, bb, target_addr, NULL, false); +} + bool is_char_array(const var_t *var) { return var && !var->ptr_level && !var->array_dim2 && !var->array_dim3 && !var->array_dim4 && - compatible_decl_type(var->type, find_type("char", true)); + compatible_decl_type(var->type, find_builtin_type("char")); } bool is_wchar_array(const var_t *var) { return var && !var->ptr_level && !var->array_dim2 && !var->array_dim3 && !var->array_dim4 && - compatible_decl_type(var->type, find_type("wchar_t", true)); + compatible_decl_type(var->type, find_builtin_type("wchar_t")); +} + +static void parse_braced_string_array_init(var_t *var, + block_t *parent, + basic_block_t **bb) +{ + if (lex_peek(T_wstring, NULL)) + parse_wstring_array_init(var, parent, bb); + else + parse_string_array_init(var, parent, bb); + lex_accept(T_comma); + lex_expect(T_close_curly); } /* C99 permits a string literal initializer only for an array of matching @@ -1221,26 +1309,7 @@ void parse_wstring_field_init(block_t *parent, const var_t *field, var_t *target_addr) { - int values[MAX_STRING_LEN]; - int length; - - length = read_wstring_units(values, MAX_STRING_LEN); - if (length > field->array_size) - error_at("Wide string initializer is too long for array", - cur_token_loc()); - - for (int i = 0; i < field->array_size; i++) { - var_t *value = require_typed_var(parent, field->type); - var_t *addr = compute_element_address(parent, bb, target_addr, i, - field->type->size); - - value->var_name = gen_name(); - value->init_val = i < length ? values[i] : 0; - value->is_const = true; - add_insn(parent, *bb, OP_load_constant, value, NULL, NULL, 0, NULL); - add_insn(parent, *bb, OP_write, NULL, addr, value, field->type->size, - NULL); - } + parse_string_array_init_to(field, parent, bb, target_addr, NULL, true); } /* A string literal also initializes one innermost row of a multidimensional @@ -1264,9 +1333,9 @@ bool string_row_starts_here(const var_t *array) (after->next->kind != T_comma && after->next->kind != T_close_curly)) return false; if ((kind == T_string && - !compatible_decl_type(array->type, find_type("char", true))) || + !compatible_decl_type(array->type, find_builtin_type("char"))) || (kind == T_wstring && - !compatible_decl_type(array->type, find_type("wchar_t", true)))) + !compatible_decl_type(array->type, find_builtin_type("wchar_t")))) error_at( "String literal initializer has incompatible array element type", next_token_loc()); @@ -1314,91 +1383,12 @@ void parse_array_field_init(block_t *parent, const var_t *field, var_t *target_addr) { - int count = 0; - - /* One past the highest element written so far. A designator can move - * backwards, so zero fill starts here rather than at `count`, which would - * clear rows that an earlier designator already stored. - */ - int filled = 0; - int elem_size = array_element_size(field); - - lex_expect(T_open_curly); - reject_empty_initializer_in_strict_c99(); - while (!lex_peek(T_close_curly, NULL)) { - var_t *value = NULL; - var_t slice; - bool comma_consumed = false; - - /* `slice` is the array whose one element the next initializer fills. - * Without a designator that is the member itself. Each subscript of a - * designator such as `[1] = { ... }` or `[1][0] = 3` descends one - * level, so the braced row, plane and scalar paths below see exactly - * the shape the designator names and reordered rows stay valid. - */ - memcpy(&slice, field, sizeof(slice)); - if (read_array_designator(initializer_name_scope(parent), field, 0, - false, &count, &slice)) - lex_expect(T_assign); - - if (count >= field->array_size) - error_at("Too many elements in array initializer", - next_token_loc()); - emit_zero_elements(parent, bb, target_addr, filled, count, elem_size); - - var_t *elem_addr = - compute_element_address(parent, bb, target_addr, count, elem_size); - if (slice.array_dim4 && lex_peek(T_open_curly, NULL)) { - parse_array_field_hyperplane_init(parent, bb, &slice, target_addr, - count); - count += fixed_array_inner_count(&slice); - } else if (slice.array_dim3 && lex_peek(T_open_curly, NULL)) { - parse_array_field_plane_init(parent, bb, &slice, target_addr, - count); - count += fixed_array_inner_count(&slice); - } else if (slice.array_dim2 && lex_peek(T_open_curly, NULL)) { - parse_array_field_row_init(parent, bb, &slice, target_addr, count); - count += fixed_array_inner_count(&slice); - } else if (string_row_starts_here(&slice)) { - parse_string_row_init(parent, bb, &slice, target_addr, count); - count += string_row_width(&slice); - } else { - if (lex_peek(T_open_curly, NULL) && is_record_type(field->type)) { - type_t *record_type = resolve_record_type(field->type); - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, record_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(field)) { - comma_consumed = parse_unbraced_record_init( - parent, bb, field->type, elem_addr, count < filled); - } else if (parent == GLOBAL_BLOCK) { - value = parse_global_constant_value(parent, bb); - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - value = opstack_pop(); - } - - if (value) { - var_t *stored = value->is_func - ? value - : resize_to(parent, bb, value, field->type, - field->ptr_level); - add_insn(parent, *bb, OP_write, NULL, elem_addr, stored, - elem_size, NULL); - } - count++; - } - - if (count > filled) - filled = count; - if (!comma_consumed && !lex_accept(T_comma)) - break; - } - lex_expect(T_close_curly); + var_t array; - emit_zero_elements(parent, bb, target_addr, filled, field->array_size, - elem_size); + memcpy(&array, field, sizeof(array)); + if (parent == GLOBAL_BLOCK) + array.scope = initializer_name_scope(parent); + parse_array_init_at(&array, parent, bb, target_addr, true); } /* Initialize the record at @addr from @value, an initializer already read that @@ -1585,14 +1575,15 @@ bool parse_struct_field_values(block_t *parent, var_t *zero = emit_zero_constant(parent, bb); if (is_union) - emit_zero_bytes(parent, bb, target_addr, struct_type->size, zero); + emit_zero_bytes(parent, bb, target_addr, struct_type->size, + alignment_type(struct_type), zero); for (int i = 0; !is_union && i < struct_type->num_fields; i++) { var_t *field = &struct_type->fields[i]; emit_zero_bytes( parent, bb, compute_field_address(parent, bb, target_addr, field), - size_var(field), zero); + size_var(field), initializer_alignment(field), zero); } } @@ -1801,7 +1792,7 @@ bool parse_struct_field_values(block_t *parent, member_address(parent, bb, target_addr, field, field_addr)); } else if (field && field->array_size && !field->ptr_level && compatible_decl_type(field->type, - find_type("wchar_t", true)) && + find_builtin_type("wchar_t")) && lex_peek(T_wstring, NULL)) { parse_wstring_field_init( parent, bb, field, @@ -1825,12 +1816,9 @@ bool parse_struct_field_values(block_t *parent, parent, bb, field->type, member_address(parent, bb, target_addr, field, field_addr), true); - } else if (parent == GLOBAL_BLOCK) { - field_val_raw = parse_global_constant_value(parent, bb); } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - field_val_raw = opstack_pop(); + field_val_raw = + read_array_scalar_initializer(parent, bb, true, NULL); } if (field_val_raw) { @@ -1917,7 +1905,7 @@ bool parse_struct_field_values(block_t *parent, break; } } - if (!nested_comma_consumed && !lex_accept(T_comma)) + if (!initializer_separator(nested_comma_consumed)) break; comma_consumed = true; if (lex_peek(T_close_curly, NULL)) @@ -1942,26 +1930,21 @@ bool parse_struct_field_values(block_t *parent, void parse_array_literal_expr(block_t *parent, basic_block_t **bb) { - var_t *array_var = require_var(parent); - array_var->var_name = gen_name(); + var_t *array_var = require_named_var(parent); array_var->is_compound_literal = true; int element_count = 0; var_t *first_element = NULL; if (!lex_peek(T_close_curly, NULL)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - first_element = opstack_pop(); + first_element = read_array_scalar_initializer(parent, bb, false, NULL); element_count = 1; while (lex_accept(T_comma)) { if (lex_peek(T_close_curly, NULL)) break; - read_expr(parent, bb); - read_ternary_operation(parent, bb); - opstack_pop(); + read_array_scalar_initializer(parent, bb, false, NULL); element_count++; } } @@ -2000,17 +1983,7 @@ basic_block_t *handle_return_statement(block_t *parent, basic_block_t *bb) error_at("void function cannot return an expression in C99", cur_token_loc()); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - while (lex_accept(T_comma)) { - discard_operand(parent, bb); - if (!read_assignment_expression(parent, &bb)) { - read_expr(parent, &bb); - read_ternary_operation(parent, &bb); - } - } + read_comma_expression(parent, &bb, false); lex_expect(T_semicolon); var_t *rs1 = opstack_pop(); @@ -2024,10 +1997,8 @@ basic_block_t *handle_return_statement(block_t *parent, basic_block_t *bb) cur_token_loc()); if (rs1 && rs1->array_size > 0 && rs1->var_name[0] == '.' && !strict_c99) { - var_t *val = require_var(parent); - val->type = rs1->type; + var_t *val = name_var(require_typed_var(parent, rs1->type)); val->init_val = rs1->init_val; - val->var_name = gen_name(); add_insn(parent, bb, OP_load_constant, val, NULL, NULL, 0, NULL); rs1 = val; } @@ -2068,19 +2039,29 @@ basic_block_t *handle_return_statement(block_t *parent, basic_block_t *bb) return NULL; } +static void emit_control_branch(block_t *parent, basic_block_t *bb) +{ + var_t *condition = opstack_pop(); + + reject_record_operand(condition); + add_insn(parent, bb, OP_branch, NULL, condition, NULL, 0, NULL); +} + +static void read_control_branch(block_t *parent, basic_block_t **bb) +{ + lex_expect(T_open_bracket); + read_control_expression(parent, bb); + lex_expect(T_close_bracket); + emit_control_branch(parent, *bb); +} + basic_block_t *handle_if_statement(block_t *parent, basic_block_t *bb) { basic_block_t *n = bb_create(parent); bb_connect(bb, n, NEXT); bb = n; - lex_expect(T_open_bracket); - read_control_expression(parent, &bb); - lex_expect(T_close_bracket); - - var_t *vd = opstack_pop(); - reject_record_operand(vd); - add_insn(parent, bb, OP_branch, NULL, vd, NULL, 0, NULL); + read_control_branch(parent, &bb); basic_block_t *then_ = bb_create(parent); basic_block_t *else_ = bb_create(parent); @@ -2128,13 +2109,7 @@ basic_block_t *handle_while_statement(block_t *parent, basic_block_t *bb) continue_bb_push(bb); basic_block_t *cond = bb; - lex_expect(T_open_bracket); - read_control_expression(parent, &bb); - lex_expect(T_close_bracket); - - var_t *vd = opstack_pop(); - reject_record_operand(vd); - add_insn(parent, bb, OP_branch, NULL, vd, NULL, 0, NULL); + read_control_branch(parent, &bb); basic_block_t *then_ = bb_create(parent); basic_block_t *else_ = bb_create(parent); @@ -2153,54 +2128,25 @@ basic_block_t *handle_while_statement(block_t *parent, basic_block_t *bb) return else_; } -basic_block_t *handle_goto_statement(block_t *parent, basic_block_t *bb) +basic_block_t *handle_goto_statement(basic_block_t *bb) { - /* Since a goto splits the current program into two basic blocks and makes - * the subsequent basic block unreachable, this causes problems for later - * CFG operations. Therefore, we create a fake if that always executes to - * wrap the goto, and connect the unreachable basic block to the else - * branch. Finally, return this else block. - * - * after: a = b + c; goto label; c *= d; - * - * before: a = b + c; if (1) - * goto label; - * c *= d; - */ - char token[MAX_ID_LEN]; if (!lex_peek(T_identifier, token)) error_at("Expected identifier after 'goto'", next_token_loc()); - lex_expect(T_identifier); lex_expect(T_semicolon); - basic_block_t *fake_if = bb_create(parent); - bb_connect(bb, fake_if, NEXT); - var_t *val = require_var(parent); - val->var_name = gen_name(); - val->init_val = 1; - add_insn(parent, fake_if, OP_load_constant, val, NULL, NULL, 0, NULL); - add_insn(parent, fake_if, OP_branch, NULL, val, NULL, 0, NULL); - - basic_block_t *then_ = bb_create(parent); - basic_block_t *else_ = bb_create(parent); - bb_connect(fake_if, then_, THEN); - bb_connect(fake_if, else_, ELSE); - - add_insn(parent, then_, OP_jump, NULL, NULL, NULL, 0, token); label_t *label = find_label(token); if (label) { label->used = true; - bb_connect(then_, label->bb, NEXT); - return else_; + bb_connect(bb, label->bb, NEXT); + } else { + if (backpatch_bb_idx >= MAX_LABELS) + error_at("Too many forward-referenced labels", cur_token_loc()); + backpatch_label[backpatch_bb_idx] = intern_string(token); + backpatch_bb[backpatch_bb_idx++] = bb; } - - if (backpatch_bb_idx > MAX_LABELS - 1) - error_at("Too many forward-referenced labels", cur_token_loc()); - - backpatch_bb[backpatch_bb_idx++] = then_; - return else_; + return NULL; } int read_const_expr(block_t *scope); @@ -2247,9 +2193,9 @@ static void reject_string_element_initializer(const var_t *var) (token->next->kind != T_comma && token->next->kind != T_close_curly)) return; if ((kind == T_string && - compatible_decl_type(var->type, find_type("char", true))) || + compatible_decl_type(var->type, find_builtin_type("char"))) || (kind == T_wstring && - compatible_decl_type(var->type, find_type("wchar_t", true)))) + compatible_decl_type(var->type, find_builtin_type("wchar_t")))) error_at("String literal cannot initialize a single array element", next_token_loc()); error_at("String literal initializer has incompatible array element type", @@ -2257,22 +2203,33 @@ static void reject_string_element_initializer(const var_t *var) } void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) +{ + parse_array_init_at(var, parent, bb, var, false); +} + +static void parse_array_init_at(var_t *var, + block_t *parent, + basic_block_t **bb, + var_t *base_addr, + bool sparse_zero) { int count = 0; int inferred_size = 0; - var_t *base_addr = NULL; + int initialized_end = 0; /* An omitted outer bound of a multidimensional declaration already has an * inner-dimension product in `array_size`. It is nevertheless inferred from * the initializer, just like a one-dimensional `int a[]`. */ bool is_implicit = (var->array_size == 0 || var->has_unsized_array); + bool runtime_initializer = + parent != GLOBAL_BLOCK || var->is_compound_literal; block_t *initializer_scope = parent; if (parent == GLOBAL_BLOCK && var->scope && var->scope != GLOBAL_BLOCK) initializer_scope = var->scope; block_t *saved_initializer_scope = global_constant_initializer_scope; - if (parent == GLOBAL_BLOCK) + if (!runtime_initializer) global_constant_initializer_scope = initializer_scope; /* Elements of a pointer array are pointer-sized. Using the base type's @@ -2289,25 +2246,17 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) if (lex_peek(T_open_curly, NULL) && string_ends_braced_initializer(var, cur_token->next->next)) { lex_expect(T_open_curly); - if (lex_peek(T_wstring, NULL)) - parse_wstring_array_init(var, parent, bb); - else - parse_string_array_init(var, parent, bb); - lex_accept(T_comma); - lex_expect(T_close_curly); + parse_braced_string_array_init(var, parent, bb); global_constant_initializer_scope = saved_initializer_scope; return; } - base_addr = var; - - /* Reordered array designators can leave holes both before and after a - * written element. Initialize the whole automatic array first, then let - * explicit elements overwrite their slots. Byte stores also cover record - * elements without relying on a backend-wide aggregate store. + /* Reordered designators can leave holes before or after written elements. + * Ordinary automatic arrays are cleared up front; field arrays clear only + * skipped ranges. */ - if (parent != GLOBAL_BLOCK && !is_implicit) - emit_zero_elements(parent, bb, base_addr, 0, var->array_size, + if (runtime_initializer && !is_implicit && !sparse_zero) + emit_zero_elements(parent, bb, base_addr, var, 0, var->array_size, elem_size); lex_expect(T_open_curly); @@ -2344,41 +2293,19 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) error_at("Too many elements in array initializer", next_token_loc()); - /* A forward designator leaves a gap. Explicit arrays were zeroed - * before parsing and global storage begins zeroed, but an inferred - * local array has no known bound yet: zero the elements skipped - * past the highest one written, whatever form the next initializer - * takes. Every element below `inferred_size` is then either zero or - * written, which a brace-elided record relies on below. - */ - if (is_implicit && parent != GLOBAL_BLOCK) - emit_zero_elements(parent, bb, base_addr, inferred_size, count, - elem_size); - - if (initializer_var->array_dim4 && lex_peek(T_open_curly, NULL)) { - parse_array_field_hyperplane_init(parent, bb, initializer_var, - base_addr, count); - count += fixed_array_inner_count(initializer_var); - if (is_implicit && count > inferred_size) - inferred_size = count; - if (!lex_accept(T_comma)) - break; - continue; - } else if (initializer_var->array_dim3 && - lex_peek(T_open_curly, NULL)) { - parse_array_field_plane_init(parent, bb, initializer_var, - base_addr, count); - count += fixed_array_inner_count(initializer_var); - if (is_implicit && count > inferred_size) - inferred_size = count; - if (!lex_accept(T_comma)) - break; - continue; - } else if (initializer_var->array_dim2 && - lex_peek(T_open_curly, NULL)) { - parse_array_field_row_init(parent, bb, initializer_var, - base_addr, count); - count += fixed_array_inner_count(initializer_var); + /* Zero any skipped range from the highest initialized slot. */ + if (runtime_initializer && (is_implicit || sparse_zero)) + emit_zero_elements( + parent, bb, base_addr, var, + is_implicit ? inferred_size : initialized_end, count, + elem_size); + + int nested = parse_nested_array_initializer( + parent, bb, initializer_var, base_addr, count); + if (nested) { + count += nested; + if (count > initialized_end) + initialized_end = count; if (is_implicit && count > inferred_size) inferred_size = count; if (!lex_accept(T_comma)) @@ -2388,6 +2315,8 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) parse_string_row_init(parent, bb, initializer_var, base_addr, count); count += string_row_width(initializer_var); + if (count > initialized_end) + initialized_end = count; if (is_implicit && count > inferred_size) inferred_size = count; if (!lex_accept(T_comma) || lex_peek(T_close_curly, NULL)) @@ -2403,192 +2332,31 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) var_t *elem_addr = compute_element_address( parent, bb, base_addr, count, elem_size); - if (is_implicit && parent != GLOBAL_BLOCK && - count >= inferred_size) - emit_zero_elements(parent, bb, base_addr, count, count + 1, - elem_size); + if (runtime_initializer && (is_implicit || sparse_zero) && + count >= (is_implicit ? inferred_size : initialized_end)) + emit_zero_elements(parent, bb, base_addr, var, count, + count + 1, elem_size); elided_comma = parse_struct_field_values( parent, bb, resolve_record_type(var->type), elem_addr, true, true, NULL); - } else if (lex_peek(T_open_curly, NULL) && - is_record_type(var->type)) { - type_t *struct_type = resolve_record_type(var->type); - - var_t *elem_addr = compute_element_address( - parent, bb, base_addr, count, elem_size); - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, struct_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(var)) { - /* Without braces a record element takes one initializer per - * member, so `struct p a[2] = { 1, 2, 3, 4 }` fills a[0] from 1 - * and 2 before a[1] starts. The record parser stops after its - * last member and leaves the comma to this loop. + } else if (is_record_type(var->type) && + (lex_peek(T_open_curly, NULL) || + unbraced_record_starts_here(var))) { + /* The shared element parser handles braced records and + * brace-elided member lists. It leaves a consumed comma for + * this loop and preserves members written by earlier + * designators. */ var_t *elem_addr = compute_element_address( parent, bb, base_addr, count, elem_size); - - /* An element already reached is zero or written, and may hold - * members a designator stored; a new one clears itself. - */ - elided_comma = parse_unbraced_record_init( - parent, bb, var->type, elem_addr, - !is_implicit || count < inferred_size); + elided_comma = parse_array_element_init( + parent, bb, var, elem_addr, elem_size, + (!is_implicit && !sparse_zero) || count < initialized_end, + NULL); } else { - /* A global initializer is restricted to simple constants, but - * it still has to be stored. Consuming the tokens and dropping - * the value left every global array zero-filled, while the same - * initializer on a local worked. - */ reject_string_element_initializer(var); - char leading_name[MAX_ID_LEN]; - - /* An arithmetic element, such as 1 ? 2 : 3 or 1 && 2, is folded - * as a constant. The runtime expression reader would branch, - * moving the global setup entry block to the branch's join and - * dropping every store emitted before it. - */ - if (parent == GLOBAL_BLOCK && - (lex_peek(T_numeric, NULL) || lex_peek(T_minus, NULL) || - global_tested_operand_starts_here(initializer_scope) || - lex_peek(T_char, NULL) || lex_peek(T_wchar, NULL) || - (lex_peek(T_identifier, leading_name) && - find_scoped_constant(leading_name, initializer_scope)) || - lex_peek(T_ampersand, NULL) || - grouped_global_function_designator_starts_here(true) || - global_function_address_dereference_starts_here() || - lex_peek(T_open_bracket, NULL) || - lex_peek(T_sizeof, NULL) || lex_peek(T_plus, NULL) || - lex_peek(T_bit_not, NULL) || lex_peek(T_log_not, NULL) || - string_element_appears_before_initializer_end( - cur_token->next) || - string_address_offset_starts_here())) { - /* Addresses, casts, sizeof and grouped or unary constant - * expressions share the reader of aggregate members. - */ - val = parse_global_constant_value(parent, bb); - } else { - char initializer_name[MAX_ID_LEN]; - char global_token[MAX_ID_LEN]; - bool function_initializer = - lex_peek(T_identifier, initializer_name) && - find_visible_func(initializer_name, initializer_scope); - var_t *object_constant = NULL; - - if (parent == GLOBAL_BLOCK && - lex_peek(T_identifier, global_token)) - object_constant = - find_var(global_token, initializer_scope); - - /* A static array named by a block-scope static's element is - * an address constant (C99 6.6p7), not an integer one, and - * takes the array-to-pointer path below. - */ - if (parent == GLOBAL_BLOCK && - initializer_scope != GLOBAL_BLOCK && - !lex_peek(T_string, NULL) && !function_initializer && - !lex_peek(T_ampersand, NULL) && - !(object_constant && object_constant->is_global && - object_constant->array_size)) { - /* Storage for a block-scope static lives globally, - * while its initializer is an integer constant - * expression in the surrounding block. Resolve local - * enumerators before emitting the global setup-store - * value. - */ - if (typed_global_literal_appears_before_initializer_end( - cur_token->next) || - string_element_appears_before_initializer_end( - cur_token->next)) { - /* read_const_expr() evaluates in an int, which - * drops the high word of a wide element. - */ - val = read_wide_global_literal_expression( - GLOBAL_BLOCK, *bb, var->scope); - } else { - val = require_var(GLOBAL_BLOCK); - val->var_name = gen_name(); - val->init_val = read_const_expr(var->scope); - val->is_const = true; - add_insn(GLOBAL_BLOCK, *bb, OP_load_constant, val, - NULL, NULL, 0, NULL); - } - } else { - if (parent == GLOBAL_BLOCK) { - char token[MAX_ID_LEN]; - bool enum_constant = - lex_peek(T_identifier, token) && - find_scoped_constant(token, parent); - bool function_constant = - lex_peek(T_identifier, token) && - find_visible_func(token, initializer_scope); - - if (!lex_peek(T_numeric, NULL) && - !lex_peek(T_minus, NULL) && - !lex_peek(T_string, NULL) && - !lex_peek(T_char, NULL) && - !lex_peek(T_wchar, NULL) && !enum_constant && - !function_constant && - !lex_peek(T_ampersand, NULL) && - !(object_constant && - object_constant->is_global && - object_constant->array_size)) - error_at( - "Global array initialization requires " - "constant " - "values", - next_token_loc()); - } - - if (parent == GLOBAL_BLOCK && object_constant && - object_constant->is_global && - object_constant->array_size) { - int stride = array_element_size(object_constant); - - /* Array-to-pointer conversion is a permitted - * address constant in static aggregate - * initializers. Preserve its row stride here rather - * than reading an object value, which global setup - * cannot do before GP is established. - */ - val = require_ref_var(parent, object_constant->type, - object_constant->ptr_level); - val->var_name = gen_name(); - lex_ident(T_identifier, global_token); - add_insn(parent, *bb, OP_address_of, val, - object_constant, NULL, 0, NULL); - if (object_constant->array_dim2) - stride *= object_constant->array_dim2; - if (object_constant->array_dim3) - stride *= object_constant->array_dim3; - if (object_constant->array_dim4) - stride *= object_constant->array_dim4; - if (object_constant->array_dim2 && - lex_peek(T_open_square, NULL)) { - var_t *row = object_constant; - - val->is_global_address = true; - val = read_global_address_designator( - initializer_scope, parent, bb, &row, val, - true); - } else if (lex_accept(T_plus)) { - int index = read_const_expr(initializer_scope); - - val = compute_element_address(parent, bb, val, - index, stride); - } else if (lex_accept(T_minus)) { - int index = read_const_expr(initializer_scope); - - val = compute_element_address(parent, bb, val, - -index, stride); - } - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - val = opstack_pop(); - } - } - } + val = read_array_scalar_initializer( + parent, bb, !runtime_initializer, initializer_scope); } if (is_implicit && count >= MAX_IMPLICIT_ARRAY) @@ -2667,34 +2435,53 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) else v = resize_to(parent, bb, val, var->type, 0); - var_t *elem_addr = compute_element_address( - parent, bb, base_addr, count, elem_size); + bool already_zeroed = + parent != GLOBAL_BLOCK && !is_implicit && !sparse_zero && + count >= initialized_end && !var->is_volatile && + !var->type->is_volatile_qualified && + !var->type->array_element_is_volatile && !var->ptr_level && + !var->is_func && !is_record_type(var->type) && + v->is_const && !v->init_val && !v->init_val_hi; /* A direct long long element is wider than a pointer on a * 32-bit target, where it is written as a register pair. */ - if (elem_size <= PTR_SIZE || - (elem_size == 8 && !var->ptr_level && !var->is_func && - !is_record_type(var->type))) { - add_insn(parent, *bb, OP_write, NULL, elem_addr, v, - elem_size, NULL); - } else { - fatal("Unsupported: array element wider than a pointer"); + if (!already_zeroed) { + var_t *elem_addr = compute_element_address( + parent, bb, base_addr, count, elem_size); + + if (elem_size <= PTR_SIZE || + (elem_size == 8 && !var->ptr_level && !var->is_func && + !is_record_type(var->type))) { + add_insn(parent, *bb, OP_write, NULL, elem_addr, v, + elem_size, NULL); + } else { + fatal( + "Unsupported: array element wider than a pointer"); + } } } + if (var->is_compound_literal && count == 0 && val) + var->init_val = val->init_val; + count++; + if (count > initialized_end) + initialized_end = count; if (is_implicit && count > inferred_size) inferred_size = count; - if (!elided_comma && !lex_accept(T_comma)) - break; - if (lex_peek(T_close_curly, NULL)) + if (!initializer_separator(elided_comma) || + lex_peek(T_close_curly, NULL)) break; } } lex_expect(T_close_curly); + if (runtime_initializer && sparse_zero && !is_implicit) + emit_zero_elements(parent, bb, base_addr, var, initialized_end, + var->array_size, elem_size); + if (is_implicit) { /* `count` walks scalars, so a brace-elided list that stops inside a * row, as in `int m[][2] = { 1, 2, 3 }`, leaves inferred_size short of @@ -2704,8 +2491,8 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) int inner = fixed_array_inner_count(var); int whole = (inferred_size + inner - 1) / inner * inner; - if (parent != GLOBAL_BLOCK) - emit_zero_elements(parent, bb, base_addr, inferred_size, whole, + if (runtime_initializer) + emit_zero_elements(parent, bb, base_addr, var, inferred_size, whole, elem_size); inferred_size = whole; var->array_size = inferred_size; @@ -2714,114 +2501,6 @@ void parse_array_init(var_t *var, block_t *parent, basic_block_t **bb) global_constant_initializer_scope = saved_initializer_scope; } -void parse_array_compound_literal(var_t *var, - block_t *parent, - basic_block_t **bb) -{ - int elem_size = var->type->size; - int count = 0; - - reject_empty_initializer_in_strict_c99(); - - /* A compound literal may spell either an inferred bound, ``int[]``, or an - * actual array type, ``int[4]``. The latter is not merely syntax: omitted - * members are zero-initialized and an excess initializer is a constraint - * violation. Keep the parsed bound until the initializer has been consumed; - * previously this routine reset it and silently turned every declared-bound - * literal into an inferred-size one. - */ - int declared_size = var->array_size; - int inferred_size = 0; - var->init_val = 0; - - /* The opening brace is already consumed. See parse_array_init(). */ - if (string_ends_braced_initializer(var, cur_token->next)) { - if (lex_peek(T_wstring, NULL)) - parse_wstring_array_init(var, parent, bb); - else - parse_string_array_init(var, parent, bb); - lex_accept(T_comma); - lex_expect(T_close_curly); - return; - } - - /* A designated element can leave holes before or after it, so initialize - * the declared object before parsing any explicit elements. - */ - emit_zero_elements(parent, bb, var, 0, declared_size, elem_size); - - if (!lex_peek(T_close_curly, NULL)) { - for (;;) { - bool elided_comma = false; - - if (read_array_designator(parent, var, 1, !declared_size, &count, - NULL)) - lex_expect(T_assign); - if (declared_size && count >= declared_size) - error_at("Too many elements in array compound literal", - next_token_loc()); - if (!declared_size && count >= MAX_IMPLICIT_ARRAY) - error_at("Too many elements in array compound literal", - next_token_loc()); - - /* An inferred-bound array gets its size only after the closing - * brace. Still zero every gap before storing a designator so the - * automatic object obeys C99's aggregate initialization rule. - * inferred_size is one past the highest initialized slot, so a - * later backward designator cannot make a forward gap overwrite an - * earlier explicit value. - */ - if (!declared_size) - emit_zero_elements(parent, bb, var, inferred_size, count, - elem_size); - - var_t *elem_addr = - compute_element_address(parent, bb, var, count, elem_size); - if (lex_peek(T_open_curly, NULL) && is_record_type(var->type)) { - /* The array compound literal owns a real aggregate object, just - * like an ordinary array initializer. A braced element must - * therefore be lowered through the shared record path; treating - * it as an expression rejected the opening brace and made - * (struct S[]){ { ... }, { ... } } unusable. - */ - type_t *record_type = resolve_record_type(var->type); - - lex_expect(T_open_curly); - parse_struct_field_init(parent, bb, record_type, elem_addr); - lex_expect(T_close_curly); - } else if (unbraced_record_starts_here(var)) { - elided_comma = parse_unbraced_record_init( - parent, bb, var->type, elem_addr, - declared_size > 0 || count < inferred_size); - } else { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - var_t *value = opstack_pop(); - if (count == 0) - var->init_val = value->init_val; - - var_t *store_val = resize_to(parent, bb, value, var->type, 0); - add_insn(parent, *bb, OP_write, NULL, elem_addr, store_val, - elem_size, NULL); - } - - if (!declared_size) { - if (count + 1 > inferred_size) - inferred_size = count + 1; - } - count++; - if (!elided_comma && !lex_accept(T_comma)) - break; - if (lex_peek(T_close_curly, NULL)) - break; - } - } - - lex_expect(T_close_curly); - - var->array_size = declared_size ? declared_size : inferred_size; -} - /* Identify compiler-emitted temporaries that hold array compound literals. They * keep array metadata without pointer indirection and are marked via * is_compound_literal when synthesized. @@ -2873,9 +2552,7 @@ var_t *scalarize_array_literal(block_t *parent, /* Create a new scalar temporary, giving it a unique name and copying over * the literal data so downstream code can treat it like a normal value. */ - var_t *scalar = require_typed_var(parent, result_type); - scalar->ptr_level = 0; - scalar->var_name = gen_name(); + var_t *scalar = name_var(require_typed_ptr_var(parent, result_type, 0)); scalar->init_val = array_var->init_val; /* Materialize the literal data into the scalar temporary via an OP_read. */ diff --git a/src/parser-sizeof.c b/src/parser-sizeof.c index 0d386d5c..d47ebddb 100644 --- a/src/parser-sizeof.c +++ b/src/parser-sizeof.c @@ -75,13 +75,11 @@ int sizeof_grouped_literal_depth(token_t *token, token_kind_t kind) */ static void push_sizeof_result(block_t *parent, basic_block_t *bb, int size) { - var_t *result = require_typed_var(parent, find_type("size_t", true)); + var_t *result = + load_constant(parent, bb, size, find_builtin_type("size_t")); - result->init_val = size; result->is_const = true; - result->var_name = gen_name(); opstack_push(result); - add_insn(parent, bb, OP_load_constant, result, NULL, NULL, 0, NULL); } int sizeof_array_object(const var_t *array) @@ -93,6 +91,22 @@ int sizeof_array_object(const var_t *array) return array->array_size * element_size; } +static void sizeof_expression_type(var_t *expr, + token_t *sizeof_token, + type_t **type, + int *ptr_cnt, + int *array_size, + bool *is_function) +{ + if (is_bitfield(expr)) + error_at("sizeof cannot be applied to a bit-field", + &sizeof_token->location); + *type = expr->type; + *ptr_cnt = expr->ptr_level; + *array_size = expr->array_size; + *is_function = expr->is_func; +} + /* What a type-only walk over a sizeof operand has seen. */ typedef struct { int operators; /* unary, postfix and cast operators applied */ @@ -198,8 +212,7 @@ static void sizeof_walk_address(var_t *object, sizeof_walk_t *walk, token_t *op) fixed_array_shape_t shape = fixed_array_shape_from_var(object); fixed_array_shape_t scalar = {0}; - fixed_array_shape_to_pointee_var(object, &shape); - object->pointee_array_element_ptr_level = object->ptr_level; + set_pointee_array_shape(object, &shape, object->ptr_level); fixed_array_shape_to_var(object, &scalar); } object->ptr_level++; @@ -231,9 +244,7 @@ static type_t *read_sizeof_cast_type(block_t *scope, int *ptr_level) while (type && lex_accept(T_asterisk)) { stars++; - while (lex_accept(T_const) || lex_accept(T_volatile) || - lex_accept(T_restrict)) - ; + skip_type_qualifiers(); } *ptr_level = stars; if (!type || type->is_floating || type->array_size || @@ -357,8 +368,7 @@ static bool scan_sizeof_postfix_operand(block_t *scope, fixed_array_shape_drop_outer(&shape); fixed_array_shape_to_var(object, &scalar); if (shape.rank) { - fixed_array_shape_to_pointee_var(object, &shape); - object->pointee_array_element_ptr_level = root->ptr_level; + set_pointee_array_shape(object, &shape, root->ptr_level); } object->ptr_level++; walk->operators++; @@ -672,7 +682,7 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) !strcmp(token, "__func__")) { lex_expect(T_identifier); push_sizeof_result(parent, *bb, - strlen(parent->func->return_def.var_name) + 1); + strlen(function_source_name(parent->func)) + 1); return; } @@ -691,13 +701,8 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) read_expr_operand(parent, &unevaluated_bb); unevaluated_expression_depth--; var_t *expr_var = opstack_pop(); - if (is_bitfield(expr_var)) - error_at("sizeof cannot be applied to a bit-field", - &sizeof_tk->location); - type = expr_var->type; - ptr_cnt = expr_var->ptr_level; - array_size = expr_var->array_size; - is_function = expr_var->is_func; + sizeof_expression_type(expr_var, sizeof_tk, &type, &ptr_cnt, + &array_size, &is_function); if (type == TY_void && ptr_cnt == 0) error_at("sizeof(void) is invalid", &sizeof_tk->location); if (is_function && ptr_cnt == 0) @@ -725,7 +730,7 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) lex_expect(T_identifier); lex_expect(T_close_bracket); push_sizeof_result(parent, *bb, - strlen(parent->func->return_def.var_name) + 1); + strlen(function_source_name(parent->func)) + 1); return; } @@ -773,19 +778,11 @@ void handle_sizeof_operator(block_t *parent, basic_block_t **bb) /* sizeof(expression) - parse the expression and get its type */ basic_block_t *unevaluated_bb = bb_create(parent); unevaluated_expression_depth++; - if (!read_assignment_expression(parent, &unevaluated_bb)) { - read_expr(parent, &unevaluated_bb); - read_ternary_operation(parent, &unevaluated_bb); - } + read_assignment_or_expression(parent, &unevaluated_bb); unevaluated_expression_depth--; var_t *expr_var = opstack_pop(); - if (is_bitfield(expr_var)) - error_at("sizeof cannot be applied to a bit-field", - &sizeof_tk->location); - type = expr_var->type; - ptr_cnt = expr_var->ptr_level; - array_size = expr_var->array_size; - is_function = expr_var->is_func; + sizeof_expression_type(expr_var, sizeof_tk, &type, &ptr_cnt, &array_size, + &is_function); if (is_incomplete_record_object(expr_var)) error_at("sizeof cannot be applied to an incomplete record type", &sizeof_tk->location); diff --git a/src/parser-stmt.c b/src/parser-stmt.c index e62da729..676aef96 100644 --- a/src/parser-stmt.c +++ b/src/parser-stmt.c @@ -87,8 +87,7 @@ bool constant_expression_needs_typed_value(token_t *token) (bracket_depth == 0 && token->kind == T_comma)) return false; else if (token->kind == T_numeric && - (numeric_literal_needs_wide_path(token->literal) || - numeric_literal_needs_typed_global_path(token->literal))) + numeric_literal_needs_global_reader(token->literal)) return true; } return false; @@ -158,16 +157,14 @@ basic_block_t *read_switch_label_statement(block_t *parent, basic_block_t *body) context->seen_cases = seen; } - constant = - require_typed_var(context->dispatch_parent, context->control->type); - constant->var_name = gen_name(); + constant = name_var(require_typed_var(context->dispatch_parent, + context->control->type)); constant->init_val = case_lo; constant->init_val_hi = case_hi; add_insn(context->dispatch_parent, context->dispatch_tail, OP_load_constant, constant, NULL, NULL, 0, NULL); - comparison = require_var(context->dispatch_parent); - comparison->var_name = gen_name(); + comparison = require_named_var(context->dispatch_parent); add_insn(context->dispatch_parent, context->dispatch_tail, OP_eq, comparison, constant, context->control, 0, NULL); add_insn(context->dispatch_parent, context->dispatch_tail, OP_branch, @@ -341,6 +338,49 @@ typedef struct { bool is_volatile; } block_decl_specifiers_t; +static bool read_block_decl_specifiers(block_decl_specifiers_t *spec) +{ + bool saw_specifier = false; + + while (lex_peek(T_static, NULL) || lex_peek(T_extern, NULL) || + lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || + lex_peek(T_register, NULL) || lex_peek(T_auto, NULL)) { + saw_specifier = true; + if (lex_accept(T_static)) { + if (spec->is_static) + error_at("duplicate static storage class specifier", + cur_token_loc()); + spec->is_static = true; + } else if (lex_accept(T_extern)) { + if (spec->is_extern) + error_at("duplicate extern storage class specifier", + cur_token_loc()); + spec->is_extern = true; + } else if (lex_accept(T_register)) { + if (spec->is_register) + error_at("duplicate register storage class specifier", + cur_token_loc()); + spec->is_register = true; + } else if (lex_accept(T_auto)) { + if (spec->is_auto) + error_at("duplicate auto storage class specifier", + cur_token_loc()); + spec->is_auto = true; + } else if (lex_accept(T_volatile)) { + spec->is_volatile = true; + } else { + lex_expect(T_const); + spec->is_const = true; + } + } + if ((spec->is_static && + (spec->is_register || spec->is_extern || spec->is_auto)) || + (spec->is_register && (spec->is_extern || spec->is_auto)) || + (spec->is_extern && spec->is_auto)) + error_at("incompatible storage class specifiers", cur_token_loc()); + return saw_specifier; +} + static basic_block_t *read_block_declarators( block_t *parent, basic_block_t *bb, @@ -354,12 +394,7 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) char token[MAX_ID_LEN]; type_t *type; var_t *vd; - bool is_const = false; - bool is_static = false; - bool is_extern = false; - bool is_register = false; - bool is_auto = false; - bool is_volatile = false; + block_decl_specifiers_t spec = {0}; bool saw_decl_specifier = false; bool for_decl_semicolon_consumed = false; @@ -383,42 +418,8 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) * emits no setup IR. */ handle_block_typedef_statement(blk, setup); - for_decl_semicolon_consumed = true; } else if (!lex_accept(T_semicolon)) { - while (lex_peek(T_static, NULL) || lex_peek(T_extern, NULL) || - lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || - lex_peek(T_register, NULL) || lex_peek(T_auto, NULL)) { - saw_decl_specifier = true; - if (lex_accept(T_static)) { - if (is_static) - error_at("duplicate static storage class specifier", - cur_token_loc()); - is_static = true; - } else if (lex_accept(T_extern)) { - if (is_extern) - error_at("duplicate extern storage class specifier", - cur_token_loc()); - is_extern = true; - } else if (lex_accept(T_register)) { - if (is_register) - error_at("duplicate register storage class specifier", - cur_token_loc()); - is_register = true; - } else if (lex_accept(T_auto)) { - if (is_auto) - error_at("duplicate auto storage class specifier", - cur_token_loc()); - is_auto = true; - } else if (lex_accept(T_volatile)) { - is_volatile = true; - } else { - lex_expect(T_const); - is_const = true; - } - } - if ((is_static && (is_register || is_extern || is_auto)) || - (is_register && (is_extern || is_auto)) || (is_extern && is_auto)) - error_at("incompatible storage class specifiers", cur_token_loc()); + saw_decl_specifier = read_block_decl_specifiers(&spec); bool has_builtin_type = lex_peek(T_signed, NULL) || lex_peek(T_unsigned, NULL) || @@ -447,22 +448,14 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) if (!type && saw_decl_specifier) error_at("declaration specifier requires a type", cur_token_loc()); if (type) { - block_decl_specifiers_t spec = {0}; - /* C99 6.8.5.3 admits only automatic or register object declarations * here. The default mode retains its historical block-scope * static/extern extension. */ - if (strict_c99 && (is_static || is_extern)) + if (strict_c99 && (spec.is_static || spec.is_extern)) error_at( "C99 for initializer permits only auto or register objects", cur_token_loc()); - spec.is_const = is_const; - spec.is_static = is_static; - spec.is_extern = is_extern; - spec.is_register = is_register; - spec.is_auto = is_auto; - spec.is_volatile = is_volatile; /* The loop scope takes the ordinary block declaration lowering, * which also consumes the terminating semicolon. @@ -491,17 +484,12 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) lex_expect(T_semicolon); } else { /* always true */ - vd = require_var(blk); - vd->init_val = 1; - vd->var_name = gen_name(); + vd = load_constant(blk, cond_, 1, TY_int); opstack_push(vd); - add_insn(blk, cond_, OP_load_constant, vd, NULL, NULL, 0, NULL); } bb_connect(cond_, for_end, ELSE); - vd = opstack_pop(); - reject_record_operand(vd); - add_insn(blk, cond_, OP_branch, NULL, vd, NULL, 0, NULL); + emit_control_branch(blk, cond_); basic_block_t *inc_ = bb_create(blk); continue_bb_push(inc_); @@ -553,8 +541,6 @@ basic_block_t *handle_for_statement(block_t *parent, basic_block_t *bb) /* A do-while loop, whose condition is tested after the body. */ basic_block_t *handle_do_statement(block_t *parent, basic_block_t *bb) { - var_t *vd; - basic_block_t *n = bb_create(parent); bb_connect(bb, n, NEXT); bb = n; @@ -570,13 +556,7 @@ basic_block_t *handle_do_statement(block_t *parent, basic_block_t *bb) bb_connect(do_body, cond_, NEXT); lex_expect(T_while); - lex_expect(T_open_bracket); - read_control_expression(parent, &cond_); - lex_expect(T_close_bracket); - - vd = opstack_pop(); - reject_record_operand(vd); - add_insn(parent, cond_, OP_branch, NULL, vd, NULL, 0, NULL); + read_control_branch(parent, &cond_); lex_expect(T_semicolon); @@ -698,7 +678,7 @@ type_t *read_enum_specifier(block_t *parent, bool *is_definition) int val = 0; type_t *type = NULL; bool has_tag = false; - block_t *scope = parent ? parent : GLOBAL_BLOCK; + block_t *scope = parent ? parent : CURRENT_TU_SCOPE; lex_expect(T_enum); if (lex_peek(T_identifier, token)) { @@ -735,6 +715,10 @@ type_t *read_enum_specifier(block_t *parent, bool *is_definition) } first = false; if (!parent) { + if (find_block_ordinary(CURRENT_TU_SCOPE, token, ORDINARY_ANY, + NULL)) + error_at("identifier redeclared as a different kind of symbol", + cur_token_loc()); add_constant(token, val); continue; } @@ -853,9 +837,7 @@ static void emit_scalar_initializer(block_t *parent, var->type->base_type == TYPE_short) && expr_result->var_name[0] == '.') { /* Extract first element from compound literal array */ - var_t *first_elem = require_var(parent); - first_elem->type = var->type; - first_elem->var_name = gen_name(); + var_t *first_elem = name_var(require_typed_var(parent, var->type)); /* Read first element from array at offset 0 expr_result is the array * itself, so we can read directly from it @@ -887,7 +869,6 @@ static void read_block_declarator_storage(block_t *parent, add_insn(spec->is_static ? GLOBAL_BLOCK : parent, spec->is_static ? GLOBAL_FUNC->bbs : *bb, OP_allocat, var, NULL, NULL, 0, NULL); - add_symbol(*bb, var); if (lex_accept(T_assign)) { validate_string_array_initializer(var); if (spec->is_static) { @@ -930,18 +911,14 @@ static void read_block_declarator_storage(block_t *parent, struct_type->base_struct) struct_type = struct_type->base_struct; - var_t *struct_addr = require_var(parent); - struct_addr->var_name = gen_name(); + var_t *struct_addr = require_named_var(parent); add_insn(parent, *bb, OP_address_of, struct_addr, var, NULL, 0, NULL); lex_expect(T_open_curly); parse_struct_field_init(parent, bb, struct_type, struct_addr); lex_expect(T_close_curly); } else { - if (!read_assignment_expression(parent, bb)) { - read_expr(parent, bb); - read_ternary_operation(parent, bb); - } + read_assignment_or_expression(parent, bb); emit_scalar_initializer(parent, bb, var, opstack_pop()); } @@ -1140,41 +1117,7 @@ basic_block_t *handle_declaration(block_t *parent, basic_block_t *bb) type_t *type; block_decl_specifiers_t spec = {0}; - while (lex_peek(T_static, NULL) || lex_peek(T_extern, NULL) || - lex_peek(T_const, NULL) || lex_peek(T_volatile, NULL) || - lex_peek(T_register, NULL) || lex_peek(T_auto, NULL)) { - if (lex_accept(T_static)) { - if (spec.is_static) - error_at("duplicate static storage class specifier", - cur_token_loc()); - spec.is_static = true; - } else if (lex_accept(T_extern)) { - if (spec.is_extern) - error_at("duplicate extern storage class specifier", - cur_token_loc()); - spec.is_extern = true; - } else if (lex_accept(T_register)) { - if (spec.is_register) - error_at("duplicate register storage class specifier", - cur_token_loc()); - spec.is_register = true; - } else if (lex_accept(T_auto)) { - if (spec.is_auto) - error_at("duplicate auto storage class specifier", - cur_token_loc()); - spec.is_auto = true; - } else if (lex_accept(T_volatile)) { - spec.is_volatile = true; - } else { - lex_expect(T_const); - spec.is_const = true; - } - } - if ((spec.is_static && - (spec.is_register || spec.is_extern || spec.is_auto)) || - (spec.is_register && (spec.is_extern || spec.is_auto)) || - (spec.is_extern && spec.is_auto)) - error_at("incompatible storage class specifiers", cur_token_loc()); + read_block_decl_specifiers(&spec); if (floating_type_starts_here()) error_at("Floating point types are not yet supported", cur_token_loc()); @@ -1298,7 +1241,6 @@ basic_block_t *handle_declaration(block_t *parent, basic_block_t *bb) basic_block_t *n = bb_create(parent); bb_connect(bb, n, NEXT); add_label(token, n); - add_insn(parent, n, OP_label, NULL, NULL, NULL, 0, token); return n; } return read_full_expression_statement(parent, bb); @@ -1412,8 +1354,10 @@ basic_block_t *handle_block_typedef_statement(block_t *parent, !direct_function_signature->returns_aggregate; bool callback_pointer_alias = decl.is_func && decl.func_signature && - decl.parenthesized_function_pointer_level == 1 && !decl.ptr_level && - !decl.array_size && !decl.pointee_array_size && !base->ptr_level && + decl.parenthesized_function_pointer_level == 1 && + !decl.array_size && !decl.pointee_array_size && + (!base->ptr_level || + (!base->func_signature && !base->pointee_func_signature)) && !is_record_type(base) && !base->is_floating && !callback_signature->va_args && !callback_signature->returns_aggregate; @@ -1581,6 +1525,7 @@ basic_block_t *handle_block_typedef_statement(block_t *parent, alias->pointer_const_mask = base->pointer_const_mask | (decl.pointer_const_mask << base->ptr_level); + alias->pointer_volatile_mask = decl.pointer_volatile_mask; /* `const ptr_t` qualifies the pointer that the base typedef hides, not * its pointee, for every declarator of the list. @@ -1598,6 +1543,14 @@ basic_block_t *handle_block_typedef_statement(block_t *parent, } alias->is_volatile_qualified = decl.is_volatile || base->is_volatile_qualified; + int volatile_depth = + base->ptr_level + (base->func_signature && + !base->is_direct_function_type && + !base->ptr_level); + if (specifier_volatile && volatile_depth && volatile_depth <= 32) { + alias->pointer_volatile_mask |= 1U << (volatile_depth - 1); + alias->is_volatile_qualified = base->is_volatile_qualified; + } if (direct_function_alias) { /* The stars of `char *name_t(void)` belong to the return type, * which the signature already records. @@ -1608,6 +1561,22 @@ basic_block_t *handle_block_typedef_statement(block_t *parent, alias->is_direct_function_type = true; } if (callback_pointer_alias) { + int return_depth = effective_pointer_depth(&decl); + alias->ptr_level = 0; + alias->pointer_const_mask = + decl.ptr_level < 32 ? decl.pointer_const_mask >> decl.ptr_level + : 0; + alias->pointer_volatile_mask = + return_depth < 32 ? decl.pointer_volatile_mask >> return_depth + : 0; + if (base->func_signature && !base->is_direct_function_type && + !base->ptr_level) { + alias->pointer_const_mask = decl.callback_is_const ? 1U : 0; + alias->pointer_volatile_mask = + decl.callback_is_volatile ? 1U : 0; + } + alias->is_const_qualified = false; + alias->is_volatile_qualified = false; alias->size = PTR_SIZE; alias->func_signature = decl.func_signature; alias->is_direct_function_type = false; @@ -1698,12 +1667,7 @@ basic_block_t *handle_block_typedef_statement(block_t *parent, if (decl.ptr_level == 1 && !direct_array && !direct_pointee_array) alias_callback_array_pointer(alias, base, decl.pointer_const_mask); if (direct_pointee_array) { - alias->pointee_array_size = decl.pointee_array_size; - alias->pointee_array_dim2 = decl.pointee_array_dim2; - alias->pointee_array_dim3 = decl.pointee_array_dim3; - alias->pointee_array_dim4 = decl.pointee_array_dim4; - alias->pointee_array_element_ptr_level = - decl.pointee_array_element_ptr_level; + copy_pointee_array_shape_to_type(alias, &decl); alias->pointee_array_element_type = base; } if (alias->ptr_level) @@ -1791,7 +1755,7 @@ basic_block_t *read_body_statement(block_t *parent, basic_block_t *bb) return handle_do_statement(parent, bb); if (lex_accept(T_goto)) - return handle_goto_statement(parent, bb); + return handle_goto_statement(bb); hoist_storage_class_specifiers(); if (lex_peek(T_typedef, NULL)) @@ -1801,9 +1765,6 @@ basic_block_t *read_body_statement(block_t *parent, basic_block_t *bb) if (lex_accept(T_semicolon)) return bb; - if (grouped_scalar_pointee_row_store_starts(parent)) - return handle_grouped_scalar_pointee_row_store(parent, bb); - /* These cannot begin a declaration, so they are unambiguously expression * statements. Identifiers remain delegated to handle_declaration(), which * first checks whether they name a type. @@ -1847,8 +1808,6 @@ basic_block_t *read_code_block(func_t *func, block_t *parent, basic_block_t *bb) return bb; } -void var_add_killed_bb(var_t *var, basic_block_t *bb); - void read_func_body(func_t *func) { block_t *blk = add_block(NULL, func); @@ -1859,9 +1818,6 @@ void read_func_body(func_t *func) func->sret_def.type = func->return_def.type; func->sret_def.ptr_level = 1; func->sret_def.var_name = "__shecc_sret"; - func->sret_def.base = &func->sret_def; - add_symbol(func->bbs, &func->sret_def); - var_add_killed_bb(&func->sret_def, func->bbs); } for (int i = 0; i < func->num_params; i++) { @@ -1872,18 +1828,15 @@ void read_func_body(func_t *func) /* A volatile parameter lives in its slot, as a volatile local does. */ if (func->param_defs[i].is_volatile) func->param_defs[i].address_taken = true; - add_symbol(func->bbs, &func->param_defs[i]); - func->param_defs[i].base = &func->param_defs[i]; - var_add_killed_bb(&func->param_defs[i], func->bbs); } + vir_frontend_begin(func); basic_block_t *body = read_code_block(func, NULL, func->bbs); if (body) bb_connect(body, func->exit, NEXT); for (int i = 0; i < backpatch_bb_idx; i++) { basic_block_t *bb = backpatch_bb[i]; - insn_t *g = bb->insn_list.tail; - label_t *label = find_label(g->str); + label_t *label = find_label(backpatch_label[i]); if (!label) error_at("goto label undefined", cur_token_loc()); @@ -1901,4 +1854,5 @@ void read_func_body(func_t *func) backpatch_bb_idx = 0; label_idx = 0; + vir_frontend_end(func); } diff --git a/src/parser.c b/src/parser.c index 5fbb8329..b7724c78 100644 --- a/src/parser.c +++ b/src/parser.c @@ -13,6 +13,8 @@ #include "defs.h" #include "globals.c" +func_t *find_visible_func(char *name, block_t *scope); + /* C language syntactic analyzer */ int global_var_idx = 0; @@ -26,6 +28,7 @@ int continue_pos_idx = 0; label_t labels[MAX_LABELS]; int label_idx = 0; basic_block_t *backpatch_bb[MAX_LABELS]; +char *backpatch_label[MAX_LABELS]; int backpatch_bb_idx = 0; /* stack of the operands of 3AC */ @@ -60,111 +63,151 @@ typedef struct fixed_array_shape { source_location_t *cur_token_loc(void); source_location_t *next_token_loc(void); +/* Advance one initializer scan token, returning false at its boundary. */ +static bool initializer_scan_continue(token_t *token, + int *depth, + bool close_curly) +{ + if (token->kind == T_open_bracket) + (*depth)++; + else if (token->kind == T_close_bracket) { + if (!*depth) + return false; + (*depth)--; + } else if (!*depth && + (token->kind == T_semicolon || token->kind == T_comma || + (close_curly && token->kind == T_close_curly))) + return false; + return true; +} + +static token_t *skip_balanced_delimiter(token_t *token, + token_kind_t open, + token_kind_t close) +{ + int depth = 0; + + for (; token; token = token->next) { + if (token->kind == open) + depth++; + else if (token->kind == close && !--depth) + return token; + } + return NULL; +} + basic_block_t *read_body_statement(block_t *parent, basic_block_t *bb); -static fixed_array_shape_t fixed_array_shape_from_type(const type_t *type) +static fixed_array_shape_t fixed_array_shape_from_bounds(int size, + int dim2, + int dim3, + int dim4) { fixed_array_shape_t shape = {0}; int trailing = 1; - if (!type->array_size) + if (!size) return shape; - if (type->array_dim2) - trailing *= type->array_dim2; - if (type->array_dim3) - trailing *= type->array_dim3; - if (type->array_dim4) - trailing *= type->array_dim4; - shape.bounds[shape.rank++] = type->array_size / trailing; - if (type->array_dim2) - shape.bounds[shape.rank++] = type->array_dim2; - if (type->array_dim3) - shape.bounds[shape.rank++] = type->array_dim3; - if (type->array_dim4) - shape.bounds[shape.rank++] = type->array_dim4; + if (dim2) + trailing *= dim2; + if (dim3) + trailing *= dim3; + if (dim4) + trailing *= dim4; + shape.bounds[shape.rank++] = size / trailing; + if (dim2) + shape.bounds[shape.rank++] = dim2; + if (dim3) + shape.bounds[shape.rank++] = dim3; + if (dim4) + shape.bounds[shape.rank++] = dim4; return shape; } -static fixed_array_shape_t fixed_array_shape_from_var(const var_t *var) +static fixed_array_shape_t fixed_array_shape_from_type(const type_t *type) { - type_t shape = {0}; + return fixed_array_shape_from_bounds(type->array_size, type->array_dim2, + type->array_dim3, type->array_dim4); +} - shape.array_size = var->array_size; - shape.array_dim2 = var->array_dim2; - shape.array_dim3 = var->array_dim3; - shape.array_dim4 = var->array_dim4; - return fixed_array_shape_from_type(&shape); +static fixed_array_shape_t fixed_array_shape_from_var(const var_t *var) +{ + return fixed_array_shape_from_bounds(var->array_size, var->array_dim2, + var->array_dim3, var->array_dim4); } static fixed_array_shape_t fixed_array_shape_from_pointee_type( const type_t *type) { - type_t shape = {0}; - - shape.array_size = type->pointee_array_size; - shape.array_dim2 = type->pointee_array_dim2; - shape.array_dim3 = type->pointee_array_dim3; - shape.array_dim4 = type->pointee_array_dim4; - return fixed_array_shape_from_type(&shape); + return fixed_array_shape_from_bounds( + type->pointee_array_size, type->pointee_array_dim2, + type->pointee_array_dim3, type->pointee_array_dim4); } static fixed_array_shape_t fixed_array_shape_from_pointee_var(const var_t *var) { - type_t shape = {0}; - - shape.array_size = var->pointee_array_size; - shape.array_dim2 = var->pointee_array_dim2; - shape.array_dim3 = var->pointee_array_dim3; - shape.array_dim4 = var->pointee_array_dim4; - return fixed_array_shape_from_type(&shape); + return fixed_array_shape_from_bounds( + var->pointee_array_size, var->pointee_array_dim2, + var->pointee_array_dim3, var->pointee_array_dim4); } -static void fixed_array_shape_to_type(type_t *type, - const fixed_array_shape_t *shape) +static void fixed_array_shape_write(const fixed_array_shape_t *shape, + int *size, + int *dim2, + int *dim3, + int *dim4) { - type->array_size = shape->rank ? shape->bounds[0] : 0; - type->array_dim2 = type->array_dim3 = type->array_dim4 = 0; + *size = shape->rank ? shape->bounds[0] : 0; + *dim2 = *dim3 = *dim4 = 0; for (int i = 1; i < shape->rank; i++) { - type->array_size *= shape->bounds[i]; + *size *= shape->bounds[i]; if (i == 1) - type->array_dim2 = shape->bounds[i]; + *dim2 = shape->bounds[i]; else if (i == 2) - type->array_dim3 = shape->bounds[i]; + *dim3 = shape->bounds[i]; else - type->array_dim4 = shape->bounds[i]; + *dim4 = shape->bounds[i]; } } +static void fixed_array_shape_to_type(type_t *type, + const fixed_array_shape_t *shape) +{ + fixed_array_shape_write(shape, &type->array_size, &type->array_dim2, + &type->array_dim3, &type->array_dim4); +} + static void fixed_array_shape_to_var(var_t *var, const fixed_array_shape_t *shape) { - var->array_size = shape->rank ? shape->bounds[0] : 0; - var->array_dim2 = var->array_dim3 = var->array_dim4 = 0; - for (int i = 1; i < shape->rank; i++) { - var->array_size *= shape->bounds[i]; - if (i == 1) - var->array_dim2 = shape->bounds[i]; - else if (i == 2) - var->array_dim3 = shape->bounds[i]; - else - var->array_dim4 = shape->bounds[i]; - } + fixed_array_shape_write(shape, &var->array_size, &var->array_dim2, + &var->array_dim3, &var->array_dim4); } static void fixed_array_shape_to_pointee_var(var_t *var, const fixed_array_shape_t *shape) { - var->pointee_array_size = shape->rank ? shape->bounds[0] : 0; - var->pointee_array_dim2 = var->pointee_array_dim3 = - var->pointee_array_dim4 = 0; - for (int i = 1; i < shape->rank; i++) { - var->pointee_array_size *= shape->bounds[i]; - if (i == 1) - var->pointee_array_dim2 = shape->bounds[i]; - else if (i == 2) - var->pointee_array_dim3 = shape->bounds[i]; - else - var->pointee_array_dim4 = shape->bounds[i]; - } + fixed_array_shape_write(shape, &var->pointee_array_size, + &var->pointee_array_dim2, &var->pointee_array_dim3, + &var->pointee_array_dim4); +} + +static void set_pointee_array_shape(var_t *var, + const fixed_array_shape_t *shape, + int element_ptr_level) +{ + fixed_array_shape_to_pointee_var(var, shape); + var->pointee_array_element_ptr_level = element_ptr_level; +} + +static void copy_pointee_array_shape_to_type(type_t *type, const var_t *source) +{ + fixed_array_shape_t shape = fixed_array_shape_from_pointee_var(source); + + fixed_array_shape_write( + &shape, &type->pointee_array_size, &type->pointee_array_dim2, + &type->pointee_array_dim3, &type->pointee_array_dim4); + type->pointee_array_element_ptr_level = + source->pointee_array_element_ptr_level; } static fixed_array_shape_t fixed_array_shape_prepend( @@ -188,21 +231,18 @@ static int fixed_array_shape_stride(const fixed_array_shape_t *shape, int element_size); static bool fixed_array_shape_drop_outer(fixed_array_shape_t *shape); +static int fixed_array_trailing_count(int dim2, int dim3, int dim4) +{ + return (dim2 ? dim2 : 1) * (dim3 ? dim3 : 1) * (dim4 ? dim4 : 1); +} + /* Elements in one outer element of @var's direct array. Only the inner bounds - * matter, which is what keeps this right while an unsized outer bound is still - * being inferred from its initializer and array_size is not yet known. + * matter while an unsized outer bound is inferred from its initializer. */ static int fixed_array_inner_count(const var_t *var) { - int count = 1; - - if (var->array_dim2) - count *= var->array_dim2; - if (var->array_dim3) - count *= var->array_dim3; - if (var->array_dim4) - count *= var->array_dim4; - return count; + return fixed_array_trailing_count(var->array_dim2, var->array_dim3, + var->array_dim4); } /* Rewrite @var's direct array bounds to those of one element of the array: a @@ -274,6 +314,19 @@ static void alias_callback_array_pointer(type_t *alias, basic_block_t *handle_block_typedef_statement(block_t *parent, basic_block_t *bb); +static var_t *scale_pointer_index(block_t *parent, + basic_block_t *bb, + var_t *index, + int stride, + bool constant_scale); +static var_t *lower_array_element_address(block_t *parent, + basic_block_t *bb, + var_t *base, + var_t *index, + type_t *element_type, + int pointer_level, + int stride, + bool rematerialize_scale); bool read_assignment_expression(block_t *parent, basic_block_t **bb); bool is_null_pointer_constant(var_t *value); bool is_record_type(const type_t *type); @@ -333,7 +386,8 @@ void push_object_at(block_t *parent, var_t *lower_reference_update(var_t *object, opcode_t op, block_t *parent, - basic_block_t **bb); + basic_block_t **bb, + bool need_updated_value); void lower_member_postfix(var_t *address, type_t *record_type, bool arrow_first, @@ -441,12 +495,12 @@ bool global_compound_literal_starts_here(void) if (!lex_peek(T_open_bracket, NULL)) return false; next = cur_token->next->next; - if (!next || - !((next->kind == T_identifier && find_type(next->literal, true)) || - next->kind == T_struct || next->kind == T_union || - next->kind == T_enum || next->kind == T_const || - next->kind == T_volatile || next->kind == T_signed || - next->kind == T_unsigned || next->kind == T_long)) + if (!next || !((next->kind == T_identifier && + find_visible_type(next->literal, CURRENT_TU_SCOPE)) || + next->kind == T_struct || next->kind == T_union || + next->kind == T_enum || next->kind == T_const || + next->kind == T_volatile || next->kind == T_signed || + next->kind == T_unsigned || next->kind == T_long)) return false; /* Only a brace after the type name makes a compound literal; otherwise the @@ -474,7 +528,7 @@ bool grouped_global_function_designator_starts_here(bool allow_address) return token && token->kind == T_open_bracket && token->next && token->next->kind == T_identifier && token->next->next && token->next->next->kind == T_close_bracket && - find_func(token->next->literal); + find_visible_func(token->next->literal, CURRENT_TU_SCOPE); } /* `&*function` and `*&function` are both the original function designator. @@ -490,7 +544,8 @@ bool global_function_designator_tokens(token_t *token, if (!token) return false; - if (token->kind == T_identifier && find_func(token->literal)) { + if (token->kind == T_identifier && + find_visible_func(token->literal, CURRENT_TU_SCOPE)) { *after = token->next; *identifier = token; return true; @@ -623,27 +678,31 @@ var_t *require_var(block_t *blk) } var_t *var = arena_calloc(BLOCK_ARENA, 1, sizeof(var_t)); + stats_var_count++; var_list->elements[var_list->size++] = var; /* var_name is a pointer now; every reader dereferences it unconditionally, * so an unnamed variable points at the empty string rather than NULL. */ var->var_name = ""; - var->consumed = -1; - var->phys_reg = -1; - var->phys_reg_hi = -1; - var->first_use = -1; - var->last_use = -1; - var->use_count = 0; - var->base = var; var->type = TY_int; var->scope = blk; var->space_is_allocated = false; var->has_backing_storage = false; - var->ofs_based_on_stack_top = false; return var; } +static var_t *name_var(var_t *var) +{ + var->var_name = gen_name(); + return var; +} + +static var_t *require_named_var(block_t *blk) +{ + return name_var(require_var(blk)); +} + var_t *require_typed_var(block_t *blk, type_t *type) { if (!type) @@ -654,6 +713,17 @@ var_t *require_typed_var(block_t *blk, type_t *type) return var; } +static var_t *load_constant(block_t *parent, + basic_block_t *bb, + int value, + type_t *type) +{ + var_t *constant = name_var(require_typed_var(parent, type)); + constant->init_val = value; + add_insn(parent, bb, OP_load_constant, constant, NULL, NULL, 0, NULL); + return constant; +} + /* Function-address operands carry a function name, but are not declarations in * the current scope. Keeping them out of the local lookup list lets find_var() * distinguish a resolved function-pointer variable from a generated function @@ -666,6 +736,17 @@ var_t *require_func_symbol_var(block_t *blk) return var; } +/* A resolved function designator used as a parser expression value. */ +var_t *function_designator_value(block_t *block, func_t *func) +{ + var_t *value = require_func_symbol_var(block); + + value->is_func = true; + value->is_function_designator = true; + value->var_name = func->return_def.var_name; + return value; +} + var_t *require_typed_ptr_var(block_t *blk, type_t *type, int ptr) { var_t *var = require_typed_var(blk, type); @@ -812,7 +893,6 @@ type_t *pointee_type_from_pointer_typedef(type_t *type) * based on the represented type. Records retain nominal identity. */ bool compatible_function_signature(const func_t *left, const func_t *right); -func_t *find_visible_func(char *name, block_t *scope); var_t *materialize_function_designator(block_t *parent, basic_block_t **bb, var_t *value); @@ -863,7 +943,147 @@ static bool is_plain_record_alias(const type_t *type) !type->func_signature && !type->pointee_func_signature; } -bool compatible_decl_type(const type_t *left, const type_t *right) +typedef struct compatible_record_pair { + const type_t *left; + const type_t *right; + const struct compatible_record_pair *next; +} compatible_record_pair_t; + +static bool compatible_decl_type_with_records( + const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen); +static bool compatible_function_signature_with_records( + const func_t *left, + const func_t *right, + const compatible_record_pair_t *seen); +static bool compatible_function_param_decl_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen); + +static bool compatible_optional_decl_type_with_records( + const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen) +{ + return !!left == !!right && + (!left || compatible_decl_type_with_records(left, right, seen)); +} + +static bool compatible_var_function_type_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) +{ + if (left->is_func != right->is_func || + !!left->pointee_func_signature != !!right->pointee_func_signature) + return false; + if (left->is_func && + (callback_pointer_indirection(left) != + callback_pointer_indirection(right) || + !compatible_function_signature_with_records( + left->func_signature, right->func_signature, seen))) + return false; + return !left->pointee_func_signature || + compatible_function_signature_with_records( + left->pointee_func_signature, right->pointee_func_signature, + seen); +} + +static bool compatible_record_member(const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) +{ + return !strcmp(left->var_name, right->var_name) && + left->ptr_level == right->ptr_level && + compatible_var_function_type_with_records(left, right, seen) && + (!left->is_func || + (left->callback_is_const == right->callback_is_const && + left->callback_is_volatile == right->callback_is_volatile)) && + left->array_size == right->array_size && + left->has_unsized_array == right->has_unsized_array && + left->array_dim2 == right->array_dim2 && + left->array_dim3 == right->array_dim3 && + left->array_dim4 == right->array_dim4 && + left->pointee_array_size == right->pointee_array_size && + left->has_direct_pointee_array_declarator == + right->has_direct_pointee_array_declarator && + left->pointee_array_dim2 == right->pointee_array_dim2 && + left->pointee_array_dim3 == right->pointee_array_dim3 && + left->pointee_array_dim4 == right->pointee_array_dim4 && + left->pointee_array_element_ptr_level == + right->pointee_array_element_ptr_level && + left->type && right->type && + compatible_optional_decl_type_with_records( + left->type->pointee_array_element_type, + right->type->pointee_array_element_type, seen) && + left->is_flexible_array_member == right->is_flexible_array_member && + left->is_const_qualified == right->is_const_qualified && + left->is_volatile == right->is_volatile && + left->pointer_const_mask == right->pointer_const_mask && + left->pointer_volatile_mask == right->pointer_volatile_mask && + left->is_bitfield == right->is_bitfield && + left->bit_width == right->bit_width && + left->bit_storage_size == right->bit_storage_size && + compatible_decl_type_with_records(left->type, right->type, seen); +} + +static bool compatible_record_type(const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen) +{ + compatible_record_pair_t pair; + + if (left->base_type != right->base_type || + strcmp(left->type_name, right->type_name)) + return false; + for (const compatible_record_pair_t *p = seen; p; p = p->next) + if ((p->left == left && p->right == right) || + (p->left == right && p->right == left)) + return true; + + /* A declaration of an incomplete tagged type is compatible with the + * completed declaration in another translation unit. + */ + if (!left->definition_started || !right->definition_started) + return true; + if (left->num_fields != right->num_fields) + return false; + + pair.left = left; + pair.right = right; + pair.next = seen; + if (left->base_type == TYPE_struct) { + for (int i = 0; i < left->num_fields; i++) + if (!compatible_record_member(&left->fields[i], &right->fields[i], + &pair)) + return false; + return true; + } + + bool matched[MAX_FIELDS] = {false}; + for (int i = 0; i < left->num_fields; i++) { + bool found = false; + + for (int j = 0; j < right->num_fields; j++) { + if (matched[j] || !compatible_record_member( + &left->fields[i], &right->fields[j], &pair)) + continue; + matched[j] = true; + found = true; + break; + } + if (!found) + return false; + } + return true; +} + +static bool compatible_decl_type_with_records( + const type_t *left, + const type_t *right, + const compatible_record_pair_t *seen) { /* A typedef of a struct or union tag denotes the tag's type. Its own * qualifiers were copied into each declarator, which the callers compare. @@ -885,8 +1105,8 @@ bool compatible_decl_type(const type_t *left, const type_t *right) left->pointer_const_mask == right->pointer_const_mask && left->is_const_qualified == right->is_const_qualified && left->is_volatile_qualified == right->is_volatile_qualified && - compatible_function_signature(left->func_signature, - right->func_signature); + compatible_function_signature_with_records( + left->func_signature, right->func_signature, seen); /* So may a callback slot typedef, `int (**slot_t)(int)`. */ if (left && right && left->pointee_func_signature && @@ -895,10 +1115,16 @@ bool compatible_decl_type(const type_t *left, const type_t *right) left->pointer_const_mask == right->pointer_const_mask && left->is_const_qualified == right->is_const_qualified && left->is_volatile_qualified == right->is_volatile_qualified && - compatible_function_signature(left->pointee_func_signature, - right->pointee_func_signature); - if (!left || !right || left->base_type != right->base_type || - left->size != right->size || left->ptr_level != right->ptr_level || + compatible_function_signature_with_records( + left->pointee_func_signature, right->pointee_func_signature, + seen); + if (!left || !right) + return false; + if ((left->base_type == TYPE_struct || left->base_type == TYPE_union) && + (right->base_type == TYPE_struct || right->base_type == TYPE_union)) + return compatible_record_type(left, right, seen); + if (left->base_type != right->base_type || left->size != right->size || + left->ptr_level != right->ptr_level || left->is_unsigned != right->is_unsigned || left->is_signed_char != right->is_signed_char || left->is_bool != right->is_bool) @@ -906,10 +1132,8 @@ bool compatible_decl_type(const type_t *left, const type_t *right) if (!!left->func_signature != !!right->func_signature) return false; if (left->func_signature && - !compatible_function_signature(left->func_signature, - right->func_signature)) - return false; - if (left->base_type == TYPE_struct || left->base_type == TYPE_union) + !compatible_function_signature_with_records( + left->func_signature, right->func_signature, seen)) return false; if (left->base_type == TYPE_typedef && (left->num_fields || right->num_fields)) @@ -917,6 +1141,11 @@ bool compatible_decl_type(const type_t *left, const type_t *right) return true; } +bool compatible_decl_type(const type_t *left, const type_t *right) +{ + return compatible_decl_type_with_records(left, right, NULL); +} + /* Function-pointer declarators carry their pointee function type separately * from the pointer-sized var_t representation. Redeclarations must compare that * syntax-only signature as well: comparing just the outer storage shape accepts @@ -924,30 +1153,29 @@ bool compatible_decl_type(const type_t *left, const type_t *right) * side has no prototype, retain the existing old-style policy. */ bool compatible_function_param_decl(const var_t *left, const var_t *right) +{ + return compatible_function_param_decl_with_records(left, right, NULL); +} + +static bool compatible_function_param_decl_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) { bool left_points_to_value; bool right_points_to_value; - if (!left || !right || left->is_func != right->is_func) + if (!left || !right || + !compatible_var_function_type_with_records(left, right, seen)) return false; if (left->is_func) { if (!left->func_signature || !right->func_signature) return false; - return callback_pointer_indirection(left) == - callback_pointer_indirection(right) && - compatible_function_signature(left->func_signature, - right->func_signature); - } - - if (left->pointee_func_signature || right->pointee_func_signature) { - if (!left->pointee_func_signature || !right->pointee_func_signature || - !compatible_function_signature(left->pointee_func_signature, - right->pointee_func_signature)) - return false; + return true; } - if (!compatible_decl_type(left->type, right->type) || + if (!compatible_decl_type_with_records(left->type, right->type, seen) || left->ptr_level != right->ptr_level) return false; @@ -990,12 +1218,60 @@ bool parameter_changes_under_default_promotion(const var_t *param) } bool compatible_function_signature(const func_t *left, const func_t *right) +{ + return compatible_function_signature_with_records(left, right, NULL); +} + +static bool compatible_function_return_decl_with_records( + const var_t *left, + const var_t *right, + const compatible_record_pair_t *seen) +{ + if (!left || !right || !left->type || !right->type) + return false; + int depth = left->ptr_level + left->type->ptr_level; + if (depth != right->ptr_level + right->type->ptr_level) + return false; + if (left->ptr_level != right->ptr_level || + !compatible_decl_type_with_records(left->type, right->type, seen)) { + if (left->type->func_signature || right->type->func_signature || + left->type->pointee_func_signature || + right->type->pointee_func_signature || left->type->array_size || + right->type->array_size || left->type->pointee_array_size || + right->type->pointee_array_size || left->pointee_array_size || + right->pointee_array_size || + !compatible_decl_type_with_records( + pointee_type_from_pointer_typedef(left->type), + pointee_type_from_pointer_typedef(right->type), seen)) + return false; + } + + /* Scalar and compact callback returns have only top-level qualifiers. An + * ordinary pointer return also keeps qualifiers on its reached objects. + */ + if (!depth) + return true; + unsigned int inner_mask = depth <= 32 ? (1U << (depth - 1)) - 1 : ~0U; + return (left->is_const_qualified || left->type->is_const_qualified) == + (right->is_const_qualified || right->type->is_const_qualified) && + (left->is_volatile || left->type->is_volatile_qualified) == + (right->is_volatile || right->type->is_volatile_qualified) && + (effective_pointer_const_mask(left) & inner_mask) == + (effective_pointer_const_mask(right) & inner_mask) && + ((left->pointer_volatile_mask | left->type->pointer_volatile_mask) & + inner_mask) == ((right->pointer_volatile_mask | + right->type->pointer_volatile_mask) & + inner_mask); +} + +static bool compatible_function_signature_with_records( + const func_t *left, + const func_t *right, + const compatible_record_pair_t *seen) { if (!left || !right || - !compatible_decl_type(left->return_def.type, right->return_def.type) || - left->return_def.ptr_level != right->return_def.ptr_level || - left->return_def.is_const_qualified != - right->return_def.is_const_qualified) + !compatible_function_return_decl_with_records(&left->return_def, + &right->return_def, seen)) return false; if (!left->has_prototype || !right->has_prototype) @@ -1004,8 +1280,8 @@ bool compatible_function_signature(const func_t *left, const func_t *right) left->va_args != right->va_args) return false; for (int i = 0; i < left->num_params; i++) - if (!compatible_function_param_decl(&left->param_defs[i], - &right->param_defs[i])) + if (!compatible_function_param_decl_with_records( + &left->param_defs[i], &right->param_defs[i], seen)) return false; return true; } @@ -1017,7 +1293,13 @@ bool compatible_function_signature(const func_t *left, const func_t *right) */ func_t *find_visible_func(char *name, block_t *scope) { - func_t *func = find_func(name); + var_t *unit_decl = find_tu_declaration(name); + func_t *func; + + if (unit_decl) + func = find_func(unit_decl->var_name); + else + func = find_func(name); var_t *binding; if (!func || !func->is_block_scope_only_declaration) @@ -1135,6 +1417,68 @@ int read_global_address_offset(block_t *scope, block_t *parent, basic_block_t *bb); +static void set_array_dimension(int *size, + int *dim2, + int *dim3, + int *dim4, + int dimension, + int bound, + bool infer_missing_size) +{ + if (!dimension) { + *size = bound; + } else { + int *dimensions[] = {dim2, dim3, dim4}; + + if (dimension > 3) + dimension = 3; + *dimensions[dimension - 1] = bound; + if (!*size && infer_missing_size) + *size = bound; + else + *size *= bound; + } +} + +static void read_array_shape_bounds(block_t *scope, + fixed_array_shape_t *shape, + char *rank_error, + char *missing_bound_error, + char *invalid_bound_error, + bool check_rank_first, + bool opening_consumed) +{ + /* Preserve each declarator's error order and any checks after `['. */ + while (opening_consumed || lex_accept(T_open_square)) { + int bound; + + opening_consumed = false; + if (check_rank_first && shape->rank >= MAX_FIXED_ARRAY_RANK) + error_at(rank_error, cur_token_loc()); + if (missing_bound_error && lex_peek(T_close_square, NULL)) + error_at(missing_bound_error, cur_token_loc()); + bound = read_const_expr(scope); + if (!check_rank_first && shape->rank >= MAX_FIXED_ARRAY_RANK) + error_at(rank_error, cur_token_loc()); + if (bound <= 0) + error_at(invalid_bound_error, cur_token_loc()); + shape->bounds[shape->rank++] = bound; + lex_expect(T_close_square); + } +} + +static void fixed_array_shape_apply(const fixed_array_shape_t *shape, + int *size, + int *dim2, + int *dim3, + int *dim4) +{ + /* Unlike fixed_array_shape_write(), retain any unused trailing dimensions. + */ + for (int i = 0; i < shape->rank; i++) + set_array_dimension(size, dim2, dim3, dim4, i, shape->bounds[i], false); +} + /* Enumerator values must be representable as int. */ bool checking_enum_constant = false; bool typed_global_literal_appears_before_initializer_end(token_t *token); @@ -1336,12 +1680,12 @@ void discard_operand(block_t *parent, basic_block_t *bb) bool unread = var && var == unread_volatile_object; unread_volatile_object = NULL; - if (!unread || var->is_func || var->array_size || - (!var->ptr_level && is_record_type(var->type))) + if (!unread || var->is_function_designator || var->array_size || + (!var->is_func && !var->ptr_level && is_record_type(var->type))) return; - var_t *copy = require_typed_ptr_var(parent, var->type, var->ptr_level); - copy->var_name = gen_name(); + var_t *copy = name_var(require_typed_ptr_var( + parent, var->type, var->is_func ? 1 : var->ptr_level)); add_insn(parent, bb, OP_assign, copy, var, NULL, 0, NULL); } @@ -1422,86 +1766,15 @@ int get_size(var_t *var) return var->type->size; } -int get_operator_prio(opcode_t op) -{ - /* https://www.cs.uic.edu/~i109/Notes/COperatorPrecedenceTable.pdf */ - switch (op) { - case OP_ternary: - return 3; - case OP_log_or: - return 4; - case OP_log_and: - return 5; - case OP_bit_or: - return 6; - case OP_bit_xor: - return 7; - case OP_bit_and: - return 8; - case OP_eq: - case OP_neq: - return 9; - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - return 10; - case OP_lshift: - case OP_rshift: - return 11; - case OP_add: - case OP_sub: - return 12; - case OP_mul: - case OP_div: - case OP_mod: - return 13; - default: - return 0; - } -} - opcode_t get_operator(void) { - opcode_t op = OP_generic; - if (lex_accept(T_plus)) - op = OP_add; - else if (lex_accept(T_minus)) - op = OP_sub; - else if (lex_accept(T_asterisk)) - op = OP_mul; - else if (lex_accept(T_divide)) - op = OP_div; - else if (lex_accept(T_mod)) - op = OP_mod; - else if (lex_accept(T_lshift)) - op = OP_lshift; - else if (lex_accept(T_rshift)) - op = OP_rshift; - else if (lex_accept(T_log_and)) - op = OP_log_and; - else if (lex_accept(T_log_or)) - op = OP_log_or; - else if (lex_accept(T_eq)) - op = OP_eq; - else if (lex_accept(T_noteq)) - op = OP_neq; - else if (lex_accept(T_lt)) - op = OP_lt; - else if (lex_accept(T_le)) - op = OP_leq; - else if (lex_accept(T_gt)) - op = OP_gt; - else if (lex_accept(T_ge)) - op = OP_geq; - else if (lex_accept(T_ampersand)) - op = OP_bit_and; - else if (lex_accept(T_bit_or)) - op = OP_bit_or; - else if (lex_accept(T_bit_xor)) - op = OP_bit_xor; - else if (lex_peek(T_question, NULL)) - op = OP_ternary; + opcode_t op = cur_token->next ? operator_for_token(cur_token->next->kind) + : OP_generic; + + if (op == OP_bit_not || op == OP_log_not) + return OP_generic; + if (op != OP_generic && op != OP_ternary) + lex_next(); return op; } @@ -1511,8 +1784,7 @@ var_t *promote_unchecked(block_t *block, type_t *target_type, int target_ptr) { - var_t *rd = require_typed_ptr_var(block, target_type, target_ptr); - rd->var_name = gen_name(); + var_t *rd = name_var(require_typed_ptr_var(block, target_type, target_ptr)); /* Encode both source and target sizes in src1: Lower 16 bits: target size * Upper 16 bits: source size This allows codegen to distinguish between @@ -1552,8 +1824,7 @@ var_t *truncate_unchecked(block_t *block, type_t *target_type, int target_ptr) { - var_t *rd = require_typed_ptr_var(block, target_type, target_ptr); - rd->var_name = gen_name(); + var_t *rd = name_var(require_typed_ptr_var(block, target_type, target_ptr)); add_insn(block, *bb, OP_trunc, rd, var, NULL, target_ptr ? PTR_SIZE : target_type->size, NULL); return rd; @@ -1567,31 +1838,33 @@ var_t *normalize_bool(block_t *block, basic_block_t **bb, var_t *var) if (is_bool_scalar(var->type, var->ptr_level)) return var; if (var->is_const && !var->ptr_level) { - rd = require_typed_var(block, TY_bool); - rd->var_name = gen_name(); - rd->init_val = var->init_val || var->init_val_hi; + rd = load_constant(block, *bb, var->init_val || var->init_val_hi, + TY_bool); rd->is_const = true; - add_insn(block, *bb, OP_load_constant, rd, NULL, NULL, 0, NULL); return rd; } - zero = require_typed_var(block, TY_int); - zero->var_name = gen_name(); - zero->init_val = 0; + zero = load_constant(block, *bb, 0, TY_int); zero->is_const = true; - add_insn(block, *bb, OP_load_constant, zero, NULL, NULL, 0, NULL); - rd = require_typed_var(block, TY_bool); - rd->var_name = gen_name(); + rd = name_var(require_typed_var(block, TY_bool)); add_insn(block, *bb, OP_neq, rd, var, zero, 0, NULL); return rd; } +var_t *resize_to(block_t *block, + basic_block_t **bb, + var_t *val, + type_t *type, + int ptr_level); + /* The value an OP_write of a scalar object of @type should store. A store * narrows to the object width by itself, which is the conversion C wants for * every integer type except _Bool: 0x100 has to become 1, not its low byte 0. - * Paths that write a plain assignment through an address share this rather than - * each resizing their value. + * It does not widen, so an int stored into a long long is converted first: + * otherwise the high word held whatever the int's register did above it, the + * carry of an overflowed sum, say. Paths that write a plain assignment through + * an address share this rather than each resizing their value. */ var_t *convert_stored_value(block_t *block, basic_block_t **bb, @@ -1599,9 +1872,13 @@ var_t *convert_stored_value(block_t *block, type_t *type, int ptr_level) { - if (!is_bool_scalar(type, ptr_level)) - return value; - return normalize_bool(block, bb, value); + if (is_bool_scalar(type, ptr_level)) + return normalize_bool(block, bb, value); + if (!ptr_level && type && !type->ptr_level && !is_record_type(type) && + !value->ptr_level && !value->array_size && !value->is_func && + get_size(value) < type->size) + return resize_to(block, bb, value, type, ptr_level); + return value; } var_t *resize_var(block_t *block, basic_block_t **bb, var_t *from, var_t *to) @@ -1983,12 +2260,60 @@ void diagnose_callback_slot_initializer(const var_t *from, const var_t *to) cur_token_loc()); } +/* Array subscripts undergo integer promotion before their byte offset is + * scaled. Keep that representation on synthetic scaling values: narrowing a + * `unsigned char` product would otherwise wrap before pointer addition. + */ +type_t *integer_promoted_type(const var_t *var) +{ + return var->type->size < TY_int->size ? TY_int : var->type; +} + /* The rest of the parser, in the order it was written. Each file depends on * what the ones before it define, so the order below is load-bearing and must * not be sorted. */ /* clang-format off */ +static void skip_type_qualifiers(void) +{ + while (lex_accept(T_const) || lex_accept(T_volatile) || + lex_accept(T_restrict)) + ; +} + +static token_t *function_pointer_declarator_suffix(token_t *open, + int *star_count); + +static void read_assignment_or_expression(block_t *parent, + basic_block_t **bb); +static void read_comma_expression(block_t *parent, + basic_block_t **bb, + bool clear_reference); +static var_t *read_subscript_address(block_t *parent, + basic_block_t **bb, + var_t *base, + type_t *element_type, + int pointer_level, + int stride) +{ + var_t *index; + + read_assignment_or_expression(parent, bb); + index = opstack_pop(); + lex_expect(T_close_square); + return lower_array_element_address(parent, *bb, base, index, element_type, + pointer_level, stride, false); +} + #include "parser-init.c" +static void read_assignment_or_expression(block_t *parent, + basic_block_t **bb) +{ + if (!read_assignment_expression(parent, bb)) { + read_expr(parent, bb); + read_ternary_operation(parent, bb); + } +} #include "parser-decl.c" #include "parser-call.c" #include "parser-sizeof.c" diff --git a/src/peephole.c b/src/peephole.c index 27b53b6a..a61ce8d5 100644 --- a/src/peephole.c +++ b/src/peephole.c @@ -9,142 +9,179 @@ #include "defs.h" #include "globals.c" -/* Determines if an instruction can be fused with a following OP_assign. Fusible - * instructions are those whose results can be directly written to the final - * destination register, eliminating intermediate moves. - */ -bool is_fusible_insn(const ph2_ir_t *ph2_ir) -{ - switch (ph2_ir->op) { - case OP_add: /* Arithmetic operations */ - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: /* Shift operations */ - case OP_rshift: - case OP_bit_and: /* Bitwise operations */ - case OP_bit_or: - case OP_bit_xor: - case OP_log_and: /* Logical operations */ - case OP_log_or: - case OP_log_not: - case OP_negate: /* Unary operations */ - case OP_load: /* Memory operations */ - case OP_global_load: - case OP_load_data_address: - case OP_load_rodata_address: - return true; - default: - return false; - } +/* Share peephole rewrite and register-write properties in one table. */ +enum { + OP_WRITES_DEST = 1, + OP_SRC0_REG = 2, + OP_SRC1_REG = 4, + OP_SRC2_REG = 8, + OP_COPY_FORWARD_SAFE = 16, + OP_FUSIBLE = 32, + OP_BINARY_ALU = 64, + OP_INTEGER_BINARY = 128, + OP_COMPARISON = 256, + OP_SCALAR_UNARY = 512, + OP_SCALAR_CAST = 1024, +}; + +#define OP_PROPERTY(op, flags) [op] = flags, +#define OP_DEST(flags) ((flags) | OP_WRITES_DEST) +#define OP_COPY(flags) ((flags) | OP_COPY_FORWARD_SAFE) +#define OP_FUSE(flags) ((flags) | OP_FUSIBLE) +#define OP_COPY_FUSE(flags) ((flags) | OP_COPY_FORWARD_SAFE | OP_FUSIBLE) +#define OP_BINARY (OP_SRC0_REG | OP_SRC1_REG | OP_BINARY_ALU) +#define OP_UNARY OP_SRC0_REG +#define OP_INTEGER (OP_BINARY | OP_INTEGER_BINARY) +/* Each macro expands to an initializer and comma. */ +/* clang-format off */ +static const unsigned short op_properties[] = { + OP_PROPERTY(OP_generic, 0) + OP_PROPERTY(OP_cmov, OP_DEST(OP_SRC0_REG | OP_SRC1_REG | OP_SRC2_REG)) + OP_PROPERTY(OP_define, 0) + OP_PROPERTY(OP_push, OP_SRC0_REG) + OP_PROPERTY(OP_call, 0) + OP_PROPERTY(OP_indirect, OP_SRC0_REG) + OP_PROPERTY(OP_return, OP_SRC0_REG) + OP_PROPERTY(OP_va_start, 0) + OP_PROPERTY(OP_allocat, 0) + OP_PROPERTY(OP_assign, OP_DEST(OP_COPY(OP_SRC0_REG))) + OP_PROPERTY(OP_load_constant, OP_DEST(0)) + OP_PROPERTY(OP_load_data_address, OP_DEST(OP_FUSE(0))) + OP_PROPERTY(OP_load_rodata_address, OP_DEST(OP_FUSE(0))) + OP_PROPERTY(OP_branch, OP_COPY(OP_SRC0_REG)) + OP_PROPERTY(OP_jump, 0) + OP_PROPERTY(OP_func_ret, 0) + OP_PROPERTY(OP_address_of_func, OP_SRC0_REG) + OP_PROPERTY(OP_load_func, OP_SRC0_REG) + OP_PROPERTY(OP_global_load_func, OP_SRC0_REG) + OP_PROPERTY(OP_address_of, OP_DEST(0)) + OP_PROPERTY(OP_global_address_of, OP_DEST(0)) + OP_PROPERTY(OP_load, OP_DEST(OP_FUSE(0))) + OP_PROPERTY(OP_global_load, OP_DEST(OP_FUSE(0))) + OP_PROPERTY(OP_store, OP_COPY(OP_SRC0_REG)) + OP_PROPERTY(OP_global_store, OP_COPY(OP_SRC0_REG)) + OP_PROPERTY(OP_read, OP_DEST(OP_COPY(OP_SRC0_REG))) + OP_PROPERTY(OP_write, OP_COPY(OP_SRC0_REG | OP_SRC1_REG)) + OP_PROPERTY(OP_add, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_sub, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_mul, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_div, OP_DEST(OP_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_mod, OP_DEST(OP_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_ternary, OP_DEST(OP_SRC0_REG)) + OP_PROPERTY(OP_lshift, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_rshift, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_log_and, OP_DEST(OP_FUSE(OP_UNARY))) + OP_PROPERTY(OP_log_or, OP_DEST(OP_FUSE(OP_UNARY))) + OP_PROPERTY(OP_log_not, OP_DEST(OP_COPY_FUSE(OP_UNARY | OP_SCALAR_UNARY))) + OP_PROPERTY(OP_eq, OP_DEST(OP_COPY_FUSE(OP_BINARY | OP_COMPARISON))) + OP_PROPERTY(OP_neq, OP_DEST(OP_COPY_FUSE(OP_BINARY | OP_COMPARISON))) + OP_PROPERTY(OP_lt, OP_DEST(OP_COPY_FUSE(OP_BINARY | OP_COMPARISON))) + OP_PROPERTY(OP_leq, OP_DEST(OP_COPY_FUSE(OP_BINARY | OP_COMPARISON))) + OP_PROPERTY(OP_gt, OP_DEST(OP_COPY_FUSE(OP_BINARY | OP_COMPARISON))) + OP_PROPERTY(OP_geq, OP_DEST(OP_COPY_FUSE(OP_BINARY | OP_COMPARISON))) + OP_PROPERTY(OP_bit_or, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_bit_and, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_bit_xor, OP_DEST(OP_COPY_FUSE(OP_INTEGER))) + OP_PROPERTY(OP_bit_not, OP_DEST(OP_COPY_FUSE(OP_UNARY | OP_SCALAR_UNARY))) + OP_PROPERTY(OP_negate, OP_DEST(OP_COPY_FUSE(OP_UNARY | OP_SCALAR_UNARY))) + OP_PROPERTY(OP_trunc, OP_DEST(OP_COPY(OP_SRC0_REG | OP_SCALAR_CAST))) + OP_PROPERTY(OP_sign_ext, OP_DEST(OP_COPY(OP_SRC0_REG | OP_SCALAR_CAST))) + OP_PROPERTY(OP_cast, OP_DEST(OP_SRC0_REG | OP_SCALAR_CAST)) + OP_PROPERTY(OP_start, 0) +}; +/* clang-format on */ +#undef OP_COPY +#undef OP_FUSE +#undef OP_COPY_FUSE +#undef OP_BINARY +#undef OP_INTEGER +#undef OP_UNARY +#undef OP_DEST +#undef OP_PROPERTY +typedef char op_property_count_check + [sizeof(op_properties) / sizeof(op_properties[0]) == OP_start + 1 ? 1 : -1]; + +static unsigned int op_property(opcode_t op, unsigned int property) +{ + if (op < OP_generic || op > OP_start) + return 0; + return op_properties[op] & property; } -/* Opcodes whose "dest" names a register they write. Only opcodes that certainly - * do are listed, so a live register is never mistaken for dead. - */ bool op_writes_dest(opcode_t op) { - switch (op) { - case OP_cmov: - case OP_load: - case OP_load_constant: - case OP_global_load: - case OP_assign: - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_bit_not: - case OP_negate: - case OP_log_not: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - case OP_read: - case OP_address_of: - case OP_global_address_of: - case OP_trunc: - case OP_sign_ext: - return true; - default: - return false; - } + return op_property(op, OP_WRITES_DEST) != 0; +} +bool op_is_binary_alu(opcode_t op) +{ + return op_property(op, OP_BINARY_ALU); +} +bool op_is_integer_binary(opcode_t op) +{ + return op_property(op, OP_INTEGER_BINARY); +} +bool op_is_comparison(opcode_t op) +{ + return op_property(op, OP_COMPARISON); +} +bool op_is_scalar_unary(opcode_t op) +{ + return op_property(op, OP_SCALAR_UNARY); +} +bool op_is_scalar_cast(opcode_t op) +{ + return op_property(op, OP_SCALAR_CAST); } - -/* Opcodes whose src0 holds something other than a register number. Anything not - * listed is assumed to read src0, which only costs a missed rewrite. - */ bool op_src0_is_reg(opcode_t op) { - switch (op) { - case OP_load: - case OP_load_constant: - case OP_global_load: - case OP_address_of: - case OP_global_address_of: - case OP_load_data_address: - case OP_load_rodata_address: - case OP_define: - case OP_label: - case OP_jump: - return false; - default: - return true; - } + return op_property(op, OP_SRC0_REG); } - -/* Opcodes whose src1 is a register rather than a width, slot or immediate. */ bool op_src1_is_reg(opcode_t op) { - switch (op) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - case OP_write: - case OP_cmov: - return true; - default: - return false; - } + return op_property(op, OP_SRC1_REG); +} +bool op_src2_is_reg(opcode_t op) +{ + return op_property(op, OP_SRC2_REG); } -/* Whether src2 names a register. Only a select does: it is the value kept when - * the condition does not hold, and a scan that missed it would take that value - * for dead and drop whatever computed it. +int func_highest_used_reg(func_t *func, int first_callee_saved) +{ + int top = first_callee_saved - 1; + + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) + for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { + if (op_writes_dest(ir->op) && ir->dest > top) + top = ir->dest; + if (op_src0_is_reg(ir->op) && ir->src0 > top) + top = ir->src0; + if (op_src1_is_reg(ir->op) && ir->src1 > top) + top = ir->src1; + if (op_src2_is_reg(ir->op) && ir->src2 > top) + top = ir->src2; + } + return top < REG_CNT ? top : REG_CNT - 1; +} + +/* These instructions can write their result straight to a following move's + * destination, eliminating the intermediate register copy. */ -bool op_src2_is_reg(opcode_t op) +bool is_fusible_insn(const ph2_ir_t *ir) { - return op == OP_cmov; + return op_property(ir->op, OP_FUSIBLE); } +int call_arg_regs(ph2_ir_t *ir); + /* Whether @ir reads @reg as an operand. A 32-bit target names the high half of - * a wide operand in src0_hi or src1_hi, which are -1 when unused. + * a wide operand in src0_hi or src1_hi, which are -1 when unused. A call reads + * its argument registers without naming any of them. */ bool ir_reads_reg(ph2_ir_t *ir, int reg) { + if ((ir->op == OP_call || ir->op == OP_indirect) && reg >= 0 && + reg < MAX_ARGS_IN_REG && reg < call_arg_regs(ir)) + return true; if (reg >= 0 && (ir->src0_hi == reg || ir->src1_hi == reg)) return true; if (op_src2_is_reg(ir->op) && ir->src2 == reg) @@ -154,6 +191,68 @@ bool ir_reads_reg(ph2_ir_t *ir, int reg) return op_src1_is_reg(ir->op) && ir->src1 == reg; } +/* Calls read their ABI arguments before clobbering the caller registers. */ +unsigned int ph2_ir_defs(ph2_ir_t *ir) +{ + unsigned int defs = 0; + if (op_writes_dest(ir->op)) { + if (ir->dest >= 0 && ir->dest < REG_CNT) + defs |= 1u << ir->dest; + if (ir->dest_hi >= 0 && ir->dest_hi < REG_CNT) + defs |= 1u << ir->dest_hi; + } + if (ir->op == OP_call || ir->op == OP_indirect) + defs |= (1u << (REG_CNT - CALLEE_SAVED_REGS)) - 1; + return defs; +} + +/* Allocated registers can cross machine edges. Three block masks suffice: + * live-in is use | (live-out & ~def), including both halves of a pair. + */ +void ph2_compute_liveness(void) +{ + FOR_EACH_FUNCTION_BODY(func) + { + int count = 0; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) + count++; + basic_block_t **blocks = malloc(count * sizeof(*blocks)); + if (!blocks) + fatal("Machine liveness allocation failed"); + int index = 0; + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + blocks[index++] = bb; + bb->machine_use = bb->machine_def = bb->machine_liveout = 0; + for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { + for (int reg = 0; reg < REG_CNT; reg++) + if (ir_reads_reg(ir, reg)) + bb->machine_use |= (1u << reg) & ~bb->machine_def; + bb->machine_def |= ph2_ir_defs(ir); + } + } + bool changed; + do { + changed = false; + for (int at = count - 1; at >= 0; at--) { + basic_block_t *bb = blocks[at]; + basic_block_t *successors[3] = {bb->next, bb->then_, bb->else_}; + unsigned int out = 0; + for (int edge = 0; edge < 3; edge++) { + basic_block_t *next = successors[edge]; + if (next) + out |= next->machine_use | + (next->machine_liveout & ~next->machine_def); + } + if (out != bb->machine_liveout) { + bb->machine_liveout = out; + changed = true; + } + } + } while (changed); + free(blocks); + } +} + /* Main peephole optimization function that applies pattern matching and * transformation rules to consecutive IR instructions. * Returns true if any optimization was applied, false otherwise. Drop the @@ -189,65 +288,20 @@ int call_arg_regs(ph2_ir_t *ir) return words < MAX_ARGS_IN_REG ? words : MAX_ARGS_IN_REG; } -/* True when @reg still holds a live value past the end of @bb. - * - * reg_alloc() hands its register file to a successor that this block is the - * only way into (bb_export_regs()), so a register can outlive the block that - * filled it. Scans that stop at the block boundary conclude there according to - * this: with nothing carried, every register dies at the end of a block and an - * unread value is dead. - */ -bool reg_live_out_of_bb(basic_block_t *bb, int reg) -{ - basic_block_t *succs[3]; - - if (!bb || reg < 0 || reg >= REG_CNT) - return false; - - /* A register the allocator pinned holds its variable on every path, so it - * is live out of every block regardless of what the successors record. - */ - if (bb->belong_to && ((bb->belong_to->pinned_regs >> reg) & 1)) - return true; - - succs[0] = bb->next; - succs[1] = bb->then_; - succs[2] = bb->else_; - - for (int i = 0; i < 3; i++) { - if (succs[i] && succs[i]->entry_regs && succs[i]->entry_regs[reg]) - return true; - } - return false; -} - -/* Whether @reg may still be read after @ir, before anything writes it again. - * - * The scan covers the rest of @bb, with a call reading its argument registers. - * Past the end of the block a register keeps its value only when it is pinned, - * or when reg_alloc() handed it to a successor through bb_export_regs(); every - * other successor loads what it reads. - */ -bool reg_read_after(basic_block_t *bb, ph2_ir_t *ir, int reg) +/* Scan local reads and definitions, then consult the successor live set. */ +static bool reg_read_after(basic_block_t *bb, ph2_ir_t *ir, int reg) { if (reg < 0) return false; if (reg >= REG_CNT) return true; - for (ph2_ir_t *p = ir->next; p; p = p->next) { - if (p->op == OP_call || p->op == OP_indirect) { - if (reg < call_arg_regs(p)) - return true; - continue; - } if (ir_reads_reg(p, reg)) return true; - if (op_writes_dest(p->op) && (p->dest == reg || p->dest_hi == reg)) + if (ph2_ir_defs(p) & (1u << reg)) return false; } - - return reg_live_out_of_bb(bb, reg); + return (bb->machine_liveout & (1u << reg)) != 0; } /* Whether folding @ir into @last loses a value that is still wanted. The folded @@ -314,26 +368,29 @@ bool pair_insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) return true; } +static bool replace_with_move(basic_block_t *bb, + ph2_ir_t *ir, + ph2_ir_t *next, + int source) +{ + ir->op = OP_assign; + ir->src0 = source; + ir->dest = next->dest; + ph2_ir_drop_after(bb, ir, next); + return true; +} + bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) { ph2_ir_t *next = ph2_ir->next; if (!next) return false; - /* ALU instruction fusion. Eliminates redundant move operations following - * arithmetic/logical operations. This is the most fundamental optimization - * that removes temporary register usage. - */ + /* Fuse result moves and fold constant identities. */ if (next->op == OP_assign) { if (is_fusible_insn(ph2_ir) && ph2_ir->dest == next->src0 && !fold_loses_dest(bb, ph2_ir, next)) { - /* Pattern: {ALU rn, rs1, rs2; mv rd, rn} → {ALU rd, rs1, rs2} - * Example: {add t1, a, b; mv result, t1} → {add result, a, b} - * - * Only when nothing reads rn afterwards. A value with two names is - * copied and then read through the first one as well, and the fused - * instruction no longer writes it. - */ + /* Do not remove a temporary that still has another live use. */ ph2_ir->dest = next->dest; ph2_ir_drop_after(bb, ph2_ir, next); return true; @@ -346,36 +403,18 @@ bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) if (next->op == OP_add && (ph2_ir->dest == next->src0 || ph2_ir->dest == next->src1) && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; add x, 0} → {mov x} (additive identity: x+0 = x) - * Handles both operand positions due to addition commutativity - * Example: {li t1, 0; add result, var, t1} → {mov result, var} - */ int non_zero_src = (ph2_ir->dest == next->src0) ? next->src1 : next->src0; - ph2_ir->op = OP_assign; - ph2_ir->src0 = non_zero_src; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + return replace_with_move(bb, ph2_ir, next, non_zero_src); } if (next->op == OP_sub && const_fold_ok(bb, ph2_ir, next)) { if (ph2_ir->dest == next->src1) { - /* Pattern: {li 0; sub x, 0} → {mov x} (x - 0 = x) - * Example: {li t1, 0; sub result, var, t1} → {mov result, var} - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + return replace_with_move(bb, ph2_ir, next, next->src0); } if (ph2_ir->dest == next->src0) { - /* Pattern: {li 0; sub 0, x} → {neg x} (0 - x = -x) - * Example: {li t1, 0; sub result, t1, var} → {neg result, var} - */ ph2_ir->op = OP_negate; ph2_ir->src0 = next->src1; ph2_ir->dest = next->dest; @@ -387,10 +426,6 @@ bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) if (next->op == OP_mul && (ph2_ir->dest == next->src0 || ph2_ir->dest == next->src1) && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; mul x, 0} → {li 0} (absorbing element: x * 0 = 0) - * Example: {li t1, 0; mul result, var, t1} → {li result, 0} - * Eliminates multiplication entirely - */ ph2_ir->op = OP_load_constant; ph2_ir->src0 = 0; ph2_ir->dest = next->dest; @@ -405,16 +440,9 @@ bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) if (next->op == OP_mul && (ph2_ir->dest == next->src0 || ph2_ir->dest == next->src1) && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 1; mul x, 1} → {mov x} (multiplicative identity: x * - * 1 = x) Example: {li t1, 1; mul result, var, t1} → {mov result, - * var} Handles both operand positions due to multiplication - * commutativity - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = ph2_ir->dest == next->src0 ? next->src1 : next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + return replace_with_move( + bb, ph2_ir, next, + ph2_ir->dest == next->src0 ? next->src1 : next->src0); } } @@ -425,93 +453,15 @@ bool insn_fusion(basic_block_t *bb, ph2_ir_t *ph2_ir) ph2_ir->src1 == (next->size_bytes > 4 ? -1 : 0) && next->op == OP_bit_and && ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li -1; and x, -1} → {mov x} (x & 0xFFFFFFFF = x) Example: - * {li t1, -1; and result, var, t1} → {mov result, var} Eliminates - * bitwise AND with all-ones mask - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + return replace_with_move(bb, ph2_ir, next, next->src0); } if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0 && (next->op == OP_lshift || next->op == OP_rshift) && - ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; shl/shr x, 0} → {mov x} (x << 0 = x >> 0 = x) - * Example: {li t1, 0; shl result, var, t1} → {mov result, var} - * Eliminates no-op shift operations - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } - - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0 && next->op == OP_bit_or && - ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; or x, 0} → {mov x} (x | 0 = x) Example: {li t1, 0; or - * result, var, t1} → {mov result, var} Eliminates bitwise OR with zero - * (identity element) - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } - - /* Power-of-2 multiplication to shift conversion. Shift operations are - * significantly faster than multiplication - */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 > 0 && - ph2_ir->src1 == 0 && next->op == OP_mul && ph2_ir->dest == next->src1) { - int shift_amount = exact_log2(ph2_ir->src0); - if (shift_amount >= 0 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 2^n; mul x, 2^n} → {li n; shl x, n} Example: {li t1, - * 4; mul result, var, t1} → - * {li t1, 2; shl result, var, t1} - */ - ph2_ir->src0 = shift_amount; - next->op = OP_lshift; - return true; - } - } - - /* XOR identity operation */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 0 && - ph2_ir->src1 == 0 && next->op == OP_bit_xor && - ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 0; xor x, 0} → {mov x} (x ^ 0 = x) Example: {li t1, 0; - * xor result, var, t1} → {mov result, var} Completes bitwise identity - * optimization coverage - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src0; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } - - /* Extended multiplicative identity (operand position variant) Handles the - * case where constant 1 is in src0 position of multiplication - */ - if (ph2_ir->op == OP_load_constant && ph2_ir->src0 == 1 && - ph2_ir->src1 == 0 && next->op == OP_mul && ph2_ir->dest == next->src0 && - const_fold_ok(bb, ph2_ir, next)) { - /* Pattern: {li 1; mul 1, x} → {mov x} (1 * x = x) Example: {li t1, 1; - * mul result, t1, var} → {mov result, var} Covers multiplication - * commutativity edge case - */ - ph2_ir->op = OP_assign; - ph2_ir->src0 = next->src1; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; - } + ph2_ir->src1 == 0 && + (next->op == OP_lshift || next->op == OP_rshift || + next->op == OP_bit_or || next->op == OP_bit_xor) && + ph2_ir->dest == next->src1 && const_fold_ok(bb, ph2_ir, next)) + return replace_with_move(bb, ph2_ir, next, next->src0); return false; } @@ -525,6 +475,14 @@ bool is_plain_register_def(const ph2_ir_t *ir) ir->op == OP_global_load || ir->op == OP_load_constant; } +static bool is_full_register_copy(const ph2_ir_t *ir) +{ + return ir && ir->op == OP_assign && ir->dest_hi < 0 && ir->src0_hi < 0 && + !(ir->is_unsigned && ir->size_bytes < PTR_SIZE) && + ir->dest != ir->src0 && ir->dest >= 0 && ir->dest < REG_CNT && + ir->src0 >= 0 && ir->src0 < REG_CNT; +} + /* Redundant move elimination: a move, load or constant load whose register is * immediately overwritten by another one is dead. {mov rd, rs1; mov rd, rs2}, * {load rd, ofs; mov rd, rs}, {li rd, imm; load rd, ofs} and the other @@ -556,6 +514,94 @@ bool redundant_move_elim(basic_block_t *bb, ph2_ir_t *ph2_ir) return true; } +/* {t = r; op ..., t, ...} -> {op ..., r, ...} when t dies there. A post + * increment ("i++" keeps i's old value in a temporary) and argument staging + * leave such copies in every loop; forwarding the source drops the move. + * + * Only a plain full-width copy qualifies: a register pair, or an unsigned + * narrowing copy that x86-64 emits as a zero extension, changes the value. + */ +bool copy_forward(basic_block_t *bb, ph2_ir_t *ph2_ir) +{ + ph2_ir_t *next = ph2_ir->next; + int t = ph2_ir->dest; + int r = ph2_ir->src0; + bool used = false; + + if (!next || !is_full_register_copy(ph2_ir)) + return false; + if (next->src0_hi >= 0 || next->src1_hi >= 0 || next->dest_hi >= 0) + return false; + + /* Only instructions that read exactly the registers they name. A call reads + * its argument registers without naming them, and several opcodes keep + * something other than a register in src0; the copy may stage an argument, + * or its number may merely match an unrelated field. + */ + if (!op_property(next->op, OP_COPY_FORWARD_SAFE)) + return false; + /* The copy has to die here, unless the instruction itself rewrites it. */ + if (!(op_writes_dest(next->op) && next->dest == t) && + reg_read_after(bb, next, t)) + return false; + + /* Backends lower "rd = op a, b" as "mov rd, a; op rd, b", so an instruction + * that writes r may read r only as its first operand: "t = r; r = sub a, t" + * must not become "r = sub a, r". + */ + if (op_writes_dest(next->op) && next->dest == r && + ((op_src1_is_reg(next->op) && next->src1 == t) || + (op_src2_is_reg(next->op) && next->src2 == t))) + return false; + if (op_src0_is_reg(next->op) && next->src0 == t) { + next->src0 = r; + used = true; + } + if (op_src1_is_reg(next->op) && next->src1 == t) { + next->src1 = r; + used = true; + } + if (op_src2_is_reg(next->op) && next->src2 == t) { + next->src2 = r; + used = true; + } + if (!used) + return false; + memcpy(ph2_ir, next, sizeof(ph2_ir_t)); + ph2_ir_drop_after(bb, ph2_ir, ph2_ir); + return true; +} + +/* {t = r; li r, K; [t] = r} -> {li t, K; [r] = t} when both die at the store. + * The copy exists only because the value was loaded into the register the + * address was in; swapping the two roles drops it, and leaves the address + * feeding the store directly, where a backend can fold it into the access. + */ +bool store_copy_swap(basic_block_t *bb, ph2_ir_t *ph2_ir) +{ + ph2_ir_t *li = ph2_ir->next; + ph2_ir_t *wr = li ? li->next : NULL; + int t = ph2_ir->dest; + int r = ph2_ir->src0; + + if (!wr || !is_full_register_copy(ph2_ir)) + return false; + if (li->op != OP_load_constant || li->dest != r || li->dest_hi >= 0) + return false; + if (wr->op != OP_write || wr->src0 != t || wr->src1 != r || + wr->src0_hi >= 0 || wr->src1_hi >= 0) + return false; + if (reg_read_after(bb, wr, t) || reg_read_after(bb, wr, r)) + return false; + + memcpy(ph2_ir, li, sizeof(ph2_ir_t)); + ph2_ir->dest = t; + wr->src0 = r; + wr->src1 = t; + ph2_ir_drop_after(bb, ph2_ir, ph2_ir); + return true; +} + /* Load/store elimination for consecutive memory operations. Removes redundant * loads and dead stores that access the same memory location. Conservative * implementation to maintain bootstrap stability. @@ -800,11 +846,7 @@ bool bitwise_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) */ if (ph2_ir->op == OP_bit_not && next->op == OP_bit_not && next->src0 == ph2_ir->dest && !fold_loses_dest(bb, ph2_ir, next)) { - /* Replace with simple assignment */ - ph2_ir->op = OP_assign; - ph2_ir->dest = next->dest; - ph2_ir_drop_after(bb, ph2_ir, next); - return true; + return replace_with_move(bb, ph2_ir, next, ph2_ir->src0); } /* Pattern 2: AND with zero → zero x & 0 = 0, and OR with all-ones → @@ -909,8 +951,8 @@ bool triple_pattern_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) * * This runs on ph2_ir_t, after register allocation, and so sees only what * assigning registers makes visible. Constant folding, common subexpression - * elimination and dead code elimination have already run over insn_t in the SSA - * optimizer and are not repeated here. + * elimination and dead code elimination have already run over VIR and are not + * repeated here. * * What is left to do at this level: * - self-assignment elimination, for assignments allocation itself created @@ -919,74 +961,75 @@ bool triple_pattern_optimization(basic_block_t *bb, ph2_ir_t *ph2_ir) * - bitwise identities on registers * - load/store pattern elimination */ -void peephole(void) +/* Apply the first rule that matches the window starting at @ir, and report + * whether one did. Every rewrite leaves @ir in place and changes what follows + * it, so the caller tries the same position again. + */ +bool peephole_at(basic_block_t *bb, ph2_ir_t *ir) { - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; + ph2_ir_t *next = ir->next; + if (!next) + return false; - /* Local peephole optimizations on post-register-allocation IR */ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { - ph2_ir_t *next = ir->next; - if (!next) - continue; - - /* Self-assignment elimination Keep this as a safety net: SSA - * handles most cases, but register allocation might create new - * self-assignments - */ - if (next->op == OP_assign && next->dest == next->src0 && - next->dest_hi == next->src0_hi) { - ph2_ir_drop_after(bb, ir, next); - continue; - } + /* Self-assignment elimination Keep this as a safety net: SSA handles most + * cases, but register allocation might create new self-assignments + */ + if (next->op == OP_assign && next->dest == next->src0 && + next->dest_hi == next->src0_hi) { + ph2_ir_drop_after(bb, ir, next); + return true; + } - /* Every rewrite below moves this instruction's result to the - * destination of the one after it, dropping the write to the - * register it named. That is fine for a temporary, whose value - * nothing wants again, and wrong for a pinned register: a - * variable lives there for the whole function and nothing else - * ever reloads it, so "li rbx, 0; add rax, rsi, rbx" must not - * become "mov rax, rsi" and leave rbx unwritten. - */ - if (ir->dest >= 0 && ir->dest < REG_CNT && - ((func->pinned_regs >> ir->dest) & 1)) - continue; - - if (PTR_SIZE < 8 && - (ph2_ir_has_pair(ir) || ph2_ir_has_pair(next) || - ph2_ir_has_pair(next->next))) { - pair_insn_fusion(bb, ir); - continue; - } + if (PTR_SIZE < 8 && (ph2_ir_has_pair(ir) || ph2_ir_has_pair(next) || + ph2_ir_has_pair(next->next))) + return pair_insn_fusion(bb, ir); + + /* Try triple pattern optimization first (3-instruction sequences) */ + if (triple_pattern_optimization(bb, ir)) + return true; + + /* Try instruction fusion (2-instruction sequences) */ + if (insn_fusion(bb, ir)) + return true; - /* Try triple pattern optimization first (3-instruction - * sequences) - */ - if (triple_pattern_optimization(bb, ir)) - continue; + /* Apply strength reduction for power-of-2 operations */ + if (strength_reduction(bb, ir)) + return true; + + /* Apply bitwise operation optimizations */ + if (bitwise_optimization(bb, ir)) + return true; - /* Try instruction fusion (2-instruction sequences) */ - if (insn_fusion(bb, ir)) - continue; + /* Swap a copied address with the constant stored through it */ + if (store_copy_swap(bb, ir)) + return true; - /* Apply strength reduction for power-of-2 operations */ - if (strength_reduction(bb, ir)) - continue; + /* Forward a copy into the instruction that consumes it */ + if (copy_forward(bb, ir)) + return true; - /* Apply bitwise operation optimizations */ - if (bitwise_optimization(bb, ir)) - continue; + /* Apply redundant move elimination */ + if (redundant_move_elim(bb, ir)) + return true; - /* Apply redundant move elimination */ - if (redundant_move_elim(bb, ir)) - continue; + /* Apply load/store elimination */ + return eliminate_load_store_pairs(bb, ir); +} - /* Apply load/store elimination */ - if (eliminate_load_store_pairs(bb, ir)) - continue; +void peephole(void) +{ + FOR_EACH_FUNCTION_BODY(func) + { + /* Local peephole optimizations on post-register-allocation IR. A + * rewrite can expose another at the same position -- a dead load + * dropped in front of a copy leaves the copy there to be forwarded -- + * so each position is retried, a bounded number of times in case a rule + * reports a change that does not shrink the window. + */ + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { + for (int tries = 0; tries < 4 && peephole_at(bb, ir); tries++) + ; } } } diff --git a/src/preprocessor.c b/src/preprocessor.c index 27bfe5f6..4adb7983 100644 --- a/src/preprocessor.c +++ b/src/preprocessor.c @@ -89,6 +89,14 @@ typedef struct macro { token_t *(*handler)(token_t *); } macro_t; +static macro_t *new_macro(char *name) +{ + macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(*macro)); + + macro->name = name; + return macro; +} + bool is_macro_defined(char *name) { const macro_t *macro = hashmap_get(MACROS, name); @@ -115,13 +123,11 @@ void define_builtin_object_macro(const char *name, token_kind_t kind, const char *replacement) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - - macro->name = intern_string((char *) name); + macro_t *macro = new_macro(intern_string((char *) name)); macro->replacement = new_token(kind, &synth_built_in_loc, strlen(replacement)); macro->replacement->literal = intern_string((char *) replacement); - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } /* Integer minimum macros are preprocessing token sequences rather than one @@ -130,17 +136,16 @@ void define_builtin_object_macro(const char *name, */ void define_builtin_negative_macro(const char *name, const char *magnitude) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + macro_t *macro = new_macro(intern_string((char *) name)); token_t *minus = new_token(T_minus, &synth_built_in_loc, 1); token_t *numeric = new_token(T_numeric, &synth_built_in_loc, strlen(magnitude)); - macro->name = intern_string((char *) name); minus->literal = "-"; numeric->literal = intern_string((char *) magnitude); minus->next = numeric; macro->replacement = minus; - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } token_t *append_builtin_macro_token(token_t **replacement, @@ -148,6 +153,38 @@ token_t *append_builtin_macro_token(token_t **replacement, token_kind_t kind, const char *literal); +typedef struct { + token_kind_t kind; + const char *literal; +} builtin_macro_token_t; + +static void append_builtin_macro_tokens(token_t **replacement, + token_t **tail, + const builtin_macro_token_t *tokens, + unsigned int count) +{ + for (unsigned int i = 0; i < count; i++) + append_builtin_macro_token(replacement, tail, tokens[i].kind, + tokens[i].literal); +} + +static macro_t *new_builtin_function_macro(const char *name, + const char *first_param, + const char *second_param) +{ + macro_t *macro = new_macro(intern_string((char *) name)); + const char *params[] = {first_param, second_param}; + + macro->is_function_like = true; + macro->param_num = second_param ? 2 : 1; + for (int i = 0; i < macro->param_num; i++) { + macro->param_names[i] = + new_token(T_identifier, &synth_built_in_loc, strlen(params[i])); + macro->param_names[i]->literal = intern_string((char *) params[i]); + } + return macro; +} + /* C99's integer construction macros append a target representation suffix to * their single integer-token argument. The preprocessor's ordinary ## path * rescans the joined spelling, so the replacement remains a numeric token. @@ -155,10 +192,9 @@ token_t *append_builtin_macro_token(token_t **replacement, void define_builtin_integer_construction_macro(const char *name, const char *suffix) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + macro_t *macro = new_macro(intern_string((char *) name)); token_t *tail = NULL; - macro->name = intern_string((char *) name); macro->is_function_like = true; macro->param_num = 1; macro->param_names[0] = new_token(T_identifier, &synth_built_in_loc, 5); @@ -171,7 +207,7 @@ void define_builtin_integer_construction_macro(const char *name, append_builtin_macro_token(¯o->replacement, &tail, T_identifier, suffix); } - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } token_t *append_builtin_macro_token(token_t **replacement, @@ -199,18 +235,16 @@ void define_builtin_minimum_macro(const char *name, const char *maximum, const char *one) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + const builtin_macro_token_t replacement[] = { + {T_open_bracket, "("}, {T_minus, "-"}, {T_numeric, maximum}, + {T_minus, "-"}, {T_numeric, one}, {T_close_bracket, ")"}, + }; + macro_t *macro = new_macro(intern_string((char *) name)); token_t *tail = NULL; - macro->name = intern_string((char *) name); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_minus, "-"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, maximum); - append_builtin_macro_token(¯o->replacement, &tail, T_minus, "-"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, one); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); + append_builtin_macro_tokens(¯o->replacement, &tail, replacement, + sizeof(replacement) / sizeof(*replacement)); + hashmap_put_borrowed(MACROS, macro->name, macro); } /* offsetof is specified as a function-like macro. Route its type/member @@ -219,64 +253,62 @@ void define_builtin_minimum_macro(const char *name, */ void install_stddef_offsetof_macro(void) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + static const builtin_macro_token_t replacement[] = { + {T_identifier, "__builtin_offsetof"}, + {T_open_bracket, "("}, + {T_identifier, "type"}, + {T_comma, ","}, + {T_identifier, "member"}, + {T_close_bracket, ")"}, + }; + macro_t *macro = new_builtin_function_macro("offsetof", "type", "member"); token_t *tail = NULL; - macro->name = intern_string("offsetof"); - macro->is_function_like = true; - macro->param_num = 2; - macro->param_names[0] = new_token(T_identifier, &synth_built_in_loc, 4); - macro->param_names[0]->literal = "type"; - macro->param_names[1] = new_token(T_identifier, &synth_built_in_loc, 6); - macro->param_names[1]->literal = "member"; - - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__builtin_offsetof"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "type"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "member"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); + append_builtin_macro_tokens(¯o->replacement, &tail, replacement, + sizeof(replacement) / sizeof(*replacement)); + hashmap_put_borrowed(MACROS, macro->name, macro); } /* shecc currently has signed plain char, 16-bit short, 32-bit int/long, and * 64-bit long long on every target. Keep tied to those actual * language types rather than to the host compiler's ABI. */ +#define LIMIT_OBJECT(name, value) \ + define_builtin_object_macro(name, T_numeric, value) +#define LIMIT_NEGATIVE(name, value) define_builtin_negative_macro(name, value) +#define LIMIT_MINIMUM(name, maximum, one) \ + define_builtin_minimum_macro(name, maximum, one) +#define LIMIT_CONSTRUCTION(name, suffix) \ + define_builtin_integer_construction_macro(name, suffix) + void install_limits_header(void) { - define_builtin_object_macro("CHAR_BIT", T_numeric, "8"); - define_builtin_object_macro("SCHAR_MAX", T_numeric, "127"); - define_builtin_object_macro("UCHAR_MAX", T_numeric, "255U"); - define_builtin_object_macro("CHAR_MAX", T_numeric, "127"); - define_builtin_object_macro("SHRT_MAX", T_numeric, "32767"); - define_builtin_object_macro("USHRT_MAX", T_numeric, "65535U"); - define_builtin_object_macro("INT_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_MAX", T_numeric, "4294967295U"); - define_builtin_object_macro("LONG_MAX", T_numeric, "2147483647L"); - define_builtin_object_macro("ULONG_MAX", T_numeric, "4294967295UL"); - define_builtin_object_macro("LLONG_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("ULLONG_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_object_macro("MB_LEN_MAX", T_numeric, "1"); - - define_builtin_negative_macro("SCHAR_MIN", "128"); - define_builtin_negative_macro("CHAR_MIN", "128"); - define_builtin_negative_macro("SHRT_MIN", "32768"); - define_builtin_minimum_macro("INT_MIN", "2147483647", "1"); - define_builtin_minimum_macro("LONG_MIN", "2147483647L", "1L"); - define_builtin_minimum_macro("LLONG_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("CHAR_BIT", "8"); + LIMIT_OBJECT("SCHAR_MAX", "127"); + LIMIT_OBJECT("UCHAR_MAX", "255U"); + LIMIT_OBJECT("CHAR_MAX", "127"); + LIMIT_OBJECT("SHRT_MAX", "32767"); + LIMIT_OBJECT("USHRT_MAX", "65535U"); + LIMIT_OBJECT("INT_MAX", "2147483647"); + LIMIT_OBJECT("UINT_MAX", "4294967295U"); + LIMIT_OBJECT("LONG_MAX", "2147483647L"); + LIMIT_OBJECT("ULONG_MAX", "4294967295UL"); + LIMIT_OBJECT("LLONG_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("ULLONG_MAX", "18446744073709551615ULL"); + LIMIT_OBJECT("MB_LEN_MAX", "1"); + + LIMIT_NEGATIVE("SCHAR_MIN", "128"); + LIMIT_NEGATIVE("CHAR_MIN", "128"); + LIMIT_NEGATIVE("SHRT_MIN", "32768"); + LIMIT_MINIMUM("INT_MIN", "2147483647", "1"); + LIMIT_MINIMUM("LONG_MIN", "2147483647L", "1L"); + LIMIT_MINIMUM("LLONG_MIN", "9223372036854775807LL", "1LL"); } /* A C null pointer constant may be the integer constant expression 0. */ void install_stddef_header(void) { - define_builtin_object_macro("NULL", T_numeric, "0"); + LIMIT_OBJECT("NULL", "0"); install_stddef_offsetof_macro(); } @@ -285,94 +317,83 @@ void install_stddef_header(void) */ void install_stdint_header(void) { - define_builtin_negative_macro("INT8_MIN", "128"); - define_builtin_object_macro("INT8_MAX", T_numeric, "127"); - define_builtin_object_macro("UINT8_MAX", T_numeric, "255U"); - define_builtin_negative_macro("INT16_MIN", "32768"); - define_builtin_object_macro("INT16_MAX", T_numeric, "32767"); - define_builtin_object_macro("UINT16_MAX", T_numeric, "65535U"); - define_builtin_minimum_macro("INT32_MIN", "2147483647", "1"); - define_builtin_object_macro("INT32_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT32_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT64_MIN", "9223372036854775807LL", "1LL"); - define_builtin_object_macro("INT64_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINT64_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("INTMAX_MIN", "9223372036854775807LL", "1LL"); - define_builtin_object_macro("INTMAX_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINTMAX_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("SIG_ATOMIC_MIN", "2147483647", "1"); - define_builtin_object_macro("SIG_ATOMIC_MAX", T_numeric, "2147483647"); - define_builtin_minimum_macro("WCHAR_MIN", "2147483647", "1"); - define_builtin_object_macro("WCHAR_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("WINT_MIN", T_numeric, "0U"); - define_builtin_object_macro("WINT_MAX", T_numeric, "4294967295U"); - define_builtin_negative_macro("INT_LEAST8_MIN", "128"); - define_builtin_object_macro("INT_LEAST8_MAX", T_numeric, "127"); - define_builtin_object_macro("UINT_LEAST8_MAX", T_numeric, "255U"); - define_builtin_negative_macro("INT_LEAST16_MIN", "32768"); - define_builtin_object_macro("INT_LEAST16_MAX", T_numeric, "32767"); - define_builtin_object_macro("UINT_LEAST16_MAX", T_numeric, "65535U"); - define_builtin_minimum_macro("INT_LEAST32_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_LEAST32_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_LEAST32_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_LEAST64_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("INT_LEAST64_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINT_LEAST64_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("INT_FAST8_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_FAST8_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_FAST8_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_FAST16_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_FAST16_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_FAST16_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_FAST32_MIN", "2147483647", "1"); - define_builtin_object_macro("INT_FAST32_MAX", T_numeric, "2147483647"); - define_builtin_object_macro("UINT_FAST32_MAX", T_numeric, "4294967295U"); - define_builtin_minimum_macro("INT_FAST64_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("INT_FAST64_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINT_FAST64_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_integer_construction_macro("INT8_C", ""); - define_builtin_integer_construction_macro("UINT8_C", ""); - define_builtin_integer_construction_macro("INT16_C", ""); - define_builtin_integer_construction_macro("UINT16_C", ""); - define_builtin_integer_construction_macro("INT32_C", ""); - define_builtin_integer_construction_macro("UINT32_C", "U"); - define_builtin_integer_construction_macro("INT64_C", "LL"); - define_builtin_integer_construction_macro("UINT64_C", "ULL"); - define_builtin_integer_construction_macro("INTMAX_C", "LL"); - define_builtin_integer_construction_macro("UINTMAX_C", "ULL"); + LIMIT_NEGATIVE("INT8_MIN", "128"); + LIMIT_OBJECT("INT8_MAX", "127"); + LIMIT_OBJECT("UINT8_MAX", "255U"); + LIMIT_NEGATIVE("INT16_MIN", "32768"); + LIMIT_OBJECT("INT16_MAX", "32767"); + LIMIT_OBJECT("UINT16_MAX", "65535U"); + LIMIT_MINIMUM("INT32_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT32_MAX", "2147483647"); + LIMIT_OBJECT("UINT32_MAX", "4294967295U"); + LIMIT_MINIMUM("INT64_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INT64_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINT64_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("INTMAX_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INTMAX_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINTMAX_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("SIG_ATOMIC_MIN", "2147483647", "1"); + LIMIT_OBJECT("SIG_ATOMIC_MAX", "2147483647"); + LIMIT_MINIMUM("WCHAR_MIN", "2147483647", "1"); + LIMIT_OBJECT("WCHAR_MAX", "2147483647"); + LIMIT_OBJECT("WINT_MIN", "0U"); + LIMIT_OBJECT("WINT_MAX", "4294967295U"); + LIMIT_NEGATIVE("INT_LEAST8_MIN", "128"); + LIMIT_OBJECT("INT_LEAST8_MAX", "127"); + LIMIT_OBJECT("UINT_LEAST8_MAX", "255U"); + LIMIT_NEGATIVE("INT_LEAST16_MIN", "32768"); + LIMIT_OBJECT("INT_LEAST16_MAX", "32767"); + LIMIT_OBJECT("UINT_LEAST16_MAX", "65535U"); + LIMIT_MINIMUM("INT_LEAST32_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_LEAST32_MAX", "2147483647"); + LIMIT_OBJECT("UINT_LEAST32_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_LEAST64_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INT_LEAST64_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINT_LEAST64_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("INT_FAST8_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_FAST8_MAX", "2147483647"); + LIMIT_OBJECT("UINT_FAST8_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_FAST16_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_FAST16_MAX", "2147483647"); + LIMIT_OBJECT("UINT_FAST16_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_FAST32_MIN", "2147483647", "1"); + LIMIT_OBJECT("INT_FAST32_MAX", "2147483647"); + LIMIT_OBJECT("UINT_FAST32_MAX", "4294967295U"); + LIMIT_MINIMUM("INT_FAST64_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INT_FAST64_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINT_FAST64_MAX", "18446744073709551615ULL"); + LIMIT_CONSTRUCTION("INT8_C", ""); + LIMIT_CONSTRUCTION("UINT8_C", ""); + LIMIT_CONSTRUCTION("INT16_C", ""); + LIMIT_CONSTRUCTION("UINT16_C", ""); + LIMIT_CONSTRUCTION("INT32_C", ""); + LIMIT_CONSTRUCTION("UINT32_C", "U"); + LIMIT_CONSTRUCTION("INT64_C", "LL"); + LIMIT_CONSTRUCTION("UINT64_C", "ULL"); + LIMIT_CONSTRUCTION("INTMAX_C", "LL"); + LIMIT_CONSTRUCTION("UINTMAX_C", "ULL"); if (PTR_SIZE == 8) { - define_builtin_minimum_macro("INTPTR_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("INTPTR_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("UINTPTR_MAX", T_numeric, - "18446744073709551615ULL"); - define_builtin_minimum_macro("PTRDIFF_MIN", "9223372036854775807LL", - "1LL"); - define_builtin_object_macro("PTRDIFF_MAX", T_numeric, - "9223372036854775807LL"); - define_builtin_object_macro("SIZE_MAX", T_numeric, - "18446744073709551615ULL"); + LIMIT_MINIMUM("INTPTR_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("INTPTR_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("UINTPTR_MAX", "18446744073709551615ULL"); + LIMIT_MINIMUM("PTRDIFF_MIN", "9223372036854775807LL", "1LL"); + LIMIT_OBJECT("PTRDIFF_MAX", "9223372036854775807LL"); + LIMIT_OBJECT("SIZE_MAX", "18446744073709551615ULL"); } else { - define_builtin_minimum_macro("INTPTR_MIN", "2147483647L", "1L"); - define_builtin_object_macro("INTPTR_MAX", T_numeric, "2147483647L"); - define_builtin_object_macro("UINTPTR_MAX", T_numeric, "4294967295UL"); - define_builtin_minimum_macro("PTRDIFF_MIN", "2147483647L", "1L"); - define_builtin_object_macro("PTRDIFF_MAX", T_numeric, "2147483647L"); - define_builtin_object_macro("SIZE_MAX", T_numeric, "4294967295UL"); + LIMIT_MINIMUM("INTPTR_MIN", "2147483647L", "1L"); + LIMIT_OBJECT("INTPTR_MAX", "2147483647L"); + LIMIT_OBJECT("UINTPTR_MAX", "4294967295UL"); + LIMIT_MINIMUM("PTRDIFF_MIN", "2147483647L", "1L"); + LIMIT_OBJECT("PTRDIFF_MAX", "2147483647L"); + LIMIT_OBJECT("SIZE_MAX", "4294967295UL"); } } +#undef LIMIT_OBJECT +#undef LIMIT_NEGATIVE +#undef LIMIT_MINIMUM +#undef LIMIT_CONSTRUCTION + /* stdbool.h is entirely macro-defined in C99. Supplying it here makes the * freestanding compiler usable with --no-libc too, rather than relying on the * private definitions prepended from lib/c.h. @@ -409,87 +430,63 @@ void install_iso646_header(void) */ void install_assert_header(void) { - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); + static const builtin_macro_token_t enabled[] = { + {T_open_bracket, "("}, + {T_open_bracket, "("}, + {T_identifier, "expr"}, + {T_close_bracket, ")"}, + {T_question, "?"}, + {T_open_bracket, "("}, + {T_identifier, "void"}, + {T_close_bracket, ")"}, + {T_numeric, "0"}, + {T_colon, ":"}, + {T_identifier, "__assert_fail"}, + {T_open_bracket, "("}, + {T_hash, "#"}, + {T_identifier, "expr"}, + {T_comma, ","}, + {T_identifier, "__FILE__"}, + {T_comma, ","}, + {T_identifier, "__LINE__"}, + {T_comma, ","}, + {T_identifier, "__func__"}, + {T_close_bracket, ")"}, + {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t disabled[] = { + {T_open_bracket, "("}, + {T_identifier, "void"}, + {T_close_bracket, ")"}, + {T_numeric, "0"}, + }; + macro_t *macro = new_builtin_function_macro("assert", "expr", NULL); token_t *tail = NULL; - macro->name = intern_string("assert"); - macro->is_function_like = true; - macro->param_num = 1; - macro->param_names[0] = new_token(T_identifier, &synth_built_in_loc, 4); - macro->param_names[0]->literal = "expr"; - - if (!is_macro_defined("NDEBUG")) { - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "expr"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_question, "?"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "void"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "0"); - append_builtin_macro_token(¯o->replacement, &tail, T_colon, ":"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__assert_fail"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_hash, "#"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "expr"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__FILE__"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__LINE__"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__func__"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - } else { + if (!is_macro_defined("NDEBUG")) + append_builtin_macro_tokens(¯o->replacement, &tail, enabled, + sizeof(enabled) / sizeof(*enabled)); + else { /* C99 requires the disabled form to remain a void expression while not * evaluating its operand. */ - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, - "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "void"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "0"); + append_builtin_macro_tokens(¯o->replacement, &tail, disabled, + sizeof(disabled) / sizeof(*disabled)); } - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); } -macro_t *new_stdarg_macro(const char *name, - const char *first_param, - const char *second_param) -{ - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - - macro->name = intern_string((char *) name); - macro->is_function_like = true; - macro->param_num = second_param ? 2 : 1; - macro->param_names[0] = - new_token(T_identifier, &synth_built_in_loc, strlen(first_param)); - macro->param_names[0]->literal = intern_string((char *) first_param); - if (second_param) { - macro->param_names[1] = - new_token(T_identifier, &synth_built_in_loc, strlen(second_param)); - macro->param_names[1]->literal = intern_string((char *) second_param); - } - return macro; -} +#define INSTALL_STDARG_MACRO(macro_name, first_param, second_param, \ + token_array) \ + do { \ + macro_t *macro = \ + new_builtin_function_macro(macro_name, first_param, second_param); \ + token_t *tail = NULL; \ + append_builtin_macro_tokens( \ + ¯o->replacement, &tail, token_array, \ + sizeof(token_array) / sizeof(*token_array)); \ + hashmap_put_borrowed(MACROS, macro->name, macro); \ + } while (0) /* The current ABI spills every variadic argument into a pointer-sized slot. * These macros deliberately cover integer and pointer arguments only; floating @@ -497,93 +494,40 @@ macro_t *new_stdarg_macro(const char *name, */ void install_stdarg_header(void) { - macro_t *macro; - token_t *tail; - - define_builtin_object_macro("__VA_SLOT_WORDS", T_numeric, - PTR_SIZE == 8 ? "2" : "1"); - define_builtin_object_macro("__VA_SLOT_BYTES", T_numeric, - PTR_SIZE == 8 ? "8" : "4"); - - macro = new_stdarg_macro("va_start", "ap", "last"); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, "ap"); - append_builtin_macro_token(¯o->replacement, &tail, T_assign, "="); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "va_list"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_ampersand, "&"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "last"); - append_builtin_macro_token(¯o->replacement, &tail, T_plus, "+"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_sizeof, "sizeof"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "last"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_plus, "+"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__VA_SLOT_BYTES"); - append_builtin_macro_token(¯o->replacement, &tail, T_minus, "-"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "1"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_divide, "/"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__VA_SLOT_BYTES"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_asterisk, "*"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__VA_SLOT_WORDS"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); - - macro = new_stdarg_macro("va_arg", "ap", "type"); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "__builtin_va_arg"); - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_ampersand, "&"); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, "ap"); - append_builtin_macro_token(¯o->replacement, &tail, T_comma, ","); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "type"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); - - macro = new_stdarg_macro("va_copy", "dest", "src"); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "dest"); - append_builtin_macro_token(¯o->replacement, &tail, T_assign, "="); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, "src"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - hashmap_put(MACROS, macro->name, macro); - - macro = new_stdarg_macro("va_end", "ap", NULL); - tail = NULL; - append_builtin_macro_token(¯o->replacement, &tail, T_open_bracket, "("); - append_builtin_macro_token(¯o->replacement, &tail, T_identifier, - "void"); - append_builtin_macro_token(¯o->replacement, &tail, T_close_bracket, - ")"); - append_builtin_macro_token(¯o->replacement, &tail, T_numeric, "0"); - hashmap_put(MACROS, macro->name, macro); -} + static const builtin_macro_token_t va_start_replacement[] = { + {T_identifier, "__builtin_va_start"}, + {T_open_bracket, "("}, + {T_ampersand, "&"}, + {T_identifier, "ap"}, + {T_comma, ","}, + {T_identifier, "last"}, + {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t va_arg_replacement[] = { + {T_identifier, "__builtin_va_arg"}, + {T_open_bracket, "("}, + {T_ampersand, "&"}, + {T_identifier, "ap"}, + {T_comma, ","}, + {T_identifier, "type"}, + {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t va_copy_replacement[] = { + {T_open_bracket, "("}, {T_identifier, "dest"}, {T_assign, "="}, + {T_identifier, "src"}, {T_close_bracket, ")"}, + }; + static const builtin_macro_token_t va_end_replacement[] = { + {T_open_bracket, "("}, + {T_identifier, "void"}, + {T_close_bracket, ")"}, + {T_numeric, "0"}, + }; + INSTALL_STDARG_MACRO("va_start", "ap", "last", va_start_replacement); + INSTALL_STDARG_MACRO("va_arg", "ap", "type", va_arg_replacement); + INSTALL_STDARG_MACRO("va_copy", "dest", "src", va_copy_replacement); + INSTALL_STDARG_MACRO("va_end", "ap", NULL, va_end_replacement); +} +#undef INSTALL_STDARG_MACRO /* An angle header is lexed as ordinary preprocessing tokens. Recover its * spelling from the immutable source buffer so dotted and slash-separated names @@ -893,45 +837,6 @@ void pp_read_line_operands(token_t *directive, error_at("Unexpected token in #line directive", &cursor->location); } -int pp_get_operator_prio(opcode_t op) -{ - /* https://www.cs.uic.edu/~i109/Notes/COperatorPrecedenceTable.pdf */ - switch (op) { - case OP_ternary: - return 3; - case OP_log_or: - return 4; - case OP_log_and: - return 5; - case OP_bit_or: - return 6; - case OP_bit_xor: - return 7; - case OP_bit_and: - return 8; - case OP_eq: - case OP_neq: - return 9; - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - return 10; - case OP_lshift: - case OP_rshift: - return 11; - case OP_add: - case OP_sub: - return 12; - case OP_mul: - case OP_div: - case OP_mod: - return 13; - default: - return 0; - } -} - int pp_get_unary_operator_prio(opcode_t op) { switch (op) { @@ -957,75 +862,9 @@ token_t *pp_get_operator(token_t *tk, opcode_t *op, bool consume) error_at("Unexpected error when trying to evaulate constant operator", &tk->location); - switch (tk->next->kind) { - case T_plus: - op[0] = OP_add; - break; - case T_minus: - op[0] = OP_sub; - break; - case T_bit_not: - op[0] = OP_bit_not; - break; - case T_log_not: - op[0] = OP_log_not; - break; - case T_asterisk: - op[0] = OP_mul; - break; - case T_divide: - op[0] = OP_div; - break; - case T_mod: - op[0] = OP_mod; - break; - case T_lshift: - op[0] = OP_lshift; - break; - case T_rshift: - op[0] = OP_rshift; - break; - case T_log_and: - op[0] = OP_log_and; - break; - case T_log_or: - op[0] = OP_log_or; - break; - case T_eq: - op[0] = OP_eq; - break; - case T_noteq: - op[0] = OP_neq; - break; - case T_lt: - op[0] = OP_lt; - break; - case T_le: - op[0] = OP_leq; - break; - case T_gt: - op[0] = OP_gt; - break; - case T_ge: - op[0] = OP_geq; - break; - case T_ampersand: - op[0] = OP_bit_and; - break; - case T_bit_or: - op[0] = OP_bit_or; - break; - case T_bit_xor: - op[0] = OP_bit_xor; - break; - case T_question: - op[0] = OP_ternary; - break; - default: - /* Maybe it's an operand, we immediately return here. */ - op[0] = OP_generic; + op[0] = operator_for_token(tk->next->kind); + if (op[0] == OP_generic) return tk; - } return consume ? pp_lex_next_token(tk, true) : tk; } @@ -1184,22 +1023,6 @@ int pp_literal_will_overflow(const pp_integer_t *value, int base, int digit) digit > remainder); } -int pp_numeric_suffix_is_valid(const char *suffix) -{ - int pos = 0; - - if ((suffix[pos] | 32) == 'u') - pos++; - if ((suffix[pos] | 32) == 'l') { - pos++; - if ((suffix[pos] | 32) == 'l') - pos++; - } - if ((suffix[pos] | 32) == 'u') - pos++; - return suffix[pos] == '\0'; -} - void pp_multiply(pp_integer_t *lhs, const pp_integer_t *rhs) { pp_integer_t multiplicand, multiplier, product = {0}; @@ -1209,7 +1032,6 @@ void pp_multiply(pp_integer_t *lhs, const pp_integer_t *rhs) multiplicand.is_unsigned = lhs->is_unsigned; multiplier.lo = rhs->lo; multiplier.hi = rhs->hi; - multiplier.is_unsigned = rhs->is_unsigned; for (int i = 0; i < 64; i++) { if (multiplier.lo & 1) @@ -1297,19 +1119,14 @@ void pp_parse_integer_literal(token_t *tk, pp_integer_t *val) int is_decimal; pp_integer_t parsed = {0}; - if (literal[0] == '0') { - if ((literal[1] | 32) == 'x') { - base = 16; - i = 2; - } else if ((literal[1] | 32) == 'b') { - base = 2; - i = 2; - } else if (literal[1] && (literal[1] | 32) != 'u' && - (literal[1] | 32) != 'l') { - base = 8; - i = 1; - } - } + base = numeric_literal_base(literal); + if (base == 8 && literal[1] && + ((literal[1] | 32) == 'u' || (literal[1] | 32) == 'l')) + base = 10; + if (base == 16 || base == 2) + i = 2; + else if (base == 8) + i = 1; is_decimal = base == 10; while (literal[i] && (literal[i] | 32) != 'u' && (literal[i] | 32) != 'l') { int digit; @@ -1339,7 +1156,7 @@ void pp_parse_integer_literal(token_t *tk, pp_integer_t *val) error_at("Invalid integer constant suffix", &tk->location); i++; } - if (!pp_numeric_suffix_is_valid(suffix)) + if (!numeric_suffix_is_valid(suffix)) error_at("Invalid integer constant suffix", &tk->location); if (!val->is_unsigned && (parsed.hi & 0x80000000U)) { if (is_decimal) @@ -1371,6 +1188,12 @@ void pp_parse_integer_literal(token_t *tk, pp_integer_t *val) } } +#define PP_SIGNED_RELATION(left, right, signed_result, unsigned_result) \ + ((!(left)->is_unsigned && !(right)->is_unsigned && \ + ((left)->hi >> 31) != ((right)->hi >> 31)) \ + ? (signed_result) \ + : (unsigned_result)) + token_t *pp_read_constant_infix_expr(int precedence, token_t *tk, pp_integer_t *val, @@ -1587,7 +1410,6 @@ token_t *pp_read_constant_infix_expr(int precedence, bool evaluate) { pp_integer_t lhs = {0}, rhs = {0}; - int comparison; /* Evaluate unary expression first */ opcode_t op; @@ -1626,7 +1448,7 @@ token_t *pp_read_constant_infix_expr(int precedence, while (true) { tk = pp_get_operator(tk, &op, false); - current_precedence = pp_get_operator_prio(op); + current_precedence = get_operator_prio(op); if (current_precedence == 0 || current_precedence <= precedence) break; @@ -1670,14 +1492,7 @@ token_t *pp_read_constant_infix_expr(int precedence, pp_enum_usual_arithmetic(&lhs, &rhs); switch (op) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: + OP_USUAL_ARITHMETIC_CASES: lhs.is_unsigned = PP_USES_UNSIGNED(&lhs, &rhs); break; default: @@ -1750,40 +1565,24 @@ token_t *pp_read_constant_infix_expr(int precedence, pp_shift_right_one(&lhs, !lhs.is_unsigned); break; case OP_gt: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, !(lhs.hi >> 31)); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison > 0); - } + pp_set_boolean(&lhs, PP_SIGNED_RELATION( + &lhs, &rhs, !(lhs.hi >> 31), + pp_compare_unsigned(&lhs, &rhs) > 0)); break; case OP_geq: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, !(lhs.hi >> 31)); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison >= 0); - } + pp_set_boolean(&lhs, PP_SIGNED_RELATION( + &lhs, &rhs, !(lhs.hi >> 31), + pp_compare_unsigned(&lhs, &rhs) >= 0)); break; case OP_lt: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, lhs.hi >> 31); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison < 0); - } + pp_set_boolean(&lhs, PP_SIGNED_RELATION( + &lhs, &rhs, lhs.hi >> 31, + pp_compare_unsigned(&lhs, &rhs) < 0)); break; case OP_leq: - if (!lhs.is_unsigned && !rhs.is_unsigned && - (lhs.hi >> 31) != (rhs.hi >> 31)) - pp_set_boolean(&lhs, lhs.hi >> 31); - else { - comparison = pp_compare_unsigned(&lhs, &rhs); - pp_set_boolean(&lhs, comparison <= 0); - } + pp_set_boolean(&lhs, PP_SIGNED_RELATION( + &lhs, &rhs, lhs.hi >> 31, + pp_compare_unsigned(&lhs, &rhs) <= 0)); break; case OP_eq: pp_set_boolean(&lhs, lhs.lo == rhs.lo && lhs.hi == rhs.hi); @@ -1815,6 +1614,8 @@ token_t *pp_read_constant_infix_expr(int precedence, return tk; } +#undef PP_SIGNED_RELATION + token_t *pp_read_constant_expr(token_t *tk, pp_integer_t *val) { tk = pp_read_constant_infix_expr(0, tk, val, true); @@ -1962,7 +1763,8 @@ token_t *pp_pragma_operator(token_t *tk, token_t *owner) while (len > 0 && (p[len - 1] == ' ' || p[len - 1] == '\t')) len--; if (len == 4 && !strncmp(p, "once", 4)) - hashmap_put(PRAGMA_ONCE, owner->location.physical_filename, NULL); + hashmap_put_borrowed(PRAGMA_ONCE, owner->location.physical_filename, + NULL); return pp_lex_expect_token(tk, T_close_bracket, true); } @@ -2027,6 +1829,23 @@ token_t *pp_paste_tokens(token_t *lhs, token_t *rhs, source_location_t *loc) * own. Membership in @args, rather than a non-empty value, is what makes a name * a parameter. */ +static bool pp_append_significant_tokens(token_t *tokens, + token_t **tail, + token_t **tail_prev) +{ + bool appended = false; + + for (token_t *tk = tokens; tk; tk = tk->next) { + if (pp_is_layout(tk)) + continue; + *tail_prev = *tail; + (*tail)->next = copy_token(tk); + *tail = (*tail)->next; + appended = true; + } + return appended; +} + token_t *pp_subst_hash(token_t *rep, hashmap_t *args) { token_t head; @@ -2073,17 +1892,8 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) !strcmp(operand->literal, "__VA_ARGS__") && hashmap_contains(args, operand->literal)) { token_t *comma_prev = tail_prev; - bool any = false; - - for (token_t *t = hashmap_get(args, operand->literal); t; - t = t->next) { - if (pp_is_layout(t)) - continue; - tail_prev = tail; - tail->next = copy_token(t); - tail = tail->next; - any = true; - } + bool any = pp_append_significant_tokens( + hashmap_get(args, operand->literal), &tail, &tail_prev); if (!any) { tail = comma_prev; tail->next = NULL; @@ -2130,13 +1940,7 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) * emit the remainder of the macro body twice. */ if (rhs_is_arg) { - for (token_t *rest = rhs->next; rest; rest = rest->next) { - if (pp_is_layout(rest)) - continue; - tail_prev = tail; - tail->next = copy_token(rest); - tail = tail->next; - } + pp_append_significant_tokens(rhs->next, &tail, &tail_prev); } lhs_empty = false; } @@ -2152,16 +1956,7 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) if (args && tk->kind == T_identifier && after && after->kind == T_hashhash && hashmap_contains(args, tk->literal)) { token_t *arg = hashmap_get(args, tk->literal); - bool any = false; - - for (token_t *t = arg; t; t = t->next) { - if (pp_is_layout(t)) - continue; - tail_prev = tail; - tail->next = copy_token(t); - tail = tail->next; - any = true; - } + bool any = pp_append_significant_tokens(arg, &tail, &tail_prev); lhs_present = true; lhs_empty = !any; continue; @@ -2183,6 +1978,81 @@ token_t *pp_subst_hash(token_t *rep, hashmap_t *args) return head.next; } +/* An expansion whose last token names a function-like macro calls it when the + * stream being rescanned follows with '(': a parameter bound to ADD in a body + * "f(a, b)", "#define F ADD" before "F(1, 2)", or "ID(ADD)(1, 2)". The standard + * rescans the expansion together with the rest of the stream; this expander + * rescans the expansion alone, so hand that name back. Splice what precedes it + * in @expanded onto *@cur and return a copy of it chained to the stream after + * @tk, or NULL when no such call follows. + * + * @hidden is the hide set the name came out of: a macro's own name in its + * expansion is never invoked again, so "#define f(x) f" leaves "f(1)(2)" as + * "f(2)", as gcc does. Each resumption consumes a '(' from the stream, so it + * terminates. + */ +static token_t *pp_resume_callee(token_t *expanded, + token_t *end, + token_t *tk, + const preprocess_ctx_t *ctx, + hide_set_t *hidden, + token_t **cur) +{ + token_t *callee = NULL; + token_t *before = NULL; + token_t *prev = NULL; + token_t *resume; + macro_t *macro; + + if (!expanded || !pp_lex_peek_token(tk, T_open_bracket, true)) + return NULL; + for (token_t *t = expanded; t; t = t->next) { + if (!pp_is_layout(t)) { + callee = t; + before = prev; + } + if (t == end) + break; + prev = t; + } + /* A parameter's own name would be substituted again, not invoked. */ + if (!callee || callee->kind != T_identifier || + (ctx->macro_args && + hashmap_contains(ctx->macro_args, callee->literal)) || + hide_set_contains(hidden, callee->literal)) + return NULL; + macro = hashmap_get(MACROS, callee->literal); + if (!macro || !macro->is_function_like || macro->is_disabled) + return NULL; + if (before) { + (*cur)->next = expanded; + *cur = before; + } + resume = copy_token(callee); + resume->next = tk->next; + return resume; +} + +static bool pp_resume_or_append(token_t *expanded, + token_t *end, + token_t **tk, + const preprocess_ctx_t *ctx, + hide_set_t *hidden, + token_t **cur) +{ + token_t *resume = pp_resume_callee(expanded, end, *tk, ctx, hidden, cur); + + if (resume) { + *tk = resume; + return true; + } + if (expanded) { + (*cur)->next = expanded; + *cur = end; + } + return false; +} + token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) { token_t head; @@ -2245,10 +2115,11 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) */ macro_arg_replacement = pp_preprocess_internal( macro_arg_replacement, &expansion_ctx); - if (macro_arg_replacement) { - cur->next = macro_arg_replacement; - cur = expansion_ctx.end_of_token; - } + + if (pp_resume_or_append(macro_arg_replacement, + expansion_ctx.end_of_token, &tk, + ctx, ctx->hide_set, &cur)) + continue; } tk = pp_lex_next_token(tk, false); continue; @@ -2374,13 +2245,14 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) */ if (empty_zero_arg_call) { if (macro->is_variadic) - hashmap_put(expansion_ctx.macro_args, - macro->variadic_tk->literal, NULL); + hashmap_put_borrowed(expansion_ctx.macro_args, + macro->variadic_tk->literal, + NULL); /* Bind argument to corresponding parameter */ } else if (arg_idx < macro->param_num) { param_tk = macro->param_names[arg_idx++]; - hashmap_put(expansion_ctx.macro_args, param_tk->literal, - arg_head.next); + hashmap_put_borrowed(expansion_ctx.macro_args, + param_tk->literal, arg_head.next); } else { /* Handle variadic macro overflow * @@ -2405,10 +2277,10 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) prev->next = new_token(T_comma, ¶m_tk->location, 1); prev->next->next = arg_head.next; - prev = arg_cur; } else { - hashmap_put(expansion_ctx.macro_args, - param_tk->literal, arg_head.next); + hashmap_put_borrowed(expansion_ctx.macro_args, + param_tk->literal, + arg_head.next); } } else { error_at( @@ -2445,8 +2317,8 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) if (macro->is_variadic && !hashmap_contains(expansion_ctx.macro_args, macro->variadic_tk->literal)) - hashmap_put(expansion_ctx.macro_args, - macro->variadic_tk->literal, NULL); + hashmap_put_borrowed(expansion_ctx.macro_args, + macro->variadic_tk->literal, NULL); /* Expand macro body with collected arguments Replace parameter * references with supplied argument tokens @@ -2454,12 +2326,17 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) token_t *expanded = pp_preprocess_internal( pp_subst_hash(macro->replacement, expansion_ctx.macro_args), &expansion_ctx); - if (expanded) { - cur->next = expanded; - cur = expansion_ctx.end_of_token; - } - hashmap_free(expansion_ctx.macro_args); + if (pp_resume_or_append(expanded, expansion_ctx.end_of_token, + &tk, ctx, expansion_ctx.hide_set, &cur)) + continue; + } else if (macro->is_function_like) { + /* A function-like macro expands only when followed by '('. Its + * bare identifier may be an ordinary identifier or a later + * macro argument, so preserve it unchanged. + */ + cur->next = copy_token(tk); + cur = cur->next; } else { /* Handle object-like macro expansion (no parameters) Simply * expand the replacement with current hide set plus this macro @@ -2469,10 +2346,9 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) hide_set_union(ctx->hide_set, new_hide_set(tk->literal)); token_t *expanded = pp_preprocess_internal( pp_subst_hash(macro->replacement, NULL), &expansion_ctx); - if (expanded) { - cur->next = expanded; - cur = expansion_ctx.end_of_token; - } + if (pp_resume_or_append(expanded, expansion_ctx.end_of_token, + &tk, ctx, expansion_ctx.hide_set, &cur)) + continue; } tk = pp_lex_next_token(tk, false); @@ -2602,6 +2478,14 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) inclusion_path[MAX_LINE_LEN - 1] = '\0'; } + /* A header that is missing is the directive's mistake, and is + * reported there. Reading it here also leaves it cached for the + * lexer, under the name the lexer will ask for. + */ + if (!try_get_file_buf(normalize_filename(inclusion_path))) + error_at("Included file cannot be found", + &include_tk->location); + tk = pp_lex_expect_token(tk, T_newline, true); tk = pp_lex_next_token(tk, false); @@ -2632,8 +2516,7 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) macro = hashmap_get(MACROS, tk->literal); if (!macro) { - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = tk->literal; + macro = new_macro(tk->literal); } else { /* Ensures that #undef effect is overwritten */ macro->is_disabled = false; @@ -2712,7 +2595,7 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) tk = pp_lex_expect_token(tk, T_newline, false); tk = pp_lex_next_token(tk, false); macro->replacement = r_head; - hashmap_put(MACROS, macro->name, macro); + hashmap_put_borrowed(MACROS, macro->name, macro); continue; } case T_cppd_undef: { @@ -2795,8 +2678,8 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) tk = pp_lex_next_token(tk, true); if (!strcmp("once", tk->literal)) - hashmap_put(PRAGMA_ONCE, tk->location.physical_filename, - NULL); + hashmap_put_borrowed(PRAGMA_ONCE, + tk->location.physical_filename, NULL); } while (!pp_lex_peek_token(tk, T_newline, true)) @@ -2834,7 +2717,6 @@ token_t *pp_preprocess_internal(token_t *tk, preprocess_ctx_t *ctx) * included previously while created by #define. */ error_at("Backslash is not allowed here", &cur->location); - break; } case T_eof: { if (ctx->trim_eof) { @@ -3001,72 +2883,62 @@ token_t *preprocess(token_t *tk) synth_built_in_loc.line = 1; synth_built_in_loc.filename = ""; - macro_t *macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__FILE__"; + macro_t *macro = new_macro("__FILE__"); macro->handler = file_macro_handler; - hashmap_put(MACROS, "__FILE__", macro); + hashmap_put_borrowed(MACROS, "__FILE__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__LINE__"; + macro = new_macro("__LINE__"); macro->handler = line_macro_handler; - hashmap_put(MACROS, "__LINE__", macro); + hashmap_put_borrowed(MACROS, "__LINE__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__DATE__"; + macro = new_macro("__DATE__"); macro->handler = date_macro_handler; - hashmap_put(MACROS, "__DATE__", macro); + hashmap_put_borrowed(MACROS, "__DATE__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__TIME__"; + macro = new_macro("__TIME__"); macro->handler = time_macro_handler; - hashmap_put(MACROS, "__TIME__", macro); + hashmap_put_borrowed(MACROS, "__TIME__", macro); /* C99-required implementation macros. shecc supplies its own small runtime * rather than a complete hosted library, so advertise freestanding mode * while retaining the C99 language-version identifier. */ - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__STDC__"; + macro = new_macro("__STDC__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, "__STDC__", macro); + hashmap_put_borrowed(MACROS, "__STDC__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__STDC_VERSION__"; + macro = new_macro("__STDC_VERSION__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 7); macro->replacement->literal = "199901L"; - hashmap_put(MACROS, "__STDC_VERSION__", macro); + hashmap_put_borrowed(MACROS, "__STDC_VERSION__", macro); - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__STDC_HOSTED__"; + macro = new_macro("__STDC_HOSTED__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "0"; - hashmap_put(MACROS, "__STDC_HOSTED__", macro); + hashmap_put_borrowed(MACROS, "__STDC_HOSTED__", macro); /* architecture defines */ - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = ARCH_PREDEFINED; + macro = new_macro(ARCH_PREDEFINED); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, ARCH_PREDEFINED, macro); + hashmap_put_borrowed(MACROS, ARCH_PREDEFINED, macro); /* shecc run-time defines */ - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__SHECC__"; + macro = new_macro("__SHECC__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, "__SHECC__", macro); + hashmap_put_borrowed(MACROS, "__SHECC__", macro); /* Tells the source being compiled that the embedded libc is not part of the * output, so the functions lib/c.c would have supplied -- '__syscall' above * all -- are unavailable and libc resolves through the PLT instead. */ if (dynlink) { - macro = arena_calloc(TOKEN_ARENA, 1, sizeof(macro_t)); - macro->name = "__SHECC_DYNLINK__"; + macro = new_macro("__SHECC_DYNLINK__"); macro->replacement = new_token(T_numeric, &synth_built_in_loc, 1); macro->replacement->literal = "1"; - hashmap_put(MACROS, "__SHECC_DYNLINK__", macro); + hashmap_put_borrowed(MACROS, "__SHECC_DYNLINK__", macro); } tk = pp_preprocess_internal(tk, &ctx); @@ -3076,6 +2948,10 @@ token_t *preprocess(token_t *tk) return tk; } +#define TOKEN_SPELLING(kind, spelling) \ + case kind: \ + return spelling + char *token_to_string(token_t *tk, char *dest) { switch (tk->kind) { @@ -3103,164 +2979,85 @@ char *token_to_string(token_t *tk, char *dest) case T_wchar: snprintf(dest, MAX_TOKEN_LEN, "L'%s'", tk->literal); return dest; - case T_comma: - return ","; - case T_open_bracket: - return "("; - case T_close_bracket: - return ")"; - case T_open_curly: - return "{"; - case T_close_curly: - return "}"; - case T_open_square: - return "["; - case T_close_square: - return "]"; - case T_asterisk: - return "*"; - case T_divide: - return "/"; - case T_mod: - return "%"; - case T_bit_or: - return "|"; - case T_bit_xor: - return "^"; - case T_bit_not: - return "~"; - case T_log_and: - return "&&"; - case T_log_or: - return "||"; - case T_log_not: - return "!"; - case T_lt: - return "<"; - case T_gt: - return ">"; - case T_le: - return "<="; - case T_ge: - return ">="; - case T_lshift: - return "<<"; - case T_rshift: - return ">>"; - case T_dot: - return "."; - case T_arrow: - return "->"; - case T_plus: - return "+"; - case T_minus: - return "-"; - case T_minuseq: - return "-="; - case T_pluseq: - return "+="; - case T_asteriskeq: - return "*="; - case T_divideeq: - return "/="; - case T_modeq: - return "%="; - case T_lshifteq: - return "<<="; - case T_rshifteq: - return ">>="; - case T_xoreq: - return "^="; - case T_oreq: - return "|="; - case T_andeq: - return "&="; - case T_eq: - return "=="; - case T_noteq: - return "!="; - case T_assign: - return "="; - case T_increment: - return "++"; - case T_decrement: - return "--"; - case T_question: - return "?"; - case T_colon: - return ":"; - case T_semicolon: - return ";"; - case T_ampersand: - return "&"; - case T_return: - return "return"; - case T_if: - return "if"; - case T_else: - return "else"; - case T_while: - return "while"; - case T_for: - return "for"; - case T_do: - return "do"; - case T_typedef: - return "typedef"; - case T_enum: - return "enum"; - case T_struct: - return "struct"; - case T_union: - return "union"; - case T_sizeof: - return "sizeof"; - case T_elipsis: - return "..."; - case T_switch: - return "switch"; - case T_case: - return "case"; - case T_break: - return "break"; - case T_default: - return "default"; - case T_continue: - return "continue"; - case T_goto: - return "goto"; - case T_const: - return "const"; - case T_volatile: - return "volatile"; - case T_static: - return "static"; - case T_extern: - return "extern"; - case T_register: - return "register"; - case T_auto: - return "auto"; - case T_restrict: - return "restrict"; - case T_inline: - return "inline"; - case T_signed: - return "signed"; - case T_unsigned: - return "unsigned"; - case T_long: - return "long"; - case T_float: - return "float"; - case T_double: - return "double"; - case T_complex: - return "_Complex"; - case T_imaginary: - return "_Imaginary"; - case T_newline: - return "\n"; + TOKEN_SPELLING(T_comma, ","); + TOKEN_SPELLING(T_open_bracket, "("); + TOKEN_SPELLING(T_close_bracket, ")"); + TOKEN_SPELLING(T_open_curly, "{"); + TOKEN_SPELLING(T_close_curly, "}"); + TOKEN_SPELLING(T_open_square, "["); + TOKEN_SPELLING(T_close_square, "]"); + TOKEN_SPELLING(T_asterisk, "*"); + TOKEN_SPELLING(T_divide, "/"); + TOKEN_SPELLING(T_mod, "%"); + TOKEN_SPELLING(T_bit_or, "|"); + TOKEN_SPELLING(T_bit_xor, "^"); + TOKEN_SPELLING(T_bit_not, "~"); + TOKEN_SPELLING(T_log_and, "&&"); + TOKEN_SPELLING(T_log_or, "||"); + TOKEN_SPELLING(T_log_not, "!"); + TOKEN_SPELLING(T_lt, "<"); + TOKEN_SPELLING(T_gt, ">"); + TOKEN_SPELLING(T_le, "<="); + TOKEN_SPELLING(T_ge, ">="); + TOKEN_SPELLING(T_lshift, "<<"); + TOKEN_SPELLING(T_rshift, ">>"); + TOKEN_SPELLING(T_dot, "."); + TOKEN_SPELLING(T_arrow, "->"); + TOKEN_SPELLING(T_plus, "+"); + TOKEN_SPELLING(T_minus, "-"); + TOKEN_SPELLING(T_minuseq, "-="); + TOKEN_SPELLING(T_pluseq, "+="); + TOKEN_SPELLING(T_asteriskeq, "*="); + TOKEN_SPELLING(T_divideeq, "/="); + TOKEN_SPELLING(T_modeq, "%="); + TOKEN_SPELLING(T_lshifteq, "<<="); + TOKEN_SPELLING(T_rshifteq, ">>="); + TOKEN_SPELLING(T_xoreq, "^="); + TOKEN_SPELLING(T_oreq, "|="); + TOKEN_SPELLING(T_andeq, "&="); + TOKEN_SPELLING(T_eq, "=="); + TOKEN_SPELLING(T_noteq, "!="); + TOKEN_SPELLING(T_assign, "="); + TOKEN_SPELLING(T_increment, "++"); + TOKEN_SPELLING(T_decrement, "--"); + TOKEN_SPELLING(T_question, "?"); + TOKEN_SPELLING(T_colon, ":"); + TOKEN_SPELLING(T_semicolon, ";"); + TOKEN_SPELLING(T_ampersand, "&"); + TOKEN_SPELLING(T_return, "return"); + TOKEN_SPELLING(T_if, "if"); + TOKEN_SPELLING(T_else, "else"); + TOKEN_SPELLING(T_while, "while"); + TOKEN_SPELLING(T_for, "for"); + TOKEN_SPELLING(T_do, "do"); + TOKEN_SPELLING(T_typedef, "typedef"); + TOKEN_SPELLING(T_enum, "enum"); + TOKEN_SPELLING(T_struct, "struct"); + TOKEN_SPELLING(T_union, "union"); + TOKEN_SPELLING(T_sizeof, "sizeof"); + TOKEN_SPELLING(T_elipsis, "..."); + TOKEN_SPELLING(T_switch, "switch"); + TOKEN_SPELLING(T_case, "case"); + TOKEN_SPELLING(T_break, "break"); + TOKEN_SPELLING(T_default, "default"); + TOKEN_SPELLING(T_continue, "continue"); + TOKEN_SPELLING(T_goto, "goto"); + TOKEN_SPELLING(T_const, "const"); + TOKEN_SPELLING(T_volatile, "volatile"); + TOKEN_SPELLING(T_static, "static"); + TOKEN_SPELLING(T_extern, "extern"); + TOKEN_SPELLING(T_register, "register"); + TOKEN_SPELLING(T_auto, "auto"); + TOKEN_SPELLING(T_restrict, "restrict"); + TOKEN_SPELLING(T_inline, "inline"); + TOKEN_SPELLING(T_signed, "signed"); + TOKEN_SPELLING(T_unsigned, "unsigned"); + TOKEN_SPELLING(T_long, "long"); + TOKEN_SPELLING(T_float, "float"); + TOKEN_SPELLING(T_double, "double"); + TOKEN_SPELLING(T_complex, "_Complex"); + TOKEN_SPELLING(T_imaginary, "_Imaginary"); + TOKEN_SPELLING(T_newline, "\n"); case T_backslash: error_at( "Internal error, backslash should be ommited after " @@ -3276,6 +3073,8 @@ char *token_to_string(token_t *tk, char *dest) } case T_tab: return "\t"; + case T_translation_unit: + return "\n"; case T_start: /* FIXME: Unused token kind */ break; @@ -3305,16 +3104,22 @@ char *token_to_string(token_t *tk, char *dest) return NULL; } +#undef TOKEN_SPELLING + void emit_preprocessed_token(token_t *tk) { char token_buffer[MAX_TOKEN_LEN], *literal; + token_kind_t previous_kind = T_start; while (tk) { - literal = token_to_string(tk, token_buffer); + literal = tk->kind == T_translation_unit && previous_kind == T_newline + ? "" + : token_to_string(tk, token_buffer); if (literal) printf("%s", literal); + previous_kind = tk->kind; tk = tk->next; } } diff --git a/src/reg-alloc.c b/src/reg-alloc.c deleted file mode 100644 index 6960043e..00000000 --- a/src/reg-alloc.c +++ /dev/null @@ -1,4143 +0,0 @@ -/* - * shecc - Self-Hosting and Educational C Compiler. - * - * shecc is freely redistributable under the BSD 2 clause license. See the file - * "LICENSE" for information on usage and redistribution of this file. - */ - -/* Allocate registers from IR. The linear-scan algorithm now expects a minimum - * of 7 available registers (typical for RISC-style architectures). - * - * TODO: Implement "-O level" optimization control. Currently the allocator - * always performs dead variable elimination without writing back to stack. - */ -#include "defs.h" -#include "globals.c" - -/* A value is pointer-like if it is a pointer itself or its type is. On LP64 - * targets these occupy PTR_SIZE bytes rather than 4. - */ -bool is_pointer_like(var_t *v) -{ - return v && (v->ptr_level > 0 || (v->type && v->type->ptr_level > 0)); -} - -/* An operand holds an address if it is pointer-like or is an array, which - * decays to one wherever it is used as a value. A subscript reaches OP_add with - * the array itself as the base, so leaving arrays out here would let an LP64 - * backend read a[i] as ordinary arithmetic and skip widening the index. - */ -bool is_address_like(var_t *v) -{ - return is_pointer_like(v) || (v && v->array_size > 0); -} - -/* Record which operands of a three-address instruction are addresses. - * - * An LP64 backend needs the operands apart, not just the instruction: pointer - * arithmetic keeps the address 64 bits wide while sign-extending the int index - * beside it. - * - * The two source flags and is_pointer deliberately ask different questions. - * Only the AArch64 backend reads the source flags, and it wants every address, - * arrays included. is_pointer is older and the x86-64 backend decides store - * widths and int narrowing by it, so it keeps counting pointer-like operands - * alone rather than acquiring arrays and changing a settled target. - */ -bool is_unsigned_scalar(const var_t *var) -{ - return var && !var->ptr_level && !var->is_func && var->type && - var->type->is_unsigned; -} - -void set_ptr_flags(ph2_ir_t *ir, insn_t *insn) -{ - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->src1_is_pointer = is_address_like(insn->rs2); - ir->is_pointer = is_pointer_like(insn->rd) || is_pointer_like(insn->rs1) || - is_pointer_like(insn->rs2); - ir->is_unsigned = is_unsigned_scalar(insn->rd) || - is_unsigned_scalar(insn->rs1) || - is_unsigned_scalar(insn->rs2); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->src1_is_unsigned = is_unsigned_scalar(insn->rs2); - - /* Addresses are ordered as unsigned values (C99 6.5.8): one above the - * signed range still compares greater than one below it. - */ - if ((insn->opcode == OP_lt || insn->opcode == OP_leq || - insn->opcode == OP_gt || insn->opcode == OP_geq) && - (ir->src0_is_pointer || ir->src1_is_pointer)) - ir->src0_is_unsigned = true; -} - -/* Width of the value a local's frame slot actually holds. - * - * Slots are pointer-sized, but a scalar occupies only its low bytes. Reading - * the whole slot back would pick up stale bytes whenever the variable was - * written through a pointer, so loads must use the declared width. Anything not - * a known scalar -- pointers, arrays, aggregates, functions -- keeps a full - * pointer in its slot and is reported as such. - */ -int var_slot_size(var_t *v) -{ - if (!v || v->ptr_level || v->is_func || v->array_size) - return PTR_SIZE; - - /* Only a variable whose address escaped can have its slot written behind - * the allocator's back, at the pointee's width rather than the slot's. - * Everything else is written and read through this same path, so the full - * slot is always valid and the wider access is safe. - * - * Narrowing these to the declared width does not work: an SSA temporary - * carries the type of the expression that made it, which for pointer - * arithmetic is still int, and a four-byte slot truncates the address. - */ - if (!v->type) - return PTR_SIZE; - - /* A direct wide scalar needs both words even when its address never - * escapes: spilling an SSA temporary is still a memory round trip. - */ - if (!is_record_type(v->type) && v->type->size == 8) - return 8; - - /* A global is memory another function reads, and its data slot is only as - * wide as its type: "unsigned int a, b;" places b four bytes after a on - * LP64, so a full-width store to a overwrites b. - */ - if (!v->address_taken && !v->is_global) - return PTR_SIZE; - - /* Classify by the stored width rather than by type identity: an enum is a - * distinct type_t that still stores as a 4-byte int, and a store through an - * enum pointer writes only those 4 bytes. Reading the slot any wider then - * picks up whatever the stack happened to hold above it. - */ - if (v->type->base_type == TYPE_struct) - return PTR_SIZE; - if (v->type->size == 1 || v->type->size == 2 || v->type->size == 4) - return v->type->size; - return PTR_SIZE; -} - -/* High halves stay out of REGS[]: its users assume a register entry names one - * ordinary scalar owner. This parallel reservation table prevents a pair's high - * machine register from being handed to a different value. - */ -var_t *pair_high_owner[REG_CNT]; - -void vreg_map_to_phys(var_t *var, int phys_reg) -{ - if (var) - var->phys_reg = phys_reg; -} - -/* A wide scalar on a 32-bit target owns two independent machine registers. - * Keeping the high mapping on the SSA value, rather than duplicating @var in - * REGS[], lets the existing register file continue to mean one complete - * single-word owner per entry while paired allocation is introduced. - */ -void vreg_map_pair_to_phys(var_t *var, int low_reg, int high_reg) -{ - if (!var) - return; - if (high_reg < 0 || high_reg >= REG_CNT) - fatal("Invalid high register for wide value"); - if (pair_high_owner[high_reg] && pair_high_owner[high_reg] != var) - fatal("Wide value high register is already reserved"); - var->phys_reg = low_reg; - var->phys_reg_hi = high_reg; - pair_high_owner[high_reg] = var; -} - -int vreg_get_phys(var_t *var) -{ - if (var) - return var->phys_reg; - return -1; -} - -int vreg_get_phys_hi(var_t *var) -{ - if (var) - return var->phys_reg_hi; - return -1; -} - -/* ABI argument locations are measured in machine words, not source parameters. - * Keeping that calculation in one place matters once a direct 64-bit scalar - * occupies two words on a 32-bit target: its first word must be even-aligned, - * and every following argument starts after both words. - */ -int abi_arg_words(const var_t *var) -{ - return var_needs_register_pair(var) ? 2 : 1; -} - -/* Where an argument starts, given the next free ABI word. - * - * AAPCS32 starts every long long at an even word, in registers and on the stack - * alike. The RV32 calling convention does so only for a variadic argument and - * on the stack, whose slots are eight-byte aligned; a named one takes the next - * two words, and when only a7 is left its low word goes there and its high word - * to the first stack slot. - */ -int abi_arg_start(int cursor, const var_t *var, bool variadic) -{ - if (abi_arg_words(var) == 2 && - (ELF_MACHINE != 0xf3 || variadic || cursor >= MAX_ARGS_IN_REG)) - cursor = ALIGN_UP(cursor, 2); - return cursor; -} - -int abi_arg_next(int cursor, const var_t *var, bool variadic) -{ - return abi_arg_start(cursor, var, variadic) + abi_arg_words(var); -} - -/* Whether an argument starting at @word is split between the last argument - * register and the stack, which only a named RV32 long long can be. - */ -bool abi_arg_is_split(int word, const var_t *var) -{ - return word < MAX_ARGS_IN_REG && - word + abi_arg_words(var) > MAX_ARGS_IN_REG; -} - -int abi_param_start(const func_t *func, int param_idx) -{ - int cursor = func->returns_aggregate ? 1 : 0; - - for (int i = 0; i < param_idx; i++) - cursor = abi_arg_next(cursor, &func->param_defs[i], false); - return abi_arg_start(cursor, &func->param_defs[param_idx], false); -} - -void vreg_clear_phys(var_t *var) -{ - if (var) { - if (var->phys_reg_hi >= 0 && var->phys_reg_hi < REG_CNT && - pair_high_owner[var->phys_reg_hi] == var) - pair_high_owner[var->phys_reg_hi] = NULL; - var->phys_reg = -1; - var->phys_reg_hi = -1; - } -} - -/* Aligns size to nearest multiple of 4, this meets ARMv7's alignment - * requirement. - * - * This function should be called whenever handling with user-defined type's - * size. - */ -int align_size(int i) -{ - return i <= 4 ? 4 : (i + 3) & ~3; -} - -bool aggregate_has_function_pointer_seen(type_t *type, - type_t **seen, - int seen_count) -{ - if (!type) - return false; - - for (int i = 0; i < seen_count; i++) - if (seen[i] == type) - return false; - - if (seen_count >= MAX_TYPES) - return false; - seen[seen_count++] = type; - - if (type->base_type == TYPE_typedef && type->base_struct) - type = type->base_struct; - - for (int i = 0; i < type->num_fields; i++) { - var_t *field = &type->fields[i]; - if (field->is_func) - return true; - - if (!field->ptr_level && field->type && !field->type->ptr_level && - (field->type->base_type == TYPE_struct || - field->type->base_type == TYPE_union || - field->type->base_type == TYPE_typedef) && - aggregate_has_function_pointer_seen(field->type, seen, seen_count)) - return true; - } - - return false; -} - -bool aggregate_has_function_pointer(type_t *type) -{ - type_t *seen[MAX_TYPES]; - return aggregate_has_function_pointer_seen(type, seen, 0); -} - -bool check_live_out(basic_block_t *bb, var_t *var) -{ - return var_list_holds(&bb->live_out, var); -} - -void track_var_use(var_t *var, int insn_idx) -{ - if (!var) - return; - - var->use_count++; - - if (var->first_use < 0) - var->first_use = insn_idx; - - var->last_use = insn_idx; -} - -void refresh(basic_block_t *bb, const insn_t *insn) -{ - for (int i = 0; i < REG_CNT; i++) { - if (!REGS[i].var) - continue; - if (check_live_out(bb, REGS[i].var)) - continue; - if (REGS[i].var->consumed < insn->idx) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - } - } -} - -ph2_ir_t *bb_add_ph2_ir(basic_block_t *bb, opcode_t op) -{ - ph2_ir_t *n = arena_alloc(BB_ARENA, sizeof(ph2_ir_t)); - n->op = op; - /* Initialize all fields explicitly */ - n->next = NULL; /* well-formed singly linked list */ - n->is_branch_detached = 0; /* arch-lowering will set for branches */ - n->src0 = 0; - n->src1 = 0; - - /* Only a select names a third source, but the allocation is not zeroed and - * every field is set here by hand. - */ - n->src2 = 0; - n->dest = 0; - n->src0_hi = -1; - n->src1_hi = -1; - n->dest_hi = -1; - n->func_name = NULL; - n->next_bb = NULL; - n->then_bb = NULL; - n->else_bb = NULL; - n->ofs_based_on_stack_top = false; - n->size_bytes = PTR_SIZE; /* default to the full slot; see add_ph2_ir */ - n->is_pointer = false; - n->is_unsigned = false; - n->src0_is_pointer = false; - n->src1_is_pointer = false; - n->src0_is_unsigned = false; - n->src1_is_unsigned = false; - n->is_volatile = false; - - if (!bb->ph2_ir_list.head) - bb->ph2_ir_list.head = n; - else - bb->ph2_ir_list.tail->next = n; - - bb->ph2_ir_list.tail = n; - return n; -} - -/* Calculate the cost of spilling a variable from a register. Higher cost means - * the variable is more valuable to keep in a register. The cost is computed - * based on multiple factors that affect performance. - */ -int calculate_spill_cost(var_t *var, basic_block_t *bb, int current_idx) -{ - int cost = 0; - - /* Variables that are live-out of the basic block must be spilled anyway, so - * give them a high cost to prefer spilling them over others - */ - if (check_live_out(bb, var)) - cost += 1000; - - /* Variables that will be used soon should have higher cost. The closer the - * next use, the higher the penalty for spilling - */ - if (var->consumed > current_idx) { - int distance = var->consumed - current_idx; - if (distance < 10) - cost += 100 - distance * 10; /* Max 100 points for immediate use */ - } - - /* Frequently used variables should stay in registers. Each use adds 5 - * points to the cost - */ - if (var->use_count > 0) - cost += var->use_count * 5; - - /* Constants can be easily reloaded, so prefer spilling them by reducing - * their cost - */ - if (var->is_const) - cost -= 50; - - /* Variables with long live ranges may benefit from spilling to free up - * registers for other variables - */ - if (var->first_use >= 0 && var->last_use >= 0) { - int range_length = var->last_use - var->first_use; - if (range_length > 100) - cost += 20; /* Small penalty for very long live ranges */ - } - - return cost; -} - -/* Variables kept in a register for the whole of the current function, named by - * the base each SSA version was renamed from. - * - * The base is the right unit. Every assignment makes a new SSA version, so - * pinning versions individually holds nothing: each one dies within a few - * instructions and the value still travels between them through a stack slot. - * Pinning the base keeps the variable itself in a register, which is what a - * loop carrying it across an iteration actually needs. - * - * Only functions that call nothing qualify, because every register in the file - * is caller-saved. - */ -var_t *pinned_base[REG_CNT]; - -/* Registers the instruction being lowered has already placed an operand in. - * - * Preparing one operand can spill another's register to make room, leaving the - * number recorded for it naming a register that no longer holds the value. An - * instruction reading more operands than the two "avoid" arguments can express - * -- a select reads three -- locks each as it is placed, and the spill searches - * leave those alone. - */ -int reg_locked; - -/* ABI argument registers already staged for the call being lowered. A staged - * register can lose its REGS[] owner while the call still needs its value: a - * variable passed twice as a pair, as in "f(g, g, ...)", is moved to its second - * argument pair, which released the first, and the next argument was loaded - * straight over it. - */ -int abi_args_staged; - -/* Whether @reg holds an operand of the instruction being lowered, or an - * argument of the call being staged. - */ -bool reg_is_locked(int reg) -{ - if (reg < 0 || reg >= REG_CNT) - return false; - return ((reg_locked | abi_args_staged) >> reg) & 1; -} - -/* The register @var's base is pinned to, or -1. */ -int pinned_reg_of(const var_t *var) -{ - if (!var || !var->base) - return -1; - - /* A value an if-converted arm computes shares its variable's base with the - * select's result, but the pinned register still holds what the arms read, - * so it takes an ordinary register instead. - */ - if (var->in_select_arm) - return -1; - for (int i = 0; i < REG_CNT; i++) { - if (pinned_base[i] && pinned_base[i] == var->base) - return i; - } - return -1; -} - -int find_best_spill(basic_block_t *bb, - int current_idx, - int avoid_reg1, - int avoid_reg2) -{ - int best_reg = -1; - int min_cost = 99999; - - for (int i = 0; i < REG_CNT; i++) { - if (i == avoid_reg1 || i == avoid_reg2) - continue; - - if (!REGS[i].var || pinned_base[i] || reg_is_locked(i)) - continue; - - int cost = calculate_spill_cost(REGS[i].var, bb, current_idx); - - if (cost < min_cost) { - min_cost = cost; - best_reg = i; - } - } - - return best_reg; -} - -/* Priority of spilling: - * - live_out variable - * - farthest local variable Slots reg_alloc() hands out inside the current - * function, in allocation order and therefore ascending by offset: both places - * that call slot_var_track() assign var->offset = stack_size and then grow the - * frame. Only these are candidates for the two cleanups below; every other - * address in the frame belongs to something a pointer may legally reach. - * - * The parallel arrays are indexed by position in slot_vars and are filled once - * per function by slot_scan(). - */ -var_t *slot_vars[MAX_LOCALS]; -int slot_var_count; -char slot_private[MAX_LOCALS]; -char slot_stores[MAX_LOCALS]; /* saturates at 2: only "exactly one" matters */ -char slot_loads[MAX_LOCALS]; -basic_block_t *slot_home[MAX_LOCALS]; - -/* The first store to and the first load from each slot, and the list node that - * precedes the store. collapse_slot_roundtrip() needs all three, and - * slot_scan() already walks past them, so recording them here saves it a search - * from the head of the block for every slot it considers. - */ -ph2_ir_t *slot_store_ir[MAX_LOCALS]; -ph2_ir_t *slot_store_prev[MAX_LOCALS]; -ph2_ir_t *slot_load_ir[MAX_LOCALS]; - -void slot_var_track(var_t *var) -{ - if (slot_var_count < MAX_LOCALS) - slot_vars[slot_var_count++] = var; -} - -/* Whether the slot's address cannot have escaped, so that the stores and loads - * naming it are the only accesses to it. address_taken covers &var; array_size - * and has_backing_storage cover aggregates whose interior is reached by pointer - * arithmetic. A store through a pointer is invisible to these scans, so - * anything else in the frame is left alone. - */ -bool slot_is_private(const var_t *var) -{ - if (var->address_taken || var->array_size || var->has_backing_storage || - var_is_volatile_object(var)) - return false; - if (var->is_global || var->ofs_based_on_stack_top) - return false; - return true; -} - -/* Position of the slot at @offset in slot_vars, or -1 when the offset does not - * name one. The array is sorted, so this is a binary search. - */ -int slot_lookup(int offset) -{ - int lo = 0, hi = slot_var_count - 1; - - while (lo <= hi) { - int mid = (lo + hi) >> 1; - int at = slot_vars[mid]->offset; - - if (at == offset) - return mid; - if (at < offset) - lo = mid + 1; - else - hi = mid - 1; - } - return -1; -} - -/* Tally the accesses to every tracked slot in one walk of the function. - * - * Both cleanups below need to know how often a slot is stored and loaded. - * Asking that question per slot means re-walking the whole function once for - * each of them, which is quadratic and, on shecc's own larger functions, was - * the single most expensive thing the compiler did. - */ -void slot_scan(func_t *func) -{ - /* alloc_var_slot() appends in increasing offset order, but phi_slot_merge() - * then rewrites the offsets of variables already in the table to put a - * chain of phis on one slot, which can leave it out of order. slot_lookup() - * binary-searches, so an unsorted table makes it miss a slot that is - * present: both scans below and dead_store_elim() miss the same one, so - * nothing is misidentified, but the two cleanups skip work they could do. - * - * Insertion sort, because the table is nearly sorted already and every - * entry a merge moved sits close to where it belongs. It costs about 0.085% - * of a self-compile and recovers optimizations worth rather less than that; - * it is here to keep the invariant alloc_var_slot() documents true, not to - * pay for itself. - */ - for (int i = 1; i < slot_var_count; i++) { - var_t *var = slot_vars[i]; - int j = i - 1; - - while (j >= 0 && slot_vars[j]->offset > var->offset) { - slot_vars[j + 1] = slot_vars[j]; - j--; - } - slot_vars[j + 1] = var; - } - - for (int i = 0; i < slot_var_count; i++) { - slot_private[i] = slot_is_private(slot_vars[i]); - slot_stores[i] = 0; - slot_loads[i] = 0; - slot_home[i] = NULL; - slot_store_ir[i] = NULL; - slot_store_prev[i] = NULL; - slot_load_ir[i] = NULL; - } - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - ph2_ir_t *prev = NULL; - for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { - if (ir->ofs_based_on_stack_top) { - prev = ir; - continue; - } - if (ir->op == OP_store) { - int i = slot_lookup(ir->src1); - if (i < 0) { - prev = ir; - continue; - } - if (slot_stores[i] < 2) { - if (!slot_stores[i]) { - slot_store_ir[i] = ir; - slot_store_prev[i] = prev; - } - slot_stores[i]++; - } - slot_home[i] = bb; - } else if (ir->op == OP_load) { - int i = slot_lookup(ir->src0); - if (i < 0) { - prev = ir; - continue; - } - if (slot_loads[i] < 2) { - if (!slot_loads[i]) - slot_load_ir[i] = ir; - slot_loads[i]++; - } - if (slot_home[i] != bb) - slot_home[i] = NULL; - } - prev = ir; - } - } -} - -/* Unlink @ir, whose predecessor in @bb's list is @prev (NULL at the head). */ -void ph2_list_remove(basic_block_t *bb, ph2_ir_t *prev, ph2_ir_t *ir) -{ - if (prev) - prev->next = ir->next; - else - bb->ph2_ir_list.head = ir->next; - if (bb->ph2_ir_list.tail == ir) - bb->ph2_ir_list.tail = prev; - - /* Clearing the link marks the node as gone. One left pointing at its old - * successor still looks like that successor's predecessor, and - * collapse_slot_roundtrip() decides whether the predecessor it recorded is - * still usable by asking exactly that. - */ - ir->next = NULL; -} - -/* Whether @ir leaves @reg holding something other than what it held before. */ -bool ph2_writes_reg(const ph2_ir_t *ir, int reg) -{ - switch (ir->op) { - case OP_store: - case OP_global_store: - case OP_write: - case OP_branch: - case OP_jump: - case OP_return: - case OP_push: - return false; - case OP_call: - case OP_indirect: - return true; /* the call clobbers the caller-saved registers */ - default: - /* A pair result writes its high register too. */ - return ir->dest == reg || (reg >= 0 && ir->dest_hi == reg); - } -} - -/* Whether @ir clobbers either register of the value stored from @store. */ -bool ph2_writes_stored_value(const ph2_ir_t *ir, const ph2_ir_t *store) -{ - return ph2_writes_reg(ir, store->src0) || - (store->src0_hi >= 0 && ph2_writes_reg(ir, store->src0_hi)); -} - -/* Collapse a value that goes out to a stack slot and comes straight back. - * - * Unwinding a phi writes the value to the phi's own slot, and the next - * instruction reads it back to copy it into the variable's slot -- a round trip - * through memory that also hides the "s = s + x" pattern from - * try_fold_alu_to_slot(), which would otherwise emit a single add to the slot. - * When the slot is written once and read once in the same block, and the source - * register survives in between, the load is just a register move and the store - * is dead. - */ -void collapse_slot_roundtrip(func_t *func) -{ - UNUSED(func); - for (int i = 0; i < slot_var_count; i++) { - if (!slot_private[i] || slot_stores[i] != 1 || slot_loads[i] != 1) - continue; - - basic_block_t *home = slot_home[i]; - if (!home) - continue; - - int offset = slot_vars[i]->offset; - ph2_ir_t *store = slot_store_ir[i], *load = slot_load_ir[i]; - ph2_ir_t *prev_store = slot_store_prev[i], *prev_load = NULL; - bool ok = false; - - /* slot_scan() already walked past this slot's only store and its only - * load, so the search that used to start at the head of the block is - * needed only when an earlier collapse in this same block unlinked the - * node the store was recorded behind. A removed node still points at - * its old successor, which is why the check is that the recorded - * predecessor is still in the list rather than that it still points - * here: ph2_list_remove() clears the link of whatever it takes out. - */ - bool linked = store && load; - if (linked) { - if (prev_store) { - if (prev_store->next != store) - linked = false; - } else if (home->ph2_ir_list.head != store) - linked = false; - } - - if (linked) { - /* The store has to come first, and its register has to still hold - * the value when the load would have run. Walking from the store to - * the load settles both: a load placed ahead of the store is never - * reached, and a clobber on the way stops the walk short. The walk - * passes the load's predecessor, so that comes from here rather - * than from a search. - */ - ph2_ir_t *prev = store; - - for (ph2_ir_t *ir = store->next; ir; ir = ir->next) { - if (ir == load) { - prev_load = prev; - ok = true; - break; - } - if (ph2_writes_stored_value(ir, store)) - break; - prev = ir; - } - } else { - ph2_ir_t *prev = NULL; - - store = NULL; - load = NULL; - prev_store = NULL; - for (ph2_ir_t *ir = home->ph2_ir_list.head; ir; ir = ir->next) { - if (!ir->ofs_based_on_stack_top) { - if (!store && ir->op == OP_store && ir->src1 == offset) { - store = ir; - prev_store = prev; - prev = ir; - continue; - } - if (ir->op == OP_load && ir->src0 == offset) { - load = ir; - prev_load = prev; - ok = store != NULL; - break; - } - } - if (store && ph2_writes_stored_value(ir, store)) - break; - prev = ir; - } - } - - /* A pair is moved as a pair: forwarding only the low register turned - * the load into a move that left the high register unwritten. - */ - if (!ok || (store->src0_hi >= 0) != (load->dest_hi >= 0)) - continue; - - load->op = OP_assign; - load->src0 = store->src0; - load->src0_hi = store->src0_hi; - - /* Removing the store first would leave prev_load stale when the store - * is the load's predecessor, so drop the later one first. - */ - if (load->dest == load->src0 && load->dest_hi == load->src0_hi) - ph2_list_remove(home, prev_load, load); - ph2_list_remove(home, prev_store, store); - } -} - -/* Drop stores to a slot that nothing in the function ever loads. - * - * Unwinding a phi copies the value into the phi's slot, and the source SSA temp - * is written to a slot of its own that no one reads -- one wasted store per phi - * operand, on every iteration of a loop. - */ -void dead_store_elim(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - ph2_ir_t *prev = NULL, *next; - for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = next) { - next = ir->next; - if (ir->op == OP_store && !ir->ofs_based_on_stack_top) { - int i = slot_lookup(ir->src1); - if (i >= 0 && slot_private[i] && !slot_loads[i]) { - ph2_list_remove(bb, prev, ir); - continue; - } - } - prev = ir; - } - } -} - -/* Write the register back to the variable's stack slot, allocating the slot on - * first use. The register keeps holding the value; only memory is made - * coherent, so callers decide separately whether to drop the association. - * Reserve @var's stack slot in @func's frame. - * - * A variable whose address escapes is reached through a pointer that carries no - * information about what it points to, so give it the alignment any object type - * could need rather than only its own. Slots stay in increasing offset order, - * which slot_lookup()'s binary search relies on. - */ -void alloc_var_slot(func_t *func, var_t *var) -{ - int slot_size = var_slot_size(var); - - if (var->address_taken) - func->stack_size = ALIGN_UP(func->stack_size, 16); - - /* A future 32-bit wide scalar occupies a low/high word pair. It must not - * start at an arbitrary four-byte offset: both AAPCS32 and the RV32 ABI - * require the pair's stack home to be eight-byte aligned. - */ - if (slot_size > PTR_SIZE) - func->stack_size = ALIGN_UP(func->stack_size, slot_size); - var->offset = func->stack_size; - var->space_is_allocated = true; - func->stack_size += slot_size; - slot_var_track(var); -} - -void store_var(basic_block_t *bb, var_t *var, int idx) -{ - if (!var->space_is_allocated) { - alloc_var_slot(bb->belong_to, var); - } - ph2_ir_t *ir = var->is_global ? bb_add_ph2_ir(bb, OP_global_store) - : bb_add_ph2_ir(bb, OP_store); - ir->src0 = idx; - ir->src0_hi = vreg_get_phys_hi(var); - ir->src1 = var->offset; - ir->ofs_based_on_stack_top = var->ofs_based_on_stack_top; - ir->is_pointer = is_pointer_like(var); - ir->is_unsigned = is_unsigned_scalar(var); - ir->size_bytes = var_slot_size(var); - ir->is_volatile = var_is_volatile_object(var); - REGS[idx].polluted = 0; -} - -void spill_var(basic_block_t *bb, var_t *var, int idx) -{ - if (REGS[idx].polluted) - store_var(bb, var, idx); - REGS[idx].var = NULL; - REGS[idx].polluted = 0; - vreg_clear_phys(var); -} - -/* Whether register @i can be handed to a value that wants one. - * - * Empty is not enough: a pinned register looks empty whenever the version it - * currently holds has been consumed, but nothing ever reloads it and the next - * read of its variable expects to find the value still there. - */ -bool reg_is_free(int i) -{ - return !REGS[i].var && !pinned_base[i] && !pair_high_owner[i] && - !((abi_args_staged >> i) & 1); -} - -/* Return the index of register for given variable. Otherwise, return -1. */ -int find_in_regs(const var_t *var) -{ - for (int i = 0; i < REG_CNT; i++) { - if (REGS[i].var == var) - return i; - } - return -1; -} - -/* Whether @var can live in a register for a whole function: nothing else may be - * able to reach it, and it must fit in one register. - */ -bool var_is_pinnable(var_t *var) -{ - if (!var || var->is_const || !var->base) - return false; - if (var_is_volatile_object(var)) - return false; - - /* slot_is_private() rules out everything reachable other than by name: - * globals, address-taken variables, arrays and backing storage. - */ - if (!slot_is_private(var)) - return false; - return var_slot_size(var) <= PTR_SIZE; -} - -/* Choose which variables this function keeps in registers. - * - * Candidates are ranked by how often they are named, which stands in well - * enough for how hot they are: a variable a loop carries is named on every - * iteration. At most half the file is given away so expression evaluation still - * has registers to work with. - */ -int pin_scan_gen; - -void pin_registers(func_t *func) -{ - int limit = REG_CNT / 2; - bool calls = false; - bool pairs = false; - - for (int i = 0; i < REG_CNT; i++) - pinned_base[i] = NULL; - func->pinned_regs = 0; - - mark_loop_depth(func); - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_call || insn->opcode == OP_indirect || - insn->opcode == OP_push) - calls = true; - if (!pairs && (var_needs_register_pair(insn->rd) || - var_needs_register_pair(insn->rs1) || - var_needs_register_pair(insn->rs2))) - pairs = true; - /* Taking an address makes the frame reachable by other means. */ - if (insn->opcode == OP_address_of) - return; - } - } - - /* A binary operation on register pairs holds two operand pairs and a - * destination pair at once. Pinning half the file left a 32-bit target with - * too few registers to lower one, so keep six of them unpinned. - */ - if (pairs && limit > REG_CNT - 6) - limit = REG_CNT - 6; - - /* Across a call only the registers the callee preserves will still hold - * their value. Those sit at the top of the file, which is the end the loop - * below hands out from, so capping the count is all that is needed. - */ - if (calls) { - if (!CALLEE_SAVED_REGS) - return; - if (limit > CALLEE_SAVED_REGS) - limit = CALLEE_SAVED_REGS; - } - - /* Tally what every variable's namings are worth. The count lives on the - * variable rather than in an array here: the file has a handful of - * registers and a function names hundreds of variables, and the one worth a - * register is not reliably among the first few met -- a pointer - * strength_reduce() introduced is named last of all. - */ - pin_scan_gen++; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *ops[4]; - - ops[0] = insn->rd; - ops[1] = insn->rs1; - ops[2] = insn->rs2; - ops[3] = insn->rs3; - - for (int k = 0; k < 4; k++) { - var_t *var = ops[k]; - - if (!var_is_pinnable(var)) - continue; - - /* A parameter passed on the stack lives at an offset into the - * caller's frame that the callee cannot pin; one passed in a - * register is moved into its pinned register on entry, so it is - * a candidate like any other -- and a good one, since a pointer - * a loop walks is usually a parameter. - */ - bool on_stack = false; - bool in_reg = false; - - for (int q = 0; q < func->num_params; q++) { - if (var->base != &func->param_defs[q]) - continue; - int word = abi_param_start(func, q); - if (word + abi_arg_words(&func->param_defs[q]) <= - MAX_ARGS_IN_REG) - in_reg = true; - else - on_stack = true; - } - if (on_stack) - continue; - - var_t *base = var->base; - - if (base->pin_gen != pin_scan_gen) { - base->pin_gen = pin_scan_gen; - base->pin_weight = 0; - - /* A parameter is written in the entry block before the body - * names it, so its range already spans blocks. - */ - base->pin_cross = in_reg; - base->pin_hot = false; - base->pin_blk = -1; - } - base->pin_weight += bb->loop_weight; - if (base->pin_blk >= 0 && base->pin_blk != bb->rpo) - base->pin_cross = true; - base->pin_blk = bb->rpo; - if (bb->loop_depth) - base->pin_hot = true; - } - } - } - - /* Take the most-named bases, highest register first: the low registers are - * where arguments and return values land. The winner is found by walking - * the tally again rather than sorting it, which costs one pass per register - * handed out -- a handful, against a function's instruction count. - */ - for (int taken = 0; taken < limit; taken++) { - var_t *best = NULL; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *ops[4]; - - ops[0] = insn->rd; - ops[1] = insn->rs1; - ops[2] = insn->rs2; - ops[3] = insn->rs3; - - for (int k = 0; k < 4; k++) { - if (!ops[k] || !ops[k]->base) - continue; - - var_t *base = ops[k]->base; - - if (base->pin_gen != pin_scan_gen) - continue; - - /* Registers are handed out from the top of the file, which - * is where the preserved ones are, so a pinned function - * saves and restores one in its prologue. A variable named - * once in straight-line code does not earn that back -- - * least of all in a small leaf called from a loop, which - * pays it on every call -- so a candidate has to be read - * inside a loop and to outlive the block that names it. - */ - if (!base->pin_cross || !base->pin_hot) - continue; - - bool already = false; - - for (int r = 0; r < REG_CNT; r++) { - if (pinned_base[r] == base) - already = true; - } - if (already) - continue; - if (!best || base->pin_weight > best->pin_weight) - best = base; - } - } - } - if (!best) - return; - - /* Incoming arguments arrive in the low registers and are placed there - * before the body runs, which would overwrite anything pinned to one of - * them; the variable would then read a parameter's value instead. - */ - int reg = REG_CNT - 1 - taken; - int arg_words_in_reg = func->returns_aggregate ? 1 : 0; - for (int i = 0; i < func->num_params; i++) { - int word = abi_param_start(func, i); - - if (word + abi_arg_words(&func->param_defs[i]) > MAX_ARGS_IN_REG) - break; - arg_words_in_reg = abi_arg_next(word, &func->param_defs[i], false); - } - - if (reg < arg_words_in_reg) - return; - - pinned_base[reg] = best; - func->pinned_regs = func->pinned_regs | (1 << reg); - } -} - -/* Defined beside OP_push handling below; allocation must not reclaim ABI - * argument registers once argument staging has begun. - */ -extern bool is_pushing_args; - -void load_var(basic_block_t *bb, var_t *var, int idx) -{ - ph2_ir_t *ir; - - /* Load constants directly, others from memory */ - if (var->is_const) { - ir = bb_add_ph2_ir(bb, OP_load_constant); - ir->src0 = var->init_val; - ir->src1 = var->init_val_hi; - } else if (var->is_global && var->array_size) { - /* A global array's address is fixed for the life of the program, and - * the initialiser recorded where its storage sits. Computing it beats - * loading back the pointer that was stored there: the address is an - * addition rather than a memory read, so it does not have to wait on - * the data cache on every trip round a loop. - */ - ir = bb_add_ph2_ir(bb, OP_global_address_of); - ir->src0 = var->init_val; - } else { - ir = var->is_global ? bb_add_ph2_ir(bb, OP_global_load) - : bb_add_ph2_ir(bb, OP_load); - ir->src0 = var->offset; - ir->ofs_based_on_stack_top = var->ofs_based_on_stack_top; - ir->is_volatile = var_is_volatile_object(var); - } - - ir->dest = idx; - ir->dest_hi = vreg_get_phys_hi(var); - ir->is_pointer = is_pointer_like(var); - ir->is_unsigned = is_unsigned_scalar(var); - - /* Incoming stack arguments use ABI-sized slots but carry a scalar in the - * low declared-width bytes. Reloading all eight bytes can retain a caller's - * sign extension; use the declaration width to preserve unsigned argument - * semantics. - */ - ir->size_bytes = var->ofs_based_on_stack_top && !var->ptr_level && - !var->is_func && var->type - ? var->type->size - : var_slot_size(var); - REGS[idx].var = var; - REGS[idx].polluted = 0; - vreg_map_to_phys(var, idx); -} - -/* The two lowest free registers, in *@low and *@high. */ -bool find_free_pair(int *low, int *high) -{ - *low = -1; - *high = -1; - for (int r = 0; r < REG_CNT; r++) { - if (!reg_is_free(r)) - continue; - if (*low < 0) { - *low = r; - } else { - *high = r; - return true; - } - } - return false; -} - -/* Two free registers for a register pair, in *@low and *@high, spilling to make - * room when the file is full. The ordinary chooser frees one register, which - * cannot help a pair, so spill whole pairs through their slots first, which - * keeps both words, and then one-word values, whether live or dead. Neither - * @operand_0 nor @operand_1 is touched, nor a pinned register, and a one-word - * spill also leaves reserved high halves and locked inputs alone. Once OP_push - * has installed ABI arguments, those registers are live until the call, so - * nothing is spilled then. - */ -bool claim_free_pair(basic_block_t *bb, - int operand_0, - int operand_1, - int *low, - int *high) -{ - if (find_free_pair(low, high) || is_pushing_args) - return *high >= 0; - - for (int scalars = 0; scalars < 2; scalars++) { - for (int r = 0; r < REG_CNT; r++) { - var_t *owner = REGS[r].var; - - if (!owner || r == operand_0 || r == operand_1 || pinned_base[r]) - continue; - if (scalars) { - if (var_needs_register_pair(owner) || pair_high_owner[r] || - reg_is_locked(r)) - continue; - } else if (!var_needs_register_pair(owner)) { - continue; - } - spill_var(bb, owner, r); - if (find_free_pair(low, high)) - return true; - } - } - return false; -} - -int prepare_operand(basic_block_t *bb, var_t *var, int operand_0) -{ - /* A pinned variable is already where it always is -- unless this version of - * it is a constant, which has no home to be in until it is written there. - * Every other version reaches the register by being defined into it or by a - * phi move, so nothing else needs materialising. - */ - int pinned = pinned_reg_of(var); - if (pinned >= 0) { - if (var->is_const && REGS[pinned].var != var) - load_var(bb, var, pinned); - REGS[pinned].var = var; - return pinned; - } - - /* Check VReg mapping first for O(1) lookup. A register holding a volatile - * variable only as its storage last held it is left to the reload below. - */ - int phys_reg = vreg_get_phys(var); - if (phys_reg >= 0 && phys_reg < REG_CNT && REGS[phys_reg].var == var) { - if (!var_is_volatile_object(var) || REGS[phys_reg].polluted) - return phys_reg; - - /* Reload a volatile variable into the register, or the pair, that - * already names it. Loading it into others left two registers naming - * the variable, and store_addressed_def(), which finds the first, took - * the stale one for the one a later assignment wrote and skipped the - * store. - */ - if (phys_reg != operand_0) { - load_var(bb, var, phys_reg); - return phys_reg; - } - } - - if (var_needs_register_pair(var)) { - int low, high; - - if (!claim_free_pair(bb, operand_0, -1, &low, &high)) - fatal("Wide operand needs two free registers"); - vreg_map_pair_to_phys(var, low, high); - load_var(bb, var, low); - return low; - } - - /* Force reload for address-taken variables (may be modified via pointer). - * - * Not when the register holds a value the slot never received, though. No - * pointer can have reached the slot since: taking the address stores the - * register first, and a write through a pointer or a call spills or drops - * every register. A parameter still in the register it arrived in is that - * case, and its slot may not exist yet, so reloading it read whatever sat - * at offset zero of the frame instead. - * - * A volatile global is reloaded for the same reason and one more: every - * read of it is an access the program has to perform (C99 6.7.3p6), not - * just a way to learn its value. A volatile local is address-taken already. - */ - int i = find_in_regs(var); - if (i > -1 && (!(var->address_taken || var_is_volatile_object(var)) || - REGS[i].polluted)) { - vreg_map_to_phys(var, i); - return i; - } - - /* As above, when the register lost the mapping but still names it. */ - if (i > -1 && i != operand_0 && var_is_volatile_object(var)) { - load_var(bb, var, i); - return i; - } - - for (i = 0; i < REG_CNT; i++) { - if (reg_is_free(i)) { - load_var(bb, var, i); - vreg_map_to_phys(var, i); - return i; - } - } - - int spilled = find_best_spill( - bb, bb->insn_list.tail ? bb->insn_list.tail->idx : 0, operand_0, -1); - - if (spilled < 0) { - for (i = 0; i < REG_CNT; i++) { - /* A pinned register is not available: nothing reloads it, so - * handing it to another variable loses the pinned value. - */ - if (i != operand_0 && REGS[i].var && !pinned_base[i] && - !reg_is_locked(i)) { - spilled = i; - break; - } - } - } - - /* spill_var() forgets the mapping once it has stored the value. Clearing it - * first dropped a pair's high register, so only the low word was stored. - */ - spill_var(bb, REGS[spilled].var, spilled); - load_var(bb, var, spilled); - vreg_map_to_phys(var, spilled); - - return spilled; -} - -/* Set while lowering the run of OP_push that stages a call's arguments. The - * pushed values have to stay put until the call consumes them, and that - * liveness is recorded on var->consumed rather than in the block's remaining - * instructions, so no register may be reused for anything here. - */ -bool is_pushing_args; - -/* Whether @var is read again before the end of @bb. - * - * var->consumed is the last use anywhere in the function, which says nothing - * about this block: a value whose only remaining reader sits on a path this - * block does not reach is dead here, and every value that outlives the block is - * in its live-out set. Asking the block directly is what makes the two arms of - * an if-else able to reuse the same register. - */ -bool var_read_later_in_bb(basic_block_t *bb, insn_t *from, const var_t *var) -{ - UNUSED(bb); - for (insn_t *insn = from; insn; insn = insn->next) { - if (insn->rs1 == var || insn->rs2 == var || insn->rs3 == var) - return true; - } - return false; -} - -/* Forget every register a call does not preserve. - * - * A pinned register the callee preserves still holds its variable, and - * forgetting that would leave it looking free for the allocator to hand to - * something else, losing the value the call went to the trouble of keeping. - */ -void clobber_caller_saved(void) -{ - for (int i = 0; i < REG_CNT; i++) { - if (!pinned_base[i] && REGS[i].var) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - } - if (!REGS[i].var) - REGS[i].polluted = 0; - } -} - -/* x86 ALU instructions are two-operand: "rd = rs1 OP rs2" is emitted as "MOV - * rd, rs1" followed by "OP rd, rs2", and the MOV disappears entirely when rd - * already names rs1's register. So when a source operand is dead after this - * instruction, its register is the best possible home for the destination. - * - * Only a plain local qualifies. A global or an address-taken variable is - * reachable through memory, so its register cannot simply be repurposed even - * once the SSA name is dead. - */ -int coalesce_candidate(basic_block_t *bb, insn_t *insn, int reg) -{ - if (reg < 0 || reg >= REG_CNT) - return -1; - - if (!insn) - return -1; - - /* Restricted to the pure ALU forms, whose lowering is exactly "MOV rd, rs1; - * OP rd, rs2". Everything else -- loads, calls, address arithmetic the - * backend folds into addressing modes -- has its own register expectations - * and is left alone. - */ - switch (insn->opcode) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_lshift: - case OP_rshift: - case OP_negate: - case OP_bit_not: - break; - default: - return -1; - } - - if (is_pushing_args) - return -1; - - /* A pinned register is not a scratch one however dead the version it - * currently holds looks: the variable living there is read again further on - * and nothing ever reloads it. Handing it to a temporary destroyed the - * value a select was about to choose between. - */ - if (pinned_base[reg]) - return -1; - - var_t *v = REGS[reg].var; - if (!v || v->is_global || v->address_taken) - return -1; - if (check_live_out(bb, v)) - return -1; - if (var_read_later_in_bb(bb, insn->next, v)) - return -1; - return reg; -} - -int prepare_dest(basic_block_t *bb, - insn_t *insn, - var_t *var, - int operand_0, - int operand_1) -{ - if (var_needs_register_pair(var)) { - int mapped_low = vreg_get_phys(var); - int mapped_high = vreg_get_phys_hi(var); - if (mapped_low >= 0 && mapped_low < REG_CNT && mapped_high >= 0 && - mapped_high < REG_CNT && REGS[mapped_low].var == var && - pair_high_owner[mapped_high] == var) { - REGS[mapped_low].polluted = 1; - return mapped_low; - } - int low, high; - - if (!claim_free_pair(bb, operand_0, operand_1, &low, &high)) - fatal("Wide destination needs two free registers"); - REGS[low].var = var; - REGS[low].polluted = 1; - vreg_map_pair_to_phys(var, low, high); - return low; - } - - int pinned = pinned_reg_of(var); - if (pinned >= 0) { - REGS[pinned].var = var; - REGS[pinned].polluted = 1; - return pinned; - } - - int phys_reg = vreg_get_phys(var); - if (phys_reg >= 0 && phys_reg < REG_CNT && REGS[phys_reg].var == var) { - REGS[phys_reg].polluted = 1; - return phys_reg; - } - - int i = find_in_regs(var); - if (i > -1) { - REGS[i].polluted = 1; - vreg_map_to_phys(var, i); - return i; - } - - /* Reuse a dying source operand's register so the two-operand lowering needs - * no MOV. Taking a free register instead would cost one on every such - * instruction. - */ - i = coalesce_candidate(bb, insn, operand_0); - if (i < 0) - i = coalesce_candidate(bb, insn, operand_1); - if (i > -1) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = var; - REGS[i].polluted = 1; - vreg_map_to_phys(var, i); - return i; - } - - for (i = 0; i < REG_CNT; i++) { - if (reg_is_free(i)) { - REGS[i].var = var; - REGS[i].polluted = 1; - vreg_map_to_phys(var, i); - return i; - } - } - - int spilled = - find_best_spill(bb, bb->insn_list.tail ? bb->insn_list.tail->idx : 0, - operand_0, operand_1); - - if (spilled < 0) { - for (i = 0; i < REG_CNT; i++) { - if (i != operand_0 && i != operand_1 && REGS[i].var && - !pinned_base[i] && !reg_is_locked(i)) { - spilled = i; - break; - } - } - } - - /* Callers normally have at least one register which is not an operand, but - * an instruction with more register inputs than the allocator's two avoid - * arguments can leave every register protected. Let that caller make an - * instruction-specific choice instead of indexing REGS[-1]. - */ - if (spilled < 0) - return -1; - - /* As in prepare_operand(), spill before forgetting the mapping, or a pair's - * high word is not stored. - */ - spill_var(bb, REGS[spilled].var, spilled); - REGS[spilled].var = var; - REGS[spilled].polluted = 1; - vreg_map_to_phys(var, spilled); - - return spilled; -} - -void spill_alive(basic_block_t *bb, const insn_t *insn) -{ - /* Spill every variable on pointer writes (conservative aliasing handling). - * A global is no exception: the pointer can hold its address, and a copy - * left in a register is what the next read of it would use. - */ - if (insn && insn->opcode == OP_write) { - for (int i = 0; i < REG_CNT; i++) { - /* A pinned variable has no address, so no write through a pointer - * can reach it. - */ - if (REGS[i].var && !pinned_base[i]) - spill_var(bb, REGS[i].var, i); - } - return; - } - - /* Standard spilling for non-pointer operations */ - for (int i = 0; i < REG_CNT; i++) { - if (!REGS[i].var || pinned_base[i]) - continue; - if (check_live_out(bb, REGS[i].var)) { - spill_var(bb, REGS[i].var, i); - continue; - } - if (REGS[i].var->consumed > insn->idx) { - spill_var(bb, REGS[i].var, i); - continue; - } - } -} - -void spill_live_out(basic_block_t *bb) -{ - for (int i = 0; i < REG_CNT; i++) { - if (!REGS[i].var || pinned_base[i]) - continue; - if (!check_live_out(bb, REGS[i].var)) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - continue; - } - if (!var_check_killed(REGS[i].var, bb)) { - vreg_clear_phys(REGS[i].var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - continue; - } - spill_var(bb, REGS[i].var, i); - } -} - -/* End a block the way spill_live_out() does -- every live-out variable written - * here is flushed to its stack slot, so memory is coherent for every path out - * of 'bb' -- but keep the register associations instead of dropping them. What - * survives the boundary is decided per successor by bb_export_regs(). - */ -void spill_live_out_keep(basic_block_t *bb) -{ - for (int i = 0; i < REG_CNT; i++) { - var_t *var = REGS[i].var; - if (!var || pinned_base[i]) - continue; - if (!check_live_out(bb, var)) { - vreg_clear_phys(var); - REGS[i].var = NULL; - REGS[i].polluted = 0; - continue; - } - if (REGS[i].polluted) - store_var(bb, var, i); - } -} - -/* Hand the register file to the successor when it is the only way out of 'bb' - * and the only way in to the successor: control then reaches it with exactly - * these registers, so it can read them in place rather than reloading from the - * slots just written. - * - * Requiring the successor to be next in reverse post-order is what makes - * "emitted immediately after" true: cfg_flatten() walks the same rpo_next chain - * the allocator does, so the two orders are the same traversal. - * - * Only a successor emitted immediately after this block qualifies. A branch - * target is emitted wherever the backend's linear walk puts it, and a block the - * walk misses is re-emitted later -- that second copy is reached with unrelated - * registers, so a file handed to it would not hold on every path that runs its - * code. Requiring the successor to be the next block in reverse post-order is - * what rules that out, since that is the order the walk emits. - * - * A conditional branch falls through as well: the backend jumps to one - * successor and lets control run on into the other, which is emitted - * contiguously exactly as a plain successor is. That edge therefore qualifies - * on the same terms, and it is the one that matters -- it is where an if/else - * chain would otherwise reload on every arm what the test just had in a - * register. - */ -void bb_export_regs(basic_block_t *bb) -{ - basic_block_t *succ = bb->next; - - if (!succ) { - /* Take whichever arm the walk placed next; the checks below confirm it - * really is contiguous and has no other way in. - */ - if (bb->then_ == bb->rpo_next) - succ = bb->then_; - else if (bb->else_ == bb->rpo_next) - succ = bb->else_; - else - return; - } - - /* The successor must also be the block reg_alloc() visits next, so that the - * file it inherits is the one just built here. - */ - if (succ != bb->rpo_next || succ->entry_regs) - return; - if (bb_pred_count(succ) != 1) - return; - - succ->entry_regs = arena_alloc(BB_ARENA, REG_CNT * sizeof(var_t *)); - for (int i = 0; i < REG_CNT; i++) { - var_t *var = REGS[i].var; - - /* Hand over only what the successor is going to read: pinning a - * register it has no use for keeps the value live across the boundary - * for nothing, and the backend then has to materialise it where it - * would otherwise have folded it into its consumer. - */ - succ->entry_regs[i] = - var && var_list_holds(&succ->live_in, var) ? var : NULL; - } -} - -/* Install the register file this block starts with: the predecessor's file when - * it handed one over, an empty one otherwise. Everything handed over has just - * been written back, so nothing is polluted on entry. - */ -void load_entry_regs(basic_block_t *bb) -{ - for (int i = 0; i < REG_CNT; i++) { - var_t *var = bb->entry_regs ? bb->entry_regs[i] : NULL; - - /* A pinned register holds the same variable on every path in. */ - if (pinned_base[i]) { - REGS[i].polluted = 0; - continue; - } - - if (REGS[i].var && REGS[i].var != var) - vreg_clear_phys(REGS[i].var); - REGS[i].var = var; - REGS[i].polluted = 0; - vreg_map_to_phys(var, i); - } -} - -/* The operand of 'OP_push' should not been killed until function called. */ -void extend_liveness(basic_block_t *bb, - const insn_t *insn, - var_t *var, - int offset) -{ - if (check_live_out(bb, var)) - return; - if (insn->idx + offset > var->consumed) - var->consumed = insn->idx + offset; -} - -/* Materialize a wide argument exactly where the ABI consumes it. Moving a pair - * after arbitrary allocation is not generally safe: the source and destination - * pairs can overlap in a cycle. A spill/reload costs a little more in that - * uncommon path, but gives both 32-bit backends an unambiguous pair in the - * required ABI registers. - */ -void prepare_pair_argument(basic_block_t *bb, var_t *var, int low) -{ - int high = low + 1; - int old_low = vreg_get_phys(var); - int old_high = vreg_get_phys_hi(var); - - if (old_low == low && old_high == high && REGS[low].var == var && - pair_high_owner[high] == var) - return; - - if (old_low >= 0 && old_low < REG_CNT && REGS[old_low].var == var) - spill_var(bb, var, old_low); - else - vreg_clear_phys(var); - - if (REGS[low].var) - spill_var(bb, REGS[low].var, low); - if (pair_high_owner[high]) { - var_t *owner = pair_high_owner[high]; - int owner_low = vreg_get_phys(owner); - - if (owner_low >= 0 && owner_low < REG_CNT && - REGS[owner_low].var == owner) - spill_var(bb, owner, owner_low); - else - vreg_clear_phys(owner); - } - - vreg_map_pair_to_phys(var, low, high); - load_var(bb, var, low); -} - -/* Reserve the outgoing stack-argument area @func's calls need. - * - * add_func() reserves one pointer-sized word for each source parameter that - * cannot go in a register, which is enough only while every argument takes one - * ABI word. A long long takes two on a 32-bit target, and may be preceded by a - * padding word, so a call can reach well past that reservation: RISC-V, which - * passes MAX_PARAMS arguments in registers, reserves nothing at all. The words - * abi_lower_call_args() stores above the reservation then overwrote the first - * locals of the caller's frame. Nothing is placed in the frame before register - * allocation, so growing the reservation here moves every slot above it. - */ -void reserve_outgoing_args(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - int cursor = 0; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_push) { - cursor = 0; - continue; - } - /* Aligning every argument bounds both placement rules. */ - cursor = abi_arg_next(cursor, insn->rs1, true); - if ((cursor - MAX_ARGS_IN_REG) * PTR_SIZE > func->stack_size) - func->stack_size = (cursor - MAX_ARGS_IN_REG) * PTR_SIZE; - } - } -} - -/* The first ABI word of each argument of the call being staged, in push order, - * and the next one OP_push takes. abi_lower_call_args() fills them in once per - * call, since only it sees the callee and so which arguments are variadic. - */ -int call_arg_starts[MAX_PARAMS + 1]; -int call_arg_next; - -/* Load only the low word of the register pair @var into @reg. */ -void load_low_word(basic_block_t *bb, var_t *var, int reg) -{ - int held = vreg_get_phys(var); - ph2_ir_t *ir; - - if (held >= 0 && held < REG_CNT && REGS[held].var == var) { - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = held; - } else if (var->is_const) { - ir = bb_add_ph2_ir(bb, OP_load_constant); - ir->src0 = var->init_val; - } else { - ir = bb_add_ph2_ir(bb, var->is_global ? OP_global_load : OP_load); - ir->src0 = var->offset; - ir->ofs_based_on_stack_top = var->ofs_based_on_stack_top; - ir->size_bytes = 4; - ir->is_volatile = var_is_volatile_object(var); - } - ir->dest = reg; -} - -/* Return whether extra arguments are pushed onto stack. */ -bool abi_lower_call_args(basic_block_t *bb, insn_t *insn) -{ - /* An aggregate-return call has one ABI-only destination in addition to the - * MAX_PARAMS source arguments. - */ - insn_t *pushes[MAX_PARAMS + 1]; - int *starts = call_arg_starts; - int num_of_args = 0; - int cursor = 0; - insn_t *call = insn; - func_t *callee = NULL; - int first_param = 0; - - while (call && call->opcode == OP_push) - call = call->next; - if (call && call->opcode == OP_call) - callee = find_func(call->str); - else if (call && call->opcode == OP_indirect) - callee = get_func_signature(call->rs1); - if (callee && callee->returns_aggregate) - first_param = 1; - - call_arg_next = 0; - while (insn && insn->opcode == OP_push) { - bool variadic; - - if (num_of_args >= MAX_PARAMS + 1) - fatal("Too many call arguments"); - variadic = callee && callee->va_args && - num_of_args - first_param >= callee->num_params; - starts[num_of_args] = abi_arg_start(cursor, insn->rs1, variadic); - pushes[num_of_args++] = insn; - cursor = abi_arg_next(cursor, insn->rs1, variadic); - insn = insn->next; - } - - if (cursor <= MAX_ARGS_IN_REG) - return false; - - for (int i = num_of_args - 1; i >= 0; i--) { - insn = pushes[i]; - - /* A split long long leaves its high word in the first stack slot. Its - * low word is placed with the register arguments. - */ - if (abi_arg_is_split(starts[i], insn->rs1)) { - int scratch = MAX_ARGS_IN_REG - 2; - - prepare_pair_argument(bb, insn->rs1, scratch); - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = scratch + 1; - ir->src1 = 0; - ir->size_bytes = 4; - spill_var(bb, insn->rs1, scratch); - continue; - } - - if (starts[i] < MAX_ARGS_IN_REG) - continue; - - if (abi_arg_words(insn->rs1) == 2) { - int scratch = MAX_ARGS_IN_REG - 2; - - prepare_pair_argument(bb, insn->rs1, scratch); - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = scratch; - ir->src0_hi = scratch + 1; - ir->src1 = (starts[i] - MAX_ARGS_IN_REG) * PTR_SIZE; - ir->size_bytes = 8; - spill_var(bb, insn->rs1, scratch); - continue; - } - - /* A pinned variable has no slot to load from: its value only ever lives - * in its register, so reading the frame here handed the callee whatever - * the slot happened to hold. - */ - int held = pinned_reg_of(insn->rs1); - - if (held >= 0) { - ph2_ir_t *mv = bb_add_ph2_ir(bb, OP_assign); - mv->src0 = held; - mv->dest = MAX_ARGS_IN_REG - 1; - } else { - load_var(bb, insn->rs1, MAX_ARGS_IN_REG - 1); - } - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = MAX_ARGS_IN_REG - 1; - ir->src1 = (starts[i] - MAX_ARGS_IN_REG) * PTR_SIZE; - } - REGS[MAX_ARGS_IN_REG - 1].var = NULL; - return true; -} - -/* The half-open range of instruction indices in @bb over which @var holds a - * value, written into *lo and *hi. - * - * Returns false when @var is not live anywhere in @bb. - * - * A value live on entry starts before the first instruction; one defined here - * starts just after the instruction that writes it, so that a value read by - * that same instruction has already ended. It ends at its last read, or past - * the last instruction when it is live on exit. Where a phi-coalescing - * candidate is live, inverted once per function. - * - * phi_slot_conflicts() and vars_interfere() each ask, for one variable at a - * time, which blocks it is live in, and the only way to answer that from the - * liveness sets is to walk every block of the function and scan that block's - * live_in and live_kill for the variable. A self-compile asks 1.59 million - * times and 97% of the answers are "nowhere in this block", so nearly all of - * that scanning finds nothing. - * - * Turning the sets inside out once per function -- from "which variables does - * this block hold" into "which blocks hold this variable" -- costs one pass - * over the sets and reduces every later query to a walk of the handful of - * blocks the variable really occupies. - * - * Candidacy is marked on the variable itself, reusing the stamp pair - * compute_live_in() left behind: liveness analysis has finished by the time - * register allocation runs, so kill_gen and in_gen are dead, and bumping the - * same liveness_gen counter keeps this pass's stamps distinct from the values - * that analysis wrote. - */ -int phi_cand_head[MAX_LOCALS]; -int phi_cand_tail[MAX_LOCALS]; -int phi_cand_num; -int phi_cand_gen; - -/* The block records, chained per candidate through live_blk_next. The buffer is - * kept between functions and only ever grows, so the whole compile pays for the - * largest function once. - */ -basic_block_t **live_blk_bb; -int *live_blk_next; -int *live_blk_cand; -int *live_blk_lo; -int *live_blk_hi; -char *live_blk_flags; -int live_blk_cap; -int live_blk_num; - -/* Bits of live_blk_flags. */ -#define LIVE_REC_IN 1 -#define LIVE_REC_OUT 2 - -/* Where each candidate is written and last read inside the block being indexed, - * filled by one walk of that block's instructions and read back only for the - * candidates that block records. - */ -int phi_cand_def[MAX_LOCALS]; -int phi_cand_last[MAX_LOCALS]; - -/* False once a function outgrows the candidate table; every query then falls - * back to walking all blocks, which is what this pass did before the index. - */ -bool phi_live_ready; - -/* The candidate number of @var under the current function's stamp, or -1 when - * @var is not one. - */ -int phi_cand_index(const var_t *var) -{ - if (!var) - return -1; - if (var->kill_gen != phi_cand_gen) - return -1; - return var->in_gen; -} - -void phi_cand_add(var_t *var) -{ - if (!var) - return; - if (var->kill_gen == phi_cand_gen) - return; - if (phi_cand_num >= MAX_LOCALS) { - phi_live_ready = false; - return; - } - var->kill_gen = phi_cand_gen; - var->in_gen = phi_cand_num; - phi_cand_head[phi_cand_num] = -1; - phi_cand_tail[phi_cand_num] = -1; - phi_cand_num++; -} - -void live_blk_reserve(int want) -{ - if (want <= live_blk_cap) - return; - - int cap = live_blk_cap; - if (cap < 1024) - cap = 1024; - while (cap < want) - cap = cap << 1; - - live_blk_bb = arena_realloc(GENERAL_ARENA, (char *) live_blk_bb, - live_blk_cap * sizeof(basic_block_t *), - cap * sizeof(basic_block_t *)); - live_blk_next = - arena_realloc(GENERAL_ARENA, (char *) live_blk_next, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_cand = - arena_realloc(GENERAL_ARENA, (char *) live_blk_cand, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_lo = arena_realloc(GENERAL_ARENA, (char *) live_blk_lo, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_hi = arena_realloc(GENERAL_ARENA, (char *) live_blk_hi, - live_blk_cap * sizeof(int), cap * sizeof(int)); - live_blk_flags = - arena_realloc(GENERAL_ARENA, live_blk_flags, live_blk_cap, cap); - live_blk_cap = cap; -} - -/* Record that @var is live somewhere in @bb, returning that block's record for - * @var. Blocks arrive in order, so a variable already recorded for this block - * -- it is in both live_in and live_kill -- is caught by looking at the tail of - * its chain rather than by searching. - */ -int phi_live_note(basic_block_t *bb, var_t *var, int flags) -{ - int idx = phi_cand_index(var); - if (idx < 0) - return -1; - - int tail = phi_cand_tail[idx]; - if (tail >= 0 && live_blk_bb[tail] == bb) { - live_blk_flags[tail] |= flags; - return tail; - } - - live_blk_reserve(live_blk_num + 1); - int rec = live_blk_num++; - live_blk_bb[rec] = bb; - live_blk_next[rec] = -1; - live_blk_cand[rec] = idx; - live_blk_flags[rec] = flags; - if (tail < 0) - phi_cand_head[idx] = rec; - else - live_blk_next[tail] = rec; - phi_cand_tail[idx] = rec; - return rec; -} - -/* Note that @var is live on exit from @bb, when @bb already records it. */ -void phi_live_mark_out(const basic_block_t *bb, var_t *var) -{ - int idx = phi_cand_index(var); - if (idx < 0) - return; - - int tail = phi_cand_tail[idx]; - if (tail >= 0 && live_blk_bb[tail] == bb) - live_blk_flags[tail] |= LIVE_REC_OUT; -} - -/* Collect the variables this pass will ask about -- the destination and operand - * of every unwound phi -- and work out, for each block one of them is live in, - * the range of instruction indices it holds a value over. - * - * The ranges are what phi_slot_conflicts() and vars_interfere() compare, and - * computing them here means one walk of each block for all of its candidates - * instead of a walk per candidate per query. - */ -void phi_live_index_build(func_t *func) -{ - phi_cand_num = 0; - live_blk_num = 0; - phi_live_ready = true; - liveness_gen++; - phi_cand_gen = liveness_gen; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_unwound_phi) - continue; - phi_cand_add(insn->rd); - phi_cand_add(insn->rs1); - } - } - - if (!phi_live_ready) - return; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - int first = live_blk_num; - - for (int i = 0; i < bb->live_in.size; i++) - phi_live_note(bb, bb->live_in.elements[i], LIVE_REC_IN); - for (int i = 0; i < bb->live_kill.size; i++) - phi_live_note(bb, bb->live_kill.elements[i], 0); - - /* Live on exit adds no block: a value live out that this block does not - * write is live in as well, so its record already exists. Marking - * rather than recording keeps that invariant from turning a dataflow - * inconsistency into a range for a block the variable is not live in. - */ - for (int i = 0; i < bb->live_out.size; i++) - phi_live_mark_out(bb, bb->live_out.elements[i]); - - int last = live_blk_num; - if (first == last) - continue; - - for (int r = first; r < last; r++) { - int c = live_blk_cand[r]; - - phi_cand_def[c] = -1; - phi_cand_last[c] = -1; - } - - /* Anything a block's instructions name is in that block's live_in or - * live_kill, so every candidate touched below has a record above and - * its scratch has just been cleared. - */ - int n = 0; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - int c = phi_cand_index(insn->rs1); - if (c >= 0) - phi_cand_last[c] = n; - c = phi_cand_index(insn->rs2); - if (c >= 0) - phi_cand_last[c] = n; - - /* A select reads a third operand. Leaving it out of the walk ends - * the value's range before the instruction that reads it, and two - * variables live at once then look free to share a slot. - */ - c = phi_cand_index(insn->rs3); - if (c >= 0) - phi_cand_last[c] = n; - c = phi_cand_index(insn->rd); - if (c >= 0 && phi_cand_def[c] < 0) - phi_cand_def[c] = n; - n++; - } - - /* A value live on entry starts before the first instruction; one - * defined here starts just after the instruction that writes it, so a - * value read by that same instruction has already ended. It ends at its - * last read, or past the last instruction when it is live on exit. - */ - for (int r = first; r < last; r++) { - int c = live_blk_cand[r]; - int def = phi_cand_def[c], last_read = phi_cand_last[c]; - int lo, hi; - - if ((live_blk_flags[r] & LIVE_REC_IN) || def < 0) - lo = -1; - else - lo = def + 1; - - if (live_blk_flags[r] & LIVE_REC_OUT) - hi = n; - else if (last_read >= 0) - hi = last_read; - else - hi = lo; - - live_blk_lo[r] = lo; - live_blk_hi[r] = hi; - } - } -} - -/* A walk of the blocks a variable can be live in. Backed by the index when it - * has an answer for the variable, and by every block of the function when it - * does not, so the two produce the same results. - */ -typedef struct { - basic_block_t *bb; - int cur; /* the record just returned, -1 when walking every block */ - int rec; - basic_block_t *all; - bool indexed; -} live_iter_t; - -void live_iter_init(live_iter_t *it, func_t *func, var_t *var) -{ - int idx = -1; - - if (phi_live_ready) - idx = phi_cand_index(var); - - if (idx < 0) { - it->indexed = false; - it->rec = -1; - it->all = func->bbs; - } else { - it->indexed = true; - it->rec = phi_cand_head[idx]; - it->all = NULL; - } - it->bb = NULL; - it->cur = -1; -} - -bool live_iter_next(live_iter_t *it) -{ - if (it->indexed) { - int rec = it->rec; - if (rec < 0) { - it->bb = NULL; - it->cur = -1; - return false; - } - it->bb = live_blk_bb[rec]; - it->cur = rec; - it->rec = live_blk_next[rec]; - return true; - } - - basic_block_t *bb = it->all; - if (!bb) { - it->bb = NULL; - it->cur = -1; - return false; - } - it->bb = bb; - it->all = bb->rpo_next; - return true; -} - -/* Advance *@cursor to @bb's record for the candidate whose chain it walks, or - * past it when that candidate is not live there. The chains and every caller's - * outer walk run in the block order of func->bbs, which is increasing rpo, so a - * cursor only ever moves forward. - */ -bool live_rec_seek(int *cursor, const basic_block_t *bb) -{ - int rec = *cursor; - - while (rec >= 0 && live_blk_bb[rec]->rpo < bb->rpo) - rec = live_blk_next[rec]; - *cursor = rec; - return rec >= 0 && live_blk_bb[rec] == bb; -} - -bool var_range_in_bb(basic_block_t *bb, var_t *var, int *lo, int *hi) -{ - bool live_in = var_list_holds(&bb->live_in, var); - - /* A value the block neither receives nor writes cannot be live anywhere in - * it: any read would have made it live on entry, and anything live on exit - * without being written here is live on entry too. Settling that from the - * liveness sets keeps the walk below off the blocks -- the large majority - * -- where the variable never appears, which is what stops this pass from - * costing a quarter of the compiler. - */ - if (!live_in && !var_check_killed(var, bb)) - return false; - - bool live_out = var_list_holds(&bb->live_out, var); - int def = -1, last_read = -1, n = 0; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->rs1 == var || insn->rs2 == var || insn->rs3 == var) - last_read = n; - if (insn->rd == var && def < 0) - def = n; - n++; - } - - if (live_in || def < 0) - *lo = -1; - else - *lo = def + 1; - - if (live_out) - *hi = n; - else if (last_read >= 0) - *hi = last_read; - else - *hi = *lo; - return true; -} - -/* Whether @a and @b are ever live at the same point in @func, and so cannot - * share one stack slot. - */ -bool vars_interfere(func_t *func, var_t *a, var_t *b) -{ - live_iter_t it; - live_iter_init(&it, func, a); - - /* Both chains run in block order, so @b's is walked with a cursor that only - * moves forward rather than searched from the start for each of @a's - * blocks. - */ - int b_idx = phi_live_ready ? phi_cand_index(b) : -1; - int b_cur = b_idx >= 0 ? phi_cand_head[b_idx] : -1; - - while (live_iter_next(&it)) { - basic_block_t *bb = it.bb; - int alo, ahi, blo, bhi; - - if (it.cur >= 0) { - alo = live_blk_lo[it.cur]; - ahi = live_blk_hi[it.cur]; - } else if (!var_range_in_bb(bb, a, &alo, &ahi)) - continue; - - if (b_idx >= 0) { - if (!live_rec_seek(&b_cur, bb)) - continue; - blo = live_blk_lo[b_cur]; - bhi = live_blk_hi[b_cur]; - } else if (!var_range_in_bb(bb, b, &blo, &bhi)) - continue; - - if (alo <= bhi && blo <= ahi) - return true; - } - return false; -} - -/* Give a phi operand the same stack slot as the phi it feeds. - * - * Leaving SSA turns each phi into a store of the operand into the phi's slot, - * one per predecessor. The operand is a value of its own, so it is spilled to a - * slot of its own first, and the phi store then reads it back -- a load and a - * store on every path into the join, and on every iteration of a loop. When the - * two never hold values at the same time they can share one slot: the operand - * is written straight into the phi's slot, and the copy the phi would make is a - * load and store of the same address, which the peephole drops. The variables - * this pass has already placed, so a newcomer can be checked against everything - * sharing the slot it is about to join rather than against one member of it. - */ -var_t *phi_slot_vars[MAX_LOCALS]; -int phi_slot_count; - -void phi_slot_record(var_t *var) -{ - /* Dropping a member silently would leave later joiners checked against only - * part of the group sharing their slot, so overflow has to stop the compile - * rather than quietly produce an unsound placement. - */ - if (phi_slot_count >= MAX_LOCALS) - fatal("Too many coalesced phi slots"); - phi_slot_vars[phi_slot_count++] = var; -} - -/* Scratch lists of the variables sharing one slot, gathered before the block - * walks below so that neither has to filter the whole placement table on every - * step of its own loop. - */ -var_t *phi_group_a[MAX_LOCALS]; -var_t *phi_group_b[MAX_LOCALS]; - -/* One cursor per member of the group being checked, so each member's chain is - * walked once across the whole query instead of being searched per block. - */ -int phi_group_cursor[MAX_LOCALS]; - -/* Whether @v is live at the same time as anything already sharing @offset. */ -bool phi_slot_conflicts(func_t *func, int offset, var_t *v) -{ - int group = 0; - - for (int i = 0; i < phi_slot_count; i++) { - var_t *other = phi_slot_vars[i]; - - if (other != v && other->offset == offset) { - int idx = phi_live_ready ? phi_cand_index(other) : -1; - - phi_group_a[group] = other; - phi_group_cursor[group] = idx >= 0 ? phi_cand_head[idx] : -1; - - /* A member the index cannot answer for is marked with a cursor of - * -2 so the walk below falls back to the scanning form for it. - */ - if (idx < 0) - phi_group_cursor[group] = -2; - group++; - } - } - if (!group) - return false; - - /* Blocks are the outer loop so that @v's range in each is worked out once - * rather than once per variable compared against; @v occupies few blocks, - * and the rest are skipped without looking at the group at all. - */ - live_iter_t it; - live_iter_init(&it, func, v); - - while (live_iter_next(&it)) { - basic_block_t *bb = it.bb; - int vlo, vhi; - - if (it.cur >= 0) { - vlo = live_blk_lo[it.cur]; - vhi = live_blk_hi[it.cur]; - } else if (!var_range_in_bb(bb, v, &vlo, &vhi)) - continue; - - for (int i = 0; i < group; i++) { - int olo, ohi; - - if (phi_group_cursor[i] == -2) { - if (!var_range_in_bb(bb, phi_group_a[i], &olo, &ohi)) - continue; - } else { - if (!live_rec_seek(&phi_group_cursor[i], bb)) - continue; - olo = live_blk_lo[phi_group_cursor[i]]; - ohi = live_blk_hi[phi_group_cursor[i]]; - } - if (vlo <= ohi && olo <= vhi) - return true; - } - } - return false; -} - -bool phi_slot_compatible(var_t *d, var_t *s) -{ - if (!slot_is_private(d) || !slot_is_private(s)) - return false; - if (var_slot_size(d) != var_slot_size(s)) - return false; - if (is_pointer_like(d) != is_pointer_like(s)) - return false; - return d->ofs_based_on_stack_top == s->ofs_based_on_stack_top; -} - -/* Move everything sharing @from onto @to, once no member of one group is ever - * live at the same time as a member of the other. Chained phis reach here as - * two groups that were built independently -- "x = phi(...)" placed with its - * own operands before "y = phi(x, ...)" was seen -- and merging them is what - * puts a whole loop-carried variable in one slot. - */ -bool phi_slot_merge(func_t *func, int to, int from) -{ - if (to == from) - return true; - - /* One pass splits the table into the two groups; the pairwise test then - * walks only those, instead of rescanning every placement to find the - * partner for each candidate. - */ - int nto = 0, nfrom = 0; - for (int i = 0; i < phi_slot_count; i++) { - var_t *var = phi_slot_vars[i]; - - if (var->offset == to) - phi_group_a[nto++] = var; - else if (var->offset == from) - phi_group_b[nfrom++] = var; - } - if (!nfrom) - return true; - - for (int i = 0; i < nto; i++) { - for (int j = 0; j < nfrom; j++) { - if (vars_interfere(func, phi_group_a[i], phi_group_b[j])) - return false; - } - } - - for (int j = 0; j < nfrom; j++) - phi_group_b[j]->offset = to; - return true; -} - -void coalesce_phi_slots(func_t *func) -{ - phi_slot_count = 0; - phi_live_index_build(func); - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_unwound_phi) - continue; - - var_t *d = insn->rd, *s = insn->rs1; - if (!d || !s || d == s) - continue; - if (!phi_slot_compatible(d, s)) - continue; - - if (d->space_is_allocated && s->space_is_allocated) { - phi_slot_merge(func, d->offset, s->offset); - continue; - } - - /* Exactly one side has a slot by now, or neither does. Chained phis - * are met operand-first, so it may be the operand that is already - * placed; whichever side has the slot is the one joined, and when - * neither does the phi takes a fresh slot for the operand to join. - */ - var_t *have, *want; - if (s->space_is_allocated) { - have = s; - want = d; - } else { - if (!d->space_is_allocated) { - alloc_var_slot(func, d); - phi_slot_record(d); - } - have = d; - want = s; - } - - if (phi_slot_conflicts(func, have->offset, want)) - continue; - want->offset = have->offset; - want->space_is_allocated = true; - phi_slot_record(want); - } - } -} - -/* Fill a lowered binary operation from its SSA instruction. Function bodies and - * global setup share it, so a wide initializer such as the element of "long - * long a[] = {1LL << 40}" names the high registers of its operands. - */ -void fill_binary_ph2_ir(ph2_ir_t *ir, - insn_t *insn, - int src0, - int src1, - int dest) -{ - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src1 = src1; - ir->src1_hi = vreg_get_phys_hi(insn->rs2); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - - /* Record whether the result is an address, and which operand it came from. - * On LP64 an int-typed result has to wrap at 32 bits, while a pointer must - * keep all 64, and pointer arithmetic has to widen the int index beside the - * address. - */ - set_ptr_flags(ir, insn); - ir->size_bytes = insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* SSA temporaries normally retain their result type, but width is a - * property of the operation as well: a wide operand must not be narrowed - * merely because an intermediate lost its annotation. This includes - * comparisons: their result is int, while CMP must inspect the common - * operand width rather than stale high halves of 32-bit register values. - */ - int left_size = insn->rs1->ptr_level ? PTR_SIZE : insn->rs1->type->size; - int right_size = insn->rs2->ptr_level ? PTR_SIZE : insn->rs2->type->size; - if (left_size > ir->size_bytes) - ir->size_bytes = left_size; - - /* A shift is the exception: its type is that of the promoted left operand - * alone, so "(x ^ y) >> (n & 31)" with a long long n is an int shift, and a - * 64-bit shift of that int register brought zeros in as its sign. - */ - if (right_size > ir->size_bytes && insn->opcode != OP_lshift && - insn->opcode != OP_rshift) - ir->size_bytes = right_size; -} - -/* The unary counterpart of fill_binary_ph2_ir(). */ -void fill_unary_ph2_ir(ph2_ir_t *ir, insn_t *insn, int src0, int dest) -{ - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - - /* As for OP_branch: the width of the test follows the operand, not the - * result. - */ - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->size_bytes = insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* !x yields an int, but it tests all of a long long operand. */ - if (insn->opcode == OP_log_not && !insn->rs1->ptr_level && - insn->rs1->type->size > ir->size_bytes) - ir->size_bytes = insn->rs1->type->size; -} - -/* Narrow a conversion result in @dest again by its own type. - * - * A byte or short sits in its register extended by its own signedness, which a - * later widening or comparison relies on. OP_cast is a move and OP_sign_ext - * extends by the source alone, so a conversion that changes signedness broke - * that: a signed char -17 became unsigned short 0xffffffef, and cast to - * unsigned char it kept every high bit. A signed destination only needs this - * when it is no wider than its unsigned source; any other value already fits. - * The 64-bit x86 backend redoes the extension itself, the others do not. - */ -void narrow_conversion_result(basic_block_t *bb, insn_t *insn, int dest) -{ - var_t *rd = insn->rd, *rs = insn->rs1; - - if (!rd || !rs || rd->ptr_level || rd->is_func || !rd->type || !rs->type || - is_record_type(rd->type) || is_record_type(rs->type)) - return; - - int size = rd->type->size; - bool rd_unsigned = is_unsigned_scalar(rd); - - if ((size != 1 && size != 2) || rd_unsigned == is_unsigned_scalar(rs)) - return; - if (!rd_unsigned && (rs->ptr_level || rs->type->size < size)) - return; - - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_trunc); - ir->src0 = dest; - ir->src1 = size; - ir->dest = dest; - ir->is_unsigned = rd_unsigned; - ir->size_bytes = size; -} - -/* Place one global initializer, which has no basic block of its own. */ -void reg_alloc_global(insn_t *global_insn) -{ - ph2_ir_t *ir; - int dest, src0; - - /* Global initializers carry no liveness information, so no operand register - * may be reused as a destination here. - */ - - switch (global_insn->opcode) { - case OP_allocat: - if (global_insn->rd->array_size) { - /* Original scheme: pointer slot + backing region. Cache the base - * offset of the backing region into init_val so later global - * initializers can address elements without loading the pointer. - */ - global_insn->rd->offset = GLOBAL_FUNC->stack_size; - global_insn->rd->space_is_allocated = true; - GLOBAL_FUNC->stack_size += PTR_SIZE; - src0 = GLOBAL_FUNC->stack_size; /* base of backing region */ - - /* Stash base offset for this array variable */ - global_insn->rd->init_val = src0; - - if (global_insn->rd->ptr_level || global_insn->rd->is_func) - GLOBAL_FUNC->stack_size += - align_size(PTR_SIZE * global_insn->rd->array_size); - else { - GLOBAL_FUNC->stack_size += align_size( - global_insn->rd->array_size * global_insn->rd->type->size); - } - - dest = - prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = src0; - ir->dest = dest; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - spill_var(GLOBAL_FUNC->bbs, global_insn->rd, dest); - } else { - global_insn->rd->offset = GLOBAL_FUNC->stack_size; - global_insn->rd->space_is_allocated = true; - if (global_insn->rd->ptr_level || global_insn->rd->is_func) - GLOBAL_FUNC->stack_size += PTR_SIZE; - else if (global_insn->rd->type != TY_int && - global_insn->rd->type != TY_short && - global_insn->rd->type != TY_char && - global_insn->rd->type != TY_bool) { - GLOBAL_FUNC->stack_size += - align_size(global_insn->rd->type->size); - } else - /* 'char' is aligned to one byte for the convenience */ - GLOBAL_FUNC->stack_size += PTR_SIZE; - } - break; - case OP_load_constant: - case OP_load_data_address: - case OP_load_rodata_address: - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - ir->src0 = global_insn->rd->init_val; - ir->src1 = global_insn->rd->init_val_hi; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(global_insn->rd); - ir->is_unsigned = is_unsigned_scalar(global_insn->rd); - ir->size_bytes = - global_insn->rd->ptr_level ? PTR_SIZE : global_insn->rd->type->size; - break; - case OP_assign: - if (global_insn->rs1 && global_insn->rs1->is_global_address) { - src0 = - prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rs1, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = global_insn->rs1->offset; - ir->dest = src0; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - } else - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, -1); - - /* An initializer such as "unsigned long long g = c ? 7U : 1U" has an - * int-sized value; a move would leave the high register unwritten, so - * extend it by its own signedness instead. - */ - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, - var_needs_register_pair(global_insn->rd) && - !var_needs_register_pair(global_insn->rs1) - ? OP_cast - : OP_assign); - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(global_insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(global_insn->rs1); - ir->src0_is_pointer = is_address_like(global_insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(global_insn->rd); - spill_var(GLOBAL_FUNC->bbs, global_insn->rd, dest); - /* release the unused constant number in register manually */ - REGS[src0].polluted = 0; - vreg_clear_phys(REGS[src0].var); - REGS[src0].var = NULL; - break; - case OP_address_of: - case OP_global_address_of: - /* A global function-pointer initializer first forms the address of its - * global storage slot. Its address is GP-relative, unlike an address - * formed in a function body. - */ - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - - /* Global arrays have a pointer slot followed by their backing region; - * an address constant for the array denotes that backing region. - */ - ir->src0 = global_insn->rs1->array_size ? global_insn->rs1->init_val - : global_insn->rs1->offset; - ir->dest = dest; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - if (global_insn->rd->is_global_address) { - global_insn->rd->offset = ir->src0; - global_insn->rd->space_is_allocated = true; - - /* This temporary carries a compile-time GP-relative offset, not a - * stack-resident value. Leaving it mapped in REGS[] lets the next - * global setup instruction spill its address to `sp + offset`; on a - * global array that offset is the backing-region address, so the - * spill overwrites an initialized element. The folded - * OP_add/OP_assign paths rematerialize this address from offset, so - * it has no runtime register lifetime. - */ - REGS[dest].polluted = 0; - vreg_clear_phys(global_insn->rd); - REGS[dest].var = NULL; - } - break; - case OP_add: { - /* Special-case address computation for globals: if rs1 is a global base - * and rs2 is a constant, propagate absolute offset to rd so OP_write - * can fold into OP_global_store. - */ - if (global_insn->rs1 && global_insn->rs1->is_global_address && - global_insn->rs2) { - global_insn->rd->offset = - global_insn->rs1->offset + global_insn->rs2->init_val; - global_insn->rd->is_global_address = true; - global_insn->rd->space_is_allocated = true; - break; - } - if (global_insn->rs1 && global_insn->rs1->is_global && - global_insn->rs2) { - int base_off = global_insn->rs1->offset; - - /* For global arrays, use backing-region base cached in init_val */ - if (global_insn->rs1->array_size > 0) - base_off = global_insn->rs1->init_val; - global_insn->rd->offset = base_off + global_insn->rs2->init_val; - global_insn->rd->space_is_allocated = true; - global_insn->rd->is_global = true; - break; - } - /* Fall through to the ordinary scalar binary lowering below. */ - goto lower_global_binary; - } - case OP_negate: - case OP_bit_not: - case OP_log_not: { - /* Unary wide global constant expressions use the normal phase-2 - * instruction too. Keeping these separate from binary lowering avoids - * preparing a nonexistent right operand. - */ - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - fill_unary_ph2_ir(ir, global_insn, src0, dest); - break; - } - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - lower_global_binary: { - /* Global scalar initializers may have been parsed as a binary constant - * expression. Use the same phase-2 operation as a function body so wide - * division, comparison, and bitwise expressions do not stop at global - * setup merely because they are not pointer-address arithmetic. - */ - int src1; - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - src1 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs2, src0); - dest = - prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, src1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - fill_binary_ph2_ir(ir, global_insn, src0, src1, dest); - break; - } - case OP_write: { - if (global_insn->rs2 && global_insn->rs2->is_func) { - /* A direct aggregate field address is itself a global object, not a - * pointer value stored in that object. Loading it here turns an - * all-zero callback slot into the destination address. - */ - if (global_insn->rs1 && global_insn->rs1->is_global) { - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rs1, - -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = global_insn->rs1->array_size - ? global_insn->rs1->init_val - : global_insn->rs1->offset; - ir->dest = dest; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - src0 = dest; - } else { - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - } - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_address_of_func); - ir->src0 = src0; - ir->func_name = intern_string(global_insn->rs2->var_name); - - /* The GP-relative address exists only to receive this one - * relocation. In particular, an address temporary for a later array - * element must not be spilled: its slot allocation can be the very - * element being initialized, which would overwrite the function - * address with the address itself. - */ - REGS[src0].polluted = 0; - vreg_clear_phys(REGS[src0].var); - REGS[src0].var = NULL; - if (dynlink) { - func_t *target_fn = find_func(ir->func_name); - if (target_fn) - target_fn->is_used = true; - } - break; - } - /* Fold (addr, val) where addr carries GP-relative offset */ - if (global_insn->rs1 && (global_insn->rs1->is_global)) { - int vreg; - - if (global_insn->rs2 && global_insn->rs2->is_global_address) { - vreg = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rs2, - -1, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_address_of); - ir->src0 = global_insn->rs2->offset; - ir->dest = vreg; - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - } else - vreg = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs2, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_global_store); - ir->src0 = vreg; - ir->src0_hi = vreg_get_phys_hi(global_insn->rs2); - - /* For array variables used as base, store to the backing region's - * base offset (cached in init_val). - */ - int base_off = global_insn->rs1->offset; - if (global_insn->rs1->array_size > 0) - base_off = global_insn->rs1->init_val; - ir->src1 = base_off; - - /* OP_global_store selects its instruction width from this field. - * Leaving it at zero falls through to an eight-byte store, which - * makes a global int field initializer overwrite its successor. - */ - ir->size_bytes = global_insn->sz; - ir->is_pointer = global_insn->rs2->ptr_level > 0; - ir->is_unsigned = is_unsigned_scalar(global_insn->rs2); - - /* A materialized global address is consumed by this one store. Its - * descriptor offset is GP-relative data, not a spill slot; - * retaining the vreg mapping lets the next initializer spill the - * address at sp + offset and overwrite the source element (for - * example, a pointer row initialized from &values[0][1]). - */ - if (global_insn->rs2 && global_insn->rs2->is_global_address) { - REGS[vreg].polluted = 0; - vreg_clear_phys(global_insn->rs2); - REGS[vreg].var = NULL; - } - break; - } - /* Fallback generic write */ - int src1; - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - src1 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs2, src0); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, OP_write); - ir->src0 = src0; - ir->src1 = src1; - ir->src1_hi = vreg_get_phys_hi(global_insn->rs2); - ir->dest = global_insn->sz; - set_ptr_flags(ir, global_insn); - break; - } - case OP_trunc: - case OP_sign_ext: - case OP_cast: - /* A narrowing initializer such as "char g[] = {65, 66}" reaches the - * global block as a conversion, so it has to be lowered here exactly as - * it is inside a function. - */ - src0 = prepare_operand(GLOBAL_FUNC->bbs, global_insn->rs1, -1); - dest = prepare_dest(GLOBAL_FUNC->bbs, NULL, global_insn->rd, src0, -1); - ir = bb_add_ph2_ir(GLOBAL_FUNC->bbs, global_insn->opcode); - ir->src0 = src0; - ir->src1 = global_insn->sz; - ir->dest = dest; - - /* Widening "long long g[] = {1}" extends into a register pair, and the - * backends select that form and the extension's sign from these fields - * exactly as they do inside a function. - */ - ir->src0_hi = vreg_get_phys_hi(global_insn->rs1); - ir->dest_hi = vreg_get_phys_hi(global_insn->rd); - ir->is_unsigned = is_unsigned_scalar(global_insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(global_insn->rs1); - if (global_insn->opcode != OP_trunc) - narrow_conversion_result(GLOBAL_FUNC->bbs, global_insn, dest); - break; - default: - printf("Unsupported global operation: %d\n", global_insn->opcode); - fflush(stdout); /* see fatal() */ - abort(); - } -} - -/* Write the value @insn just gave an address-taken scalar to its slot. - * - * A pointer to the variable reads the slot, never the register, and nothing the - * allocator tracks says when that happens: the variable need not be named again - * for a read through the pointer, or a callee handed it, to want this value. A - * global is stored straight after its assignment for the same reason. Records - * and arrays are left alone, since their name holds an address and what a - * pointer reaches is the storage behind it. - */ -void store_addressed_def(basic_block_t *bb, const insn_t *insn) -{ - var_t *var = insn->rd; - - if (!var || !var->address_taken || var->is_global) - return; - if (insn->opcode == OP_allocat) - return; - if (var->array_size || var->has_backing_storage) - return; - if (!var->ptr_level && (!var->type || is_record_type(var->type))) - return; - - int reg = find_in_regs(var); - if (reg < 0 || !REGS[reg].polluted) - return; - store_var(bb, var, reg); -} - -/* Assign registers across one basic block, and emit the phase-2 IR that carries - * the assignment. - */ -void reg_alloc_bb(func_t *func, basic_block_t *bb) -{ - bool handle_abi = false, args_on_stack = false; - bool riscv_indirect_target_staged = false; - - is_pushing_args = false; - abi_args_staged = 0; - int args = 0; - - bb->visited++; - - /* The entry block starts with the incoming arguments already in their - * registers; every other block takes what its predecessor handed over, or - * nothing. - */ - if (bb != func->bbs) - load_entry_regs(bb); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - func_t *callee_func; - ph2_ir_t *ir; - int dest, src0, src1; - int sz, clear_reg; - - refresh(bb, insn); - - switch (insn->opcode) { - case OP_unwound_phi: - track_var_use(insn->rs1, insn->idx); - - /* A pinned destination lives in the same register on every path, so - * the copy this phi stands for is a register move rather than a - * write into a slot nothing reads back. - */ - int to = pinned_reg_of(insn->rd); - if (to >= 0) { - src0 = prepare_operand(bb, insn->rs1, -1); - if (src0 != to) { - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = src0; - ir->dest = to; - ir->is_pointer = is_pointer_like(insn->rd); - ir->size_bytes = var_slot_size(insn->rd); - } - REGS[to].var = insn->rd; - REGS[to].polluted = 1; - break; - } - - if (!insn->rd->space_is_allocated) - alloc_var_slot(bb->belong_to, insn->rd); - - /* Sharing a slot with the phi turns the copy into a write of the - * operand into the place it already lives. Only a register the - * block has changed still needs storing -- and reading the slot - * back first, as the general path would, is a load whose value goes - * straight home again. - */ - if (insn->rs1->space_is_allocated && - insn->rs1->offset == insn->rd->offset && - insn->rs1->ofs_based_on_stack_top == - insn->rd->ofs_based_on_stack_top) { - int held = find_in_regs(insn->rs1); - - if (held < 0 || !REGS[held].polluted) - break; /* the slot already holds the value */ - store_var(bb, insn->rs1, held); - break; - } - - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = src0; - - /* The phi's slot is eight bytes for a pair; a store of the low - * register alone left the high word of the previous iteration. - */ - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src1 = insn->rd->offset; - ir->ofs_based_on_stack_top = insn->rd->ofs_based_on_stack_top; - ir->is_pointer = is_pointer_like(insn->rd); - ir->size_bytes = var_slot_size(insn->rd); - break; - case OP_allocat: - if ((insn->rd->type == TY_void || insn->rd->type == TY_int || - insn->rd->type == TY_short || insn->rd->type == TY_char || - insn->rd->type == TY_bool) && - insn->rd->array_size == 0) - break; - - insn->rd->offset = func->stack_size; - insn->rd->space_is_allocated = true; - func->stack_size += PTR_SIZE; - src0 = func->stack_size; - - if (insn->rd->ptr_level || insn->rd->is_func) - sz = PTR_SIZE; - else { - sz = insn->rd->type->size; - } - - if (insn->rd->array_size) - func->stack_size += align_size(insn->rd->array_size * sz); - else - func->stack_size += align_size(sz); - - if (!insn->rd->is_global && - aggregate_has_function_pointer(insn->rd->type)) { - insn->rd->has_backing_storage = true; - } - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = src0; - ir->dest = dest; - ir->ofs_based_on_stack_top = insn->rd->ofs_based_on_stack_top; - - /* For arrays, store the base address just like global arrays do */ - if (insn->rd->array_size) - spill_var(bb, insn->rd, dest); - else if (insn->rd->is_func || - (!insn->rd->ptr_level && !insn->rd->type->ptr_level && - is_record_type(insn->rd->type) && - !insn->rd->has_backing_storage)) { - /* OP_allocat's result is the backing address for a callback - * object, not the pointer value stored in that object. Keeping - * it mapped as the object lets a later conservative spill - * overwrite an initialized callback with its own address. A - * record is reached through &record, which names the slot - * itself, so the same spill would overwrite its first member. - */ - REGS[dest].var = NULL; - REGS[dest].polluted = 0; - vreg_clear_phys(insn->rd); - } - break; - case OP_load_constant: - case OP_load_data_address: - case OP_load_rodata_address: - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, insn->opcode); - ir->src0 = insn->rd->init_val; - ir->src1 = insn->rd->init_val_hi; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* store global variable immediately after assignment */ - if (insn->rd->is_global) { - ir = bb_add_ph2_ir(bb, OP_global_store); - ir->src0 = dest; - ir->src0_hi = vreg_get_phys_hi(insn->rd); - ir->src1 = insn->rd->offset; - ir->is_volatile = var_is_volatile_object(insn->rd); - REGS[dest].polluted = 0; - } - - break; - case OP_address_of: - case OP_global_address_of: - /* Mark variable as address-taken, disable constant optimization */ - insn->rs1->address_taken = true; - insn->rs1->is_const = false; - - /* Source-level record parameters arrive as hidden pointers to the - * caller's by-value copy. Their declaration must stay a record for - * parsing and type checks, but taking its address produces the ABI - * pointer rather than allocating a scalar spill slot. - */ - if (insn->rs1->is_aggregate_param) { - int param_idx = -1; - for (int i = 0; i < func->num_params; i++) - if (insn->rs1->base == &func->param_defs[i] || - insn->rs1 == &func->param_defs[i]) { - param_idx = i; - break; - } - if (param_idx < 0) - fatal("Aggregate parameter is not owned by its function"); - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - - /* A variadic callee reserves the final named aggregate's - * complete ABI footprint in its contiguous argument-save area. - * `va_start` obtains that slot's address, then advances by the - * same rounded extent to the first unnamed argument. - */ - if (func->va_args && param_idx + 1 == func->num_params && - insn->rs1->space_is_allocated) { - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = insn->rs1->offset; - ir->dest = dest; - ir->ofs_based_on_stack_top = - insn->rs1->ofs_based_on_stack_top; - } else if (insn->rs1->space_is_allocated) { - ir = bb_add_ph2_ir(bb, OP_load); - ir->src0 = insn->rs1->offset; - ir->dest = dest; - ir->ofs_based_on_stack_top = - insn->rs1->ofs_based_on_stack_top; - } else if (abi_param_start(func, param_idx) < MAX_ARGS_IN_REG) { - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = abi_param_start(func, param_idx); - ir->dest = dest; - } else { - ir = bb_add_ph2_ir(bb, OP_load); - ir->src0 = - (abi_param_start(func, param_idx) - MAX_ARGS_IN_REG) * - PTR_SIZE; - ir->dest = dest; - ir->ofs_based_on_stack_top = true; - } - ir->is_pointer = true; - ir->size_bytes = PTR_SIZE; - break; - } - - /* OP_allocat puts a local aggregate's spill slot before its backing - * storage. &aggregate must name the backing storage, not the spill - * slot. - * - * FIXME: This does not support aggregate parameter for now. - */ - bool is_pointer = insn->rs1->ptr_level || - (insn->rs1->type && insn->rs1->type->ptr_level); - if (!insn->rs1->is_global && !is_pointer && - aggregate_has_function_pointer(insn->rs1->type)) { - if (!insn->rs1->has_backing_storage) { - insn->rs1->offset = func->stack_size; - insn->rs1->space_is_allocated = true; - insn->rs1->ofs_based_on_stack_top = false; - func->stack_size += PTR_SIZE; - if (insn->rs1->ptr_level) - sz = PTR_SIZE; - else - sz = insn->rs1->type->size; - if (insn->rs1->array_size) - func->stack_size += - align_size(insn->rs1->array_size * sz); - else - func->stack_size += align_size(sz); - insn->rs1->has_backing_storage = true; - } - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = insn->rs1->offset + PTR_SIZE; - ir->dest = dest; - ir->ofs_based_on_stack_top = insn->rs1->ofs_based_on_stack_top; - break; - } - - /* make sure variable is on stack */ - if (!insn->rs1->space_is_allocated) { - alloc_var_slot(bb->belong_to, insn->rs1); - - for (int i = 0; i < REG_CNT; i++) - if (REGS[i].var == insn->rs1 && !pinned_base[i]) { - ir = bb_add_ph2_ir(bb, OP_store); - ir->src0 = i; - - /* A pair spills both words: the slot is about to be - * read through the address, not from this register. - */ - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src1 = insn->rs1->offset; - ir->ofs_based_on_stack_top = - insn->rs1->ofs_based_on_stack_top; - /* Clear stale register tracking */ - REGS[i].var = NULL; - vreg_clear_phys(insn->rs1); - } - } - - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - if (insn->rs1->is_global || insn->opcode == OP_global_address_of) - ir = bb_add_ph2_ir(bb, OP_global_address_of); - else - ir = bb_add_ph2_ir(bb, OP_address_of); - ir->src0 = insn->rs1->offset; - ir->dest = dest; - ir->ofs_based_on_stack_top = insn->rs1->ofs_based_on_stack_top; - break; - case OP_cmov: { - /* A select reads three registers, one more than the allocator's - * avoid arguments can protect, so each is locked as it is placed. - * With all three safe the destination may land anywhere. - */ - int cond, taken, other; - - track_var_use(insn->rs1, insn->idx); - track_var_use(insn->rs2, insn->idx); - track_var_use(insn->rs3, insn->idx); - - reg_locked = 0; - taken = prepare_operand(bb, insn->rs1, -1); - reg_locked = reg_locked | (1 << taken); - other = prepare_operand(bb, insn->rs3, taken); - reg_locked = reg_locked | (1 << other); - cond = prepare_operand(bb, insn->rs2, taken); - reg_locked = reg_locked | (1 << cond); - dest = prepare_dest(bb, insn, insn->rd, taken, other); - - if (dest < 0) { - /* A select needs a fourth register only while all three inputs - * remain live. Save one unpinned input first, then use its - * physical register as the result. The CMOV emitter - * deliberately supports the destination aliasing either arm; it - * tests the condition before overwriting anything, so the - * condition is safe too if it is the only choice. - * - * spill_var() leaves the machine register unchanged, which is - * exactly what the select still needs. It only removes the - * allocator's association, making the value available for the - * result and forcing a later use of the saved input to reload - * its slot. - */ - int reuse = -1; - const int sources[] = {taken, other, cond}; - - for (int i = 0; i < 3; i++) { - int reg = sources[i]; - if (!pinned_base[reg]) { - reuse = reg; - break; - } - } - - /* pin_registers() reserves at most half the file, so one of a - * select's inputs is always reclaimable. - */ - if (reuse < 0) - abort(); - spill_var(bb, REGS[reuse].var, reuse); - dest = prepare_dest(bb, insn, insn->rd, taken, other); - if (dest != reuse) - abort(); - } - reg_locked = 0; - ir = bb_add_ph2_ir(bb, OP_cmov); - ir->src0 = cond; - ir->src1 = taken; - ir->src2 = other; - ir->dest = dest; - - /* As for OP_branch, the recorded width is the condition's: a select - * tests that value, and the arms are moved whole. An int condition - * is decided by its low word alone, whatever the width of the - * result. - */ - ir->size_bytes = - insn->rs2->ptr_level ? PTR_SIZE : insn->rs2->type->size; - ir->src0_is_pointer = is_address_like(insn->rs2); - ir->is_pointer = is_pointer_like(insn->rd); - break; - } - case OP_assign: - /* A copy nothing reads is dropped, unless it copies from or to a - * volatile object. Copying from one is the read a discarded - * expression such as "status;" asks for, and a write to one is a - * side effect however little the program reads it back. - */ - if (insn->rd->consumed == -1 && - !var_is_volatile_object(insn->rs1) && - !var_is_volatile_object(insn->rd)) - break; - - track_var_use(insn->rs1, insn->idx); - src0 = find_in_regs(insn->rs1); - - /* A volatile object whose register matches its storage is read - * again, as prepare_operand() does. - */ - if (src0 > -1 && var_is_volatile_object(insn->rs1) && - !REGS[src0].polluted) - src0 = -1; - - /* If operand is loaded from stack, clear the original slot after - * moving. - */ - if (src0 > -1) - clear_reg = 0; - else { - clear_reg = 1; - src0 = prepare_operand(bb, insn->rs1, -1); - } - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - - /* store global variable immediately after assignment */ - if (insn->rd->is_global) { - ir = bb_add_ph2_ir(bb, OP_global_store); - ir->src0 = dest; - ir->src0_hi = vreg_get_phys_hi(insn->rd); - ir->src1 = insn->rd->offset; - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->size_bytes = var_slot_size(insn->rd); - ir->is_volatile = var_is_volatile_object(insn->rd); - REGS[dest].polluted = 0; - } - - if (clear_reg) { - vreg_clear_phys(REGS[src0].var); - REGS[src0].var = NULL; - } - - /* An assignment creates storage-backed state. It must not retain a - * literal's compile-time cache for later reads. - */ - insn->rd->is_const = false; - - break; - case OP_read: - src0 = prepare_operand(bb, insn->rs1, -1); - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, OP_read); - ir->src0 = src0; - ir->src1 = insn->sz; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_volatile = insn_reads_volatile(insn); - set_ptr_flags(ir, insn); - break; - case OP_write: - if (insn->rs2->is_func) { - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_address_of_func); - ir->src0 = src0; - ir->func_name = intern_string(insn->rs2->var_name); - if (dynlink) { - func_t *target_fn = find_func(ir->func_name); - if (target_fn) - target_fn->is_used = true; - } - } else { - /* FIXME: Register content becomes stale after store operation. - * Current workaround causes redundant spilling - need better - * register invalidation strategy. - */ - spill_alive(bb, insn); - src0 = prepare_operand(bb, insn->rs1, -1); - src1 = prepare_operand(bb, insn->rs2, src0); - ir = bb_add_ph2_ir(bb, OP_write); - ir->src0 = src0; - ir->src1 = src1; - ir->src1_hi = vreg_get_phys_hi(insn->rs2); - ir->dest = insn->sz; - set_ptr_flags(ir, insn); - } - break; - case OP_branch: - src0 = prepare_operand(bb, insn->rs1, -1); - - /* Read the high register now: spill_live_out_keep() below forgets - * the mapping of a condition that does not outlive the block. - */ - src1 = vreg_get_phys_hi(insn->rs1); - - /* REGS[src0].var had been set to NULL, but the actual content is - * still holded in the register. - * - * Write every live-out value back but keep it in its register: the - * arm reached by the jump starts with an empty file and loads from - * the slots just written, while the arm that falls through can - * inherit the registers through bb_export_regs(). - */ - spill_live_out_keep(bb); - - ir = bb_add_ph2_ir(bb, OP_branch); - ir->src0 = src0; - ir->src0_hi = src1; - - /* An LP64 backend tests an address over its full width and an int - * over its low word only. - */ - ir->src0_is_pointer = is_address_like(insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - - /* A long long is tested whole too, which AArch64 selects by the - * width recorded here. - */ - ir->size_bytes = - insn->rs1->ptr_level ? PTR_SIZE : insn->rs1->type->size; - ir->then_bb = bb->then_; - ir->else_bb = bb->else_; - break; - case OP_push: - extend_liveness(bb, insn, insn->rs1, insn->sz); - - if (!is_pushing_args) { - /* RV32 has four ABI argument registers. A call with two aligned - * 64-bit arguments fills all four, leaving no virtual register - * in which OP_indirect may materialize its target. Stage it - * before assigning that file. The RV32 backend keeps this value - * in s2, an otherwise-unused callee-saved register. - */ - if (ELF_MACHINE == 0xf3) { - insn_t *call = insn; - - while (call && call->opcode == OP_push) - call = call->next; - if (call && call->opcode == OP_indirect) { - src0 = prepare_operand(bb, call->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_load_func); - ir->src0 = src0; - riscv_indirect_target_staged = true; - } - } - spill_alive(bb, insn); - is_pushing_args = true; - } - if (!handle_abi) { - args_on_stack = abi_lower_call_args(bb, insn); - handle_abi = true; - } - - args = call_arg_starts[call_arg_next++]; - if (args_on_stack && args >= MAX_ARGS_IN_REG) { - args += abi_arg_words(insn->rs1); - break; - } - - if (abi_arg_words(insn->rs1) == 2) { - if (abi_arg_is_split(args, insn->rs1)) { - /* Nothing is placed after it in a register, so the last one - * takes the low word without being claimed. - */ - load_low_word(bb, insn->rs1, args); - abi_args_staged |= 1 << args; - args += 2; - break; - } - prepare_pair_argument(bb, insn->rs1, args); - abi_args_staged |= 3 << args; - args += 2; - break; - } - - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = src0; - ir->dest = args; - ir->is_unsigned = is_unsigned_scalar(insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - REGS[ir->dest].var = insn->rs1; - REGS[ir->dest].polluted = 0; - abi_args_staged |= 1 << args; - args++; - break; - case OP_call: - callee_func = find_func(insn->str); - if (!callee_func->num_params) - spill_alive(bb, insn); - - if (dynlink) - callee_func->is_used = true; - - ir = bb_add_ph2_ir(bb, OP_call); - /* add_insn() interned this when the call was created. */ - ir->func_name = insn->str; - - is_pushing_args = false; - abi_args_staged = 0; - args = 0; - handle_abi = false; - - clobber_caller_saved(); - - break; - case OP_indirect: - if (!args) - spill_alive(bb, insn); - - if (!riscv_indirect_target_staged) { - src0 = prepare_operand(bb, insn->rs1, -1); - ir = bb_add_ph2_ir(bb, OP_load_func); - ir->src0 = src0; - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - } - - bb_add_ph2_ir(bb, OP_indirect); - - is_pushing_args = false; - abi_args_staged = 0; - args = 0; - handle_abi = false; - riscv_indirect_target_staged = false; - - clobber_caller_saved(); - break; - case OP_func_ret: - dest = prepare_dest(bb, insn, insn->rd, -1, -1); - ir = bb_add_ph2_ir(bb, OP_assign); - ir->src0 = 0; - if (var_needs_register_pair(insn->rd)) - ir->src0_hi = 1; - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - break; - case OP_return: - if (insn->rs1) - src0 = prepare_operand(bb, insn->rs1, -1); - else - src0 = -1; - - ir = bb_add_ph2_ir(bb, OP_return); - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - if (insn->rs1) - ir->size_bytes = - insn->rs1->ptr_level ? PTR_SIZE : insn->rs1->type->size; - break; - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_lshift: - case OP_rshift: - case OP_eq: - case OP_neq: - case OP_gt: - case OP_geq: - case OP_lt: - case OP_leq: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - track_var_use(insn->rs1, insn->idx); - track_var_use(insn->rs2, insn->idx); - src0 = prepare_operand(bb, insn->rs1, -1); - src1 = prepare_operand(bb, insn->rs2, src0); - dest = prepare_dest(bb, insn, insn->rd, src0, src1); - ir = bb_add_ph2_ir(bb, insn->opcode); - fill_binary_ph2_ir(ir, insn, src0, src1, dest); - break; - case OP_negate: - case OP_bit_not: - case OP_log_not: - src0 = prepare_operand(bb, insn->rs1, -1); - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, insn->opcode); - fill_unary_ph2_ir(ir, insn, src0, dest); - break; - case OP_trunc: - case OP_sign_ext: - case OP_cast: - src0 = prepare_operand(bb, insn->rs1, -1); - dest = prepare_dest(bb, insn, insn->rd, src0, -1); - ir = bb_add_ph2_ir(bb, insn->opcode); - ir->src1 = insn->sz; - ir->src0 = src0; - ir->src0_hi = vreg_get_phys_hi(insn->rs1); - ir->dest = dest; - ir->dest_hi = vreg_get_phys_hi(insn->rd); - ir->is_unsigned = is_unsigned_scalar(insn->rd); - ir->src0_is_unsigned = is_unsigned_scalar(insn->rs1); - ir->size_bytes = - insn->rd->ptr_level ? PTR_SIZE : insn->rd->type->size; - if (insn->opcode != OP_trunc) - narrow_conversion_result(bb, insn, dest); - break; - default: - printf("Unknown opcode\n"); - fflush(stdout); /* see fatal() */ - abort(); - } - - store_addressed_def(bb, insn); - } - - if (bb->next) { - spill_live_out_keep(bb); - bb_export_regs(bb); - } else if (bb->then_ || bb->else_) { - /* A conditional branch has already written its live-out values back at - * OP_branch; only the handover is left. - */ - bb_export_regs(bb); - } - - if (bb == func->exit) - return; - - /* append jump instruction for the normal block only */ - if (!bb->next) - return; - - if (bb->next == func->exit) - return; - - /* jump to the beginning of loop or over the else block */ - if (bb->next->visited == func->visited || bb->next->rpo != bb->rpo + 1) { - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_jump); - ir->next_bb = bb->next; - } -} - -void reg_alloc(void) -{ - /* TODO: Add proper .bss and .data section support for uninitialized / - * initialized globals - */ - for (insn_t *global_insn = GLOBAL_FUNC->bbs->insn_list.head; global_insn; - global_insn = global_insn->next) { - reg_alloc_global(global_insn); - } - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - func->visited++; - fatal_function_context = func->return_def.var_name; - - if (!strcmp(func->return_def.var_name, "main")) - MAIN_BB = func->bbs; - - for (int i = 0; i < REG_CNT; i++) { - REGS[i].var = NULL; - pair_high_owner[i] = NULL; - } - - slot_var_count = 0; - reserve_outgoing_args(func); - coalesce_phi_slots(func); - pin_registers(func); - - /* Set arguments available. The register file is indexed by ABI word, - * whereas param_defs is indexed by source parameter. - */ - int args_in_reg = 0; - if (func->returns_aggregate) { - REGS[0].var = var_subscript0(&func->sret_def); - REGS[0].polluted = 1; - } - for (int i = 0; i < func->num_params; i++) { - int word = abi_param_start(func, i); - - if (word + abi_arg_words(&func->param_defs[i]) > MAX_ARGS_IN_REG) - break; - REGS[word].var = var_subscript0(&func->param_defs[i]); - REGS[word].polluted = 1; - if (abi_arg_words(&func->param_defs[i]) == 2) - vreg_map_pair_to_phys(REGS[word].var, word, word + 1); - args_in_reg++; - } - - /* Move a pinned parameter out of the argument register it arrived in - * before anything else claims that register. Pinned registers are - * handed out from the top of the file and pin_registers() refuses any - * that an argument occupies, so this never overwrites another one. - */ - for (int i = 0; i < args_in_reg; i++) { - var_t *param = var_subscript0(&func->param_defs[i]); - int home = pinned_reg_of(param); - int word = abi_param_start(func, i); - - if (home < 0) - continue; - - ph2_ir_t *mv = bb_add_ph2_ir(func->bbs, OP_assign); - mv->src0 = word; - mv->dest = home; - REGS[home].var = param; - REGS[home].polluted = 0; - REGS[word].var = NULL; - REGS[word].polluted = 0; - } - - /* variadic function implementation */ - if (func->va_args) { - /* When encountering a variadic function, allocate space for all - * arguments on the local stack to ensure their addresses are - * contiguous. - * - * On a 32-bit target a long long argument takes two words, after a - * padding word when it would start at an odd one, so a call can - * pass twice as many words as it has arguments: save that many. - * va_arg rounds its cursor to an even word by rounding the address - * to eight bytes, so the first word must sit at an address that is - * a multiple of eight too. - */ - int va_words = PTR_SIZE < 8 ? 2 * MAX_PARAMS : MAX_PARAMS; - - /* A stack word is fetched through the register after the argument - * registers. RISC-V passes as many words in registers as the file - * holds, so it borrows the last argument register instead, which - * has been saved by then. - */ - int scratch = - MAX_ARGS_IN_REG < REG_CNT ? MAX_ARGS_IN_REG : REG_CNT - 1; - - if (PTR_SIZE < 8) - func->stack_size = ALIGN_UP(func->stack_size, 8); - for (int i = 0; i < va_words; i++) { - ph2_ir_t *ir; - int src0 = i; - - if (i >= MAX_ARGS_IN_REG) { - /* Callee should access caller's stack to obtain the extra - * arguments. - */ - ir = bb_add_ph2_ir(func->bbs, OP_load); - ir->dest = scratch; - ir->src0 = (i - MAX_ARGS_IN_REG) * PTR_SIZE; - ir->ofs_based_on_stack_top = true; - src0 = scratch; - } - - /* Slot i saves ABI word i. The parameter homed there is found - * by its ABI position, since a hidden aggregate-return pointer - * takes word 0 and shifts every named parameter up. - */ - int param_idx = -1; - for (int q = 0; q < func->num_params; q++) { - if (abi_param_start(func, q) == i) { - param_idx = q; - break; - } - } - - if (param_idx >= 0) { - var_t *param = var_subscript0(&func->param_defs[param_idx]); - param->offset = func->stack_size; - param->space_is_allocated = true; - } - - ir = bb_add_ph2_ir(func->bbs, OP_store); - ir->src0 = src0; - ir->src1 = func->stack_size; - func->stack_size += PTR_SIZE; - if (param_idx >= 0) { - var_t *param = var_subscript0(&func->param_defs[param_idx]); - - if (param_idx + 1 == func->num_params && - param->is_aggregate_param) { - int footprint = ALIGN_UP(param->type->size, PTR_SIZE); - - /* The first word was just saved above. Reserve the - * remaining words before spilling unnamed arguments. - */ - func->stack_size += footprint - PTR_SIZE; - } - } - } - - /* A parameter that arrived in the borrowed register now lives only - * in its save slot. - */ - if (va_words > MAX_ARGS_IN_REG && scratch < MAX_ARGS_IN_REG) { - var_t *owner = pair_high_owner[scratch]; - - if (owner) { - int low = vreg_get_phys(owner); - - if (low >= 0 && low < REG_CNT && REGS[low].var == owner) { - REGS[low].var = NULL; - REGS[low].polluted = 0; - } - vreg_clear_phys(owner); - } else if (REGS[scratch].var) { - vreg_clear_phys(REGS[scratch].var); - REGS[scratch].var = NULL; - REGS[scratch].polluted = 0; - } - } - } else { - /* If the number of function arguments is fixed, the extra arguments - * are directly placed in the caller's stack space instead of the - * callee's. - * - * +----------> +---------------+ - * | | local vars | - * | +---------------+ - * | | extra arg 4 | - * | +---------------+ <-- sp + stack_size + 12 - * caller's space | extra arg 3 | - * | +---------------+ <-- sp + stack_size + 8 - * | | extra arg 2 | - * | +---------------+ <-- sp + stack_size + 4 - * | | extra arg 1 | - * +----------> +---------------+ <-- sp + stack_size - * | | local vars | - * | +---------------+ <-- sp + 16 - * callee's space | Next callee's | - * | | additional | - * | | arguments | - * +----------> +---------------+ <-- sp - * - * Note that: - * - For the Arm architecture, extra arg1 ~ argX correspond to - * arg5 ~ arg(X + 4). - * - For the RISC-V architecture, extra arg1 ~ argX correspond to - * arg9 ~ arg(X + 8). - * - * If any instruction use one of these additional arguments, it - * inherits 'offset' and 'ofs_based_on_stack_top'. When calling - * cfg_flatten(), the operand's offset will be recalculated by - * adding the function's stack size. - */ - for (int i = 0; i < func->num_params; i++) { - var_t *param = var_subscript0(&func->param_defs[i]); - int word = abi_param_start(func, i); - - if (abi_arg_is_split(word, param)) { - /* A long long split between a7 and the first stack slot - * gets a slot of its own: store the low word, then fetch - * the high word through a7, which holds nothing else. - */ - ph2_ir_t *ir; - - if (!param->space_is_allocated) - alloc_var_slot(func, param); - ir = bb_add_ph2_ir(func->bbs, OP_store); - ir->src0 = word; - ir->src1 = param->offset; - ir->size_bytes = 4; - ir = bb_add_ph2_ir(func->bbs, OP_load); - ir->dest = word; - ir->src0 = 0; - ir->ofs_based_on_stack_top = true; - ir->size_bytes = 4; - ir = bb_add_ph2_ir(func->bbs, OP_store); - ir->src0 = word; - ir->src1 = param->offset + 4; - ir->size_bytes = 4; - continue; - } - if (word < MAX_ARGS_IN_REG) - continue; - param->offset = (word - MAX_ARGS_IN_REG) * PTR_SIZE; - param->space_is_allocated = true; - param->ofs_based_on_stack_top = true; - } - } - - /* A volatile parameter lives in its slot, so write it there on entry. - * Its register then matches the slot, and every read reloads it. - */ - for (int i = 0; i < args_in_reg; i++) { - var_t *param = var_subscript0(&func->param_defs[i]); - int word = abi_param_start(func, i); - - if (var_is_volatile_object(param) && REGS[word].var == param) - store_var(func->bbs, param, word); - } - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - reg_alloc_bb(func, bb); - } - - /* handle implicit return */ - for (int i = 0; i < func->exit->prev_idx; i++) { - basic_block_t *bb = func->exit->prev[i].bb; - if (!bb) - continue; - - if (func->return_def.type != TY_void) - continue; - - if (bb->insn_list.tail) - if (bb->insn_list.tail->opcode == OP_return) - continue; - - ph2_ir_t *ir = bb_add_ph2_ir(bb, OP_return); - ir->src0 = -1; - } - - slot_scan(func); - collapse_slot_roundtrip(func); - dead_store_elim(func); - } -} - -/* Name the high register of a 32-bit target's register pair after its low one, - * as "%x2:%x3", when @reg is one. - */ -void dump_pair_high(int reg) -{ - if (reg >= 0) - printf(":%%x%c", reg + 48); -} - -void dump_ph2_ir(void) -{ - for (int i = 0; i < ph2_ir_idx; i++) { - ph2_ir_t *ph2_ir = PH2_IR_FLATTEN[i]; - - const int rd = ph2_ir->dest + 48; - const int rs1 = ph2_ir->src0 + 48; - const int rs2 = ph2_ir->src1 + 48; - - switch (ph2_ir->op) { - case OP_define: - printf("%s:", ph2_ir->func_name); - break; - case OP_allocat: - continue; - case OP_assign: - printf("\t%%x%c = %%x%c", rd, rs1); - break; - case OP_cmov: - printf("\t%%x%c = %%x%c if %%x%c else %%x%c", rd, rs2, rs1, - ph2_ir->src2 + 48); - break; - case OP_load_constant: - printf("\tli %%x%c, $%d", rd, ph2_ir->src0); - break; - case OP_load_data_address: - printf("\t%%x%c = .data(%d)", rd, ph2_ir->src0); - break; - case OP_load_rodata_address: - printf("\t%%x%c = .rodata(%d)", rd, ph2_ir->src0); - break; - case OP_address_of: - printf("\t%%x%c = %%sp + %d", rd, ph2_ir->src0); - break; - case OP_global_address_of: - printf("\t%%x%c = %%gp + %d", rd, ph2_ir->src0); - break; - case OP_branch: - printf("\tbr %%x%c", rs1); - break; - case OP_jump: - printf("\tj %s", ph2_ir->func_name ? ph2_ir->func_name : ""); - break; - case OP_call: - printf("\tcall @%s", ph2_ir->func_name); - break; - case OP_return: - if (ph2_ir->src0 == -1) - printf("\tret"); - else - printf("\tret %%x%c", rs1); - break; - case OP_load: - printf("\tload %%x%c", rd); - dump_pair_high(ph2_ir->dest_hi); - printf(", %d(sp)", ph2_ir->src0); - break; - case OP_store: - printf("\tstore %%x%c", rs1); - dump_pair_high(ph2_ir->src0_hi); - printf(", %d(sp)", ph2_ir->src1); - break; - case OP_global_load: - printf("\tload %%x%c", rd); - dump_pair_high(ph2_ir->dest_hi); - printf(", %d(gp)", ph2_ir->src0); - break; - case OP_global_store: - printf("\tstore %%x%c", rs1); - dump_pair_high(ph2_ir->src0_hi); - printf(", %d(gp)", ph2_ir->src1); - break; - case OP_read: - printf("\t%%x%c = (%%x%c)", rd, rs1); - break; - case OP_write: - printf("\t(%%x%c) = %%x%c", rs1, rs2); - break; - case OP_address_of_func: - printf("\t(%%x%c) = @%s", rs1, ph2_ir->func_name); - break; - case OP_load_func: - printf("\tload %%t0, %d(sp)", ph2_ir->src0); - break; - case OP_global_load_func: - printf("\tload %%t0, %d(gp)", ph2_ir->src0); - break; - case OP_indirect: - printf("\tindirect call @(%%t0)"); - break; - case OP_negate: - printf("\tneg %%x%c, %%x%c", rd, rs1); - break; - case OP_add: - printf("\t%%x%c = add %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_sub: - printf("\t%%x%c = sub %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_mul: - printf("\t%%x%c = mul %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_div: - printf("\t%%x%c = div %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_mod: - printf("\t%%x%c = mod %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_eq: - printf("\t%%x%c = eq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_neq: - printf("\t%%x%c = neq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_gt: - printf("\t%%x%c = gt %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_lt: - printf("\t%%x%c = lt %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_geq: - printf("\t%%x%c = geq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_leq: - printf("\t%%x%c = leq %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_bit_and: - printf("\t%%x%c = and %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_bit_or: - printf("\t%%x%c = or %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_bit_not: - printf("\t%%x%c = not %%x%c", rd, rs1); - break; - case OP_bit_xor: - printf("\t%%x%c = xor %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_log_not: - printf("\t%%x%c = not %%x%c", rd, rs1); - break; - case OP_rshift: - printf("\t%%x%c = rshift %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_lshift: - printf("\t%%x%c = lshift %%x%c, %%x%c", rd, rs1, rs2); - break; - case OP_trunc: - printf("\t%%x%c = trunc %%x%c, %d", rd, rs1, ph2_ir->src1); - break; - case OP_sign_ext: - printf("\t%%x%c = sign_ext %%x%c, %d", rd, rs1, ph2_ir->src1); - break; - case OP_cast: - printf("\t%%x%c = cast %%x%c", rd, rs1); - break; - default: - break; - } - printf("\n"); - } -} diff --git a/src/riscv-codegen.c b/src/riscv-codegen.c index fbc1bfc0..c935634b 100644 --- a/src/riscv-codegen.c +++ b/src/riscv-codegen.c @@ -12,388 +12,234 @@ #define RV32_ALIGNMENT 16 -/* Explanation: registers preservation/restoration - * - * The following table illustrates which registers are caller-saved or - * callee-saved: - * +-----------+--------+ - * | Register | Saver | - * | ABI Name | | - * +-----------+--------+ - * | zero, | | - * | gp, | (None) | - * | tp | | - * +-----------+--------+ - * | ra | Caller | - * +-----------+--------+ - * | sp | Callee | - * +-----------+--------+ - * | a0 - a7 | Caller | - * +-----------+--------+ - * | s0 - s11 | Callee | - * +-----------+--------+ - * | t0 - t6 | Caller | - * +-----------+--------+ - * - * - ra and sp: are properly handled by the code generator. - * - * - a0 - a7: are implicitly handled in register allocation phase. - * - * Register allocation use 8 virtual registers (vreg0 - vreg7) to - * allocate registers from IR. When encountering a function call, - * all virtual registers are spilled properly before the call, and - * their contents will also be restored if they are used after the - * call. - * - * Since vreg0 - vreg7 map to a0 - a7 directly, these physical - * registers are also preserved/restored naturally whan a function - * is invoked or returned. - * - * - s0 and s1: handling depends on the linking mode. - * - * Static linking: - * For s0 register, it is used at the program entry point under - * static linking. The program entry point will use s0 to store - * the value of sp register and obtain 'argc' and 'argv' via s0 - * before calling the main function. Then, the content of s0 is - * no longer used after calling the main function, so its - * preservation and restoration can also be ignored. - * - * s1 is not used under static linking, so it is not necessary to - * be handled. - * - * Dynamic linking: - * Both registers are used to preserve 'argc' and 'argv' and pass - * them to the main function. Therefore, the code generator ensures - * that proper instructions are generated to preserve/restore - * their contents via stack. - * - * - s2 - s11: are not necessary to be preserved or restored. - * - * The current code generator does not use s2 - s11 to generate - * instructions, so these registers are not needed to be processed. - * - * - t0 - t6: are not necessary to be handled. - * - * These registers are used to store certain temporary values; - * however, the code generator ensures that these values do not - * persist across function calls. +static bool rv_count_only; +static int rv_counted_words; + +void emit_ph2_ir(ph2_ir_t *ph2_ir); + +/* a0-a7 hold virtual registers 0-7; s3-s5 hold the remaining allocatable + * registers and are saved when used. s2 stages indirect callees, while s0/s1 + * belong to startup argument handling. The backend does not use other saved + * registers; ra and sp are handled by the function prologue and epilogue. */ -void update_elf_offset(ph2_ir_t *ph2_ir) +/* The machine register for IR register @ir_reg. The first eight are the + * argument registers a0-a7; the rest are s3-s5, which a callee preserves, so a + * value pinned there survives a call. s0 and s1 belong to the entry code and s2 + * holds an indirect call's target. + */ +int rv_reg_of(int ir_reg) { - switch (ph2_ir->op) { - case OP_load_constant: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) - elf_offset += 8; - else - elf_offset += 4; - if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) - elf_offset += 8; - else - elf_offset += 4; - } - return; - case OP_address_of: - case OP_global_address_of: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) - elf_offset += 12; - else - elf_offset += 4; - return; - case OP_assign: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) - elf_offset += 4; - /* Exchanging the two registers of a pair takes a third move. */ - if (ph2_ir->dest_hi >= 0 && ph2_ir->dest == ph2_ir->src0_hi && - ph2_ir->dest_hi == ph2_ir->src0) - elf_offset += 4; - return; - case OP_load: - case OP_global_load: - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) - if (ph2_ir->dest_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->dest_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_store: - case OP_global_store: - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) - if (ph2_ir->src0_hi >= 0) - elf_offset += 20; - else - elf_offset += 16; - else if (ph2_ir->src0_hi >= 0) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_read: - elf_offset += ph2_ir->dest_hi >= 0 ? 8 : 4; - return; - case OP_write: - elf_offset += ph2_ir->src1_hi >= 0 ? 8 : 4; - return; - case OP_call: - /* Any call may be farther than a JAL reaches. */ - elf_offset += 8; - return; - case OP_jump: - case OP_load_func: - case OP_indirect: - case OP_lshift: - case OP_rshift: - elf_offset += ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 ? 52 : 4; - return; - case OP_gt: - case OP_lt: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 28 : 4; - return; - case OP_negate: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 12; - return; - case OP_add: - case OP_sub: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 12; - return; - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 4; - return; - case OP_bit_not: - elf_offset += 4; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_mul: - if (hard_mul_div) { - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 24; - else - elf_offset += 4; - } else if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - elf_offset += 100; - else - elf_offset += 52; - return; - case OP_div: - case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) + return ir_reg < 8 ? ir_reg + __a0 : __s3 + ir_reg - 8; +} - /* Four copied input words, a quotient pair, a remainder pair, and a - * fixed 64-round restoring loop. The modulo form copies the final - * remainder out after the loop. Signed operands add magnitude - * conversion and a sign-restoration sequence. - */ - elf_offset += ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned - ? (ph2_ir->op == OP_mod ? 168 : 160) - : 244; - else if (hard_mul_div) - elf_offset += 4; - else - elf_offset += 116; - return; - case OP_load_data_address: - case OP_load_rodata_address: - case OP_geq: - case OP_leq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 32 : 8; - return; - case OP_log_not: - elf_offset += ph2_ir->src0_hi >= 0 ? 12 : 8; - return; - case OP_address_of_func: - elf_offset += 12; - return; - case OP_eq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 20 : 12; - return; - case OP_neq: - elf_offset += ph2_ir->src0_hi >= 0 && ph2_ir->src1_hi >= 0 ? 16 : 8; - return; - case OP_branch: - elf_offset += ph2_ir->src0_hi >= 0 ? 24 : 20; - return; - case OP_return: - elf_offset += 24; - if (ph2_ir->src0_hi >= 0) - elf_offset += 4; - return; - case OP_trunc: - /* A byte and a short each need a shift pair to keep the sign. */ - if (ph2_ir->src1 == 1 || ph2_ir->src1 == 2) - elf_offset += 8; - else - elf_offset += 4; - return; - case OP_sign_ext: { - /* Decode source size from upper 16 bits */ - int source_size = (ph2_ir->src1 >> 16) & 0xFFFF; - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi < 0) { - /* Mirror the emitter: a zero-extended byte is one ANDI, a - * sign-extended byte and either halfword take a shift pair, and a - * word is one move; the high word adds one more instruction. The - * byte cases were swapped, so a (long long) cast of an unsigned - * char moved every later branch target by four bytes. - */ - bool zero_ext = ph2_ir->src0_is_unsigned || ph2_ir->src0_is_pointer; - elf_offset += - source_size == 2 || (source_size == 1 && !zero_ext) ? 12 : 8; - return; +/* The callee-saved registers @func writes, and so must save on entry and + * restore on return: how many of s3-s5, which are handed out in order so the + * highest one used decides, and in bit 8 whether it stages an indirect call's + * target in s2. + */ +int rv_saved_regs(func_t *func) +{ + int top = func_highest_used_reg(func, 8); + int s2 = 0; + + for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { + for (ph2_ir_t *ir = bb->ph2_ir_list.head; ir; ir = ir->next) { + if (ir->dest_hi > top) + top = ir->dest_hi; + if (ir->src0_hi > top) + top = ir->src0_hi; + if (ir->src1_hi > top) + top = ir->src1_hi; + if (ir->op == OP_load_func) + s2 = 1 << 8; } - if (source_size == 2) - elf_offset += 8; /* short extension: 2 instructions */ - else - elf_offset += 12; /* byte extension: 3 instructions */ - return; - } - case OP_cast: - elf_offset += - ph2_ir->dest_hi >= 0 && - (ph2_ir->src0_hi < 0 || ph2_ir->dest_hi != ph2_ir->src0_hi) - ? 8 - : 4; - return; - default: - fatal("Unknown opcode"); } + if (top >= REG_CNT) + top = REG_CNT - 1; + return (top - 7) | s2; +} + +/* How many words rv_saved_regs()'s @saved occupies in the frame. */ +int rv_saved_words(int saved) +{ + return (saved & 0xff) + (saved >> 8); +} + +/* The frame a function allocates: its locals, ra, and the saved registers, kept + * 16-byte aligned. + */ +int rv_frame_bytes(int stack_size, int saved) +{ + return ALIGN_UP(stack_size + 4 + rv_saved_words(saved) * 4, RV32_ALIGNMENT); +} + +/* The register saved in frame word @i: s3 onwards, then s2. */ +int rv_saved_reg(int saved, int i) +{ + return i < (saved & 0xff) ? __s3 + i : __s2; +} + +static int rv_count_ph2_ir_words(ph2_ir_t *ph2_ir) +{ + bool saved_count_only = rv_count_only; + int saved_counted_words = rv_counted_words; + char *saved_fatal_function_context = fatal_function_context; + + rv_count_only = true; + rv_counted_words = 0; + emit_ph2_ir(ph2_ir); + int words = rv_counted_words; + + rv_count_only = saved_count_only; + rv_counted_words = saved_counted_words; + fatal_function_context = saved_fatal_function_context; + return words; +} + +void update_elf_offset(ph2_ir_t *ph2_ir) +{ + if (ph2_ir->op != OP_allocat) + elf_offset += rv_count_ph2_ir_words(ph2_ir) * 4; +} + +static ph2_frame_layout_t rv_frame_layout(func_t *func, ph2_ir_t *define) +{ + int saved = rv_saved_regs(func); + + define->src2 = saved; + return (ph2_frame_layout_t) {rv_frame_bytes(func->stack_size, saved), + rv_count_ph2_ir_words(define) * 4, saved, + true}; } void cfg_flatten(void) { func_t *func; + ph2_ir_prepare(false); if (dynlink) { - /* When using dynamic linking, 25 instructions are generated at the + /* When using dynamic linking, 26 instructions are generated at the * program entry point to perform the following operations: * - prepare arguments and call __libc_start_main() * - preserve a0 ('argc'), a1 ('argv') and sp. * - allocate and clear a global stack, then jump to global init * function. */ - elf_offset = 100; + elf_offset = 104; } else { /* Under static linking, "__syscall" must be generated to allow the * program to invoke system calls. * - * "__syscall" consists of 9 instructions, preceded by 6 initial - * instructions. Consequently, the elf offset for "__syscall" is is 24 - * bytes, and the offset for the subsequent function (GLOBAL_FUNC) is 60 - * bytes. + * "__syscall" consists of 9 instructions, preceded by 7 initial + * instructions because the global-init call is an AUIPC/JALR pair. + * Consequently, "__syscall" starts at byte 28 and the subsequent + * function (GLOBAL_FUNC) starts at byte 64. */ func = find_func("__syscall"); - func->bbs->elf_offset = 24; - elf_offset = 60; + func->bbs->elf_offset = 28; + elf_offset = 64; } GLOBAL_FUNC->bbs->elf_offset = elf_offset; - for (ph2_ir_t *ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) { - update_elf_offset(ph2_ir); - } + ph2_ir_visit_globals(update_elf_offset); - /* prepare 'argc' and 'argv', then proceed to 'main' function. The stub - * precedes every function, so 'main' can lie beyond the 1 MiB a JAL - * reaches; it is called through AUIPC and JALR instead. + /* Prepare argc/argv and reach main through a range-independent call. + * code_generate() emits this only when there is a main, so a translation + * unit without one must not be charged for it either: every function after + * would be placed 28 bytes from where it is emitted. */ - if (dynlink) - elf_offset += 48; - else - elf_offset += 28; - - for (func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; + if (MAIN_BB) + elf_offset += dynlink ? 48 : 28; - /* reserve stack */ - ph2_ir_t *flatten_ir = add_ph2_ir(OP_define); - flatten_ir->src0 = func->stack_size; - flatten_ir->func_name = intern_string(func->return_def.var_name); + ph2_ir_flatten_functions(rv_frame_layout, update_elf_offset); +} - /* Except for local variables, it must allocate additional space to - * preserve the content of ra at each function entry point. - * - * 'stack_size' doesn't include the additional space, so an extra number - * '4' is added to 'stack_size'. - */ - int stack_top_ofs = ALIGN_UP(func->stack_size + 4, RV32_ALIGNMENT); +void emit(int code) +{ + if (rv_count_only) { + if (rv_counted_words < INT_MAX) + rv_counted_words++; + } else + elf_write_int(elf_code, code); +} - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->elf_offset = elf_offset; +static void emit_rv32_split_address(bool pc_relative, + rv_reg rd, + rv_reg base, + int offset) +{ + int hi = rv_hi(offset), lo = rv_lo(offset); - if (bb == func->bbs) { - /* save ra, sp */ - elf_offset += 16; - } + emit(pc_relative ? __auipc(base, hi) : __lui(base, hi)); + emit(pc_relative ? __jalr(rd, base, lo) : __addi(rd, base, lo)); +} - for (ph2_ir_t *insn = bb->ph2_ir_list.head; insn; - insn = insn->next) { - if (insn->ofs_based_on_stack_top) { - switch (insn->op) { - case OP_load: - case OP_address_of: - insn->src0 = insn->src0 + stack_top_ofs; - break; - case OP_store: - insn->src1 = insn->src1 + stack_top_ofs; - break; - default: - /* Ignore opcodes with the ofs_based_on_stack_top flag - * set since only the three opcodes above needs to - * access a variable's address. - */ - break; - } - } - flatten_ir = add_existed_ph2_ir(insn); +static rv_reg rv_memory_base(rv_reg base, int offset, int pair, int *disp) +{ + *disp = offset; + if (offset >= -2048 && offset <= 2047 - pair * 4) + return base; + emit_rv32_split_address(false, __t0, __t0, offset); + emit(__add(__t0, base, __t0)); + *disp = 0; + return __t0; +} - if (insn->op == OP_return) { - /* restore sp */ - flatten_ir->src1 = bb->belong_to->stack_size; - } +static void rv_load(rv_reg dest, + rv_reg base, + int offset, + int size, + bool is_unsigned) +{ + if (size == 1) + emit(is_unsigned ? __lbu(dest, base, offset) + : __lb(dest, base, offset)); + else if (size == 2) + emit(is_unsigned ? __lhu(dest, base, offset) + : __lh(dest, base, offset)); + else + emit(__lw(dest, base, offset)); +} - update_elf_offset(flatten_ir); - } - } - } +static void rv_store(rv_reg src, rv_reg base, int offset, int size) +{ + if (size == 1) + emit(__sb(src, base, offset)); + else if (size == 2) + emit(__sh(src, base, offset)); + else + emit(__sw(src, base, offset)); } -void emit(int code) +static int riscv_bitwise_insn(opcode_t op, + rv_reg dest, + rv_reg left, + rv_reg right) { - elf_write_int(elf_code, code); + switch (op) { + case OP_bit_and: + return __and(dest, left, right); + case OP_bit_or: + return __or(dest, left, right); + case OP_bit_xor: + return __xor(dest, left, right); + default: + fatal("unsupported RISC-V bitwise operation"); + } + return 0; } void emit_ph2_ir(ph2_ir_t *ph2_ir) { func_t *func; - int rd = ph2_ir->dest + 10; - int rs1 = ph2_ir->src0 + 10; - int rs2 = ph2_ir->src1 + 10; - int rd_hi = ph2_ir->dest_hi + 10; - int rs1_hi = ph2_ir->src0_hi + 10; - int rs2_hi = ph2_ir->src1_hi + 10; + int rd = rv_reg_of(ph2_ir->dest); + int rs1 = rv_reg_of(ph2_ir->src0); + int rs2 = rv_reg_of(ph2_ir->src1); + int rd_hi = rv_reg_of(ph2_ir->dest_hi); + int rs1_hi = rv_reg_of(ph2_ir->src0_hi); + int rs2_hi = rv_reg_of(ph2_ir->src1_hi); int ofs; /* Prepare the variables to reuse the same code for the instruction sequence @@ -407,23 +253,28 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) switch (ph2_ir->op) { case OP_define: fatal_function_context = ph2_ir->func_name; - ofs = ALIGN_UP(ph2_ir->src0 + 4, RV32_ALIGNMENT); - emit(__sw(__ra, __sp, -4)); - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + ofs = rv_frame_bytes(ph2_ir->src0, ph2_ir->src2); + + /* Lower sp before saving below the frame's top: there is no red zone, + * and a signal delivered in between would overwrite anything stored + * below sp. t1 holds the old sp to address the saves from. + */ + emit_rv32_split_address(false, __t0, __t0, ofs); + emit(__addi(__t1, __sp, 0)); emit(__sub(__sp, __sp, __t0)); + emit(__sw(__ra, __t1, -4)); + for (int i = 0; i < rv_saved_words(ph2_ir->src2); i++) + emit(__sw(rv_saved_reg(ph2_ir->src2, i), __t1, -8 - i * 4)); return; case OP_load_constant: if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(rd, rv_hi(ph2_ir->src0))); - emit(__addi(rd, rd, rv_lo(ph2_ir->src0))); + emit_rv32_split_address(false, rd, rd, ph2_ir->src0); } else emit(__addi(rd, __zero, ph2_ir->src0)); if (ph2_ir->dest_hi >= 0) { if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) { - emit(__lui(rd_hi, rv_hi(ph2_ir->src1))); - emit(__addi(rd_hi, rd_hi, rv_lo(ph2_ir->src1))); + emit_rv32_split_address(false, rd_hi, rd_hi, ph2_ir->src1); } else emit(__addi(rd_hi, __zero, ph2_ir->src1)); } @@ -432,14 +283,13 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) case OP_global_address_of: interm = ph2_ir->op == OP_address_of ? __sp : __gp; if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src0))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src0))); + emit_rv32_split_address(false, __t0, __t0, ph2_ir->src0); emit(__add(rd, interm, __t0)); } else emit(__addi(rd, interm, ph2_ir->src0)); return; case OP_assign: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && rd == rs1_hi) { + if (PH2_PAIR_DEST_SRC0(ph2_ir) && rd == rs1_hi) { /* A pair moved one register over, as a slot round trip collapsed * into a move can leave it: the low destination is the high source, * so move the high word first, through t0 when the two registers @@ -456,105 +306,57 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; } emit(__addi(rd, rs1, 0)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->dest_hi != ph2_ir->src0_hi) + if (PH2_PAIR_DEST_SRC0(ph2_ir) && ph2_ir->dest_hi != ph2_ir->src0_hi) emit(__addi(rd_hi, rs1_hi, 0)); return; case OP_load: case OP_global_load: interm = ph2_ir->op == OP_load ? __sp : __gp; if (ph2_ir->dest_hi >= 0) { - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src0))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src0))); - emit(__add(__t0, interm, __t0)); - emit(__lw(rd, __t0, 0)); - emit(__lw(rd_hi, __t0, 4)); - } else { - emit(__lw(rd, interm, ph2_ir->src0)); - emit(__lw(rd_hi, interm, ph2_ir->src0 + 4)); - } + int disp; + rv_reg base = rv_memory_base(interm, ph2_ir->src0, 1, &disp); + + rv_load(rd, base, disp, 4, true); + rv_load(rd_hi, base, disp + 4, 4, true); return; } - if (ph2_ir->src0 < -2048 || ph2_ir->src0 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src0))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src0))); - emit(__add(__t0, interm, __t0)); - if (ph2_ir->size_bytes == 1) - emit(ph2_ir->is_unsigned ? __lbu(rd, __t0, 0) - : __lb(rd, __t0, 0)); - else if (ph2_ir->size_bytes == 2) - emit(ph2_ir->is_unsigned ? __lhu(rd, __t0, 0) - : __lh(rd, __t0, 0)); - else - emit(__lw(rd, __t0, 0)); - } else if (ph2_ir->size_bytes == 1) - emit(ph2_ir->is_unsigned ? __lbu(rd, interm, ph2_ir->src0) - : __lb(rd, interm, ph2_ir->src0)); - else if (ph2_ir->size_bytes == 2) - emit(ph2_ir->is_unsigned ? __lhu(rd, interm, ph2_ir->src0) - : __lh(rd, interm, ph2_ir->src0)); - else - emit(__lw(rd, interm, ph2_ir->src0)); + { + int disp; + rv_reg base = rv_memory_base(interm, ph2_ir->src0, 0, &disp); + + rv_load(rd, base, disp, ph2_ir->size_bytes, ph2_ir->is_unsigned); + } return; case OP_store: case OP_global_store: interm = ph2_ir->op == OP_store ? __sp : __gp; if (ph2_ir->src0_hi >= 0) { - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src1))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src1))); - emit(__add(__t0, interm, __t0)); - emit(__sw(rs1, __t0, 0)); - emit(__sw(rs1_hi, __t0, 4)); - } else { - emit(__sw(rs1, interm, ph2_ir->src1)); - emit(__sw(rs1_hi, interm, ph2_ir->src1 + 4)); - } + int disp; + rv_reg base = rv_memory_base(interm, ph2_ir->src1, 1, &disp); + + rv_store(rs1, base, disp, 4); + rv_store(rs1_hi, base, disp + 4, 4); return; } - if (ph2_ir->src1 < -2048 || ph2_ir->src1 > 2047) { - emit(__lui(__t0, rv_hi(ph2_ir->src1))); - emit(__addi(__t0, __t0, rv_lo(ph2_ir->src1))); - emit(__add(__t0, interm, __t0)); - if (ph2_ir->size_bytes == 1) - emit(__sb(rs1, __t0, 0)); - else if (ph2_ir->size_bytes == 2) - emit(__sh(rs1, __t0, 0)); - else - emit(__sw(rs1, __t0, 0)); - } else if (ph2_ir->size_bytes == 1) - emit(__sb(rs1, interm, ph2_ir->src1)); - else if (ph2_ir->size_bytes == 2) - emit(__sh(rs1, interm, ph2_ir->src1)); - else - emit(__sw(rs1, interm, ph2_ir->src1)); + { + int disp; + rv_reg base = rv_memory_base(interm, ph2_ir->src1, 0, &disp); + + rv_store(rs1, base, disp, ph2_ir->size_bytes); + } return; case OP_read: if (ph2_ir->dest_hi >= 0) { - /* A pair read through a pointer. When the low destination is the - * address register itself, fetch the high word first so the address - * survives until both words are loaded. - */ if (ph2_ir->src1 != 8) fatal("unsupported RISC-V pair load width"); - if (rd == rs1) { - emit(__lw(rd_hi, rs1, 4)); - emit(__lw(rd, rs1, 0)); - } else { - emit(__lw(rd, rs1, 0)); - emit(__lw(rd_hi, rs1, 4)); - } +#define RV_WORD_LOAD(dst, base, offset) emit(__lw(dst, base, offset)) + EMIT_PAIR_LOAD(RV_WORD_LOAD, rd, rd_hi, rs1); +#undef RV_WORD_LOAD return; } - if (ph2_ir->src1 == 1) - emit(ph2_ir->is_unsigned ? __lbu(rd, rs1, 0) : __lb(rd, rs1, 0)); - else if (ph2_ir->src1 == 2) - emit(ph2_ir->is_unsigned ? __lhu(rd, rs1, 0) : __lh(rd, rs1, 0)); - else if (ph2_ir->src1 == 4) - emit(__lw(rd, rs1, 0)); - else + if (ph2_ir->src1 != 1 && ph2_ir->src1 != 2 && ph2_ir->src1 != 4) fatal("unsupported RISC-V load width"); + rv_load(rd, rs1, 0, ph2_ir->src1, ph2_ir->is_unsigned); return; case OP_write: if (ph2_ir->src1_hi >= 0) { @@ -564,19 +366,13 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sw(rs2_hi, rs1, 4)); return; } - if (ph2_ir->dest == 1) - emit(__sb(rs2, rs1, 0)); - else if (ph2_ir->dest == 2) - emit(__sh(rs2, rs1, 0)); - else if (ph2_ir->dest == 4) - emit(__sw(rs2, rs1, 0)); - else + if (ph2_ir->dest != 1 && ph2_ir->dest != 2 && ph2_ir->dest != 4) fatal("unsupported RISC-V store width"); + rv_store(rs2, rs1, 0, ph2_ir->dest); return; case OP_branch: ofs = elf_code_start + ph2_ir->then_bb->elf_offset; - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + emit_rv32_split_address(false, __t0, __t0, ofs); /* A pair is true when either word is nonzero. */ if (ph2_ir->src0_hi >= 0) { @@ -585,52 +381,43 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) } emit(__beq(rs1, __zero, 8)); emit(__jalr(__zero, __t0, 0)); - emit(__jal(__zero, ph2_ir->else_bb->elf_offset - elf_code->size)); + ofs = elf_code_start + ph2_ir->else_bb->elf_offset; + emit_rv32_split_address(false, __t0, __t0, ofs); + emit(__jalr(__zero, __t0, 0)); return; case OP_jump: - emit(__jal(__zero, ph2_ir->next_bb->elf_offset - elf_code->size)); + ofs = ph2_ir->next_bb->elf_offset - elf_code->size; + emit_rv32_split_address(true, __zero, __t0, ofs); return; case OP_call: func = find_func(ph2_ir->func_name); - - /* A self-hosted image is past the 1 MiB a JAL reaches, and the PLT - * follows the code and read-only data. Call PC-relative through AUIPC - * and JALR, which reach anywhere. - */ - if (func->bbs) + if (func->bbs) { ofs = func->bbs->elf_offset - elf_code->size; - else if (dynlink) + emit_rv32_split_address(true, __ra, __ra, ofs); + return; + } else if (dynlink) { + /* The PLT follows the code and read-only data, so a large image + * puts it beyond the 1 MiB a JAL reaches. Call it PC-relative + * through AUIPC and JALR, which reach anywhere. + */ ofs = (dynamic_sections.elf_plt_start + func->plt_offset) - (elf_code_start + elf_code->size); - else { + emit_rv32_split_address(true, __ra, __ra, ofs); + return; + } else { printf("The '%s' function is not implemented\n", ph2_ir->func_name); fflush(stdout); /* see fatal() */ abort(); } - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); - return; case OP_load_data_address: - emit(__lui(rd, rv_hi(elf_data_start + ph2_ir->src0))); - emit(__addi(rd, rd, rv_lo(elf_data_start + ph2_ir->src0))); + emit_rv32_split_address(false, rd, rd, elf_data_start + ph2_ir->src0); return; case OP_load_rodata_address: - emit(__lui(rd, rv_hi(elf_rodata_start + ph2_ir->src0))); - emit(__addi(rd, rd, rv_lo(elf_rodata_start + ph2_ir->src0))); + emit_rv32_split_address(false, rd, rd, elf_rodata_start + ph2_ir->src0); return; case OP_address_of_func: - func = find_func(ph2_ir->func_name); - if (func->bbs) - ofs = elf_code_start + func->bbs->elf_offset; - else if (dynlink) - ofs = dynamic_sections.elf_plt_start + func->plt_offset; - else { - printf("The '%s' function is not implemented\n", ph2_ir->func_name); - fflush(stdout); /* see fatal() */ - abort(); - } - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + ofs = function_entry_address(ph2_ir->func_name); + emit_rv32_split_address(false, __t0, __t0, ofs); emit(__sw(__t0, rs1, 0)); return; case OP_load_func: @@ -646,57 +433,62 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__addi(__a0, rs1, 0)); if (ph2_ir->src0_hi >= 0) emit(__addi(__a1, rs1_hi, 0)); - ofs = ALIGN_UP(ph2_ir->src1 + 4, RV32_ALIGNMENT); - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); - emit(__add(__sp, __sp, __t0)); - emit(__lw(__ra, __sp, -4)); + ofs = rv_frame_bytes(ph2_ir->src1, ph2_ir->src2); + /* Reload the saved registers while sp still covers them. */ + emit_rv32_split_address(false, __t0, __t0, ofs); + emit(__add(__t1, __sp, __t0)); + emit(__lw(__ra, __t1, -4)); + for (int i = 0; i < rv_saved_words(ph2_ir->src2); i++) + emit(__lw(rv_saved_reg(ph2_ir->src2, i), __t1, -8 - i * 4)); + emit(__addi(__sp, __t1, 0)); emit(__jalr(__zero, __ra, 0)); return; case OP_add: - emit(__add(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - emit(__sltu(__t0, rd, rs1)); + if (PH2_PAIR_BINARY(ph2_ir)) { + /* Preserve the original low words until carry is known, including + * an in-place destination and x + x using the same register pair. + */ + emit(__add(__t0, rs1, rs2)); + emit(__sltu(__t1, __t0, rs1)); emit(__add(rd_hi, rs1_hi, rs2_hi)); - emit(__add(rd_hi, rd_hi, __t0)); - } + emit(__add(rd_hi, rd_hi, __t1)); + emit(__addi(rd, __t0, 0)); + } else + emit(__add(rd, rs1, rs2)); return; case OP_sub: - emit(__sub(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) emit(__sltu(__t0, rs1, rs2)); + emit(__sub(rd, rs1, rs2)); + if (PH2_PAIR_BINARY(ph2_ir)) { emit(__sub(rd_hi, rs1_hi, rs2_hi)); emit(__sub(rd_hi, rd_hi, __t0)); } return; case OP_mul: if (hard_mul_div) { - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { - emit(__mul(rd, rs1, rs2)); - emit(__mulhu(rd_hi, rs1, rs2)); + if (PH2_PAIR_BINARY(ph2_ir)) { emit(__mul(__t0, rs1, rs2_hi)); emit(__mul(__t1, rs1_hi, rs2)); - emit(__add(rd_hi, rd_hi, __t0)); + emit(__mulhu(__t2, rs1, rs2)); + emit(__mul(rd, rs1, rs2)); + emit(__add(rd_hi, __t2, __t0)); emit(__add(rd_hi, rd_hi, __t1)); } else emit(__mul(rd, rs1, rs2)); } else { - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) { /* Multiply two 64-bit pairs modulo 2^64 without the M * extension. Keep the product in the destination pair, so t0-t3 * can hold the shifting multiplicand and multiplier; the loop * ends once both multiplier words are zero. */ - emit(__addi(rd, __zero, 0)); - emit(__addi(rd_hi, __zero, 0)); emit(__addi(__t0, rs1, 0)); emit(__addi(__t1, rs1_hi, 0)); emit(__addi(__t2, rs2, 0)); emit(__addi(__t3, rs2_hi, 0)); + emit(__addi(rd, __zero, 0)); + emit(__addi(rd_hi, __zero, 0)); emit(__andi(__t4, __t2, 1)); emit(__sub(__t4, __zero, __t4)); emit(__and(__t5, __t0, __t4)); @@ -735,8 +527,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_div: case OP_mod: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) { + if (PH2_PAIR_BINARY(ph2_ir)) { bool is_unsigned = ph2_ir->src0_is_unsigned || ph2_ir->src1_is_unsigned; @@ -913,7 +704,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sub(rd, __zero, rd)); return; case OP_lshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (PH2_PAIR_DEST_SRC0(ph2_ir)) { emit(__addi(__t0, __zero, 32)); emit(__bltu(rs2, __t0, 20)); emit(__sub(__t1, rs2, __t0)); @@ -938,7 +729,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) emit(__sll(rd, rs1, rs2)); return; case OP_rshift: - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (PH2_PAIR_DEST_SRC0(ph2_ir)) { bool is_unsigned = ph2_ir->src0_is_unsigned; emit(__addi(__t0, __zero, 32)); @@ -1031,7 +822,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_negate: emit(__sub(rd, __zero, rs1)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) { + if (PH2_PAIR_DEST_SRC0(ph2_ir)) { emit(__sltu(__t0, __zero, rs1)); emit(__sub(rd_hi, __zero, rs1_hi)); emit(__sub(rd_hi, rd_hi, __t0)); @@ -1039,26 +830,15 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) return; case OP_bit_not: emit(__xori(rd, rs1, -1)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0) + if (PH2_PAIR_DEST_SRC0(ph2_ir)) emit(__xori(rd_hi, rs1_hi, -1)); return; case OP_bit_and: - emit(__and(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__and(rd_hi, rs1_hi, rs2_hi)); - return; case OP_bit_or: - emit(__or(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__or(rd_hi, rs1_hi, rs2_hi)); - return; case OP_bit_xor: - emit(__xor(rd, rs1, rs2)); - if (ph2_ir->dest_hi >= 0 && ph2_ir->src0_hi >= 0 && - ph2_ir->src1_hi >= 0) - emit(__xor(rd_hi, rs1_hi, rs2_hi)); + emit(riscv_bitwise_insn(ph2_ir->op, rd, rs1, rs2)); + if (PH2_PAIR_BINARY(ph2_ir)) + emit(riscv_bitwise_insn(ph2_ir->op, rd_hi, rs1_hi, rs2_hi)); return; case OP_log_not: if (ph2_ir->src0_hi >= 0) { @@ -1160,6 +940,7 @@ void emit_ph2_ir(ph2_ir_t *ph2_ir) void plt_generate(void); void code_generate(void) { + int global_stack_size; int ofs; if (dynlink) { @@ -1194,8 +975,7 @@ void code_generate(void) * __libc_start_main(main_wrapper, argc, argv, NULL, * NULL, NULL, stack_end) */ - emit(__lui(__a0, rv_hi(elf_code_start + 40))); - emit(__addi(__a0, __a0, rv_lo(elf_code_start + 40))); + emit_rv32_split_address(false, __a0, __a0, elf_code_start + 40); emit(__lw(__a1, __sp, 0)); emit(__addi(__a2, __sp, 4)); emit(__addi(__a3, __zero, 0)); @@ -1208,8 +988,7 @@ void code_generate(void) */ ofs = (dynamic_sections.elf_plt_start + PLT_FIXUP_SIZE) - (elf_code_start + elf_code->size); - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); + emit_rv32_split_address(true, __ra, __ra, ofs); /* The main wrapper is located here under the dynamic linking mode * @@ -1226,14 +1005,15 @@ void code_generate(void) emit(__sw(__s0, __sp, 0)); /* callee-saved */ emit(__addi(__s0, __a0, 0)); /* argc */ emit(__addi(__s1, __a1, 0)); /* argv */ - ofs = ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT) + 4; + global_stack_size = + ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT) + 4; } else { /* When using static linking, the starting address of the main wrapper * is here. * * Save original sp in s0 first. */ - ofs = ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT); + global_stack_size = ALIGN_UP(GLOBAL_FUNC->stack_size, RV32_ALIGNMENT); emit(__addi(__s0, __sp, 0)); } @@ -1242,8 +1022,7 @@ void code_generate(void) * 2. jump to global init function * 3. call the main function */ - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + emit_rv32_split_address(false, __t0, __t0, global_stack_size); emit(__sub(__sp, __sp, __t0)); emit(__addi(__gp, __sp, 0)); /* Set up global pointer */ @@ -1260,7 +1039,8 @@ void code_generate(void) emit(__sw(__zero, __t1, 0)); emit(__bne(__t0, __zero, -12)); } - emit(__jal(__ra, GLOBAL_FUNC->bbs->elf_offset - elf_code->size)); + ofs = GLOBAL_FUNC->bbs->elf_offset - elf_code->size; + emit_rv32_split_address(true, __ra, __ra, ofs); if (!dynlink) { /* syscall trampoline for __syscall */ @@ -1275,10 +1055,7 @@ void code_generate(void) emit(__jalr(__zero, __ra, 0)); } - ph2_ir_t *ph2_ir; - for (ph2_ir = GLOBAL_FUNC->bbs->ph2_ir_list.head; ph2_ir; - ph2_ir = ph2_ir->next) - emit_ph2_ir(ph2_ir); + ph2_ir_visit_globals(emit_ph2_ir); /* prepare 'argc' and 'argv', then proceed to 'main' function */ if (MAIN_BB) { @@ -1286,15 +1063,13 @@ void code_generate(void) emit(__addi(__a0, __s0, 0)); emit(__addi(__a1, __s1, 0)); ofs = MAIN_BB->elf_offset - elf_code->size; - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); + emit_rv32_split_address(true, __ra, __ra, ofs); /* - Restore sp, s0 and s1. * - Transfer control back to __libc_start_main() using * the preserved ra. */ - emit(__lui(__t0, rv_hi(ofs))); - emit(__addi(__t0, __t0, rv_lo(ofs))); + emit_rv32_split_address(false, __t0, __t0, global_stack_size); emit(__add(__sp, __sp, __t0)); emit(__lw(__ra, __sp, 8)); emit(__lw(__s1, __sp, 4)); @@ -1307,8 +1082,7 @@ void code_generate(void) emit(__lw(__a0, __t0, 0)); emit(__addi(__a1, __t0, 4)); ofs = MAIN_BB->elf_offset - elf_code->size; - emit(__auipc(__ra, rv_hi(ofs))); - emit(__jalr(__ra, __ra, rv_lo(ofs))); + emit_rv32_split_address(true, __ra, __ra, ofs); /* exit with main's return value in a0 */ emit(__addi(__a7, __zero, 93)); @@ -1317,7 +1091,7 @@ void code_generate(void) } for (int i = 0; i < ph2_ir_idx; i++) { - ph2_ir = PH2_IR_FLATTEN[i]; + ph2_ir_t *ph2_ir = PH2_IR_FLATTEN[i]; emit_ph2_ir(ph2_ir); } } diff --git a/src/riscv.c b/src/riscv.c index 82cd9d1c..b1af8c2c 100644 --- a/src/riscv.c +++ b/src/riscv.c @@ -122,27 +122,22 @@ int rv_encode_R(rv_op op, rv_reg rd, rv_reg rs1, rv_reg rs2) return op + (rd << 7) + (rs1 << 15) + (rs2 << 20); } -int rv_encode_I(rv_op op, rv_reg rd, rv_reg rs1, int imm) +static int rv_imm12(int imm) { if (imm > 2047 || imm < -2048) fatal("Offset too large"); + return imm < 0 ? imm + 4096 : imm; +} - if (imm < 0) { - imm += 4096; - imm &= (1 << 13) - 1; - } +int rv_encode_I(rv_op op, rv_reg rd, rv_reg rs1, int imm) +{ + imm = rv_imm12(imm); return op + (rd << 7) + (rs1 << 15) + (imm << 20); } int rv_encode_S(rv_op op, rv_reg rs1, rv_reg rs2, int imm) { - if (imm > 2047 || imm < -2048) - fatal("Offset too large"); - - if (imm < 0) { - imm += 4096; - imm &= (1 << 13) - 1; - } + imm = rv_imm12(imm); return op + (rs1 << 15) + (rs2 << 20) + rv_extract_bits(imm, 0, 4, 7, 11) + rv_extract_bits(imm, 5, 11, 25, 31); } diff --git a/src/ssa.c b/src/ssa.c deleted file mode 100644 index fc32239c..00000000 --- a/src/ssa.c +++ /dev/null @@ -1,4913 +0,0 @@ -/* - * shecc - Self-Hosting and Educational C Compiler. - * - * shecc is freely redistributable under the BSD 2 clause license. See the file - * "LICENSE" for information on usage and redistribution of this file. - */ -#include -#include - -#include "defs.h" -#include "globals.c" - -/* Constant cast optimization. Despite the file name this is not SCCP: there is - * no lattice and no CFG-edge worklist anywhere in the tree. - */ -#include "opt-sccp.c" - -/* Configuration constants - replace magic numbers */ -#define PHI_WORKLIST_SIZE 128 -#define DCE_WORKLIST_SIZE 2048 - -/* Dead store elimination window size */ -#define OVERWRITE_WINDOW 3 - -/* Whether @var lives in memory that other code can reach without naming it: a - * global, or a variable whose address is taken. SSA gives such a variable one - * name, and a write to it is observable however unused it looks. - */ -bool var_in_memory(const var_t *var) -{ - return var->is_global || var->address_taken; -} - -/* Whether @insn accesses a volatile object, which C99 6.7.3p6 counts as a side - * effect whether or not anything uses the value: it names such an object as an - * operand, or reads through an address mark_volatile_addresses() traced to one. - * Taking an object's address does not access it. - */ -bool insn_reads_volatile(const insn_t *insn) -{ - if (insn->opcode == OP_address_of || insn->opcode == OP_allocat) - return false; - if (insn->opcode == OP_read && insn->rs1 && insn->rs1->is_volatile) - return !insn->rd || !insn->rd->is_assignment_reload; - return var_is_volatile_object(insn->rs1) || - var_is_volatile_object(insn->rs2) || - var_is_volatile_object(insn->rs3); -} - -/* Whether @var holds an address into a volatile object: a volatile pointer or - * array, or a temporary already traced to one. - */ -bool var_is_volatile_address(const var_t *var) -{ - if (!var || !var->is_volatile) - return false; - return var->ptr_level || var->array_size || - (var->type && var->type->ptr_level) || var->var_name[0] == '.'; -} - -/* Mark every temporary that holds an address into a volatile object. - * - * A declaration says its object is volatile, but a member or an element of the - * object, or whatever a pointer to volatile designates, is read through an - * address the parser computed into a temporary, and nothing on the temporary - * says what it points into. Follow each address from the object or pointer it - * was derived from, so that the read at the end of the chain is recognizable as - * the side effect it is. Only addresses are followed: an int computed from a - * volatile int is an ordinary value. - */ -void mark_volatile_addresses(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *rd = insn->rd; - bool from_volatile = false; - - if (!rd || rd->is_volatile || rd->var_name[0] != '.') - continue; - - switch (insn->opcode) { - case OP_address_of: - from_volatile = insn->rs1 && insn->rs1->is_volatile; - break; - case OP_add: - case OP_sub: - from_volatile = var_is_volatile_address(insn->rs1) || - var_is_volatile_address(insn->rs2); - break; - case OP_assign: - case OP_cast: - from_volatile = var_is_volatile_address(insn->rs1); - break; - default: - break; - } - if (from_volatile) - rd->is_volatile = true; - } - } -} - -void var_list_ensure_capacity(var_list_t *list, int min_capacity) -{ - if (list->capacity >= min_capacity) - return; - - int new_capacity = list->capacity ? list->capacity : HOST_PTR_SIZE; - - while (new_capacity < min_capacity) - new_capacity <<= 1; - - /* arena_realloc() extends the block in place when this list was the last - * thing allocated, which the liveness sets often are; allocating a fresh - * array and copying abandoned the old one in the arena every time. - */ - list->elements = arena_realloc(BB_ARENA, (char *) list->elements, - list->capacity * HOST_PTR_SIZE, - new_capacity * HOST_PTR_SIZE); - list->capacity = new_capacity; -} - -/* Whether @var appears in @list. */ -bool var_list_holds(const var_list_t *list, var_t *var) -{ - for (int i = 0; i < list->size; i++) { - if (list->elements[i] == var) - return true; - } - return false; -} - -void var_list_add_var(var_list_t *list, var_t *var) -{ - if (var_list_holds(list, var)) - return; - - var_list_ensure_capacity(list, list->size + 1); - list->elements[list->size++] = var; -} - -/* Append without the membership scan var_list_add_var() performs. Callers that - * use this must establish uniqueness themselves. - */ -void var_list_append(var_list_t *list, var_t *var) -{ - var_list_ensure_capacity(list, list->size + 1); - list->elements[list->size++] = var; -} - -void var_list_assign_array(var_list_t *list, var_t **data, int count) -{ - var_list_ensure_capacity(list, count); - memcpy(list->elements, data, count * HOST_PTR_SIZE); - list->size = count; -} - -/* cfront does not accept structure as an argument, pass pointer The only thing - * a step of either traversal changes in the argument block is the block it - * names, so a step sets that field and puts it back on the way out. Copying the - * whole structure per edge instead -- which is what these did -- costs a copy - * on every block of every traversal, and the traversals are how nearly every - * analysis in the middle end walks a function. - */ -void bb_forward_traversal(bb_traversal_args_t *args) -{ - basic_block_t *bb = args->bb; - func_t *func = args->func; - - bb->visited = func->visited; - - if (args->preorder_cb) - args->preorder_cb(func, bb); - - if (bb->next) { - if (bb->next->visited < func->visited) { - args->bb = bb->next; - bb_forward_traversal(args); - } - } - if (bb->then_) { - if (bb->then_->visited < func->visited) { - args->bb = bb->then_; - bb_forward_traversal(args); - } - } - if (bb->else_) { - if (bb->else_->visited < func->visited) { - args->bb = bb->else_; - bb_forward_traversal(args); - } - } - - args->bb = bb; - - if (args->postorder_cb) - args->postorder_cb(func, bb); -} - -/* cfront does not accept structure as an argument, pass pointer */ -void bb_backward_traversal(bb_traversal_args_t *args) -{ - basic_block_t *bb = args->bb; - func_t *func = args->func; - - bb->visited = func->visited; - - if (args->preorder_cb) - args->preorder_cb(func, bb); - - for (int i = 0; i < bb->prev_idx; i++) { - basic_block_t *pred = bb->prev[i].bb; - - if (!pred) - continue; - if (pred->visited < func->visited) { - args->bb = pred; - bb_backward_traversal(args); - } - } - - args->bb = bb; - - if (args->postorder_cb) - args->postorder_cb(func, bb); -} - -void bb_index_rpo(func_t *func, basic_block_t *bb) -{ - bb->rpo = func->bb_cnt++; -} - -void bb_reverse_index(func_t *func, basic_block_t *bb) -{ - bb->rpo = func->bb_cnt - bb->rpo; -} - -void bb_build_rpo(func_t *func, basic_block_t *bb) -{ - if (func->bbs == bb) - return; - - basic_block_t *prev = func->bbs; - basic_block_t *curr = prev->rpo_next; - for (; curr; curr = curr->rpo_next) { - if (curr->rpo < bb->rpo) { - prev = curr; - continue; - } - bb->rpo_next = curr; - prev->rpo_next = bb; - return; - } - - prev->rpo_next = bb; -} - -void build_rpo(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_index_rpo; - bb_forward_traversal(args); - - func->visited++; - args->postorder_cb = bb_reverse_index; - bb_forward_traversal(args); - - func->visited++; - args->postorder_cb = bb_build_rpo; - bb_forward_traversal(args); - } -} - -basic_block_t *intersect(basic_block_t *i, basic_block_t *j) -{ - while (i != j) { - while (i->rpo > j->rpo) - i = i->idom; - while (j->rpo > i->rpo) - j = j->idom; - } - return i; -} - -/* Find the immediate dominator of each basic block to build the dominator tree. - * - * Once the dominator tree is built, we can perform the more advanced - * optimiaztion according to the liveness analysis and the reachability - * analysis, e.g. common subexpression elimination, loop optimiaztion or dead - * code elimination . - * - * Reference: - * Cooper, Keith D.; Harvey, Timothy J.; Kennedy, Ken (2001). - * "A Simple, Fast Dominance Algorithm" - */ -void build_idom(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - bool changed; - - func->bbs->idom = func->bbs; - - do { - changed = false; - - for (basic_block_t *bb = func->bbs->rpo_next; bb; - bb = bb->rpo_next) { - /* pick one predecessor */ - basic_block_t *pred = NULL; - for (int i = 0; i < bb->prev_idx; i++) { - if (!bb->prev[i].bb) - continue; - if (!bb->prev[i].bb->idom) - continue; - pred = bb->prev[i].bb; - break; - } - - /* Reverse postorder puts a predecessor of every reachable block - * ahead of it, so one is normally settled by now. A block where - * none is cannot be given an immediate dominator yet; leaving - * it for a later round is what keeps the walk off an - * uninitialised pointer. - */ - if (!pred) - continue; - - for (int i = 0; i < bb->prev_idx; i++) { - if (!bb->prev[i].bb) - continue; - if (bb->prev[i].bb == pred) - continue; - if (bb->prev[i].bb->idom) - pred = intersect(bb->prev[i].bb, pred); - } - if (bb->idom != pred) { - bb->idom = pred; - changed = true; - } - } - } while (changed); - } -} - -bool dom_connect(basic_block_t *pred, basic_block_t *succ) -{ - if (succ->dom_prev) - return false; - - for (int i = 0; i < pred->dom_next_idx; i++) { - if (pred->dom_next[i] == succ) - return false; - } - - if (pred->dom_next_idx >= pred->dom_next_cap) - pred->dom_next = - arena_grow(BB_ARENA, (char *) pred->dom_next, &pred->dom_next_cap, - sizeof(basic_block_t *), 4, MAX_BB_DOM_SUCC, - "Too many children in dominator tree"); - - pred->dom_next[pred->dom_next_idx++] = succ; - succ->dom_prev = pred; - return true; -} - -void bb_build_dom(func_t *func, basic_block_t *bb) -{ - basic_block_t *curr = bb; - while (curr != func->bbs) { - if (!dom_connect(curr->idom, curr)) - break; - curr = curr->idom; - } -} - -void build_dom(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->preorder_cb = bb_build_dom; - bb_forward_traversal(args); - } -} - -/* Recompute the dominator tree over the CFG as it now stands. - * - * if_convert() only removes edges, and losing one can only strengthen - * dominance, so a stale tree stays conservative there. thread_const_branch() - * adds "pred -> target", a path that bypasses the join: a join that dominated - * the target no longer does. strength_reduce() and mark_loop_depth() both ask - * is_dominate() which edges close a loop, and against a stale tree a forward - * edge reads as a back edge -- mark_natural_loop() then walks predecessors out - * of the region it was meant to stay inside, since it has no stop at the header - * and relies on the header dominating the latch. - * - * The order has to be rebuilt along with the tree. Both passes keep the - * rpo_next chain consistent -- a block they drop comes out of it -- but - * dropping blocks and adding edges does not preserve reverse postorder, and - * build_idom() depends on it: it reaches each block once a predecessor already - * has an idom, and out of order it can meet one where none does. - */ -void rebuild_dom(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - - /* dom_connect() refuses a block that already has a parent, and - * bb_build_rpo() splices into the chain rather than replacing it, so - * both have to be taken down before they can be built again. The next - * link is read before it is cleared: clearing as the walk goes would - * cut the chain out from under it. - */ - basic_block_t *bb = func->bbs; - - while (bb) { - basic_block_t *next = bb->rpo_next; - - bb->idom = NULL; - bb->dom_prev = NULL; - bb->dom_next_idx = 0; - bb->rpo_next = NULL; - bb = next; - } - func->bb_cnt = 0; - } - - build_rpo(); - build_idom(); - build_dom(); -} - -void bb_build_df(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - int cnt = 0; - for (int i = 0; i < bb->prev_idx; i++) { - if (bb->prev[i].bb) - cnt++; - } - if (cnt <= 0) - return; - - for (int i = 0; i < bb->prev_idx; i++) { - if (bb->prev[i].bb) { - /* Walk up from the predecessor to this block's immediate dominator. - * The walk normally stops there, since a block's immediate - * dominator dominates all of its predecessors -- but an edge the - * dominator tree does not account for runs off the top instead, so - * stop at the root as well. Ending early only widens the frontier, - * which costs a phi that turns out to be trivial. - */ - for (basic_block_t *curr = bb->prev[i].bb; curr && curr != bb->idom; - curr = curr->idom) - bb_add_df(curr, bb); - } - } -} - -void build_df(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_build_df; - bb_forward_traversal(args); - } -} - -basic_block_t *reverse_intersect(basic_block_t *i, basic_block_t *j) -{ - while (i != j) { - while (i->rpo_r > j->rpo_r) - i = i->r_idom; - while (j->rpo_r > i->rpo_r) - j = j->r_idom; - } - return i; -} - -void build_r_idom(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - bool changed; - - func->exit->r_idom = func->exit; - - do { - changed = false; - - for (basic_block_t *bb = func->exit->rpo_r_next; bb; - bb = bb->rpo_r_next) { - /* pick one predecessor */ - basic_block_t *pred = NULL; - if (bb->next && bb->next->r_idom) { - pred = bb->next; - } else if (bb->else_ && bb->else_->r_idom) { - pred = bb->else_; - } else if (bb->then_ && bb->then_->r_idom) { - pred = bb->then_; - } - - /* The mirror of the rule in build_idom(): reverse postorder - * from the exit puts a successor of every block that reaches it - * ahead of that block, so one is normally settled by now. A - * block where none is cannot be given an immediate - * postdominator yet; leaving it for a later round is what keeps - * the walk off an uninitialised pointer. - */ - if (!pred) - continue; - - if (bb->next && bb->next != pred && bb->next->r_idom) - pred = reverse_intersect(bb->next, pred); - if (bb->else_ && bb->else_ != pred && bb->else_->r_idom) - pred = reverse_intersect(bb->else_, pred); - if (bb->then_ && bb->then_ != pred && bb->then_->r_idom) - pred = reverse_intersect(bb->then_, pred); - if (bb->r_idom != pred) { - bb->r_idom = pred; - changed = true; - } - } - } while (changed); - } -} - -bool rdom_connect(basic_block_t *pred, basic_block_t *succ) -{ - /* A block is attached to the reverse-dominator tree at most once, so the - * rdom_prev guard above already rules out a duplicate edge. - */ - if (succ->rdom_prev) - return false; - - /* rdom_count only ever guarded a MAX_BB_RDOM_SUCC ceiling on an array of - * reverse-dominator children that no longer exists -- nothing reads the - * count, and nothing walks those children -- so the field is gone. - */ - succ->rdom_prev = pred; - return true; -} - -void bb_build_rdom(func_t *func, basic_block_t *bb) -{ - if (!func->bbs) - return; - for (basic_block_t *curr = bb; curr != func->exit; curr = curr->r_idom) { - if (!rdom_connect(curr->r_idom, curr)) - break; - } -} - -void build_rdom(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->exit; - - func->visited++; - args->preorder_cb = bb_build_rdom; - bb_backward_traversal(args); - } -} - -void bb_build_rdf(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - int cnt = 0; - if (bb->next) - cnt++; - if (bb->then_) - cnt++; - if (bb->else_) - cnt++; - if (cnt <= 0) - return; - - /* As in bb_build_df(), the walk up the post-dominator tree stops at this - * block's immediate post-dominator, and at the root when an edge the tree - * does not account for takes it past that. - */ - if (bb->next) { - for (basic_block_t *curr = bb->next; curr && curr != bb->r_idom; - curr = curr->r_idom) - bb_add_rdf(curr, bb); - } - if (bb->else_) { - for (basic_block_t *curr = bb->else_; curr && curr != bb->r_idom; - curr = curr->r_idom) - bb_add_rdf(curr, bb); - } - if (bb->then_) { - for (basic_block_t *curr = bb->then_; curr && curr != bb->r_idom; - curr = curr->r_idom) - bb_add_rdf(curr, bb); - } -} - -void build_rdf(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->exit; - - func->visited++; - args->postorder_cb = bb_build_rdf; - bb_backward_traversal(args); - } -} - -void use_chain_add_tail(insn_t *i, var_t *var) -{ - use_chain_t *u = arena_calloc(INSN_ARENA, 1, sizeof(use_chain_t)); - - u->insn = i; - if (!var->users_head) - var->users_head = u; - else - var->users_tail->next = u; - u->prev = var->users_tail; - var->users_tail = u; -} - -void use_chain_build(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *i = bb->insn_list.head; i; i = i->next) { - if (i->rs1) - use_chain_add_tail(i, i->rs1); - if (i->rs2) - use_chain_add_tail(i, i->rs2); - } - } - } -} - -bool var_check_killed(const var_t *var, const basic_block_t *bb) -{ - for (int i = 0; i < bb->live_kill.size; i++) { - if (bb->live_kill.elements[i] == var) - return true; - } - return false; -} - -void bb_add_killed_var(basic_block_t *bb, var_t *var) -{ - var_list_add_var(&bb->live_kill, var); -} - -void var_add_killed_bb(var_t *var, basic_block_t *bb) -{ - bool found = false; - ref_block_t *ref; - for (ref = var->ref_block_list.head; ref; ref = ref->next) { - if (ref->bb == bb) { - found = true; - break; - } - } - if (found) - return; - - ref = arena_calloc(GENERAL_ARENA, 1, sizeof(ref_block_t)); - ref->bb = bb; - if (!var->ref_block_list.head) - var->ref_block_list.head = ref; - else - var->ref_block_list.tail->next = ref; - - var->ref_block_list.tail = ref; -} - -void fn_add_global(func_t *func, var_t *var) -{ - bool found = false; - symbol_t *sym; - for (sym = func->global_sym_list.head; sym; sym = sym->next) { - if (sym->var == var) { - found = true; - break; - } - } - if (found) - return; - - sym = arena_alloc_symbol(); - sym->var = var; - if (!func->global_sym_list.head) { - sym->index = 0; - func->global_sym_list.head = sym; - func->global_sym_list.tail = sym; - } else { - sym->index = func->global_sym_list.tail->index + 1; - func->global_sym_list.tail->next = sym; - func->global_sym_list.tail = sym; - } -} - -void bb_solve_globals(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->rs1) - if (!var_check_killed(insn->rs1, bb)) - fn_add_global(bb->belong_to, insn->rs1); - if (insn->rs2) - if (!var_check_killed(insn->rs2, bb)) - fn_add_global(bb->belong_to, insn->rs2); - if (insn->rd) { - bb_add_killed_var(bb, insn->rd); - var_add_killed_bb(insn->rd, bb); - } - } -} - -void solve_globals(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_solve_globals; - bb_forward_traversal(args); - } -} - -bool var_check_in_scope(const var_t *var, block_t *block) -{ - const func_t *func = block->func; - - while (block) { - /* Only the first 'size' entries hold a variable; the rest of the - * allocation was never written. Reading them compares against whatever - * the allocator left there, and a stray match puts a variable in scope - * that is not, which changes where phis are inserted and so what code - * comes out -- differently from one build to the next. - */ - for (int i = 0; i < block->locals.size; i++) { - if (var == block->locals.elements[i]) - return true; - } - block = block->parent; - } - - for (int i = 0; i < func->num_params; i++) { - if (&func->param_defs[i] == var) - return true; - } - - if (func->returns_aggregate && &func->sret_def == var) - return true; - - return false; -} - -bool insert_phi_insn(basic_block_t *bb, var_t *var) -{ - bool found = false; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if ((insn->opcode == OP_phi) && (insn->rd == var)) { - found = true; - break; - } - } - if (found) - return false; - - insn_t *head = bb->insn_list.head; - insn_t *n = arena_calloc(INSN_ARENA, 1, sizeof(insn_t)); - n->opcode = OP_phi; - n->rd = var; - n->rs1 = var; - n->rs2 = var; - if (!head) { - bb->insn_list.head = n; - bb->insn_list.tail = n; - } else { - head->prev = n; - n->next = head; - bb->insn_list.head = n; - } - return true; -} - -void solve_phi_insertion(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - for (symbol_t *sym = func->global_sym_list.head; sym; sym = sym->next) { - var_t *var = sym->var; - - basic_block_t *work_list[PHI_WORKLIST_SIZE]; - int work_list_idx = 0; - - for (ref_block_t *ref = var->ref_block_list.head; ref; - ref = ref->next) { - if (work_list_idx >= PHI_WORKLIST_SIZE - 1) - fatal("PHI worklist overflow"); - work_list[work_list_idx++] = ref->bb; - } - - for (int i = 0; i < work_list_idx; i++) { - basic_block_t *bb = work_list[i]; - for (int j = 0; j < bb->df_idx; j++) { - basic_block_t *df = bb->DF[j]; - if (!var_check_in_scope(var, df->scope)) - continue; - - bool is_decl = false; - for (symbol_t *s = df->symbol_list.head; s; s = s->next) { - if (s->var == var) { - is_decl = true; - break; - } - } - - if (is_decl) - continue; - - if (df == func->exit) - continue; - - /* Neither a global nor an address-taken variable is - * renamed, so there is no version for a phi to choose. - */ - if (var_in_memory(var)) - continue; - - if (insert_phi_insn(df, var)) { - bool found = false; - - /* Restrict phi insertion of ternary operation, and - * logical-and/or operation. - * - * The ternary and logical-and/or operations don't - * create new scope, so prevent temporary variable from - * propagating through the dominance tree. - */ - if (var->is_ternary_ret || var->is_logical_ret) - continue; - - for (int l = 0; l < work_list_idx; l++) { - if (work_list[l] == df) { - found = true; - break; - } - } - if (!found) { - if (work_list_idx >= PHI_WORKLIST_SIZE - 1) - fatal("PHI worklist overflow"); - work_list[work_list_idx++] = df; - } - } - } - } - } - } -} - -var_t *require_var(block_t *blk); - -/* A new variable holding @val, for use as an instruction operand. - * - * It has to be new. rename_var() gives every use reached by one definition the - * same var_t, and mark_const() stamps init_val onto that shared object, so - * rewriting an existing constant's value changes what every other use sees. The - * caller emits the OP_load_constant that defines it. - */ -var_t *new_const_var(block_t *scope, int val) -{ - var_t *var = require_var(scope); - - var->var_name = gen_name(); - var->is_const = true; - var->init_val = val; - var->init_val_hi = 0; - return var; -} -bool is_dominate(const basic_block_t *pred, basic_block_t *succ); - -/* The renaming state of @v, created on first use. */ -rename_t *var_rename(var_t *v) -{ - if (!v->rename) - v->rename = arena_calloc(BLOCK_ARENA, 1, sizeof(rename_t)); - return v->rename; -} - -/* Push a fresh subscript onto @base's renaming stack, growing it as needed. */ -void rename_stack_push(var_t *base, int sub) -{ - rename_t *r = var_rename(base); - if (r->stack_idx >= r->stack_cap) - r->stack = arena_grow(BLOCK_ARENA, (char *) r->stack, &r->stack_cap, - sizeof(int), 8, MAX_RENAME_STACK, - "Too many nested definitions of a variable"); - r->stack[r->stack_idx++] = sub; -} - -/* The one name an address-taken variable keeps for its whole life. - * - * A store through a pointer changes such a variable without any instruction - * naming it, and a read through one sees whatever its last assignment by name - * wrote. Separate SSA versions would give each assignment a place of its own, - * so neither kind of access would reach the other. Like a global, it keeps one - * var_t for every definition and use: the entry version for a parameter, under - * which the allocator finds its incoming register, and the declaration itself - * for anything else. - */ -var_t *addressed_name(var_t *v) -{ - var_t *entry = var_subscript0(v->base); - return entry ? entry : v->base; -} - -void new_name(block_t *block, var_t **var) -{ - var_t *v = *var; - if (!v->base) - v->base = v; - if (v->is_global) - return; - if (v->address_taken) { - var[0] = addressed_name(v); - return; - } - - rename_t *r = var_rename(v->base); - int i = r->counter++; - rename_stack_push(v->base, i); - var_t *vd = require_var(block); - memcpy(vd, *var, sizeof(var_t)); - var_reset_subscripts(vd); /* the copy shares nothing with its base */ - /* A fresh SSA definition has no physical-register residence. In particular, - * it must not inherit either half of a future wide pair from the version it - * was copied from. - */ - vd->phys_reg = -1; - vd->phys_reg_hi = -1; - vd->base = *var; - vd->subscript = i; - var_add_subscript(v, vd); - var[0] = vd; -} - -var_t *get_stack_top_subscript_var(var_t *var) -{ - rename_t *r = var_rename(var->base); - if (r->stack_idx < 1) - return var; /* fallback: use base when no prior definition */ - - int sub = r->stack[r->stack_idx - 1]; - for (int i = 0; i < var->base->subscripts_idx; i++) { - if (var->base->subscripts[i]->subscript == sub) - return var->base->subscripts[i]; - } - - fatal("Failed to find subscript variable on rename stack"); - return NULL; /* unreachable, but silences compiler warning */ -} - -void rename_var(var_t **var) -{ - var_t *v = *var; - if (!v->base) - v->base = v; - if (v->is_global) - return; - if (v->address_taken) { - var[0] = addressed_name(v); - return; - } - - var[0] = get_stack_top_subscript_var(*var); -} - -void pop_name(var_t *var) -{ - if (var_in_memory(var)) - return; - - /* Pop unconditionally, creating the state if the variable has none: the - * inline rename_t this replaced was always present, so a pop with nothing - * pushed drove stack_idx to -1, and the next push then landed one slot - * below the stack. Preserve that exactly. - */ - rename_t *r = var_rename(var->base); - r->stack_idx--; -} - -void append_phi_operand(insn_t *insn, var_t *var, basic_block_t *bb_from) -{ - phi_operand_t *op = arena_calloc(GENERAL_ARENA, 1, sizeof(phi_operand_t)); - op->from = bb_from; - op->var = get_stack_top_subscript_var(var); - - phi_operand_t *tail = insn->phi_ops; - if (tail) { - while (tail->next) - tail = tail->next; - tail->next = op; - } else - insn->phi_ops = op; -} - -void bb_solve_phi_params(basic_block_t *bb) -{ - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_phi) - new_name(bb->scope, &insn->rd); - else { - if (insn->rs1) - rename_var(&insn->rs1); - if (insn->rs2) - if (!insn->rs2->is_func) - rename_var(&insn->rs2); - if (insn->rd) - new_name(bb->scope, &insn->rd); - } - } - - if (bb->next) { - for (insn_t *insn = bb->next->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_phi) - append_phi_operand(insn, insn->rd, bb); - } - } - - if (bb->then_) { - for (insn_t *insn = bb->then_->insn_list.head; insn; - insn = insn->next) { - if (insn->opcode == OP_phi) - append_phi_operand(insn, insn->rd, bb); - } - } - - if (bb->else_) { - for (insn_t *insn = bb->else_->insn_list.head; insn; - insn = insn->next) { - if (insn->opcode == OP_phi) - append_phi_operand(insn, insn->rd, bb); - } - } - - for (int i = 0; i < bb->dom_next_idx; i++) - bb_solve_phi_params(bb->dom_next[i]); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_phi) - pop_name(insn->rd); - else if (insn->rd) - pop_name(insn->rd); - } -} - -void solve_phi_params(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - /* The aggregate return destination is an ABI parameter even though it - * is not a source-language parameter. Build its entry SSA version - * before the visible parameters. - */ - for (int i = -1; i < func->num_params; i++) { - /* FIXME: Direct argument renaming in SSA construction phase may - * interfere with later optimization passes - */ - var_t *var = require_var(func->bbs->scope); - if (i < 0 && !func->returns_aggregate) - continue; - var_t *base = i < 0 ? &func->sret_def : &func->param_defs[i]; - memcpy(var, base, sizeof(var_t)); - var_reset_subscripts(var); /* the copy shares nothing with base */ - var->phys_reg = -1; - var->phys_reg_hi = -1; - var->base = base; - var->subscript = 0; - - rename_t *r = var_rename(base); - rename_stack_push(base, r->counter++); - var_add_subscript(base, var); - } - - bb_solve_phi_params(func->bbs); - } -} - -void append_unwound_phi_insn(basic_block_t *bb, var_t *dest, var_t *rs) -{ - insn_t *n = arena_calloc(INSN_ARENA, 1, sizeof(insn_t)); - n->opcode = OP_unwound_phi; - n->rd = dest; - n->rs1 = rs; - n->belong_to = bb; - - insn_t *tail = bb->insn_list.tail; - if (!tail) { - bb->insn_list.head = n; - bb->insn_list.tail = n; - } else { - /* insert it before branch instruction */ - if (tail->opcode == OP_branch) { - if (tail->prev) { - tail->prev->next = n; - n->prev = tail->prev; - } else - bb->insn_list.head = n; - - n->next = tail; - tail->prev = n; - } else { - tail->next = n; - bb->insn_list.tail = n; - } - } -} - -void bb_unwind_phi(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - insn_t *insn; - for (insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_phi) - break; - - for (phi_operand_t *operand = insn->phi_ops; operand; - operand = operand->next) - append_unwound_phi_insn(operand->from, insn->rd, operand->var); - } - - bb->insn_list.head = insn; - if (!insn) - bb->insn_list.tail = NULL; - else - insn->prev = NULL; -} - -int loop_scan_gen; - -/* Raise the depth of every block in the natural loop that the edge from @latch - * back to @header closes: the header, and everything that can reach the latch - * without leaving the loop. - */ -bool mark_natural_loop(basic_block_t *header, basic_block_t *latch) -{ - basic_block_t *stack[MAX_LOOP_WALK]; - int sp = 0; - - loop_scan_gen++; - header->loop_mark = loop_scan_gen; - header->loop_depth++; - - if (latch != header) { - latch->loop_mark = loop_scan_gen; - latch->loop_depth++; - stack[sp++] = latch; - } - - while (sp) { - basic_block_t *bb = stack[--sp]; - - for (int i = 0; i < bb->prev_idx; i++) { - basic_block_t *p = bb->prev[i].bb; - - if (!p || p->loop_mark == loop_scan_gen) - continue; - - /* Out of room to widen. Marking this block and then not walking - * through it would leave the rest of the loop unmarked while - * looking marked, and a reader of loop_mark would take blocks - * inside the loop for blocks outside it. Report the whole answer as - * unusable instead. - */ - if (sp >= MAX_LOOP_WALK) - return false; - p->loop_mark = loop_scan_gen; - p->loop_depth++; - stack[sp++] = p; - } - } - return true; -} - -/* Count the loops enclosing each block of @func. - * - * A loop is closed by an edge to a block that dominates its source. Position - * along the rpo_next chain cannot stand in for that: the passes that splice - * blocks out of the chain leave an if's arms sitting after the block they - * rejoin, and every one of those edges would read as a loop. - */ -void mark_loop_depth(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->loop_depth = 0; - bb->loop_mark = 0; - } - - - for (basic_block_t *p = func->bbs; p; p = p->rpo_next) { - basic_block_t *succ[3]; - - succ[0] = p->next; - succ[1] = p->then_; - succ[2] = p->else_; - - for (int k = 0; k < 3; k++) { - if (!succ[k]) - continue; - if (succ[k] != p && !is_dominate(succ[k], p)) - continue; - - /* A walk that ran out of room stopped partway, leaving the depths - * it had already raised standing over a region it never finished - * measuring. Those feed the weights pin_registers() compares, so a - * half-counted loop would outbid a whole one; drop the tally for - * the function instead and let every block weigh the same. - */ - if (!mark_natural_loop(succ[k], p)) { - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - bb->loop_depth = 0; - bb->loop_weight = 1; - } - return; - } - } - } - - /* What one naming in each block is worth, so that the tally in - * pin_registers() reads it rather than deriving it per operand. - */ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - int w = 1; - - for (int d = 0; d < bb->loop_depth && d < MAX_WEIGHTED_LOOP_DEPTH; d++) - w = w * LOOP_USE_WEIGHT; - bb->loop_weight = w; - } -} - -/* Put @insn into @bb straight after @after, or at the head when it is NULL. */ -void bb_insert_after(basic_block_t *bb, insn_t *after, insn_t *insn) -{ - insn->belong_to = bb; - insn->prev = after; - if (after) { - insn->next = after->next; - after->next = insn; - } else { - insn->next = bb->insn_list.head; - bb->insn_list.head = insn; - } - if (insn->next) - insn->next->prev = insn; - else - bb->insn_list.tail = insn; -} - -/* Append @insn to @bb, detaching it from wherever it was. */ -void bb_append_insn(basic_block_t *bb, insn_t *insn) -{ - bb_insert_after(bb, bb->insn_list.tail, insn); -} - -/* A fresh instruction naming @rd, @rs1 and @rs2, not yet in any block. The - * allocation is zeroed, so every other field starts empty -- which is what the - * passes that build instructions want, and one less thing for each of them to - * remember. - */ -insn_t *new_insn(opcode_t op, var_t *rd, var_t *rs1, var_t *rs2) -{ - insn_t *insn = arena_calloc(INSN_ARENA, 1, sizeof(insn_t)); - - insn->opcode = op; - insn->rd = rd; - insn->rs1 = rs1; - insn->rs2 = rs2; - return insn; -} - -/* Whether @insn computes a value with no side effect and no way to fault, so - * running it on a path that would not have reached it changes nothing. - */ -bool insn_is_speculatable(const insn_t *insn) -{ - /* Running an arm unconditionally would read a volatile object on a path - * that never evaluates it, and that read is a side effect. - */ - if (insn_reads_volatile(insn)) - return false; - - switch (insn->opcode) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_negate: - case OP_bit_not: - case OP_log_not: - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - case OP_assign: - case OP_load_constant: - return true; - default: - /* Division can trap, a read can fault, a write or a call is a side - * effect, and anything else is not worth reasoning about here. - */ - return false; - } -} - -/* Walk the arm starting at @arm, gathering the blocks it runs before control - * rejoins. - * - * An arm is not one block: the copy carrying its value to the join is appended - * to whichever block immediately precedes the join, which is separate from the - * one holding the arm's computation. The walk follows single-entry, single-exit - * blocks and stops at the first block something else can also reach, which is - * the join. - */ -basic_block_t *if_arm_chain(basic_block_t *arm, - basic_block_t **chain, - int *len, - int *count) -{ - basic_block_t *bb = arm; - - *len = 0; - *count = 0; - - while (bb) { - if (bb->then_ || bb->else_ || !bb->next) - return NULL; - if (bb_pred_count(bb) != 1) - return NULL; - if (*len >= MAX_IF_ARM_BLOCKS) - return NULL; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode == OP_unwound_phi) - continue; - if (!insn_is_speculatable(insn)) - return NULL; - - /* Writing a global or an address-taken variable is a store, and a - * store is a side effect however plain the arithmetic producing it - * looks. - */ - if (insn->rd && var_in_memory(insn->rd)) - return NULL; - *count = *count + 1; - } - - chain[*len] = bb; - *len = *len + 1; - - if (bb_pred_count(bb->next) != 1) - return bb->next; - bb = bb->next; - } - return NULL; -} - -/* The single copy an arm hands to the join, or NULL when it hands over more or - * fewer than one. - */ -insn_t *if_chain_phi(basic_block_t **chain, int len) -{ - insn_t *phi = NULL; - - for (int i = 0; i < len; i++) { - for (insn_t *insn = chain[i]->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_unwound_phi) - continue; - if (phi) - return NULL; - phi = insn; - } - } - return phi; -} - -/* Whether any instruction in @b reads a value defined in @a. */ -bool if_arm_reads_other(basic_block_t **a, - int alen, - basic_block_t **b, - int blen) -{ - for (int i = 0; i < alen; i++) { - for (insn_t *def = a[i]->insn_list.head; def; def = def->next) { - if (!def->rd) - continue; - for (int j = 0; j < blen; j++) { - for (insn_t *use = b[j]->insn_list.head; use; use = use->next) { - if (use->rs1 == def->rd || use->rs2 == def->rd || - use->rs3 == def->rd) - return true; - } - } - } - } - return false; -} - -/* Whether @var is an integer scalar a 32-bit target keeps in a register pair. - * The widening pass below, the allocator and the backends all ask this. - */ -bool var_needs_register_pair(const var_t *var) -{ - return PTR_SIZE < 8 && var && !var->ptr_level && !var->is_func && - !var->array_size && var->type && var->type->size == 8 && - !is_record_type(var->type); -} - -/* Flatten "if (c) x = A; else x = B;" into both computations and a select. - * - * A branch the hardware cannot predict costs far more than the arm it skips: - * the CRC loop in the benchmark suite mispredicts 17% of 283 million branches - * and pays about a thousand million cycles for it, most of its run time. - * Running both arms and selecting between the results has nothing to - * mispredict, and neither arm may write memory, call anything or divide, so - * running the one that would have been skipped costs only itself. - */ -bool if_convert_bb(func_t *func, basic_block_t *bb) -{ - basic_block_t *t = bb->then_; - basic_block_t *e = bb->else_; - basic_block_t *t_chain[MAX_IF_ARM_BLOCKS]; - basic_block_t *e_chain[MAX_IF_ARM_BLOCKS]; - int t_len = 0; - int e_len = 0; - int tn = 0; - int en = 0; - - if (!t || !e || t == e) - return false; - - basic_block_t *join = if_arm_chain(t, t_chain, &t_len, &tn); - basic_block_t *e_join = if_arm_chain(e, e_chain, &e_len, &en); - - if (!join || join != e_join) - return false; - - /* Flattening removes both arms, so the join must be reached from them and - * nothing else. - */ - if (bb_pred_count(join) != 2) - return false; - - insn_t *t_phi = if_chain_phi(t_chain, t_len); - insn_t *e_phi = if_chain_phi(e_chain, e_len); - if (!t_phi || !e_phi || t_phi->rd != e_phi->rd) - return false; - if (var_in_memory(t_phi->rd)) - return false; - /* Both arms handing over the same value makes the select pointless. */ - if (t_phi->rs1 == e_phi->rs1) - return false; - if (tn + en > MAX_SPECULATED_INSNS) - return false; - - insn_t *br = bb->insn_list.tail; - if (!br || br->opcode != OP_branch || !br->rs1) - return false; - - /* The arms run one after the other now, so whichever computes a value the - * other reads has to go first -- common subexpression elimination leaves - * exactly that. If each reads something the other defines, no order works. - */ - bool e_needs_t = if_arm_reads_other(t_chain, t_len, e_chain, e_len); - bool t_needs_e = if_arm_reads_other(e_chain, e_len, t_chain, t_len); - - if (e_needs_t && t_needs_e) - return false; - - var_t *cond = br->rs1; - - bb->insn_list.tail = br->prev; - if (br->prev) - br->prev->next = NULL; - else - bb->insn_list.head = NULL; - - insn_t *next; - - for (int step = 0; step < 2; step++) { - bool take_t = e_needs_t ? step == 0 : step == 1; - basic_block_t **chain = take_t ? t_chain : e_chain; - const insn_t *skip = take_t ? t_phi : e_phi; - int len = take_t ? t_len : e_len; - - for (int i = 0; i < len; i++) { - for (insn_t *insn = chain[i]->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn == skip) - continue; - - /* Whatever the arm computes is a version of the variable the - * select writes as often as not, and that variable's pinned - * register still has to carry the value the arms read. - */ - if (insn->rd) - insn->rd->in_select_arm = true; - bb_append_insn(bb, insn); - } - chain[i]->insn_list.head = NULL; - chain[i]->insn_list.tail = NULL; - } - } - - /* One instruction naming all three inputs: split into a copy and a - * read-modify move, the allocator settles their registers separately and - * need not agree on one. - */ - t_phi->opcode = OP_cmov; - t_phi->rs2 = cond; - t_phi->rs3 = e_phi->rs1; - bb_append_insn(bb, t_phi); - - /* Rewire: the test now falls straight into the join. */ - bb_disconnect(t_chain[t_len - 1], join); - bb_disconnect(e_chain[e_len - 1], join); - bb_disconnect(bb, t); - bb_disconnect(bb, e); - bb_connect(bb, join, NEXT); - - /* Take the arms out of the order the rest of the compiler walks. */ - for (basic_block_t *p = func->bbs; p; p = p->rpo_next) { - bool drop = true; - - while (drop && p->rpo_next) { - drop = false; - for (int i = 0; i < t_len; i++) { - if (p->rpo_next == t_chain[i]) - drop = true; - } - for (int i = 0; i < e_len; i++) { - if (p->rpo_next == e_chain[i]) - drop = true; - } - if (drop) - p->rpo_next = p->rpo_next->rpo_next; - } - } - return true; -} - -/* The generation stamping the loop strength_reduce() is looking at, so that - * var_read_by() can be asked about that loop alone. - */ -int sr_gen; - -/* Whether anything in @func reads @var, ignoring the @nskip instructions in - * @skip. - * - * With @loop_only set, only the blocks the stamped loop covers are searched: a - * reader outside it does not keep a literal alive inside, because the allocator - * materialises one wherever the value is wanted. - */ -bool var_read_by(func_t *func, - const var_t *var, - insn_t **skip, - int nskip, - bool loop_only) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (loop_only && bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - bool skipped = false; - - for (int i = 0; i < nskip; i++) { - if (skip[i] == insn) - skipped = true; - } - if (skipped) - continue; - if (insn->rs1 == var || insn->rs2 == var || insn->rs3 == var) - return true; - } - } - return false; -} - -/* Detach @insn from the block holding it. */ -void bb_remove_insn(basic_block_t *bb, insn_t *insn) -{ - if (insn->prev) - insn->prev->next = insn->next; - else - bb->insn_list.head = insn->next; - if (insn->next) - insn->next->prev = insn->prev; - else - bb->insn_list.tail = insn->prev; - insn->prev = NULL; - insn->next = NULL; -} - -/* Send the arms of "if (a && b)" straight where they are going. - * - * A short-circuit condition becomes a block per arm, each writing 0 or 1 into - * one temporary, and a join that does nothing but branch on it. The value is - * born a constant and dies at that branch, so the arm already knows which way - * it goes: writing the answer to a stack slot, reading it back and testing it - * is five instructions to reach a label the arm could have jumped to. Every - * loop guarded by "i < n && p[i]" pays them on each iteration. - */ -bool thread_const_branch(func_t *func, basic_block_t *join) -{ - insn_t *br = join->insn_list.head; - - if (!join->then_ || !join->else_) - return false; - - /* Only a block that does nothing else: anything before the branch would be - * skipped, and a phi copy left in it belongs to a successor. - */ - if (!br || br->next || br->opcode != OP_branch || !br->rs1) - return false; - - var_t *cond = br->rs1; - bool threaded = false; - - /* Two passes: the first only looks, because the reader scan between them - * walks the whole function and is not worth paying for a join no - * predecessor can be threaded to anyway. - */ - for (int pass = 0; pass < 2; pass++) { - for (int i = 0; i < join->prev_idx; i++) { - basic_block_t *pred = join->prev[i].bb; - - if (!pred || pred == join) - continue; - - /* A predecessor that also goes somewhere else keeps a branch of its - * own, and rewiring one of its edges would need that branch - * rewritten. - */ - if (pred->then_ || pred->else_ || pred->next != join) - continue; - - insn_t *def = pred->insn_list.tail; - - if (!def || def->rd != cond) - continue; - if (def->opcode != OP_unwound_phi && def->opcode != OP_assign) - continue; - if (!def->rs1 || !def->rs1->is_const) - continue; - - if (!pass) { - threaded = true; - break; - } - - /* A wide constant with a zero low word, such as 1ULL << 32, is - * still true. - */ - basic_block_t *target = def->rs1->init_val || def->rs1->init_val_hi - ? join->then_ - : join->else_; - - bb_remove_insn(pred, def); - bb_disconnect(pred, join); - bb_connect(pred, target, NEXT); - } - - if (!threaded) - return false; - if (!pass) { - /* The condition must die here: threading removes the writes, so - * anything else reading it would read nothing. - */ - insn_t *skip[1]; - - skip[0] = br; - if (var_read_by(func, cond, skip, 1, false)) - return false; - } - } - - /* Nothing reaches the join any more, so take it out of the order the rest - * of the compiler walks -- and with it the branch, whose condition no - * longer has a definition on any path. - */ - if (!bb_pred_count(join)) { - bb_disconnect(join, join->then_); - bb_disconnect(join, join->else_); - for (basic_block_t *p = func->bbs; p && p->rpo_next; p = p->rpo_next) { - if (p->rpo_next == join) { - p->rpo_next = join->rpo_next; - break; - } - } - } - return true; -} - -void thread_const_branches(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - const insn_t *br = bb->insn_list.head; - - if (!br || br->next || br->opcode != OP_branch || !br->rs1) - continue; - thread_const_branch(func, bb); - } - } -} - -/* Copy a small function into its callers. - * - * A call to a four-line helper costs more than the helper: the arguments go - * into the argument registers, everything live crosses a call boundary and so - * goes to the frame, and the callee builds and tears down a frame of its own. - * The benchmark suite's "calls" case spends three instructions on that for - * every one it spends computing. Copying the body in removes all of it and lets - * the optimizer see the caller and the callee together. - * - * Only a body with no control flow of its own is copied, which keeps the - * transformation to splicing one instruction list into another. - */ -var_t *inline_from[MAX_INLINE_VARS]; -var_t *inline_to[MAX_INLINE_VARS]; -int inline_map_n; - -/* The caller's stand-in for the callee's @var: the argument for a parameter, a - * fresh variable for anything the body computes, and the variable itself for - * anything shared. - */ -var_t *inline_lookup(var_t *var, block_t *scope) -{ - /* A global is shared, not copied. */ - if (!var || var->is_global) - return var; - - for (int i = 0; i < inline_map_n; i++) { - if (inline_from[i] == var) - return inline_to[i]; - - /* Parameters are mapped by the variable they are versions of, so that a - * body naming a later version still finds the argument. - */ - if (inline_from[i] && inline_from[i] == var->base) - return inline_to[i]; - } - - if (inline_map_n >= MAX_INLINE_VARS) - return NULL; - - var_t *copy = require_var(scope); - - copy->var_name = gen_name(); - copy->type = var->type; - copy->ptr_level = var->ptr_level; - copy->is_const = var->is_const; - copy->init_val = var->init_val; - copy->init_val_hi = var->init_val_hi; - inline_from[inline_map_n] = var; - inline_to[inline_map_n] = copy; - inline_map_n++; - return copy; -} - -/* Whether @func is small and straight-line enough to copy into its callers. - * - * A function is a chain of blocks even when its source has no control flow: the - * entry block holds the declarations and falls into the body. Any block that - * branches, or that something else can reach, ends the chain and makes the - * function too complicated to splice into an instruction list. - */ -int inline_round; - -/* Whether cloning @func's body would need more renaming entries than the map - * holds. - * - * inline_lookup() returns NULL once it is full, and inline_call_at() has - * already removed the push/call/retval sequence by the time cloning starts, so - * a failure there cannot be declined -- it leaves instructions carrying null - * operands behind, which the register allocator dereferences. Counting before - * anything is touched is what keeps that from needing to be undone. - * - * The count mirrors inline_lookup()'s matching exactly: a global is shared - * rather than copied, a parameter is found through the variable it is a version - * of, and every other version gets an entry of its own. - */ -bool inline_map_fits(func_t *func) -{ - var_t *seen[MAX_INLINE_VARS]; - int n = 0; - - for (int q = 0; q < func->num_params; q++) { - if (n >= MAX_INLINE_VARS) - return false; - seen[n++] = &func->param_defs[q]; - } - - for (basic_block_t *bb = func->bbs; bb; bb = bb->next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *ops[4]; - - ops[0] = insn->rd; - ops[1] = insn->rs1; - ops[2] = insn->rs2; - ops[3] = insn->rs3; - - for (int k = 0; k < 4; k++) { - var_t *var = ops[k]; - - if (!var || var->is_global) - continue; - - bool found = false; - - for (int i = 0; i < n; i++) { - if (seen[i] == var || seen[i] == var->base) - found = true; - } - if (found) - continue; - if (n >= MAX_INLINE_VARS) - return false; - seen[n++] = var; - } - } - } - return true; -} - -bool func_is_inlinable(func_t *func) -{ - int n = 0, blocks = 0; - insn_t *last = NULL; - - /* The answer only changes when a round changes the body. */ - if (func->inline_gen == inline_round) - return func->inline_ok; - func->inline_gen = inline_round; - func->inline_ok = false; - func->inline_ret = NULL; - - if (!func->bbs || func->va_args) - return false; - if (func->num_params > MAX_ARGS_IN_REG) - return false; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->next) { - if (bb->then_ || bb->else_) - return false; - if (bb != func->bbs && bb_pred_count(bb) != 1) - return false; - - /* The single-predecessor test above is skipped for the entry block, so - * a "next" chain that came back round to it would walk for ever. A - * chain longer than the instruction budget cannot be inlinable in any - * case, so stopping there costs nothing and bounds the walk. - */ - if (++blocks > MAX_INLINE_INSNS) - return false; - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - n++; - if (n > MAX_INLINE_INSNS) - return false; - last = insn; - if (insn->opcode == OP_return) - continue; - - /* A call of its own would have to be copied as a call, which is - * what the copying is meant to remove; the pass runs again once - * that callee has been copied in, and by then this body qualifies. - */ - if (!insn_is_speculatable(insn)) - return false; - if (insn->rd && var_in_memory(insn->rd)) - return false; - - /* Writing a parameter would mean the copy assigns to the caller's - * own variable, since a parameter maps onto the argument. - */ - for (int q = 0; q < func->num_params; q++) { - if (insn->rd && insn->rd->base == &func->param_defs[q]) - return false; - } - } - } - - if (!last || last->opcode != OP_return) - return false; - if (!inline_map_fits(func)) - return false; - - func->inline_ok = true; - func->inline_ret = last; - return true; -} - -/* Copy @src into @bb after @after, renaming what it names. */ -insn_t *inline_clone(basic_block_t *bb, - insn_t *after, - insn_t *src, - block_t *scope) -{ - insn_t *copy = new_insn(src->opcode, inline_lookup(src->rd, scope), - inline_lookup(src->rs1, scope), - inline_lookup(src->rs2, scope)); - - copy->rs3 = inline_lookup(src->rs3, scope); - copy->sz = src->sz; - copy->str = src->str; - bb_insert_after(bb, after, copy); - return copy; -} - -/* Replace the push/call/retval sequence ending at @call with the callee's body. - * - * Returns the instruction to carry on scanning from, which is NULL when the - * copy lands at the end of the block; *@done says whether the call was replaced - * at all. The two have to be reported separately -- a NULL return read as "left - * alone" stopped the round from being counted as progress and left the rest of - * the block unscanned. - * - * @done is an int and not a bool on purpose. A _Bool is one byte, and writing - * one byte through a pointer into the caller's slot -- which is pointer-sized - * -- leaves the bytes above it holding whatever was there before, so the read - * back can be true when false was written. Compiling shecc with itself for - * RISC-V is where that shows: the copy of the compiler so built inlines less - * than the one gcc built, and stage1 and stage2 stop matching. Widening the - * flag sidesteps it; the underlying narrow-store bug is still there for any - * other "bool *" to find. - */ -insn_t *inline_call_at(basic_block_t *bb, insn_t *call, int *done) -{ - func_t *callee = find_func(call->str); - - *done = 0; - if (!callee || !func_is_inlinable(callee)) - return NULL; - - /* The arguments are the pushes immediately before the call. */ - var_t *args[MAX_PARAMS]; - insn_t *first = call; - int argc = 0; - - for (insn_t *p = call->prev; p && p->opcode == OP_push; p = p->prev) { - first = p; - argc++; - } - if (argc != callee->num_params || argc > MAX_PARAMS) - return NULL; - - /* A call nested in another call's argument list -- "f(g(x), y)" -- has the - * outer call's pushes already standing before it. The register allocator - * hands those the argument registers as it meets them, so anything spliced - * in between would overwrite arguments the outer call is still waiting to - * make. - */ - for (insn_t *p = first->prev; p; p = p->prev) { - if (p->opcode == OP_call || p->opcode == OP_indirect) - break; - if (p->opcode == OP_push) - return NULL; - } - - insn_t *walk = first; - for (int i = 0; i < argc; i++) { - args[i] = walk->rs1; - walk = walk->next; - } - - insn_t *last = call; - var_t *dest = NULL; - - if (call->next && call->next->opcode == OP_func_ret) { - last = call->next; - dest = last->rd; - } - - insn_t *ret = callee->inline_ret; - - /* A value the caller wants and the callee does not produce, or the other - * way round, would need a fabricated definition. - */ - if (!dest != !ret->rs1) - return NULL; - - inline_map_n = 0; - for (int i = 0; i < argc; i++) { - if (inline_map_n >= MAX_INLINE_VARS) - return NULL; - inline_from[inline_map_n] = &callee->param_defs[i]; - inline_to[inline_map_n] = args[i]; - inline_map_n++; - } - - block_t *scope = bb->scope; - insn_t *after = first->prev; - insn_t *next; - - for (insn_t *dead = first; dead; dead = next) { - next = dead->next; - bb_remove_insn(bb, dead); - if (dead == last) - break; - } - - for (basic_block_t *src_bb = callee->bbs; src_bb; src_bb = src_bb->next) { - for (insn_t *src = src_bb->insn_list.head; src; src = src->next) { - if (src == ret) - break; - after = inline_clone(bb, after, src, scope); - } - } - - if (dest) { - insn_t *copy = - new_insn(OP_assign, dest, inline_lookup(ret->rs1, scope), NULL); - - bb_insert_after(bb, after, copy); - after = copy; - } - - *done = 1; - if (after) - return after->next; - return bb->insn_list.head; -} - -void inline_calls(void) -{ - for (int round = 0; round < MAX_INLINE_ROUNDS; round++) { - bool changed = false; - - inline_round++; - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - insn_t *next; - - for (insn_t *insn = bb->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn->opcode != OP_call || !insn->str) - continue; - - int done; - insn_t *resume = inline_call_at(bb, insn, &done); - - if (done) { - next = resume; - changed = true; - - /* The body just grew, so the verdict cached for it this - * round no longer describes it -- and losing a call may - * be exactly what makes it copyable. - */ - func->inline_gen = 0; - } - } - } - } - if (!changed) - break; - } -} - -/* Walk an array with a pointer instead of recomputing its address. - * - * "a[i * 64 + k]" inside the k loop spends four instructions turning k into an - * address, every one of which changes by the same amount from one iteration to - * the next. Computing the first address before the loop and adding that amount - * at the bottom leaves the body with the access alone -- matmul's innermost - * loop spent six of its sixteen instructions on the two subscripts. - * - * Unlike hoisting a loop-invariant value, this pays even when the result has to - * live on the frame: the loop trades a four-instruction recomputation for one - * addition, where hoisting trades a recomputation for a reload. The variable - * the loop counts with, where the chain walk stops: its value before the loop - * is what the first address is computed from. - */ -var_t *sr_iv; - -var_t *sr_base(var_t *var) -{ - if (!var) - return NULL; - return var->base ? var->base : var; -} - -/* Whether @var is written inside the loop being examined. A constant never is, - * wherever its materialisation sits: the allocator emits one wherever the value - * is wanted. - */ -bool sr_varies(var_t *var) -{ - const var_t *base; - - if (!var || var->is_const) - return false; - base = sr_base(var); - return base && base->loop_stamp == sr_gen; -} - -/* How much @var moves per iteration, into *step. Reports false when that is not - * known -- which for a variable the loop writes means "not yet derived". - */ -bool sr_step_of(var_t *var, int *step) -{ - const var_t *base; - - if (!var) - return false; - if (var->is_const) { - *step = 0; - return true; - } - base = sr_base(var); - if (!base) - return false; - if (base->iv_gen == sr_gen) { - *step = base->iv_step; - return true; - } - if (base->loop_stamp != sr_gen) { - *step = 0; /* nothing in the loop writes it */ - return true; - } - return false; -} - -void sr_set_step(var_t *var, int step) -{ - var_t *base = sr_base(var); - - if (!base) - return; - base->iv_gen = sr_gen; - base->iv_step = step; -} - -/* Whether @insn is one the pass is willing to lift out of a loop: it computes a - * value from its operands, touches no memory and faults on nothing. Taking an - * array's address qualifies -- that address is the same on every iteration -- - * and a literal does not, because the allocator materialises one wherever it is - * wanted and moving it would only lengthen a live range. - */ -bool sr_movable(const insn_t *insn) -{ - switch (insn->opcode) { - case OP_add: - case OP_sub: - case OP_mul: - case OP_lshift: - case OP_rshift: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - case OP_assign: - case OP_address_of: - case OP_global_address_of: - case OP_load_data_address: - case OP_load_rodata_address: - return true; - default: - return false; - } -} - -/* The instruction inside the loop that defines @var, or NULL. */ -insn_t *sr_def_of(func_t *func, const var_t *var) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->rd == var) - return insn; - } - } - return NULL; -} - -/* Find the variable the loop counts with, and how far it counts each time. - * - * It is the one whose only write in the loop adds a literal to itself, which - * after phi unwinding is a pair: a temporary holding the sum, and a copy of - * that temporary back into the variable. - */ -var_t *sr_basic_iv(func_t *func, basic_block_t *latch, int *step) -{ - var_t *found = NULL; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_assign && insn->opcode != OP_unwound_phi) - continue; - if (!insn->rd || !insn->rs1) - continue; - - var_t *k = sr_base(insn->rd); - insn_t *sum = sr_def_of(func, insn->rs1); - - if (!k || !sum) - continue; - if (sum->opcode != OP_add && sum->opcode != OP_sub) - continue; - if (!sum->rs1 || !sum->rs2) - continue; - - const var_t *addend = NULL; - var_t *carried = NULL; - int sign = 1; - - if (sr_base(sum->rs1) == k && sum->rs2->is_const) { - addend = sum->rs2; - carried = sum->rs1; - if (sum->opcode == OP_sub) - sign = -1; /* counting down */ - } else if (sum->opcode == OP_add && sr_base(sum->rs2) == k && - sum->rs1->is_const) { - addend = sum->rs1; - carried = sum->rs2; - } - if (!addend || !addend->init_val) - continue; - - /* The literal has to be added to the variable itself, not to - * something computed from it: a temporary made from a variable - * carries that variable as its base, so "g = g * 2 + 1" matches the - * shape as readily as "i = i + 1" and would be taken for a counter - * stepping by one. The value flowing in is the one the loop either - * carries round in a phi or never writes. - */ - insn_t *src = sr_def_of(func, carried); - - if (src) { - if (src->opcode != OP_assign && src->opcode != OP_unwound_phi) - continue; - if (sr_base(src->rs1) != k) - continue; - } - /* Two counters would each need their own analysis. */ - if (found && found != k) - return NULL; - found = k; - *step = sign * addend->init_val; - } - } - if (!found) - return NULL; - - /* One variable the loop writes twice is not a counter: "i++" in one arm and - * "i--" in another moves it by neither step, and an address derived from it - * would advance by a fixed amount that matches neither. - */ - int writes = 0; - basic_block_t *wblk = NULL; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - /* Only a copy back into the variable counts. The arithmetic that - * produces the new value writes a temporary, and a temporary made - * from a variable carries that variable as its base. Only a write - * of the variable itself counts. The arithmetic that produces the - * new value writes a temporary, and a temporary made from a - * variable carries that variable as its base -- but a select names - * the variable, and one writing the counter means the counter does - * not advance on every trip round. - */ - if (insn->opcode != OP_assign && insn->opcode != OP_unwound_phi && - insn->opcode != OP_cmov) - continue; - if (!insn->rd || sr_base(insn->rd) != found) - continue; - - /* A phi copying the variable to itself carries it around the loop - * rather than giving it a new value. - */ - if (insn->opcode != OP_cmov && sr_base(insn->rs1) == found) - continue; - writes++; - wblk = bb; - } - } - if (writes != 1) - return NULL; - - /* The write has to happen on every trip round, because the pointer derived - * from the counter is advanced on every trip round. A "continue" that jumps - * over the sole increment leaves the two disagreeing, and the loop reads - * one element too far from then on. - */ - if (wblk != latch && !is_dominate(wblk, latch)) - return NULL; - return found; -} - -/* Derive how far every value in the loop moves, from the counter outwards. - * Several rounds because a chain is only as derivable as its operands. - */ -void sr_derive_steps(func_t *func) -{ - for (int round = 0; round < MAX_IV_ROUNDS; round++) { - bool changed = false; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - int a = 0, b = 0, s = 0; - - if (!insn->rd) - continue; - - const var_t *base = sr_base(insn->rd); - - if (!base || base->iv_gen == sr_gen) - continue; - if (base->def_cnt != 1) - continue; - if (!sr_step_of(insn->rs1, &a)) - continue; - - switch (insn->opcode) { - case OP_assign: - s = a; - break; - case OP_add: - if (!sr_step_of(insn->rs2, &b)) - continue; - s = a + b; - break; - case OP_sub: - if (!sr_step_of(insn->rs2, &b)) - continue; - s = a - b; - break; - case OP_mul: - if (!insn->rs2 || !insn->rs2->is_const) - continue; - s = a * insn->rs2->init_val; - break; - case OP_lshift: - if (!insn->rs2 || !insn->rs2->is_const) - continue; - if (insn->rs2->init_val < 0 || insn->rs2->init_val > 30) - continue; - s = a << insn->rs2->init_val; - break; - default: - continue; - } - sr_set_step(insn->rd, s); - changed = true; - } - } - if (!changed) - return; - } -} - -/* Gather, into @chain, the instructions of the loop that compute @var, in the - * order they appear. Reports false when any of them is shared, has a side - * effect, or the chain is longer than the pass is willing to move. - */ -bool sr_collect_chain(func_t *func, var_t *var, insn_t **chain, int *len) -{ - if (sr_base(var) == sr_iv) - return true; /* the counter itself: the preheader has its first value */ - - insn_t *def = sr_def_of(func, var); - - if (!def) - return false; - for (int i = 0; i < *len; i++) { - if (chain[i] == def) - return true; /* already collected */ - } - if (*len >= MAX_IV_CHAIN) - return false; - if (!sr_movable(def)) - return false; - - const var_t *base = sr_base(def->rd); - - if (!base || base->def_cnt != 1) - return false; - if (var_in_memory(base)) - return false; - - chain[*len] = def; - *len = *len + 1; - - var_t *ops[3]; - - ops[0] = def->rs1; - ops[1] = def->rs2; - ops[2] = def->rs3; - for (int k = 0; k < 3; k++) { - if (!sr_varies(ops[k])) - continue; /* a constant or something the loop never writes */ - if (!sr_collect_chain(func, ops[k], chain, len)) - return false; - } - return true; -} - -/* Add "@var += @step" immediately before @latch's terminator. */ -void sr_emit_advance(basic_block_t *latch, var_t *var, int step) -{ - block_t *scope = latch->scope; - var_t *amount = new_const_var(scope, step); - var_t *sum = require_var(scope); - insn_t *after = latch->insn_list.tail; - insn_t *load; - insn_t *add; - insn_t *assign; - - /* A loop latch ends in the jump back to its header. Appending the advance - * after that jump leaves it unreachable, so place the whole sequence before - * the terminator instead. - */ - if (after && (after->opcode == OP_branch || after->opcode == OP_jump || - after->opcode == OP_return || after->opcode == OP_func_ret)) - after = after->prev; - - sum->var_name = gen_name(); - sum->type = var->type; - sum->ptr_level = var->ptr_level; - - load = new_insn(OP_load_constant, amount, NULL, NULL); - add = new_insn(OP_add, sum, var, amount); - assign = new_insn(OP_assign, var, sum, NULL); - bb_insert_after(latch, after, load); - bb_insert_after(latch, load, add); - bb_insert_after(latch, add, assign); -} - -/* Turn the address @use reads or writes into a pointer the loop advances. - * Reports whether it did. - */ -bool sr_reduce_access(func_t *func, - basic_block_t *pre, - basic_block_t *latch, - insn_t *use) -{ - /* One slot past the chain holds the access itself, so that the scan below - * can be told to ignore the whole of what is moving in one list. - */ - insn_t *chain[MAX_IV_CHAIN + 1]; - int len = 0, step = 0; - var_t *addr = use->rs1; - - if (!addr || !sr_step_of(addr, &step) || !step) - return false; - if (!sr_collect_chain(func, addr, chain, &len) || !len) - return false; - if (len < MIN_IV_CHAIN) - return false; - - /* Every value the chain computes has to belong to it alone: what moves out - * of the loop stops being recomputed there. - * - * The access is skipped along with the chain, so its own operands are - * checked by hand: it may read the address the chain ends at, and nothing - * else. A value the chain computes that the access also stores would stay - * behind in the preheader at its first-iteration value while the address - * went on advancing. - */ - chain[len] = use; - for (int i = 0; i < len; i++) { - var_t *val = chain[i]->rd; - - if (use->rs2 == val || use->rs3 == val) - return false; - if (use->rs1 == val && val != addr) - return false; - if (var_read_by(func, val, chain, len + 1, false)) - return false; - } - - /* Order the chain so that each value lands after the ones it reads. - * - * The walk collected every definition before its operands, so reversing it - * is nearly right -- but only while the chain is a straight line. A value - * two of them share is collected under the first, and reversing then puts - * it after the second, which would read a variable nothing had defined yet. - * Choosing repeatedly instead is correct either way: take an entry once - * every chain value it reads has been taken. Nothing is moved until the - * whole order is settled, so a chain that cannot be ordered at all is - * declined rather than half-moved. - */ - int order[MAX_IV_CHAIN + 1]; - int placed[MAX_IV_CHAIN + 1]; - int n_placed = 0; - - for (int i = 0; i < len; i++) - placed[i] = 0; - - while (n_placed < len) { - int progress = 0; - - for (int i = len - 1; i >= 0; i--) { - if (placed[i]) - continue; - - var_t *ops[3]; - int ready = 1; - - ops[0] = chain[i]->rs1; - ops[1] = chain[i]->rs2; - ops[2] = chain[i]->rs3; - for (int k = 0; k < 3; k++) { - if (!ops[k]) - continue; - for (int j = 0; j < len; j++) { - if (j != i && !placed[j] && chain[j]->rd == ops[k]) - ready = 0; - } - } - if (!ready) - continue; - - order[n_placed++] = i; - placed[i] = 1; - progress = 1; - } - if (!progress) - return false; - } - - for (int i = 0; i < n_placed; i++) { - insn_t *moving = chain[order[i]]; - - bb_remove_insn(moving->belong_to, moving); - bb_append_insn(pre, moving); - } - - sr_emit_advance(latch, addr, step); - return true; -} - -/* Drop the literals the loop no longer needs. - * - * The shift counts and element widths a subscript used are materialised inside - * the loop, and moving the arithmetic that read them out of it leaves those - * materialisations behind with nothing to serve -- four of the fourteen - * instructions in matmul's innermost loop. - */ -void sr_sweep_dead_consts(func_t *func) -{ - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - - insn_t *next; - for (insn_t *insn = bb->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn->opcode != OP_load_constant || !insn->rd) - continue; - if (var_in_memory(insn->rd)) - continue; - if (var_read_by(func, insn->rd, NULL, 0, true)) - continue; - bb_remove_insn(bb, insn); - } - } -} - -/* The one block outside the loop that falls into @header, or NULL. */ -basic_block_t *sr_preheader(basic_block_t *header) -{ - basic_block_t *pre = NULL; - - for (int i = 0; i < header->prev_idx; i++) { - basic_block_t *p = header->prev[i].bb; - - if (!p) - continue; - if (p->loop_mark == sr_gen) - continue; /* the latch, or anything else inside */ - if (p->then_ || p->else_ || p->next != header) - return NULL; - if (pre) - return NULL; - pre = p; - } - return pre; -} - -/* How many blocks inside the loop jump back to @header. */ -int sr_latch_count(basic_block_t *header) -{ - int n = 0; - - for (int i = 0; i < header->prev_idx; i++) { - const basic_block_t *p = header->prev[i].bb; - - if (p && p->loop_mark == sr_gen) - n++; - } - return n; -} - -void sr_loop(func_t *func, basic_block_t *header, basic_block_t *latch) -{ - int step = 0, made = 0; - - /* One way back, and one way in that runs everything before the loop: the - * first address is computed there and advanced there. The test on the latch - * costs nothing and comes before the walk that marks the loop. - */ - if (latch->then_ || latch->else_ || latch->next != header) - return; - - sr_gen = loop_scan_gen + 1; - if (!mark_natural_loop(header, latch)) - return; /* the walk ran out of room, so loop_mark says nothing */ - - if (sr_latch_count(header) != 1) - return; - - basic_block_t *pre = sr_preheader(header); - - if (!pre) - return; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *base = sr_base(insn->rd); - - if (base) - base->loop_stamp = sr_gen; - } - } - - var_t *iv = sr_basic_iv(func, latch, &step); - - if (!iv || !step) - return; - sr_iv = iv; - sr_set_step(iv, step); - sr_derive_steps(func); - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - if (bb->loop_mark != sr_gen) - continue; - - insn_t *next; - for (insn_t *insn = bb->insn_list.head; insn; insn = next) { - next = insn->next; - if (insn->opcode != OP_read && insn->opcode != OP_write) - continue; - - /* A reduction takes instructions out of this block, so the walk - * starts again rather than following a pointer into the block they - * moved to. - */ - if (made >= MAX_IV_PER_LOOP) - break; - if (sr_reduce_access(func, pre, latch, insn)) { - made++; - next = bb->insn_list.head; - } - } - } - - if (made) - sr_sweep_dead_consts(func); -} - -/* Whether @var is an integer scalar narrower than a register pair. */ -bool ssa_is_narrow_scalar(const var_t *var) -{ - return var && !var->ptr_level && !var->is_func && !var->array_size && - var->type && var->type->size < 8 && !var->type->ptr_level && - !is_record_type(var->type); -} - -/* The int type of @var's signedness, which a narrow operand converts to. */ -type_t *int_type_of(const var_t *var) -{ - return var->type->is_unsigned ? TY_uint : TY_int; -} - -/* Insert before @at a copy of the narrow @var widened to @type, sign- or - * zero-extended by @var's own signedness as the usual arithmetic conversions - * require, and return it. - */ -var_t *widen_before(basic_block_t *bb, insn_t *at, var_t *var, type_t *type) -{ - var_t *wide = require_var(bb->scope); - insn_t *ext; - - wide->var_name = gen_name(); - wide->type = type; - ext = new_insn(OP_sign_ext, wide, var, NULL); - ext->sz = (var->type->size << 16) | 8; - bb_insert_after(bb, at->prev, ext); - return wide; -} - -/* Make @insn define a new variable of @type and convert that into its original - * destination straight after, with @conv (an extension or a truncation). - */ -void convert_result_after(basic_block_t *bb, - insn_t *insn, - type_t *type, - opcode_t conv) -{ - var_t *result = insn->rd; - var_t *inner = require_var(bb->scope); - insn_t *copy; - - inner->var_name = gen_name(); - inner->type = type; - insn->rd = inner; - copy = new_insn(conv, result, inner, NULL); - copy->sz = conv == OP_trunc ? result->type->size : (4 << 16) | 8; - bb_insert_after(bb, insn, copy); -} - -/* Give every register-pair operation operands of its own width. - * - * The 32-bit backends lower a wide operation only when all of its values are - * pairs; with a narrow operand they fall back to the one-word form, which keeps - * no high word. The parser does not convert every operand it builds, and an - * LP64 target never noticed, since its registers are already wide: a long long - * compared with an int temporary, or stored from one, lost the high word on Arm - * and RISC-V. Convert those operands here, after the optimizer, so that the - * allocator and the backends see one width per operation. Where the low word of - * a narrow result is exact -- addition, subtraction, multiplication, bitwise - * operations and left shifts -- a narrow destination is left as it is. - */ -void widen_pair_operands(void) -{ - if (PTR_SIZE >= 8) - return; - - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *rd = insn->rd; - bool wide1 = var_needs_register_pair(insn->rs1); - bool wide2 = var_needs_register_pair(insn->rs2); - - switch (insn->opcode) { - case OP_eq: - case OP_neq: - case OP_lt: - case OP_leq: - case OP_gt: - case OP_geq: - if (wide1 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = - widen_before(bb, insn, insn->rs2, insn->rs1->type); - else if (wide2 && ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = - widen_before(bb, insn, insn->rs1, insn->rs2->type); - break; - case OP_add: - case OP_sub: - case OP_mul: - case OP_div: - case OP_mod: - case OP_bit_and: - case OP_bit_or: - case OP_bit_xor: - if (var_needs_register_pair(rd)) { - if (!wide1 && !wide2) { - /* Both operands narrow: the operation is an int one - * whose result is then converted. - */ - if (ssa_is_narrow_scalar(insn->rs1) && - ssa_is_narrow_scalar(insn->rs2)) - convert_result_after( - bb, insn, - insn->rs1->type->is_unsigned || - insn->rs2->type->is_unsigned - ? TY_uint - : TY_int, - OP_sign_ext); - break; - } - if (!wide1 && ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = - widen_before(bb, insn, insn->rs1, rd->type); - if (!wide2 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = - widen_before(bb, insn, insn->rs2, rd->type); - break; - } - - /* A quotient's low word depends on the high words. */ - if ((insn->opcode == OP_div || insn->opcode == OP_mod) && - ssa_is_narrow_scalar(rd) && (wide1 || wide2)) { - type_t *type = - wide1 ? insn->rs1->type : insn->rs2->type; - - if (!wide1 && ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = widen_before(bb, insn, insn->rs1, type); - if (!wide2 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = widen_before(bb, insn, insn->rs2, type); - convert_result_after(bb, insn, type, OP_trunc); - } - break; - case OP_lshift: - case OP_rshift: - /* The result has the left operand's type. */ - if (var_needs_register_pair(rd) && - ssa_is_narrow_scalar(insn->rs1)) - convert_result_after(bb, insn, int_type_of(insn->rs1), - OP_sign_ext); - else if (insn->opcode == OP_rshift && wide1 && - ssa_is_narrow_scalar(rd)) - convert_result_after(bb, insn, insn->rs1->type, - OP_trunc); - break; - case OP_negate: - case OP_bit_not: - if (var_needs_register_pair(rd) && - ssa_is_narrow_scalar(insn->rs1)) - convert_result_after(bb, insn, int_type_of(insn->rs1), - OP_sign_ext); - break; - case OP_assign: - case OP_unwound_phi: - if (var_needs_register_pair(rd) && - ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = widen_before(bb, insn, insn->rs1, rd->type); - break; - case OP_read: - if (var_needs_register_pair(rd) && insn->sz == 4) - convert_result_after(bb, insn, int_type_of(rd), - OP_sign_ext); - break; - case OP_write: - if (insn->sz == 8 && ssa_is_narrow_scalar(insn->rs2)) - insn->rs2 = widen_before(bb, insn, insn->rs2, - insn->rs2->type->is_unsigned - ? TY_ulong_long - : TY_long_long); - break; - case OP_return: - if (var_needs_register_pair(&func->return_def) && - ssa_is_narrow_scalar(insn->rs1)) - insn->rs1 = widen_before(bb, insn, insn->rs1, - func->return_def.type); - break; - default: - break; - } - } - } - } -} - -void strength_reduce(void) -{ - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *base = sr_base(insn->rd); - - if (base) - base->def_cnt = 0; - } - } - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) { - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - var_t *base = sr_base(insn->rd); - - if (base) - base->def_cnt++; - } - } - - for (basic_block_t *latch = func->bbs; latch; latch = latch->rpo_next) { - basic_block_t *succ[3]; - - succ[0] = latch->next; - succ[1] = latch->then_; - succ[2] = latch->else_; - - for (int k = 0; k < 3; k++) { - if (!succ[k] || succ[k] == latch) - continue; - if (!is_dominate(succ[k], latch)) - continue; - sr_loop(func, succ[k], latch); - } - } - } -} - -void if_convert(void) -{ - /* The guard wraps the body rather than returning early: shecc parses its - * own source, and a target without conditional moves would be left with a - * loop after a return, which it rejects as unreachable code. - */ -#ifdef HAVE_COND_MOVE - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - if (!func->bbs) - continue; - for (basic_block_t *bb = func->bbs; bb; bb = bb->rpo_next) - if_convert_bb(func, bb); - } -#endif -} - -void unwind_phi(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->preorder_cb = bb_unwind_phi; - bb_forward_traversal(args); - } -} - -/* Whether @pred is a proper ancestor of @succ in the dominator tree. - * - * The walk goes up from @succ rather than down through everything @pred - * dominates: the answer lies on the one path to the root, where the downward - * search visited @pred's whole subtree and did not stop when it found it. - * dom_prev is the inverse of the dom_next the search followed, so the two agree - * on every pair. - */ -bool is_dominate(const basic_block_t *pred, basic_block_t *succ) -{ - for (basic_block_t *bb = succ; bb; bb = bb->dom_prev) { - if (bb->dom_prev == pred) - return true; - } - return false; -} - -/* For any variable, the basic block that defines it must dominate all the basic - * blocks where it is used; otherwise, it is an invalid cross-block - * initialization. - */ -void bb_check_var_cross_init(func_t *func, basic_block_t *bb) -{ - UNUSED(func); - - for (insn_t *insn = bb->insn_list.head; insn; insn = insn->next) { - if (insn->opcode != OP_allocat) - continue; - - const var_t *var = insn->rd; - ref_block_t *ref; - for (ref = var->ref_block_list.head; ref; ref = ref->next) { - if (ref->bb == bb) - continue; - - if (!is_dominate(bb, ref->bb)) - printf("Warning: Variable '%s' cross-initialized\n", - var->var_name); - } - } -} - -/** - * A variable's initialization lives in a basic block that does not dominate all - * of its uses, so control flow can reach a use without first passing through - * its initialization (i.e., a possibly-uninitialized use). - * - * For Example: // Jumps directly to 'label', skipping the declaration below - * goto label; if (1) { - * // This line is never executed when 'goto' is taken - * int x; - * label: - * // Uses 'x' after its declaration was bypassed - * x = 5; - * } - */ -void check_var_cross_init(void) -{ - bb_traversal_args_t *args = arena_alloc_traversal_args(); - for (func_t *func = FUNC_LIST.head; func; func = func->next) { - /* Skip function declarations without bodies */ - if (!func->bbs) - continue; - - args->func = func; - args->bb = func->bbs; - - func->visited++; - args->postorder_cb = bb_check_var_cross_init; - bb_forward_traversal(args); - } -} - -void bb_dump_connection(FILE *fd, - basic_block_t *curr, - basic_block_t *next, - bb_connection_type_t type) -{ - const char *str; - - switch (type) { - case NEXT: - str = "%s_%p:s->%s_%p:n\n"; - break; - case THEN: - str = "%s_%p:sw->%s_%p:n\n"; - break; - case ELSE: - str = "%s_%p:se->%s_%p:n\n"; - break; - default: - fatal("Unknown basic block connection type"); - } - - const char *pred; - const void *pred_id; - if (curr->insn_list.tail) { - pred = "insn"; - pred_id = curr->insn_list.tail; - } else { - pred = "pseudo"; - pred_id = curr; - } - - const char *succ; - const void *succ_id; - if (next->insn_list.tail) { - succ = "insn"; - succ_id = next->insn_list.head; - } else { - succ = "pseudo"; - succ_id = next; - } - - fprintf(fd, str, pred, pred_id, succ, succ_id); -} - -/* escape character for the tag in dot file */ -char *get_insn_op(const insn_t *insn) -{ - switch (insn->opcode) { - case OP_add: - return "+"; - case OP_sub: - return "-"; - case OP_mul: - return "*"; - case OP_div: - return "/"; - case OP_mod: - return "%%"; - case OP_lshift: - return "<<"; - case OP_rshift: - return ">>"; - case OP_eq: - return "=="; - case OP_neq: - return "!="; - case OP_gt: - return ">"; - case OP_lt: - return "<"; - case OP_geq: - return ">="; - case OP_leq: - return "<="; - case OP_bit_and: - return "&"; - case OP_bit_or: - return "|"; - case OP_bit_xor: - return "^"; - case OP_log_and: - return "&&"; - case OP_log_or: - return "||"; - default: - fatal("Unknown opcode in operator string conversion"); - return ""; /* unreachable, but silences compiler warning */ - } -} - -void bb_dump(FILE *fd, func_t *func, basic_block_t *bb) -{ - bb->visited++; - - bool next_ = false, then_ = false, else_ = false; - if (bb->next) - next_ = true; - if (bb->then_) - then_ = true; - if (bb->else_) - else_ = true; - if (then_ && !else_) - printf("Warning: missing false branch\n"); - if (!then_ && else_) - printf("Warning: missing true branch\n"); - if (next_ && (then_ || else_)) - printf("Warning: normal BB with condition\n"); - - fprintf(fd, "subgraph cluster_%p {\n", (void *) bb); - fprintf(fd, "label=\"BasicBlock %p (%s)\"\n", (void *) bb, - bb->bb_label_name); - - insn_t *insn = bb->insn_list.head; - if (!insn) - fprintf(fd, "pseudo_%p [label=\"pseudo\"]\n", (void *) bb); - if (!insn && (then_ || else_)) - printf("Warning: pseudo node should only have NEXT\n"); - - for (; insn; insn = insn->next) { - if (insn->opcode == OP_phi) { - fprintf(fd, "insn_%p [label=", (void *) insn); - fprintf(fd, "<%s%d := PHI(%s%d", - insn->rd->var_name, insn->rd->subscript, - insn->phi_ops->var->var_name, - insn->phi_ops->var->subscript); - - for (phi_operand_t *op = insn->phi_ops->next; op; op = op->next) { - fprintf(fd, ", %s%d", op->var->var_name, - op->var->subscript); - } - fprintf(fd, ")>]\n"); - } else { - char str[DUMP_INSN_LEN]; - switch (insn->opcode) { - case OP_allocat: - sprintf(str, "<%s%d := ALLOC>", insn->rd->var_name, - insn->rd->subscript); - break; - case OP_load_constant: - sprintf(str, "<%s%d := CONST %d>", - insn->rd->var_name, insn->rd->subscript, - insn->rd->init_val); - break; - case OP_load_data_address: - sprintf(str, "<%s%d := [.data] + %d>", - insn->rd->var_name, insn->rd->subscript, - insn->rd->init_val); - break; - case OP_load_rodata_address: - sprintf(str, "<%s%d := [.rodata] + %d>", - insn->rd->var_name, insn->rd->subscript, - insn->rd->init_val); - break; - case OP_address_of: - sprintf(str, "<%s%d := &%s%d>", - insn->rd->var_name, insn->rd->subscript, - insn->rs1->var_name, insn->rs1->subscript); - break; - case OP_assign: - sprintf(str, "<%s%d := %s%d>", - insn->rd->var_name, insn->rd->subscript, - insn->rs1->var_name, insn->rs1->subscript); - break; - case OP_read: - sprintf(str, "<%s%d := (%s%d)>", - insn->rd->var_name, insn->rd->subscript, - insn->rs1->var_name, insn->rs1->subscript); - break; - case OP_write: - if (insn->rs2->is_func) - sprintf(str, "<(%s%d) := %s>", - insn->rs1->var_name, insn->rs1->subscript, - insn->rs2->var_name); - else - sprintf(str, "<(%s%d) := %s%d>", - insn->rs1->var_name, insn->rs1->subscript, - insn->rs2->var_name, insn->rs2->subscript); - break; - case OP_branch: - sprintf(str, "%d>", insn->rs1->var_name, - insn->rs1->subscript); - break; - case OP_jump: - sprintf(str, ""); - break; - case OP_label: - sprintf(str, "